Line data Source code
1 : /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 : calls to the runtime library (libgomp), data marshalling to implement data
3 : sharing and copying clauses, offloading to accelerators, and more.
4 :
5 : Contributed by Diego Novillo <dnovillo@redhat.com>
6 :
7 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
8 :
9 : This file is part of GCC.
10 :
11 : GCC is free software; you can redistribute it and/or modify it under
12 : the terms of the GNU General Public License as published by the Free
13 : Software Foundation; either version 3, or (at your option) any later
14 : version.
15 :
16 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 : for more details.
20 :
21 : You should have received a copy of the GNU General Public License
22 : along with GCC; see the file COPYING3. If not see
23 : <http://www.gnu.org/licenses/>. */
24 :
25 : #include "config.h"
26 : #include "system.h"
27 : #include "coretypes.h"
28 : #include "backend.h"
29 : #include "target.h"
30 : #include "tree.h"
31 : #include "gimple.h"
32 : #include "tree-pass.h"
33 : #include "ssa.h"
34 : #include "cgraph.h"
35 : #include "pretty-print.h"
36 : #include "diagnostic-core.h"
37 : #include "fold-const.h"
38 : #include "stor-layout.h"
39 : #include "internal-fn.h"
40 : #include "gimple-iterator.h"
41 : #include "gimple-fold.h"
42 : #include "gimplify.h"
43 : #include "gimplify-me.h"
44 : #include "gimple-walk.h"
45 : #include "tree-iterator.h"
46 : #include "tree-inline.h"
47 : #include "langhooks.h"
48 : #include "tree-dfa.h"
49 : #include "tree-ssa.h"
50 : #include "splay-tree.h"
51 : #include "omp-general.h"
52 : #include "omp-low.h"
53 : #include "gimple-low.h"
54 : #include "alloc-pool.h"
55 : #include "symbol-summary.h"
56 : #include "tree-nested.h"
57 : #include "context.h"
58 : #include "gomp-constants.h"
59 : #include "gimple-pretty-print.h"
60 : #include "stringpool.h"
61 : #include "attribs.h"
62 : #include "omp-offload.h"
63 :
64 : /* Lowering of OMP parallel and workshare constructs proceeds in two
65 : phases. The first phase scans the function looking for OMP statements
66 : and then for variables that must be replaced to satisfy data sharing
67 : clauses. The second phase expands code for the constructs, as well as
68 : re-gimplifying things when variables have been replaced with complex
69 : expressions.
70 :
71 : Final code generation is done by pass_expand_omp. The flowgraph is
72 : scanned for regions which are then moved to a new
73 : function, to be invoked by the thread library, or offloaded. */
74 :
75 : /* Context structure. Used to store information about each parallel
76 : directive in the code. */
77 :
78 : struct omp_context
79 : {
80 : /* This field must be at the beginning, as we do "inheritance": Some
81 : callback functions for tree-inline.cc (e.g., omp_copy_decl)
82 : receive a copy_body_data pointer that is up-casted to an
83 : omp_context pointer. */
84 : copy_body_data cb;
85 :
86 : /* The tree of contexts corresponding to the encountered constructs. */
87 : struct omp_context *outer;
88 : gimple *stmt;
89 :
90 : /* Map variables to fields in a structure that allows communication
91 : between sending and receiving threads. */
92 : splay_tree field_map;
93 : tree record_type;
94 : tree sender_decl;
95 : tree receiver_decl;
96 :
97 : /* These are used just by task contexts, if task firstprivate fn is
98 : needed. srecord_type is used to communicate from the thread
99 : that encountered the task construct to task firstprivate fn,
100 : record_type is allocated by GOMP_task, initialized by task firstprivate
101 : fn and passed to the task body fn. */
102 : splay_tree sfield_map;
103 : tree srecord_type;
104 :
105 : /* A chain of variables to add to the top-level block surrounding the
106 : construct. In the case of a parallel, this is in the child function. */
107 : tree block_vars;
108 :
109 : /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
110 : barriers should jump to during omplower pass. */
111 : tree cancel_label;
112 :
113 : /* The sibling GIMPLE_OMP_FOR simd with _simt_ clause or NULL
114 : otherwise. */
115 : gimple *simt_stmt;
116 :
117 : /* For task reductions registered in this context, a vector containing
118 : the length of the private copies block (if constant, otherwise NULL)
119 : and then offsets (if constant, otherwise NULL) for each entry. */
120 : vec<tree> task_reductions;
121 :
122 : /* A hash map from the reduction clauses to the registered array
123 : elts. */
124 : hash_map<tree, unsigned> *task_reduction_map;
125 :
126 : /* And a hash map from the lastprivate(conditional:) variables to their
127 : corresponding tracking loop iteration variables. */
128 : hash_map<tree, tree> *lastprivate_conditional_map;
129 :
130 : /* And a hash map from the allocate variables to their corresponding
131 : allocators. */
132 : hash_map<tree, tree> *allocate_map;
133 :
134 : /* A tree_list of the reduction clauses in this context. This is
135 : only used for checking the consistency of OpenACC reduction
136 : clauses in scan_omp_for and is not guaranteed to contain a valid
137 : value outside of this function. */
138 : tree local_reduction_clauses;
139 :
140 : /* A tree_list of the reduction clauses in outer contexts. This is
141 : only used for checking the consistency of OpenACC reduction
142 : clauses in scan_omp_for and is not guaranteed to contain a valid
143 : value outside of this function. */
144 : tree outer_reduction_clauses;
145 :
146 : /* Nesting depth of this context. Used to beautify error messages re
147 : invalid gotos. The outermost ctx is depth 1, with depth 0 being
148 : reserved for the main body of the function. */
149 : int depth;
150 :
151 : /* True if this parallel directive is nested within another. */
152 : bool is_nested;
153 :
154 : /* True if this construct can be cancelled. */
155 : bool cancellable;
156 :
157 : /* True if lower_omp_1 should look up lastprivate conditional in parent
158 : context. */
159 : bool combined_into_simd_safelen1;
160 :
161 : /* True if there is nested scan context with inclusive clause. */
162 : bool scan_inclusive;
163 :
164 : /* True if there is nested scan context with exclusive clause. */
165 : bool scan_exclusive;
166 :
167 : /* True in the second simd loop of for simd with inscan reductions. */
168 : bool for_simd_scan_phase;
169 :
170 : /* True if there is order(concurrent) clause on the construct. */
171 : bool order_concurrent;
172 :
173 : /* True if there is bind clause on the construct (i.e. a loop construct). */
174 : bool loop_p;
175 :
176 : /* Only used for omp target contexts. True if a teams construct is
177 : strictly nested in it. */
178 : bool teams_nested_p;
179 :
180 : /* Only used for omp target contexts. True if an OpenMP construct other
181 : than teams is strictly nested in it. */
182 : bool nonteams_nested_p;
183 :
184 : /* Candidates for adjusting OpenACC privatization level. */
185 : vec<tree> oacc_privatization_candidates;
186 : };
187 :
188 : static splay_tree all_contexts;
189 : static int taskreg_nesting_level;
190 : static int target_nesting_level;
191 : static bitmap make_addressable_vars;
192 : static bitmap global_nonaddressable_vars;
193 : static vec<omp_context *> taskreg_contexts;
194 : static vec<gomp_task *> task_cpyfns;
195 :
196 : static void scan_omp (gimple_seq *, omp_context *);
197 : static tree scan_omp_1_op (tree *, int *, void *);
198 : static bool omp_maybe_offloaded_ctx (omp_context *ctx);
199 :
200 : #define WALK_SUBSTMTS \
201 : case GIMPLE_BIND: \
202 : case GIMPLE_TRY: \
203 : case GIMPLE_CATCH: \
204 : case GIMPLE_EH_FILTER: \
205 : case GIMPLE_ASSUME: \
206 : case GIMPLE_TRANSACTION: \
207 : /* The sub-statements for these should be walked. */ \
208 : *handled_ops_p = false; \
209 : break;
210 :
211 : /* Return whether CTX represents an OpenACC 'parallel' or 'serial' construct.
212 : (This doesn't include OpenACC 'kernels' decomposed parts.) */
213 :
214 : static bool
215 33665 : is_oacc_parallel_or_serial (omp_context *ctx)
216 : {
217 33665 : enum gimple_code outer_type = gimple_code (ctx->stmt);
218 33665 : return ((outer_type == GIMPLE_OMP_TARGET)
219 33665 : && ((gimple_omp_target_kind (ctx->stmt)
220 : == GF_OMP_TARGET_KIND_OACC_PARALLEL)
221 15323 : || (gimple_omp_target_kind (ctx->stmt)
222 33665 : == GF_OMP_TARGET_KIND_OACC_SERIAL)));
223 : }
224 :
225 : /* Return whether CTX represents an OpenACC 'kernels' construct.
226 : (This doesn't include OpenACC 'kernels' decomposed parts.) */
227 :
228 : static bool
229 49900 : is_oacc_kernels (omp_context *ctx)
230 : {
231 49900 : enum gimple_code outer_type = gimple_code (ctx->stmt);
232 49900 : return ((outer_type == GIMPLE_OMP_TARGET)
233 49900 : && (gimple_omp_target_kind (ctx->stmt)
234 49900 : == GF_OMP_TARGET_KIND_OACC_KERNELS));
235 : }
236 :
237 : /* Return whether CTX represents an OpenACC 'kernels' decomposed part. */
238 :
239 : static bool
240 21645 : is_oacc_kernels_decomposed_part (omp_context *ctx)
241 : {
242 21645 : enum gimple_code outer_type = gimple_code (ctx->stmt);
243 21645 : return ((outer_type == GIMPLE_OMP_TARGET)
244 21645 : && ((gimple_omp_target_kind (ctx->stmt)
245 : == GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED)
246 14552 : || (gimple_omp_target_kind (ctx->stmt)
247 : == GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE)
248 14552 : || (gimple_omp_target_kind (ctx->stmt)
249 21645 : == GF_OMP_TARGET_KIND_OACC_DATA_KERNELS)));
250 : }
251 :
252 : /* Return true if STMT corresponds to an OpenMP target region. */
253 : static bool
254 42850 : is_omp_target (gimple *stmt)
255 : {
256 42850 : if (gimple_code (stmt) == GIMPLE_OMP_TARGET)
257 : {
258 6511 : int kind = gimple_omp_target_kind (stmt);
259 6511 : return (kind == GF_OMP_TARGET_KIND_REGION
260 6511 : || kind == GF_OMP_TARGET_KIND_DATA
261 6511 : || kind == GF_OMP_TARGET_KIND_ENTER_DATA
262 11147 : || kind == GF_OMP_TARGET_KIND_EXIT_DATA);
263 : }
264 : return false;
265 : }
266 :
267 : /* If DECL is the artificial dummy VAR_DECL created for non-static
268 : data member privatization, return the underlying "this" parameter,
269 : otherwise return NULL. */
270 :
271 : tree
272 540411 : omp_member_access_dummy_var (tree decl)
273 : {
274 540411 : if (!VAR_P (decl)
275 331519 : || !DECL_ARTIFICIAL (decl)
276 162807 : || !DECL_IGNORED_P (decl)
277 157379 : || !DECL_HAS_VALUE_EXPR_P (decl)
278 553197 : || !lang_hooks.decls.omp_disregard_value_expr (decl, false))
279 538151 : return NULL_TREE;
280 :
281 2260 : tree v = DECL_VALUE_EXPR (decl);
282 2260 : if (TREE_CODE (v) != COMPONENT_REF)
283 : return NULL_TREE;
284 :
285 16164 : while (1)
286 9197 : switch (TREE_CODE (v))
287 : {
288 6967 : case COMPONENT_REF:
289 6967 : case MEM_REF:
290 6967 : case INDIRECT_REF:
291 6967 : CASE_CONVERT:
292 6967 : case POINTER_PLUS_EXPR:
293 6967 : v = TREE_OPERAND (v, 0);
294 6967 : continue;
295 2211 : case PARM_DECL:
296 2211 : if (DECL_CONTEXT (v) == current_function_decl
297 2211 : && DECL_ARTIFICIAL (v)
298 4422 : && TREE_CODE (TREE_TYPE (v)) == POINTER_TYPE)
299 : return v;
300 : return NULL_TREE;
301 : default:
302 : return NULL_TREE;
303 : }
304 : }
305 :
306 : /* Helper for unshare_and_remap, called through walk_tree. */
307 :
308 : static tree
309 1579 : unshare_and_remap_1 (tree *tp, int *walk_subtrees, void *data)
310 : {
311 1579 : tree *pair = (tree *) data;
312 1579 : if (*tp == pair[0])
313 : {
314 233 : *tp = unshare_expr (pair[1]);
315 233 : *walk_subtrees = 0;
316 : }
317 1346 : else if (IS_TYPE_OR_DECL_P (*tp))
318 244 : *walk_subtrees = 0;
319 1579 : return NULL_TREE;
320 : }
321 :
322 : /* Return unshare_expr (X) with all occurrences of FROM
323 : replaced with TO. */
324 :
325 : static tree
326 150 : unshare_and_remap (tree x, tree from, tree to)
327 : {
328 150 : tree pair[2] = { from, to };
329 150 : x = unshare_expr (x);
330 150 : walk_tree (&x, unshare_and_remap_1, pair, NULL);
331 150 : return x;
332 : }
333 :
334 : /* Convenience function for calling scan_omp_1_op on tree operands. */
335 :
336 : static inline tree
337 338975 : scan_omp_op (tree *tp, omp_context *ctx)
338 : {
339 338975 : struct walk_stmt_info wi;
340 :
341 338975 : memset (&wi, 0, sizeof (wi));
342 338975 : wi.info = ctx;
343 338975 : wi.want_locations = true;
344 :
345 338975 : return walk_tree (tp, scan_omp_1_op, &wi, NULL);
346 : }
347 :
348 : static void lower_omp (gimple_seq *, omp_context *);
349 : static tree lookup_decl_in_outer_ctx (tree, omp_context *);
350 : static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *);
351 :
352 : /* Return true if CTX is for an omp parallel. */
353 :
354 : static inline bool
355 776492 : is_parallel_ctx (omp_context *ctx)
356 : {
357 61162 : return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL;
358 : }
359 :
360 :
361 : /* Return true if CTX is for an omp task. */
362 :
363 : static inline bool
364 236062 : is_task_ctx (omp_context *ctx)
365 : {
366 38793 : return gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
367 : }
368 :
369 :
370 : /* Return true if CTX is for an omp taskloop. */
371 :
372 : static inline bool
373 8182 : is_taskloop_ctx (omp_context *ctx)
374 : {
375 8182 : return gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
376 8182 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_TASKLOOP;
377 : }
378 :
379 :
380 : /* Return true if CTX is for a host omp teams. */
381 :
382 : static inline bool
383 521421 : is_host_teams_ctx (omp_context *ctx)
384 : {
385 521421 : return gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
386 521421 : && gimple_omp_teams_host (as_a <gomp_teams *> (ctx->stmt));
387 : }
388 :
389 : /* Return true if CTX is for an omp parallel or omp task or host omp teams
390 : (the last one is strictly not a task region in OpenMP speak, but we
391 : need to treat it similarly). */
392 :
393 : static inline bool
394 663889 : is_taskreg_ctx (omp_context *ctx)
395 : {
396 663889 : return is_parallel_ctx (ctx) || is_task_ctx (ctx) || is_host_teams_ctx (ctx);
397 : }
398 :
399 : /* Return true if EXPR is variable sized. If IS_REF is true, then
400 : EXPR is assumed to be a reference and the object it refers
401 : to is checked instead. */
402 :
403 : static inline bool
404 906209 : is_variable_sized (const_tree expr, bool is_ref = false)
405 : {
406 906209 : if (is_ref)
407 2166 : expr = TREE_TYPE (expr);
408 906209 : return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr)));
409 : }
410 :
411 : /* Lookup variables. The "maybe" form
412 : allows for the variable form to not have been entered, otherwise we
413 : assert that the variable must have been entered. */
414 :
415 : static inline tree
416 716431 : lookup_decl (tree var, omp_context *ctx)
417 : {
418 1432763 : tree *n = ctx->cb.decl_map->get (var);
419 716431 : return *n;
420 : }
421 :
422 : static inline tree
423 719334 : maybe_lookup_decl (const_tree var, omp_context *ctx)
424 : {
425 719334 : tree *n = ctx->cb.decl_map->get (const_cast<tree> (var));
426 719334 : return n ? *n : NULL_TREE;
427 : }
428 :
429 : static inline tree
430 127317 : lookup_field (tree var, omp_context *ctx)
431 : {
432 127317 : splay_tree_node n;
433 254634 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
434 127317 : return (tree) n->value;
435 : }
436 :
437 : static inline tree
438 169404 : lookup_sfield (splay_tree_key key, omp_context *ctx)
439 : {
440 169404 : splay_tree_node n;
441 169404 : n = splay_tree_lookup (ctx->sfield_map
442 : ? ctx->sfield_map : ctx->field_map, key);
443 169404 : return (tree) n->value;
444 : }
445 :
446 : static inline tree
447 967 : lookup_sfield (tree var, omp_context *ctx)
448 : {
449 967 : return lookup_sfield ((splay_tree_key) var, ctx);
450 : }
451 :
452 : static inline tree
453 221552 : maybe_lookup_field (splay_tree_key key, omp_context *ctx)
454 : {
455 221552 : splay_tree_node n;
456 221552 : n = splay_tree_lookup (ctx->field_map, key);
457 221552 : return n ? (tree) n->value : NULL_TREE;
458 : }
459 :
460 : static inline tree
461 221552 : maybe_lookup_field (tree var, omp_context *ctx)
462 : {
463 2555 : return maybe_lookup_field ((splay_tree_key) var, ctx);
464 : }
465 :
466 : /* Return true if DECL should be copied by pointer. SHARED_CTX is
467 : the parallel context if DECL is to be shared. */
468 :
469 : static bool
470 232483 : use_pointer_for_field (tree decl, omp_context *shared_ctx)
471 : {
472 453428 : if (AGGREGATE_TYPE_P (TREE_TYPE (decl))
473 215786 : || TYPE_ATOMIC (TREE_TYPE (decl))
474 448269 : || POLY_INT_CST_P (DECL_SIZE (decl)))
475 : return true;
476 :
477 : /* We can only use copy-in/copy-out semantics for shared variables
478 : when we know the value is not accessible from an outer scope. */
479 215756 : if (shared_ctx)
480 : {
481 29413 : gcc_assert (!is_gimple_omp_oacc (shared_ctx->stmt));
482 :
483 : /* ??? Trivially accessible from anywhere. But why would we even
484 : be passing an address in this case? Should we simply assert
485 : this to be false, or should we have a cleanup pass that removes
486 : these from the list of mappings? */
487 29413 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, shared_ctx)))
488 : return true;
489 :
490 : /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
491 : without analyzing the expression whether or not its location
492 : is accessible to anyone else. In the case of nested parallel
493 : regions it certainly may be. */
494 29413 : if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl))
495 : return true;
496 :
497 : /* Do not use copy-in/copy-out for variables that have their
498 : address taken. */
499 28499 : if (is_global_var (decl))
500 : {
501 : /* For file scope vars, track whether we've seen them as
502 : non-addressable initially and in that case, keep the same
503 : answer for the duration of the pass, even when they are made
504 : addressable later on e.g. through reduction expansion. Global
505 : variables which weren't addressable before the pass will not
506 : have their privatized copies address taken. See PR91216. */
507 1232 : if (!TREE_ADDRESSABLE (decl))
508 : {
509 479 : if (!global_nonaddressable_vars)
510 74 : global_nonaddressable_vars = BITMAP_ALLOC (NULL);
511 479 : bitmap_set_bit (global_nonaddressable_vars, DECL_UID (decl));
512 : }
513 753 : else if (!global_nonaddressable_vars
514 986 : || !bitmap_bit_p (global_nonaddressable_vars,
515 233 : DECL_UID (decl)))
516 731 : return true;
517 : }
518 27267 : else if (TREE_ADDRESSABLE (decl))
519 : return true;
520 :
521 : /* lower_send_shared_vars only uses copy-in, but not copy-out
522 : for these. */
523 18362 : if (TREE_READONLY (decl)
524 18362 : || ((TREE_CODE (decl) == RESULT_DECL
525 11779 : || TREE_CODE (decl) == PARM_DECL)
526 673 : && DECL_BY_REFERENCE (decl)))
527 : return false;
528 :
529 : /* Disallow copy-in/out in nested parallel if
530 : decl is shared in outer parallel, otherwise
531 : each thread could store the shared variable
532 : in its own copy-in location, making the
533 : variable no longer really shared. */
534 11547 : if (shared_ctx->is_nested)
535 : {
536 2470 : omp_context *up;
537 :
538 5975 : for (up = shared_ctx->outer; up; up = up->outer)
539 5702 : if ((is_taskreg_ctx (up)
540 3643 : || (gimple_code (up->stmt) == GIMPLE_OMP_TARGET
541 411 : && is_gimple_omp_offloaded (up->stmt)))
542 6113 : && maybe_lookup_decl (decl, up))
543 : break;
544 :
545 2470 : if (up)
546 : {
547 2197 : tree c;
548 :
549 2197 : if (gimple_code (up->stmt) == GIMPLE_OMP_TARGET)
550 : {
551 309 : for (c = gimple_omp_target_clauses (up->stmt);
552 2031 : c; c = OMP_CLAUSE_CHAIN (c))
553 1932 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
554 1932 : && OMP_CLAUSE_DECL (c) == decl)
555 : break;
556 : }
557 : else
558 1888 : for (c = gimple_omp_taskreg_clauses (up->stmt);
559 6909 : c; c = OMP_CLAUSE_CHAIN (c))
560 6454 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
561 6454 : && OMP_CLAUSE_DECL (c) == decl)
562 : break;
563 :
564 2197 : if (c)
565 1643 : goto maybe_mark_addressable_and_ret;
566 : }
567 : }
568 :
569 : /* For tasks avoid using copy-in/out. As tasks can be
570 : deferred or executed in different thread, when GOMP_task
571 : returns, the task hasn't necessarily terminated. */
572 9904 : if (is_task_ctx (shared_ctx))
573 : {
574 2193 : tree outer;
575 550 : maybe_mark_addressable_and_ret:
576 2193 : outer = maybe_lookup_decl_in_outer_ctx (decl, shared_ctx);
577 2193 : if (is_gimple_reg (outer) && !omp_member_access_dummy_var (outer))
578 : {
579 : /* Taking address of OUTER in lower_send_shared_vars
580 : might need regimplification of everything that uses the
581 : variable. */
582 982 : if (!make_addressable_vars)
583 489 : make_addressable_vars = BITMAP_ALLOC (NULL);
584 982 : bitmap_set_bit (make_addressable_vars, DECL_UID (outer));
585 982 : TREE_ADDRESSABLE (outer) = 1;
586 : }
587 2193 : return true;
588 : }
589 : }
590 :
591 : return false;
592 : }
593 :
594 : /* Construct a new automatic decl similar to VAR. */
595 :
596 : static tree
597 229663 : omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx)
598 : {
599 229663 : tree copy = copy_var_decl (var, name, type);
600 :
601 229663 : DECL_CONTEXT (copy) = current_function_decl;
602 :
603 229663 : if (ctx)
604 : {
605 229382 : DECL_CHAIN (copy) = ctx->block_vars;
606 229382 : ctx->block_vars = copy;
607 : }
608 : else
609 281 : record_vars (copy);
610 :
611 : /* If VAR is listed in make_addressable_vars, it wasn't
612 : originally addressable, but was only later made so.
613 : We don't need to take address of privatizations
614 : from that var. */
615 229663 : if (TREE_ADDRESSABLE (var)
616 229663 : && ((make_addressable_vars
617 3467 : && bitmap_bit_p (make_addressable_vars, DECL_UID (var)))
618 39728 : || (global_nonaddressable_vars
619 515 : && bitmap_bit_p (global_nonaddressable_vars, DECL_UID (var)))))
620 2077 : TREE_ADDRESSABLE (copy) = 0;
621 :
622 229663 : return copy;
623 : }
624 :
625 : static tree
626 229663 : omp_copy_decl_1 (tree var, omp_context *ctx)
627 : {
628 229663 : return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx);
629 : }
630 :
631 : /* Build tree nodes to access the field for VAR on the receiver side. */
632 :
633 : static tree
634 122861 : build_receiver_ref (tree var, bool by_ref, omp_context *ctx)
635 : {
636 122861 : tree x, field = lookup_field (var, ctx);
637 :
638 : /* If the receiver record type was remapped in the child function,
639 : remap the field into the new record type. */
640 122861 : x = maybe_lookup_field (field, ctx);
641 122861 : if (x != NULL)
642 11245 : field = x;
643 :
644 122861 : x = build_simple_mem_ref (ctx->receiver_decl);
645 122861 : TREE_THIS_NOTRAP (x) = 1;
646 122861 : x = omp_build_component_ref (x, field);
647 122861 : if (by_ref)
648 : {
649 36186 : x = build_simple_mem_ref (x);
650 36186 : TREE_THIS_NOTRAP (x) = 1;
651 : }
652 :
653 122861 : return x;
654 : }
655 :
656 : /* Build tree nodes to access VAR in the scope outer to CTX. In the case
657 : of a parallel, this is a component reference; for workshare constructs
658 : this is some variable. */
659 :
660 : static tree
661 114404 : build_outer_var_ref (tree var, omp_context *ctx,
662 : enum omp_clause_code code = OMP_CLAUSE_ERROR)
663 : {
664 114404 : tree x;
665 114404 : omp_context *outer = ctx->outer;
666 114793 : for (; outer; outer = outer->outer)
667 : {
668 82474 : if (gimple_code (outer->stmt) == GIMPLE_OMP_TASKGROUP)
669 373 : continue;
670 82117 : if (gimple_code (outer->stmt) == GIMPLE_OMP_SCOPE
671 82101 : && !maybe_lookup_decl (var, outer))
672 16 : continue;
673 : break;
674 : }
675 :
676 114404 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx)))
677 : x = var;
678 106299 : else if (is_variable_sized (var))
679 : {
680 47 : x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0);
681 47 : x = build_outer_var_ref (x, ctx, code);
682 47 : x = build_simple_mem_ref (x);
683 : }
684 106252 : else if (is_taskreg_ctx (ctx))
685 : {
686 63715 : bool by_ref = use_pointer_for_field (var, NULL);
687 63715 : x = build_receiver_ref (var, by_ref, ctx);
688 : }
689 42537 : else if ((gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
690 41200 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
691 14075 : || ctx->loop_p
692 13302 : || code == OMP_CLAUSE_ALLOCATE
693 55587 : || (code == OMP_CLAUSE_PRIVATE
694 40 : && (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
695 : || gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS
696 : || gimple_code (ctx->stmt) == GIMPLE_OMP_SINGLE)))
697 : {
698 : /* #pragma omp simd isn't a worksharing construct, and can reference
699 : even private vars in its linear etc. clauses.
700 : Similarly for OMP_CLAUSE_PRIVATE with outer ref, that can refer
701 : to private vars in all worksharing constructs. */
702 29507 : x = NULL_TREE;
703 29507 : if (outer && is_taskreg_ctx (outer))
704 1002 : x = lookup_decl (var, outer);
705 28505 : else if (outer)
706 22315 : x = maybe_lookup_decl_in_outer_ctx (var, ctx);
707 23317 : if (x == NULL_TREE)
708 : x = var;
709 : }
710 13030 : else if (code == OMP_CLAUSE_LASTPRIVATE && is_taskloop_ctx (ctx))
711 : {
712 689 : gcc_assert (outer);
713 689 : splay_tree_node n
714 1378 : = splay_tree_lookup (outer->field_map,
715 689 : (splay_tree_key) &DECL_UID (var));
716 689 : if (n == NULL)
717 : {
718 610 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, outer)))
719 : x = var;
720 : else
721 384 : x = lookup_decl (var, outer);
722 : }
723 : else
724 : {
725 79 : tree field = (tree) n->value;
726 : /* If the receiver record type was remapped in the child function,
727 : remap the field into the new record type. */
728 79 : x = maybe_lookup_field (field, outer);
729 79 : if (x != NULL)
730 0 : field = x;
731 :
732 79 : x = build_simple_mem_ref (outer->receiver_decl);
733 79 : x = omp_build_component_ref (x, field);
734 79 : if (use_pointer_for_field (var, outer))
735 59 : x = build_simple_mem_ref (x);
736 : }
737 : }
738 12341 : else if (outer)
739 12149 : x = lookup_decl (var, outer);
740 192 : else if (omp_privatize_by_reference (var))
741 : /* This can happen with orphaned constructs. If var is reference, it is
742 : possible it is shared and as such valid. */
743 : x = var;
744 94 : else if (omp_member_access_dummy_var (var))
745 : x = var;
746 : else
747 0 : gcc_unreachable ();
748 :
749 114404 : if (x == var)
750 : {
751 16287 : tree t = omp_member_access_dummy_var (var);
752 16287 : if (t)
753 : {
754 220 : x = DECL_VALUE_EXPR (var);
755 220 : tree o = maybe_lookup_decl_in_outer_ctx (t, ctx);
756 220 : if (o != t)
757 54 : x = unshare_and_remap (x, t, o);
758 : else
759 166 : x = unshare_expr (x);
760 : }
761 : }
762 :
763 114404 : if (omp_privatize_by_reference (var))
764 3092 : x = build_simple_mem_ref (x);
765 :
766 114404 : return x;
767 : }
768 :
769 : /* Build tree nodes to access the field for VAR on the sender side. */
770 :
771 : static tree
772 168358 : build_sender_ref (splay_tree_key key, omp_context *ctx)
773 : {
774 168358 : tree field = lookup_sfield (key, ctx);
775 168358 : tree tmp = ctx->sender_decl;
776 168358 : if (POINTER_TYPE_P (TREE_TYPE (tmp)))
777 451 : tmp = build_fold_indirect_ref (tmp);
778 168358 : return omp_build_component_ref (tmp, field);
779 : }
780 :
781 : static tree
782 164204 : build_sender_ref (tree var, omp_context *ctx)
783 : {
784 0 : return build_sender_ref ((splay_tree_key) var, ctx);
785 : }
786 :
787 : /* Add a new field for VAR inside the structure CTX. If BY_REF is true,
788 : use a pointer to the VAR rather than VAR itself.
789 : MASK is a bit mask of other options. Bits are interpreted as:
790 : 1: Install VAR in ctx->field_map.
791 : 2: Install VAR in ctx->sfield_map.
792 : 4: VAR is an array, convert it to a pointer.
793 : 8: Use DECL_UID (VAR) instead of VAR as key.
794 : 16: Use DECL_NAME (VAR) instead of VAR as key.
795 : 32: Don't dereference omp_is_reference types.
796 : KEY_EXPR allows specifying something other than VAR as the lookup key.
797 : If specified, it also overrides the 8 and 16 MASK bits. */
798 :
799 : static void
800 131207 : install_var_field (tree var, bool by_ref, int mask, omp_context *ctx,
801 : tree key_expr = NULL_TREE)
802 : {
803 131207 : tree field, type, sfield = NULL_TREE;
804 131207 : splay_tree_key key = (splay_tree_key) var;
805 :
806 131207 : if (key_expr)
807 : /* Allow caller to explicitly set the expression used as the key. */
808 44748 : key = (splay_tree_key) key_expr;
809 : else
810 : {
811 86459 : if ((mask & 16) != 0)
812 : {
813 624 : key = (splay_tree_key) &DECL_NAME (var);
814 624 : gcc_checking_assert (key != (splay_tree_key) var);
815 : }
816 86459 : if ((mask & 8) != 0)
817 : {
818 1619 : key = (splay_tree_key) &DECL_UID (var);
819 1619 : gcc_checking_assert (key != (splay_tree_key) var);
820 : }
821 : }
822 131207 : gcc_assert ((mask & 1) == 0
823 : || !splay_tree_lookup (ctx->field_map, key));
824 131207 : gcc_assert ((mask & 2) == 0 || !ctx->sfield_map
825 : || !splay_tree_lookup (ctx->sfield_map, key));
826 131207 : gcc_assert ((mask & 3) == 3
827 : || !is_gimple_omp_oacc (ctx->stmt));
828 :
829 131207 : type = TREE_TYPE (var);
830 131207 : if ((mask & 16) != 0)
831 624 : type = lang_hooks.decls.omp_array_data (var, true);
832 :
833 : /* Prevent redeclaring the var in the split-off function with a restrict
834 : pointer type. Note that we only clear type itself, restrict qualifiers in
835 : the pointed-to type will be ignored by points-to analysis. */
836 131207 : if (POINTER_TYPE_P (type)
837 131207 : && TYPE_RESTRICT (type))
838 7006 : type = build_qualified_type (type, TYPE_QUALS (type) & ~TYPE_QUAL_RESTRICT);
839 :
840 131207 : if (mask & 4)
841 : {
842 425 : gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
843 425 : type = build_pointer_type (build_pointer_type (type));
844 : }
845 130782 : else if (by_ref)
846 64104 : type = build_pointer_type (type);
847 66678 : else if ((mask & (32 | 3)) == 1
848 66678 : && omp_privatize_by_reference (var))
849 162 : type = TREE_TYPE (type);
850 :
851 131207 : field = build_decl (DECL_SOURCE_LOCATION (var),
852 131207 : FIELD_DECL, DECL_NAME (var), type);
853 :
854 : /* Remember what variable this field was created for. This does have a
855 : side effect of making dwarf2out ignore this member, so for helpful
856 : debugging we clear it later in delete_omp_context. */
857 131207 : DECL_ABSTRACT_ORIGIN (field) = var;
858 131207 : if ((mask & 16) == 0 && type == TREE_TYPE (var))
859 : {
860 62191 : SET_DECL_ALIGN (field, DECL_ALIGN (var));
861 62191 : DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
862 62191 : TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
863 : }
864 : else
865 69016 : SET_DECL_ALIGN (field, TYPE_ALIGN (type));
866 :
867 131207 : if ((mask & 3) == 3)
868 : {
869 130155 : insert_field_into_struct (ctx->record_type, field);
870 130155 : if (ctx->srecord_type)
871 : {
872 678 : sfield = build_decl (DECL_SOURCE_LOCATION (var),
873 678 : FIELD_DECL, DECL_NAME (var), type);
874 678 : DECL_ABSTRACT_ORIGIN (sfield) = var;
875 678 : SET_DECL_ALIGN (sfield, DECL_ALIGN (field));
876 678 : DECL_USER_ALIGN (sfield) = DECL_USER_ALIGN (field);
877 678 : TREE_THIS_VOLATILE (sfield) = TREE_THIS_VOLATILE (field);
878 678 : insert_field_into_struct (ctx->srecord_type, sfield);
879 : }
880 : }
881 : else
882 : {
883 1052 : if (ctx->srecord_type == NULL_TREE)
884 : {
885 544 : tree t;
886 :
887 544 : ctx->srecord_type = lang_hooks.types.make_type (RECORD_TYPE);
888 544 : ctx->sfield_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
889 2027 : for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t))
890 : {
891 1483 : sfield = build_decl (DECL_SOURCE_LOCATION (t),
892 1483 : FIELD_DECL, DECL_NAME (t), TREE_TYPE (t));
893 1483 : DECL_ABSTRACT_ORIGIN (sfield) = DECL_ABSTRACT_ORIGIN (t);
894 1483 : insert_field_into_struct (ctx->srecord_type, sfield);
895 2966 : splay_tree_insert (ctx->sfield_map,
896 1483 : (splay_tree_key) DECL_ABSTRACT_ORIGIN (t),
897 : (splay_tree_value) sfield);
898 : }
899 : }
900 1052 : sfield = field;
901 1052 : insert_field_into_struct ((mask & 1) ? ctx->record_type
902 : : ctx->srecord_type, field);
903 : }
904 :
905 131207 : if (mask & 1)
906 130820 : splay_tree_insert (ctx->field_map, key, (splay_tree_value) field);
907 131207 : if ((mask & 2) && ctx->sfield_map)
908 1065 : splay_tree_insert (ctx->sfield_map, key, (splay_tree_value) sfield);
909 131207 : }
910 :
911 : static tree
912 229232 : install_var_local (tree var, omp_context *ctx)
913 : {
914 229232 : tree new_var = omp_copy_decl_1 (var, ctx);
915 229232 : insert_decl_map (&ctx->cb, var, new_var);
916 229232 : return new_var;
917 : }
918 :
919 : /* Adjust the replacement for DECL in CTX for the new context. This means
920 : copying the DECL_VALUE_EXPR, and fixing up the type. */
921 :
922 : static void
923 204504 : fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug)
924 : {
925 204504 : tree new_decl, size;
926 :
927 204504 : new_decl = lookup_decl (decl, ctx);
928 :
929 204504 : TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), &ctx->cb);
930 :
931 407875 : if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug)
932 204693 : && DECL_HAS_VALUE_EXPR_P (decl))
933 : {
934 1322 : tree ve = DECL_VALUE_EXPR (decl);
935 1322 : walk_tree (&ve, copy_tree_body_r, &ctx->cb, NULL);
936 1322 : SET_DECL_VALUE_EXPR (new_decl, ve);
937 1322 : DECL_HAS_VALUE_EXPR_P (new_decl) = 1;
938 : }
939 :
940 204504 : if (!TREE_CONSTANT (DECL_SIZE (new_decl)))
941 : {
942 1133 : size = remap_decl (DECL_SIZE (decl), &ctx->cb);
943 1133 : if (size == error_mark_node)
944 316 : size = TYPE_SIZE (TREE_TYPE (new_decl));
945 1133 : DECL_SIZE (new_decl) = size;
946 :
947 1133 : size = remap_decl (DECL_SIZE_UNIT (decl), &ctx->cb);
948 1133 : if (size == error_mark_node)
949 303 : size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl));
950 1133 : DECL_SIZE_UNIT (new_decl) = size;
951 : }
952 204504 : }
953 :
954 : /* The callback for remap_decl. Search all containing contexts for a
955 : mapping of the variable; this avoids having to duplicate the splay
956 : tree ahead of time. We know a mapping doesn't already exist in the
957 : given context. Create new mappings to implement default semantics. */
958 :
959 : static tree
960 464108 : omp_copy_decl (tree var, copy_body_data *cb)
961 : {
962 464108 : omp_context *ctx = (omp_context *) cb;
963 464108 : tree new_var;
964 :
965 464108 : if (TREE_CODE (var) == LABEL_DECL)
966 : {
967 188714 : if (FORCED_LABEL (var) || DECL_NONLOCAL (var))
968 : return var;
969 188708 : new_var = create_artificial_label (DECL_SOURCE_LOCATION (var));
970 188708 : DECL_CONTEXT (new_var) = current_function_decl;
971 188708 : insert_decl_map (&ctx->cb, var, new_var);
972 188708 : return new_var;
973 : }
974 :
975 357226 : while (!is_taskreg_ctx (ctx))
976 : {
977 337998 : ctx = ctx->outer;
978 337998 : if (ctx == NULL)
979 : return var;
980 279767 : new_var = maybe_lookup_decl (var, ctx);
981 279767 : if (new_var)
982 : return new_var;
983 : }
984 :
985 19228 : if (is_global_var (var) || decl_function_context (var) != ctx->cb.src_fn)
986 13371 : return var;
987 :
988 5857 : return error_mark_node;
989 : }
990 :
991 : /* Create a new context, with OUTER_CTX being the surrounding context. */
992 :
993 : static omp_context *
994 137611 : new_omp_context (gimple *stmt, omp_context *outer_ctx)
995 : {
996 137611 : omp_context *ctx = XCNEW (omp_context);
997 :
998 137611 : splay_tree_insert (all_contexts, (splay_tree_key) stmt,
999 : (splay_tree_value) ctx);
1000 137611 : ctx->stmt = stmt;
1001 :
1002 137611 : if (outer_ctx)
1003 : {
1004 74450 : ctx->outer = outer_ctx;
1005 74450 : ctx->cb = outer_ctx->cb;
1006 74450 : ctx->cb.block = NULL;
1007 74450 : ctx->depth = outer_ctx->depth + 1;
1008 : }
1009 : else
1010 : {
1011 63161 : ctx->cb.src_fn = current_function_decl;
1012 63161 : ctx->cb.dst_fn = current_function_decl;
1013 63161 : ctx->cb.src_node = cgraph_node::get (current_function_decl);
1014 63161 : gcc_checking_assert (ctx->cb.src_node);
1015 63161 : ctx->cb.dst_node = ctx->cb.src_node;
1016 63161 : ctx->cb.src_cfun = cfun;
1017 63161 : ctx->cb.copy_decl = omp_copy_decl;
1018 63161 : ctx->cb.eh_lp_nr = 0;
1019 63161 : ctx->cb.transform_call_graph_edges = CB_CGE_MOVE;
1020 63161 : ctx->cb.adjust_array_error_bounds = true;
1021 63161 : ctx->cb.dont_remap_vla_if_no_change = true;
1022 63161 : ctx->depth = 1;
1023 : }
1024 :
1025 137611 : ctx->cb.decl_map = new hash_map<tree, tree>;
1026 :
1027 137611 : return ctx;
1028 : }
1029 :
1030 : static gimple_seq maybe_catch_exception (gimple_seq);
1031 :
1032 : /* Finalize task copyfn. */
1033 :
1034 : static void
1035 504 : finalize_task_copyfn (gomp_task *task_stmt)
1036 : {
1037 504 : struct function *child_cfun;
1038 504 : tree child_fn;
1039 504 : gimple_seq seq = NULL, new_seq;
1040 504 : gbind *bind;
1041 :
1042 504 : child_fn = gimple_omp_task_copy_fn (task_stmt);
1043 504 : if (child_fn == NULL_TREE)
1044 0 : return;
1045 :
1046 504 : child_cfun = DECL_STRUCT_FUNCTION (child_fn);
1047 504 : DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
1048 :
1049 504 : push_cfun (child_cfun);
1050 504 : bind = gimplify_body (child_fn, false);
1051 504 : gimple_seq_add_stmt (&seq, bind);
1052 504 : new_seq = maybe_catch_exception (seq);
1053 504 : if (new_seq != seq)
1054 : {
1055 332 : bind = gimple_build_bind (NULL, new_seq, NULL);
1056 332 : seq = NULL;
1057 332 : gimple_seq_add_stmt (&seq, bind);
1058 : }
1059 504 : gimple_set_body (child_fn, seq);
1060 504 : pop_cfun ();
1061 :
1062 : /* Inform the callgraph about the new function. */
1063 504 : cgraph_node *node = cgraph_node::get_create (child_fn);
1064 504 : node->parallelized_function = 1;
1065 504 : cgraph_node::add_new_function (child_fn, false);
1066 : }
1067 :
1068 : /* Destroy a omp_context data structures. Called through the splay tree
1069 : value delete callback. */
1070 :
1071 : static void
1072 137605 : delete_omp_context (splay_tree_value value)
1073 : {
1074 137605 : omp_context *ctx = (omp_context *) value;
1075 :
1076 275210 : delete ctx->cb.decl_map;
1077 :
1078 137605 : if (ctx->field_map)
1079 69089 : splay_tree_delete (ctx->field_map);
1080 137605 : if (ctx->sfield_map)
1081 544 : splay_tree_delete (ctx->sfield_map);
1082 :
1083 : /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1084 : it produces corrupt debug information. */
1085 137605 : if (ctx->record_type)
1086 : {
1087 53297 : tree t;
1088 222311 : for (t = TYPE_FIELDS (ctx->record_type); t ; t = DECL_CHAIN (t))
1089 169014 : DECL_ABSTRACT_ORIGIN (t) = NULL;
1090 : }
1091 137605 : if (ctx->srecord_type)
1092 : {
1093 544 : tree t;
1094 3092 : for (t = TYPE_FIELDS (ctx->srecord_type); t ; t = DECL_CHAIN (t))
1095 2548 : DECL_ABSTRACT_ORIGIN (t) = NULL;
1096 : }
1097 :
1098 137605 : if (ctx->task_reduction_map)
1099 : {
1100 1165 : ctx->task_reductions.release ();
1101 2330 : delete ctx->task_reduction_map;
1102 : }
1103 :
1104 137878 : delete ctx->lastprivate_conditional_map;
1105 138984 : delete ctx->allocate_map;
1106 :
1107 137605 : XDELETE (ctx);
1108 137605 : }
1109 :
1110 : /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1111 : context. */
1112 :
1113 : static void
1114 36688 : fixup_child_record_type (omp_context *ctx)
1115 : {
1116 36688 : tree f, type = ctx->record_type;
1117 :
1118 36688 : if (!ctx->receiver_decl)
1119 : return;
1120 : /* ??? It isn't sufficient to just call remap_type here, because
1121 : variably_modified_type_p doesn't work the way we expect for
1122 : record types. Testing each field for whether it needs remapping
1123 : and creating a new record by hand works, however. */
1124 161462 : for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
1125 126055 : if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
1126 : break;
1127 36688 : if (f)
1128 : {
1129 1281 : tree name, new_fields = NULL;
1130 :
1131 1281 : type = lang_hooks.types.make_type (RECORD_TYPE);
1132 1281 : name = DECL_NAME (TYPE_NAME (ctx->record_type));
1133 1281 : name = build_decl (DECL_SOURCE_LOCATION (ctx->receiver_decl),
1134 : TYPE_DECL, name, type);
1135 1281 : TYPE_NAME (type) = name;
1136 :
1137 13520 : for (f = TYPE_FIELDS (ctx->record_type); f ; f = DECL_CHAIN (f))
1138 : {
1139 12239 : tree new_f = copy_node (f);
1140 12239 : DECL_CONTEXT (new_f) = type;
1141 12239 : TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &ctx->cb);
1142 12239 : DECL_CHAIN (new_f) = new_fields;
1143 12239 : walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &ctx->cb, NULL);
1144 12239 : walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r,
1145 : &ctx->cb, NULL);
1146 12239 : walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
1147 : &ctx->cb, NULL);
1148 12239 : new_fields = new_f;
1149 :
1150 : /* Arrange to be able to look up the receiver field
1151 : given the sender field. */
1152 12239 : splay_tree_insert (ctx->field_map, (splay_tree_key) f,
1153 : (splay_tree_value) new_f);
1154 : }
1155 1281 : TYPE_FIELDS (type) = nreverse (new_fields);
1156 1281 : layout_type (type);
1157 : }
1158 :
1159 : /* In a target region we never modify any of the pointers in *.omp_data_i,
1160 : so attempt to help the optimizers. */
1161 36688 : if (is_gimple_omp_offloaded (ctx->stmt))
1162 17530 : type = build_qualified_type (type, TYPE_QUAL_CONST);
1163 :
1164 36688 : TREE_TYPE (ctx->receiver_decl)
1165 110064 : = build_qualified_type (flexible_array_type_p (type)
1166 107 : ? build_pointer_type (type)
1167 36581 : : build_reference_type (type), TYPE_QUAL_RESTRICT);
1168 : }
1169 :
1170 : /* Instantiate decls as necessary in CTX to satisfy the data sharing
1171 : specified by CLAUSES. */
1172 :
1173 : static void
1174 129774 : scan_sharing_clauses (tree clauses, omp_context *ctx)
1175 : {
1176 129774 : tree c, decl;
1177 129774 : bool scan_array_reductions = false;
1178 129774 : bool flex_array_ptr = false;
1179 :
1180 550051 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1181 420277 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_ALLOCATE
1182 420277 : && (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) == NULL_TREE
1183 : /* omp_default_mem_alloc is 1 */
1184 1284 : || !integer_onep (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c))
1185 572 : || OMP_CLAUSE_ALLOCATE_ALIGN (c) != NULL_TREE))
1186 : {
1187 : /* The allocate clauses that appear on a target construct or on
1188 : constructs in a target region must specify an allocator expression
1189 : unless a requires directive with the dynamic_allocators clause
1190 : is present in the same compilation unit. */
1191 1848 : if (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) == NULL_TREE
1192 1092 : && ((omp_requires_mask & OMP_REQUIRES_DYNAMIC_ALLOCATORS) == 0)
1193 2928 : && omp_maybe_offloaded_ctx (ctx))
1194 10 : error_at (OMP_CLAUSE_LOCATION (c), "%<allocate%> clause must"
1195 : " specify an allocator here");
1196 1848 : if (ctx->allocate_map == NULL)
1197 1379 : ctx->allocate_map = new hash_map<tree, tree>;
1198 1848 : tree val = integer_zero_node;
1199 1848 : if (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c))
1200 756 : val = OMP_CLAUSE_ALLOCATE_ALLOCATOR (c);
1201 1848 : if (OMP_CLAUSE_ALLOCATE_ALIGN (c))
1202 287 : val = build_tree_list (val, OMP_CLAUSE_ALLOCATE_ALIGN (c));
1203 1848 : ctx->allocate_map->put (OMP_CLAUSE_DECL (c), val);
1204 : }
1205 :
1206 550051 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1207 : {
1208 420277 : bool by_ref;
1209 :
1210 420277 : switch (OMP_CLAUSE_CODE (c))
1211 : {
1212 73217 : case OMP_CLAUSE_PRIVATE:
1213 73217 : decl = OMP_CLAUSE_DECL (c);
1214 73217 : if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
1215 165 : goto do_private;
1216 73052 : else if (!is_variable_sized (decl))
1217 72689 : install_var_local (decl, ctx);
1218 : break;
1219 :
1220 46772 : case OMP_CLAUSE_SHARED:
1221 46772 : decl = OMP_CLAUSE_DECL (c);
1222 46772 : if (ctx->allocate_map && ctx->allocate_map->get (decl))
1223 37 : ctx->allocate_map->remove (decl);
1224 : /* Ignore shared directives in teams construct inside of
1225 : target construct. */
1226 46772 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
1227 46772 : && !is_host_teams_ctx (ctx))
1228 : {
1229 : /* Global variables don't need to be copied,
1230 : the receiver side will use them directly. */
1231 11856 : tree odecl = maybe_lookup_decl_in_outer_ctx (decl, ctx);
1232 11856 : if (is_global_var (odecl))
1233 : break;
1234 8127 : insert_decl_map (&ctx->cb, decl, odecl);
1235 8127 : break;
1236 : }
1237 34916 : gcc_assert (is_taskreg_ctx (ctx));
1238 34916 : gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl))
1239 : || !is_variable_sized (decl));
1240 : /* Global variables don't need to be copied,
1241 : the receiver side will use them directly. */
1242 34916 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1243 : break;
1244 28851 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
1245 : {
1246 86 : use_pointer_for_field (decl, ctx);
1247 86 : break;
1248 : }
1249 28765 : by_ref = use_pointer_for_field (decl, NULL);
1250 55065 : if ((! TREE_READONLY (decl) && !OMP_CLAUSE_SHARED_READONLY (c))
1251 19261 : || TREE_ADDRESSABLE (decl)
1252 18904 : || by_ref
1253 47669 : || omp_privatize_by_reference (decl))
1254 : {
1255 11972 : by_ref = use_pointer_for_field (decl, ctx);
1256 11972 : install_var_field (decl, by_ref, 3, ctx);
1257 11972 : install_var_local (decl, ctx);
1258 11972 : break;
1259 : }
1260 : /* We don't need to copy const scalar vars back. */
1261 16793 : OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE);
1262 16793 : goto do_private;
1263 :
1264 14502 : case OMP_CLAUSE_REDUCTION:
1265 : /* Collect 'reduction' clauses on OpenACC compute construct. */
1266 14502 : if (is_gimple_omp_oacc (ctx->stmt)
1267 14502 : && is_gimple_omp_offloaded (ctx->stmt))
1268 : {
1269 : /* No 'reduction' clauses on OpenACC 'kernels'. */
1270 790 : gcc_checking_assert (!is_oacc_kernels (ctx));
1271 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
1272 790 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
1273 :
1274 790 : ctx->local_reduction_clauses
1275 790 : = tree_cons (NULL, c, ctx->local_reduction_clauses);
1276 : }
1277 : /* FALLTHRU */
1278 :
1279 16576 : case OMP_CLAUSE_IN_REDUCTION:
1280 16576 : decl = OMP_CLAUSE_DECL (c);
1281 16576 : if (ctx->allocate_map
1282 16576 : && ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1283 389 : && (OMP_CLAUSE_REDUCTION_INSCAN (c)
1284 389 : || OMP_CLAUSE_REDUCTION_TASK (c)))
1285 397 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
1286 389 : || is_task_ctx (ctx)))
1287 : {
1288 : /* For now. */
1289 17 : if (ctx->allocate_map->get (decl))
1290 17 : ctx->allocate_map->remove (decl);
1291 : }
1292 16576 : if (TREE_CODE (decl) == MEM_REF)
1293 : {
1294 2366 : tree t = TREE_OPERAND (decl, 0);
1295 2366 : if (TREE_CODE (t) == POINTER_PLUS_EXPR)
1296 436 : t = TREE_OPERAND (t, 0);
1297 2366 : if (INDIRECT_REF_P (t)
1298 2272 : || TREE_CODE (t) == ADDR_EXPR)
1299 1378 : t = TREE_OPERAND (t, 0);
1300 2366 : if (is_omp_target (ctx->stmt))
1301 : {
1302 96 : if (is_variable_sized (t))
1303 : {
1304 32 : gcc_assert (DECL_HAS_VALUE_EXPR_P (t));
1305 32 : t = DECL_VALUE_EXPR (t);
1306 32 : gcc_assert (INDIRECT_REF_P (t));
1307 32 : t = TREE_OPERAND (t, 0);
1308 32 : gcc_assert (DECL_P (t));
1309 : }
1310 96 : tree at = t;
1311 96 : if (ctx->outer)
1312 96 : scan_omp_op (&at, ctx->outer);
1313 96 : tree nt = omp_copy_decl_1 (at, ctx->outer);
1314 192 : splay_tree_insert (ctx->field_map,
1315 96 : (splay_tree_key) &DECL_CONTEXT (t),
1316 : (splay_tree_value) nt);
1317 96 : if (at != t)
1318 96 : splay_tree_insert (ctx->field_map,
1319 48 : (splay_tree_key) &DECL_CONTEXT (at),
1320 : (splay_tree_value) nt);
1321 96 : break;
1322 : }
1323 2270 : install_var_local (t, ctx);
1324 2270 : if (is_taskreg_ctx (ctx)
1325 1822 : && (!is_global_var (maybe_lookup_decl_in_outer_ctx (t, ctx))
1326 387 : || (is_task_ctx (ctx)
1327 310 : && (TREE_CODE (TREE_TYPE (t)) == POINTER_TYPE
1328 234 : || (TREE_CODE (TREE_TYPE (t)) == REFERENCE_TYPE
1329 0 : && (TREE_CODE (TREE_TYPE (TREE_TYPE (t)))
1330 : == POINTER_TYPE)))))
1331 1511 : && !is_variable_sized (t)
1332 3722 : && (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
1333 617 : || (!OMP_CLAUSE_REDUCTION_TASK (c)
1334 578 : && !is_task_ctx (ctx))))
1335 : {
1336 1335 : by_ref = use_pointer_for_field (t, NULL);
1337 1335 : if (is_task_ctx (ctx)
1338 835 : && TREE_CODE (TREE_TYPE (t)) == REFERENCE_TYPE
1339 1495 : && TREE_CODE (TREE_TYPE (TREE_TYPE (t))) == POINTER_TYPE)
1340 : {
1341 32 : install_var_field (t, false, 1, ctx);
1342 32 : install_var_field (t, by_ref, 2, ctx);
1343 : }
1344 : else
1345 1303 : install_var_field (t, by_ref, 3, ctx);
1346 : }
1347 : break;
1348 : }
1349 14210 : if (is_omp_target (ctx->stmt))
1350 : {
1351 335 : tree at = decl;
1352 335 : if (ctx->outer)
1353 54 : scan_omp_op (&at, ctx->outer);
1354 335 : tree nt = omp_copy_decl_1 (at, ctx->outer);
1355 670 : splay_tree_insert (ctx->field_map,
1356 335 : (splay_tree_key) &DECL_CONTEXT (decl),
1357 : (splay_tree_value) nt);
1358 335 : if (at != decl)
1359 24 : splay_tree_insert (ctx->field_map,
1360 12 : (splay_tree_key) &DECL_CONTEXT (at),
1361 : (splay_tree_value) nt);
1362 335 : break;
1363 : }
1364 13875 : if (is_task_ctx (ctx)
1365 26680 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1366 12805 : && OMP_CLAUSE_REDUCTION_TASK (c)
1367 373 : && is_parallel_ctx (ctx)))
1368 : {
1369 : /* Global variables don't need to be copied,
1370 : the receiver side will use them directly. */
1371 1150 : if (!is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1372 : {
1373 462 : by_ref = use_pointer_for_field (decl, ctx);
1374 462 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION)
1375 304 : install_var_field (decl, by_ref, 3, ctx);
1376 : }
1377 1150 : install_var_local (decl, ctx);
1378 1150 : break;
1379 : }
1380 12725 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1381 12725 : && OMP_CLAUSE_REDUCTION_TASK (c))
1382 : {
1383 293 : install_var_local (decl, ctx);
1384 293 : break;
1385 : }
1386 12432 : goto do_private;
1387 :
1388 22736 : case OMP_CLAUSE_LASTPRIVATE:
1389 : /* Let the corresponding firstprivate clause create
1390 : the variable. */
1391 22736 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1392 : break;
1393 : /* FALLTHRU */
1394 :
1395 59369 : case OMP_CLAUSE_FIRSTPRIVATE:
1396 59369 : case OMP_CLAUSE_LINEAR:
1397 59369 : decl = OMP_CLAUSE_DECL (c);
1398 89147 : do_private:
1399 89147 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1400 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR
1401 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1402 45776 : && is_gimple_omp_offloaded (ctx->stmt))
1403 : {
1404 16262 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1405 16262 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR
1406 239 : && lang_hooks.decls.omp_array_data (decl, true)))
1407 : {
1408 : /* OpenACC firstprivate clauses are later processed with same
1409 : code path as map clauses in lower_omp_target, so follow
1410 : the same convention of using the whole clause expression
1411 : as splay-tree key. */
1412 15893 : tree k = (is_oacc_parallel_or_serial (ctx) ? c : NULL_TREE);
1413 15893 : by_ref = !omp_privatize_by_reference (decl);
1414 15893 : install_var_field (decl, by_ref, 3, ctx, k);
1415 : }
1416 369 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1417 : {
1418 220 : if (INDIRECT_REF_P (decl))
1419 0 : decl = TREE_OPERAND (decl, 0);
1420 220 : install_var_field (decl, true, 3, ctx);
1421 : }
1422 149 : else if (TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1423 8 : install_var_field (decl, true, 3, ctx);
1424 : else
1425 141 : install_var_field (decl, false, 3, ctx);
1426 : }
1427 89147 : if (is_variable_sized (decl))
1428 : {
1429 60 : if (is_task_ctx (ctx))
1430 : {
1431 14 : if (ctx->allocate_map
1432 14 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
1433 : {
1434 : /* For now. */
1435 0 : if (ctx->allocate_map->get (decl))
1436 0 : ctx->allocate_map->remove (decl);
1437 : }
1438 14 : install_var_field (decl, false, 1, ctx);
1439 : }
1440 : break;
1441 : }
1442 89087 : else if (is_taskreg_ctx (ctx))
1443 : {
1444 38338 : bool global
1445 38338 : = is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx));
1446 38338 : by_ref = use_pointer_for_field (decl, NULL);
1447 :
1448 38338 : if (is_task_ctx (ctx)
1449 38338 : && (global || by_ref || omp_privatize_by_reference (decl)))
1450 : {
1451 619 : if (ctx->allocate_map
1452 619 : && ctx->allocate_map->get (decl))
1453 59 : install_var_field (decl, by_ref, 32 | 1, ctx);
1454 : else
1455 560 : install_var_field (decl, false, 1, ctx);
1456 619 : if (!global)
1457 355 : install_var_field (decl, by_ref, 2, ctx);
1458 : }
1459 37719 : else if (!global)
1460 36450 : install_var_field (decl, by_ref, 3, ctx);
1461 : }
1462 89087 : install_var_local (decl, ctx);
1463 : /* For descr arrays on target: firstprivatize data + attach ptr. */
1464 89087 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1465 45341 : && is_gimple_omp_offloaded (ctx->stmt)
1466 15868 : && !is_gimple_omp_oacc (ctx->stmt)
1467 11620 : && lang_hooks.decls.omp_array_data (decl, true)
1468 89118 : && lang_hooks.decls.omp_array_data_privatize (decl))
1469 : {
1470 23 : install_var_field (decl, false, 16 | 3, ctx);
1471 23 : install_var_field (decl, true, 8 | 3, ctx);
1472 : }
1473 : break;
1474 :
1475 2118 : case OMP_CLAUSE_USE_DEVICE_PTR:
1476 2118 : case OMP_CLAUSE_USE_DEVICE_ADDR:
1477 2118 : decl = OMP_CLAUSE_DECL (c);
1478 :
1479 : /* Fortran array descriptors. */
1480 2118 : if (lang_hooks.decls.omp_array_data (decl, true))
1481 601 : install_var_field (decl, false, 19, ctx);
1482 1517 : else if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
1483 1312 : && !omp_privatize_by_reference (decl)
1484 513 : && !omp_is_allocatable_or_ptr (decl))
1485 2454 : || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1486 407 : install_var_field (decl, true, 11, ctx);
1487 : else
1488 1110 : install_var_field (decl, false, 11, ctx);
1489 2118 : if (DECL_SIZE (decl)
1490 2118 : && !poly_int_tree_p (DECL_SIZE (decl)))
1491 : {
1492 6 : tree decl2 = DECL_VALUE_EXPR (decl);
1493 6 : gcc_assert (INDIRECT_REF_P (decl2));
1494 6 : decl2 = TREE_OPERAND (decl2, 0);
1495 6 : gcc_assert (DECL_P (decl2));
1496 6 : install_var_local (decl2, ctx);
1497 : }
1498 2118 : install_var_local (decl, ctx);
1499 2118 : break;
1500 :
1501 239 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
1502 239 : decl = OMP_CLAUSE_DECL (c);
1503 239 : while (INDIRECT_REF_P (decl)
1504 256 : || TREE_CODE (decl) == ARRAY_REF)
1505 17 : decl = TREE_OPERAND (decl, 0);
1506 239 : goto do_private;
1507 :
1508 149 : case OMP_CLAUSE_IS_DEVICE_PTR:
1509 149 : decl = OMP_CLAUSE_DECL (c);
1510 149 : goto do_private;
1511 :
1512 20111 : case OMP_CLAUSE__LOOPTEMP_:
1513 20111 : case OMP_CLAUSE__REDUCTEMP_:
1514 20111 : gcc_assert (is_taskreg_ctx (ctx));
1515 20111 : decl = OMP_CLAUSE_DECL (c);
1516 20111 : install_var_field (decl, false, 3, ctx);
1517 20111 : install_var_local (decl, ctx);
1518 20111 : break;
1519 :
1520 892 : case OMP_CLAUSE_COPYPRIVATE:
1521 892 : case OMP_CLAUSE_COPYIN:
1522 892 : decl = OMP_CLAUSE_DECL (c);
1523 892 : by_ref = use_pointer_for_field (decl, NULL);
1524 892 : install_var_field (decl, by_ref, 3, ctx);
1525 892 : break;
1526 :
1527 55243 : case OMP_CLAUSE_FINAL:
1528 55243 : case OMP_CLAUSE_IF:
1529 55243 : case OMP_CLAUSE_SELF:
1530 55243 : case OMP_CLAUSE_NUM_THREADS:
1531 55243 : case OMP_CLAUSE_NUM_TEAMS:
1532 55243 : case OMP_CLAUSE_THREAD_LIMIT:
1533 55243 : case OMP_CLAUSE_DEVICE:
1534 55243 : case OMP_CLAUSE_SCHEDULE:
1535 55243 : case OMP_CLAUSE_DIST_SCHEDULE:
1536 55243 : case OMP_CLAUSE_DEPEND:
1537 55243 : case OMP_CLAUSE_PRIORITY:
1538 55243 : case OMP_CLAUSE_GRAINSIZE:
1539 55243 : case OMP_CLAUSE_NUM_TASKS:
1540 55243 : case OMP_CLAUSE_NUM_GANGS:
1541 55243 : case OMP_CLAUSE_NUM_WORKERS:
1542 55243 : case OMP_CLAUSE_VECTOR_LENGTH:
1543 55243 : case OMP_CLAUSE_DETACH:
1544 55243 : case OMP_CLAUSE_FILTER:
1545 55243 : if (ctx->outer)
1546 17564 : scan_omp_op (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer);
1547 : break;
1548 :
1549 88308 : case OMP_CLAUSE_TO:
1550 88308 : case OMP_CLAUSE_FROM:
1551 88308 : case OMP_CLAUSE_MAP:
1552 88308 : if (ctx->outer)
1553 15385 : scan_omp_op (&OMP_CLAUSE_SIZE (c), ctx->outer);
1554 88308 : decl = OMP_CLAUSE_DECL (c);
1555 : /* If requested, make 'decl' addressable. */
1556 88308 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1557 88308 : && OMP_CLAUSE_MAP_DECL_MAKE_ADDRESSABLE (c))
1558 : {
1559 433 : gcc_checking_assert (DECL_P (decl));
1560 :
1561 433 : bool decl_addressable = TREE_ADDRESSABLE (decl);
1562 433 : if (!decl_addressable)
1563 : {
1564 219 : if (!make_addressable_vars)
1565 131 : make_addressable_vars = BITMAP_ALLOC (NULL);
1566 219 : bitmap_set_bit (make_addressable_vars, DECL_UID (decl));
1567 219 : TREE_ADDRESSABLE (decl) = 1;
1568 : }
1569 :
1570 433 : if (dump_enabled_p ())
1571 : {
1572 421 : location_t loc = OMP_CLAUSE_LOCATION (c);
1573 421 : const dump_user_location_t d_u_loc
1574 421 : = dump_user_location_t::from_location_t (loc);
1575 : /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */
1576 : #if __GNUC__ >= 10
1577 421 : # pragma GCC diagnostic push
1578 421 : # pragma GCC diagnostic ignored "-Wformat"
1579 : #endif
1580 421 : if (!decl_addressable)
1581 207 : dump_printf_loc (MSG_NOTE, d_u_loc,
1582 : "variable %<%T%>"
1583 : " made addressable\n",
1584 : decl);
1585 : else
1586 214 : dump_printf_loc (MSG_NOTE, d_u_loc,
1587 : "variable %<%T%>"
1588 : " already made addressable\n",
1589 : decl);
1590 : #if __GNUC__ >= 10
1591 421 : # pragma GCC diagnostic pop
1592 : #endif
1593 : }
1594 :
1595 : /* Done. */
1596 433 : OMP_CLAUSE_MAP_DECL_MAKE_ADDRESSABLE (c) = 0;
1597 : }
1598 : /* Global variables with "omp declare target" attribute
1599 : don't need to be copied, the receiver side will use them
1600 : directly. However, global variables with "omp declare target link"
1601 : attribute need to be copied. Or when ALWAYS modifier is used. */
1602 88308 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1603 80525 : && DECL_P (decl)
1604 44346 : && ((OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER
1605 40529 : && (OMP_CLAUSE_MAP_KIND (c)
1606 : != GOMP_MAP_FIRSTPRIVATE_REFERENCE)
1607 40100 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ATTACH
1608 39866 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_DETACH)
1609 4545 : || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1610 41004 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_TO
1611 40869 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_FROM
1612 40840 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_TOFROM
1613 40352 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_TO
1614 40322 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_FROM
1615 40300 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_TOFROM
1616 40286 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_TO_PSET
1617 36297 : && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1618 7560 : && varpool_node::get_create (decl)->offloadable
1619 93574 : && !lookup_attribute ("omp declare target link",
1620 5266 : DECL_ATTRIBUTES (decl)))
1621 : break;
1622 83070 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1623 83070 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER)
1624 : {
1625 : /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
1626 : not offloaded; there is nothing to map for those. */
1627 10487 : if (!is_gimple_omp_offloaded (ctx->stmt)
1628 4978 : && !POINTER_TYPE_P (TREE_TYPE (decl))
1629 10751 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
1630 : break;
1631 : }
1632 82806 : if (!flex_array_ptr)
1633 82578 : flex_array_ptr = lang_hooks.decls.omp_deep_mapping_p (ctx->stmt, c);
1634 82806 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1635 75023 : && DECL_P (decl)
1636 38844 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
1637 38610 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
1638 83105 : && is_omp_target (ctx->stmt))
1639 : {
1640 : /* If this is an offloaded region, an attach operation should
1641 : only exist when the pointer variable is mapped in a prior
1642 : clause. An exception is if we have a reference (to pointer):
1643 : in that case we should have mapped "*decl" in a previous
1644 : mapping instead of "decl". Skip the assertion in that case.
1645 : If we had an error, we may not have attempted to sort clauses
1646 : properly, so avoid the test. */
1647 261 : if (TREE_CODE (TREE_TYPE (decl)) != REFERENCE_TYPE
1648 229 : && is_gimple_omp_offloaded (ctx->stmt)
1649 331 : && !seen_error ())
1650 62 : gcc_assert
1651 : (maybe_lookup_decl (decl, ctx)
1652 : || (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1653 : && lookup_attribute ("omp declare target",
1654 : DECL_ATTRIBUTES (decl))));
1655 :
1656 : /* By itself, attach/detach is generated as part of pointer
1657 : variable mapping and should not create new variables in the
1658 : offloaded region, however sender refs for it must be created
1659 : for its address to be passed to the runtime. */
1660 261 : tree field
1661 261 : = build_decl (OMP_CLAUSE_LOCATION (c),
1662 : FIELD_DECL, NULL_TREE, ptr_type_node);
1663 261 : SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
1664 261 : insert_field_into_struct (ctx->record_type, field);
1665 : /* To not clash with a map of the pointer variable itself,
1666 : attach/detach maps have their field looked up by the *clause*
1667 : tree expression, not the decl. */
1668 261 : gcc_assert (!splay_tree_lookup (ctx->field_map,
1669 : (splay_tree_key) c));
1670 261 : splay_tree_insert (ctx->field_map, (splay_tree_key) c,
1671 : (splay_tree_value) field);
1672 261 : break;
1673 : }
1674 82545 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1675 82545 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
1676 70947 : || (OMP_CLAUSE_MAP_KIND (c)
1677 : == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
1678 : {
1679 4244 : if (TREE_CODE (decl) == COMPONENT_REF
1680 4244 : || (INDIRECT_REF_P (decl)
1681 0 : && TREE_CODE (TREE_OPERAND (decl, 0)) == COMPONENT_REF
1682 0 : && (((TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0)))
1683 : == REFERENCE_TYPE)
1684 0 : || (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0)))
1685 : == POINTER_TYPE)))))
1686 : break;
1687 4244 : if (DECL_SIZE (decl)
1688 4244 : && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
1689 : {
1690 710 : tree decl2 = DECL_VALUE_EXPR (decl);
1691 710 : gcc_assert (INDIRECT_REF_P (decl2));
1692 710 : decl2 = TREE_OPERAND (decl2, 0);
1693 710 : gcc_assert (DECL_P (decl2));
1694 710 : install_var_local (decl2, ctx);
1695 : }
1696 4244 : install_var_local (decl, ctx);
1697 4244 : break;
1698 : }
1699 78301 : if (DECL_P (decl))
1700 : {
1701 40500 : if (DECL_SIZE (decl)
1702 40500 : && !poly_int_tree_p (DECL_SIZE (decl)))
1703 : {
1704 0 : tree decl2 = DECL_VALUE_EXPR (decl);
1705 0 : gcc_assert (INDIRECT_REF_P (decl2));
1706 0 : decl2 = TREE_OPERAND (decl2, 0);
1707 0 : gcc_assert (DECL_P (decl2));
1708 0 : install_var_field (decl2, true, 3, ctx, c);
1709 0 : install_var_local (decl2, ctx);
1710 0 : install_var_local (decl, ctx);
1711 : }
1712 : else
1713 : {
1714 40500 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1715 34339 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
1716 5204 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
1717 45704 : && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1718 425 : install_var_field (decl, true, 7, ctx, c);
1719 : else
1720 40075 : install_var_field (decl, true, 3, ctx, c);
1721 40500 : if (is_gimple_omp_offloaded (ctx->stmt)
1722 40500 : && !(is_gimple_omp_oacc (ctx->stmt)
1723 13857 : && OMP_CLAUSE_MAP_IN_REDUCTION (c)))
1724 23818 : install_var_local (decl, ctx);
1725 : }
1726 : }
1727 : else
1728 : {
1729 37801 : tree base = get_base_address (decl);
1730 37801 : tree nc = OMP_CLAUSE_CHAIN (c);
1731 37801 : if (DECL_P (base)
1732 9266 : && nc != NULL_TREE
1733 6676 : && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP
1734 5514 : && OMP_CLAUSE_DECL (nc) == base
1735 1184 : && OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_POINTER
1736 38469 : && integer_zerop (OMP_CLAUSE_SIZE (nc)))
1737 : {
1738 0 : OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) = 1;
1739 0 : OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc) = 1;
1740 : }
1741 : else
1742 : {
1743 37801 : if (ctx->outer)
1744 : {
1745 7152 : scan_omp_op (&OMP_CLAUSE_DECL (c), ctx->outer);
1746 7152 : decl = OMP_CLAUSE_DECL (c);
1747 : }
1748 37801 : gcc_assert (!splay_tree_lookup (ctx->field_map,
1749 : (splay_tree_key) decl));
1750 37801 : tree field
1751 37801 : = build_decl (OMP_CLAUSE_LOCATION (c),
1752 : FIELD_DECL, NULL_TREE, ptr_type_node);
1753 37801 : SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
1754 37801 : insert_field_into_struct (ctx->record_type, field);
1755 37801 : splay_tree_insert (ctx->field_map, (splay_tree_key) c,
1756 : (splay_tree_value) field);
1757 : }
1758 : }
1759 : break;
1760 :
1761 3973 : case OMP_CLAUSE_ORDER:
1762 3973 : ctx->order_concurrent = true;
1763 3973 : break;
1764 :
1765 1907 : case OMP_CLAUSE_BIND:
1766 1907 : ctx->loop_p = true;
1767 1907 : break;
1768 :
1769 : case OMP_CLAUSE_NOWAIT:
1770 : case OMP_CLAUSE_ORDERED:
1771 : case OMP_CLAUSE_COLLAPSE:
1772 : case OMP_CLAUSE_UNTIED:
1773 : case OMP_CLAUSE_MERGEABLE:
1774 : case OMP_CLAUSE_PROC_BIND:
1775 : case OMP_CLAUSE_SAFELEN:
1776 : case OMP_CLAUSE_SIMDLEN:
1777 : case OMP_CLAUSE_THREADS:
1778 : case OMP_CLAUSE_SIMD:
1779 : case OMP_CLAUSE_NOGROUP:
1780 : case OMP_CLAUSE_DEFAULTMAP:
1781 : case OMP_CLAUSE_ASYNC:
1782 : case OMP_CLAUSE_WAIT:
1783 : case OMP_CLAUSE_GANG:
1784 : case OMP_CLAUSE_WORKER:
1785 : case OMP_CLAUSE_VECTOR:
1786 : case OMP_CLAUSE_INDEPENDENT:
1787 : case OMP_CLAUSE_AUTO:
1788 : case OMP_CLAUSE_SEQ:
1789 : case OMP_CLAUSE_TILE:
1790 : case OMP_CLAUSE__SIMT_:
1791 : case OMP_CLAUSE_DEFAULT:
1792 : case OMP_CLAUSE_NONTEMPORAL:
1793 : case OMP_CLAUSE_IF_PRESENT:
1794 : case OMP_CLAUSE_FINALIZE:
1795 : case OMP_CLAUSE_TASK_REDUCTION:
1796 : case OMP_CLAUSE_ALLOCATE:
1797 : case OMP_CLAUSE_DEVICE_TYPE:
1798 : break;
1799 :
1800 162 : case OMP_CLAUSE_ALIGNED:
1801 162 : decl = OMP_CLAUSE_DECL (c);
1802 162 : if (is_global_var (decl)
1803 162 : && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1804 86 : install_var_local (decl, ctx);
1805 : break;
1806 :
1807 361 : case OMP_CLAUSE__CONDTEMP_:
1808 361 : decl = OMP_CLAUSE_DECL (c);
1809 361 : if (is_parallel_ctx (ctx))
1810 : {
1811 118 : install_var_field (decl, false, 3, ctx);
1812 118 : install_var_local (decl, ctx);
1813 : }
1814 243 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
1815 243 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
1816 368 : && !OMP_CLAUSE__CONDTEMP__ITER (c))
1817 125 : install_var_local (decl, ctx);
1818 : break;
1819 :
1820 : case OMP_CLAUSE_INIT:
1821 : case OMP_CLAUSE_USE:
1822 : case OMP_CLAUSE_DESTROY:
1823 : break;
1824 :
1825 0 : case OMP_CLAUSE__CACHE_:
1826 0 : case OMP_CLAUSE_NOHOST:
1827 0 : default:
1828 0 : gcc_unreachable ();
1829 : }
1830 : }
1831 :
1832 550051 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1833 : {
1834 420277 : switch (OMP_CLAUSE_CODE (c))
1835 : {
1836 22736 : case OMP_CLAUSE_LASTPRIVATE:
1837 : /* Let the corresponding firstprivate clause create
1838 : the variable. */
1839 22736 : if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
1840 7245 : scan_array_reductions = true;
1841 22736 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1842 : break;
1843 : /* FALLTHRU */
1844 :
1845 149767 : case OMP_CLAUSE_FIRSTPRIVATE:
1846 149767 : case OMP_CLAUSE_PRIVATE:
1847 149767 : case OMP_CLAUSE_LINEAR:
1848 149767 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
1849 149767 : case OMP_CLAUSE_IS_DEVICE_PTR:
1850 149767 : decl = OMP_CLAUSE_DECL (c);
1851 149767 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1852 : {
1853 256 : while (INDIRECT_REF_P (decl)
1854 256 : || TREE_CODE (decl) == ARRAY_REF)
1855 17 : decl = TREE_OPERAND (decl, 0);
1856 : }
1857 :
1858 149767 : if (is_variable_sized (decl))
1859 : {
1860 423 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1861 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR
1862 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1863 53 : && is_gimple_omp_offloaded (ctx->stmt))
1864 : {
1865 12 : tree decl2 = DECL_VALUE_EXPR (decl);
1866 12 : gcc_assert (INDIRECT_REF_P (decl2));
1867 12 : decl2 = TREE_OPERAND (decl2, 0);
1868 12 : gcc_assert (DECL_P (decl2));
1869 12 : install_var_local (decl2, ctx);
1870 12 : fixup_remapped_decl (decl2, ctx, false);
1871 : }
1872 423 : install_var_local (decl, ctx);
1873 : }
1874 299534 : fixup_remapped_decl (decl, ctx,
1875 149767 : OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE
1876 149767 : && OMP_CLAUSE_PRIVATE_DEBUG (c));
1877 149767 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
1878 149767 : && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
1879 : scan_array_reductions = true;
1880 : break;
1881 :
1882 16576 : case OMP_CLAUSE_REDUCTION:
1883 16576 : case OMP_CLAUSE_IN_REDUCTION:
1884 16576 : decl = OMP_CLAUSE_DECL (c);
1885 16576 : if (TREE_CODE (decl) != MEM_REF && !is_omp_target (ctx->stmt))
1886 : {
1887 13875 : if (is_variable_sized (decl))
1888 0 : install_var_local (decl, ctx);
1889 13875 : fixup_remapped_decl (decl, ctx, false);
1890 : }
1891 16576 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1892 2840 : scan_array_reductions = true;
1893 : break;
1894 :
1895 530 : case OMP_CLAUSE_TASK_REDUCTION:
1896 530 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1897 2840 : scan_array_reductions = true;
1898 : break;
1899 :
1900 29979 : case OMP_CLAUSE_SHARED:
1901 : /* Ignore shared directives in teams construct inside of
1902 : target construct. */
1903 29979 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
1904 29979 : && !is_host_teams_ctx (ctx))
1905 : break;
1906 18123 : decl = OMP_CLAUSE_DECL (c);
1907 18123 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1908 : break;
1909 12058 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
1910 : {
1911 86 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl,
1912 : ctx->outer)))
1913 : break;
1914 79 : bool by_ref = use_pointer_for_field (decl, ctx);
1915 79 : install_var_field (decl, by_ref, 11, ctx);
1916 79 : break;
1917 : }
1918 11972 : fixup_remapped_decl (decl, ctx, false);
1919 11972 : break;
1920 :
1921 80525 : case OMP_CLAUSE_MAP:
1922 80525 : if (!is_gimple_omp_offloaded (ctx->stmt))
1923 : break;
1924 53789 : decl = OMP_CLAUSE_DECL (c);
1925 53789 : if (DECL_P (decl)
1926 33874 : && ((OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER
1927 30225 : && (OMP_CLAUSE_MAP_KIND (c)
1928 : != GOMP_MAP_FIRSTPRIVATE_REFERENCE))
1929 4078 : || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1930 30999 : && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1931 61172 : && varpool_node::get_create (decl)->offloadable)
1932 : break;
1933 48107 : if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
1934 45895 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
1935 2212 : && is_omp_target (ctx->stmt)
1936 50110 : && !is_gimple_omp_offloaded (ctx->stmt))
1937 : break;
1938 48107 : if (DECL_P (decl))
1939 : {
1940 28192 : if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
1941 25139 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER)
1942 6700 : && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
1943 29817 : && !COMPLETE_TYPE_P (TREE_TYPE (decl)))
1944 : {
1945 24 : tree new_decl = lookup_decl (decl, ctx);
1946 24 : TREE_TYPE (new_decl)
1947 48 : = remap_type (TREE_TYPE (decl), &ctx->cb);
1948 : }
1949 28168 : else if (DECL_SIZE (decl)
1950 28168 : && !poly_int_tree_p (DECL_SIZE (decl)))
1951 : {
1952 710 : tree decl2 = DECL_VALUE_EXPR (decl);
1953 710 : gcc_assert (INDIRECT_REF_P (decl2));
1954 710 : decl2 = TREE_OPERAND (decl2, 0);
1955 710 : gcc_assert (DECL_P (decl2));
1956 710 : fixup_remapped_decl (decl2, ctx, false);
1957 710 : fixup_remapped_decl (decl, ctx, true);
1958 : }
1959 : else
1960 27458 : fixup_remapped_decl (decl, ctx, false);
1961 : }
1962 : break;
1963 :
1964 : case OMP_CLAUSE_COPYPRIVATE:
1965 : case OMP_CLAUSE_COPYIN:
1966 : case OMP_CLAUSE_DEFAULT:
1967 : case OMP_CLAUSE_IF:
1968 : case OMP_CLAUSE_SELF:
1969 : case OMP_CLAUSE_NUM_THREADS:
1970 : case OMP_CLAUSE_NUM_TEAMS:
1971 : case OMP_CLAUSE_THREAD_LIMIT:
1972 : case OMP_CLAUSE_DEVICE:
1973 : case OMP_CLAUSE_SCHEDULE:
1974 : case OMP_CLAUSE_DIST_SCHEDULE:
1975 : case OMP_CLAUSE_NOWAIT:
1976 : case OMP_CLAUSE_ORDERED:
1977 : case OMP_CLAUSE_COLLAPSE:
1978 : case OMP_CLAUSE_UNTIED:
1979 : case OMP_CLAUSE_FINAL:
1980 : case OMP_CLAUSE_MERGEABLE:
1981 : case OMP_CLAUSE_PROC_BIND:
1982 : case OMP_CLAUSE_SAFELEN:
1983 : case OMP_CLAUSE_SIMDLEN:
1984 : case OMP_CLAUSE_ALIGNED:
1985 : case OMP_CLAUSE_DEPEND:
1986 : case OMP_CLAUSE_DETACH:
1987 : case OMP_CLAUSE_ALLOCATE:
1988 : case OMP_CLAUSE__LOOPTEMP_:
1989 : case OMP_CLAUSE__REDUCTEMP_:
1990 : case OMP_CLAUSE_TO:
1991 : case OMP_CLAUSE_FROM:
1992 : case OMP_CLAUSE_PRIORITY:
1993 : case OMP_CLAUSE_GRAINSIZE:
1994 : case OMP_CLAUSE_NUM_TASKS:
1995 : case OMP_CLAUSE_THREADS:
1996 : case OMP_CLAUSE_SIMD:
1997 : case OMP_CLAUSE_NOGROUP:
1998 : case OMP_CLAUSE_DEFAULTMAP:
1999 : case OMP_CLAUSE_ORDER:
2000 : case OMP_CLAUSE_BIND:
2001 : case OMP_CLAUSE_USE_DEVICE_PTR:
2002 : case OMP_CLAUSE_USE_DEVICE_ADDR:
2003 : case OMP_CLAUSE_NONTEMPORAL:
2004 : case OMP_CLAUSE_ASYNC:
2005 : case OMP_CLAUSE_WAIT:
2006 : case OMP_CLAUSE_NUM_GANGS:
2007 : case OMP_CLAUSE_NUM_WORKERS:
2008 : case OMP_CLAUSE_VECTOR_LENGTH:
2009 : case OMP_CLAUSE_GANG:
2010 : case OMP_CLAUSE_WORKER:
2011 : case OMP_CLAUSE_VECTOR:
2012 : case OMP_CLAUSE_INDEPENDENT:
2013 : case OMP_CLAUSE_AUTO:
2014 : case OMP_CLAUSE_SEQ:
2015 : case OMP_CLAUSE_TILE:
2016 : case OMP_CLAUSE__SIMT_:
2017 : case OMP_CLAUSE_IF_PRESENT:
2018 : case OMP_CLAUSE_FINALIZE:
2019 : case OMP_CLAUSE_FILTER:
2020 : case OMP_CLAUSE__CONDTEMP_:
2021 : case OMP_CLAUSE_INIT:
2022 : case OMP_CLAUSE_USE:
2023 : case OMP_CLAUSE_DESTROY:
2024 : case OMP_CLAUSE_DEVICE_TYPE:
2025 : break;
2026 :
2027 0 : case OMP_CLAUSE__CACHE_:
2028 0 : case OMP_CLAUSE_NOHOST:
2029 0 : default:
2030 0 : gcc_unreachable ();
2031 : }
2032 : }
2033 :
2034 129774 : gcc_checking_assert (!scan_array_reductions
2035 : || !is_gimple_omp_oacc (ctx->stmt));
2036 : if (scan_array_reductions)
2037 : {
2038 34630 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2039 29440 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
2040 27058 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
2041 26434 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
2042 30144 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
2043 : {
2044 2184 : omp_context *rctx = ctx;
2045 2184 : if (is_omp_target (ctx->stmt))
2046 60 : rctx = ctx->outer;
2047 2184 : scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), rctx);
2048 2184 : scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), rctx);
2049 : }
2050 27256 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
2051 27256 : && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
2052 7245 : scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
2053 20011 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
2054 20011 : && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
2055 656 : scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
2056 : }
2057 129774 : if (flex_array_ptr)
2058 : {
2059 144 : tree field = build_range_type (size_type_node,
2060 : build_int_cstu (size_type_node, 0),
2061 : NULL_TREE);
2062 144 : field = build_array_type (ptr_type_node, field);
2063 144 : field = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE, field);
2064 144 : SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
2065 144 : DECL_CONTEXT (field) = ctx->record_type;
2066 144 : DECL_CHAIN (field) = TYPE_FIELDS (ctx->record_type);
2067 144 : TYPE_FIELDS (ctx->record_type) = field;
2068 : }
2069 129774 : }
2070 :
2071 : /* Create a new name for omp child function. Returns an identifier. */
2072 :
2073 : static tree
2074 51384 : create_omp_child_function_name (bool task_copy)
2075 : {
2076 102224 : return clone_function_name_numbered (current_function_decl,
2077 51384 : task_copy ? "_omp_cpyfn" : "_omp_fn");
2078 : }
2079 :
2080 : /* Return true if CTX may belong to offloaded code: either if current function
2081 : is offloaded, or any enclosing context corresponds to a target region. */
2082 :
2083 : static bool
2084 122812 : omp_maybe_offloaded_ctx (omp_context *ctx)
2085 : {
2086 122812 : if (cgraph_node::get (current_function_decl)->offloadable)
2087 : return true;
2088 203244 : for (; ctx; ctx = ctx->outer)
2089 120186 : if (is_gimple_omp_offloaded (ctx->stmt))
2090 : return true;
2091 : return false;
2092 : }
2093 :
2094 : /* Build a decl for the omp child function. It'll not contain a body
2095 : yet, just the bare decl. */
2096 :
2097 : static void
2098 51384 : create_omp_child_function (omp_context *ctx, bool task_copy)
2099 : {
2100 51384 : tree decl, type, name, t;
2101 :
2102 51384 : name = create_omp_child_function_name (task_copy);
2103 51384 : if (task_copy)
2104 544 : type = build_function_type_list (void_type_node, ptr_type_node,
2105 : ptr_type_node, NULL_TREE);
2106 : else
2107 50840 : type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE);
2108 :
2109 51384 : decl = build_decl (gimple_location (ctx->stmt), FUNCTION_DECL, name, type);
2110 :
2111 51384 : gcc_checking_assert (!is_gimple_omp_oacc (ctx->stmt)
2112 : || !task_copy);
2113 39998 : if (!task_copy)
2114 50840 : ctx->cb.dst_fn = decl;
2115 : else
2116 544 : gimple_omp_task_set_copy_fn (ctx->stmt, decl);
2117 :
2118 51384 : TREE_STATIC (decl) = 1;
2119 51384 : TREE_USED (decl) = 1;
2120 51384 : DECL_ARTIFICIAL (decl) = 1;
2121 51384 : DECL_IGNORED_P (decl) = 0;
2122 51384 : TREE_PUBLIC (decl) = 0;
2123 51384 : DECL_UNINLINABLE (decl) = 1;
2124 51384 : DECL_EXTERNAL (decl) = 0;
2125 51384 : DECL_CONTEXT (decl) = NULL_TREE;
2126 51384 : DECL_INITIAL (decl) = make_node (BLOCK);
2127 51384 : BLOCK_SUPERCONTEXT (DECL_INITIAL (decl)) = decl;
2128 51384 : DECL_ATTRIBUTES (decl) = DECL_ATTRIBUTES (current_function_decl);
2129 : /* Remove omp declare simd attribute from the new attributes. */
2130 51384 : if (tree a = lookup_attribute ("omp declare simd", DECL_ATTRIBUTES (decl)))
2131 : {
2132 9 : while (tree a2 = lookup_attribute ("omp declare simd", TREE_CHAIN (a)))
2133 : a = a2;
2134 9 : a = TREE_CHAIN (a);
2135 22 : for (tree *p = &DECL_ATTRIBUTES (decl); *p != a;)
2136 13 : if (is_attribute_p ("omp declare simd", get_attribute_name (*p)))
2137 9 : *p = TREE_CHAIN (*p);
2138 : else
2139 : {
2140 4 : tree chain = TREE_CHAIN (*p);
2141 4 : *p = copy_node (*p);
2142 4 : p = &TREE_CHAIN (*p);
2143 4 : *p = chain;
2144 : }
2145 : }
2146 102768 : DECL_FUNCTION_SPECIFIC_OPTIMIZATION (decl)
2147 51384 : = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (current_function_decl);
2148 102768 : DECL_FUNCTION_SPECIFIC_TARGET (decl)
2149 51384 : = DECL_FUNCTION_SPECIFIC_TARGET (current_function_decl);
2150 102768 : DECL_FUNCTION_VERSIONED (decl)
2151 51384 : = DECL_FUNCTION_VERSIONED (current_function_decl);
2152 :
2153 51384 : if (omp_maybe_offloaded_ctx (ctx))
2154 : {
2155 31025 : cgraph_node::get_create (decl)->offloadable = 1;
2156 31025 : if (ENABLE_OFFLOADING)
2157 : g->have_offload = true;
2158 : }
2159 :
2160 51384 : if (cgraph_node::get_create (decl)->offloadable)
2161 : {
2162 31025 : const char *target_attr = (is_gimple_omp_offloaded (ctx->stmt)
2163 31025 : ? "omp target entrypoint"
2164 6199 : : "omp declare target");
2165 31025 : if (lookup_attribute ("omp declare target",
2166 31025 : DECL_ATTRIBUTES (current_function_decl)))
2167 : {
2168 3032 : if (is_gimple_omp_offloaded (ctx->stmt))
2169 1620 : DECL_ATTRIBUTES (decl)
2170 1620 : = remove_attribute ("omp declare target",
2171 1620 : copy_list (DECL_ATTRIBUTES (decl)));
2172 : else
2173 : target_attr = NULL;
2174 : }
2175 1620 : if (target_attr
2176 29613 : && is_gimple_omp_offloaded (ctx->stmt)
2177 24826 : && lookup_attribute ("noclone", DECL_ATTRIBUTES (decl)) == NULL_TREE)
2178 19319 : DECL_ATTRIBUTES (decl) = tree_cons (get_identifier ("noclone"),
2179 19319 : NULL_TREE, DECL_ATTRIBUTES (decl));
2180 31025 : if (target_attr)
2181 29613 : DECL_ATTRIBUTES (decl)
2182 59226 : = tree_cons (get_identifier (target_attr),
2183 29613 : NULL_TREE, DECL_ATTRIBUTES (decl));
2184 : }
2185 :
2186 51384 : t = build_decl (DECL_SOURCE_LOCATION (decl),
2187 : RESULT_DECL, NULL_TREE, void_type_node);
2188 51384 : DECL_ARTIFICIAL (t) = 1;
2189 51384 : DECL_IGNORED_P (t) = 1;
2190 51384 : DECL_CONTEXT (t) = decl;
2191 51384 : DECL_RESULT (decl) = t;
2192 :
2193 51384 : tree data_name = get_identifier (".omp_data_i");
2194 51384 : t = build_decl (DECL_SOURCE_LOCATION (decl), PARM_DECL, data_name,
2195 : ptr_type_node);
2196 51384 : DECL_ARTIFICIAL (t) = 1;
2197 51384 : DECL_NAMELESS (t) = 1;
2198 51384 : DECL_ARG_TYPE (t) = ptr_type_node;
2199 51384 : DECL_CONTEXT (t) = current_function_decl;
2200 51384 : TREE_USED (t) = 1;
2201 51384 : TREE_READONLY (t) = 1;
2202 51384 : DECL_ARGUMENTS (decl) = t;
2203 51384 : if (!task_copy)
2204 50840 : ctx->receiver_decl = t;
2205 : else
2206 : {
2207 544 : t = build_decl (DECL_SOURCE_LOCATION (decl),
2208 : PARM_DECL, get_identifier (".omp_data_o"),
2209 : ptr_type_node);
2210 544 : DECL_ARTIFICIAL (t) = 1;
2211 544 : DECL_NAMELESS (t) = 1;
2212 544 : DECL_ARG_TYPE (t) = ptr_type_node;
2213 544 : DECL_CONTEXT (t) = current_function_decl;
2214 544 : TREE_USED (t) = 1;
2215 544 : TREE_ADDRESSABLE (t) = 1;
2216 544 : DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2217 544 : DECL_ARGUMENTS (decl) = t;
2218 : }
2219 :
2220 : /* Allocate memory for the function structure. The call to
2221 : allocate_struct_function clobbers CFUN, so we need to restore
2222 : it afterward. */
2223 51384 : push_struct_function (decl);
2224 51384 : cfun->function_end_locus = gimple_location (ctx->stmt);
2225 51384 : init_tree_ssa (cfun);
2226 51384 : pop_cfun ();
2227 51384 : }
2228 :
2229 : /* Callback for walk_gimple_seq. Check if combined parallel
2230 : contains gimple_omp_for_combined_into_p OMP_FOR. */
2231 :
2232 : tree
2233 35665 : omp_find_combined_for (gimple_stmt_iterator *gsi_p,
2234 : bool *handled_ops_p,
2235 : struct walk_stmt_info *wi)
2236 : {
2237 35665 : gimple *stmt = gsi_stmt (*gsi_p);
2238 :
2239 35665 : *handled_ops_p = true;
2240 35665 : switch (gimple_code (stmt))
2241 : {
2242 19981 : WALK_SUBSTMTS;
2243 :
2244 13511 : case GIMPLE_OMP_FOR:
2245 13511 : if (gimple_omp_for_combined_into_p (stmt)
2246 13511 : && gimple_omp_for_kind (stmt)
2247 7960 : == *(const enum gf_mask *) (wi->info))
2248 : {
2249 7960 : wi->info = stmt;
2250 7960 : return integer_zero_node;
2251 : }
2252 : break;
2253 : default:
2254 : break;
2255 : }
2256 : return NULL;
2257 : }
2258 :
2259 : /* Add _LOOPTEMP_/_REDUCTEMP_ clauses on OpenMP parallel or task. */
2260 :
2261 : static void
2262 14096 : add_taskreg_looptemp_clauses (enum gf_mask msk, gimple *stmt,
2263 : omp_context *outer_ctx)
2264 : {
2265 14096 : struct walk_stmt_info wi;
2266 :
2267 14096 : memset (&wi, 0, sizeof (wi));
2268 14096 : wi.val_only = true;
2269 14096 : wi.info = (void *) &msk;
2270 14096 : walk_gimple_seq (gimple_omp_body (stmt), omp_find_combined_for, NULL, &wi);
2271 14096 : if (wi.info != (void *) &msk)
2272 : {
2273 7960 : gomp_for *for_stmt = as_a <gomp_for *> ((gimple *) wi.info);
2274 7960 : struct omp_for_data fd;
2275 7960 : omp_extract_for_data (for_stmt, &fd, NULL);
2276 : /* We need two temporaries with fd.loop.v type (istart/iend)
2277 : and then (fd.collapse - 1) temporaries with the same
2278 : type for count2 ... countN-1 vars if not constant. */
2279 7960 : size_t count = 2, i;
2280 7960 : tree type = fd.iter_type;
2281 7960 : if (fd.collapse > 1
2282 2648 : && TREE_CODE (fd.loop.n2) != INTEGER_CST)
2283 : {
2284 1466 : count += fd.collapse - 1;
2285 : /* If there are lastprivate clauses on the inner
2286 : GIMPLE_OMP_FOR, add one more temporaries for the total number
2287 : of iterations (product of count1 ... countN-1). */
2288 1466 : if (omp_find_clause (gimple_omp_for_clauses (for_stmt),
2289 : OMP_CLAUSE_LASTPRIVATE)
2290 1466 : || (msk == GF_OMP_FOR_KIND_FOR
2291 1335 : && omp_find_clause (gimple_omp_parallel_clauses (stmt),
2292 : OMP_CLAUSE_LASTPRIVATE)))
2293 : {
2294 726 : tree temp = create_tmp_var (type);
2295 726 : tree c = build_omp_clause (UNKNOWN_LOCATION,
2296 : OMP_CLAUSE__LOOPTEMP_);
2297 726 : insert_decl_map (&outer_ctx->cb, temp, temp);
2298 726 : OMP_CLAUSE_DECL (c) = temp;
2299 726 : OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
2300 726 : gimple_omp_taskreg_set_clauses (stmt, c);
2301 : }
2302 1466 : if (fd.non_rect
2303 24 : && fd.last_nonrect == fd.first_nonrect + 1)
2304 12 : if (tree v = gimple_omp_for_index (for_stmt, fd.last_nonrect))
2305 12 : if (!TYPE_UNSIGNED (TREE_TYPE (v)))
2306 : {
2307 12 : v = gimple_omp_for_index (for_stmt, fd.first_nonrect);
2308 12 : tree type2 = TREE_TYPE (v);
2309 12 : count++;
2310 48 : for (i = 0; i < 3; i++)
2311 : {
2312 36 : tree temp = create_tmp_var (type2);
2313 36 : tree c = build_omp_clause (UNKNOWN_LOCATION,
2314 : OMP_CLAUSE__LOOPTEMP_);
2315 36 : insert_decl_map (&outer_ctx->cb, temp, temp);
2316 36 : OMP_CLAUSE_DECL (c) = temp;
2317 36 : OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
2318 36 : gimple_omp_taskreg_set_clauses (stmt, c);
2319 : }
2320 : }
2321 : }
2322 26725 : for (i = 0; i < count; i++)
2323 : {
2324 18765 : tree temp = create_tmp_var (type);
2325 18765 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
2326 18765 : insert_decl_map (&outer_ctx->cb, temp, temp);
2327 18765 : OMP_CLAUSE_DECL (c) = temp;
2328 18765 : OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
2329 18765 : gimple_omp_taskreg_set_clauses (stmt, c);
2330 : }
2331 : }
2332 14096 : if (msk == GF_OMP_FOR_KIND_TASKLOOP
2333 15663 : && omp_find_clause (gimple_omp_task_clauses (stmt),
2334 : OMP_CLAUSE_REDUCTION))
2335 : {
2336 517 : tree type = build_pointer_type (pointer_sized_int_node);
2337 517 : tree temp = create_tmp_var (type);
2338 517 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
2339 517 : insert_decl_map (&outer_ctx->cb, temp, temp);
2340 517 : OMP_CLAUSE_DECL (c) = temp;
2341 517 : OMP_CLAUSE_CHAIN (c) = gimple_omp_task_clauses (stmt);
2342 517 : gimple_omp_task_set_clauses (stmt, c);
2343 : }
2344 14096 : }
2345 :
2346 : /* Scan an OpenMP parallel directive. */
2347 :
2348 : static void
2349 18173 : scan_omp_parallel (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2350 : {
2351 18173 : omp_context *ctx;
2352 18173 : tree name;
2353 18173 : gomp_parallel *stmt = as_a <gomp_parallel *> (gsi_stmt (*gsi));
2354 :
2355 : /* Ignore parallel directives with empty bodies, unless there
2356 : are copyin clauses. */
2357 18173 : if (optimize > 0
2358 12484 : && empty_body_p (gimple_omp_body (stmt))
2359 18218 : && omp_find_clause (gimple_omp_parallel_clauses (stmt),
2360 : OMP_CLAUSE_COPYIN) == NULL)
2361 : {
2362 39 : gsi_replace (gsi, gimple_build_nop (), false);
2363 39 : return;
2364 : }
2365 :
2366 18134 : if (gimple_omp_parallel_combined_p (stmt))
2367 12529 : add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR, stmt, outer_ctx);
2368 18134 : for (tree c = omp_find_clause (gimple_omp_parallel_clauses (stmt),
2369 : OMP_CLAUSE_REDUCTION);
2370 20964 : c; c = omp_find_clause (OMP_CLAUSE_CHAIN (c), OMP_CLAUSE_REDUCTION))
2371 3909 : if (OMP_CLAUSE_REDUCTION_TASK (c))
2372 : {
2373 67 : tree type = build_pointer_type (pointer_sized_int_node);
2374 67 : tree temp = create_tmp_var (type);
2375 67 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
2376 67 : if (outer_ctx)
2377 18 : insert_decl_map (&outer_ctx->cb, temp, temp);
2378 67 : OMP_CLAUSE_DECL (c) = temp;
2379 67 : OMP_CLAUSE_CHAIN (c) = gimple_omp_parallel_clauses (stmt);
2380 67 : gimple_omp_parallel_set_clauses (stmt, c);
2381 67 : break;
2382 : }
2383 3842 : else if (OMP_CLAUSE_CHAIN (c) == NULL_TREE)
2384 : break;
2385 :
2386 18134 : ctx = new_omp_context (stmt, outer_ctx);
2387 18134 : taskreg_contexts.safe_push (ctx);
2388 18134 : if (taskreg_nesting_level > 1)
2389 6191 : ctx->is_nested = true;
2390 18134 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2391 18134 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2392 18134 : name = create_tmp_var_name (".omp_data_s");
2393 18134 : name = build_decl (gimple_location (stmt),
2394 : TYPE_DECL, name, ctx->record_type);
2395 18134 : DECL_ARTIFICIAL (name) = 1;
2396 18134 : DECL_NAMELESS (name) = 1;
2397 18134 : TYPE_NAME (ctx->record_type) = name;
2398 18134 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
2399 18134 : create_omp_child_function (ctx, false);
2400 18134 : gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn);
2401 :
2402 18134 : scan_sharing_clauses (gimple_omp_parallel_clauses (stmt), ctx);
2403 18134 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
2404 :
2405 18134 : if (TYPE_FIELDS (ctx->record_type) == NULL)
2406 3572 : ctx->record_type = ctx->receiver_decl = NULL;
2407 : }
2408 :
2409 : /* Scan an OpenMP task directive. */
2410 :
2411 : static void
2412 5374 : scan_omp_task (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2413 : {
2414 5374 : omp_context *ctx;
2415 5374 : tree name, t;
2416 5374 : gomp_task *stmt = as_a <gomp_task *> (gsi_stmt (*gsi));
2417 :
2418 : /* Ignore task directives with empty bodies, unless they have depend
2419 : clause. */
2420 5374 : if (optimize > 0
2421 2582 : && gimple_omp_body (stmt)
2422 2539 : && empty_body_p (gimple_omp_body (stmt))
2423 5428 : && !omp_find_clause (gimple_omp_task_clauses (stmt), OMP_CLAUSE_DEPEND))
2424 : {
2425 19 : gsi_replace (gsi, gimple_build_nop (), false);
2426 131 : return;
2427 : }
2428 :
2429 5355 : if (gimple_omp_task_taskloop_p (stmt))
2430 1567 : add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP, stmt, outer_ctx);
2431 :
2432 5355 : ctx = new_omp_context (stmt, outer_ctx);
2433 :
2434 5355 : if (gimple_omp_task_taskwait_p (stmt))
2435 : {
2436 93 : scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
2437 93 : return;
2438 : }
2439 :
2440 5262 : taskreg_contexts.safe_push (ctx);
2441 5262 : if (taskreg_nesting_level > 1)
2442 2326 : ctx->is_nested = true;
2443 5262 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2444 5262 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2445 5262 : name = create_tmp_var_name (".omp_data_s");
2446 5262 : name = build_decl (gimple_location (stmt),
2447 : TYPE_DECL, name, ctx->record_type);
2448 5262 : DECL_ARTIFICIAL (name) = 1;
2449 5262 : DECL_NAMELESS (name) = 1;
2450 5262 : TYPE_NAME (ctx->record_type) = name;
2451 5262 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
2452 5262 : create_omp_child_function (ctx, false);
2453 5262 : gimple_omp_task_set_child_fn (stmt, ctx->cb.dst_fn);
2454 :
2455 5262 : scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
2456 :
2457 5262 : if (ctx->srecord_type)
2458 : {
2459 544 : name = create_tmp_var_name (".omp_data_a");
2460 544 : name = build_decl (gimple_location (stmt),
2461 : TYPE_DECL, name, ctx->srecord_type);
2462 544 : DECL_ARTIFICIAL (name) = 1;
2463 544 : DECL_NAMELESS (name) = 1;
2464 544 : TYPE_NAME (ctx->srecord_type) = name;
2465 544 : TYPE_ARTIFICIAL (ctx->srecord_type) = 1;
2466 544 : create_omp_child_function (ctx, true);
2467 : }
2468 :
2469 5262 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
2470 :
2471 5262 : if (TYPE_FIELDS (ctx->record_type) == NULL)
2472 : {
2473 1838 : ctx->record_type = ctx->receiver_decl = NULL;
2474 1838 : t = build_int_cst (long_integer_type_node, 0);
2475 1838 : gimple_omp_task_set_arg_size (stmt, t);
2476 1838 : t = build_int_cst (long_integer_type_node, 1);
2477 1838 : gimple_omp_task_set_arg_align (stmt, t);
2478 : }
2479 : }
2480 :
2481 : /* Helper function for finish_taskreg_scan, called through walk_tree.
2482 : If maybe_lookup_decl_in_outer_context returns non-NULL for some
2483 : tree, replace it in the expression. */
2484 :
2485 : static tree
2486 276 : finish_taskreg_remap (tree *tp, int *walk_subtrees, void *data)
2487 : {
2488 276 : if (VAR_P (*tp))
2489 : {
2490 96 : omp_context *ctx = (omp_context *) data;
2491 96 : tree t = maybe_lookup_decl_in_outer_ctx (*tp, ctx);
2492 96 : if (t != *tp)
2493 : {
2494 9 : if (DECL_HAS_VALUE_EXPR_P (t))
2495 0 : t = unshare_expr (DECL_VALUE_EXPR (t));
2496 9 : *tp = t;
2497 : }
2498 96 : *walk_subtrees = 0;
2499 : }
2500 180 : else if (IS_TYPE_OR_DECL_P (*tp))
2501 0 : *walk_subtrees = 0;
2502 276 : return NULL_TREE;
2503 : }
2504 :
2505 : /* If any decls have been made addressable during scan_omp,
2506 : adjust their fields if needed, and layout record types
2507 : of parallel/task constructs. */
2508 :
2509 : static void
2510 26014 : finish_taskreg_scan (omp_context *ctx)
2511 : {
2512 26014 : if (ctx->record_type == NULL_TREE)
2513 : return;
2514 :
2515 : /* If any make_addressable_vars were needed, verify all
2516 : OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK,TEAMS}
2517 : statements if use_pointer_for_field hasn't changed
2518 : because of that. If it did, update field types now. */
2519 19158 : if (make_addressable_vars)
2520 : {
2521 2209 : tree c;
2522 :
2523 12338 : for (c = gimple_omp_taskreg_clauses (ctx->stmt);
2524 12338 : c; c = OMP_CLAUSE_CHAIN (c))
2525 10129 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
2526 10129 : && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
2527 : {
2528 2617 : tree decl = OMP_CLAUSE_DECL (c);
2529 :
2530 : /* Global variables don't need to be copied,
2531 : the receiver side will use them directly. */
2532 2617 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
2533 281 : continue;
2534 2336 : if (!bitmap_bit_p (make_addressable_vars, DECL_UID (decl))
2535 2336 : || !use_pointer_for_field (decl, ctx))
2536 1787 : continue;
2537 549 : tree field = lookup_field (decl, ctx);
2538 549 : if (TREE_CODE (TREE_TYPE (field)) == POINTER_TYPE
2539 549 : && TREE_TYPE (TREE_TYPE (field)) == TREE_TYPE (decl))
2540 495 : continue;
2541 54 : TREE_TYPE (field) = build_pointer_type (TREE_TYPE (decl));
2542 54 : TREE_THIS_VOLATILE (field) = 0;
2543 54 : DECL_USER_ALIGN (field) = 0;
2544 54 : SET_DECL_ALIGN (field, TYPE_ALIGN (TREE_TYPE (field)));
2545 54 : if (TYPE_ALIGN (ctx->record_type) < DECL_ALIGN (field))
2546 16 : SET_TYPE_ALIGN (ctx->record_type, DECL_ALIGN (field));
2547 54 : if (ctx->srecord_type)
2548 : {
2549 0 : tree sfield = lookup_sfield (decl, ctx);
2550 0 : TREE_TYPE (sfield) = TREE_TYPE (field);
2551 0 : TREE_THIS_VOLATILE (sfield) = 0;
2552 0 : DECL_USER_ALIGN (sfield) = 0;
2553 0 : SET_DECL_ALIGN (sfield, DECL_ALIGN (field));
2554 0 : if (TYPE_ALIGN (ctx->srecord_type) < DECL_ALIGN (sfield))
2555 0 : SET_TYPE_ALIGN (ctx->srecord_type, DECL_ALIGN (sfield));
2556 : }
2557 : }
2558 : }
2559 :
2560 19158 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL)
2561 : {
2562 14562 : tree clauses = gimple_omp_parallel_clauses (ctx->stmt);
2563 14562 : tree c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
2564 14562 : if (c)
2565 : {
2566 : /* Move the _reductemp_ clause first. GOMP_parallel_reductions
2567 : expects to find it at the start of data. */
2568 67 : tree f = lookup_field (OMP_CLAUSE_DECL (c), ctx);
2569 67 : tree *p = &TYPE_FIELDS (ctx->record_type);
2570 153 : while (*p)
2571 153 : if (*p == f)
2572 : {
2573 67 : *p = DECL_CHAIN (*p);
2574 67 : break;
2575 : }
2576 : else
2577 86 : p = &DECL_CHAIN (*p);
2578 67 : DECL_CHAIN (f) = TYPE_FIELDS (ctx->record_type);
2579 67 : TYPE_FIELDS (ctx->record_type) = f;
2580 : }
2581 14562 : layout_type (ctx->record_type);
2582 14562 : fixup_child_record_type (ctx);
2583 : }
2584 4596 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
2585 : {
2586 1172 : layout_type (ctx->record_type);
2587 1172 : fixup_child_record_type (ctx);
2588 : }
2589 : else
2590 : {
2591 3424 : location_t loc = gimple_location (ctx->stmt);
2592 3424 : tree *p, vla_fields = NULL_TREE, *q = &vla_fields;
2593 3424 : tree detach_clause
2594 3424 : = omp_find_clause (gimple_omp_task_clauses (ctx->stmt),
2595 : OMP_CLAUSE_DETACH);
2596 : /* Move VLA fields to the end. */
2597 3424 : p = &TYPE_FIELDS (ctx->record_type);
2598 15325 : while (*p)
2599 11901 : if (!TYPE_SIZE_UNIT (TREE_TYPE (*p))
2600 11901 : || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p))))
2601 : {
2602 96 : *q = *p;
2603 96 : *p = TREE_CHAIN (*p);
2604 96 : TREE_CHAIN (*q) = NULL_TREE;
2605 96 : q = &TREE_CHAIN (*q);
2606 : }
2607 : else
2608 11805 : p = &DECL_CHAIN (*p);
2609 3424 : *p = vla_fields;
2610 3424 : if (gimple_omp_task_taskloop_p (ctx->stmt))
2611 : {
2612 : /* Move fields corresponding to first and second _looptemp_
2613 : clause first. There are filled by GOMP_taskloop
2614 : and thus need to be in specific positions. */
2615 1567 : tree clauses = gimple_omp_task_clauses (ctx->stmt);
2616 1567 : tree c1 = omp_find_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
2617 1567 : tree c2 = omp_find_clause (OMP_CLAUSE_CHAIN (c1),
2618 : OMP_CLAUSE__LOOPTEMP_);
2619 1567 : tree c3 = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
2620 1567 : tree f1 = lookup_field (OMP_CLAUSE_DECL (c1), ctx);
2621 1567 : tree f2 = lookup_field (OMP_CLAUSE_DECL (c2), ctx);
2622 2084 : tree f3 = c3 ? lookup_field (OMP_CLAUSE_DECL (c3), ctx) : NULL_TREE;
2623 1567 : p = &TYPE_FIELDS (ctx->record_type);
2624 8871 : while (*p)
2625 7304 : if (*p == f1 || *p == f2 || *p == f3)
2626 3651 : *p = DECL_CHAIN (*p);
2627 : else
2628 3653 : p = &DECL_CHAIN (*p);
2629 1567 : DECL_CHAIN (f1) = f2;
2630 1567 : if (c3)
2631 : {
2632 517 : DECL_CHAIN (f2) = f3;
2633 517 : DECL_CHAIN (f3) = TYPE_FIELDS (ctx->record_type);
2634 : }
2635 : else
2636 1050 : DECL_CHAIN (f2) = TYPE_FIELDS (ctx->record_type);
2637 1567 : TYPE_FIELDS (ctx->record_type) = f1;
2638 1567 : if (ctx->srecord_type)
2639 : {
2640 389 : f1 = lookup_sfield (OMP_CLAUSE_DECL (c1), ctx);
2641 389 : f2 = lookup_sfield (OMP_CLAUSE_DECL (c2), ctx);
2642 389 : if (c3)
2643 186 : f3 = lookup_sfield (OMP_CLAUSE_DECL (c3), ctx);
2644 389 : p = &TYPE_FIELDS (ctx->srecord_type);
2645 2097 : while (*p)
2646 1708 : if (*p == f1 || *p == f2 || *p == f3)
2647 964 : *p = DECL_CHAIN (*p);
2648 : else
2649 744 : p = &DECL_CHAIN (*p);
2650 389 : DECL_CHAIN (f1) = f2;
2651 389 : DECL_CHAIN (f2) = TYPE_FIELDS (ctx->srecord_type);
2652 389 : if (c3)
2653 : {
2654 186 : DECL_CHAIN (f2) = f3;
2655 186 : DECL_CHAIN (f3) = TYPE_FIELDS (ctx->srecord_type);
2656 : }
2657 : else
2658 203 : DECL_CHAIN (f2) = TYPE_FIELDS (ctx->srecord_type);
2659 389 : TYPE_FIELDS (ctx->srecord_type) = f1;
2660 : }
2661 : }
2662 3424 : if (detach_clause)
2663 : {
2664 189 : tree c, field;
2665 :
2666 : /* Look for a firstprivate clause with the detach event handle. */
2667 540 : for (c = gimple_omp_taskreg_clauses (ctx->stmt);
2668 540 : c; c = OMP_CLAUSE_CHAIN (c))
2669 : {
2670 540 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FIRSTPRIVATE)
2671 347 : continue;
2672 193 : if (maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_DECL (c), ctx)
2673 193 : == OMP_CLAUSE_DECL (detach_clause))
2674 : break;
2675 : }
2676 :
2677 189 : gcc_assert (c);
2678 189 : field = lookup_field (OMP_CLAUSE_DECL (c), ctx);
2679 :
2680 : /* Move field corresponding to the detach clause first.
2681 : This is filled by GOMP_task and needs to be in a
2682 : specific position. */
2683 189 : p = &TYPE_FIELDS (ctx->record_type);
2684 518 : while (*p)
2685 329 : if (*p == field)
2686 189 : *p = DECL_CHAIN (*p);
2687 : else
2688 140 : p = &DECL_CHAIN (*p);
2689 189 : DECL_CHAIN (field) = TYPE_FIELDS (ctx->record_type);
2690 189 : TYPE_FIELDS (ctx->record_type) = field;
2691 189 : if (ctx->srecord_type)
2692 : {
2693 3 : field = lookup_sfield (OMP_CLAUSE_DECL (c), ctx);
2694 3 : p = &TYPE_FIELDS (ctx->srecord_type);
2695 9 : while (*p)
2696 6 : if (*p == field)
2697 3 : *p = DECL_CHAIN (*p);
2698 : else
2699 3 : p = &DECL_CHAIN (*p);
2700 3 : DECL_CHAIN (field) = TYPE_FIELDS (ctx->srecord_type);
2701 3 : TYPE_FIELDS (ctx->srecord_type) = field;
2702 : }
2703 : }
2704 3424 : layout_type (ctx->record_type);
2705 3424 : fixup_child_record_type (ctx);
2706 3424 : if (ctx->srecord_type)
2707 544 : layout_type (ctx->srecord_type);
2708 3424 : tree t = fold_convert_loc (loc, long_integer_type_node,
2709 3424 : TYPE_SIZE_UNIT (ctx->record_type));
2710 3424 : if (TREE_CODE (t) != INTEGER_CST)
2711 : {
2712 21 : t = unshare_expr (t);
2713 21 : walk_tree (&t, finish_taskreg_remap, ctx, NULL);
2714 : }
2715 3424 : gimple_omp_task_set_arg_size (ctx->stmt, t);
2716 3424 : t = build_int_cst (long_integer_type_node,
2717 3424 : TYPE_ALIGN_UNIT (ctx->record_type));
2718 3424 : gimple_omp_task_set_arg_align (ctx->stmt, t);
2719 : }
2720 : }
2721 :
2722 : /* Find the enclosing offload context. */
2723 :
2724 : static omp_context *
2725 9665 : enclosing_target_ctx (omp_context *ctx)
2726 : {
2727 43359 : for (; ctx; ctx = ctx->outer)
2728 41745 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET)
2729 : break;
2730 :
2731 22111 : return ctx;
2732 : }
2733 :
2734 : /* Return whether CTX's parent compute construct is an OpenACC 'kernels'
2735 : construct.
2736 : (This doesn't include OpenACC 'kernels' decomposed parts.) */
2737 :
2738 : static bool
2739 11325 : ctx_in_oacc_kernels_region (omp_context *ctx)
2740 : {
2741 39137 : for (;ctx != NULL; ctx = ctx->outer)
2742 : {
2743 29472 : gimple *stmt = ctx->stmt;
2744 29472 : if (gimple_code (stmt) == GIMPLE_OMP_TARGET
2745 29472 : && gimple_omp_target_kind (stmt) == GF_OMP_TARGET_KIND_OACC_KERNELS)
2746 : return true;
2747 : }
2748 :
2749 : return false;
2750 : }
2751 :
2752 : /* Check the parallelism clauses inside a OpenACC 'kernels' region.
2753 : (This doesn't include OpenACC 'kernels' decomposed parts.)
2754 : Until kernels handling moves to use the same loop indirection
2755 : scheme as parallel, we need to do this checking early. */
2756 :
2757 : static unsigned
2758 7151 : check_oacc_kernel_gwv (gomp_for *stmt, omp_context *ctx)
2759 : {
2760 7151 : bool checking = true;
2761 7151 : unsigned outer_mask = 0;
2762 7151 : unsigned this_mask = 0;
2763 7151 : bool has_seq = false, has_auto = false;
2764 :
2765 7151 : if (ctx->outer)
2766 4819 : outer_mask = check_oacc_kernel_gwv (NULL, ctx->outer);
2767 7151 : if (!stmt)
2768 : {
2769 4819 : checking = false;
2770 4819 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR)
2771 : return outer_mask;
2772 1730 : stmt = as_a <gomp_for *> (ctx->stmt);
2773 : }
2774 :
2775 11330 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
2776 : {
2777 7268 : switch (OMP_CLAUSE_CODE (c))
2778 : {
2779 645 : case OMP_CLAUSE_GANG:
2780 645 : this_mask |= GOMP_DIM_MASK (GOMP_DIM_GANG);
2781 645 : break;
2782 439 : case OMP_CLAUSE_WORKER:
2783 439 : this_mask |= GOMP_DIM_MASK (GOMP_DIM_WORKER);
2784 439 : break;
2785 342 : case OMP_CLAUSE_VECTOR:
2786 342 : this_mask |= GOMP_DIM_MASK (GOMP_DIM_VECTOR);
2787 342 : break;
2788 63 : case OMP_CLAUSE_SEQ:
2789 63 : has_seq = true;
2790 63 : break;
2791 94 : case OMP_CLAUSE_AUTO:
2792 94 : has_auto = true;
2793 94 : break;
2794 : default:
2795 : break;
2796 : }
2797 : }
2798 :
2799 4062 : if (checking)
2800 : {
2801 2332 : if (has_seq && (this_mask || has_auto))
2802 36 : error_at (gimple_location (stmt), "%<seq%> overrides other"
2803 : " OpenACC loop specifiers");
2804 2296 : else if (has_auto && this_mask)
2805 30 : error_at (gimple_location (stmt), "%<auto%> conflicts with other"
2806 : " OpenACC loop specifiers");
2807 :
2808 2332 : if (this_mask & outer_mask)
2809 15 : error_at (gimple_location (stmt), "inner loop uses same"
2810 : " OpenACC parallelism as containing loop");
2811 : }
2812 :
2813 4062 : return outer_mask | this_mask;
2814 : }
2815 :
2816 : /* Scan a GIMPLE_OMP_FOR. */
2817 :
2818 : static omp_context *
2819 52643 : scan_omp_for (gomp_for *stmt, omp_context *outer_ctx)
2820 : {
2821 52643 : omp_context *ctx;
2822 52643 : size_t i;
2823 52643 : tree clauses = gimple_omp_for_clauses (stmt);
2824 :
2825 52643 : ctx = new_omp_context (stmt, outer_ctx);
2826 :
2827 52643 : if (is_gimple_omp_oacc (stmt))
2828 : {
2829 12446 : omp_context *tgt = enclosing_target_ctx (outer_ctx);
2830 :
2831 12446 : if (!(tgt && is_oacc_kernels (tgt)))
2832 32251 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2833 : {
2834 22137 : tree c_op0;
2835 22137 : switch (OMP_CLAUSE_CODE (c))
2836 : {
2837 1952 : case OMP_CLAUSE_GANG:
2838 1952 : c_op0 = OMP_CLAUSE_GANG_EXPR (c);
2839 1952 : break;
2840 :
2841 1426 : case OMP_CLAUSE_WORKER:
2842 1426 : c_op0 = OMP_CLAUSE_WORKER_EXPR (c);
2843 1426 : break;
2844 :
2845 1627 : case OMP_CLAUSE_VECTOR:
2846 1627 : c_op0 = OMP_CLAUSE_VECTOR_EXPR (c);
2847 1627 : break;
2848 :
2849 17132 : default:
2850 17132 : continue;
2851 : }
2852 :
2853 5005 : if (c_op0)
2854 : {
2855 : /* By construction, this is impossible for OpenACC 'kernels'
2856 : decomposed parts. */
2857 210 : gcc_assert (!(tgt && is_oacc_kernels_decomposed_part (tgt)));
2858 :
2859 210 : error_at (OMP_CLAUSE_LOCATION (c),
2860 : "argument not permitted on %qs clause",
2861 210 : omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
2862 210 : if (tgt)
2863 180 : inform (gimple_location (tgt->stmt),
2864 : "enclosing parent compute construct");
2865 30 : else if (oacc_get_fn_attrib (current_function_decl))
2866 30 : inform (DECL_SOURCE_LOCATION (current_function_decl),
2867 : "enclosing routine");
2868 : else
2869 0 : gcc_unreachable ();
2870 : }
2871 : }
2872 :
2873 12446 : if (tgt && is_oacc_kernels (tgt))
2874 2332 : check_oacc_kernel_gwv (stmt, ctx);
2875 :
2876 : /* Collect all variables named in reductions on this loop. Ensure
2877 : that, if this loop has a reduction on some variable v, and there is
2878 : a reduction on v somewhere in an outer context, then there is a
2879 : reduction on v on all intervening loops as well. */
2880 12446 : tree local_reduction_clauses = NULL;
2881 39374 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
2882 : {
2883 26928 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
2884 4712 : local_reduction_clauses
2885 4712 : = tree_cons (NULL, c, local_reduction_clauses);
2886 : }
2887 12446 : if (ctx->outer_reduction_clauses == NULL && ctx->outer != NULL)
2888 11920 : ctx->outer_reduction_clauses
2889 11920 : = chainon (unshare_expr (ctx->outer->local_reduction_clauses),
2890 : ctx->outer->outer_reduction_clauses);
2891 12446 : tree outer_reduction_clauses = ctx->outer_reduction_clauses;
2892 12446 : tree local_iter = local_reduction_clauses;
2893 17158 : for (; local_iter; local_iter = TREE_CHAIN (local_iter))
2894 : {
2895 4712 : tree local_clause = TREE_VALUE (local_iter);
2896 4712 : tree local_var = OMP_CLAUSE_DECL (local_clause);
2897 4712 : tree_code local_op = OMP_CLAUSE_REDUCTION_CODE (local_clause);
2898 4712 : bool have_outer_reduction = false;
2899 4712 : tree ctx_iter = outer_reduction_clauses;
2900 5826 : for (; ctx_iter; ctx_iter = TREE_CHAIN (ctx_iter))
2901 : {
2902 3325 : tree outer_clause = TREE_VALUE (ctx_iter);
2903 3325 : tree outer_var = OMP_CLAUSE_DECL (outer_clause);
2904 3325 : tree_code outer_op = OMP_CLAUSE_REDUCTION_CODE (outer_clause);
2905 3325 : if (outer_var == local_var && outer_op != local_op)
2906 : {
2907 535 : if (warning_at (OMP_CLAUSE_LOCATION (local_clause),
2908 535 : OPT_Wopenmp, "conflicting reduction "
2909 : "operations for %qE",
2910 : local_var))
2911 535 : inform (OMP_CLAUSE_LOCATION (outer_clause),
2912 : "location of the previous reduction for %qE",
2913 : outer_var);
2914 : }
2915 3325 : if (outer_var == local_var)
2916 : {
2917 : have_outer_reduction = true;
2918 : break;
2919 : }
2920 : }
2921 4712 : if (have_outer_reduction)
2922 : {
2923 : /* There is a reduction on outer_var both on this loop and on
2924 : some enclosing loop. Walk up the context tree until such a
2925 : loop with a reduction on outer_var is found, and complain
2926 : about all intervening loops that do not have such a
2927 : reduction. */
2928 2211 : struct omp_context *curr_loop = ctx->outer;
2929 2211 : bool found = false;
2930 2826 : while (curr_loop != NULL)
2931 : {
2932 2826 : tree curr_iter = curr_loop->local_reduction_clauses;
2933 3460 : for (; curr_iter; curr_iter = TREE_CHAIN (curr_iter))
2934 : {
2935 2845 : tree curr_clause = TREE_VALUE (curr_iter);
2936 2845 : tree curr_var = OMP_CLAUSE_DECL (curr_clause);
2937 2845 : if (curr_var == local_var)
2938 : {
2939 : found = true;
2940 : break;
2941 : }
2942 : }
2943 2826 : if (!found)
2944 615 : warning_at (gimple_location (curr_loop->stmt), OPT_Wopenmp,
2945 : "nested loop in reduction needs "
2946 : "reduction clause for %qE",
2947 : local_var);
2948 : else
2949 : break;
2950 615 : curr_loop = curr_loop->outer;
2951 : }
2952 : }
2953 : }
2954 12446 : ctx->local_reduction_clauses = local_reduction_clauses;
2955 12446 : ctx->outer_reduction_clauses
2956 12446 : = chainon (unshare_expr (ctx->local_reduction_clauses),
2957 : ctx->outer_reduction_clauses);
2958 :
2959 12446 : if (tgt && is_oacc_kernels (tgt))
2960 : {
2961 : /* Strip out reductions, as they are not handled yet. */
2962 : tree *prev_ptr = &clauses;
2963 :
2964 7123 : while (tree probe = *prev_ptr)
2965 : {
2966 4791 : tree *next_ptr = &OMP_CLAUSE_CHAIN (probe);
2967 :
2968 4791 : if (OMP_CLAUSE_CODE (probe) == OMP_CLAUSE_REDUCTION)
2969 718 : *prev_ptr = *next_ptr;
2970 : else
2971 : prev_ptr = next_ptr;
2972 : }
2973 :
2974 2332 : gimple_omp_for_set_clauses (stmt, clauses);
2975 : }
2976 : }
2977 :
2978 52643 : scan_sharing_clauses (clauses, ctx);
2979 :
2980 52643 : scan_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
2981 179967 : for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
2982 : {
2983 74681 : scan_omp_op (gimple_omp_for_index_ptr (stmt, i), ctx);
2984 74681 : scan_omp_op (gimple_omp_for_initial_ptr (stmt, i), ctx);
2985 74681 : scan_omp_op (gimple_omp_for_final_ptr (stmt, i), ctx);
2986 74681 : scan_omp_op (gimple_omp_for_incr_ptr (stmt, i), ctx);
2987 : }
2988 52643 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
2989 52643 : return ctx;
2990 : }
2991 :
2992 : /* Duplicate #pragma omp simd, one for SIMT, another one for SIMD. */
2993 :
2994 : static void
2995 0 : scan_omp_simd (gimple_stmt_iterator *gsi, gomp_for *stmt,
2996 : omp_context *outer_ctx)
2997 : {
2998 0 : gbind *bind = gimple_build_bind (NULL, NULL, NULL);
2999 0 : gsi_replace (gsi, bind, false);
3000 0 : gimple_seq seq = NULL;
3001 0 : gimple *g = gimple_build_call_internal (IFN_GOMP_USE_SIMT, 0);
3002 0 : tree cond = create_tmp_var_raw (integer_type_node);
3003 0 : DECL_CONTEXT (cond) = current_function_decl;
3004 0 : DECL_SEEN_IN_BIND_EXPR_P (cond) = 1;
3005 0 : gimple_bind_set_vars (bind, cond);
3006 0 : gimple_call_set_lhs (g, cond);
3007 0 : gimple_seq_add_stmt (&seq, g);
3008 0 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
3009 0 : tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
3010 0 : tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
3011 0 : g = gimple_build_cond (NE_EXPR, cond, integer_zero_node, lab1, lab2);
3012 0 : gimple_seq_add_stmt (&seq, g);
3013 0 : g = gimple_build_label (lab1);
3014 0 : gimple_seq_add_stmt (&seq, g);
3015 0 : gimple_seq new_seq = copy_gimple_seq_and_replace_locals (stmt);
3016 0 : gomp_for *new_stmt = as_a <gomp_for *> (new_seq);
3017 0 : tree clause = build_omp_clause (gimple_location (stmt), OMP_CLAUSE__SIMT_);
3018 0 : OMP_CLAUSE_CHAIN (clause) = gimple_omp_for_clauses (new_stmt);
3019 0 : gimple_omp_for_set_clauses (new_stmt, clause);
3020 0 : gimple_seq_add_stmt (&seq, new_stmt);
3021 0 : g = gimple_build_goto (lab3);
3022 0 : gimple_seq_add_stmt (&seq, g);
3023 0 : g = gimple_build_label (lab2);
3024 0 : gimple_seq_add_stmt (&seq, g);
3025 0 : gimple_seq_add_stmt (&seq, stmt);
3026 0 : g = gimple_build_label (lab3);
3027 0 : gimple_seq_add_stmt (&seq, g);
3028 0 : gimple_bind_set_body (bind, seq);
3029 0 : update_stmt (bind);
3030 0 : scan_omp_for (new_stmt, outer_ctx);
3031 0 : scan_omp_for (stmt, outer_ctx)->simt_stmt = new_stmt;
3032 0 : }
3033 :
3034 : static tree omp_find_scan (gimple_stmt_iterator *, bool *,
3035 : struct walk_stmt_info *);
3036 : static omp_context *maybe_lookup_ctx (gimple *);
3037 :
3038 : /* Duplicate #pragma omp simd, one for the scan input phase loop and one
3039 : for scan phase loop. */
3040 :
3041 : static void
3042 83 : scan_omp_simd_scan (gimple_stmt_iterator *gsi, gomp_for *stmt,
3043 : omp_context *outer_ctx)
3044 : {
3045 : /* The only change between inclusive and exclusive scan will be
3046 : within the first simd loop, so just use inclusive in the
3047 : worksharing loop. */
3048 83 : outer_ctx->scan_inclusive = true;
3049 83 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_INCLUSIVE);
3050 83 : OMP_CLAUSE_DECL (c) = integer_zero_node;
3051 :
3052 83 : gomp_scan *input_stmt = gimple_build_omp_scan (NULL, NULL_TREE);
3053 83 : gomp_scan *scan_stmt = gimple_build_omp_scan (NULL, c);
3054 83 : gsi_replace (gsi, input_stmt, false);
3055 83 : gimple_seq input_body = NULL;
3056 83 : gimple_seq_add_stmt (&input_body, stmt);
3057 83 : gsi_insert_after (gsi, scan_stmt, GSI_NEW_STMT);
3058 :
3059 83 : gimple_stmt_iterator input1_gsi = gsi_none ();
3060 83 : struct walk_stmt_info wi;
3061 83 : memset (&wi, 0, sizeof (wi));
3062 83 : wi.val_only = true;
3063 83 : wi.info = (void *) &input1_gsi;
3064 83 : walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), omp_find_scan, NULL, &wi);
3065 83 : gcc_assert (!gsi_end_p (input1_gsi));
3066 :
3067 83 : gimple *input_stmt1 = gsi_stmt (input1_gsi);
3068 83 : gsi_next (&input1_gsi);
3069 83 : gimple *scan_stmt1 = gsi_stmt (input1_gsi);
3070 83 : gcc_assert (scan_stmt1 && gimple_code (scan_stmt1) == GIMPLE_OMP_SCAN);
3071 83 : c = gimple_omp_scan_clauses (as_a <gomp_scan *> (scan_stmt1));
3072 166 : if (c && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_EXCLUSIVE)
3073 : std::swap (input_stmt1, scan_stmt1);
3074 :
3075 83 : gimple_seq input_body1 = gimple_omp_body (input_stmt1);
3076 83 : gimple_omp_set_body (input_stmt1, NULL);
3077 :
3078 83 : gimple_seq scan_body = copy_gimple_seq_and_replace_locals (stmt);
3079 83 : gomp_for *new_stmt = as_a <gomp_for *> (scan_body);
3080 :
3081 83 : gimple_omp_set_body (input_stmt1, input_body1);
3082 83 : gimple_omp_set_body (scan_stmt1, NULL);
3083 :
3084 83 : gimple_stmt_iterator input2_gsi = gsi_none ();
3085 83 : memset (&wi, 0, sizeof (wi));
3086 83 : wi.val_only = true;
3087 83 : wi.info = (void *) &input2_gsi;
3088 83 : walk_gimple_seq_mod (gimple_omp_body_ptr (new_stmt), omp_find_scan,
3089 : NULL, &wi);
3090 83 : gcc_assert (!gsi_end_p (input2_gsi));
3091 :
3092 83 : gimple *input_stmt2 = gsi_stmt (input2_gsi);
3093 83 : gsi_next (&input2_gsi);
3094 83 : gimple *scan_stmt2 = gsi_stmt (input2_gsi);
3095 83 : gcc_assert (scan_stmt2 && gimple_code (scan_stmt2) == GIMPLE_OMP_SCAN);
3096 166 : if (c && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_EXCLUSIVE)
3097 : std::swap (input_stmt2, scan_stmt2);
3098 :
3099 83 : gimple_omp_set_body (input_stmt2, NULL);
3100 :
3101 83 : gimple_omp_set_body (input_stmt, input_body);
3102 83 : gimple_omp_set_body (scan_stmt, scan_body);
3103 :
3104 83 : omp_context *ctx = new_omp_context (input_stmt, outer_ctx);
3105 83 : scan_omp (gimple_omp_body_ptr (input_stmt), ctx);
3106 :
3107 83 : ctx = new_omp_context (scan_stmt, outer_ctx);
3108 83 : scan_omp (gimple_omp_body_ptr (scan_stmt), ctx);
3109 :
3110 83 : maybe_lookup_ctx (new_stmt)->for_simd_scan_phase = true;
3111 83 : }
3112 :
3113 : /* Scan an OpenMP sections directive. */
3114 :
3115 : static void
3116 581 : scan_omp_sections (gomp_sections *stmt, omp_context *outer_ctx)
3117 : {
3118 581 : omp_context *ctx;
3119 :
3120 581 : ctx = new_omp_context (stmt, outer_ctx);
3121 581 : scan_sharing_clauses (gimple_omp_sections_clauses (stmt), ctx);
3122 581 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3123 581 : }
3124 :
3125 : /* Scan an OpenMP single directive. */
3126 :
3127 : static void
3128 1228 : scan_omp_single (gomp_single *stmt, omp_context *outer_ctx)
3129 : {
3130 1228 : omp_context *ctx;
3131 1228 : tree name;
3132 :
3133 1228 : ctx = new_omp_context (stmt, outer_ctx);
3134 1228 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
3135 1228 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
3136 1228 : name = create_tmp_var_name (".omp_copy_s");
3137 1228 : name = build_decl (gimple_location (stmt),
3138 : TYPE_DECL, name, ctx->record_type);
3139 1228 : TYPE_NAME (ctx->record_type) = name;
3140 :
3141 1228 : scan_sharing_clauses (gimple_omp_single_clauses (stmt), ctx);
3142 1228 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3143 :
3144 1228 : if (TYPE_FIELDS (ctx->record_type) == NULL)
3145 1081 : ctx->record_type = NULL;
3146 : else
3147 147 : layout_type (ctx->record_type);
3148 1228 : }
3149 :
3150 : /* Scan a GIMPLE_OMP_TARGET. */
3151 :
3152 : static void
3153 41853 : scan_omp_target (gomp_target *stmt, omp_context *outer_ctx)
3154 : {
3155 41853 : omp_context *ctx;
3156 41853 : tree name;
3157 41853 : bool offloaded = is_gimple_omp_offloaded (stmt);
3158 41853 : tree clauses = gimple_omp_target_clauses (stmt);
3159 :
3160 41853 : ctx = new_omp_context (stmt, outer_ctx);
3161 41853 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
3162 41853 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
3163 41853 : name = create_tmp_var_name (".omp_data_t");
3164 41853 : name = build_decl (gimple_location (stmt),
3165 : TYPE_DECL, name, ctx->record_type);
3166 41853 : DECL_ARTIFICIAL (name) = 1;
3167 41853 : DECL_NAMELESS (name) = 1;
3168 41853 : TYPE_NAME (ctx->record_type) = name;
3169 41853 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
3170 :
3171 41853 : if (offloaded)
3172 : {
3173 24826 : create_omp_child_function (ctx, false);
3174 24826 : gimple_omp_target_set_child_fn (stmt, ctx->cb.dst_fn);
3175 : }
3176 :
3177 41853 : scan_sharing_clauses (clauses, ctx);
3178 41853 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3179 :
3180 41853 : if (TYPE_FIELDS (ctx->record_type) == NULL)
3181 7855 : ctx->record_type = ctx->receiver_decl = NULL;
3182 : else
3183 : {
3184 33998 : TYPE_FIELDS (ctx->record_type)
3185 33998 : = nreverse (TYPE_FIELDS (ctx->record_type));
3186 33998 : if (flag_checking)
3187 : {
3188 33998 : unsigned int align = DECL_ALIGN (TYPE_FIELDS (ctx->record_type));
3189 33998 : for (tree field = TYPE_FIELDS (ctx->record_type);
3190 131130 : field;
3191 97132 : field = DECL_CHAIN (field))
3192 97132 : gcc_assert (DECL_ALIGN (field) == align);
3193 : }
3194 33998 : layout_type (ctx->record_type);
3195 33998 : if (offloaded)
3196 17530 : fixup_child_record_type (ctx);
3197 : }
3198 :
3199 41853 : if (ctx->teams_nested_p && ctx->nonteams_nested_p)
3200 : {
3201 4 : error_at (gimple_location (stmt),
3202 : "%<target%> construct with nested %<teams%> construct "
3203 : "contains directives outside of the %<teams%> construct");
3204 4 : gimple_omp_set_body (stmt, gimple_build_bind (NULL, NULL, NULL));
3205 : }
3206 41853 : }
3207 :
3208 : /* Scan an OpenMP teams directive. */
3209 :
3210 : static void
3211 8736 : scan_omp_teams (gomp_teams *stmt, omp_context *outer_ctx)
3212 : {
3213 8736 : omp_context *ctx = new_omp_context (stmt, outer_ctx);
3214 :
3215 8736 : if (!gimple_omp_teams_host (stmt))
3216 : {
3217 6118 : scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
3218 6118 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3219 6118 : return;
3220 : }
3221 2618 : taskreg_contexts.safe_push (ctx);
3222 2618 : gcc_assert (taskreg_nesting_level == 1);
3223 2618 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
3224 2618 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
3225 2618 : tree name = create_tmp_var_name (".omp_data_s");
3226 2618 : name = build_decl (gimple_location (stmt),
3227 : TYPE_DECL, name, ctx->record_type);
3228 2618 : DECL_ARTIFICIAL (name) = 1;
3229 2618 : DECL_NAMELESS (name) = 1;
3230 2618 : TYPE_NAME (ctx->record_type) = name;
3231 2618 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
3232 2618 : create_omp_child_function (ctx, false);
3233 2618 : gimple_omp_teams_set_child_fn (stmt, ctx->cb.dst_fn);
3234 :
3235 2618 : scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
3236 2618 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3237 :
3238 2618 : if (TYPE_FIELDS (ctx->record_type) == NULL)
3239 1446 : ctx->record_type = ctx->receiver_decl = NULL;
3240 : }
3241 :
3242 : /* Check nesting restrictions. */
3243 : static bool
3244 156891 : check_omp_nesting_restrictions (gimple *stmt, omp_context *ctx)
3245 : {
3246 156891 : tree c;
3247 :
3248 : /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
3249 : inside an OpenACC CTX. */
3250 156891 : if (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
3251 156891 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE)
3252 : /* ..., except for the atomic codes that OpenACC shares with OpenMP. */
3253 : ;
3254 281921 : else if (!(is_gimple_omp (stmt)
3255 138850 : && is_gimple_omp_oacc (stmt)))
3256 : {
3257 112807 : if (oacc_get_fn_attrib (cfun->decl) != NULL)
3258 : {
3259 48 : error_at (gimple_location (stmt),
3260 : "non-OpenACC construct inside of OpenACC routine");
3261 48 : return false;
3262 : }
3263 : else
3264 236202 : for (omp_context *octx = ctx; octx != NULL; octx = octx->outer)
3265 247062 : if (is_gimple_omp (octx->stmt)
3266 123619 : && is_gimple_omp_oacc (octx->stmt))
3267 : {
3268 176 : error_at (gimple_location (stmt),
3269 : "non-OpenACC construct inside of OpenACC region");
3270 176 : return false;
3271 : }
3272 : }
3273 :
3274 156667 : if (ctx != NULL)
3275 : {
3276 84141 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
3277 84141 : && gimple_omp_target_kind (ctx->stmt) == GF_OMP_TARGET_KIND_REGION)
3278 : {
3279 8314 : c = omp_find_clause (gimple_omp_target_clauses (ctx->stmt),
3280 : OMP_CLAUSE_DEVICE);
3281 8314 : if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
3282 : {
3283 9 : error_at (gimple_location (stmt),
3284 : "OpenMP constructs are not allowed in target region "
3285 : "with %<ancestor%>");
3286 9 : return false;
3287 : }
3288 :
3289 8305 : if (gimple_code (stmt) == GIMPLE_OMP_TEAMS && !ctx->teams_nested_p)
3290 6114 : ctx->teams_nested_p = true;
3291 : else
3292 2191 : ctx->nonteams_nested_p = true;
3293 : }
3294 84132 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_SCAN
3295 172 : && ctx->outer
3296 84304 : && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR)
3297 : ctx = ctx->outer;
3298 84132 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3299 28308 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
3300 85941 : && !ctx->loop_p)
3301 : {
3302 1589 : c = NULL_TREE;
3303 1589 : if (ctx->order_concurrent
3304 1589 : && (gimple_code (stmt) == GIMPLE_OMP_ORDERED
3305 280 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
3306 190 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE))
3307 : {
3308 210 : error_at (gimple_location (stmt),
3309 : "OpenMP constructs other than %<parallel%>, %<loop%>"
3310 : " or %<simd%> may not be nested inside a region with"
3311 : " the %<order(concurrent)%> clause");
3312 210 : return false;
3313 : }
3314 1379 : if (gimple_code (stmt) == GIMPLE_OMP_ORDERED)
3315 : {
3316 133 : c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
3317 133 : if (omp_find_clause (c, OMP_CLAUSE_SIMD))
3318 : {
3319 133 : if (omp_find_clause (c, OMP_CLAUSE_THREADS)
3320 133 : && (ctx->outer == NULL
3321 29 : || !gimple_omp_for_combined_into_p (ctx->stmt)
3322 25 : || gimple_code (ctx->outer->stmt) != GIMPLE_OMP_FOR
3323 25 : || (gimple_omp_for_kind (ctx->outer->stmt)
3324 : != GF_OMP_FOR_KIND_FOR)
3325 25 : || !gimple_omp_for_combined_p (ctx->outer->stmt)))
3326 : {
3327 8 : error_at (gimple_location (stmt),
3328 : "%<ordered simd threads%> must be closely "
3329 : "nested inside of %<%s simd%> region",
3330 8 : lang_GNU_Fortran () ? "do" : "for");
3331 8 : return false;
3332 : }
3333 : return true;
3334 : }
3335 : }
3336 : else if (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
3337 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE
3338 : || gimple_code (stmt) == GIMPLE_OMP_SCAN
3339 : || gimple_code (stmt) == GIMPLE_OMP_STRUCTURED_BLOCK)
3340 : return true;
3341 : else if (gimple_code (stmt) == GIMPLE_OMP_FOR
3342 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
3343 : return true;
3344 60 : error_at (gimple_location (stmt),
3345 : "OpenMP constructs other than "
3346 : "%<ordered simd%>, %<simd%>, %<loop%> or %<atomic%> may "
3347 : "not be nested inside %<simd%> region");
3348 60 : return false;
3349 : }
3350 82543 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
3351 : {
3352 5829 : if ((gimple_code (stmt) != GIMPLE_OMP_FOR
3353 5525 : || (gimple_omp_for_kind (stmt) != GF_OMP_FOR_KIND_DISTRIBUTE
3354 25 : && omp_find_clause (gimple_omp_for_clauses (stmt),
3355 : OMP_CLAUSE_BIND) == NULL_TREE))
3356 5845 : && gimple_code (stmt) != GIMPLE_OMP_PARALLEL)
3357 : {
3358 132 : error_at (gimple_location (stmt),
3359 : "only %<distribute%>, %<parallel%> or %<loop%> "
3360 : "regions are allowed to be strictly nested inside "
3361 : "%<teams%> region");
3362 132 : return false;
3363 : }
3364 : }
3365 76714 : else if (ctx->order_concurrent
3366 1944 : && gimple_code (stmt) != GIMPLE_OMP_PARALLEL
3367 1465 : && (gimple_code (stmt) != GIMPLE_OMP_FOR
3368 1191 : || gimple_omp_for_kind (stmt) != GF_OMP_FOR_KIND_SIMD)
3369 289 : && gimple_code (stmt) != GIMPLE_OMP_SCAN
3370 77003 : && gimple_code (stmt) != GIMPLE_OMP_STRUCTURED_BLOCK)
3371 : {
3372 285 : if (ctx->loop_p)
3373 135 : error_at (gimple_location (stmt),
3374 : "OpenMP constructs other than %<parallel%>, %<loop%> or "
3375 : "%<simd%> may not be nested inside a %<loop%> region");
3376 : else
3377 150 : error_at (gimple_location (stmt),
3378 : "OpenMP constructs other than %<parallel%>, %<loop%> or "
3379 : "%<simd%> may not be nested inside a region with "
3380 : "the %<order(concurrent)%> clause");
3381 285 : return false;
3382 : }
3383 : }
3384 154652 : switch (gimple_code (stmt))
3385 : {
3386 52762 : case GIMPLE_OMP_FOR:
3387 52762 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_SIMD)
3388 : return true;
3389 42004 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
3390 : {
3391 8217 : if (ctx != NULL && gimple_code (ctx->stmt) != GIMPLE_OMP_TEAMS)
3392 : {
3393 0 : error_at (gimple_location (stmt),
3394 : "%<distribute%> region must be strictly nested "
3395 : "inside %<teams%> construct");
3396 0 : return false;
3397 : }
3398 : return true;
3399 : }
3400 : /* We split taskloop into task and nested taskloop in it. */
3401 33787 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_TASKLOOP)
3402 : return true;
3403 : /* For now, hope this will change and loop bind(parallel) will not
3404 : be allowed in lots of contexts. */
3405 30653 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR
3406 48795 : && omp_find_clause (gimple_omp_for_clauses (stmt), OMP_CLAUSE_BIND))
3407 : return true;
3408 30024 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
3409 : {
3410 12511 : bool ok = false;
3411 :
3412 12511 : if (ctx)
3413 11964 : switch (gimple_code (ctx->stmt))
3414 : {
3415 4049 : case GIMPLE_OMP_FOR:
3416 4049 : ok = (gimple_omp_for_kind (ctx->stmt)
3417 : == GF_OMP_FOR_KIND_OACC_LOOP);
3418 4049 : break;
3419 :
3420 7887 : case GIMPLE_OMP_TARGET:
3421 7887 : switch (gimple_omp_target_kind (ctx->stmt))
3422 : {
3423 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
3424 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
3425 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
3426 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
3427 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
3428 : ok = true;
3429 : break;
3430 :
3431 : default:
3432 : break;
3433 : }
3434 :
3435 : default:
3436 : break;
3437 : }
3438 547 : else if (oacc_get_fn_attrib (current_function_decl))
3439 : ok = true;
3440 4049 : if (!ok)
3441 : {
3442 65 : error_at (gimple_location (stmt),
3443 : "OpenACC loop directive must be associated with"
3444 : " an OpenACC compute region");
3445 65 : return false;
3446 : }
3447 : }
3448 : /* FALLTHRU */
3449 34120 : case GIMPLE_CALL:
3450 34120 : if (is_gimple_call (stmt)
3451 34120 : && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3452 : == BUILT_IN_GOMP_CANCEL
3453 3057 : || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3454 : == BUILT_IN_GOMP_CANCELLATION_POINT))
3455 : {
3456 1907 : const char *bad = NULL;
3457 1907 : const char *kind = NULL;
3458 1907 : const char *construct
3459 1907 : = (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3460 : == BUILT_IN_GOMP_CANCEL)
3461 1907 : ? "cancel"
3462 803 : : "cancellation point";
3463 1907 : if (ctx == NULL)
3464 : {
3465 41 : error_at (gimple_location (stmt), "orphaned %qs construct",
3466 : construct);
3467 41 : return false;
3468 : }
3469 1866 : switch (tree_fits_shwi_p (gimple_call_arg (stmt, 0))
3470 1866 : ? tree_to_shwi (gimple_call_arg (stmt, 0))
3471 : : 0)
3472 : {
3473 528 : case 1:
3474 528 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_PARALLEL)
3475 : bad = "parallel";
3476 208 : else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3477 : == BUILT_IN_GOMP_CANCEL
3478 208 : && !integer_zerop (gimple_call_arg (stmt, 1)))
3479 164 : ctx->cancellable = true;
3480 164 : kind = "parallel";
3481 164 : break;
3482 427 : case 2:
3483 427 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
3484 427 : || gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR)
3485 : bad = "for";
3486 117 : else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3487 : == BUILT_IN_GOMP_CANCEL
3488 117 : && !integer_zerop (gimple_call_arg (stmt, 1)))
3489 : {
3490 106 : ctx->cancellable = true;
3491 106 : if (omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3492 : OMP_CLAUSE_NOWAIT))
3493 5 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3494 : "%<cancel for%> inside "
3495 : "%<nowait%> for construct");
3496 106 : if (omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3497 : OMP_CLAUSE_ORDERED))
3498 5 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3499 : "%<cancel for%> inside "
3500 : "%<ordered%> for construct");
3501 : }
3502 5 : kind = "for";
3503 5 : break;
3504 383 : case 4:
3505 383 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_SECTIONS
3506 383 : && gimple_code (ctx->stmt) != GIMPLE_OMP_SECTION)
3507 : bad = "sections";
3508 93 : else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3509 : == BUILT_IN_GOMP_CANCEL
3510 93 : && !integer_zerop (gimple_call_arg (stmt, 1)))
3511 : {
3512 73 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
3513 : {
3514 0 : ctx->cancellable = true;
3515 0 : if (omp_find_clause (gimple_omp_sections_clauses
3516 0 : (ctx->stmt),
3517 : OMP_CLAUSE_NOWAIT))
3518 0 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3519 : "%<cancel sections%> inside "
3520 : "%<nowait%> sections construct");
3521 : }
3522 : else
3523 : {
3524 73 : gcc_assert (ctx->outer
3525 : && gimple_code (ctx->outer->stmt)
3526 : == GIMPLE_OMP_SECTIONS);
3527 73 : ctx->outer->cancellable = true;
3528 73 : if (omp_find_clause (gimple_omp_sections_clauses
3529 73 : (ctx->outer->stmt),
3530 : OMP_CLAUSE_NOWAIT))
3531 10 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3532 : "%<cancel sections%> inside "
3533 : "%<nowait%> sections construct");
3534 : }
3535 : }
3536 10 : kind = "sections";
3537 10 : break;
3538 528 : case 8:
3539 528 : if (!is_task_ctx (ctx)
3540 528 : && (!is_taskloop_ctx (ctx)
3541 54 : || ctx->outer == NULL
3542 54 : || !is_task_ctx (ctx->outer)))
3543 : bad = "task";
3544 : else
3545 : {
3546 248 : for (omp_context *octx = ctx->outer;
3547 439 : octx; octx = octx->outer)
3548 : {
3549 425 : switch (gimple_code (octx->stmt))
3550 : {
3551 : case GIMPLE_OMP_TASKGROUP:
3552 : break;
3553 40 : case GIMPLE_OMP_TARGET:
3554 40 : if (gimple_omp_target_kind (octx->stmt)
3555 : != GF_OMP_TARGET_KIND_REGION)
3556 20 : continue;
3557 : /* FALLTHRU */
3558 99 : case GIMPLE_OMP_PARALLEL:
3559 99 : case GIMPLE_OMP_TEAMS:
3560 99 : error_at (gimple_location (stmt),
3561 : "%<%s taskgroup%> construct not closely "
3562 : "nested inside of %<taskgroup%> region",
3563 : construct);
3564 99 : return false;
3565 104 : case GIMPLE_OMP_TASK:
3566 104 : if (gimple_omp_task_taskloop_p (octx->stmt)
3567 94 : && octx->outer
3568 198 : && is_taskloop_ctx (octx->outer))
3569 : {
3570 94 : tree clauses
3571 94 : = gimple_omp_for_clauses (octx->outer->stmt);
3572 94 : if (!omp_find_clause (clauses, OMP_CLAUSE_NOGROUP))
3573 : break;
3574 : }
3575 60 : continue;
3576 111 : default:
3577 111 : continue;
3578 171 : }
3579 : break;
3580 : }
3581 149 : ctx->cancellable = true;
3582 : }
3583 149 : kind = "taskgroup";
3584 149 : break;
3585 0 : default:
3586 0 : error_at (gimple_location (stmt), "invalid arguments");
3587 0 : return false;
3588 : }
3589 328 : if (bad)
3590 : {
3591 1200 : error_at (gimple_location (stmt),
3592 : "%<%s %s%> construct not closely nested inside of %qs",
3593 : construct, kind, bad);
3594 1200 : return false;
3595 : }
3596 : }
3597 : /* FALLTHRU */
3598 : case GIMPLE_OMP_SECTIONS:
3599 : case GIMPLE_OMP_SINGLE:
3600 55580 : for (; ctx != NULL; ctx = ctx->outer)
3601 37517 : switch (gimple_code (ctx->stmt))
3602 : {
3603 6873 : case GIMPLE_OMP_FOR:
3604 6873 : if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
3605 6873 : && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
3606 : break;
3607 : /* FALLTHRU */
3608 778 : case GIMPLE_OMP_SECTIONS:
3609 778 : case GIMPLE_OMP_SINGLE:
3610 778 : case GIMPLE_OMP_ORDERED:
3611 778 : case GIMPLE_OMP_MASTER:
3612 778 : case GIMPLE_OMP_MASKED:
3613 778 : case GIMPLE_OMP_TASK:
3614 778 : case GIMPLE_OMP_CRITICAL:
3615 778 : if (is_gimple_call (stmt))
3616 : {
3617 707 : if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3618 : != BUILT_IN_GOMP_BARRIER)
3619 : return true;
3620 23 : error_at (gimple_location (stmt),
3621 : "barrier region may not be closely nested inside "
3622 : "of work-sharing, %<loop%>, %<critical%>, "
3623 : "%<ordered%>, %<master%>, %<masked%>, explicit "
3624 : "%<task%> or %<taskloop%> region");
3625 23 : return false;
3626 : }
3627 71 : error_at (gimple_location (stmt),
3628 : "work-sharing region may not be closely nested inside "
3629 : "of work-sharing, %<loop%>, %<critical%>, %<ordered%>, "
3630 : "%<master%>, %<masked%>, explicit %<task%> or "
3631 : "%<taskloop%> region");
3632 71 : return false;
3633 : case GIMPLE_OMP_PARALLEL:
3634 : case GIMPLE_OMP_TEAMS:
3635 : return true;
3636 13437 : case GIMPLE_OMP_TARGET:
3637 13437 : if (gimple_omp_target_kind (ctx->stmt)
3638 : == GF_OMP_TARGET_KIND_REGION)
3639 : return true;
3640 : break;
3641 : default:
3642 : break;
3643 : }
3644 : break;
3645 : case GIMPLE_OMP_MASTER:
3646 : case GIMPLE_OMP_MASKED:
3647 1519 : for (; ctx != NULL; ctx = ctx->outer)
3648 1014 : switch (gimple_code (ctx->stmt))
3649 : {
3650 14 : case GIMPLE_OMP_FOR:
3651 14 : if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
3652 14 : && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
3653 : break;
3654 : /* FALLTHRU */
3655 30 : case GIMPLE_OMP_SECTIONS:
3656 30 : case GIMPLE_OMP_SINGLE:
3657 30 : case GIMPLE_OMP_TASK:
3658 60 : error_at (gimple_location (stmt),
3659 : "%qs region may not be closely nested inside "
3660 : "of work-sharing, %<loop%>, explicit %<task%> or "
3661 : "%<taskloop%> region",
3662 30 : gimple_code (stmt) == GIMPLE_OMP_MASTER
3663 : ? "master" : "masked");
3664 30 : return false;
3665 : case GIMPLE_OMP_PARALLEL:
3666 : case GIMPLE_OMP_TEAMS:
3667 : return true;
3668 10 : case GIMPLE_OMP_TARGET:
3669 10 : if (gimple_omp_target_kind (ctx->stmt)
3670 : == GF_OMP_TARGET_KIND_REGION)
3671 : return true;
3672 : break;
3673 : default:
3674 : break;
3675 : }
3676 : break;
3677 : case GIMPLE_OMP_SCOPE:
3678 274 : for (; ctx != NULL; ctx = ctx->outer)
3679 175 : switch (gimple_code (ctx->stmt))
3680 : {
3681 7 : case GIMPLE_OMP_FOR:
3682 7 : if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
3683 7 : && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
3684 : break;
3685 : /* FALLTHRU */
3686 25 : case GIMPLE_OMP_SECTIONS:
3687 25 : case GIMPLE_OMP_SINGLE:
3688 25 : case GIMPLE_OMP_TASK:
3689 25 : case GIMPLE_OMP_CRITICAL:
3690 25 : case GIMPLE_OMP_ORDERED:
3691 25 : case GIMPLE_OMP_MASTER:
3692 25 : case GIMPLE_OMP_MASKED:
3693 25 : error_at (gimple_location (stmt),
3694 : "%<scope%> region may not be closely nested inside "
3695 : "of work-sharing, %<loop%>, explicit %<task%>, "
3696 : "%<taskloop%>, %<critical%>, %<ordered%>, %<master%>, "
3697 : "or %<masked%> region");
3698 25 : return false;
3699 : case GIMPLE_OMP_PARALLEL:
3700 : case GIMPLE_OMP_TEAMS:
3701 : return true;
3702 5 : case GIMPLE_OMP_TARGET:
3703 5 : if (gimple_omp_target_kind (ctx->stmt)
3704 : == GF_OMP_TARGET_KIND_REGION)
3705 : return true;
3706 : break;
3707 : default:
3708 : break;
3709 : }
3710 : break;
3711 5378 : case GIMPLE_OMP_TASK:
3712 22939 : for (c = gimple_omp_task_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
3713 17565 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS)
3714 : {
3715 4 : enum omp_clause_doacross_kind kind = OMP_CLAUSE_DOACROSS_KIND (c);
3716 4 : error_at (OMP_CLAUSE_LOCATION (c),
3717 : "%<%s(%s)%> is only allowed in %<omp ordered%>",
3718 4 : OMP_CLAUSE_DOACROSS_DEPEND (c) ? "depend" : "doacross",
3719 : kind == OMP_CLAUSE_DOACROSS_SOURCE ? "source" : "sink");
3720 4 : return false;
3721 : }
3722 : break;
3723 1631 : case GIMPLE_OMP_ORDERED:
3724 2873 : for (c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
3725 2873 : c; c = OMP_CLAUSE_CHAIN (c))
3726 : {
3727 1258 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DOACROSS)
3728 : {
3729 169 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
3730 : {
3731 4 : error_at (OMP_CLAUSE_LOCATION (c),
3732 : "invalid depend kind in omp %<ordered%> %<depend%>");
3733 4 : return false;
3734 : }
3735 165 : gcc_assert (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_THREADS
3736 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SIMD);
3737 165 : continue;
3738 : }
3739 :
3740 1089 : tree oclause;
3741 : /* Look for containing ordered(N) loop. */
3742 1089 : if (ctx == NULL
3743 1077 : || gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
3744 1089 : || (oclause
3745 1077 : = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3746 : OMP_CLAUSE_ORDERED)) == NULL_TREE)
3747 : {
3748 12 : error_at (OMP_CLAUSE_LOCATION (c),
3749 : "%<ordered%> construct with %<depend%> clause "
3750 : "must be closely nested inside an %<ordered%> loop");
3751 12 : return false;
3752 : }
3753 : }
3754 1615 : c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
3755 1615 : if (omp_find_clause (c, OMP_CLAUSE_SIMD))
3756 : {
3757 : /* ordered simd must be closely nested inside of simd region,
3758 : and simd region must not encounter constructs other than
3759 : ordered simd, therefore ordered simd may be either orphaned,
3760 : or ctx->stmt must be simd. The latter case is handled already
3761 : earlier. */
3762 34 : if (ctx != NULL)
3763 : {
3764 10 : error_at (gimple_location (stmt),
3765 : "%<ordered%> %<simd%> must be closely nested inside "
3766 : "%<simd%> region");
3767 10 : return false;
3768 : }
3769 : }
3770 1605 : for (; ctx != NULL; ctx = ctx->outer)
3771 1474 : switch (gimple_code (ctx->stmt))
3772 : {
3773 38 : case GIMPLE_OMP_CRITICAL:
3774 38 : case GIMPLE_OMP_TASK:
3775 38 : case GIMPLE_OMP_ORDERED:
3776 38 : ordered_in_taskloop:
3777 38 : error_at (gimple_location (stmt),
3778 : "%<ordered%> region may not be closely nested inside "
3779 : "of %<critical%>, %<ordered%>, explicit %<task%> or "
3780 : "%<taskloop%> region");
3781 38 : return false;
3782 1416 : case GIMPLE_OMP_FOR:
3783 1416 : if (gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_TASKLOOP)
3784 10 : goto ordered_in_taskloop;
3785 1406 : tree o;
3786 1406 : o = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3787 : OMP_CLAUSE_ORDERED);
3788 1406 : if (o == NULL)
3789 : {
3790 22 : error_at (gimple_location (stmt),
3791 : "%<ordered%> region must be closely nested inside "
3792 : "a loop region with an %<ordered%> clause");
3793 22 : return false;
3794 : }
3795 1384 : if (!gimple_omp_ordered_standalone_p (stmt))
3796 : {
3797 375 : if (OMP_CLAUSE_ORDERED_DOACROSS (o))
3798 : {
3799 10 : error_at (gimple_location (stmt),
3800 : "%<ordered%> construct without %<doacross%> or "
3801 : "%<depend%> clauses must not have the same "
3802 : "binding region as %<ordered%> construct with "
3803 : "those clauses");
3804 10 : return false;
3805 : }
3806 365 : else if (OMP_CLAUSE_ORDERED_EXPR (o))
3807 : {
3808 23 : tree co
3809 23 : = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3810 : OMP_CLAUSE_COLLAPSE);
3811 23 : HOST_WIDE_INT
3812 23 : o_n = tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (o));
3813 23 : HOST_WIDE_INT c_n = 1;
3814 23 : if (co)
3815 5 : c_n = tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (co));
3816 23 : if (o_n != c_n)
3817 : {
3818 10 : error_at (gimple_location (stmt),
3819 : "%<ordered%> construct without %<doacross%> "
3820 : "or %<depend%> clauses binds to loop where "
3821 : "%<collapse%> argument %wd is different from "
3822 : "%<ordered%> argument %wd", c_n, o_n);
3823 10 : return false;
3824 : }
3825 : }
3826 : }
3827 : return true;
3828 10 : case GIMPLE_OMP_TARGET:
3829 10 : if (gimple_omp_target_kind (ctx->stmt)
3830 : != GF_OMP_TARGET_KIND_REGION)
3831 : break;
3832 : /* FALLTHRU */
3833 30 : case GIMPLE_OMP_PARALLEL:
3834 30 : case GIMPLE_OMP_TEAMS:
3835 30 : error_at (gimple_location (stmt),
3836 : "%<ordered%> region must be closely nested inside "
3837 : "a loop region with an %<ordered%> clause");
3838 30 : return false;
3839 : default:
3840 : break;
3841 : }
3842 : break;
3843 462 : case GIMPLE_OMP_CRITICAL:
3844 462 : {
3845 462 : tree this_stmt_name
3846 462 : = gimple_omp_critical_name (as_a <gomp_critical *> (stmt));
3847 888 : for (; ctx != NULL; ctx = ctx->outer)
3848 432 : if (gomp_critical *other_crit
3849 451 : = dyn_cast <gomp_critical *> (ctx->stmt))
3850 25 : if (this_stmt_name == gimple_omp_critical_name (other_crit))
3851 : {
3852 6 : error_at (gimple_location (stmt),
3853 : "%<critical%> region may not be nested inside "
3854 : "a %<critical%> region with the same name");
3855 6 : return false;
3856 : }
3857 : }
3858 : break;
3859 8780 : case GIMPLE_OMP_TEAMS:
3860 8780 : if (ctx == NULL)
3861 : break;
3862 6162 : else if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET
3863 6162 : || (gimple_omp_target_kind (ctx->stmt)
3864 : != GF_OMP_TARGET_KIND_REGION))
3865 : {
3866 : /* Teams construct can appear either strictly nested inside of
3867 : target construct with no intervening stmts, or can be encountered
3868 : only by initial task (so must not appear inside any OpenMP
3869 : construct. */
3870 44 : error_at (gimple_location (stmt),
3871 : "%<teams%> construct must be closely nested inside of "
3872 : "%<target%> construct or not nested in any OpenMP "
3873 : "construct");
3874 44 : return false;
3875 : }
3876 : break;
3877 42236 : case GIMPLE_OMP_TARGET:
3878 189231 : for (c = gimple_omp_target_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
3879 146999 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS)
3880 : {
3881 4 : enum omp_clause_doacross_kind kind = OMP_CLAUSE_DOACROSS_KIND (c);
3882 4 : error_at (OMP_CLAUSE_LOCATION (c),
3883 : "%<depend(%s)%> is only allowed in %<omp ordered%>",
3884 : kind == OMP_CLAUSE_DOACROSS_SOURCE ? "source" : "sink");
3885 4 : return false;
3886 : }
3887 42232 : if (is_gimple_omp_offloaded (stmt)
3888 42232 : && oacc_get_fn_attrib (cfun->decl) != NULL)
3889 : {
3890 4 : error_at (gimple_location (stmt),
3891 : "OpenACC region inside of OpenACC routine, nested "
3892 : "parallelism not supported yet");
3893 4 : return false;
3894 : }
3895 49642 : for (; ctx != NULL; ctx = ctx->outer)
3896 : {
3897 7820 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET)
3898 : {
3899 768 : if (is_gimple_omp (stmt)
3900 768 : && is_gimple_omp_oacc (stmt)
3901 242 : && is_gimple_omp (ctx->stmt))
3902 : {
3903 242 : error_at (gimple_location (stmt),
3904 : "OpenACC construct inside of non-OpenACC region");
3905 242 : return false;
3906 : }
3907 526 : continue;
3908 : }
3909 :
3910 7052 : const char *stmt_name, *ctx_stmt_name;
3911 7052 : switch (gimple_omp_target_kind (stmt))
3912 : {
3913 : case GF_OMP_TARGET_KIND_REGION: stmt_name = "target"; break;
3914 399 : case GF_OMP_TARGET_KIND_DATA: stmt_name = "target data"; break;
3915 1688 : case GF_OMP_TARGET_KIND_UPDATE: stmt_name = "target update"; break;
3916 12 : case GF_OMP_TARGET_KIND_ENTER_DATA:
3917 12 : stmt_name = "target enter data"; break;
3918 18 : case GF_OMP_TARGET_KIND_EXIT_DATA:
3919 18 : stmt_name = "target exit data"; break;
3920 1442 : case GF_OMP_TARGET_KIND_OACC_PARALLEL: stmt_name = "parallel"; break;
3921 1621 : case GF_OMP_TARGET_KIND_OACC_KERNELS: stmt_name = "kernels"; break;
3922 183 : case GF_OMP_TARGET_KIND_OACC_SERIAL: stmt_name = "serial"; break;
3923 386 : case GF_OMP_TARGET_KIND_OACC_DATA: stmt_name = "data"; break;
3924 256 : case GF_OMP_TARGET_KIND_OACC_UPDATE: stmt_name = "update"; break;
3925 94 : case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
3926 94 : stmt_name = "enter data"; break;
3927 86 : case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
3928 86 : stmt_name = "exit data"; break;
3929 64 : case GF_OMP_TARGET_KIND_OACC_DECLARE: stmt_name = "declare"; break;
3930 128 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA: stmt_name = "host_data";
3931 128 : break;
3932 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
3933 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
3934 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
3935 : /* OpenACC 'kernels' decomposed parts. */
3936 1621 : stmt_name = "kernels"; break;
3937 0 : default: gcc_unreachable ();
3938 : }
3939 7052 : switch (gimple_omp_target_kind (ctx->stmt))
3940 : {
3941 : case GF_OMP_TARGET_KIND_REGION: ctx_stmt_name = "target"; break;
3942 2712 : case GF_OMP_TARGET_KIND_DATA: ctx_stmt_name = "target data"; break;
3943 35 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
3944 35 : ctx_stmt_name = "parallel"; break;
3945 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
3946 1037 : ctx_stmt_name = "kernels"; break;
3947 35 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
3948 35 : ctx_stmt_name = "serial"; break;
3949 3117 : case GF_OMP_TARGET_KIND_OACC_DATA: ctx_stmt_name = "data"; break;
3950 8 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
3951 8 : ctx_stmt_name = "host_data"; break;
3952 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
3953 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
3954 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
3955 : /* OpenACC 'kernels' decomposed parts. */
3956 1037 : ctx_stmt_name = "kernels"; break;
3957 0 : default: gcc_unreachable ();
3958 : }
3959 :
3960 : /* OpenACC/OpenMP mismatch? */
3961 14104 : if (is_gimple_omp_oacc (stmt)
3962 7052 : != is_gimple_omp_oacc (ctx->stmt))
3963 : {
3964 56 : error_at (gimple_location (stmt),
3965 : "%s %qs construct inside of %s %qs region",
3966 : (is_gimple_omp_oacc (stmt)
3967 : ? "OpenACC" : "OpenMP"), stmt_name,
3968 : (is_gimple_omp_oacc (ctx->stmt)
3969 : ? "OpenACC" : "OpenMP"), ctx_stmt_name);
3970 28 : return false;
3971 : }
3972 7024 : if (is_gimple_omp_offloaded (ctx->stmt))
3973 : {
3974 : /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3975 185 : if (is_gimple_omp_oacc (ctx->stmt))
3976 : {
3977 105 : error_at (gimple_location (stmt),
3978 : "%qs construct inside of %qs region",
3979 : stmt_name, ctx_stmt_name);
3980 105 : return false;
3981 : }
3982 : else
3983 : {
3984 80 : if ((gimple_omp_target_kind (ctx->stmt)
3985 : == GF_OMP_TARGET_KIND_REGION)
3986 80 : && (gimple_omp_target_kind (stmt)
3987 : == GF_OMP_TARGET_KIND_REGION))
3988 : {
3989 58 : c = omp_find_clause (gimple_omp_target_clauses (stmt),
3990 : OMP_CLAUSE_DEVICE);
3991 58 : if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
3992 : break;
3993 : }
3994 49 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3995 : "%qs construct inside of %qs region",
3996 : stmt_name, ctx_stmt_name);
3997 : }
3998 : }
3999 : }
4000 : break;
4001 : default:
4002 : break;
4003 : }
4004 : return true;
4005 : }
4006 :
4007 :
4008 : /* Helper function scan_omp.
4009 :
4010 : Callback for walk_tree or operators in walk_gimple_stmt used to
4011 : scan for OMP directives in TP. */
4012 :
4013 : static tree
4014 9813749 : scan_omp_1_op (tree *tp, int *walk_subtrees, void *data)
4015 : {
4016 9813749 : struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
4017 9813749 : omp_context *ctx = (omp_context *) wi->info;
4018 9813749 : tree t = *tp;
4019 9813749 : tree tmp;
4020 :
4021 9813749 : switch (TREE_CODE (t))
4022 : {
4023 4155424 : case VAR_DECL:
4024 4155424 : case PARM_DECL:
4025 4155424 : case LABEL_DECL:
4026 4155424 : case RESULT_DECL:
4027 4155424 : if (ctx)
4028 : {
4029 2129349 : tmp = NULL_TREE;
4030 2129349 : if (TREE_CODE (t) == VAR_DECL
4031 3815034 : && (tmp = lookup_attribute ("omp allocate var",
4032 1685685 : DECL_ATTRIBUTES (t))) != NULL_TREE)
4033 69 : t = TREE_VALUE (TREE_VALUE (tmp));
4034 2129349 : tree repl = remap_decl (t, &ctx->cb);
4035 2129349 : gcc_checking_assert (TREE_CODE (repl) != ERROR_MARK);
4036 2129349 : if (tmp != NULL_TREE && t != repl)
4037 31 : *tp = build_fold_addr_expr (repl);
4038 2129318 : else if (tmp == NULL_TREE)
4039 2129280 : *tp = repl;
4040 : }
4041 : break;
4042 :
4043 193632 : case INDIRECT_REF:
4044 193632 : case MEM_REF:
4045 193632 : if (ctx
4046 78072 : && TREE_CODE (TREE_OPERAND (t, 0)) == VAR_DECL
4047 252687 : && ((tmp = lookup_attribute ("omp allocate var",
4048 59055 : DECL_ATTRIBUTES (TREE_OPERAND (t, 0))))
4049 : != NULL_TREE))
4050 : {
4051 131 : tmp = TREE_VALUE (TREE_VALUE (tmp));
4052 131 : tree repl = remap_decl (tmp, &ctx->cb);
4053 131 : gcc_checking_assert (TREE_CODE (repl) != ERROR_MARK);
4054 131 : if (tmp != repl)
4055 120 : *tp = repl;
4056 : break;
4057 : }
4058 5658194 : gcc_fallthrough ();
4059 :
4060 5658194 : default:
4061 5658194 : if (ctx && TYPE_P (t))
4062 5 : *tp = remap_type (t, &ctx->cb);
4063 5658189 : else if (!DECL_P (t))
4064 : {
4065 4694097 : *walk_subtrees = 1;
4066 4694097 : if (ctx)
4067 : {
4068 1427239 : tree tem = remap_type (TREE_TYPE (t), &ctx->cb);
4069 1427239 : if (tem != TREE_TYPE (t))
4070 : {
4071 9946 : if (TREE_CODE (t) == INTEGER_CST)
4072 4202 : *tp = wide_int_to_tree (tem, wi::to_wide (t));
4073 : else
4074 5744 : TREE_TYPE (t) = tem;
4075 : }
4076 : }
4077 : }
4078 : break;
4079 : }
4080 :
4081 9813749 : return NULL_TREE;
4082 : }
4083 :
4084 : /* Return true if FNDECL is a setjmp or a longjmp. */
4085 :
4086 : static bool
4087 3042 : setjmp_or_longjmp_p (const_tree fndecl)
4088 : {
4089 3042 : if (fndecl_built_in_p (fndecl, BUILT_IN_SETJMP, BUILT_IN_LONGJMP))
4090 : return true;
4091 :
4092 3042 : tree declname = DECL_NAME (fndecl);
4093 3042 : if (!declname
4094 3042 : || (DECL_CONTEXT (fndecl) != NULL_TREE
4095 2347 : && TREE_CODE (DECL_CONTEXT (fndecl)) != TRANSLATION_UNIT_DECL)
4096 5833 : || !TREE_PUBLIC (fndecl))
4097 : return false;
4098 :
4099 2629 : const char *name = IDENTIFIER_POINTER (declname);
4100 2629 : return !strcmp (name, "setjmp") || !strcmp (name, "longjmp");
4101 : }
4102 :
4103 : /* Helper function for scan_omp.
4104 :
4105 : Callback for walk_gimple_stmt used to scan for OMP directives in
4106 : the current statement in GSI. */
4107 :
4108 : static tree
4109 3409594 : scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
4110 : struct walk_stmt_info *wi)
4111 : {
4112 3409594 : gimple *stmt = gsi_stmt (*gsi);
4113 3409594 : omp_context *ctx = (omp_context *) wi->info;
4114 :
4115 3409594 : if (gimple_has_location (stmt))
4116 2782658 : input_location = gimple_location (stmt);
4117 :
4118 : /* Check the nesting restrictions. */
4119 3409594 : bool remove = false;
4120 3409594 : if (is_gimple_omp (stmt))
4121 152670 : remove = !check_omp_nesting_restrictions (stmt, ctx);
4122 3256924 : else if (is_gimple_call (stmt))
4123 : {
4124 231357 : tree fndecl = gimple_call_fndecl (stmt);
4125 231357 : if (fndecl)
4126 : {
4127 215530 : if (ctx
4128 67776 : && gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
4129 14654 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
4130 3042 : && setjmp_or_longjmp_p (fndecl)
4131 215538 : && !ctx->loop_p)
4132 : {
4133 4 : remove = true;
4134 4 : error_at (gimple_location (stmt),
4135 : "setjmp/longjmp inside %<simd%> construct");
4136 : }
4137 215526 : else if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
4138 63370 : switch (DECL_FUNCTION_CODE (fndecl))
4139 : {
4140 4221 : case BUILT_IN_GOMP_BARRIER:
4141 4221 : case BUILT_IN_GOMP_CANCEL:
4142 4221 : case BUILT_IN_GOMP_CANCELLATION_POINT:
4143 4221 : case BUILT_IN_GOMP_TASKYIELD:
4144 4221 : case BUILT_IN_GOMP_TASKWAIT:
4145 4221 : case BUILT_IN_GOMP_TASKGROUP_START:
4146 4221 : case BUILT_IN_GOMP_TASKGROUP_END:
4147 4221 : remove = !check_omp_nesting_restrictions (stmt, ctx);
4148 4221 : break;
4149 : default:
4150 : break;
4151 : }
4152 152156 : else if (ctx)
4153 : {
4154 45850 : omp_context *octx = ctx;
4155 45850 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_SCAN && ctx->outer)
4156 45850 : octx = ctx->outer;
4157 45850 : if (octx->order_concurrent && omp_runtime_api_call (fndecl))
4158 : {
4159 240 : remove = true;
4160 240 : error_at (gimple_location (stmt),
4161 : "OpenMP runtime API call %qD in a region with "
4162 : "%<order(concurrent)%> clause", fndecl);
4163 : }
4164 45850 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
4165 3293 : && omp_runtime_api_call (fndecl)
4166 315 : && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
4167 : != strlen ("omp_get_num_teams"))
4168 182 : || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
4169 : "omp_get_num_teams") != 0)
4170 45998 : && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
4171 : != strlen ("omp_get_team_num"))
4172 121 : || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
4173 : "omp_get_team_num") != 0))
4174 : {
4175 27 : remove = true;
4176 27 : error_at (gimple_location (stmt),
4177 : "OpenMP runtime API call %qD strictly nested in a "
4178 : "%<teams%> region", fndecl);
4179 : }
4180 45850 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
4181 13316 : && (gimple_omp_target_kind (ctx->stmt)
4182 : == GF_OMP_TARGET_KIND_REGION)
4183 50780 : && omp_runtime_api_call (fndecl))
4184 : {
4185 365 : tree tgt_clauses = gimple_omp_target_clauses (ctx->stmt);
4186 365 : tree c = omp_find_clause (tgt_clauses, OMP_CLAUSE_DEVICE);
4187 365 : if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
4188 5 : error_at (gimple_location (stmt),
4189 : "OpenMP runtime API call %qD in a region with "
4190 : "%<device(ancestor)%> clause", fndecl);
4191 : }
4192 : }
4193 : }
4194 : }
4195 202745 : if (remove)
4196 : {
4197 3326 : stmt = gimple_build_nop ();
4198 3326 : gsi_replace (gsi, stmt, false);
4199 : }
4200 :
4201 3409594 : *handled_ops_p = true;
4202 :
4203 3409594 : switch (gimple_code (stmt))
4204 : {
4205 18173 : case GIMPLE_OMP_PARALLEL:
4206 18173 : taskreg_nesting_level++;
4207 18173 : scan_omp_parallel (gsi, ctx);
4208 18173 : taskreg_nesting_level--;
4209 18173 : break;
4210 :
4211 5374 : case GIMPLE_OMP_TASK:
4212 5374 : taskreg_nesting_level++;
4213 5374 : scan_omp_task (gsi, ctx);
4214 5374 : taskreg_nesting_level--;
4215 5374 : break;
4216 :
4217 52726 : case GIMPLE_OMP_FOR:
4218 52726 : if ((gimple_omp_for_kind (as_a <gomp_for *> (stmt))
4219 : == GF_OMP_FOR_KIND_SIMD)
4220 10810 : && gimple_omp_for_combined_into_p (stmt)
4221 60418 : && gimple_code (ctx->stmt) != GIMPLE_OMP_SCAN)
4222 : {
4223 7526 : tree clauses = gimple_omp_for_clauses (as_a <gomp_for *> (stmt));
4224 7526 : tree c = omp_find_clause (clauses, OMP_CLAUSE_REDUCTION);
4225 7526 : if (c && OMP_CLAUSE_REDUCTION_INSCAN (c) && !seen_error ())
4226 : {
4227 83 : scan_omp_simd_scan (gsi, as_a <gomp_for *> (stmt), ctx);
4228 83 : break;
4229 : }
4230 : }
4231 52643 : if ((gimple_omp_for_kind (as_a <gomp_for *> (stmt))
4232 : == GF_OMP_FOR_KIND_SIMD)
4233 10727 : && omp_maybe_offloaded_ctx (ctx)
4234 3641 : && omp_max_simt_vf ()
4235 52643 : && gimple_omp_for_collapse (stmt) == 1)
4236 0 : scan_omp_simd (gsi, as_a <gomp_for *> (stmt), ctx);
4237 : else
4238 52643 : scan_omp_for (as_a <gomp_for *> (stmt), ctx);
4239 : break;
4240 :
4241 184 : case GIMPLE_OMP_SCOPE:
4242 184 : ctx = new_omp_context (stmt, ctx);
4243 184 : scan_sharing_clauses (gimple_omp_scope_clauses (stmt), ctx);
4244 184 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4245 184 : break;
4246 :
4247 852 : case GIMPLE_OMP_DISPATCH:
4248 852 : ctx = new_omp_context (stmt, ctx);
4249 852 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4250 852 : break;
4251 :
4252 612 : case GIMPLE_OMP_INTEROP:
4253 612 : ctx = new_omp_context (stmt, ctx);
4254 612 : break;
4255 :
4256 581 : case GIMPLE_OMP_SECTIONS:
4257 581 : scan_omp_sections (as_a <gomp_sections *> (stmt), ctx);
4258 581 : break;
4259 :
4260 1228 : case GIMPLE_OMP_SINGLE:
4261 1228 : scan_omp_single (as_a <gomp_single *> (stmt), ctx);
4262 1228 : break;
4263 :
4264 1284 : case GIMPLE_OMP_SCAN:
4265 1284 : if (tree clauses = gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt)))
4266 : {
4267 606 : if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_INCLUSIVE)
4268 304 : ctx->scan_inclusive = true;
4269 302 : else if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_EXCLUSIVE)
4270 302 : ctx->scan_exclusive = true;
4271 : }
4272 : /* FALLTHRU */
4273 6207 : case GIMPLE_OMP_SECTION:
4274 6207 : case GIMPLE_OMP_STRUCTURED_BLOCK:
4275 6207 : case GIMPLE_OMP_MASTER:
4276 6207 : case GIMPLE_OMP_ORDERED:
4277 6207 : case GIMPLE_OMP_CRITICAL:
4278 6207 : ctx = new_omp_context (stmt, ctx);
4279 6207 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4280 6207 : break;
4281 :
4282 453 : case GIMPLE_OMP_MASKED:
4283 453 : ctx = new_omp_context (stmt, ctx);
4284 453 : scan_sharing_clauses (gimple_omp_masked_clauses (stmt), ctx);
4285 453 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4286 453 : break;
4287 :
4288 607 : case GIMPLE_OMP_TASKGROUP:
4289 607 : ctx = new_omp_context (stmt, ctx);
4290 607 : scan_sharing_clauses (gimple_omp_taskgroup_clauses (stmt), ctx);
4291 607 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4292 607 : break;
4293 :
4294 41853 : case GIMPLE_OMP_TARGET:
4295 41853 : if (is_gimple_omp_offloaded (stmt))
4296 : {
4297 24826 : taskreg_nesting_level++;
4298 24826 : scan_omp_target (as_a <gomp_target *> (stmt), ctx);
4299 24826 : taskreg_nesting_level--;
4300 : }
4301 : else
4302 17027 : scan_omp_target (as_a <gomp_target *> (stmt), ctx);
4303 : break;
4304 :
4305 8736 : case GIMPLE_OMP_TEAMS:
4306 8736 : if (gimple_omp_teams_host (as_a <gomp_teams *> (stmt)))
4307 : {
4308 2618 : taskreg_nesting_level++;
4309 2618 : scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
4310 2618 : taskreg_nesting_level--;
4311 : }
4312 : else
4313 6118 : scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
4314 : break;
4315 :
4316 281679 : case GIMPLE_BIND:
4317 281679 : {
4318 281679 : tree var;
4319 :
4320 281679 : *handled_ops_p = false;
4321 281679 : if (ctx)
4322 152628 : for (var = gimple_bind_vars (as_a <gbind *> (stmt));
4323 579632 : var ;
4324 427004 : var = DECL_CHAIN (var))
4325 427004 : insert_decl_map (&ctx->cb, var, var);
4326 : }
4327 : break;
4328 2990329 : default:
4329 2990329 : *handled_ops_p = false;
4330 2990329 : break;
4331 : }
4332 :
4333 3409594 : return NULL_TREE;
4334 : }
4335 :
4336 :
4337 : /* Scan all the statements starting at the current statement. CTX
4338 : contains context information about the OMP directives and
4339 : clauses found during the scan. */
4340 :
4341 : static void
4342 258754 : scan_omp (gimple_seq *body_p, omp_context *ctx)
4343 : {
4344 258754 : location_t saved_location;
4345 258754 : struct walk_stmt_info wi;
4346 :
4347 258754 : memset (&wi, 0, sizeof (wi));
4348 258754 : wi.info = ctx;
4349 258754 : wi.want_locations = true;
4350 :
4351 258754 : saved_location = input_location;
4352 258754 : walk_gimple_seq_mod (body_p, scan_omp_1_stmt, scan_omp_1_op, &wi);
4353 258754 : input_location = saved_location;
4354 258754 : }
4355 :
4356 : /* Re-gimplification and code generation routines. */
4357 :
4358 : /* Remove omp_member_access_dummy_var variables from gimple_bind_vars
4359 : of BIND if in a method. */
4360 :
4361 : static void
4362 258418 : maybe_remove_omp_member_access_dummy_vars (gbind *bind)
4363 : {
4364 258418 : if (DECL_ARGUMENTS (current_function_decl)
4365 97888 : && DECL_ARTIFICIAL (DECL_ARGUMENTS (current_function_decl))
4366 269168 : && (TREE_CODE (TREE_TYPE (DECL_ARGUMENTS (current_function_decl)))
4367 : == POINTER_TYPE))
4368 : {
4369 3520 : tree vars = gimple_bind_vars (bind);
4370 19340 : for (tree *pvar = &vars; *pvar; )
4371 15820 : if (omp_member_access_dummy_var (*pvar))
4372 719 : *pvar = DECL_CHAIN (*pvar);
4373 : else
4374 15101 : pvar = &DECL_CHAIN (*pvar);
4375 3520 : gimple_bind_set_vars (bind, vars);
4376 : }
4377 258418 : }
4378 :
4379 : /* Remove omp_member_access_dummy_var variables from BLOCK_VARS of
4380 : block and its subblocks. */
4381 :
4382 : static void
4383 12876 : remove_member_access_dummy_vars (tree block)
4384 : {
4385 22435 : for (tree *pvar = &BLOCK_VARS (block); *pvar; )
4386 9559 : if (omp_member_access_dummy_var (*pvar))
4387 108 : *pvar = DECL_CHAIN (*pvar);
4388 : else
4389 9451 : pvar = &DECL_CHAIN (*pvar);
4390 :
4391 16761 : for (block = BLOCK_SUBBLOCKS (block); block; block = BLOCK_CHAIN (block))
4392 3885 : remove_member_access_dummy_vars (block);
4393 12876 : }
4394 :
4395 : /* If a context was created for STMT when it was scanned, return it. */
4396 :
4397 : static omp_context *
4398 121191 : maybe_lookup_ctx (gimple *stmt)
4399 : {
4400 121191 : splay_tree_node n;
4401 121191 : n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt);
4402 121191 : return n ? (omp_context *) n->value : NULL;
4403 : }
4404 :
4405 :
4406 : /* Find the mapping for DECL in CTX or the immediately enclosing
4407 : context that has a mapping for DECL.
4408 :
4409 : If CTX is a nested parallel directive, we may have to use the decl
4410 : mappings created in CTX's parent context. Suppose that we have the
4411 : following parallel nesting (variable UIDs showed for clarity):
4412 :
4413 : iD.1562 = 0;
4414 : #omp parallel shared(iD.1562) -> outer parallel
4415 : iD.1562 = iD.1562 + 1;
4416 :
4417 : #omp parallel shared (iD.1562) -> inner parallel
4418 : iD.1562 = iD.1562 - 1;
4419 :
4420 : Each parallel structure will create a distinct .omp_data_s structure
4421 : for copying iD.1562 in/out of the directive:
4422 :
4423 : outer parallel .omp_data_s.1.i -> iD.1562
4424 : inner parallel .omp_data_s.2.i -> iD.1562
4425 :
4426 : A shared variable mapping will produce a copy-out operation before
4427 : the parallel directive and a copy-in operation after it. So, in
4428 : this case we would have:
4429 :
4430 : iD.1562 = 0;
4431 : .omp_data_o.1.i = iD.1562;
4432 : #omp parallel shared(iD.1562) -> outer parallel
4433 : .omp_data_i.1 = &.omp_data_o.1
4434 : .omp_data_i.1->i = .omp_data_i.1->i + 1;
4435 :
4436 : .omp_data_o.2.i = iD.1562; -> **
4437 : #omp parallel shared(iD.1562) -> inner parallel
4438 : .omp_data_i.2 = &.omp_data_o.2
4439 : .omp_data_i.2->i = .omp_data_i.2->i - 1;
4440 :
4441 :
4442 : ** This is a problem. The symbol iD.1562 cannot be referenced
4443 : inside the body of the outer parallel region. But since we are
4444 : emitting this copy operation while expanding the inner parallel
4445 : directive, we need to access the CTX structure of the outer
4446 : parallel directive to get the correct mapping:
4447 :
4448 : .omp_data_o.2.i = .omp_data_i.1->i
4449 :
4450 : Since there may be other workshare or parallel directives enclosing
4451 : the parallel directive, it may be necessary to walk up the context
4452 : parent chain. This is not a problem in general because nested
4453 : parallelism happens only rarely. */
4454 :
4455 : static tree
4456 126285 : lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
4457 : {
4458 126285 : tree t;
4459 126285 : omp_context *up;
4460 :
4461 183015 : for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
4462 56730 : t = maybe_lookup_decl (decl, up);
4463 :
4464 126285 : gcc_assert (!ctx->is_nested || t || is_global_var (decl));
4465 :
4466 97962 : return t ? t : decl;
4467 : }
4468 :
4469 :
4470 : /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
4471 : in outer contexts. */
4472 :
4473 : static tree
4474 357021 : maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
4475 : {
4476 357021 : tree t = NULL;
4477 357021 : omp_context *up;
4478 :
4479 628158 : for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
4480 271137 : t = maybe_lookup_decl (decl, up);
4481 :
4482 357021 : return t ? t : decl;
4483 : }
4484 :
4485 :
4486 : /* Construct the initialization value for reduction operation OP. */
4487 :
4488 : tree
4489 13078 : omp_reduction_init_op (location_t loc, enum tree_code op, tree type)
4490 : {
4491 13078 : switch (op)
4492 : {
4493 10743 : case PLUS_EXPR:
4494 10743 : case MINUS_EXPR:
4495 10743 : case BIT_IOR_EXPR:
4496 10743 : case BIT_XOR_EXPR:
4497 10743 : case TRUTH_OR_EXPR:
4498 10743 : case TRUTH_ORIF_EXPR:
4499 10743 : case TRUTH_XOR_EXPR:
4500 10743 : case NE_EXPR:
4501 10743 : return build_zero_cst (type);
4502 :
4503 1431 : case MULT_EXPR:
4504 1431 : case TRUTH_AND_EXPR:
4505 1431 : case TRUTH_ANDIF_EXPR:
4506 1431 : case EQ_EXPR:
4507 1431 : return fold_convert_loc (loc, type, integer_one_node);
4508 :
4509 208 : case BIT_AND_EXPR:
4510 208 : return fold_convert_loc (loc, type, integer_minus_one_node);
4511 :
4512 388 : case MAX_EXPR:
4513 388 : if (SCALAR_FLOAT_TYPE_P (type))
4514 : {
4515 158 : REAL_VALUE_TYPE min;
4516 158 : if (HONOR_INFINITIES (type))
4517 158 : real_arithmetic (&min, NEGATE_EXPR, &dconstinf, NULL);
4518 : else
4519 0 : real_maxval (&min, 1, TYPE_MODE (type));
4520 158 : return build_real (type, min);
4521 : }
4522 230 : else if (POINTER_TYPE_P (type))
4523 : {
4524 2 : wide_int min
4525 2 : = wi::min_value (TYPE_PRECISION (type), TYPE_SIGN (type));
4526 2 : return wide_int_to_tree (type, min);
4527 2 : }
4528 : else
4529 : {
4530 228 : gcc_assert (INTEGRAL_TYPE_P (type));
4531 228 : return TYPE_MIN_VALUE (type);
4532 : }
4533 :
4534 308 : case MIN_EXPR:
4535 308 : if (SCALAR_FLOAT_TYPE_P (type))
4536 : {
4537 108 : REAL_VALUE_TYPE max;
4538 108 : if (HONOR_INFINITIES (type))
4539 108 : max = dconstinf;
4540 : else
4541 0 : real_maxval (&max, 0, TYPE_MODE (type));
4542 108 : return build_real (type, max);
4543 : }
4544 200 : else if (POINTER_TYPE_P (type))
4545 : {
4546 2 : wide_int max
4547 2 : = wi::max_value (TYPE_PRECISION (type), TYPE_SIGN (type));
4548 2 : return wide_int_to_tree (type, max);
4549 2 : }
4550 : else
4551 : {
4552 198 : gcc_assert (INTEGRAL_TYPE_P (type));
4553 198 : return TYPE_MAX_VALUE (type);
4554 : }
4555 :
4556 0 : default:
4557 0 : gcc_unreachable ();
4558 : }
4559 : }
4560 :
4561 : /* Construct the initialization value for reduction CLAUSE. */
4562 :
4563 : tree
4564 10457 : omp_reduction_init (tree clause, tree type)
4565 : {
4566 10457 : return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause),
4567 10457 : OMP_CLAUSE_REDUCTION_CODE (clause), type);
4568 : }
4569 :
4570 : /* Return alignment to be assumed for var in CLAUSE, which should be
4571 : OMP_CLAUSE_ALIGNED. */
4572 :
4573 : static tree
4574 140 : omp_clause_aligned_alignment (tree clause)
4575 : {
4576 140 : if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause))
4577 123 : return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause);
4578 :
4579 : /* Otherwise return implementation defined alignment. */
4580 17 : unsigned int al = 1;
4581 17 : opt_scalar_mode mode_iter;
4582 17 : auto_vector_modes modes;
4583 17 : targetm.vectorize.autovectorize_vector_modes (&modes, true);
4584 17 : static enum mode_class classes[]
4585 : = { MODE_INT, MODE_VECTOR_INT, MODE_FLOAT, MODE_VECTOR_FLOAT };
4586 51 : for (int i = 0; i < 4; i += 2)
4587 : /* The for loop above dictates that we only walk through scalar classes. */
4588 255 : FOR_EACH_MODE_IN_CLASS (mode_iter, classes[i])
4589 : {
4590 221 : scalar_mode mode = mode_iter.require ();
4591 221 : machine_mode vmode = targetm.vectorize.preferred_simd_mode (mode);
4592 221 : if (GET_MODE_CLASS (vmode) != classes[i + 1])
4593 323 : continue;
4594 : machine_mode alt_vmode;
4595 476 : for (unsigned int j = 0; j < modes.length (); ++j)
4596 527 : if (related_vector_mode (modes[j], mode).exists (&alt_vmode)
4597 578 : && known_ge (GET_MODE_SIZE (alt_vmode), GET_MODE_SIZE (vmode)))
4598 119 : vmode = alt_vmode;
4599 :
4600 119 : tree type = lang_hooks.types.type_for_mode (mode, 1);
4601 119 : if (type == NULL_TREE || TYPE_MODE (type) != mode)
4602 17 : continue;
4603 102 : type = build_vector_type_for_mode (type, vmode);
4604 102 : if (TYPE_MODE (type) != vmode)
4605 0 : continue;
4606 102 : if (TYPE_ALIGN_UNIT (type) > al)
4607 17 : al = TYPE_ALIGN_UNIT (type);
4608 : }
4609 17 : return build_int_cst (integer_type_node, al);
4610 17 : }
4611 :
4612 :
4613 : /* This structure is part of the interface between lower_rec_simd_input_clauses
4614 : and lower_rec_input_clauses. */
4615 :
4616 : class omplow_simd_context {
4617 : public:
4618 77144 : omplow_simd_context () { memset (this, 0, sizeof (*this)); }
4619 : tree idx;
4620 : tree lane;
4621 : tree lastlane;
4622 : vec<tree, va_heap> simt_eargs;
4623 : gimple_seq simt_dlist;
4624 : poly_uint64 max_vf;
4625 : bool is_simt;
4626 : };
4627 :
4628 : /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
4629 : privatization. */
4630 :
4631 : static bool
4632 10228 : lower_rec_simd_input_clauses (tree new_var, omp_context *ctx,
4633 : omplow_simd_context *sctx, tree &ivar,
4634 : tree &lvar, tree *rvar = NULL,
4635 : tree *rvar2 = NULL)
4636 : {
4637 10228 : if (known_eq (sctx->max_vf, 0U))
4638 : {
4639 4790 : sctx->max_vf = (sctx->is_simt ? omp_max_simt_vf ()
4640 4790 : : omp_max_vf (omp_maybe_offloaded_ctx (ctx)));
4641 4790 : if (maybe_gt (sctx->max_vf, 1U))
4642 : {
4643 3459 : tree c = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
4644 : OMP_CLAUSE_SAFELEN);
4645 3459 : if (c)
4646 : {
4647 89 : poly_uint64 safe_len;
4648 89 : if (!poly_int_tree_p (OMP_CLAUSE_SAFELEN_EXPR (c), &safe_len)
4649 89 : || maybe_lt (safe_len, 1U))
4650 6 : sctx->max_vf = 1;
4651 : else
4652 83 : sctx->max_vf = lower_bound (sctx->max_vf, safe_len);
4653 : }
4654 : }
4655 4790 : if (sctx->is_simt && !known_eq (sctx->max_vf, 1U))
4656 : {
4657 0 : for (tree c = gimple_omp_for_clauses (ctx->stmt); c;
4658 0 : c = OMP_CLAUSE_CHAIN (c))
4659 : {
4660 0 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
4661 0 : continue;
4662 :
4663 0 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
4664 : {
4665 : /* UDR reductions are not supported yet for SIMT, disable
4666 : SIMT. */
4667 0 : sctx->max_vf = 1;
4668 0 : break;
4669 : }
4670 :
4671 0 : if (truth_value_p (OMP_CLAUSE_REDUCTION_CODE (c))
4672 0 : && !INTEGRAL_TYPE_P (TREE_TYPE (new_var)))
4673 : {
4674 : /* Doing boolean operations on non-integral types is
4675 : for conformance only, it's not worth supporting this
4676 : for SIMT. */
4677 0 : sctx->max_vf = 1;
4678 0 : break;
4679 : }
4680 : }
4681 : }
4682 4790 : if (maybe_gt (sctx->max_vf, 1U))
4683 : {
4684 3440 : sctx->idx = create_tmp_var (unsigned_type_node);
4685 3440 : sctx->lane = create_tmp_var (unsigned_type_node);
4686 : }
4687 : }
4688 10228 : if (known_eq (sctx->max_vf, 1U))
4689 : return false;
4690 :
4691 7189 : if (sctx->is_simt)
4692 : {
4693 0 : if (is_gimple_reg (new_var))
4694 : {
4695 0 : ivar = lvar = new_var;
4696 0 : return true;
4697 : }
4698 0 : tree type = TREE_TYPE (new_var), ptype = build_pointer_type (type);
4699 0 : ivar = lvar = create_tmp_var (type);
4700 0 : TREE_ADDRESSABLE (ivar) = 1;
4701 0 : DECL_ATTRIBUTES (ivar) = tree_cons (get_identifier ("omp simt private"),
4702 0 : NULL, DECL_ATTRIBUTES (ivar));
4703 0 : sctx->simt_eargs.safe_push (build1 (ADDR_EXPR, ptype, ivar));
4704 0 : tree clobber = build_clobber (type);
4705 0 : gimple *g = gimple_build_assign (ivar, clobber);
4706 0 : gimple_seq_add_stmt (&sctx->simt_dlist, g);
4707 : }
4708 : else
4709 : {
4710 7189 : tree atype = build_array_type_nelts (TREE_TYPE (new_var), sctx->max_vf);
4711 7189 : tree avar = create_tmp_var_raw (atype);
4712 7189 : if (TREE_ADDRESSABLE (new_var))
4713 912 : TREE_ADDRESSABLE (avar) = 1;
4714 7189 : DECL_ATTRIBUTES (avar)
4715 7189 : = tree_cons (get_identifier ("omp simd array"), NULL,
4716 7189 : DECL_ATTRIBUTES (avar));
4717 7189 : gimple_add_tmp_var (avar);
4718 7189 : tree iavar = avar;
4719 7189 : if (rvar && !ctx->for_simd_scan_phase)
4720 : {
4721 : /* For inscan reductions, create another array temporary,
4722 : which will hold the reduced value. */
4723 232 : iavar = create_tmp_var_raw (atype);
4724 232 : if (TREE_ADDRESSABLE (new_var))
4725 33 : TREE_ADDRESSABLE (iavar) = 1;
4726 232 : DECL_ATTRIBUTES (iavar)
4727 232 : = tree_cons (get_identifier ("omp simd array"), NULL,
4728 : tree_cons (get_identifier ("omp simd inscan"), NULL,
4729 232 : DECL_ATTRIBUTES (iavar)));
4730 232 : gimple_add_tmp_var (iavar);
4731 232 : ctx->cb.decl_map->put (avar, iavar);
4732 232 : if (sctx->lastlane == NULL_TREE)
4733 184 : sctx->lastlane = create_tmp_var (unsigned_type_node);
4734 232 : *rvar = build4 (ARRAY_REF, TREE_TYPE (new_var), iavar,
4735 : sctx->lastlane, NULL_TREE, NULL_TREE);
4736 232 : TREE_THIS_NOTRAP (*rvar) = 1;
4737 :
4738 232 : if (ctx->scan_exclusive)
4739 : {
4740 : /* And for exclusive scan yet another one, which will
4741 : hold the value during the scan phase. */
4742 114 : tree savar = create_tmp_var_raw (atype);
4743 114 : if (TREE_ADDRESSABLE (new_var))
4744 17 : TREE_ADDRESSABLE (savar) = 1;
4745 114 : DECL_ATTRIBUTES (savar)
4746 114 : = tree_cons (get_identifier ("omp simd array"), NULL,
4747 : tree_cons (get_identifier ("omp simd inscan "
4748 : "exclusive"), NULL,
4749 114 : DECL_ATTRIBUTES (savar)));
4750 114 : gimple_add_tmp_var (savar);
4751 114 : ctx->cb.decl_map->put (iavar, savar);
4752 114 : *rvar2 = build4 (ARRAY_REF, TREE_TYPE (new_var), savar,
4753 : sctx->idx, NULL_TREE, NULL_TREE);
4754 114 : TREE_THIS_NOTRAP (*rvar2) = 1;
4755 : }
4756 : }
4757 7189 : ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), iavar, sctx->idx,
4758 : NULL_TREE, NULL_TREE);
4759 7189 : lvar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, sctx->lane,
4760 : NULL_TREE, NULL_TREE);
4761 7189 : TREE_THIS_NOTRAP (ivar) = 1;
4762 7189 : TREE_THIS_NOTRAP (lvar) = 1;
4763 : }
4764 7189 : if (DECL_P (new_var))
4765 : {
4766 7035 : SET_DECL_VALUE_EXPR (new_var, lvar);
4767 7035 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
4768 : }
4769 : return true;
4770 : }
4771 :
4772 : /* Helper function of lower_rec_input_clauses. For a reference
4773 : in simd reduction, add an underlying variable it will reference. */
4774 :
4775 : static void
4776 64 : handle_simd_reference (location_t loc, tree new_vard, gimple_seq *ilist)
4777 : {
4778 64 : tree z = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard)));
4779 64 : if (TREE_CONSTANT (z))
4780 : {
4781 64 : z = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard)),
4782 : get_name (new_vard));
4783 64 : gimple_add_tmp_var (z);
4784 64 : TREE_ADDRESSABLE (z) = 1;
4785 64 : z = build_fold_addr_expr_loc (loc, z);
4786 64 : gimplify_assign (new_vard, z, ilist);
4787 : }
4788 64 : }
4789 :
4790 : /* Helper function for lower_rec_input_clauses. Emit into ilist sequence
4791 : code to emit (type) (tskred_temp[idx]). */
4792 :
4793 : static tree
4794 1120 : task_reduction_read (gimple_seq *ilist, tree tskred_temp, tree type,
4795 : unsigned idx)
4796 : {
4797 1120 : unsigned HOST_WIDE_INT sz
4798 1120 : = tree_to_uhwi (TYPE_SIZE_UNIT (pointer_sized_int_node));
4799 1120 : tree r = build2 (MEM_REF, pointer_sized_int_node,
4800 1120 : tskred_temp, build_int_cst (TREE_TYPE (tskred_temp),
4801 1120 : idx * sz));
4802 1120 : tree v = create_tmp_var (pointer_sized_int_node);
4803 1120 : gimple *g = gimple_build_assign (v, r);
4804 1120 : gimple_seq_add_stmt (ilist, g);
4805 1120 : if (!useless_type_conversion_p (type, pointer_sized_int_node))
4806 : {
4807 872 : v = create_tmp_var (type);
4808 872 : g = gimple_build_assign (v, NOP_EXPR, gimple_assign_lhs (g));
4809 872 : gimple_seq_add_stmt (ilist, g);
4810 : }
4811 1120 : return v;
4812 : }
4813 :
4814 : /* Lower early initialization of privatized variable NEW_VAR
4815 : if it needs an allocator (has allocate clause). */
4816 :
4817 : static bool
4818 144567 : lower_private_allocate (tree var, tree new_var, tree &allocator,
4819 : tree &allocate_ptr, gimple_seq *ilist,
4820 : omp_context *ctx, bool is_ref, tree size)
4821 : {
4822 144567 : if (allocator)
4823 : return false;
4824 144473 : gcc_assert (allocate_ptr == NULL_TREE);
4825 144473 : if (ctx->allocate_map
4826 3082 : && (DECL_P (new_var) || (TYPE_P (new_var) && size)))
4827 3079 : if (tree *allocatorp = ctx->allocate_map->get (var))
4828 1140 : allocator = *allocatorp;
4829 144473 : if (allocator == NULL_TREE)
4830 : return false;
4831 1140 : if (!is_ref && omp_privatize_by_reference (var))
4832 : {
4833 0 : allocator = NULL_TREE;
4834 0 : return false;
4835 : }
4836 :
4837 1140 : unsigned HOST_WIDE_INT ialign = 0;
4838 1140 : if (TREE_CODE (allocator) == TREE_LIST)
4839 : {
4840 236 : ialign = tree_to_uhwi (TREE_VALUE (allocator));
4841 236 : allocator = TREE_PURPOSE (allocator);
4842 : }
4843 1140 : if (TREE_CODE (allocator) != INTEGER_CST)
4844 428 : allocator = build_outer_var_ref (allocator, ctx, OMP_CLAUSE_ALLOCATE);
4845 1140 : allocator = fold_convert (pointer_sized_int_node, allocator);
4846 1140 : if (TREE_CODE (allocator) != INTEGER_CST)
4847 : {
4848 428 : tree var = create_tmp_var (TREE_TYPE (allocator));
4849 428 : gimplify_assign (var, allocator, ilist);
4850 428 : allocator = var;
4851 : }
4852 :
4853 1140 : tree ptr_type, align, sz = size;
4854 1140 : if (TYPE_P (new_var))
4855 : {
4856 225 : ptr_type = build_pointer_type (new_var);
4857 225 : ialign = MAX (ialign, TYPE_ALIGN_UNIT (new_var));
4858 : }
4859 915 : else if (is_ref)
4860 : {
4861 114 : ptr_type = build_pointer_type (TREE_TYPE (TREE_TYPE (new_var)));
4862 114 : ialign = MAX (ialign, TYPE_ALIGN_UNIT (TREE_TYPE (ptr_type)));
4863 : }
4864 : else
4865 : {
4866 801 : ptr_type = build_pointer_type (TREE_TYPE (new_var));
4867 801 : ialign = MAX (ialign, DECL_ALIGN_UNIT (new_var));
4868 801 : if (sz == NULL_TREE)
4869 703 : sz = fold_convert (size_type_node, DECL_SIZE_UNIT (new_var));
4870 : }
4871 1140 : align = build_int_cst (size_type_node, ialign);
4872 1140 : if (TREE_CODE (sz) != INTEGER_CST)
4873 : {
4874 54 : tree szvar = create_tmp_var (size_type_node);
4875 54 : gimplify_assign (szvar, sz, ilist);
4876 54 : sz = szvar;
4877 : }
4878 1140 : allocate_ptr = create_tmp_var (ptr_type);
4879 1140 : tree a = builtin_decl_explicit (BUILT_IN_GOMP_ALLOC);
4880 1140 : gimple *g = gimple_build_call (a, 3, align, sz, allocator);
4881 1140 : gimple_call_set_lhs (g, allocate_ptr);
4882 1140 : gimple_seq_add_stmt (ilist, g);
4883 1140 : if (!is_ref)
4884 : {
4885 801 : tree x = build_simple_mem_ref (allocate_ptr);
4886 801 : TREE_THIS_NOTRAP (x) = 1;
4887 801 : SET_DECL_VALUE_EXPR (new_var, x);
4888 801 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
4889 : }
4890 : return true;
4891 : }
4892 :
4893 : /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
4894 : from the receiver (aka child) side and initializers for REFERENCE_TYPE
4895 : private variables. Initialization statements go in ILIST, while calls
4896 : to destructors go in DLIST. */
4897 :
4898 : static void
4899 77144 : lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist,
4900 : omp_context *ctx, struct omp_for_data *fd)
4901 : {
4902 77144 : tree c, copyin_seq, x, ptr;
4903 77144 : bool copyin_by_ref = false;
4904 77144 : bool lastprivate_firstprivate = false;
4905 77144 : bool reduction_omp_orig_ref = false;
4906 77144 : int pass;
4907 77144 : bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
4908 77144 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD);
4909 77144 : omplow_simd_context sctx = omplow_simd_context ();
4910 77144 : tree simt_lane = NULL_TREE, simtrec = NULL_TREE;
4911 77144 : tree ivar = NULL_TREE, lvar = NULL_TREE, uid = NULL_TREE;
4912 77144 : gimple_seq llist[4] = { };
4913 77144 : tree nonconst_simd_if = NULL_TREE;
4914 :
4915 77144 : copyin_seq = NULL;
4916 77144 : sctx.is_simt = is_simd && omp_find_clause (clauses, OMP_CLAUSE__SIMT_);
4917 :
4918 : /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
4919 : with data sharing clauses referencing variable sized vars. That
4920 : is unnecessarily hard to support and very unlikely to result in
4921 : vectorized code anyway. */
4922 9355 : if (is_simd)
4923 61463 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
4924 52108 : switch (OMP_CLAUSE_CODE (c))
4925 : {
4926 7579 : case OMP_CLAUSE_LINEAR:
4927 7579 : if (OMP_CLAUSE_LINEAR_ARRAY (c))
4928 216 : sctx.max_vf = 1;
4929 : /* FALLTHRU */
4930 25155 : case OMP_CLAUSE_PRIVATE:
4931 25155 : case OMP_CLAUSE_FIRSTPRIVATE:
4932 25155 : case OMP_CLAUSE_LASTPRIVATE:
4933 25155 : if (is_variable_sized (OMP_CLAUSE_DECL (c)))
4934 0 : sctx.max_vf = 1;
4935 25155 : else if (omp_privatize_by_reference (OMP_CLAUSE_DECL (c)))
4936 : {
4937 279 : tree rtype = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
4938 279 : if (!TREE_CONSTANT (TYPE_SIZE_UNIT (rtype)))
4939 96 : sctx.max_vf = 1;
4940 : }
4941 : break;
4942 2578 : case OMP_CLAUSE_REDUCTION:
4943 2578 : case OMP_CLAUSE_IN_REDUCTION:
4944 2578 : if (TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF
4945 2578 : || is_variable_sized (OMP_CLAUSE_DECL (c)))
4946 180 : sctx.max_vf = 1;
4947 2398 : else if (omp_privatize_by_reference (OMP_CLAUSE_DECL (c)))
4948 : {
4949 144 : tree rtype = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
4950 144 : if (!TREE_CONSTANT (TYPE_SIZE_UNIT (rtype)))
4951 0 : sctx.max_vf = 1;
4952 : }
4953 : break;
4954 759 : case OMP_CLAUSE_IF:
4955 759 : if (integer_zerop (OMP_CLAUSE_IF_EXPR (c)))
4956 124 : sctx.max_vf = 1;
4957 635 : else if (TREE_CODE (OMP_CLAUSE_IF_EXPR (c)) != INTEGER_CST)
4958 625 : nonconst_simd_if = OMP_CLAUSE_IF_EXPR (c);
4959 : break;
4960 626 : case OMP_CLAUSE_SIMDLEN:
4961 626 : if (integer_onep (OMP_CLAUSE_SIMDLEN_EXPR (c)))
4962 110 : sctx.max_vf = 1;
4963 : break;
4964 184 : case OMP_CLAUSE__CONDTEMP_:
4965 : /* FIXME: lastprivate(conditional:) not handled for SIMT yet. */
4966 184 : if (sctx.is_simt)
4967 0 : sctx.max_vf = 1;
4968 : break;
4969 22806 : default:
4970 22806 : continue;
4971 22806 : }
4972 :
4973 : /* Add a placeholder for simduid. */
4974 77144 : if (sctx.is_simt && maybe_ne (sctx.max_vf, 1U))
4975 0 : sctx.simt_eargs.safe_push (NULL_TREE);
4976 :
4977 : unsigned task_reduction_cnt = 0;
4978 : unsigned task_reduction_cntorig = 0;
4979 : unsigned task_reduction_cnt_full = 0;
4980 : unsigned task_reduction_cntorig_full = 0;
4981 : unsigned task_reduction_other_cnt = 0;
4982 234487 : tree tskred_atype = NULL_TREE, tskred_avar = NULL_TREE;
4983 : tree tskred_base = NULL_TREE, tskred_temp = NULL_TREE;
4984 : /* Do all the fixed sized types in the first pass, and the variable sized
4985 : types in the second pass. This makes sure that the scalar arguments to
4986 : the variable sized types are processed before we use them in the
4987 : variable sized operations. For task reductions we use 4 passes, in the
4988 : first two we ignore them, in the third one gather arguments for
4989 : GOMP_task_reduction_remap call and in the last pass actually handle
4990 : the task reductions. */
4991 391830 : for (pass = 0; pass < ((task_reduction_cnt || task_reduction_other_cnt)
4992 234487 : ? 4 : 2); ++pass)
4993 : {
4994 157347 : if (pass == 2 && task_reduction_cnt)
4995 : {
4996 871 : tskred_atype
4997 871 : = build_array_type_nelts (ptr_type_node, task_reduction_cnt
4998 871 : + task_reduction_cntorig);
4999 871 : tskred_avar = create_tmp_var_raw (tskred_atype);
5000 871 : gimple_add_tmp_var (tskred_avar);
5001 871 : TREE_ADDRESSABLE (tskred_avar) = 1;
5002 871 : task_reduction_cnt_full = task_reduction_cnt;
5003 871 : task_reduction_cntorig_full = task_reduction_cntorig;
5004 : }
5005 156476 : else if (pass == 3 && task_reduction_cnt)
5006 : {
5007 871 : x = builtin_decl_explicit (BUILT_IN_GOMP_TASK_REDUCTION_REMAP);
5008 871 : gimple *g
5009 871 : = gimple_build_call (x, 3, size_int (task_reduction_cnt),
5010 871 : size_int (task_reduction_cntorig),
5011 : build_fold_addr_expr (tskred_avar));
5012 871 : gimple_seq_add_stmt (ilist, g);
5013 : }
5014 157347 : if (pass == 3 && task_reduction_other_cnt)
5015 : {
5016 : /* For reduction clauses, build
5017 : tskred_base = (void *) tskred_temp[2]
5018 : + omp_get_thread_num () * tskred_temp[1]
5019 : or if tskred_temp[1] is known to be constant, that constant
5020 : directly. This is the start of the private reduction copy block
5021 : for the current thread. */
5022 838 : tree v = create_tmp_var (integer_type_node);
5023 838 : x = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
5024 838 : gimple *g = gimple_build_call (x, 0);
5025 838 : gimple_call_set_lhs (g, v);
5026 838 : gimple_seq_add_stmt (ilist, g);
5027 838 : c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
5028 838 : tskred_temp = OMP_CLAUSE_DECL (c);
5029 838 : if (is_taskreg_ctx (ctx))
5030 557 : tskred_temp = lookup_decl (tskred_temp, ctx);
5031 838 : tree v2 = create_tmp_var (sizetype);
5032 838 : g = gimple_build_assign (v2, NOP_EXPR, v);
5033 838 : gimple_seq_add_stmt (ilist, g);
5034 838 : if (ctx->task_reductions[0])
5035 821 : v = fold_convert (sizetype, ctx->task_reductions[0]);
5036 : else
5037 17 : v = task_reduction_read (ilist, tskred_temp, sizetype, 1);
5038 838 : tree v3 = create_tmp_var (sizetype);
5039 838 : g = gimple_build_assign (v3, MULT_EXPR, v2, v);
5040 838 : gimple_seq_add_stmt (ilist, g);
5041 838 : v = task_reduction_read (ilist, tskred_temp, ptr_type_node, 2);
5042 838 : tskred_base = create_tmp_var (ptr_type_node);
5043 838 : g = gimple_build_assign (tskred_base, POINTER_PLUS_EXPR, v, v3);
5044 838 : gimple_seq_add_stmt (ilist, g);
5045 : }
5046 157347 : task_reduction_cnt = 0;
5047 157347 : task_reduction_cntorig = 0;
5048 157347 : task_reduction_other_cnt = 0;
5049 761488 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
5050 : {
5051 604145 : enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c);
5052 604145 : tree var, new_var;
5053 604145 : bool by_ref;
5054 604145 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
5055 604145 : bool task_reduction_p = false;
5056 604145 : bool task_reduction_needs_orig_p = false;
5057 604145 : tree cond = NULL_TREE;
5058 604145 : tree allocator, allocate_ptr;
5059 :
5060 604145 : switch (c_kind)
5061 : {
5062 136012 : case OMP_CLAUSE_PRIVATE:
5063 136012 : if (OMP_CLAUSE_PRIVATE_DEBUG (c))
5064 435702 : continue;
5065 : break;
5066 60044 : case OMP_CLAUSE_SHARED:
5067 : /* Ignore shared directives in teams construct inside
5068 : of target construct. */
5069 60044 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
5070 60044 : && !is_host_teams_ctx (ctx))
5071 23496 : continue;
5072 36548 : if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL)
5073 : {
5074 12080 : gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c)
5075 : || is_global_var (OMP_CLAUSE_DECL (c)));
5076 12080 : continue;
5077 : }
5078 : case OMP_CLAUSE_FIRSTPRIVATE:
5079 : case OMP_CLAUSE_COPYIN:
5080 : break;
5081 15612 : case OMP_CLAUSE_LINEAR:
5082 15612 : if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c)
5083 15612 : && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
5084 : lastprivate_firstprivate = true;
5085 : break;
5086 37162 : case OMP_CLAUSE_REDUCTION:
5087 37162 : case OMP_CLAUSE_IN_REDUCTION:
5088 37162 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
5089 28914 : || is_task_ctx (ctx)
5090 63736 : || OMP_CLAUSE_REDUCTION_TASK (c))
5091 : {
5092 12780 : task_reduction_p = true;
5093 12780 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
5094 : {
5095 4532 : task_reduction_other_cnt++;
5096 4532 : if (pass == 2)
5097 1133 : continue;
5098 : }
5099 : else
5100 8248 : task_reduction_cnt++;
5101 11647 : if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5102 : {
5103 696 : var = OMP_CLAUSE_DECL (c);
5104 : /* If var is a global variable that isn't privatized
5105 : in outer contexts, we don't need to look up the
5106 : original address, it is always the address of the
5107 : global variable itself. */
5108 696 : if (!DECL_P (var)
5109 272 : || omp_privatize_by_reference (var)
5110 923 : || !is_global_var
5111 227 : (maybe_lookup_decl_in_outer_ctx (var, ctx)))
5112 : {
5113 594 : task_reduction_needs_orig_p = true;
5114 594 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5115 492 : task_reduction_cntorig++;
5116 : }
5117 : }
5118 : }
5119 24382 : else if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5120 357581 : reduction_omp_orig_ref = true;
5121 : break;
5122 3352 : case OMP_CLAUSE__REDUCTEMP_:
5123 3352 : if (!is_taskreg_ctx (ctx))
5124 1124 : continue;
5125 : /* FALLTHRU */
5126 111456 : case OMP_CLAUSE__LOOPTEMP_:
5127 : /* Handle _looptemp_/_reductemp_ clauses only on
5128 : parallel/task. */
5129 111456 : if (fd)
5130 69210 : continue;
5131 : break;
5132 43702 : case OMP_CLAUSE_LASTPRIVATE:
5133 43702 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
5134 : {
5135 2058 : lastprivate_firstprivate = true;
5136 2058 : if (pass != 0 || is_taskloop_ctx (ctx))
5137 1343 : continue;
5138 : }
5139 : /* Even without corresponding firstprivate, if
5140 : decl is Fortran allocatable, it needs outer var
5141 : reference. */
5142 41644 : else if (pass == 0
5143 62430 : && lang_hooks.decls.omp_private_outer_ref
5144 20786 : (OMP_CLAUSE_DECL (c)))
5145 : lastprivate_firstprivate = true;
5146 : break;
5147 280 : case OMP_CLAUSE_ALIGNED:
5148 280 : if (pass != 1)
5149 140 : continue;
5150 140 : var = OMP_CLAUSE_DECL (c);
5151 140 : if (TREE_CODE (TREE_TYPE (var)) == POINTER_TYPE
5152 140 : && !is_global_var (var))
5153 : {
5154 76 : new_var = maybe_lookup_decl (var, ctx);
5155 76 : if (new_var == NULL_TREE)
5156 6 : new_var = maybe_lookup_decl_in_outer_ctx (var, ctx);
5157 76 : x = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
5158 76 : tree alarg = omp_clause_aligned_alignment (c);
5159 76 : alarg = fold_convert_loc (clause_loc, size_type_node, alarg);
5160 76 : x = build_call_expr_loc (clause_loc, x, 2, new_var, alarg);
5161 76 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
5162 76 : x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
5163 76 : gimplify_and_add (x, ilist);
5164 : }
5165 64 : else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
5166 64 : && is_global_var (var))
5167 : {
5168 64 : tree ptype = build_pointer_type (TREE_TYPE (var)), t, t2;
5169 64 : new_var = lookup_decl (var, ctx);
5170 64 : t = maybe_lookup_decl_in_outer_ctx (var, ctx);
5171 64 : t = build_fold_addr_expr_loc (clause_loc, t);
5172 64 : t2 = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
5173 64 : tree alarg = omp_clause_aligned_alignment (c);
5174 64 : alarg = fold_convert_loc (clause_loc, size_type_node, alarg);
5175 64 : t = build_call_expr_loc (clause_loc, t2, 2, t, alarg);
5176 64 : t = fold_convert_loc (clause_loc, ptype, t);
5177 64 : x = create_tmp_var (ptype);
5178 64 : t = build2 (MODIFY_EXPR, ptype, x, t);
5179 64 : gimplify_and_add (t, ilist);
5180 64 : t = build_simple_mem_ref_loc (clause_loc, x);
5181 64 : SET_DECL_VALUE_EXPR (new_var, t);
5182 64 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5183 : }
5184 140 : continue;
5185 1514 : case OMP_CLAUSE__CONDTEMP_:
5186 1514 : if (is_parallel_ctx (ctx)
5187 1514 : || (is_simd && !OMP_CLAUSE__CONDTEMP__ITER (c)))
5188 : break;
5189 1094 : continue;
5190 135882 : default:
5191 135882 : continue;
5192 137116 : }
5193 :
5194 357581 : if (task_reduction_p != (pass >= 2))
5195 15616 : continue;
5196 :
5197 341965 : allocator = NULL_TREE;
5198 341965 : allocate_ptr = NULL_TREE;
5199 341965 : new_var = var = OMP_CLAUSE_DECL (c);
5200 341965 : if ((c_kind == OMP_CLAUSE_REDUCTION
5201 341965 : || c_kind == OMP_CLAUSE_IN_REDUCTION)
5202 29585 : && TREE_CODE (var) == MEM_REF)
5203 : {
5204 4356 : var = TREE_OPERAND (var, 0);
5205 4356 : if (TREE_CODE (var) == POINTER_PLUS_EXPR)
5206 773 : var = TREE_OPERAND (var, 0);
5207 4356 : if (TREE_CODE (var) == INDIRECT_REF
5208 4168 : || TREE_CODE (var) == ADDR_EXPR)
5209 2506 : var = TREE_OPERAND (var, 0);
5210 4356 : if (is_variable_sized (var))
5211 : {
5212 179 : gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
5213 179 : var = DECL_VALUE_EXPR (var);
5214 179 : gcc_assert (TREE_CODE (var) == INDIRECT_REF);
5215 179 : var = TREE_OPERAND (var, 0);
5216 179 : gcc_assert (DECL_P (var));
5217 : }
5218 4356 : new_var = var;
5219 : }
5220 29585 : if (c_kind == OMP_CLAUSE_IN_REDUCTION && is_omp_target (ctx->stmt))
5221 : {
5222 844 : splay_tree_key key = (splay_tree_key) &DECL_CONTEXT (var);
5223 844 : new_var = (tree) splay_tree_lookup (ctx->field_map, key)->value;
5224 : }
5225 341121 : else if (c_kind != OMP_CLAUSE_COPYIN)
5226 340101 : new_var = lookup_decl (var, ctx);
5227 :
5228 341965 : if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN)
5229 : {
5230 24812 : if (pass != 0)
5231 12405 : continue;
5232 : }
5233 : /* C/C++ array section reductions. */
5234 317153 : else if ((c_kind == OMP_CLAUSE_REDUCTION
5235 : || c_kind == OMP_CLAUSE_IN_REDUCTION)
5236 317153 : && var != OMP_CLAUSE_DECL (c))
5237 : {
5238 4356 : if (pass == 0)
5239 873 : continue;
5240 :
5241 3483 : tree bias = TREE_OPERAND (OMP_CLAUSE_DECL (c), 1);
5242 3483 : tree orig_var = TREE_OPERAND (OMP_CLAUSE_DECL (c), 0);
5243 :
5244 3483 : if (TREE_CODE (orig_var) == POINTER_PLUS_EXPR)
5245 : {
5246 723 : tree b = TREE_OPERAND (orig_var, 1);
5247 723 : if (is_omp_target (ctx->stmt))
5248 : b = NULL_TREE;
5249 : else
5250 723 : b = maybe_lookup_decl (b, ctx);
5251 723 : if (b == NULL)
5252 : {
5253 17 : b = TREE_OPERAND (orig_var, 1);
5254 17 : b = maybe_lookup_decl_in_outer_ctx (b, ctx);
5255 : }
5256 723 : if (integer_zerop (bias))
5257 : bias = b;
5258 : else
5259 : {
5260 0 : bias = fold_convert_loc (clause_loc,
5261 0 : TREE_TYPE (b), bias);
5262 0 : bias = fold_build2_loc (clause_loc, PLUS_EXPR,
5263 0 : TREE_TYPE (b), b, bias);
5264 : }
5265 723 : orig_var = TREE_OPERAND (orig_var, 0);
5266 : }
5267 3483 : if (pass == 2)
5268 : {
5269 1149 : tree out = maybe_lookup_decl_in_outer_ctx (var, ctx);
5270 1149 : if (is_global_var (out)
5271 241 : && TREE_CODE (TREE_TYPE (out)) != POINTER_TYPE
5272 1336 : && (TREE_CODE (TREE_TYPE (out)) != REFERENCE_TYPE
5273 0 : || (TREE_CODE (TREE_TYPE (TREE_TYPE (out)))
5274 : != POINTER_TYPE)))
5275 : x = var;
5276 962 : else if (is_omp_target (ctx->stmt))
5277 : x = out;
5278 : else
5279 : {
5280 866 : bool by_ref = use_pointer_for_field (var, NULL);
5281 866 : x = build_receiver_ref (var, by_ref, ctx);
5282 866 : if (TREE_CODE (TREE_TYPE (var)) == REFERENCE_TYPE
5283 866 : && (TREE_CODE (TREE_TYPE (TREE_TYPE (var)))
5284 : == POINTER_TYPE))
5285 32 : x = build_fold_addr_expr (x);
5286 : }
5287 1149 : if (TREE_CODE (orig_var) == INDIRECT_REF)
5288 40 : x = build_simple_mem_ref (x);
5289 1109 : else if (TREE_CODE (orig_var) == ADDR_EXPR)
5290 : {
5291 646 : if (var == TREE_OPERAND (orig_var, 0))
5292 583 : x = build_fold_addr_expr (x);
5293 : }
5294 1149 : bias = fold_convert (sizetype, bias);
5295 1149 : x = fold_convert (ptr_type_node, x);
5296 1149 : x = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
5297 1149 : TREE_TYPE (x), x, bias);
5298 1149 : unsigned cnt = task_reduction_cnt - 1;
5299 1149 : if (!task_reduction_needs_orig_p)
5300 1061 : cnt += (task_reduction_cntorig_full
5301 1061 : - task_reduction_cntorig);
5302 : else
5303 88 : cnt = task_reduction_cntorig - 1;
5304 1149 : tree r = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5305 1149 : size_int (cnt), NULL_TREE, NULL_TREE);
5306 1149 : gimplify_assign (r, x, ilist);
5307 1149 : continue;
5308 1149 : }
5309 :
5310 2334 : if (TREE_CODE (orig_var) == INDIRECT_REF
5311 2240 : || TREE_CODE (orig_var) == ADDR_EXPR)
5312 1362 : orig_var = TREE_OPERAND (orig_var, 0);
5313 2334 : tree d = OMP_CLAUSE_DECL (c);
5314 2334 : tree type = TREE_TYPE (d);
5315 2334 : gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
5316 2334 : tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
5317 2334 : tree sz = v;
5318 2334 : const char *name = get_name (orig_var);
5319 2334 : if (pass != 3 && !TREE_CONSTANT (v))
5320 : {
5321 93 : tree t;
5322 93 : if (is_omp_target (ctx->stmt))
5323 : t = NULL_TREE;
5324 : else
5325 93 : t = maybe_lookup_decl (v, ctx);
5326 93 : if (t)
5327 88 : v = t;
5328 : else
5329 5 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
5330 93 : gimplify_expr (&v, ilist, NULL, is_gimple_val, fb_rvalue);
5331 186 : t = fold_build2_loc (clause_loc, PLUS_EXPR,
5332 93 : TREE_TYPE (v), v,
5333 93 : build_int_cst (TREE_TYPE (v), 1));
5334 93 : sz = fold_build2_loc (clause_loc, MULT_EXPR,
5335 93 : TREE_TYPE (v), t,
5336 93 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5337 : }
5338 2334 : if (pass == 3)
5339 : {
5340 1461 : tree xv = create_tmp_var (ptr_type_node);
5341 1461 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5342 : {
5343 1149 : unsigned cnt = task_reduction_cnt - 1;
5344 1149 : if (!task_reduction_needs_orig_p)
5345 1061 : cnt += (task_reduction_cntorig_full
5346 1061 : - task_reduction_cntorig);
5347 : else
5348 88 : cnt = task_reduction_cntorig - 1;
5349 1149 : x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5350 1149 : size_int (cnt), NULL_TREE, NULL_TREE);
5351 :
5352 1149 : gimple *g = gimple_build_assign (xv, x);
5353 1149 : gimple_seq_add_stmt (ilist, g);
5354 : }
5355 : else
5356 : {
5357 312 : unsigned int idx = *ctx->task_reduction_map->get (c);
5358 312 : tree off;
5359 312 : if (ctx->task_reductions[1 + idx])
5360 81 : off = fold_convert (sizetype,
5361 : ctx->task_reductions[1 + idx]);
5362 : else
5363 231 : off = task_reduction_read (ilist, tskred_temp, sizetype,
5364 231 : 7 + 3 * idx + 1);
5365 312 : gimple *g = gimple_build_assign (xv, POINTER_PLUS_EXPR,
5366 : tskred_base, off);
5367 312 : gimple_seq_add_stmt (ilist, g);
5368 : }
5369 1461 : x = fold_convert (build_pointer_type (boolean_type_node),
5370 : xv);
5371 1461 : if (TREE_CONSTANT (v))
5372 1065 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (x), x,
5373 : TYPE_SIZE_UNIT (type));
5374 : else
5375 : {
5376 396 : tree t;
5377 396 : if (is_omp_target (ctx->stmt))
5378 : t = NULL_TREE;
5379 : else
5380 336 : t = maybe_lookup_decl (v, ctx);
5381 336 : if (t)
5382 316 : v = t;
5383 : else
5384 80 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
5385 396 : gimplify_expr (&v, ilist, NULL, is_gimple_val,
5386 : fb_rvalue);
5387 792 : t = fold_build2_loc (clause_loc, PLUS_EXPR,
5388 396 : TREE_TYPE (v), v,
5389 396 : build_int_cst (TREE_TYPE (v), 1));
5390 396 : t = fold_build2_loc (clause_loc, MULT_EXPR,
5391 396 : TREE_TYPE (v), t,
5392 396 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5393 396 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (x), x, t);
5394 : }
5395 1461 : cond = create_tmp_var (TREE_TYPE (x));
5396 1461 : gimplify_assign (cond, x, ilist);
5397 1461 : x = xv;
5398 : }
5399 873 : else if (lower_private_allocate (var, type, allocator,
5400 : allocate_ptr, ilist, ctx,
5401 : true,
5402 873 : TREE_CONSTANT (v)
5403 780 : ? TYPE_SIZE_UNIT (type)
5404 : : sz))
5405 225 : x = allocate_ptr;
5406 648 : else if (TREE_CONSTANT (v))
5407 : {
5408 560 : x = create_tmp_var_raw (type, name);
5409 560 : gimple_add_tmp_var (x);
5410 560 : TREE_ADDRESSABLE (x) = 1;
5411 560 : x = build_fold_addr_expr_loc (clause_loc, x);
5412 : }
5413 : else
5414 : {
5415 88 : tree atmp
5416 88 : = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
5417 88 : tree al = size_int (TYPE_ALIGN (TREE_TYPE (type)));
5418 88 : x = build_call_expr_loc (clause_loc, atmp, 2, sz, al);
5419 : }
5420 :
5421 2334 : tree ptype = build_pointer_type (TREE_TYPE (type));
5422 2334 : x = fold_convert_loc (clause_loc, ptype, x);
5423 2334 : tree y = create_tmp_var (ptype, name);
5424 2334 : gimplify_assign (y, x, ilist);
5425 2334 : x = y;
5426 2334 : tree yb = y;
5427 :
5428 2334 : if (!integer_zerop (bias))
5429 : {
5430 1512 : bias = fold_convert_loc (clause_loc, pointer_sized_int_node,
5431 : bias);
5432 1512 : yb = fold_convert_loc (clause_loc, pointer_sized_int_node,
5433 : x);
5434 1512 : yb = fold_build2_loc (clause_loc, MINUS_EXPR,
5435 : pointer_sized_int_node, yb, bias);
5436 1512 : x = fold_convert_loc (clause_loc, TREE_TYPE (x), yb);
5437 1512 : yb = create_tmp_var (ptype, name);
5438 1512 : gimplify_assign (yb, x, ilist);
5439 1512 : x = yb;
5440 : }
5441 :
5442 2334 : d = TREE_OPERAND (d, 0);
5443 2334 : if (TREE_CODE (d) == POINTER_PLUS_EXPR)
5444 420 : d = TREE_OPERAND (d, 0);
5445 2334 : if (TREE_CODE (d) == ADDR_EXPR)
5446 : {
5447 1268 : if (orig_var != var)
5448 : {
5449 91 : gcc_assert (is_variable_sized (orig_var));
5450 91 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var),
5451 : x);
5452 91 : gimplify_assign (new_var, x, ilist);
5453 91 : tree new_orig_var = lookup_decl (orig_var, ctx);
5454 91 : tree t = build_fold_indirect_ref (new_var);
5455 91 : DECL_IGNORED_P (new_var) = 0;
5456 91 : TREE_THIS_NOTRAP (t) = 1;
5457 91 : SET_DECL_VALUE_EXPR (new_orig_var, t);
5458 91 : DECL_HAS_VALUE_EXPR_P (new_orig_var) = 1;
5459 : }
5460 : else
5461 : {
5462 1177 : x = build2 (MEM_REF, TREE_TYPE (new_var), x,
5463 : build_int_cst (ptype, 0));
5464 1177 : SET_DECL_VALUE_EXPR (new_var, x);
5465 1177 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5466 : }
5467 : }
5468 : else
5469 : {
5470 1066 : gcc_assert (orig_var == var);
5471 1066 : if (TREE_CODE (d) == INDIRECT_REF)
5472 : {
5473 94 : x = create_tmp_var (ptype, name);
5474 94 : TREE_ADDRESSABLE (x) = 1;
5475 94 : gimplify_assign (x, yb, ilist);
5476 94 : x = build_fold_addr_expr_loc (clause_loc, x);
5477 : }
5478 1066 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
5479 1066 : gimplify_assign (new_var, x, ilist);
5480 : }
5481 : /* GOMP_taskgroup_reduction_register memsets the whole
5482 : array to zero. If the initializer is zero, we don't
5483 : need to initialize it again, just mark it as ever
5484 : used unconditionally, i.e. cond = true. */
5485 2334 : if (cond
5486 1461 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) == NULL_TREE
5487 3267 : && initializer_zerop (omp_reduction_init (c,
5488 933 : TREE_TYPE (type))))
5489 : {
5490 624 : gimple *g = gimple_build_assign (build_simple_mem_ref (cond),
5491 : boolean_true_node);
5492 624 : gimple_seq_add_stmt (ilist, g);
5493 624 : continue;
5494 624 : }
5495 1710 : tree end = create_artificial_label (UNKNOWN_LOCATION);
5496 1710 : if (cond)
5497 : {
5498 837 : gimple *g;
5499 837 : if (!is_parallel_ctx (ctx))
5500 : {
5501 786 : tree condv = create_tmp_var (boolean_type_node);
5502 786 : g = gimple_build_assign (condv,
5503 : build_simple_mem_ref (cond));
5504 786 : gimple_seq_add_stmt (ilist, g);
5505 786 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
5506 786 : g = gimple_build_cond (NE_EXPR, condv,
5507 : boolean_false_node, end, lab1);
5508 786 : gimple_seq_add_stmt (ilist, g);
5509 786 : gimple_seq_add_stmt (ilist, gimple_build_label (lab1));
5510 : }
5511 837 : g = gimple_build_assign (build_simple_mem_ref (cond),
5512 : boolean_true_node);
5513 837 : gimple_seq_add_stmt (ilist, g);
5514 : }
5515 :
5516 1710 : tree y1 = create_tmp_var (ptype);
5517 1710 : gimplify_assign (y1, y, ilist);
5518 1710 : tree i2 = NULL_TREE, y2 = NULL_TREE;
5519 1710 : tree body2 = NULL_TREE, end2 = NULL_TREE;
5520 1710 : tree y3 = NULL_TREE, y4 = NULL_TREE;
5521 1710 : if (task_reduction_needs_orig_p)
5522 : {
5523 112 : y3 = create_tmp_var (ptype);
5524 112 : tree ref;
5525 112 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5526 88 : ref = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5527 88 : size_int (task_reduction_cnt_full
5528 : + task_reduction_cntorig - 1),
5529 : NULL_TREE, NULL_TREE);
5530 : else
5531 : {
5532 24 : unsigned int idx = *ctx->task_reduction_map->get (c);
5533 24 : ref = task_reduction_read (ilist, tskred_temp, ptype,
5534 24 : 7 + 3 * idx);
5535 : }
5536 112 : gimplify_assign (y3, ref, ilist);
5537 : }
5538 1598 : else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) || is_simd)
5539 : {
5540 668 : if (pass != 3)
5541 : {
5542 252 : y2 = create_tmp_var (ptype);
5543 252 : gimplify_assign (y2, y, ilist);
5544 : }
5545 668 : if (is_simd || OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5546 : {
5547 188 : tree ref = build_outer_var_ref (var, ctx);
5548 : /* For ref build_outer_var_ref already performs this. */
5549 188 : if (TREE_CODE (d) == INDIRECT_REF)
5550 0 : gcc_assert (omp_privatize_by_reference (var));
5551 188 : else if (TREE_CODE (d) == ADDR_EXPR)
5552 96 : ref = build_fold_addr_expr (ref);
5553 92 : else if (omp_privatize_by_reference (var))
5554 8 : ref = build_fold_addr_expr (ref);
5555 188 : ref = fold_convert_loc (clause_loc, ptype, ref);
5556 188 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)
5557 188 : && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5558 : {
5559 8 : y3 = create_tmp_var (ptype);
5560 8 : gimplify_assign (y3, unshare_expr (ref), ilist);
5561 : }
5562 188 : if (is_simd)
5563 : {
5564 180 : y4 = create_tmp_var (ptype);
5565 180 : gimplify_assign (y4, ref, dlist);
5566 : }
5567 : }
5568 : }
5569 1710 : tree i = create_tmp_var (TREE_TYPE (v));
5570 1710 : gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), ilist);
5571 1710 : tree body = create_artificial_label (UNKNOWN_LOCATION);
5572 1710 : gimple_seq_add_stmt (ilist, gimple_build_label (body));
5573 1710 : if (y2)
5574 : {
5575 252 : i2 = create_tmp_var (TREE_TYPE (v));
5576 252 : gimplify_assign (i2, build_int_cst (TREE_TYPE (v), 0), dlist);
5577 252 : body2 = create_artificial_label (UNKNOWN_LOCATION);
5578 252 : end2 = create_artificial_label (UNKNOWN_LOCATION);
5579 252 : gimple_seq_add_stmt (dlist, gimple_build_label (body2));
5580 : }
5581 1710 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
5582 : {
5583 600 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
5584 600 : tree decl_placeholder
5585 600 : = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
5586 600 : SET_DECL_VALUE_EXPR (decl_placeholder,
5587 : build_simple_mem_ref (y1));
5588 600 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
5589 600 : SET_DECL_VALUE_EXPR (placeholder,
5590 : y3 ? build_simple_mem_ref (y3)
5591 : : error_mark_node);
5592 600 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
5593 600 : x = lang_hooks.decls.omp_clause_default_ctor
5594 720 : (c, build_simple_mem_ref (y1),
5595 120 : y3 ? build_simple_mem_ref (y3) : NULL_TREE);
5596 600 : if (x)
5597 152 : gimplify_and_add (x, ilist);
5598 600 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
5599 : {
5600 576 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
5601 576 : lower_omp (&tseq, ctx);
5602 576 : gimple_seq_add_seq (ilist, tseq);
5603 : }
5604 600 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
5605 600 : if (is_simd)
5606 : {
5607 0 : SET_DECL_VALUE_EXPR (decl_placeholder,
5608 : build_simple_mem_ref (y2));
5609 0 : SET_DECL_VALUE_EXPR (placeholder,
5610 : build_simple_mem_ref (y4));
5611 0 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
5612 0 : lower_omp (&tseq, ctx);
5613 0 : gimple_seq_add_seq (dlist, tseq);
5614 0 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
5615 : }
5616 600 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
5617 600 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 0;
5618 600 : if (y2)
5619 : {
5620 72 : x = lang_hooks.decls.omp_clause_dtor
5621 72 : (c, build_simple_mem_ref (y2));
5622 72 : if (x)
5623 40 : gimplify_and_add (x, dlist);
5624 : }
5625 : }
5626 : else
5627 : {
5628 1110 : x = omp_reduction_init (c, TREE_TYPE (type));
5629 1110 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
5630 :
5631 : /* reduction(-:var) sums up the partial results, so it
5632 : acts identically to reduction(+:var). */
5633 1110 : if (code == MINUS_EXPR)
5634 0 : code = PLUS_EXPR;
5635 :
5636 1110 : gimplify_assign (build_simple_mem_ref (y1), x, ilist);
5637 1110 : if (is_simd)
5638 : {
5639 180 : x = build2 (code, TREE_TYPE (type),
5640 : build_simple_mem_ref (y4),
5641 : build_simple_mem_ref (y2));
5642 180 : gimplify_assign (build_simple_mem_ref (y4), x, dlist);
5643 : }
5644 : }
5645 1710 : gimple *g
5646 1710 : = gimple_build_assign (y1, POINTER_PLUS_EXPR, y1,
5647 1710 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5648 1710 : gimple_seq_add_stmt (ilist, g);
5649 1710 : if (y3)
5650 : {
5651 120 : g = gimple_build_assign (y3, POINTER_PLUS_EXPR, y3,
5652 120 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5653 120 : gimple_seq_add_stmt (ilist, g);
5654 : }
5655 1710 : g = gimple_build_assign (i, PLUS_EXPR, i,
5656 1710 : build_int_cst (TREE_TYPE (i), 1));
5657 1710 : gimple_seq_add_stmt (ilist, g);
5658 1710 : g = gimple_build_cond (LE_EXPR, i, v, body, end);
5659 1710 : gimple_seq_add_stmt (ilist, g);
5660 1710 : gimple_seq_add_stmt (ilist, gimple_build_label (end));
5661 1710 : if (y2)
5662 : {
5663 252 : g = gimple_build_assign (y2, POINTER_PLUS_EXPR, y2,
5664 252 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5665 252 : gimple_seq_add_stmt (dlist, g);
5666 252 : if (y4)
5667 : {
5668 180 : g = gimple_build_assign
5669 180 : (y4, POINTER_PLUS_EXPR, y4,
5670 180 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5671 180 : gimple_seq_add_stmt (dlist, g);
5672 : }
5673 252 : g = gimple_build_assign (i2, PLUS_EXPR, i2,
5674 252 : build_int_cst (TREE_TYPE (i2), 1));
5675 252 : gimple_seq_add_stmt (dlist, g);
5676 252 : g = gimple_build_cond (LE_EXPR, i2, v, body2, end2);
5677 252 : gimple_seq_add_stmt (dlist, g);
5678 252 : gimple_seq_add_stmt (dlist, gimple_build_label (end2));
5679 : }
5680 1710 : if (allocator)
5681 : {
5682 225 : tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
5683 225 : g = gimple_build_call (f, 2, allocate_ptr, allocator);
5684 225 : gimple_seq_add_stmt (dlist, g);
5685 : }
5686 1710 : continue;
5687 1710 : }
5688 312797 : else if (pass == 2)
5689 : {
5690 913 : tree out = maybe_lookup_decl_in_outer_ctx (var, ctx);
5691 913 : if (is_global_var (out))
5692 : x = var;
5693 361 : else if (is_omp_target (ctx->stmt))
5694 : x = out;
5695 : else
5696 : {
5697 301 : bool by_ref = use_pointer_for_field (var, ctx);
5698 301 : x = build_receiver_ref (var, by_ref, ctx);
5699 : }
5700 913 : if (!omp_privatize_by_reference (var))
5701 823 : x = build_fold_addr_expr (x);
5702 913 : x = fold_convert (ptr_type_node, x);
5703 913 : unsigned cnt = task_reduction_cnt - 1;
5704 913 : if (!task_reduction_needs_orig_p)
5705 878 : cnt += task_reduction_cntorig_full - task_reduction_cntorig;
5706 : else
5707 35 : cnt = task_reduction_cntorig - 1;
5708 913 : tree r = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5709 913 : size_int (cnt), NULL_TREE, NULL_TREE);
5710 913 : gimplify_assign (r, x, ilist);
5711 913 : continue;
5712 913 : }
5713 311884 : else if (pass == 3)
5714 : {
5715 1734 : tree type = TREE_TYPE (new_var);
5716 1734 : if (!omp_privatize_by_reference (var))
5717 1628 : type = build_pointer_type (type);
5718 1734 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5719 : {
5720 913 : unsigned cnt = task_reduction_cnt - 1;
5721 913 : if (!task_reduction_needs_orig_p)
5722 878 : cnt += (task_reduction_cntorig_full
5723 878 : - task_reduction_cntorig);
5724 : else
5725 35 : cnt = task_reduction_cntorig - 1;
5726 913 : x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5727 913 : size_int (cnt), NULL_TREE, NULL_TREE);
5728 : }
5729 : else
5730 : {
5731 821 : unsigned int idx = *ctx->task_reduction_map->get (c);
5732 821 : tree off;
5733 821 : if (ctx->task_reductions[1 + idx])
5734 821 : off = fold_convert (sizetype,
5735 : ctx->task_reductions[1 + idx]);
5736 : else
5737 0 : off = task_reduction_read (ilist, tskred_temp, sizetype,
5738 0 : 7 + 3 * idx + 1);
5739 821 : x = fold_build2 (POINTER_PLUS_EXPR, ptr_type_node,
5740 : tskred_base, off);
5741 : }
5742 1734 : x = fold_convert (type, x);
5743 1734 : tree t;
5744 1734 : if (omp_privatize_by_reference (var))
5745 : {
5746 106 : gimplify_assign (new_var, x, ilist);
5747 106 : t = new_var;
5748 106 : new_var = build_simple_mem_ref (new_var);
5749 : }
5750 : else
5751 : {
5752 1628 : t = create_tmp_var (type);
5753 1628 : gimplify_assign (t, x, ilist);
5754 1628 : SET_DECL_VALUE_EXPR (new_var, build_simple_mem_ref (t));
5755 1628 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5756 : }
5757 1734 : t = fold_convert (build_pointer_type (boolean_type_node), t);
5758 1734 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t,
5759 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5760 1734 : cond = create_tmp_var (TREE_TYPE (t));
5761 1734 : gimplify_assign (cond, t, ilist);
5762 : }
5763 310150 : else if (is_variable_sized (var))
5764 : {
5765 : /* For variable sized types, we need to allocate the
5766 : actual storage here. Call alloca and store the
5767 : result in the pointer decl that we created elsewhere. */
5768 262 : if (pass == 0)
5769 134 : continue;
5770 :
5771 128 : if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx))
5772 : {
5773 114 : tree tmp;
5774 :
5775 114 : ptr = DECL_VALUE_EXPR (new_var);
5776 114 : gcc_assert (TREE_CODE (ptr) == INDIRECT_REF);
5777 114 : ptr = TREE_OPERAND (ptr, 0);
5778 114 : gcc_assert (DECL_P (ptr));
5779 114 : x = TYPE_SIZE_UNIT (TREE_TYPE (new_var));
5780 :
5781 114 : if (lower_private_allocate (var, new_var, allocator,
5782 : allocate_ptr, ilist, ctx,
5783 : false, x))
5784 8 : tmp = allocate_ptr;
5785 : else
5786 : {
5787 : /* void *tmp = __builtin_alloca */
5788 106 : tree atmp
5789 106 : = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
5790 106 : gcall *stmt
5791 106 : = gimple_build_call (atmp, 2, x,
5792 106 : size_int (DECL_ALIGN (var)));
5793 106 : cfun->calls_alloca = 1;
5794 106 : tmp = create_tmp_var_raw (ptr_type_node);
5795 106 : gimple_add_tmp_var (tmp);
5796 106 : gimple_call_set_lhs (stmt, tmp);
5797 :
5798 106 : gimple_seq_add_stmt (ilist, stmt);
5799 : }
5800 :
5801 114 : x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp);
5802 114 : gimplify_assign (ptr, x, ilist);
5803 : }
5804 : }
5805 309888 : else if (omp_privatize_by_reference (var)
5806 309888 : && (c_kind != OMP_CLAUSE_FIRSTPRIVATE
5807 2016 : || !OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c)))
5808 : {
5809 : /* For references that are being privatized for Fortran,
5810 : allocate new backing storage for the new pointer
5811 : variable. This allows us to avoid changing all the
5812 : code that expects a pointer to something that expects
5813 : a direct variable. */
5814 4895 : if (pass == 0)
5815 2550 : continue;
5816 :
5817 2345 : x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
5818 2345 : if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx))
5819 : {
5820 146 : x = build_receiver_ref (var, false, ctx);
5821 146 : if (ctx->allocate_map)
5822 16 : if (tree *allocatep = ctx->allocate_map->get (var))
5823 : {
5824 16 : allocator = *allocatep;
5825 16 : if (TREE_CODE (allocator) == TREE_LIST)
5826 0 : allocator = TREE_PURPOSE (allocator);
5827 16 : if (TREE_CODE (allocator) != INTEGER_CST)
5828 8 : allocator = build_outer_var_ref (allocator, ctx);
5829 16 : allocator = fold_convert (pointer_sized_int_node,
5830 : allocator);
5831 16 : allocate_ptr = unshare_expr (x);
5832 : }
5833 146 : if (allocator == NULL_TREE)
5834 130 : x = build_fold_addr_expr_loc (clause_loc, x);
5835 : }
5836 2199 : else if (lower_private_allocate (var, new_var, allocator,
5837 : allocate_ptr,
5838 : ilist, ctx, true, x))
5839 86 : x = allocate_ptr;
5840 2113 : else if (TREE_CONSTANT (x))
5841 : {
5842 : /* For reduction in SIMD loop, defer adding the
5843 : initialization of the reference, because if we decide
5844 : to use SIMD array for it, the initialization could cause
5845 : expansion ICE. Ditto for other privatization clauses. */
5846 1404 : if (is_simd)
5847 : x = NULL_TREE;
5848 : else
5849 : {
5850 1077 : x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
5851 : get_name (var));
5852 1077 : gimple_add_tmp_var (x);
5853 1077 : TREE_ADDRESSABLE (x) = 1;
5854 1077 : x = build_fold_addr_expr_loc (clause_loc, x);
5855 : }
5856 : }
5857 : else
5858 : {
5859 709 : tree atmp
5860 709 : = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
5861 709 : tree rtype = TREE_TYPE (TREE_TYPE (new_var));
5862 709 : tree al = size_int (TYPE_ALIGN (rtype));
5863 709 : x = build_call_expr_loc (clause_loc, atmp, 2, x, al);
5864 : }
5865 :
5866 2018 : if (x)
5867 : {
5868 2018 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
5869 2018 : gimplify_assign (new_var, x, ilist);
5870 : }
5871 :
5872 2345 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
5873 : }
5874 304993 : else if ((c_kind == OMP_CLAUSE_REDUCTION
5875 : || c_kind == OMP_CLAUSE_IN_REDUCTION)
5876 304993 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
5877 : {
5878 1956 : if (pass == 0)
5879 978 : continue;
5880 : }
5881 303037 : else if (pass != 0)
5882 151264 : continue;
5883 :
5884 169365 : switch (OMP_CLAUSE_CODE (c))
5885 : {
5886 11897 : case OMP_CLAUSE_SHARED:
5887 : /* Ignore shared directives in teams construct inside
5888 : target construct. */
5889 11897 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
5890 11897 : && !is_host_teams_ctx (ctx))
5891 0 : continue;
5892 : /* Shared global vars are just accessed directly. */
5893 11897 : if (is_global_var (new_var))
5894 : break;
5895 : /* For taskloop firstprivate/lastprivate, represented
5896 : as firstprivate and shared clause on the task, new_var
5897 : is the firstprivate var. */
5898 11743 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
5899 : break;
5900 : /* Set up the DECL_VALUE_EXPR for shared variables now. This
5901 : needs to be delayed until after fixup_child_record_type so
5902 : that we get the correct type during the dereference. */
5903 11431 : by_ref = use_pointer_for_field (var, ctx);
5904 11431 : x = build_receiver_ref (var, by_ref, ctx);
5905 11431 : SET_DECL_VALUE_EXPR (new_var, x);
5906 11431 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5907 :
5908 : /* ??? If VAR is not passed by reference, and the variable
5909 : hasn't been initialized yet, then we'll get a warning for
5910 : the store into the omp_data_s structure. Ideally, we'd be
5911 : able to notice this and not store anything at all, but
5912 : we're generating code too early. Suppress the warning. */
5913 11431 : if (!by_ref)
5914 5203 : suppress_warning (var, OPT_Wuninitialized);
5915 : break;
5916 :
5917 210 : case OMP_CLAUSE__CONDTEMP_:
5918 210 : if (is_parallel_ctx (ctx))
5919 : {
5920 103 : x = build_receiver_ref (var, false, ctx);
5921 103 : SET_DECL_VALUE_EXPR (new_var, x);
5922 103 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5923 : }
5924 107 : else if (is_simd && !OMP_CLAUSE__CONDTEMP__ITER (c))
5925 : {
5926 107 : x = build_zero_cst (TREE_TYPE (var));
5927 107 : goto do_private;
5928 : }
5929 : break;
5930 :
5931 21290 : case OMP_CLAUSE_LASTPRIVATE:
5932 21290 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
5933 : break;
5934 : /* FALLTHRU */
5935 :
5936 87388 : case OMP_CLAUSE_PRIVATE:
5937 87388 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_PRIVATE)
5938 20786 : x = build_outer_var_ref (var, ctx);
5939 66602 : else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
5940 : {
5941 163 : if (is_task_ctx (ctx))
5942 12 : x = build_receiver_ref (var, false, ctx);
5943 : else
5944 151 : x = build_outer_var_ref (var, ctx, OMP_CLAUSE_PRIVATE);
5945 : }
5946 : else
5947 : x = NULL;
5948 94032 : do_private:
5949 94032 : tree nx;
5950 94032 : bool copy_ctor;
5951 94032 : copy_ctor = false;
5952 94032 : lower_private_allocate (var, new_var, allocator, allocate_ptr,
5953 : ilist, ctx, false, NULL_TREE);
5954 94032 : nx = unshare_expr (new_var);
5955 94032 : if (is_simd
5956 24199 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
5957 101418 : && OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c))
5958 22 : copy_ctor = true;
5959 94032 : if (copy_ctor)
5960 22 : nx = lang_hooks.decls.omp_clause_copy_ctor (c, nx, x);
5961 : else
5962 94010 : nx = lang_hooks.decls.omp_clause_default_ctor (c, nx, x);
5963 94032 : if (is_simd)
5964 : {
5965 24199 : tree y = lang_hooks.decls.omp_clause_dtor (c, new_var);
5966 23517 : if ((TREE_ADDRESSABLE (new_var) || nx || y
5967 23501 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
5968 7010 : && (gimple_omp_for_collapse (ctx->stmt) != 1
5969 1375 : || (gimple_omp_for_index (ctx->stmt, 0)
5970 : != new_var)))
5971 16793 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE__CONDTEMP_
5972 16686 : || omp_privatize_by_reference (var))
5973 31064 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
5974 : ivar, lvar))
5975 : {
5976 5996 : if (omp_privatize_by_reference (var))
5977 : {
5978 52 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
5979 52 : tree new_vard = TREE_OPERAND (new_var, 0);
5980 52 : gcc_assert (DECL_P (new_vard));
5981 52 : SET_DECL_VALUE_EXPR (new_vard,
5982 : build_fold_addr_expr (lvar));
5983 52 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
5984 : }
5985 :
5986 5996 : if (nx)
5987 : {
5988 30 : tree iv = unshare_expr (ivar);
5989 30 : if (copy_ctor)
5990 22 : x = lang_hooks.decls.omp_clause_copy_ctor (c, iv,
5991 : x);
5992 : else
5993 8 : x = lang_hooks.decls.omp_clause_default_ctor (c,
5994 : iv,
5995 : x);
5996 : }
5997 5966 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__CONDTEMP_)
5998 : {
5999 37 : x = build2 (MODIFY_EXPR, TREE_TYPE (ivar),
6000 : unshare_expr (ivar), x);
6001 37 : nx = x;
6002 : }
6003 5996 : if (nx && x)
6004 67 : gimplify_and_add (x, &llist[0]);
6005 5996 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6006 5996 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
6007 : {
6008 37 : tree v = new_var;
6009 37 : if (!DECL_P (v))
6010 : {
6011 4 : gcc_assert (TREE_CODE (v) == MEM_REF);
6012 4 : v = TREE_OPERAND (v, 0);
6013 4 : gcc_assert (DECL_P (v));
6014 : }
6015 37 : v = *ctx->lastprivate_conditional_map->get (v);
6016 37 : tree t = create_tmp_var (TREE_TYPE (v));
6017 37 : tree z = build_zero_cst (TREE_TYPE (v));
6018 37 : tree orig_v
6019 37 : = build_outer_var_ref (var, ctx,
6020 : OMP_CLAUSE_LASTPRIVATE);
6021 37 : gimple_seq_add_stmt (dlist,
6022 37 : gimple_build_assign (t, z));
6023 37 : gcc_assert (DECL_HAS_VALUE_EXPR_P (v));
6024 37 : tree civar = DECL_VALUE_EXPR (v);
6025 37 : gcc_assert (TREE_CODE (civar) == ARRAY_REF);
6026 37 : civar = unshare_expr (civar);
6027 37 : TREE_OPERAND (civar, 1) = sctx.idx;
6028 37 : x = build2 (MODIFY_EXPR, TREE_TYPE (t), t,
6029 : unshare_expr (civar));
6030 74 : x = build2 (COMPOUND_EXPR, TREE_TYPE (orig_v), x,
6031 37 : build2 (MODIFY_EXPR, TREE_TYPE (orig_v),
6032 : orig_v, unshare_expr (ivar)));
6033 37 : tree cond = build2 (LT_EXPR, boolean_type_node, t,
6034 : civar);
6035 37 : x = build3 (COND_EXPR, void_type_node, cond, x,
6036 : void_node);
6037 37 : gimple_seq tseq = NULL;
6038 37 : gimplify_and_add (x, &tseq);
6039 37 : if (ctx->outer)
6040 27 : lower_omp (&tseq, ctx->outer);
6041 37 : gimple_seq_add_seq (&llist[1], tseq);
6042 : }
6043 5996 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6044 5996 : && ctx->for_simd_scan_phase)
6045 : {
6046 9 : x = unshare_expr (ivar);
6047 9 : tree orig_v
6048 9 : = build_outer_var_ref (var, ctx,
6049 : OMP_CLAUSE_LASTPRIVATE);
6050 9 : x = lang_hooks.decls.omp_clause_assign_op (c, x,
6051 : orig_v);
6052 9 : gimplify_and_add (x, &llist[0]);
6053 : }
6054 5996 : if (y)
6055 : {
6056 30 : y = lang_hooks.decls.omp_clause_dtor (c, ivar);
6057 30 : if (y)
6058 30 : gimplify_and_add (y, &llist[1]);
6059 : }
6060 : break;
6061 : }
6062 18203 : if (omp_privatize_by_reference (var))
6063 : {
6064 28 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6065 28 : tree new_vard = TREE_OPERAND (new_var, 0);
6066 28 : gcc_assert (DECL_P (new_vard));
6067 28 : tree type = TREE_TYPE (TREE_TYPE (new_vard));
6068 28 : x = TYPE_SIZE_UNIT (type);
6069 28 : if (TREE_CONSTANT (x))
6070 : {
6071 28 : x = create_tmp_var_raw (type, get_name (var));
6072 28 : gimple_add_tmp_var (x);
6073 28 : TREE_ADDRESSABLE (x) = 1;
6074 28 : x = build_fold_addr_expr_loc (clause_loc, x);
6075 28 : x = fold_convert_loc (clause_loc,
6076 28 : TREE_TYPE (new_vard), x);
6077 28 : gimplify_assign (new_vard, x, ilist);
6078 : }
6079 : }
6080 : }
6081 88036 : if (nx)
6082 486 : gimplify_and_add (nx, ilist);
6083 88036 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6084 15148 : && is_simd
6085 89784 : && ctx->for_simd_scan_phase)
6086 : {
6087 5 : tree orig_v = build_outer_var_ref (var, ctx,
6088 : OMP_CLAUSE_LASTPRIVATE);
6089 5 : x = lang_hooks.decls.omp_clause_assign_op (c, new_var,
6090 : orig_v);
6091 5 : gimplify_and_add (x, ilist);
6092 : }
6093 : /* FALLTHRU */
6094 :
6095 118860 : do_dtor:
6096 118860 : x = lang_hooks.decls.omp_clause_dtor (c, new_var);
6097 118860 : if (x)
6098 1132 : gimplify_and_add (x, dlist);
6099 118860 : if (allocator)
6100 : {
6101 847 : if (!is_gimple_val (allocator))
6102 : {
6103 22 : tree avar = create_tmp_var (TREE_TYPE (allocator));
6104 22 : gimplify_assign (avar, allocator, dlist);
6105 22 : allocator = avar;
6106 : }
6107 847 : if (!is_gimple_val (allocate_ptr))
6108 : {
6109 50 : tree apvar = create_tmp_var (TREE_TYPE (allocate_ptr));
6110 50 : gimplify_assign (apvar, allocate_ptr, dlist);
6111 50 : allocate_ptr = apvar;
6112 : }
6113 847 : tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
6114 847 : gimple *g
6115 847 : = gimple_build_call (f, 2, allocate_ptr, allocator);
6116 847 : gimple_seq_add_stmt (dlist, g);
6117 : }
6118 : break;
6119 :
6120 7806 : case OMP_CLAUSE_LINEAR:
6121 7806 : if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c))
6122 1269 : goto do_firstprivate;
6123 6537 : if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
6124 : x = NULL;
6125 : else
6126 3598 : x = build_outer_var_ref (var, ctx);
6127 6537 : goto do_private;
6128 :
6129 28673 : case OMP_CLAUSE_FIRSTPRIVATE:
6130 28673 : if (is_task_ctx (ctx))
6131 : {
6132 4630 : if ((omp_privatize_by_reference (var)
6133 146 : && !OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c))
6134 4630 : || is_variable_sized (var))
6135 160 : goto do_dtor;
6136 4470 : else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var,
6137 : ctx))
6138 4470 : || use_pointer_for_field (var, NULL))
6139 : {
6140 421 : x = build_receiver_ref (var, false, ctx);
6141 421 : if (ctx->allocate_map)
6142 34 : if (tree *allocatep = ctx->allocate_map->get (var))
6143 : {
6144 34 : allocator = *allocatep;
6145 34 : if (TREE_CODE (allocator) == TREE_LIST)
6146 4 : allocator = TREE_PURPOSE (allocator);
6147 34 : if (TREE_CODE (allocator) != INTEGER_CST)
6148 14 : allocator = build_outer_var_ref (allocator, ctx);
6149 34 : allocator = fold_convert (pointer_sized_int_node,
6150 : allocator);
6151 34 : allocate_ptr = unshare_expr (x);
6152 34 : x = build_simple_mem_ref (x);
6153 34 : TREE_THIS_NOTRAP (x) = 1;
6154 : }
6155 421 : SET_DECL_VALUE_EXPR (new_var, x);
6156 421 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
6157 421 : goto do_dtor;
6158 : }
6159 : }
6160 28092 : if (OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c)
6161 28092 : && omp_privatize_by_reference (var))
6162 : {
6163 0 : x = build_outer_var_ref (var, ctx);
6164 0 : gcc_assert (TREE_CODE (x) == MEM_REF
6165 : && integer_zerop (TREE_OPERAND (x, 1)));
6166 0 : x = TREE_OPERAND (x, 0);
6167 0 : x = lang_hooks.decls.omp_clause_copy_ctor
6168 0 : (c, unshare_expr (new_var), x);
6169 0 : gimplify_and_add (x, ilist);
6170 0 : goto do_dtor;
6171 : }
6172 29361 : do_firstprivate:
6173 29361 : lower_private_allocate (var, new_var, allocator, allocate_ptr,
6174 : ilist, ctx, false, NULL_TREE);
6175 29361 : x = build_outer_var_ref (var, ctx);
6176 29361 : if (is_simd)
6177 : {
6178 1063 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
6179 1063 : && gimple_omp_for_combined_into_p (ctx->stmt))
6180 : {
6181 668 : tree t = OMP_CLAUSE_LINEAR_STEP (c);
6182 668 : if (DECL_P (t))
6183 114 : t = build_outer_var_ref (t, ctx);
6184 668 : tree stept = TREE_TYPE (t);
6185 668 : tree ct = omp_find_clause (clauses,
6186 : OMP_CLAUSE__LOOPTEMP_);
6187 668 : gcc_assert (ct);
6188 668 : tree l = OMP_CLAUSE_DECL (ct);
6189 668 : tree n1 = fd->loop.n1;
6190 668 : tree step = fd->loop.step;
6191 668 : tree itype = TREE_TYPE (l);
6192 668 : if (POINTER_TYPE_P (itype))
6193 0 : itype = signed_type_for (itype);
6194 668 : l = fold_build2 (MINUS_EXPR, itype, l, n1);
6195 668 : if (TYPE_UNSIGNED (itype)
6196 668 : && fd->loop.cond_code == GT_EXPR)
6197 0 : l = fold_build2 (TRUNC_DIV_EXPR, itype,
6198 : fold_build1 (NEGATE_EXPR, itype, l),
6199 : fold_build1 (NEGATE_EXPR,
6200 : itype, step));
6201 : else
6202 668 : l = fold_build2 (TRUNC_DIV_EXPR, itype, l, step);
6203 668 : t = fold_build2 (MULT_EXPR, stept,
6204 : fold_convert (stept, l), t);
6205 :
6206 668 : if (OMP_CLAUSE_LINEAR_ARRAY (c))
6207 : {
6208 108 : if (omp_privatize_by_reference (var))
6209 : {
6210 72 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6211 72 : tree new_vard = TREE_OPERAND (new_var, 0);
6212 72 : gcc_assert (DECL_P (new_vard));
6213 72 : tree type = TREE_TYPE (TREE_TYPE (new_vard));
6214 72 : nx = TYPE_SIZE_UNIT (type);
6215 72 : if (TREE_CONSTANT (nx))
6216 : {
6217 24 : nx = create_tmp_var_raw (type,
6218 : get_name (var));
6219 24 : gimple_add_tmp_var (nx);
6220 24 : TREE_ADDRESSABLE (nx) = 1;
6221 24 : nx = build_fold_addr_expr_loc (clause_loc,
6222 : nx);
6223 24 : nx = fold_convert_loc (clause_loc,
6224 24 : TREE_TYPE (new_vard),
6225 : nx);
6226 24 : gimplify_assign (new_vard, nx, ilist);
6227 : }
6228 : }
6229 :
6230 108 : x = lang_hooks.decls.omp_clause_linear_ctor
6231 108 : (c, new_var, x, t);
6232 108 : gimplify_and_add (x, ilist);
6233 108 : goto do_dtor;
6234 : }
6235 :
6236 560 : if (POINTER_TYPE_P (TREE_TYPE (x)))
6237 11 : x = fold_build_pointer_plus (x, t);
6238 : else
6239 549 : x = fold_build2 (PLUS_EXPR, TREE_TYPE (x), x,
6240 : fold_convert (TREE_TYPE (x), t));
6241 : }
6242 :
6243 955 : if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LINEAR
6244 934 : || TREE_ADDRESSABLE (new_var)
6245 799 : || omp_privatize_by_reference (var))
6246 1217 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
6247 : ivar, lvar))
6248 : {
6249 152 : if (omp_privatize_by_reference (var))
6250 : {
6251 22 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6252 22 : tree new_vard = TREE_OPERAND (new_var, 0);
6253 22 : gcc_assert (DECL_P (new_vard));
6254 22 : SET_DECL_VALUE_EXPR (new_vard,
6255 : build_fold_addr_expr (lvar));
6256 22 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6257 : }
6258 152 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR)
6259 : {
6260 131 : tree iv = create_tmp_var (TREE_TYPE (new_var));
6261 131 : x = lang_hooks.decls.omp_clause_copy_ctor (c, iv, x);
6262 131 : gimplify_and_add (x, ilist);
6263 131 : gimple_stmt_iterator gsi
6264 131 : = gsi_start (*gimple_omp_body_ptr (ctx->stmt));
6265 131 : gassign *g
6266 131 : = gimple_build_assign (unshare_expr (lvar), iv);
6267 131 : gsi_insert_before_without_update (&gsi, g,
6268 : GSI_SAME_STMT);
6269 131 : tree t = OMP_CLAUSE_LINEAR_STEP (c);
6270 131 : enum tree_code code = PLUS_EXPR;
6271 131 : if (POINTER_TYPE_P (TREE_TYPE (new_var)))
6272 : code = POINTER_PLUS_EXPR;
6273 131 : g = gimple_build_assign (iv, code, iv, t);
6274 131 : gsi_insert_before_without_update (&gsi, g,
6275 : GSI_SAME_STMT);
6276 131 : break;
6277 : }
6278 21 : x = lang_hooks.decls.omp_clause_copy_ctor
6279 21 : (c, unshare_expr (ivar), x);
6280 21 : gimplify_and_add (x, &llist[0]);
6281 21 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6282 21 : if (x)
6283 21 : gimplify_and_add (x, &llist[1]);
6284 : break;
6285 : }
6286 803 : if (omp_privatize_by_reference (var))
6287 : {
6288 105 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6289 105 : tree new_vard = TREE_OPERAND (new_var, 0);
6290 105 : gcc_assert (DECL_P (new_vard));
6291 105 : tree type = TREE_TYPE (TREE_TYPE (new_vard));
6292 105 : nx = TYPE_SIZE_UNIT (type);
6293 105 : if (TREE_CONSTANT (nx))
6294 : {
6295 57 : nx = create_tmp_var_raw (type, get_name (var));
6296 57 : gimple_add_tmp_var (nx);
6297 57 : TREE_ADDRESSABLE (nx) = 1;
6298 57 : nx = build_fold_addr_expr_loc (clause_loc, nx);
6299 57 : nx = fold_convert_loc (clause_loc,
6300 57 : TREE_TYPE (new_vard), nx);
6301 57 : gimplify_assign (new_vard, nx, ilist);
6302 : }
6303 : }
6304 : }
6305 29101 : x = lang_hooks.decls.omp_clause_copy_ctor
6306 29101 : (c, unshare_expr (new_var), x);
6307 29097 : gimplify_and_add (x, ilist);
6308 29097 : goto do_dtor;
6309 :
6310 19352 : case OMP_CLAUSE__LOOPTEMP_:
6311 19352 : case OMP_CLAUSE__REDUCTEMP_:
6312 19352 : gcc_assert (is_taskreg_ctx (ctx));
6313 19352 : x = build_outer_var_ref (var, ctx);
6314 19352 : x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
6315 19352 : gimplify_and_add (x, ilist);
6316 19352 : break;
6317 :
6318 510 : case OMP_CLAUSE_COPYIN:
6319 510 : by_ref = use_pointer_for_field (var, NULL);
6320 510 : x = build_receiver_ref (var, by_ref, ctx);
6321 510 : x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x);
6322 510 : append_to_statement_list (x, ©in_seq);
6323 510 : copyin_by_ref |= by_ref;
6324 510 : break;
6325 :
6326 13025 : case OMP_CLAUSE_REDUCTION:
6327 13025 : case OMP_CLAUSE_IN_REDUCTION:
6328 : /* OpenACC reductions are initialized using the
6329 : GOACC_REDUCTION internal function. */
6330 13025 : if (is_gimple_omp_oacc (ctx->stmt))
6331 : break;
6332 9043 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
6333 : {
6334 1446 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
6335 1446 : gimple *tseq;
6336 1446 : tree ptype = TREE_TYPE (placeholder);
6337 1446 : if (cond)
6338 : {
6339 295 : x = error_mark_node;
6340 295 : if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)
6341 295 : && !task_reduction_needs_orig_p)
6342 28 : x = var;
6343 267 : else if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
6344 : {
6345 45 : tree pptype = build_pointer_type (ptype);
6346 45 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
6347 35 : x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
6348 35 : size_int (task_reduction_cnt_full
6349 : + task_reduction_cntorig - 1),
6350 : NULL_TREE, NULL_TREE);
6351 : else
6352 : {
6353 10 : unsigned int idx
6354 10 : = *ctx->task_reduction_map->get (c);
6355 10 : x = task_reduction_read (ilist, tskred_temp,
6356 10 : pptype, 7 + 3 * idx);
6357 : }
6358 45 : x = fold_convert (pptype, x);
6359 45 : x = build_simple_mem_ref (x);
6360 : }
6361 : }
6362 : else
6363 : {
6364 1151 : lower_private_allocate (var, new_var, allocator,
6365 : allocate_ptr, ilist, ctx, false,
6366 : NULL_TREE);
6367 1151 : x = build_outer_var_ref (var, ctx);
6368 :
6369 1151 : if (omp_privatize_by_reference (var)
6370 1151 : && !useless_type_conversion_p (ptype, TREE_TYPE (x)))
6371 49 : x = build_fold_addr_expr_loc (clause_loc, x);
6372 : }
6373 1446 : SET_DECL_VALUE_EXPR (placeholder, x);
6374 1446 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
6375 1446 : tree new_vard = new_var;
6376 1446 : if (omp_privatize_by_reference (var))
6377 : {
6378 203 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6379 203 : new_vard = TREE_OPERAND (new_var, 0);
6380 203 : gcc_assert (DECL_P (new_vard));
6381 : }
6382 1446 : tree rvar = NULL_TREE, *rvarp = NULL, rvar2 = NULL_TREE;
6383 1446 : if (is_simd
6384 311 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6385 1757 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6386 : rvarp = &rvar;
6387 1446 : if (is_simd
6388 1446 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
6389 : ivar, lvar, rvarp,
6390 : &rvar2))
6391 : {
6392 174 : if (new_vard == new_var)
6393 : {
6394 128 : gcc_assert (DECL_VALUE_EXPR (new_var) == lvar);
6395 128 : SET_DECL_VALUE_EXPR (new_var, ivar);
6396 : }
6397 : else
6398 : {
6399 46 : SET_DECL_VALUE_EXPR (new_vard,
6400 : build_fold_addr_expr (ivar));
6401 46 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6402 : }
6403 174 : x = lang_hooks.decls.omp_clause_default_ctor
6404 174 : (c, unshare_expr (ivar),
6405 : build_outer_var_ref (var, ctx));
6406 174 : if (rvarp && ctx->for_simd_scan_phase)
6407 : {
6408 16 : if (x)
6409 8 : gimplify_and_add (x, &llist[0]);
6410 16 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6411 16 : if (x)
6412 8 : gimplify_and_add (x, &llist[1]);
6413 469 : break;
6414 : }
6415 78 : else if (rvarp)
6416 : {
6417 78 : if (x)
6418 : {
6419 38 : gimplify_and_add (x, &llist[0]);
6420 :
6421 38 : tree ivar2 = unshare_expr (lvar);
6422 38 : TREE_OPERAND (ivar2, 1) = sctx.idx;
6423 38 : x = lang_hooks.decls.omp_clause_default_ctor
6424 38 : (c, ivar2, build_outer_var_ref (var, ctx));
6425 38 : gimplify_and_add (x, &llist[0]);
6426 :
6427 38 : if (rvar2)
6428 : {
6429 16 : x = lang_hooks.decls.omp_clause_default_ctor
6430 16 : (c, unshare_expr (rvar2),
6431 : build_outer_var_ref (var, ctx));
6432 16 : gimplify_and_add (x, &llist[0]);
6433 : }
6434 :
6435 : /* For types that need construction, add another
6436 : private var which will be default constructed
6437 : and optionally initialized with
6438 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT, as in the
6439 : loop we want to assign this value instead of
6440 : constructing and destructing it in each
6441 : iteration. */
6442 38 : tree nv = create_tmp_var_raw (TREE_TYPE (ivar));
6443 38 : gimple_add_tmp_var (nv);
6444 60 : ctx->cb.decl_map->put (TREE_OPERAND (rvar2
6445 : ? rvar2
6446 : : ivar, 0),
6447 : nv);
6448 38 : x = lang_hooks.decls.omp_clause_default_ctor
6449 38 : (c, nv, build_outer_var_ref (var, ctx));
6450 38 : gimplify_and_add (x, ilist);
6451 :
6452 38 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6453 : {
6454 19 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6455 19 : x = DECL_VALUE_EXPR (new_vard);
6456 19 : tree vexpr = nv;
6457 19 : if (new_vard != new_var)
6458 11 : vexpr = build_fold_addr_expr (nv);
6459 19 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
6460 19 : lower_omp (&tseq, ctx);
6461 19 : SET_DECL_VALUE_EXPR (new_vard, x);
6462 19 : gimple_seq_add_seq (ilist, tseq);
6463 19 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6464 : }
6465 :
6466 38 : x = lang_hooks.decls.omp_clause_dtor (c, nv);
6467 38 : if (x)
6468 38 : gimplify_and_add (x, dlist);
6469 : }
6470 :
6471 78 : tree ref = build_outer_var_ref (var, ctx);
6472 78 : x = unshare_expr (ivar);
6473 78 : x = lang_hooks.decls.omp_clause_assign_op (c, x,
6474 : ref);
6475 78 : gimplify_and_add (x, &llist[0]);
6476 :
6477 78 : ref = build_outer_var_ref (var, ctx);
6478 78 : x = lang_hooks.decls.omp_clause_assign_op (c, ref,
6479 : rvar);
6480 78 : gimplify_and_add (x, &llist[3]);
6481 :
6482 78 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6483 78 : if (new_vard == new_var)
6484 48 : SET_DECL_VALUE_EXPR (new_var, lvar);
6485 : else
6486 30 : SET_DECL_VALUE_EXPR (new_vard,
6487 : build_fold_addr_expr (lvar));
6488 :
6489 78 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6490 78 : if (x)
6491 38 : gimplify_and_add (x, &llist[1]);
6492 :
6493 78 : tree ivar2 = unshare_expr (lvar);
6494 78 : TREE_OPERAND (ivar2, 1) = sctx.idx;
6495 78 : x = lang_hooks.decls.omp_clause_dtor (c, ivar2);
6496 78 : if (x)
6497 38 : gimplify_and_add (x, &llist[1]);
6498 :
6499 78 : if (rvar2)
6500 : {
6501 36 : x = lang_hooks.decls.omp_clause_dtor (c, rvar2);
6502 36 : if (x)
6503 16 : gimplify_and_add (x, &llist[1]);
6504 : }
6505 : break;
6506 : }
6507 80 : if (x)
6508 34 : gimplify_and_add (x, &llist[0]);
6509 80 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6510 : {
6511 76 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6512 76 : lower_omp (&tseq, ctx);
6513 76 : gimple_seq_add_seq (&llist[0], tseq);
6514 : }
6515 80 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6516 80 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
6517 80 : lower_omp (&tseq, ctx);
6518 80 : gimple_seq_add_seq (&llist[1], tseq);
6519 80 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
6520 80 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6521 80 : if (new_vard == new_var)
6522 70 : SET_DECL_VALUE_EXPR (new_var, lvar);
6523 : else
6524 10 : SET_DECL_VALUE_EXPR (new_vard,
6525 : build_fold_addr_expr (lvar));
6526 80 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6527 80 : if (x)
6528 38 : gimplify_and_add (x, &llist[1]);
6529 : break;
6530 : }
6531 : /* If this is a reference to constant size reduction var
6532 : with placeholder, we haven't emitted the initializer
6533 : for it because it is undesirable if SIMD arrays are used.
6534 : But if they aren't used, we need to emit the deferred
6535 : initialization now. */
6536 1272 : else if (omp_privatize_by_reference (var) && is_simd)
6537 46 : handle_simd_reference (clause_loc, new_vard, ilist);
6538 :
6539 1272 : tree lab2 = NULL_TREE;
6540 1272 : if (cond)
6541 : {
6542 295 : gimple *g;
6543 295 : if (!is_parallel_ctx (ctx))
6544 : {
6545 278 : tree condv = create_tmp_var (boolean_type_node);
6546 278 : tree m = build_simple_mem_ref (cond);
6547 278 : g = gimple_build_assign (condv, m);
6548 278 : gimple_seq_add_stmt (ilist, g);
6549 278 : tree lab1
6550 278 : = create_artificial_label (UNKNOWN_LOCATION);
6551 278 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
6552 278 : g = gimple_build_cond (NE_EXPR, condv,
6553 : boolean_false_node,
6554 : lab2, lab1);
6555 278 : gimple_seq_add_stmt (ilist, g);
6556 278 : gimple_seq_add_stmt (ilist,
6557 278 : gimple_build_label (lab1));
6558 : }
6559 295 : g = gimple_build_assign (build_simple_mem_ref (cond),
6560 : boolean_true_node);
6561 295 : gimple_seq_add_stmt (ilist, g);
6562 : }
6563 295 : x = lang_hooks.decls.omp_clause_default_ctor
6564 1272 : (c, unshare_expr (new_var),
6565 : cond ? NULL_TREE
6566 977 : : build_outer_var_ref (var, ctx));
6567 1272 : if (x)
6568 286 : gimplify_and_add (x, ilist);
6569 :
6570 1272 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6571 1272 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6572 : {
6573 158 : if (ctx->for_simd_scan_phase)
6574 977 : goto do_dtor;
6575 142 : if (x || (!is_simd
6576 32 : && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)))
6577 : {
6578 70 : tree nv = create_tmp_var_raw (TREE_TYPE (new_var));
6579 70 : gimple_add_tmp_var (nv);
6580 70 : ctx->cb.decl_map->put (new_vard, nv);
6581 70 : x = lang_hooks.decls.omp_clause_default_ctor
6582 70 : (c, nv, build_outer_var_ref (var, ctx));
6583 70 : if (x)
6584 70 : gimplify_and_add (x, ilist);
6585 70 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6586 : {
6587 35 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6588 35 : tree vexpr = nv;
6589 35 : if (new_vard != new_var)
6590 19 : vexpr = build_fold_addr_expr (nv);
6591 35 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
6592 35 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6593 35 : lower_omp (&tseq, ctx);
6594 35 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
6595 35 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
6596 35 : gimple_seq_add_seq (ilist, tseq);
6597 : }
6598 70 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6599 70 : if (is_simd && ctx->scan_exclusive)
6600 : {
6601 16 : tree nv2
6602 16 : = create_tmp_var_raw (TREE_TYPE (new_var));
6603 16 : gimple_add_tmp_var (nv2);
6604 16 : ctx->cb.decl_map->put (nv, nv2);
6605 16 : x = lang_hooks.decls.omp_clause_default_ctor
6606 16 : (c, nv2, build_outer_var_ref (var, ctx));
6607 16 : gimplify_and_add (x, ilist);
6608 16 : x = lang_hooks.decls.omp_clause_dtor (c, nv2);
6609 16 : if (x)
6610 16 : gimplify_and_add (x, dlist);
6611 : }
6612 70 : x = lang_hooks.decls.omp_clause_dtor (c, nv);
6613 70 : if (x)
6614 70 : gimplify_and_add (x, dlist);
6615 : }
6616 72 : else if (is_simd
6617 40 : && ctx->scan_exclusive
6618 92 : && TREE_ADDRESSABLE (TREE_TYPE (new_var)))
6619 : {
6620 0 : tree nv2 = create_tmp_var_raw (TREE_TYPE (new_var));
6621 0 : gimple_add_tmp_var (nv2);
6622 0 : ctx->cb.decl_map->put (new_vard, nv2);
6623 0 : x = lang_hooks.decls.omp_clause_dtor (c, nv2);
6624 0 : if (x)
6625 0 : gimplify_and_add (x, dlist);
6626 : }
6627 142 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6628 142 : goto do_dtor;
6629 : }
6630 :
6631 1114 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6632 : {
6633 1066 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6634 1066 : if (c_kind == OMP_CLAUSE_IN_REDUCTION
6635 1066 : && is_omp_target (ctx->stmt))
6636 : {
6637 12 : tree d = maybe_lookup_decl_in_outer_ctx (var, ctx);
6638 12 : tree oldv = NULL_TREE;
6639 12 : gcc_assert (d);
6640 12 : if (DECL_HAS_VALUE_EXPR_P (d))
6641 6 : oldv = DECL_VALUE_EXPR (d);
6642 12 : SET_DECL_VALUE_EXPR (d, new_vard);
6643 12 : DECL_HAS_VALUE_EXPR_P (d) = 1;
6644 12 : lower_omp (&tseq, ctx);
6645 12 : if (oldv)
6646 6 : SET_DECL_VALUE_EXPR (d, oldv);
6647 : else
6648 : {
6649 6 : SET_DECL_VALUE_EXPR (d, NULL_TREE);
6650 6 : DECL_HAS_VALUE_EXPR_P (d) = 0;
6651 : }
6652 : }
6653 : else
6654 1054 : lower_omp (&tseq, ctx);
6655 1066 : gimple_seq_add_seq (ilist, tseq);
6656 : }
6657 1114 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6658 1114 : if (is_simd)
6659 : {
6660 43 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
6661 43 : lower_omp (&tseq, ctx);
6662 43 : gimple_seq_add_seq (dlist, tseq);
6663 43 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
6664 : }
6665 1114 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6666 1114 : if (cond)
6667 : {
6668 295 : if (lab2)
6669 278 : gimple_seq_add_stmt (ilist, gimple_build_label (lab2));
6670 : break;
6671 : }
6672 819 : goto do_dtor;
6673 : }
6674 : else
6675 : {
6676 7597 : x = omp_reduction_init (c, TREE_TYPE (new_var));
6677 7597 : gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE);
6678 7597 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
6679 :
6680 7597 : if (cond)
6681 : {
6682 1439 : gimple *g;
6683 1439 : tree lab2 = NULL_TREE;
6684 : /* GOMP_taskgroup_reduction_register memsets the whole
6685 : array to zero. If the initializer is zero, we don't
6686 : need to initialize it again, just mark it as ever
6687 : used unconditionally, i.e. cond = true. */
6688 1439 : if (initializer_zerop (x))
6689 : {
6690 1258 : g = gimple_build_assign (build_simple_mem_ref (cond),
6691 : boolean_true_node);
6692 1258 : gimple_seq_add_stmt (ilist, g);
6693 3223 : break;
6694 : }
6695 :
6696 : /* Otherwise, emit
6697 : if (!cond) { cond = true; new_var = x; } */
6698 181 : if (!is_parallel_ctx (ctx))
6699 : {
6700 177 : tree condv = create_tmp_var (boolean_type_node);
6701 177 : tree m = build_simple_mem_ref (cond);
6702 177 : g = gimple_build_assign (condv, m);
6703 177 : gimple_seq_add_stmt (ilist, g);
6704 177 : tree lab1
6705 177 : = create_artificial_label (UNKNOWN_LOCATION);
6706 177 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
6707 177 : g = gimple_build_cond (NE_EXPR, condv,
6708 : boolean_false_node,
6709 : lab2, lab1);
6710 177 : gimple_seq_add_stmt (ilist, g);
6711 177 : gimple_seq_add_stmt (ilist,
6712 177 : gimple_build_label (lab1));
6713 : }
6714 181 : g = gimple_build_assign (build_simple_mem_ref (cond),
6715 : boolean_true_node);
6716 181 : gimple_seq_add_stmt (ilist, g);
6717 181 : gimplify_assign (new_var, x, ilist);
6718 181 : if (lab2)
6719 177 : gimple_seq_add_stmt (ilist, gimple_build_label (lab2));
6720 : break;
6721 : }
6722 :
6723 : /* reduction(-:var) sums up the partial results, so it
6724 : acts identically to reduction(+:var). */
6725 6158 : if (code == MINUS_EXPR)
6726 33 : code = PLUS_EXPR;
6727 :
6728 6158 : bool is_truth_op
6729 6158 : = (code == TRUTH_ANDIF_EXPR || code == TRUTH_ORIF_EXPR);
6730 6158 : tree new_vard = new_var;
6731 6158 : if (is_simd && omp_privatize_by_reference (var))
6732 : {
6733 52 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6734 52 : new_vard = TREE_OPERAND (new_var, 0);
6735 52 : gcc_assert (DECL_P (new_vard));
6736 : }
6737 6158 : tree rvar = NULL_TREE, *rvarp = NULL, rvar2 = NULL_TREE;
6738 6158 : if (is_simd
6739 2087 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6740 8245 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6741 : rvarp = &rvar;
6742 6158 : if (is_simd
6743 6158 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
6744 : ivar, lvar, rvarp,
6745 : &rvar2))
6746 : {
6747 867 : if (new_vard != new_var)
6748 : {
6749 34 : SET_DECL_VALUE_EXPR (new_vard,
6750 : build_fold_addr_expr (lvar));
6751 34 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6752 : }
6753 :
6754 867 : tree ref = build_outer_var_ref (var, ctx);
6755 :
6756 867 : if (rvarp)
6757 : {
6758 193 : if (ctx->for_simd_scan_phase)
6759 : break;
6760 154 : gimplify_assign (ivar, ref, &llist[0]);
6761 154 : ref = build_outer_var_ref (var, ctx);
6762 154 : gimplify_assign (ref, rvar, &llist[3]);
6763 154 : break;
6764 : }
6765 :
6766 674 : gimplify_assign (unshare_expr (ivar), x, &llist[0]);
6767 :
6768 674 : if (sctx.is_simt)
6769 : {
6770 0 : if (!simt_lane)
6771 0 : simt_lane = create_tmp_var (unsigned_type_node);
6772 0 : x = build_call_expr_internal_loc
6773 0 : (UNKNOWN_LOCATION, IFN_GOMP_SIMT_XCHG_BFLY,
6774 0 : TREE_TYPE (ivar), 2, ivar, simt_lane);
6775 : /* Make sure x is evaluated unconditionally. */
6776 0 : tree bfly_var = create_tmp_var (TREE_TYPE (ivar));
6777 0 : gimplify_assign (bfly_var, x, &llist[2]);
6778 0 : x = build2 (code, TREE_TYPE (ivar), ivar, bfly_var);
6779 0 : gimplify_assign (ivar, x, &llist[2]);
6780 : }
6781 674 : tree ivar2 = ivar;
6782 674 : tree ref2 = ref;
6783 674 : if (is_truth_op)
6784 : {
6785 99 : tree zero = build_zero_cst (TREE_TYPE (ivar));
6786 99 : ivar2 = fold_build2_loc (clause_loc, NE_EXPR,
6787 : boolean_type_node, ivar,
6788 : zero);
6789 99 : ref2 = fold_build2_loc (clause_loc, NE_EXPR,
6790 : boolean_type_node, ref,
6791 : zero);
6792 : }
6793 674 : x = build2 (code, TREE_TYPE (ref), ref2, ivar2);
6794 674 : if (is_truth_op)
6795 99 : x = fold_convert (TREE_TYPE (ref), x);
6796 674 : ref = build_outer_var_ref (var, ctx);
6797 674 : gimplify_assign (ref, x, &llist[1]);
6798 :
6799 : }
6800 : else
6801 : {
6802 5291 : lower_private_allocate (var, new_var, allocator,
6803 : allocate_ptr, ilist, ctx,
6804 : false, NULL_TREE);
6805 5291 : if (omp_privatize_by_reference (var) && is_simd)
6806 18 : handle_simd_reference (clause_loc, new_vard, ilist);
6807 5291 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6808 5291 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6809 : break;
6810 4958 : gimplify_assign (new_var, x, ilist);
6811 4958 : if (is_simd)
6812 : {
6813 1028 : tree ref = build_outer_var_ref (var, ctx);
6814 1028 : tree new_var2 = new_var;
6815 1028 : tree ref2 = ref;
6816 1028 : if (is_truth_op)
6817 : {
6818 24 : tree zero = build_zero_cst (TREE_TYPE (new_var));
6819 24 : new_var2
6820 24 : = fold_build2_loc (clause_loc, NE_EXPR,
6821 : boolean_type_node, new_var,
6822 : zero);
6823 24 : ref2 = fold_build2_loc (clause_loc, NE_EXPR,
6824 : boolean_type_node, ref,
6825 : zero);
6826 : }
6827 1028 : x = build2 (code, TREE_TYPE (ref2), ref2, new_var2);
6828 1028 : if (is_truth_op)
6829 24 : x = fold_convert (TREE_TYPE (new_var), x);
6830 1028 : ref = build_outer_var_ref (var, ctx);
6831 1028 : gimplify_assign (ref, x, dlist);
6832 : }
6833 4958 : if (allocator)
6834 61 : goto do_dtor;
6835 : }
6836 : }
6837 5571 : break;
6838 :
6839 0 : default:
6840 0 : gcc_unreachable ();
6841 : }
6842 : }
6843 : }
6844 77140 : if (tskred_avar)
6845 : {
6846 871 : tree clobber = build_clobber (TREE_TYPE (tskred_avar));
6847 871 : gimple_seq_add_stmt (ilist, gimple_build_assign (tskred_avar, clobber));
6848 : }
6849 :
6850 77140 : if (known_eq (sctx.max_vf, 1U))
6851 : {
6852 1772 : sctx.is_simt = false;
6853 1772 : if (ctx->lastprivate_conditional_map)
6854 : {
6855 52 : if (gimple_omp_for_combined_into_p (ctx->stmt))
6856 : {
6857 : /* Signal to lower_omp_1 that it should use parent context. */
6858 43 : ctx->combined_into_simd_safelen1 = true;
6859 370 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
6860 327 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6861 327 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
6862 : {
6863 57 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
6864 57 : omp_context *outer = ctx->outer;
6865 57 : if (gimple_code (outer->stmt) == GIMPLE_OMP_SCAN)
6866 6 : outer = outer->outer;
6867 57 : tree *v = ctx->lastprivate_conditional_map->get (o);
6868 57 : tree po = lookup_decl (OMP_CLAUSE_DECL (c), outer);
6869 57 : tree *pv = outer->lastprivate_conditional_map->get (po);
6870 57 : *v = *pv;
6871 : }
6872 : }
6873 : else
6874 : {
6875 : /* When not vectorized, treat lastprivate(conditional:) like
6876 : normal lastprivate, as there will be just one simd lane
6877 : writing the privatized variable. */
6878 9 : delete ctx->lastprivate_conditional_map;
6879 9 : ctx->lastprivate_conditional_map = NULL;
6880 : }
6881 : }
6882 : }
6883 :
6884 77140 : if (nonconst_simd_if)
6885 : {
6886 625 : if (sctx.lane == NULL_TREE)
6887 : {
6888 570 : sctx.idx = create_tmp_var (unsigned_type_node);
6889 570 : sctx.lane = create_tmp_var (unsigned_type_node);
6890 : }
6891 : /* FIXME: For now. */
6892 625 : sctx.is_simt = false;
6893 : }
6894 :
6895 77140 : if (sctx.lane || sctx.is_simt)
6896 : {
6897 4010 : uid = create_tmp_var (ptr_type_node, "simduid");
6898 : /* Don't want uninit warnings on simduid, it is always uninitialized,
6899 : but we use it not for the value, but for the DECL_UID only. */
6900 4010 : suppress_warning (uid, OPT_Wuninitialized);
6901 4010 : c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SIMDUID_);
6902 4010 : OMP_CLAUSE__SIMDUID__DECL (c) = uid;
6903 4010 : OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
6904 4010 : gimple_omp_for_set_clauses (ctx->stmt, c);
6905 : }
6906 : /* Emit calls denoting privatized variables and initializing a pointer to
6907 : structure that holds private variables as fields after ompdevlow pass. */
6908 77140 : if (sctx.is_simt)
6909 : {
6910 0 : sctx.simt_eargs[0] = uid;
6911 0 : gimple *g
6912 0 : = gimple_build_call_internal_vec (IFN_GOMP_SIMT_ENTER, sctx.simt_eargs);
6913 0 : gimple_call_set_lhs (g, uid);
6914 0 : gimple_seq_add_stmt (ilist, g);
6915 0 : sctx.simt_eargs.release ();
6916 :
6917 0 : simtrec = create_tmp_var (ptr_type_node, ".omp_simt");
6918 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_ENTER_ALLOC, 1, uid);
6919 0 : gimple_call_set_lhs (g, simtrec);
6920 0 : gimple_seq_add_stmt (ilist, g);
6921 : }
6922 77140 : if (sctx.lane)
6923 : {
6924 7395 : gimple *g = gimple_build_call_internal (IFN_GOMP_SIMD_LANE,
6925 : 2 + (nonconst_simd_if != NULL),
6926 : uid, integer_zero_node,
6927 : nonconst_simd_if);
6928 4010 : gimple_call_set_lhs (g, sctx.lane);
6929 4010 : gimple_stmt_iterator gsi = gsi_start (*gimple_omp_body_ptr (ctx->stmt));
6930 4010 : gsi_insert_before_without_update (&gsi, g, GSI_SAME_STMT);
6931 4010 : g = gimple_build_assign (sctx.lane, INTEGER_CST,
6932 : build_int_cst (unsigned_type_node, 0));
6933 4010 : gimple_seq_add_stmt (ilist, g);
6934 4010 : if (sctx.lastlane)
6935 : {
6936 184 : g = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
6937 : 2, uid, sctx.lane);
6938 184 : gimple_call_set_lhs (g, sctx.lastlane);
6939 184 : gimple_seq_add_stmt (dlist, g);
6940 184 : gimple_seq_add_seq (dlist, llist[3]);
6941 : }
6942 : /* Emit reductions across SIMT lanes in log_2(simt_vf) steps. */
6943 4010 : if (llist[2])
6944 : {
6945 0 : tree simt_vf = create_tmp_var (unsigned_type_node);
6946 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_VF, 0);
6947 0 : gimple_call_set_lhs (g, simt_vf);
6948 0 : gimple_seq_add_stmt (dlist, g);
6949 :
6950 0 : tree t = build_int_cst (unsigned_type_node, 1);
6951 0 : g = gimple_build_assign (simt_lane, INTEGER_CST, t);
6952 0 : gimple_seq_add_stmt (dlist, g);
6953 :
6954 0 : t = build_int_cst (unsigned_type_node, 0);
6955 0 : g = gimple_build_assign (sctx.idx, INTEGER_CST, t);
6956 0 : gimple_seq_add_stmt (dlist, g);
6957 :
6958 0 : tree body = create_artificial_label (UNKNOWN_LOCATION);
6959 0 : tree header = create_artificial_label (UNKNOWN_LOCATION);
6960 0 : tree end = create_artificial_label (UNKNOWN_LOCATION);
6961 0 : gimple_seq_add_stmt (dlist, gimple_build_goto (header));
6962 0 : gimple_seq_add_stmt (dlist, gimple_build_label (body));
6963 :
6964 0 : gimple_seq_add_seq (dlist, llist[2]);
6965 :
6966 0 : g = gimple_build_assign (simt_lane, LSHIFT_EXPR, simt_lane, integer_one_node);
6967 0 : gimple_seq_add_stmt (dlist, g);
6968 :
6969 0 : gimple_seq_add_stmt (dlist, gimple_build_label (header));
6970 0 : g = gimple_build_cond (LT_EXPR, simt_lane, simt_vf, body, end);
6971 0 : gimple_seq_add_stmt (dlist, g);
6972 :
6973 0 : gimple_seq_add_stmt (dlist, gimple_build_label (end));
6974 : }
6975 12030 : for (int i = 0; i < 2; i++)
6976 8020 : if (llist[i])
6977 : {
6978 1716 : tree vf = create_tmp_var (unsigned_type_node);
6979 1716 : g = gimple_build_call_internal (IFN_GOMP_SIMD_VF, 1, uid);
6980 1716 : gimple_call_set_lhs (g, vf);
6981 1716 : gimple_seq *seq = i == 0 ? ilist : dlist;
6982 1716 : gimple_seq_add_stmt (seq, g);
6983 1716 : tree t = build_int_cst (unsigned_type_node, 0);
6984 1716 : g = gimple_build_assign (sctx.idx, INTEGER_CST, t);
6985 1716 : gimple_seq_add_stmt (seq, g);
6986 1716 : tree body = create_artificial_label (UNKNOWN_LOCATION);
6987 1716 : tree header = create_artificial_label (UNKNOWN_LOCATION);
6988 1716 : tree end = create_artificial_label (UNKNOWN_LOCATION);
6989 1716 : gimple_seq_add_stmt (seq, gimple_build_goto (header));
6990 1716 : gimple_seq_add_stmt (seq, gimple_build_label (body));
6991 1716 : gimple_seq_add_seq (seq, llist[i]);
6992 1716 : t = build_int_cst (unsigned_type_node, 1);
6993 1716 : g = gimple_build_assign (sctx.idx, PLUS_EXPR, sctx.idx, t);
6994 1716 : gimple_seq_add_stmt (seq, g);
6995 1716 : gimple_seq_add_stmt (seq, gimple_build_label (header));
6996 1716 : g = gimple_build_cond (LT_EXPR, sctx.idx, vf, body, end);
6997 1716 : gimple_seq_add_stmt (seq, g);
6998 1716 : gimple_seq_add_stmt (seq, gimple_build_label (end));
6999 : }
7000 : }
7001 77140 : if (sctx.is_simt)
7002 : {
7003 0 : gimple_seq_add_seq (dlist, sctx.simt_dlist);
7004 0 : gimple *g
7005 0 : = gimple_build_call_internal (IFN_GOMP_SIMT_EXIT, 1, simtrec);
7006 0 : gimple_seq_add_stmt (dlist, g);
7007 : }
7008 :
7009 : /* The copyin sequence is not to be executed by the main thread, since
7010 : that would result in self-copies. Perhaps not visible to scalars,
7011 : but it certainly is to C++ operator=. */
7012 77140 : if (copyin_seq)
7013 : {
7014 436 : x = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM),
7015 : 0);
7016 436 : x = build2 (NE_EXPR, boolean_type_node, x,
7017 436 : build_int_cst (TREE_TYPE (x), 0));
7018 436 : x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL);
7019 436 : gimplify_and_add (x, ilist);
7020 : }
7021 :
7022 : /* If any copyin variable is passed by reference, we must ensure the
7023 : master thread doesn't modify it before it is copied over in all
7024 : threads. Similarly for variables in both firstprivate and
7025 : lastprivate clauses we need to ensure the lastprivate copying
7026 : happens after firstprivate copying in all threads. And similarly
7027 : for UDRs if initializer expression refers to omp_orig. */
7028 77140 : if (copyin_by_ref || lastprivate_firstprivate
7029 75210 : || (reduction_omp_orig_ref
7030 89 : && !ctx->scan_inclusive
7031 89 : && !ctx->scan_exclusive))
7032 : {
7033 : /* Don't add any barrier for #pragma omp simd or
7034 : #pragma omp distribute. */
7035 2019 : if (!is_task_ctx (ctx)
7036 2019 : && (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
7037 1485 : || gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_FOR))
7038 882 : gimple_seq_add_stmt (ilist, omp_build_barrier (NULL_TREE));
7039 : }
7040 :
7041 : /* If max_vf is non-zero, then we can use only a vectorization factor
7042 : up to the max_vf we chose. So stick it into the safelen clause. */
7043 77140 : if (maybe_ne (sctx.max_vf, 0U))
7044 : {
7045 5212 : tree c = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
7046 : OMP_CLAUSE_SAFELEN);
7047 5212 : poly_uint64 safe_len;
7048 5212 : if (c == NULL_TREE
7049 5212 : || (poly_int_tree_p (OMP_CLAUSE_SAFELEN_EXPR (c), &safe_len)
7050 613 : && maybe_gt (safe_len, sctx.max_vf)))
7051 : {
7052 5122 : c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_SAFELEN);
7053 5122 : OMP_CLAUSE_SAFELEN_EXPR (c) = build_int_cst (integer_type_node,
7054 5122 : sctx.max_vf);
7055 5122 : OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
7056 5122 : gimple_omp_for_set_clauses (ctx->stmt, c);
7057 : }
7058 : }
7059 77140 : }
7060 :
7061 : /* Create temporary variables for lastprivate(conditional:) implementation
7062 : in context CTX with CLAUSES. */
7063 :
7064 : static void
7065 47268 : lower_lastprivate_conditional_clauses (tree *clauses, omp_context *ctx)
7066 : {
7067 47268 : tree iter_type = NULL_TREE;
7068 47268 : tree cond_ptr = NULL_TREE;
7069 47268 : tree iter_var = NULL_TREE;
7070 47268 : bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
7071 47268 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD);
7072 47268 : tree next = *clauses;
7073 227054 : for (tree c = *clauses; c; c = OMP_CLAUSE_CHAIN (c))
7074 179786 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7075 179786 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
7076 : {
7077 386 : if (is_simd)
7078 : {
7079 107 : tree cc = omp_find_clause (next, OMP_CLAUSE__CONDTEMP_);
7080 107 : gcc_assert (cc);
7081 107 : if (iter_type == NULL_TREE)
7082 : {
7083 77 : iter_type = TREE_TYPE (OMP_CLAUSE_DECL (cc));
7084 77 : iter_var = create_tmp_var_raw (iter_type);
7085 77 : DECL_CONTEXT (iter_var) = current_function_decl;
7086 77 : DECL_SEEN_IN_BIND_EXPR_P (iter_var) = 1;
7087 77 : DECL_CHAIN (iter_var) = ctx->block_vars;
7088 77 : ctx->block_vars = iter_var;
7089 77 : tree c3
7090 77 : = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__CONDTEMP_);
7091 77 : OMP_CLAUSE__CONDTEMP__ITER (c3) = 1;
7092 77 : OMP_CLAUSE_DECL (c3) = iter_var;
7093 77 : OMP_CLAUSE_CHAIN (c3) = *clauses;
7094 77 : *clauses = c3;
7095 77 : ctx->lastprivate_conditional_map = new hash_map<tree, tree>;
7096 : }
7097 107 : next = OMP_CLAUSE_CHAIN (cc);
7098 107 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
7099 107 : tree v = lookup_decl (OMP_CLAUSE_DECL (cc), ctx);
7100 107 : ctx->lastprivate_conditional_map->put (o, v);
7101 107 : continue;
7102 107 : }
7103 279 : if (iter_type == NULL)
7104 : {
7105 205 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR)
7106 : {
7107 193 : struct omp_for_data fd;
7108 193 : omp_extract_for_data (as_a <gomp_for *> (ctx->stmt), &fd,
7109 : NULL);
7110 193 : iter_type = unsigned_type_for (fd.iter_type);
7111 : }
7112 12 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
7113 12 : iter_type = unsigned_type_node;
7114 205 : tree c2 = omp_find_clause (*clauses, OMP_CLAUSE__CONDTEMP_);
7115 205 : if (c2)
7116 : {
7117 103 : cond_ptr
7118 103 : = lookup_decl_in_outer_ctx (OMP_CLAUSE_DECL (c2), ctx);
7119 103 : OMP_CLAUSE_DECL (c2) = cond_ptr;
7120 : }
7121 : else
7122 : {
7123 102 : cond_ptr = create_tmp_var_raw (build_pointer_type (iter_type));
7124 102 : DECL_CONTEXT (cond_ptr) = current_function_decl;
7125 102 : DECL_SEEN_IN_BIND_EXPR_P (cond_ptr) = 1;
7126 102 : DECL_CHAIN (cond_ptr) = ctx->block_vars;
7127 102 : ctx->block_vars = cond_ptr;
7128 102 : c2 = build_omp_clause (UNKNOWN_LOCATION,
7129 : OMP_CLAUSE__CONDTEMP_);
7130 102 : OMP_CLAUSE_DECL (c2) = cond_ptr;
7131 102 : OMP_CLAUSE_CHAIN (c2) = *clauses;
7132 102 : *clauses = c2;
7133 : }
7134 205 : iter_var = create_tmp_var_raw (iter_type);
7135 205 : DECL_CONTEXT (iter_var) = current_function_decl;
7136 205 : DECL_SEEN_IN_BIND_EXPR_P (iter_var) = 1;
7137 205 : DECL_CHAIN (iter_var) = ctx->block_vars;
7138 205 : ctx->block_vars = iter_var;
7139 205 : tree c3
7140 205 : = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__CONDTEMP_);
7141 205 : OMP_CLAUSE__CONDTEMP__ITER (c3) = 1;
7142 205 : OMP_CLAUSE_DECL (c3) = iter_var;
7143 205 : OMP_CLAUSE_CHAIN (c3) = OMP_CLAUSE_CHAIN (c2);
7144 205 : OMP_CLAUSE_CHAIN (c2) = c3;
7145 205 : ctx->lastprivate_conditional_map = new hash_map<tree, tree>;
7146 : }
7147 279 : tree v = create_tmp_var_raw (iter_type);
7148 279 : DECL_CONTEXT (v) = current_function_decl;
7149 279 : DECL_SEEN_IN_BIND_EXPR_P (v) = 1;
7150 279 : DECL_CHAIN (v) = ctx->block_vars;
7151 279 : ctx->block_vars = v;
7152 279 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
7153 279 : ctx->lastprivate_conditional_map->put (o, v);
7154 : }
7155 47268 : }
7156 :
7157 :
7158 : /* Generate code to implement the LASTPRIVATE clauses. This is used for
7159 : both parallel and workshare constructs. PREDICATE may be NULL if it's
7160 : always true. BODY_P is the sequence to insert early initialization
7161 : if needed, STMT_LIST is where the non-conditional lastprivate handling
7162 : goes into and CSTMT_LIST is a sequence that needs to be run in a critical
7163 : section. */
7164 :
7165 : static void
7166 47268 : lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *body_p,
7167 : gimple_seq *stmt_list, gimple_seq *cstmt_list,
7168 : omp_context *ctx)
7169 : {
7170 47268 : tree x, c, label = NULL, orig_clauses = clauses;
7171 47268 : bool par_clauses = false;
7172 47268 : tree simduid = NULL, lastlane = NULL, simtcond = NULL, simtlast = NULL;
7173 47268 : unsigned HOST_WIDE_INT conditional_off = 0;
7174 47268 : gimple_seq post_stmt_list = NULL;
7175 :
7176 : /* Early exit if there are no lastprivate or linear clauses. */
7177 196314 : for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
7178 161528 : if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LASTPRIVATE
7179 161528 : || (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LINEAR
7180 6763 : && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses)))
7181 : break;
7182 47268 : if (clauses == NULL)
7183 : {
7184 : /* If this was a workshare clause, see if it had been combined
7185 : with its parallel. In that case, look for the clauses on the
7186 : parallel statement itself. */
7187 34786 : if (is_parallel_ctx (ctx))
7188 30787 : return;
7189 :
7190 34786 : ctx = ctx->outer;
7191 34786 : if (ctx == NULL || !is_parallel_ctx (ctx))
7192 : return;
7193 :
7194 12258 : clauses = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
7195 : OMP_CLAUSE_LASTPRIVATE);
7196 12258 : if (clauses == NULL)
7197 : return;
7198 : par_clauses = true;
7199 : }
7200 :
7201 16481 : bool maybe_simt = false;
7202 16481 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
7203 16481 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
7204 : {
7205 7424 : maybe_simt = omp_find_clause (orig_clauses, OMP_CLAUSE__SIMT_);
7206 7424 : simduid = omp_find_clause (orig_clauses, OMP_CLAUSE__SIMDUID_);
7207 7424 : if (simduid)
7208 3558 : simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
7209 : }
7210 :
7211 16481 : if (predicate)
7212 : {
7213 16362 : gcond *stmt;
7214 16362 : tree label_true, arm1, arm2;
7215 16362 : enum tree_code pred_code = TREE_CODE (predicate);
7216 :
7217 16362 : label = create_artificial_label (UNKNOWN_LOCATION);
7218 16362 : label_true = create_artificial_label (UNKNOWN_LOCATION);
7219 16362 : if (TREE_CODE_CLASS (pred_code) == tcc_comparison)
7220 : {
7221 16362 : arm1 = TREE_OPERAND (predicate, 0);
7222 16362 : arm2 = TREE_OPERAND (predicate, 1);
7223 16362 : gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
7224 16362 : gimplify_expr (&arm2, stmt_list, NULL, is_gimple_val, fb_rvalue);
7225 : }
7226 : else
7227 : {
7228 0 : arm1 = predicate;
7229 0 : gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
7230 0 : arm2 = boolean_false_node;
7231 0 : pred_code = NE_EXPR;
7232 : }
7233 16362 : if (maybe_simt)
7234 : {
7235 0 : c = build2 (pred_code, boolean_type_node, arm1, arm2);
7236 0 : c = fold_convert (integer_type_node, c);
7237 0 : simtcond = create_tmp_var (integer_type_node);
7238 0 : gimplify_assign (simtcond, c, stmt_list);
7239 0 : gcall *g = gimple_build_call_internal (IFN_GOMP_SIMT_VOTE_ANY,
7240 : 1, simtcond);
7241 0 : c = create_tmp_var (integer_type_node);
7242 0 : gimple_call_set_lhs (g, c);
7243 0 : gimple_seq_add_stmt (stmt_list, g);
7244 0 : stmt = gimple_build_cond (NE_EXPR, c, integer_zero_node,
7245 : label_true, label);
7246 : }
7247 : else
7248 16362 : stmt = gimple_build_cond (pred_code, arm1, arm2, label_true, label);
7249 16362 : gimple_seq_add_stmt (stmt_list, stmt);
7250 16362 : gimple_seq_add_stmt (stmt_list, gimple_build_label (label_true));
7251 : }
7252 :
7253 16481 : tree cond_ptr = NULL_TREE;
7254 61575 : for (c = clauses; c ;)
7255 : {
7256 56348 : tree var, new_var;
7257 56348 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
7258 56348 : gimple_seq *this_stmt_list = stmt_list;
7259 56348 : tree lab2 = NULL_TREE;
7260 :
7261 56348 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7262 21813 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c)
7263 386 : && ctx->lastprivate_conditional_map
7264 56721 : && !ctx->combined_into_simd_safelen1)
7265 : {
7266 316 : gcc_assert (body_p);
7267 316 : if (simduid)
7268 37 : goto next;
7269 279 : if (cond_ptr == NULL_TREE)
7270 : {
7271 205 : cond_ptr = omp_find_clause (orig_clauses, OMP_CLAUSE__CONDTEMP_);
7272 205 : cond_ptr = OMP_CLAUSE_DECL (cond_ptr);
7273 : }
7274 279 : tree type = TREE_TYPE (TREE_TYPE (cond_ptr));
7275 279 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
7276 279 : tree v = *ctx->lastprivate_conditional_map->get (o);
7277 279 : gimplify_assign (v, build_zero_cst (type), body_p);
7278 279 : this_stmt_list = cstmt_list;
7279 279 : tree mem;
7280 279 : if (POINTER_TYPE_P (TREE_TYPE (cond_ptr)))
7281 : {
7282 136 : mem = build2 (MEM_REF, type, cond_ptr,
7283 136 : build_int_cst (TREE_TYPE (cond_ptr),
7284 136 : conditional_off));
7285 136 : conditional_off += tree_to_uhwi (TYPE_SIZE_UNIT (type));
7286 : }
7287 : else
7288 143 : mem = build4 (ARRAY_REF, type, cond_ptr,
7289 143 : size_int (conditional_off++), NULL_TREE, NULL_TREE);
7290 279 : tree mem2 = copy_node (mem);
7291 279 : gimple_seq seq = NULL;
7292 279 : mem = force_gimple_operand (mem, &seq, true, NULL_TREE);
7293 279 : gimple_seq_add_seq (this_stmt_list, seq);
7294 279 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
7295 279 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
7296 279 : gimple *g = gimple_build_cond (GT_EXPR, v, mem, lab1, lab2);
7297 279 : gimple_seq_add_stmt (this_stmt_list, g);
7298 279 : gimple_seq_add_stmt (this_stmt_list, gimple_build_label (lab1));
7299 279 : gimplify_assign (mem2, v, this_stmt_list);
7300 : }
7301 56032 : else if (predicate
7302 55794 : && ctx->combined_into_simd_safelen1
7303 116 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7304 57 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c)
7305 56089 : && ctx->lastprivate_conditional_map)
7306 : this_stmt_list = &post_stmt_list;
7307 :
7308 56311 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7309 56311 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
7310 4867 : && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c)))
7311 : {
7312 26643 : var = OMP_CLAUSE_DECL (c);
7313 26643 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7314 21776 : && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)
7315 27670 : && is_taskloop_ctx (ctx))
7316 : {
7317 312 : gcc_checking_assert (ctx->outer && is_task_ctx (ctx->outer));
7318 312 : new_var = lookup_decl (var, ctx->outer);
7319 : }
7320 : else
7321 : {
7322 26331 : new_var = lookup_decl (var, ctx);
7323 : /* Avoid uninitialized warnings for lastprivate and
7324 : for linear iterators. */
7325 26331 : if (predicate
7326 52473 : && (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7327 4867 : || OMP_CLAUSE_LINEAR_NO_COPYIN (c)))
7328 24873 : suppress_warning (new_var, OPT_Wuninitialized);
7329 : }
7330 :
7331 26643 : if (!maybe_simt && simduid && DECL_HAS_VALUE_EXPR_P (new_var))
7332 : {
7333 5879 : tree val = DECL_VALUE_EXPR (new_var);
7334 5879 : if (TREE_CODE (val) == ARRAY_REF
7335 5831 : && VAR_P (TREE_OPERAND (val, 0))
7336 11710 : && lookup_attribute ("omp simd array",
7337 5831 : DECL_ATTRIBUTES (TREE_OPERAND (val,
7338 : 0))))
7339 : {
7340 5831 : if (lastlane == NULL)
7341 : {
7342 2766 : lastlane = create_tmp_var (unsigned_type_node);
7343 2766 : gcall *g
7344 2766 : = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
7345 : 2, simduid,
7346 2766 : TREE_OPERAND (val, 1));
7347 2766 : gimple_call_set_lhs (g, lastlane);
7348 2766 : gimple_seq_add_stmt (this_stmt_list, g);
7349 : }
7350 5831 : new_var = build4 (ARRAY_REF, TREE_TYPE (val),
7351 5831 : TREE_OPERAND (val, 0), lastlane,
7352 : NULL_TREE, NULL_TREE);
7353 5831 : TREE_THIS_NOTRAP (new_var) = 1;
7354 : }
7355 : }
7356 20764 : else if (maybe_simt)
7357 : {
7358 0 : tree val = (DECL_HAS_VALUE_EXPR_P (new_var)
7359 0 : ? DECL_VALUE_EXPR (new_var)
7360 : : new_var);
7361 0 : if (simtlast == NULL)
7362 : {
7363 0 : simtlast = create_tmp_var (unsigned_type_node);
7364 0 : gcall *g = gimple_build_call_internal
7365 0 : (IFN_GOMP_SIMT_LAST_LANE, 1, simtcond);
7366 0 : gimple_call_set_lhs (g, simtlast);
7367 0 : gimple_seq_add_stmt (this_stmt_list, g);
7368 : }
7369 0 : x = build_call_expr_internal_loc
7370 0 : (UNKNOWN_LOCATION, IFN_GOMP_SIMT_XCHG_IDX,
7371 0 : TREE_TYPE (val), 2, val, simtlast);
7372 0 : new_var = unshare_expr (new_var);
7373 0 : gimplify_assign (new_var, x, this_stmt_list);
7374 0 : new_var = unshare_expr (new_var);
7375 : }
7376 :
7377 26643 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7378 26643 : && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
7379 : {
7380 7159 : lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
7381 7159 : gimple_seq_add_seq (this_stmt_list,
7382 7159 : OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c));
7383 7159 : OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL;
7384 : }
7385 19484 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
7386 19484 : && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
7387 : {
7388 426 : lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
7389 426 : gimple_seq_add_seq (this_stmt_list,
7390 426 : OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c));
7391 426 : OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c) = NULL;
7392 : }
7393 :
7394 26643 : x = NULL_TREE;
7395 26643 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7396 21776 : && OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c)
7397 26703 : && is_taskloop_ctx (ctx))
7398 : {
7399 76 : tree ovar = maybe_lookup_decl_in_outer_ctx (var,
7400 38 : ctx->outer->outer);
7401 38 : if (is_global_var (ovar))
7402 7 : x = ovar;
7403 : }
7404 7 : if (!x)
7405 26636 : x = build_outer_var_ref (var, ctx, OMP_CLAUSE_LASTPRIVATE);
7406 26643 : if (omp_privatize_by_reference (var))
7407 657 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
7408 26643 : x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var);
7409 26643 : gimplify_and_add (x, this_stmt_list);
7410 :
7411 26643 : if (lab2)
7412 279 : gimple_seq_add_stmt (this_stmt_list, gimple_build_label (lab2));
7413 : }
7414 :
7415 56348 : next:
7416 56348 : c = OMP_CLAUSE_CHAIN (c);
7417 56348 : if (c == NULL && !par_clauses)
7418 : {
7419 : /* If this was a workshare clause, see if it had been combined
7420 : with its parallel. In that case, continue looking for the
7421 : clauses also on the parallel statement itself. */
7422 12482 : if (is_parallel_ctx (ctx))
7423 : break;
7424 :
7425 12482 : ctx = ctx->outer;
7426 12482 : if (ctx == NULL || !is_parallel_ctx (ctx))
7427 : break;
7428 :
7429 1228 : c = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
7430 : OMP_CLAUSE_LASTPRIVATE);
7431 1228 : par_clauses = true;
7432 : }
7433 : }
7434 :
7435 16481 : if (label)
7436 16362 : gimple_seq_add_stmt (stmt_list, gimple_build_label (label));
7437 16481 : gimple_seq_add_seq (stmt_list, post_stmt_list);
7438 : }
7439 :
7440 : /* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL
7441 : (which might be a placeholder). INNER is true if this is an inner
7442 : axis of a multi-axis loop. FORK and JOIN are (optional) fork and
7443 : join markers. Generate the before-loop forking sequence in
7444 : FORK_SEQ and the after-loop joining sequence to JOIN_SEQ. The
7445 : general form of these sequences is
7446 :
7447 : GOACC_REDUCTION_SETUP
7448 : GOACC_FORK
7449 : GOACC_REDUCTION_INIT
7450 : ...
7451 : GOACC_REDUCTION_FINI
7452 : GOACC_JOIN
7453 : GOACC_REDUCTION_TEARDOWN. */
7454 :
7455 : static void
7456 25946 : lower_oacc_reductions (location_t loc, tree clauses, tree level, bool inner,
7457 : gcall *fork, gcall *private_marker, gcall *join,
7458 : gimple_seq *fork_seq, gimple_seq *join_seq,
7459 : omp_context *ctx)
7460 : {
7461 25946 : gimple_seq before_fork = NULL;
7462 25946 : gimple_seq after_fork = NULL;
7463 25946 : gimple_seq before_join = NULL;
7464 25946 : gimple_seq after_join = NULL;
7465 25946 : tree init_code = NULL_TREE, fini_code = NULL_TREE,
7466 25946 : setup_code = NULL_TREE, teardown_code = NULL_TREE;
7467 25946 : unsigned offset = 0;
7468 :
7469 96516 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
7470 70570 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
7471 : {
7472 : /* No 'reduction' clauses on OpenACC 'kernels'. */
7473 7751 : gcc_checking_assert (!is_oacc_kernels (ctx));
7474 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
7475 7751 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
7476 :
7477 7751 : tree orig = OMP_CLAUSE_DECL (c);
7478 7751 : tree orig_clause;
7479 7751 : tree var = maybe_lookup_decl (orig, ctx);
7480 7751 : tree ref_to_res = NULL_TREE;
7481 7751 : tree incoming, outgoing, v1, v2, v3;
7482 7751 : bool is_private = false;
7483 :
7484 7751 : enum tree_code rcode = OMP_CLAUSE_REDUCTION_CODE (c);
7485 7751 : if (rcode == MINUS_EXPR)
7486 : rcode = PLUS_EXPR;
7487 6609 : else if (rcode == TRUTH_ANDIF_EXPR)
7488 : rcode = BIT_AND_EXPR;
7489 6431 : else if (rcode == TRUTH_ORIF_EXPR)
7490 433 : rcode = BIT_IOR_EXPR;
7491 7751 : tree op = build_int_cst (unsigned_type_node, rcode);
7492 :
7493 7751 : if (!var)
7494 4 : var = orig;
7495 :
7496 7751 : incoming = outgoing = var;
7497 :
7498 7751 : if (!inner)
7499 : {
7500 : /* See if an outer construct also reduces this variable. */
7501 : omp_context *outer = ctx;
7502 :
7503 7385 : while (omp_context *probe = outer->outer)
7504 : {
7505 4708 : enum gimple_code type = gimple_code (probe->stmt);
7506 4708 : tree cls;
7507 :
7508 4708 : switch (type)
7509 : {
7510 2220 : case GIMPLE_OMP_FOR:
7511 2220 : cls = gimple_omp_for_clauses (probe->stmt);
7512 2220 : break;
7513 :
7514 2488 : case GIMPLE_OMP_TARGET:
7515 : /* No 'reduction' clauses inside OpenACC 'kernels'
7516 : regions. */
7517 2488 : gcc_checking_assert (!is_oacc_kernels (probe));
7518 :
7519 2488 : if (!is_gimple_omp_offloaded (probe->stmt))
7520 26 : goto do_lookup;
7521 :
7522 2462 : cls = gimple_omp_target_clauses (probe->stmt);
7523 2462 : break;
7524 :
7525 0 : default:
7526 0 : goto do_lookup;
7527 : }
7528 :
7529 14249 : outer = probe;
7530 14249 : for (; cls; cls = OMP_CLAUSE_CHAIN (cls))
7531 11632 : if (OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_REDUCTION
7532 11632 : && orig == OMP_CLAUSE_DECL (cls))
7533 : {
7534 1772 : incoming = outgoing = lookup_decl (orig, probe);
7535 1772 : goto has_outer_reduction;
7536 : }
7537 9860 : else if ((OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_FIRSTPRIVATE
7538 8971 : || OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_PRIVATE)
7539 11164 : && orig == OMP_CLAUSE_DECL (cls))
7540 : {
7541 293 : is_private = true;
7542 293 : goto do_lookup;
7543 : }
7544 : }
7545 :
7546 2677 : do_lookup:
7547 : /* This is the outermost construct with this reduction,
7548 : see if there's a mapping for it. */
7549 2996 : orig_clause = NULL_TREE;
7550 2996 : if (gimple_code (outer->stmt) == GIMPLE_OMP_TARGET)
7551 7557 : for (tree cls = gimple_omp_target_clauses (outer->stmt);
7552 7557 : cls; cls = OMP_CLAUSE_CHAIN (cls))
7553 7250 : if (OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_MAP
7554 4823 : && orig == OMP_CLAUSE_DECL (cls)
7555 9805 : && maybe_lookup_field (cls, outer))
7556 : {
7557 : orig_clause = cls;
7558 : break;
7559 : }
7560 2996 : if (orig_clause != NULL_TREE && !is_private)
7561 : {
7562 2553 : ref_to_res = build_receiver_ref (orig_clause, false, outer);
7563 2553 : if (omp_privatize_by_reference (orig))
7564 79 : ref_to_res = build_simple_mem_ref (ref_to_res);
7565 :
7566 2553 : tree type = TREE_TYPE (var);
7567 2553 : if (POINTER_TYPE_P (type))
7568 79 : type = TREE_TYPE (type);
7569 :
7570 2553 : outgoing = var;
7571 2553 : incoming = omp_reduction_init_op (loc, rcode, type);
7572 : }
7573 : else
7574 : {
7575 : /* Try to look at enclosing contexts for reduction var,
7576 : use original if no mapping found. */
7577 443 : tree t = NULL_TREE;
7578 443 : omp_context *c = ctx->outer;
7579 895 : while (c && !t)
7580 : {
7581 452 : t = maybe_lookup_decl (orig, c);
7582 452 : c = c->outer;
7583 : }
7584 443 : incoming = outgoing = (t ? t : orig);
7585 : }
7586 :
7587 2553 : has_outer_reduction:;
7588 : }
7589 :
7590 4325 : if (!ref_to_res)
7591 5198 : ref_to_res = integer_zero_node;
7592 :
7593 7751 : if (omp_privatize_by_reference (orig))
7594 : {
7595 159 : tree type = TREE_TYPE (var);
7596 159 : const char *id = IDENTIFIER_POINTER (DECL_NAME (var));
7597 :
7598 159 : if (!inner)
7599 : {
7600 110 : tree x = create_tmp_var (TREE_TYPE (type), id);
7601 110 : gimplify_assign (var, build_fold_addr_expr (x), fork_seq);
7602 : }
7603 :
7604 159 : v1 = create_tmp_var (type, id);
7605 159 : v2 = create_tmp_var (type, id);
7606 159 : v3 = create_tmp_var (type, id);
7607 :
7608 159 : gimplify_assign (v1, var, fork_seq);
7609 159 : gimplify_assign (v2, var, fork_seq);
7610 159 : gimplify_assign (v3, var, fork_seq);
7611 :
7612 159 : var = build_simple_mem_ref (var);
7613 159 : v1 = build_simple_mem_ref (v1);
7614 159 : v2 = build_simple_mem_ref (v2);
7615 159 : v3 = build_simple_mem_ref (v3);
7616 159 : outgoing = build_simple_mem_ref (outgoing);
7617 :
7618 159 : if (!TREE_CONSTANT (incoming))
7619 80 : incoming = build_simple_mem_ref (incoming);
7620 : }
7621 : else
7622 : /* Note that 'var' might be a mem ref. */
7623 : v1 = v2 = v3 = var;
7624 :
7625 : /* Determine position in reduction buffer, which may be used
7626 : by target. The parser has ensured that this is not a
7627 : variable-sized type. */
7628 7751 : fixed_size_mode mode
7629 7751 : = as_a <fixed_size_mode> (TYPE_MODE (TREE_TYPE (var)));
7630 7751 : unsigned align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT;
7631 7751 : offset = (offset + align - 1) & ~(align - 1);
7632 7751 : tree off = build_int_cst (sizetype, offset);
7633 7751 : offset += GET_MODE_SIZE (mode);
7634 :
7635 7751 : if (!init_code)
7636 : {
7637 6969 : init_code = build_int_cst (integer_type_node,
7638 : IFN_GOACC_REDUCTION_INIT);
7639 6969 : fini_code = build_int_cst (integer_type_node,
7640 : IFN_GOACC_REDUCTION_FINI);
7641 6969 : setup_code = build_int_cst (integer_type_node,
7642 : IFN_GOACC_REDUCTION_SETUP);
7643 6969 : teardown_code = build_int_cst (integer_type_node,
7644 : IFN_GOACC_REDUCTION_TEARDOWN);
7645 : }
7646 :
7647 7751 : tree setup_call
7648 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7649 7751 : TREE_TYPE (var), 6, setup_code,
7650 : unshare_expr (ref_to_res),
7651 : unshare_expr (incoming),
7652 : level, op, off);
7653 7751 : tree init_call
7654 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7655 7751 : TREE_TYPE (var), 6, init_code,
7656 : unshare_expr (ref_to_res),
7657 : unshare_expr (v1), level, op, off);
7658 7751 : tree fini_call
7659 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7660 7751 : TREE_TYPE (var), 6, fini_code,
7661 : unshare_expr (ref_to_res),
7662 : unshare_expr (v2), level, op, off);
7663 7751 : tree teardown_call
7664 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7665 7751 : TREE_TYPE (var), 6, teardown_code,
7666 : ref_to_res, unshare_expr (v3),
7667 : level, op, off);
7668 :
7669 7751 : gimplify_assign (unshare_expr (v1), setup_call, &before_fork);
7670 7751 : gimplify_assign (unshare_expr (v2), init_call, &after_fork);
7671 7751 : gimplify_assign (unshare_expr (v3), fini_call, &before_join);
7672 7751 : gimplify_assign (unshare_expr (outgoing), teardown_call, &after_join);
7673 : }
7674 :
7675 : /* Now stitch things together. */
7676 25946 : gimple_seq_add_seq (fork_seq, before_fork);
7677 25946 : if (private_marker)
7678 259 : gimple_seq_add_stmt (fork_seq, private_marker);
7679 25946 : if (fork)
7680 16551 : gimple_seq_add_stmt (fork_seq, fork);
7681 25946 : gimple_seq_add_seq (fork_seq, after_fork);
7682 :
7683 25946 : gimple_seq_add_seq (join_seq, before_join);
7684 25946 : if (join)
7685 16551 : gimple_seq_add_stmt (join_seq, join);
7686 25946 : gimple_seq_add_seq (join_seq, after_join);
7687 25946 : }
7688 :
7689 : /* Generate code to implement the REDUCTION clauses, append it
7690 : to STMT_SEQP. CLIST if non-NULL is a pointer to a sequence
7691 : that should be emitted also inside of the critical section,
7692 : in that case clear *CLIST afterwards, otherwise leave it as is
7693 : and let the caller emit it itself. */
7694 :
7695 : static void
7696 71929 : lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp,
7697 : gimple_seq *clist, omp_context *ctx)
7698 : {
7699 71929 : gimple_seq sub_seq = NULL;
7700 71929 : gimple *stmt;
7701 71929 : tree x, c;
7702 71929 : int count = 0;
7703 :
7704 : /* OpenACC loop reductions are handled elsewhere. */
7705 71929 : if (is_gimple_omp_oacc (ctx->stmt))
7706 70902 : return;
7707 :
7708 : /* SIMD reductions are handled in lower_rec_input_clauses. */
7709 60604 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
7710 60604 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
7711 : return;
7712 :
7713 : /* inscan reductions are handled elsewhere. */
7714 51249 : if (ctx->scan_inclusive || ctx->scan_exclusive)
7715 : return;
7716 :
7717 : /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
7718 : update in that case, otherwise use a lock. */
7719 242209 : for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c))
7720 191983 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
7721 191983 : && !OMP_CLAUSE_REDUCTION_TASK (c))
7722 : {
7723 4583 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)
7724 4583 : || TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF)
7725 : {
7726 : /* Never use OMP_ATOMIC for array reductions or UDRs. */
7727 : count = -1;
7728 : break;
7729 : }
7730 3733 : count++;
7731 : }
7732 :
7733 51076 : if (count == 0)
7734 : return;
7735 :
7736 14918 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
7737 : {
7738 13891 : tree var, ref, new_var, orig_var;
7739 13891 : enum tree_code code;
7740 13891 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
7741 :
7742 22387 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
7743 13891 : || OMP_CLAUSE_REDUCTION_TASK (c))
7744 8496 : continue;
7745 :
7746 5395 : enum omp_clause_code ccode = OMP_CLAUSE_REDUCTION;
7747 5395 : orig_var = var = OMP_CLAUSE_DECL (c);
7748 5395 : if (TREE_CODE (var) == MEM_REF)
7749 : {
7750 689 : var = TREE_OPERAND (var, 0);
7751 689 : if (TREE_CODE (var) == POINTER_PLUS_EXPR)
7752 50 : var = TREE_OPERAND (var, 0);
7753 689 : if (TREE_CODE (var) == ADDR_EXPR)
7754 304 : var = TREE_OPERAND (var, 0);
7755 : else
7756 : {
7757 : /* If this is a pointer or referenced based array
7758 : section, the var could be private in the outer
7759 : context e.g. on orphaned loop construct. Pretend this
7760 : is private variable's outer reference. */
7761 385 : ccode = OMP_CLAUSE_PRIVATE;
7762 385 : if (INDIRECT_REF_P (var))
7763 54 : var = TREE_OPERAND (var, 0);
7764 : }
7765 689 : orig_var = var;
7766 689 : if (is_variable_sized (var))
7767 : {
7768 25 : gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
7769 25 : var = DECL_VALUE_EXPR (var);
7770 25 : gcc_assert (INDIRECT_REF_P (var));
7771 25 : var = TREE_OPERAND (var, 0);
7772 25 : gcc_assert (DECL_P (var));
7773 : }
7774 : }
7775 5395 : new_var = lookup_decl (var, ctx);
7776 5395 : if (var == OMP_CLAUSE_DECL (c)
7777 5395 : && omp_privatize_by_reference (var))
7778 152 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
7779 5395 : ref = build_outer_var_ref (var, ctx, ccode);
7780 5395 : code = OMP_CLAUSE_REDUCTION_CODE (c);
7781 :
7782 : /* reduction(-:var) sums up the partial results, so it acts
7783 : identically to reduction(+:var). */
7784 5395 : if (code == MINUS_EXPR)
7785 57 : code = PLUS_EXPR;
7786 :
7787 5395 : bool is_truth_op = (code == TRUTH_ANDIF_EXPR || code == TRUTH_ORIF_EXPR);
7788 5395 : if (count == 1)
7789 : {
7790 3324 : tree addr = build_fold_addr_expr_loc (clause_loc, ref);
7791 :
7792 3324 : addr = save_expr (addr);
7793 3324 : ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr);
7794 3324 : tree new_var2 = new_var;
7795 3324 : tree ref2 = ref;
7796 3324 : if (is_truth_op)
7797 : {
7798 784 : tree zero = build_zero_cst (TREE_TYPE (new_var));
7799 784 : new_var2 = fold_build2_loc (clause_loc, NE_EXPR,
7800 : boolean_type_node, new_var, zero);
7801 784 : ref2 = fold_build2_loc (clause_loc, NE_EXPR, boolean_type_node,
7802 : ref, zero);
7803 : }
7804 3324 : x = fold_build2_loc (clause_loc, code, TREE_TYPE (new_var2), ref2,
7805 : new_var2);
7806 3324 : if (is_truth_op)
7807 784 : x = fold_convert (TREE_TYPE (new_var), x);
7808 3324 : x = build2 (OMP_ATOMIC, void_type_node, addr, x);
7809 3324 : OMP_ATOMIC_MEMORY_ORDER (x) = OMP_MEMORY_ORDER_RELAXED;
7810 3324 : gimplify_and_add (x, stmt_seqp);
7811 3324 : return;
7812 : }
7813 2071 : else if (TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF)
7814 : {
7815 689 : tree d = OMP_CLAUSE_DECL (c);
7816 689 : tree type = TREE_TYPE (d);
7817 689 : tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
7818 689 : tree i = create_tmp_var (TREE_TYPE (v));
7819 689 : tree ptype = build_pointer_type (TREE_TYPE (type));
7820 689 : tree bias = TREE_OPERAND (d, 1);
7821 689 : d = TREE_OPERAND (d, 0);
7822 689 : if (TREE_CODE (d) == POINTER_PLUS_EXPR)
7823 : {
7824 50 : tree b = TREE_OPERAND (d, 1);
7825 50 : b = maybe_lookup_decl (b, ctx);
7826 50 : if (b == NULL)
7827 : {
7828 2 : b = TREE_OPERAND (d, 1);
7829 2 : b = maybe_lookup_decl_in_outer_ctx (b, ctx);
7830 : }
7831 50 : if (integer_zerop (bias))
7832 : bias = b;
7833 : else
7834 : {
7835 0 : bias = fold_convert_loc (clause_loc, TREE_TYPE (b), bias);
7836 0 : bias = fold_build2_loc (clause_loc, PLUS_EXPR,
7837 0 : TREE_TYPE (b), b, bias);
7838 : }
7839 50 : d = TREE_OPERAND (d, 0);
7840 : }
7841 : /* For ref build_outer_var_ref already performs this, so
7842 : only new_var needs a dereference. */
7843 689 : if (INDIRECT_REF_P (d))
7844 : {
7845 54 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
7846 54 : gcc_assert (omp_privatize_by_reference (var)
7847 : && var == orig_var);
7848 : }
7849 635 : else if (TREE_CODE (d) == ADDR_EXPR)
7850 : {
7851 304 : if (orig_var == var)
7852 : {
7853 279 : new_var = build_fold_addr_expr (new_var);
7854 279 : ref = build_fold_addr_expr (ref);
7855 : }
7856 : }
7857 : else
7858 : {
7859 331 : gcc_assert (orig_var == var);
7860 331 : if (omp_privatize_by_reference (var))
7861 79 : ref = build_fold_addr_expr (ref);
7862 : }
7863 689 : if (DECL_P (v))
7864 : {
7865 93 : tree t = maybe_lookup_decl (v, ctx);
7866 93 : if (t)
7867 88 : v = t;
7868 : else
7869 5 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
7870 93 : gimplify_expr (&v, stmt_seqp, NULL, is_gimple_val, fb_rvalue);
7871 : }
7872 689 : if (!integer_zerop (bias))
7873 : {
7874 402 : bias = fold_convert_loc (clause_loc, sizetype, bias);
7875 804 : new_var = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
7876 402 : TREE_TYPE (new_var), new_var,
7877 : unshare_expr (bias));
7878 402 : ref = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
7879 402 : TREE_TYPE (ref), ref, bias);
7880 : }
7881 689 : new_var = fold_convert_loc (clause_loc, ptype, new_var);
7882 689 : ref = fold_convert_loc (clause_loc, ptype, ref);
7883 689 : tree m = create_tmp_var (ptype);
7884 689 : gimplify_assign (m, new_var, stmt_seqp);
7885 689 : new_var = m;
7886 689 : m = create_tmp_var (ptype);
7887 689 : gimplify_assign (m, ref, stmt_seqp);
7888 689 : ref = m;
7889 689 : gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), stmt_seqp);
7890 689 : tree body = create_artificial_label (UNKNOWN_LOCATION);
7891 689 : tree end = create_artificial_label (UNKNOWN_LOCATION);
7892 689 : gimple_seq_add_stmt (&sub_seq, gimple_build_label (body));
7893 689 : tree priv = build_simple_mem_ref_loc (clause_loc, new_var);
7894 689 : tree out = build_simple_mem_ref_loc (clause_loc, ref);
7895 689 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
7896 : {
7897 72 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
7898 72 : tree decl_placeholder
7899 72 : = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
7900 72 : SET_DECL_VALUE_EXPR (placeholder, out);
7901 72 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
7902 72 : SET_DECL_VALUE_EXPR (decl_placeholder, priv);
7903 72 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
7904 72 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
7905 72 : gimple_seq_add_seq (&sub_seq,
7906 72 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
7907 72 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
7908 72 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
7909 72 : OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c) = NULL;
7910 : }
7911 : else
7912 : {
7913 617 : tree out2 = out;
7914 617 : tree priv2 = priv;
7915 617 : if (is_truth_op)
7916 : {
7917 0 : tree zero = build_zero_cst (TREE_TYPE (out));
7918 0 : out2 = fold_build2_loc (clause_loc, NE_EXPR,
7919 : boolean_type_node, out, zero);
7920 0 : priv2 = fold_build2_loc (clause_loc, NE_EXPR,
7921 : boolean_type_node, priv, zero);
7922 : }
7923 617 : x = build2 (code, TREE_TYPE (out2), out2, priv2);
7924 617 : if (is_truth_op)
7925 0 : x = fold_convert (TREE_TYPE (out), x);
7926 617 : out = unshare_expr (out);
7927 617 : gimplify_assign (out, x, &sub_seq);
7928 : }
7929 689 : gimple *g = gimple_build_assign (new_var, POINTER_PLUS_EXPR, new_var,
7930 689 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
7931 689 : gimple_seq_add_stmt (&sub_seq, g);
7932 689 : g = gimple_build_assign (ref, POINTER_PLUS_EXPR, ref,
7933 689 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
7934 689 : gimple_seq_add_stmt (&sub_seq, g);
7935 689 : g = gimple_build_assign (i, PLUS_EXPR, i,
7936 689 : build_int_cst (TREE_TYPE (i), 1));
7937 689 : gimple_seq_add_stmt (&sub_seq, g);
7938 689 : g = gimple_build_cond (LE_EXPR, i, v, body, end);
7939 689 : gimple_seq_add_stmt (&sub_seq, g);
7940 689 : gimple_seq_add_stmt (&sub_seq, gimple_build_label (end));
7941 : }
7942 1382 : else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
7943 : {
7944 776 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
7945 :
7946 776 : if (omp_privatize_by_reference (var)
7947 833 : && !useless_type_conversion_p (TREE_TYPE (placeholder),
7948 57 : TREE_TYPE (ref)))
7949 49 : ref = build_fold_addr_expr_loc (clause_loc, ref);
7950 776 : SET_DECL_VALUE_EXPR (placeholder, ref);
7951 776 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
7952 776 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
7953 776 : gimple_seq_add_seq (&sub_seq, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
7954 776 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
7955 776 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
7956 : }
7957 : else
7958 : {
7959 606 : tree new_var2 = new_var;
7960 606 : tree ref2 = ref;
7961 606 : if (is_truth_op)
7962 : {
7963 78 : tree zero = build_zero_cst (TREE_TYPE (new_var));
7964 78 : new_var2 = fold_build2_loc (clause_loc, NE_EXPR,
7965 : boolean_type_node, new_var, zero);
7966 78 : ref2 = fold_build2_loc (clause_loc, NE_EXPR, boolean_type_node,
7967 : ref, zero);
7968 : }
7969 606 : x = build2 (code, TREE_TYPE (ref), ref2, new_var2);
7970 606 : if (is_truth_op)
7971 78 : x = fold_convert (TREE_TYPE (new_var), x);
7972 606 : ref = build_outer_var_ref (var, ctx);
7973 606 : gimplify_assign (ref, x, &sub_seq);
7974 : }
7975 : }
7976 :
7977 1027 : stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START),
7978 : 0);
7979 1027 : gimple_seq_add_stmt (stmt_seqp, stmt);
7980 :
7981 1027 : gimple_seq_add_seq (stmt_seqp, sub_seq);
7982 :
7983 1027 : if (clist)
7984 : {
7985 157 : gimple_seq_add_seq (stmt_seqp, *clist);
7986 157 : *clist = NULL;
7987 : }
7988 :
7989 1027 : stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END),
7990 : 0);
7991 1027 : gimple_seq_add_stmt (stmt_seqp, stmt);
7992 : }
7993 :
7994 :
7995 : /* Generate code to implement the COPYPRIVATE clauses. */
7996 :
7997 : static void
7998 115 : lower_copyprivate_clauses (tree clauses, gimple_seq *slist, gimple_seq *rlist,
7999 : omp_context *ctx)
8000 : {
8001 115 : tree c;
8002 :
8003 463 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
8004 : {
8005 348 : tree var, new_var, ref, x;
8006 348 : bool by_ref;
8007 348 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
8008 :
8009 348 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE)
8010 6 : continue;
8011 :
8012 342 : var = OMP_CLAUSE_DECL (c);
8013 342 : by_ref = use_pointer_for_field (var, NULL);
8014 :
8015 342 : ref = build_sender_ref (var, ctx);
8016 342 : x = new_var = lookup_decl_in_outer_ctx (var, ctx);
8017 342 : if (by_ref)
8018 : {
8019 74 : x = build_fold_addr_expr_loc (clause_loc, new_var);
8020 74 : x = fold_convert_loc (clause_loc, TREE_TYPE (ref), x);
8021 : }
8022 342 : gimplify_assign (ref, x, slist);
8023 :
8024 342 : ref = build_receiver_ref (var, false, ctx);
8025 342 : if (by_ref)
8026 : {
8027 74 : ref = fold_convert_loc (clause_loc,
8028 74 : build_pointer_type (TREE_TYPE (new_var)),
8029 : ref);
8030 74 : ref = build_fold_indirect_ref_loc (clause_loc, ref);
8031 : }
8032 342 : if (omp_privatize_by_reference (var))
8033 : {
8034 171 : ref = fold_convert_loc (clause_loc, TREE_TYPE (new_var), ref);
8035 171 : ref = build_simple_mem_ref_loc (clause_loc, ref);
8036 171 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
8037 : }
8038 342 : x = lang_hooks.decls.omp_clause_assign_op (c, new_var, ref);
8039 342 : gimplify_and_add (x, rlist);
8040 : }
8041 115 : }
8042 :
8043 :
8044 : /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
8045 : and REDUCTION from the sender (aka parent) side. */
8046 :
8047 : static void
8048 22389 : lower_send_clauses (tree clauses, gimple_seq *ilist, gimple_seq *olist,
8049 : omp_context *ctx)
8050 : {
8051 22389 : tree c, t;
8052 22389 : int ignored_looptemp = 0;
8053 22389 : bool is_taskloop = false;
8054 :
8055 : /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
8056 : by GOMP_taskloop. */
8057 22389 : if (is_task_ctx (ctx) && gimple_omp_task_taskloop_p (ctx->stmt))
8058 : {
8059 : ignored_looptemp = 2;
8060 : is_taskloop = true;
8061 : }
8062 :
8063 116969 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
8064 : {
8065 94580 : tree val, ref, x, var;
8066 94580 : bool by_ref, do_in = false, do_out = false;
8067 94580 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
8068 :
8069 94580 : switch (OMP_CLAUSE_CODE (c))
8070 : {
8071 4658 : case OMP_CLAUSE_PRIVATE:
8072 4658 : if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
8073 : break;
8074 4503 : continue;
8075 : case OMP_CLAUSE_FIRSTPRIVATE:
8076 : case OMP_CLAUSE_COPYIN:
8077 : case OMP_CLAUSE_LASTPRIVATE:
8078 : case OMP_CLAUSE_IN_REDUCTION:
8079 : case OMP_CLAUSE__REDUCTEMP_:
8080 : break;
8081 4787 : case OMP_CLAUSE_REDUCTION:
8082 4787 : if (is_task_ctx (ctx) || OMP_CLAUSE_REDUCTION_TASK (c))
8083 745 : continue;
8084 : break;
8085 17561 : case OMP_CLAUSE_SHARED:
8086 17561 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
8087 : break;
8088 17249 : continue;
8089 18795 : case OMP_CLAUSE__LOOPTEMP_:
8090 18795 : if (ignored_looptemp)
8091 : {
8092 2660 : ignored_looptemp--;
8093 2660 : continue;
8094 : }
8095 : break;
8096 11936 : default:
8097 11936 : continue;
8098 : }
8099 :
8100 57487 : val = OMP_CLAUSE_DECL (c);
8101 57487 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
8102 53445 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION)
8103 59127 : && TREE_CODE (val) == MEM_REF)
8104 : {
8105 1578 : val = TREE_OPERAND (val, 0);
8106 1578 : if (TREE_CODE (val) == POINTER_PLUS_EXPR)
8107 351 : val = TREE_OPERAND (val, 0);
8108 1578 : if (INDIRECT_REF_P (val)
8109 1494 : || TREE_CODE (val) == ADDR_EXPR)
8110 906 : val = TREE_OPERAND (val, 0);
8111 1578 : if (is_variable_sized (val))
8112 56 : continue;
8113 : }
8114 :
8115 : /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
8116 : outer taskloop region. */
8117 57431 : omp_context *ctx_for_o = ctx;
8118 57431 : if (is_taskloop
8119 3911 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
8120 57743 : && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
8121 312 : ctx_for_o = ctx->outer;
8122 :
8123 57431 : var = lookup_decl_in_outer_ctx (val, ctx_for_o);
8124 :
8125 57431 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN
8126 56921 : && is_global_var (var)
8127 59736 : && (val == OMP_CLAUSE_DECL (c)
8128 273 : || !is_task_ctx (ctx)
8129 241 : || (TREE_CODE (TREE_TYPE (val)) != POINTER_TYPE
8130 187 : && (TREE_CODE (TREE_TYPE (val)) != REFERENCE_TYPE
8131 0 : || (TREE_CODE (TREE_TYPE (TREE_TYPE (val)))
8132 : != POINTER_TYPE)))))
8133 2251 : continue;
8134 :
8135 55180 : t = omp_member_access_dummy_var (var);
8136 55180 : if (t)
8137 : {
8138 249 : var = DECL_VALUE_EXPR (var);
8139 249 : tree o = maybe_lookup_decl_in_outer_ctx (t, ctx_for_o);
8140 249 : if (o != t)
8141 69 : var = unshare_and_remap (var, t, o);
8142 : else
8143 180 : var = unshare_expr (var);
8144 : }
8145 :
8146 55180 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED)
8147 : {
8148 : /* Handle taskloop firstprivate/lastprivate, where the
8149 : lastprivate on GIMPLE_OMP_TASK is represented as
8150 : OMP_CLAUSE_SHARED_FIRSTPRIVATE. */
8151 79 : tree f = lookup_sfield ((splay_tree_key) &DECL_UID (val), ctx);
8152 79 : x = omp_build_component_ref (ctx->sender_decl, f);
8153 79 : if (use_pointer_for_field (val, ctx))
8154 59 : var = build_fold_addr_expr (var);
8155 79 : gimplify_assign (x, var, ilist);
8156 79 : DECL_ABSTRACT_ORIGIN (f) = NULL;
8157 79 : continue;
8158 79 : }
8159 :
8160 55155 : if (((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
8161 51804 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IN_REDUCTION)
8162 4433 : || val == OMP_CLAUSE_DECL (c))
8163 108899 : && is_variable_sized (val))
8164 54 : continue;
8165 55047 : by_ref = use_pointer_for_field (val, NULL);
8166 :
8167 55047 : switch (OMP_CLAUSE_CODE (c))
8168 : {
8169 26795 : case OMP_CLAUSE_FIRSTPRIVATE:
8170 26795 : if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c)
8171 7255 : && !by_ref
8172 33998 : && is_task_ctx (ctx))
8173 2330 : suppress_warning (var);
8174 : do_in = true;
8175 : break;
8176 :
8177 : case OMP_CLAUSE_PRIVATE:
8178 : case OMP_CLAUSE_COPYIN:
8179 : case OMP_CLAUSE__LOOPTEMP_:
8180 : case OMP_CLAUSE__REDUCTEMP_:
8181 : do_in = true;
8182 : break;
8183 :
8184 6463 : case OMP_CLAUSE_LASTPRIVATE:
8185 6463 : if (by_ref || omp_privatize_by_reference (val))
8186 : {
8187 383 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
8188 239 : continue;
8189 : do_in = true;
8190 : }
8191 : else
8192 : {
8193 6080 : do_out = true;
8194 6080 : if (lang_hooks.decls.omp_private_outer_ref (val))
8195 : do_in = true;
8196 : }
8197 : break;
8198 :
8199 4433 : case OMP_CLAUSE_REDUCTION:
8200 4433 : case OMP_CLAUSE_IN_REDUCTION:
8201 4433 : do_in = true;
8202 4433 : if (val == OMP_CLAUSE_DECL (c))
8203 : {
8204 3130 : if (is_task_ctx (ctx))
8205 301 : by_ref = use_pointer_for_field (val, ctx);
8206 : else
8207 2829 : do_out = !(by_ref || omp_privatize_by_reference (val));
8208 : }
8209 : else
8210 1303 : by_ref = TREE_CODE (TREE_TYPE (val)) == ARRAY_TYPE;
8211 : break;
8212 :
8213 0 : default:
8214 0 : gcc_unreachable ();
8215 : }
8216 :
8217 1604 : if (do_in)
8218 : {
8219 48752 : ref = build_sender_ref (val, ctx);
8220 48752 : x = by_ref ? build_fold_addr_expr_loc (clause_loc, var) : var;
8221 48752 : gimplify_assign (ref, x, ilist);
8222 48752 : if (is_task_ctx (ctx))
8223 6231 : DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref, 1)) = NULL;
8224 : }
8225 :
8226 54808 : if (do_out)
8227 : {
8228 8374 : ref = build_sender_ref (val, ctx);
8229 8374 : gimplify_assign (var, ref, olist);
8230 : }
8231 : }
8232 22389 : }
8233 :
8234 : /* Generate code to implement SHARED from the sender (aka parent)
8235 : side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
8236 : list things that got automatically shared. */
8237 :
8238 : static void
8239 22389 : lower_send_shared_vars (gimple_seq *ilist, gimple_seq *olist, omp_context *ctx)
8240 : {
8241 22389 : tree var, ovar, nvar, t, f, x, record_type;
8242 :
8243 22389 : if (ctx->record_type == NULL)
8244 4451 : return;
8245 :
8246 17938 : record_type = ctx->srecord_type ? ctx->srecord_type : ctx->record_type;
8247 86764 : for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
8248 : {
8249 68826 : ovar = DECL_ABSTRACT_ORIGIN (f);
8250 68826 : if (!ovar || TREE_CODE (ovar) == FIELD_DECL)
8251 6310 : continue;
8252 :
8253 62516 : nvar = maybe_lookup_decl (ovar, ctx);
8254 113397 : if (!nvar
8255 62107 : || !DECL_HAS_VALUE_EXPR_P (nvar)
8256 74332 : || (ctx->allocate_map
8257 274 : && ctx->allocate_map->get (ovar)))
8258 50881 : continue;
8259 :
8260 : /* If CTX is a nested parallel directive. Find the immediately
8261 : enclosing parallel or workshare construct that contains a
8262 : mapping for OVAR. */
8263 11635 : var = lookup_decl_in_outer_ctx (ovar, ctx);
8264 :
8265 11635 : t = omp_member_access_dummy_var (var);
8266 11635 : if (t)
8267 : {
8268 49 : var = DECL_VALUE_EXPR (var);
8269 49 : tree o = maybe_lookup_decl_in_outer_ctx (t, ctx);
8270 49 : if (o != t)
8271 18 : var = unshare_and_remap (var, t, o);
8272 : else
8273 31 : var = unshare_expr (var);
8274 : }
8275 :
8276 11635 : if (use_pointer_for_field (ovar, ctx))
8277 : {
8278 6434 : x = build_sender_ref (ovar, ctx);
8279 6434 : if (TREE_CODE (TREE_TYPE (f)) == ARRAY_TYPE
8280 6434 : && TREE_TYPE (f) == TREE_TYPE (ovar))
8281 : {
8282 103 : gcc_assert (is_parallel_ctx (ctx)
8283 : && DECL_ARTIFICIAL (ovar));
8284 : /* _condtemp_ clause. */
8285 103 : var = build_constructor (TREE_TYPE (x), NULL);
8286 : }
8287 : else
8288 6331 : var = build_fold_addr_expr (var);
8289 6434 : gimplify_assign (x, var, ilist);
8290 : }
8291 : else
8292 : {
8293 5201 : x = build_sender_ref (ovar, ctx);
8294 5201 : gimplify_assign (x, var, ilist);
8295 :
8296 5201 : if (!TREE_READONLY (var)
8297 : /* We don't need to receive a new reference to a result
8298 : or parm decl. In fact we may not store to it as we will
8299 : invalidate any pending RSO and generate wrong gimple
8300 : during inlining. */
8301 5201 : && !((TREE_CODE (var) == RESULT_DECL
8302 3304 : || TREE_CODE (var) == PARM_DECL)
8303 182 : && DECL_BY_REFERENCE (var)))
8304 : {
8305 3233 : x = build_sender_ref (ovar, ctx);
8306 3233 : gimplify_assign (var, x, olist);
8307 : }
8308 : }
8309 : }
8310 : }
8311 :
8312 : /* Emit an OpenACC head marker call, encapulating the partitioning and
8313 : other information that must be processed by the target compiler.
8314 : Return the maximum number of dimensions the associated loop might
8315 : be partitioned over. */
8316 :
8317 : static unsigned
8318 9665 : lower_oacc_head_mark (location_t loc, tree ddvar, tree clauses,
8319 : gimple_seq *seq, omp_context *ctx)
8320 : {
8321 9665 : unsigned levels = 0;
8322 9665 : unsigned tag = 0;
8323 9665 : tree gang_static = NULL_TREE;
8324 9665 : auto_vec<tree, 5> args;
8325 :
8326 19330 : args.quick_push (build_int_cst
8327 9665 : (integer_type_node, IFN_UNIQUE_OACC_HEAD_MARK));
8328 9665 : args.quick_push (ddvar);
8329 30854 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
8330 : {
8331 21189 : switch (OMP_CLAUSE_CODE (c))
8332 : {
8333 1856 : case OMP_CLAUSE_GANG:
8334 1856 : tag |= OLF_DIM_GANG;
8335 1856 : gang_static = OMP_CLAUSE_GANG_STATIC_EXPR (c);
8336 : /* static:* is represented by -1, and we can ignore it, as
8337 : scheduling is always static. */
8338 1856 : if (gang_static && integer_minus_onep (gang_static))
8339 70 : gang_static = NULL_TREE;
8340 1856 : levels++;
8341 1856 : break;
8342 :
8343 1330 : case OMP_CLAUSE_WORKER:
8344 1330 : tag |= OLF_DIM_WORKER;
8345 1330 : levels++;
8346 1330 : break;
8347 :
8348 1523 : case OMP_CLAUSE_VECTOR:
8349 1523 : tag |= OLF_DIM_VECTOR;
8350 1523 : levels++;
8351 1523 : break;
8352 :
8353 314 : case OMP_CLAUSE_SEQ:
8354 314 : tag |= OLF_SEQ;
8355 314 : break;
8356 :
8357 409 : case OMP_CLAUSE_AUTO:
8358 409 : tag |= OLF_AUTO;
8359 409 : break;
8360 :
8361 260 : case OMP_CLAUSE_INDEPENDENT:
8362 260 : tag |= OLF_INDEPENDENT;
8363 260 : break;
8364 :
8365 136 : case OMP_CLAUSE_TILE:
8366 136 : tag |= OLF_TILE;
8367 136 : break;
8368 :
8369 3986 : case OMP_CLAUSE_REDUCTION:
8370 3986 : tag |= OLF_REDUCTION;
8371 3986 : break;
8372 :
8373 11375 : default:
8374 11375 : continue;
8375 : }
8376 : }
8377 :
8378 9665 : if (gang_static)
8379 : {
8380 146 : if (DECL_P (gang_static))
8381 16 : gang_static = build_outer_var_ref (gang_static, ctx);
8382 146 : tag |= OLF_GANG_STATIC;
8383 : }
8384 :
8385 9665 : omp_context *tgt = enclosing_target_ctx (ctx);
8386 9665 : if (!tgt || is_oacc_parallel_or_serial (tgt))
8387 : ;
8388 62 : else if (is_oacc_kernels (tgt))
8389 : /* Not using this loops handling inside OpenACC 'kernels' regions. */
8390 0 : gcc_unreachable ();
8391 62 : else if (is_oacc_kernels_decomposed_part (tgt))
8392 : ;
8393 : else
8394 0 : gcc_unreachable ();
8395 :
8396 : /* In a parallel region, loops are implicitly INDEPENDENT. */
8397 9665 : if (!tgt || is_oacc_parallel_or_serial (tgt))
8398 9603 : tag |= OLF_INDEPENDENT;
8399 :
8400 : /* Loops inside OpenACC 'kernels' decomposed parts' regions are expected to
8401 : have an explicit 'seq' or 'independent' clause, and no 'auto' clause. */
8402 9665 : if (tgt && is_oacc_kernels_decomposed_part (tgt))
8403 : {
8404 62 : gcc_assert (tag & (OLF_SEQ | OLF_INDEPENDENT));
8405 62 : gcc_assert (!(tag & OLF_AUTO));
8406 : }
8407 :
8408 9665 : if (tag & OLF_TILE)
8409 : /* Tiling could use all 3 levels. */
8410 : levels = 3;
8411 : else
8412 : {
8413 : /* A loop lacking SEQ, GANG, WORKER and/or VECTOR could be AUTO.
8414 : Ensure at least one level, or 2 for possible auto
8415 : partitioning */
8416 9529 : bool maybe_auto = !(tag & (((GOMP_DIM_MASK (GOMP_DIM_MAX) - 1)
8417 : << OLF_DIM_BASE) | OLF_SEQ));
8418 :
8419 9529 : if (levels < 1u + maybe_auto)
8420 : levels = 1u + maybe_auto;
8421 : }
8422 :
8423 9665 : args.quick_push (build_int_cst (integer_type_node, levels));
8424 9665 : args.quick_push (build_int_cst (integer_type_node, tag));
8425 9665 : if (gang_static)
8426 146 : args.quick_push (gang_static);
8427 :
8428 9665 : gcall *call = gimple_build_call_internal_vec (IFN_UNIQUE, args);
8429 9665 : gimple_set_location (call, loc);
8430 9665 : gimple_set_lhs (call, ddvar);
8431 9665 : gimple_seq_add_stmt (seq, call);
8432 :
8433 9665 : return levels;
8434 9665 : }
8435 :
8436 : /* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the
8437 : partitioning level of the enclosed region. */
8438 :
8439 : static void
8440 42767 : lower_oacc_loop_marker (location_t loc, tree ddvar, bool head,
8441 : tree tofollow, gimple_seq *seq)
8442 : {
8443 42767 : int marker_kind = (head ? IFN_UNIQUE_OACC_HEAD_MARK
8444 : : IFN_UNIQUE_OACC_TAIL_MARK);
8445 42767 : tree marker = build_int_cst (integer_type_node, marker_kind);
8446 42767 : int nargs = 2 + (tofollow != NULL_TREE);
8447 42767 : gcall *call = gimple_build_call_internal (IFN_UNIQUE, nargs,
8448 : marker, ddvar, tofollow);
8449 42767 : gimple_set_location (call, loc);
8450 42767 : gimple_set_lhs (call, ddvar);
8451 42767 : gimple_seq_add_stmt (seq, call);
8452 42767 : }
8453 :
8454 : /* Generate the before and after OpenACC loop sequences. CLAUSES are
8455 : the loop clauses, from which we extract reductions. Initialize
8456 : HEAD and TAIL. */
8457 :
8458 : static void
8459 9665 : lower_oacc_head_tail (location_t loc, tree clauses, gcall *private_marker,
8460 : gimple_seq *head, gimple_seq *tail, omp_context *ctx)
8461 : {
8462 9665 : bool inner = false;
8463 9665 : tree ddvar = create_tmp_var (integer_type_node, ".data_dep");
8464 9665 : gimple_seq_add_stmt (head, gimple_build_assign (ddvar, integer_zero_node));
8465 :
8466 9665 : unsigned count = lower_oacc_head_mark (loc, ddvar, clauses, head, ctx);
8467 :
8468 9665 : if (private_marker)
8469 : {
8470 229 : gimple_set_location (private_marker, loc);
8471 229 : gimple_call_set_lhs (private_marker, ddvar);
8472 229 : gimple_call_set_arg (private_marker, 1, ddvar);
8473 : }
8474 :
8475 9665 : tree fork_kind = build_int_cst (unsigned_type_node, IFN_UNIQUE_OACC_FORK);
8476 9665 : tree join_kind = build_int_cst (unsigned_type_node, IFN_UNIQUE_OACC_JOIN);
8477 :
8478 9665 : gcc_assert (count);
8479 26216 : for (unsigned done = 1; count; count--, done++)
8480 : {
8481 16551 : gimple_seq fork_seq = NULL;
8482 16551 : gimple_seq join_seq = NULL;
8483 :
8484 16551 : tree place = build_int_cst (integer_type_node, -1);
8485 16551 : gcall *fork = gimple_build_call_internal (IFN_UNIQUE, 3,
8486 : fork_kind, ddvar, place);
8487 16551 : gimple_set_location (fork, loc);
8488 16551 : gimple_set_lhs (fork, ddvar);
8489 :
8490 16551 : gcall *join = gimple_build_call_internal (IFN_UNIQUE, 3,
8491 : join_kind, ddvar, place);
8492 16551 : gimple_set_location (join, loc);
8493 16551 : gimple_set_lhs (join, ddvar);
8494 :
8495 : /* Mark the beginning of this level sequence. */
8496 16551 : if (inner)
8497 6886 : lower_oacc_loop_marker (loc, ddvar, true,
8498 6886 : build_int_cst (integer_type_node, count),
8499 : &fork_seq);
8500 16551 : lower_oacc_loop_marker (loc, ddvar, false,
8501 16551 : build_int_cst (integer_type_node, done),
8502 : &join_seq);
8503 :
8504 23437 : lower_oacc_reductions (loc, clauses, place, inner,
8505 : fork, (count == 1) ? private_marker : NULL,
8506 : join, &fork_seq, &join_seq, ctx);
8507 :
8508 : /* Append this level to head. */
8509 16551 : gimple_seq_add_seq (head, fork_seq);
8510 : /* Prepend it to tail. */
8511 16551 : gimple_seq_add_seq (&join_seq, *tail);
8512 16551 : *tail = join_seq;
8513 :
8514 16551 : inner = true;
8515 : }
8516 :
8517 : /* Mark the end of the sequence. */
8518 9665 : lower_oacc_loop_marker (loc, ddvar, true, NULL_TREE, head);
8519 9665 : lower_oacc_loop_marker (loc, ddvar, false, NULL_TREE, tail);
8520 9665 : }
8521 :
8522 : /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
8523 : catch handler and return it. This prevents programs from violating the
8524 : structured block semantics with throws. */
8525 :
8526 : static gimple_seq
8527 95019 : maybe_catch_exception (gimple_seq body)
8528 : {
8529 95019 : gimple *g;
8530 95019 : tree decl;
8531 :
8532 95019 : if (!flag_exceptions)
8533 : return body;
8534 :
8535 42535 : if (lang_hooks.eh_protect_cleanup_actions != NULL)
8536 42507 : decl = lang_hooks.eh_protect_cleanup_actions ();
8537 : else
8538 28 : decl = builtin_decl_explicit (BUILT_IN_TRAP);
8539 :
8540 42535 : g = gimple_build_eh_must_not_throw (decl);
8541 42535 : g = gimple_build_try (body, gimple_seq_alloc_with_stmt (g),
8542 : GIMPLE_TRY_CATCH);
8543 :
8544 42535 : return gimple_seq_alloc_with_stmt (g);
8545 : }
8546 :
8547 :
8548 : /* Routines to lower OMP directives into OMP-GIMPLE. */
8549 :
8550 : /* If ctx is a worksharing context inside of a cancellable parallel
8551 : region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8552 : and conditional branch to parallel's cancel_label to handle
8553 : cancellation in the implicit barrier. */
8554 :
8555 : static void
8556 48520 : maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple *omp_return,
8557 : gimple_seq *body)
8558 : {
8559 48520 : gcc_assert (gimple_code (omp_return) == GIMPLE_OMP_RETURN);
8560 48520 : if (gimple_omp_return_nowait_p (omp_return))
8561 : return;
8562 19559 : for (omp_context *outer = ctx->outer; outer; outer = outer->outer)
8563 16010 : if (gimple_code (outer->stmt) == GIMPLE_OMP_PARALLEL
8564 16010 : && outer->cancellable)
8565 : {
8566 64 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_CANCEL);
8567 64 : tree c_bool_type = TREE_TYPE (TREE_TYPE (fndecl));
8568 64 : tree lhs = create_tmp_var (c_bool_type);
8569 64 : gimple_omp_return_set_lhs (omp_return, lhs);
8570 64 : tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
8571 64 : gimple *g = gimple_build_cond (NE_EXPR, lhs,
8572 : fold_convert (c_bool_type,
8573 : boolean_false_node),
8574 : outer->cancel_label, fallthru_label);
8575 64 : gimple_seq_add_stmt (body, g);
8576 64 : gimple_seq_add_stmt (body, gimple_build_label (fallthru_label));
8577 : }
8578 15946 : else if (gimple_code (outer->stmt) != GIMPLE_OMP_TASKGROUP
8579 15946 : && gimple_code (outer->stmt) != GIMPLE_OMP_SCOPE)
8580 : return;
8581 : }
8582 :
8583 : /* Find the first task_reduction or reduction clause or return NULL
8584 : if there are none. */
8585 :
8586 : static inline tree
8587 48775 : omp_task_reductions_find_first (tree clauses, enum tree_code code,
8588 : enum omp_clause_code ccode)
8589 : {
8590 64093 : while (1)
8591 : {
8592 56434 : clauses = omp_find_clause (clauses, ccode);
8593 56434 : if (clauses == NULL_TREE)
8594 : return NULL_TREE;
8595 9105 : if (ccode != OMP_CLAUSE_REDUCTION
8596 9105 : || code == OMP_TASKLOOP
8597 9105 : || OMP_CLAUSE_REDUCTION_TASK (clauses))
8598 : return clauses;
8599 7659 : clauses = OMP_CLAUSE_CHAIN (clauses);
8600 : }
8601 : }
8602 :
8603 : static void lower_omp_task_reductions (omp_context *, enum tree_code, tree,
8604 : gimple_seq *, gimple_seq *);
8605 :
8606 : /* Lower the OpenMP sections directive in the current statement in GSI_P.
8607 : CTX is the enclosing OMP context for the current statement. */
8608 :
8609 : static void
8610 378 : lower_omp_sections (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8611 : {
8612 378 : tree block, control;
8613 378 : gimple_stmt_iterator tgsi;
8614 378 : gomp_sections *stmt;
8615 378 : gimple *t;
8616 378 : gbind *new_stmt, *bind;
8617 378 : gimple_seq ilist, dlist, olist, tred_dlist = NULL, clist = NULL, new_body;
8618 :
8619 378 : stmt = as_a <gomp_sections *> (gsi_stmt (*gsi_p));
8620 :
8621 378 : push_gimplify_context ();
8622 :
8623 378 : dlist = NULL;
8624 378 : ilist = NULL;
8625 :
8626 378 : tree rclauses
8627 378 : = omp_task_reductions_find_first (gimple_omp_sections_clauses (stmt),
8628 : OMP_SECTIONS, OMP_CLAUSE_REDUCTION);
8629 378 : tree rtmp = NULL_TREE;
8630 378 : if (rclauses)
8631 : {
8632 8 : tree type = build_pointer_type (pointer_sized_int_node);
8633 8 : tree temp = create_tmp_var (type);
8634 8 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
8635 8 : OMP_CLAUSE_DECL (c) = temp;
8636 8 : OMP_CLAUSE_CHAIN (c) = gimple_omp_sections_clauses (stmt);
8637 8 : gimple_omp_sections_set_clauses (stmt, c);
8638 8 : lower_omp_task_reductions (ctx, OMP_SECTIONS,
8639 : gimple_omp_sections_clauses (stmt),
8640 : &ilist, &tred_dlist);
8641 8 : rclauses = c;
8642 8 : rtmp = make_ssa_name (type);
8643 8 : gimple_seq_add_stmt (&ilist, gimple_build_assign (rtmp, temp));
8644 : }
8645 :
8646 378 : tree *clauses_ptr = gimple_omp_sections_clauses_ptr (stmt);
8647 378 : lower_lastprivate_conditional_clauses (clauses_ptr, ctx);
8648 :
8649 378 : lower_rec_input_clauses (gimple_omp_sections_clauses (stmt),
8650 : &ilist, &dlist, ctx, NULL);
8651 :
8652 378 : control = create_tmp_var (unsigned_type_node, ".section");
8653 378 : gimple_omp_sections_set_control (stmt, control);
8654 :
8655 378 : new_body = gimple_omp_body (stmt);
8656 378 : gimple_omp_set_body (stmt, NULL);
8657 378 : tgsi = gsi_start (new_body);
8658 1233 : for (; !gsi_end_p (tgsi); gsi_next (&tgsi))
8659 : {
8660 855 : omp_context *sctx;
8661 855 : gimple *sec_start;
8662 :
8663 855 : sec_start = gsi_stmt (tgsi);
8664 855 : sctx = maybe_lookup_ctx (sec_start);
8665 855 : gcc_assert (sctx);
8666 :
8667 855 : lower_omp (gimple_omp_body_ptr (sec_start), sctx);
8668 855 : gsi_insert_seq_after (&tgsi, gimple_omp_body (sec_start),
8669 : GSI_CONTINUE_LINKING);
8670 855 : gimple_omp_set_body (sec_start, NULL);
8671 :
8672 855 : if (gsi_one_before_end_p (tgsi))
8673 : {
8674 378 : gimple_seq l = NULL;
8675 378 : lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt), NULL,
8676 : &ilist, &l, &clist, ctx);
8677 378 : gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING);
8678 378 : gimple_omp_section_set_last (sec_start);
8679 : }
8680 :
8681 855 : gsi_insert_after (&tgsi, gimple_build_omp_return (false),
8682 : GSI_CONTINUE_LINKING);
8683 : }
8684 :
8685 378 : block = make_node (BLOCK);
8686 378 : bind = gimple_build_bind (NULL, new_body, block);
8687 :
8688 378 : olist = NULL;
8689 378 : lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist,
8690 : &clist, ctx);
8691 378 : if (clist)
8692 : {
8693 10 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START);
8694 10 : gcall *g = gimple_build_call (fndecl, 0);
8695 10 : gimple_seq_add_stmt (&olist, g);
8696 10 : gimple_seq_add_seq (&olist, clist);
8697 10 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END);
8698 10 : g = gimple_build_call (fndecl, 0);
8699 10 : gimple_seq_add_stmt (&olist, g);
8700 : }
8701 :
8702 378 : block = make_node (BLOCK);
8703 378 : new_stmt = gimple_build_bind (NULL, NULL, block);
8704 378 : gsi_replace (gsi_p, new_stmt, true);
8705 :
8706 378 : pop_gimplify_context (new_stmt);
8707 378 : gimple_bind_append_vars (new_stmt, ctx->block_vars);
8708 378 : BLOCK_VARS (block) = gimple_bind_vars (bind);
8709 378 : if (BLOCK_VARS (block))
8710 0 : TREE_USED (block) = 1;
8711 :
8712 378 : new_body = NULL;
8713 378 : gimple_seq_add_seq (&new_body, ilist);
8714 378 : gimple_seq_add_stmt (&new_body, stmt);
8715 378 : gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ());
8716 378 : gimple_seq_add_stmt (&new_body, bind);
8717 :
8718 378 : t = gimple_build_omp_continue (control, control);
8719 378 : gimple_seq_add_stmt (&new_body, t);
8720 :
8721 378 : gimple_seq_add_seq (&new_body, olist);
8722 378 : if (ctx->cancellable)
8723 19 : gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
8724 378 : gimple_seq_add_seq (&new_body, dlist);
8725 :
8726 378 : new_body = maybe_catch_exception (new_body);
8727 :
8728 378 : bool nowait = omp_find_clause (gimple_omp_sections_clauses (stmt),
8729 378 : OMP_CLAUSE_NOWAIT) != NULL_TREE;
8730 378 : t = gimple_build_omp_return (nowait);
8731 378 : gimple_seq_add_stmt (&new_body, t);
8732 378 : gimple_seq_add_seq (&new_body, tred_dlist);
8733 378 : maybe_add_implicit_barrier_cancel (ctx, t, &new_body);
8734 :
8735 378 : if (rclauses)
8736 8 : OMP_CLAUSE_DECL (rclauses) = rtmp;
8737 :
8738 378 : gimple_bind_set_body (new_stmt, new_body);
8739 378 : }
8740 :
8741 :
8742 : /* A subroutine of lower_omp_single. Expand the simple form of
8743 : a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8744 :
8745 : if (GOMP_single_start ())
8746 : BODY;
8747 : [ GOMP_barrier (); ] -> unless 'nowait' is present.
8748 :
8749 : FIXME. It may be better to delay expanding the logic of this until
8750 : pass_expand_omp. The expanded logic may make the job more difficult
8751 : to a synchronization analysis pass. */
8752 :
8753 : static void
8754 1004 : lower_omp_single_simple (gomp_single *single_stmt, gimple_seq *pre_p)
8755 : {
8756 1004 : location_t loc = gimple_location (single_stmt);
8757 1004 : tree tlabel = create_artificial_label (loc);
8758 1004 : tree flabel = create_artificial_label (loc);
8759 1004 : gimple *call, *cond;
8760 1004 : tree lhs, decl;
8761 :
8762 1004 : decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START);
8763 1004 : lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)));
8764 1004 : call = gimple_build_call (decl, 0);
8765 1004 : gimple_call_set_lhs (call, lhs);
8766 1004 : gimple_seq_add_stmt (pre_p, call);
8767 :
8768 1004 : cond = gimple_build_cond (EQ_EXPR, lhs,
8769 1004 : fold_convert_loc (loc, TREE_TYPE (lhs),
8770 : boolean_true_node),
8771 : tlabel, flabel);
8772 1004 : gimple_seq_add_stmt (pre_p, cond);
8773 1004 : gimple_seq_add_stmt (pre_p, gimple_build_label (tlabel));
8774 1004 : gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
8775 1004 : gimple_seq_add_stmt (pre_p, gimple_build_label (flabel));
8776 1004 : }
8777 :
8778 :
8779 : /* A subroutine of lower_omp_single. Expand the simple form of
8780 : a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8781 :
8782 : #pragma omp single copyprivate (a, b, c)
8783 :
8784 : Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
8785 :
8786 : {
8787 : if ((copyout_p = GOMP_single_copy_start ()) == NULL)
8788 : {
8789 : BODY;
8790 : copyout.a = a;
8791 : copyout.b = b;
8792 : copyout.c = c;
8793 : GOMP_single_copy_end (©out);
8794 : }
8795 : else
8796 : {
8797 : a = copyout_p->a;
8798 : b = copyout_p->b;
8799 : c = copyout_p->c;
8800 : }
8801 : GOMP_barrier ();
8802 : }
8803 :
8804 : FIXME. It may be better to delay expanding the logic of this until
8805 : pass_expand_omp. The expanded logic may make the job more difficult
8806 : to a synchronization analysis pass. */
8807 :
8808 : static void
8809 115 : lower_omp_single_copy (gomp_single *single_stmt, gimple_seq *pre_p,
8810 : omp_context *ctx)
8811 : {
8812 115 : tree ptr_type, t, l0, l1, l2, bfn_decl;
8813 115 : gimple_seq copyin_seq;
8814 115 : location_t loc = gimple_location (single_stmt);
8815 :
8816 115 : ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o");
8817 :
8818 115 : ptr_type = build_pointer_type (ctx->record_type);
8819 115 : ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i");
8820 :
8821 115 : l0 = create_artificial_label (loc);
8822 115 : l1 = create_artificial_label (loc);
8823 115 : l2 = create_artificial_label (loc);
8824 :
8825 115 : bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START);
8826 115 : t = build_call_expr_loc (loc, bfn_decl, 0);
8827 115 : t = fold_convert_loc (loc, ptr_type, t);
8828 115 : gimplify_assign (ctx->receiver_decl, t, pre_p);
8829 :
8830 115 : t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl,
8831 : build_int_cst (ptr_type, 0));
8832 115 : t = build3 (COND_EXPR, void_type_node, t,
8833 : build_and_jump (&l0), build_and_jump (&l1));
8834 115 : gimplify_and_add (t, pre_p);
8835 :
8836 115 : gimple_seq_add_stmt (pre_p, gimple_build_label (l0));
8837 :
8838 115 : gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
8839 :
8840 115 : copyin_seq = NULL;
8841 115 : lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt), pre_p,
8842 : ©in_seq, ctx);
8843 :
8844 115 : t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
8845 115 : bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END);
8846 115 : t = build_call_expr_loc (loc, bfn_decl, 1, t);
8847 115 : gimplify_and_add (t, pre_p);
8848 :
8849 115 : t = build_and_jump (&l2);
8850 115 : gimplify_and_add (t, pre_p);
8851 :
8852 115 : gimple_seq_add_stmt (pre_p, gimple_build_label (l1));
8853 :
8854 115 : gimple_seq_add_seq (pre_p, copyin_seq);
8855 :
8856 115 : gimple_seq_add_stmt (pre_p, gimple_build_label (l2));
8857 115 : }
8858 :
8859 :
8860 : /* Expand code for an OpenMP single directive. */
8861 :
8862 : static void
8863 1119 : lower_omp_single (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8864 : {
8865 1119 : tree block;
8866 1119 : gomp_single *single_stmt = as_a <gomp_single *> (gsi_stmt (*gsi_p));
8867 1119 : gbind *bind;
8868 1119 : gimple_seq bind_body, bind_body_tail = NULL, dlist;
8869 :
8870 1119 : push_gimplify_context ();
8871 :
8872 1119 : block = make_node (BLOCK);
8873 1119 : bind = gimple_build_bind (NULL, NULL, block);
8874 1119 : gsi_replace (gsi_p, bind, true);
8875 1119 : bind_body = NULL;
8876 1119 : dlist = NULL;
8877 1119 : lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt),
8878 : &bind_body, &dlist, ctx, NULL);
8879 1119 : lower_omp (gimple_omp_body_ptr (single_stmt), ctx);
8880 :
8881 1119 : gimple_seq_add_stmt (&bind_body, single_stmt);
8882 :
8883 1119 : if (ctx->record_type)
8884 115 : lower_omp_single_copy (single_stmt, &bind_body, ctx);
8885 : else
8886 1004 : lower_omp_single_simple (single_stmt, &bind_body);
8887 :
8888 1119 : gimple_omp_set_body (single_stmt, NULL);
8889 :
8890 1119 : gimple_seq_add_seq (&bind_body, dlist);
8891 :
8892 1119 : bind_body = maybe_catch_exception (bind_body);
8893 :
8894 1119 : bool nowait = omp_find_clause (gimple_omp_single_clauses (single_stmt),
8895 1119 : OMP_CLAUSE_NOWAIT) != NULL_TREE;
8896 1119 : gimple *g = gimple_build_omp_return (nowait);
8897 1119 : gimple_seq_add_stmt (&bind_body_tail, g);
8898 1119 : maybe_add_implicit_barrier_cancel (ctx, g, &bind_body_tail);
8899 1119 : if (ctx->record_type)
8900 : {
8901 115 : gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
8902 115 : tree clobber = build_clobber (ctx->record_type);
8903 115 : gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
8904 : clobber), GSI_SAME_STMT);
8905 : }
8906 1119 : gimple_seq_add_seq (&bind_body, bind_body_tail);
8907 1119 : gimple_bind_set_body (bind, bind_body);
8908 :
8909 1119 : pop_gimplify_context (bind);
8910 :
8911 1119 : gimple_bind_append_vars (bind, ctx->block_vars);
8912 1119 : BLOCK_VARS (block) = ctx->block_vars;
8913 1119 : if (BLOCK_VARS (block))
8914 26 : TREE_USED (block) = 1;
8915 1119 : }
8916 :
8917 :
8918 : /* Lower code for an OMP scope directive. */
8919 :
8920 : static void
8921 133 : lower_omp_scope (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8922 : {
8923 133 : tree block;
8924 133 : gimple *scope_stmt = gsi_stmt (*gsi_p);
8925 133 : gbind *bind;
8926 133 : gimple_seq bind_body, bind_body_tail = NULL, dlist;
8927 133 : gimple_seq tred_dlist = NULL;
8928 :
8929 133 : push_gimplify_context ();
8930 :
8931 133 : block = make_node (BLOCK);
8932 133 : bind = gimple_build_bind (NULL, NULL, block);
8933 133 : gsi_replace (gsi_p, bind, true);
8934 133 : bind_body = NULL;
8935 133 : dlist = NULL;
8936 :
8937 133 : tree rclauses
8938 133 : = omp_task_reductions_find_first (gimple_omp_scope_clauses (scope_stmt),
8939 : OMP_SCOPE, OMP_CLAUSE_REDUCTION);
8940 133 : if (rclauses)
8941 : {
8942 46 : tree type = build_pointer_type (pointer_sized_int_node);
8943 46 : tree temp = create_tmp_var (type);
8944 46 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
8945 46 : OMP_CLAUSE_DECL (c) = temp;
8946 46 : OMP_CLAUSE_CHAIN (c) = gimple_omp_scope_clauses (scope_stmt);
8947 46 : gimple_omp_scope_set_clauses (scope_stmt, c);
8948 46 : lower_omp_task_reductions (ctx, OMP_SCOPE,
8949 : gimple_omp_scope_clauses (scope_stmt),
8950 : &bind_body, &tred_dlist);
8951 46 : rclauses = c;
8952 46 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_SCOPE_START);
8953 46 : gimple *stmt = gimple_build_call (fndecl, 1, temp);
8954 46 : gimple_seq_add_stmt (&bind_body, stmt);
8955 : }
8956 :
8957 133 : lower_rec_input_clauses (gimple_omp_scope_clauses (scope_stmt),
8958 : &bind_body, &dlist, ctx, NULL);
8959 133 : lower_omp (gimple_omp_body_ptr (scope_stmt), ctx);
8960 :
8961 133 : gimple_seq_add_stmt (&bind_body, scope_stmt);
8962 :
8963 133 : gimple_seq_add_seq (&bind_body, gimple_omp_body (scope_stmt));
8964 :
8965 133 : gimple_omp_set_body (scope_stmt, NULL);
8966 :
8967 133 : gimple_seq clist = NULL;
8968 133 : lower_reduction_clauses (gimple_omp_scope_clauses (scope_stmt),
8969 : &bind_body, &clist, ctx);
8970 133 : if (clist)
8971 : {
8972 0 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START);
8973 0 : gcall *g = gimple_build_call (fndecl, 0);
8974 0 : gimple_seq_add_stmt (&bind_body, g);
8975 0 : gimple_seq_add_seq (&bind_body, clist);
8976 0 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END);
8977 0 : g = gimple_build_call (fndecl, 0);
8978 0 : gimple_seq_add_stmt (&bind_body, g);
8979 : }
8980 :
8981 133 : gimple_seq_add_seq (&bind_body, dlist);
8982 :
8983 133 : bind_body = maybe_catch_exception (bind_body);
8984 :
8985 133 : bool nowait = omp_find_clause (gimple_omp_scope_clauses (scope_stmt),
8986 133 : OMP_CLAUSE_NOWAIT) != NULL_TREE;
8987 133 : gimple *g = gimple_build_omp_return (nowait);
8988 133 : gimple_seq_add_stmt (&bind_body_tail, g);
8989 133 : gimple_seq_add_seq (&bind_body_tail, tred_dlist);
8990 133 : maybe_add_implicit_barrier_cancel (ctx, g, &bind_body_tail);
8991 133 : if (ctx->record_type)
8992 : {
8993 0 : gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
8994 0 : tree clobber = build_clobber (ctx->record_type);
8995 0 : gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
8996 : clobber), GSI_SAME_STMT);
8997 : }
8998 133 : gimple_seq_add_seq (&bind_body, bind_body_tail);
8999 :
9000 133 : gimple_bind_set_body (bind, bind_body);
9001 :
9002 133 : pop_gimplify_context (bind);
9003 :
9004 133 : gimple_bind_append_vars (bind, ctx->block_vars);
9005 133 : BLOCK_VARS (block) = ctx->block_vars;
9006 133 : if (BLOCK_VARS (block))
9007 105 : TREE_USED (block) = 1;
9008 133 : }
9009 :
9010 : /* Lower code for an OMP dispatch directive. */
9011 :
9012 : static void
9013 659 : lower_omp_dispatch (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9014 : {
9015 659 : tree block;
9016 659 : gimple *stmt = gsi_stmt (*gsi_p);
9017 659 : gbind *bind;
9018 :
9019 659 : push_gimplify_context ();
9020 :
9021 659 : block = make_node (BLOCK);
9022 659 : bind = gimple_build_bind (NULL, NULL, block);
9023 659 : gsi_replace (gsi_p, bind, true);
9024 :
9025 659 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
9026 659 : gimple_bind_set_body (bind, maybe_catch_exception (gimple_omp_body (stmt)));
9027 :
9028 659 : pop_gimplify_context (bind);
9029 :
9030 659 : gimple_bind_append_vars (bind, ctx->block_vars);
9031 659 : BLOCK_VARS (block) = ctx->block_vars;
9032 659 : }
9033 :
9034 : /* Expand code for an OpenMP master or masked directive. */
9035 :
9036 : static void
9037 1114 : lower_omp_master (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9038 : {
9039 1114 : tree block, lab = NULL, x, bfn_decl;
9040 1114 : gimple *stmt = gsi_stmt (*gsi_p);
9041 1114 : gbind *bind;
9042 1114 : location_t loc = gimple_location (stmt);
9043 1114 : gimple_seq tseq;
9044 1114 : tree filter = integer_zero_node;
9045 :
9046 1114 : push_gimplify_context ();
9047 :
9048 1114 : if (gimple_code (stmt) == GIMPLE_OMP_MASKED)
9049 : {
9050 383 : filter = omp_find_clause (gimple_omp_masked_clauses (stmt),
9051 : OMP_CLAUSE_FILTER);
9052 383 : if (filter)
9053 252 : filter = fold_convert (integer_type_node,
9054 : OMP_CLAUSE_FILTER_EXPR (filter));
9055 : else
9056 131 : filter = integer_zero_node;
9057 : }
9058 1114 : block = make_node (BLOCK);
9059 1114 : bind = gimple_build_bind (NULL, NULL, block);
9060 1114 : gsi_replace (gsi_p, bind, true);
9061 1114 : gimple_bind_add_stmt (bind, stmt);
9062 :
9063 1114 : bfn_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
9064 1114 : x = build_call_expr_loc (loc, bfn_decl, 0);
9065 1114 : x = build2 (EQ_EXPR, boolean_type_node, x, filter);
9066 1114 : x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab));
9067 1114 : tseq = NULL;
9068 1114 : gimplify_and_add (x, &tseq);
9069 1114 : gimple_bind_add_seq (bind, tseq);
9070 :
9071 1114 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
9072 1114 : gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
9073 1114 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
9074 1114 : gimple_omp_set_body (stmt, NULL);
9075 :
9076 1114 : gimple_bind_add_stmt (bind, gimple_build_label (lab));
9077 :
9078 1114 : gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
9079 :
9080 1114 : pop_gimplify_context (bind);
9081 :
9082 1114 : gimple_bind_append_vars (bind, ctx->block_vars);
9083 1114 : BLOCK_VARS (block) = ctx->block_vars;
9084 1114 : }
9085 :
9086 : /* Helper function for lower_omp_task_reductions. For a specific PASS
9087 : find out the current clause it should be processed, or return false
9088 : if all have been processed already. */
9089 :
9090 : static inline bool
9091 7952 : omp_task_reduction_iterate (int pass, enum tree_code code,
9092 : enum omp_clause_code ccode, tree *c, tree *decl,
9093 : tree *type, tree *next)
9094 : {
9095 11300 : for (; *c; *c = omp_find_clause (OMP_CLAUSE_CHAIN (*c), ccode))
9096 : {
9097 6696 : if (ccode == OMP_CLAUSE_REDUCTION
9098 6640 : && code != OMP_TASKLOOP
9099 6640 : && !OMP_CLAUSE_REDUCTION_TASK (*c))
9100 56 : continue;
9101 6584 : *decl = OMP_CLAUSE_DECL (*c);
9102 6584 : *type = TREE_TYPE (*decl);
9103 6584 : if (TREE_CODE (*decl) == MEM_REF)
9104 : {
9105 1940 : if (pass != 1)
9106 970 : continue;
9107 : }
9108 : else
9109 : {
9110 4644 : if (omp_privatize_by_reference (*decl))
9111 152 : *type = TREE_TYPE (*type);
9112 4644 : if (pass != (!TREE_CONSTANT (TYPE_SIZE_UNIT (*type))))
9113 2322 : continue;
9114 : }
9115 3292 : *next = omp_find_clause (OMP_CLAUSE_CHAIN (*c), ccode);
9116 3292 : return true;
9117 : }
9118 4660 : *decl = NULL_TREE;
9119 4660 : *type = NULL_TREE;
9120 4660 : *next = NULL_TREE;
9121 4660 : return false;
9122 : }
9123 :
9124 : /* Lower task_reduction and reduction clauses (the latter unless CODE is
9125 : OMP_TASKGROUP only with task modifier). Register mapping of those in
9126 : START sequence and reducing them and unregister them in the END sequence. */
9127 :
9128 : static void
9129 1374 : lower_omp_task_reductions (omp_context *ctx, enum tree_code code, tree clauses,
9130 : gimple_seq *start, gimple_seq *end)
9131 : {
9132 838 : enum omp_clause_code ccode
9133 : = (code == OMP_TASKGROUP
9134 1374 : ? OMP_CLAUSE_TASK_REDUCTION : OMP_CLAUSE_REDUCTION);
9135 1374 : tree cancellable = NULL_TREE;
9136 1374 : clauses = omp_task_reductions_find_first (clauses, code, ccode);
9137 1374 : if (clauses == NULL_TREE)
9138 209 : return;
9139 1165 : if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE)
9140 : {
9141 316 : for (omp_context *outer = ctx->outer; outer; outer = outer->outer)
9142 238 : if (gimple_code (outer->stmt) == GIMPLE_OMP_PARALLEL
9143 238 : && outer->cancellable)
9144 : {
9145 14 : cancellable = error_mark_node;
9146 14 : break;
9147 : }
9148 224 : else if (gimple_code (outer->stmt) != GIMPLE_OMP_TASKGROUP
9149 224 : && gimple_code (outer->stmt) != GIMPLE_OMP_SCOPE)
9150 : break;
9151 : }
9152 1165 : tree record_type = lang_hooks.types.make_type (RECORD_TYPE);
9153 1165 : tree *last = &TYPE_FIELDS (record_type);
9154 1165 : unsigned cnt = 0;
9155 1165 : if (cancellable)
9156 : {
9157 14 : tree field = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE,
9158 : ptr_type_node);
9159 14 : tree ifield = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE,
9160 : integer_type_node);
9161 14 : *last = field;
9162 14 : DECL_CHAIN (field) = ifield;
9163 14 : last = &DECL_CHAIN (ifield);
9164 14 : DECL_CONTEXT (field) = record_type;
9165 14 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (field))
9166 0 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (field));
9167 14 : DECL_CONTEXT (ifield) = record_type;
9168 14 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (ifield))
9169 0 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (ifield));
9170 : }
9171 3495 : for (int pass = 0; pass < 2; pass++)
9172 : {
9173 2330 : tree decl, type, next;
9174 2330 : for (tree c = clauses;
9175 3976 : omp_task_reduction_iterate (pass, code, ccode,
9176 1646 : &c, &decl, &type, &next); c = next)
9177 : {
9178 1646 : ++cnt;
9179 1646 : tree new_type = type;
9180 1646 : if (ctx->outer)
9181 1165 : new_type = remap_type (type, &ctx->outer->cb);
9182 1646 : tree field
9183 1646 : = build_decl (OMP_CLAUSE_LOCATION (c), FIELD_DECL,
9184 1646 : DECL_P (decl) ? DECL_NAME (decl) : NULL_TREE,
9185 : new_type);
9186 1646 : if (DECL_P (decl) && type == TREE_TYPE (decl))
9187 : {
9188 1123 : SET_DECL_ALIGN (field, DECL_ALIGN (decl));
9189 1123 : DECL_USER_ALIGN (field) = DECL_USER_ALIGN (decl);
9190 1123 : TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (decl);
9191 : }
9192 : else
9193 523 : SET_DECL_ALIGN (field, TYPE_ALIGN (type));
9194 1646 : DECL_CONTEXT (field) = record_type;
9195 1646 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (field))
9196 1187 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (field));
9197 1646 : *last = field;
9198 1646 : last = &DECL_CHAIN (field);
9199 1646 : tree bfield
9200 1646 : = build_decl (OMP_CLAUSE_LOCATION (c), FIELD_DECL, NULL_TREE,
9201 : boolean_type_node);
9202 1646 : DECL_CONTEXT (bfield) = record_type;
9203 1646 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (bfield))
9204 0 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (bfield));
9205 1646 : *last = bfield;
9206 1646 : last = &DECL_CHAIN (bfield);
9207 : }
9208 : }
9209 1165 : *last = NULL_TREE;
9210 1165 : layout_type (record_type);
9211 :
9212 : /* Build up an array which registers with the runtime all the reductions
9213 : and deregisters them at the end. Format documented in libgomp/task.c. */
9214 1165 : tree atype = build_array_type_nelts (pointer_sized_int_node, 7 + cnt * 3);
9215 1165 : tree avar = create_tmp_var_raw (atype);
9216 1165 : gimple_add_tmp_var (avar);
9217 1165 : TREE_ADDRESSABLE (avar) = 1;
9218 1165 : tree r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_zero_node,
9219 : NULL_TREE, NULL_TREE);
9220 1165 : tree t = build_int_cst (pointer_sized_int_node, cnt);
9221 1165 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9222 1165 : gimple_seq seq = NULL;
9223 1165 : tree sz = fold_convert (pointer_sized_int_node,
9224 : TYPE_SIZE_UNIT (record_type));
9225 1165 : int cachesz = 64;
9226 1165 : sz = fold_build2 (PLUS_EXPR, pointer_sized_int_node, sz,
9227 : build_int_cst (pointer_sized_int_node, cachesz - 1));
9228 1165 : sz = fold_build2 (BIT_AND_EXPR, pointer_sized_int_node, sz,
9229 : build_int_cst (pointer_sized_int_node, ~(cachesz - 1)));
9230 1165 : ctx->task_reductions.create (1 + cnt);
9231 1165 : ctx->task_reduction_map = new hash_map<tree, unsigned>;
9232 2330 : ctx->task_reductions.quick_push (TREE_CODE (sz) == INTEGER_CST
9233 1205 : ? sz : NULL_TREE);
9234 1165 : sz = force_gimple_operand (sz, &seq, true, NULL_TREE);
9235 1165 : gimple_seq_add_seq (start, seq);
9236 1165 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_one_node,
9237 : NULL_TREE, NULL_TREE);
9238 1165 : gimple_seq_add_stmt (start, gimple_build_assign (r, sz));
9239 1165 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (2),
9240 : NULL_TREE, NULL_TREE);
9241 1165 : t = build_int_cst (pointer_sized_int_node,
9242 1165 : MAX (TYPE_ALIGN_UNIT (record_type), (unsigned) cachesz));
9243 1165 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9244 1165 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (3),
9245 : NULL_TREE, NULL_TREE);
9246 1165 : t = build_int_cst (pointer_sized_int_node, -1);
9247 1165 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9248 1165 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (4),
9249 : NULL_TREE, NULL_TREE);
9250 1165 : t = build_int_cst (pointer_sized_int_node, 0);
9251 1165 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9252 :
9253 : /* In end, build a loop that iterates from 0 to < omp_get_num_threads ()
9254 : and for each task reduction checks a bool right after the private variable
9255 : within that thread's chunk; if the bool is clear, it hasn't been
9256 : initialized and thus isn't going to be reduced nor destructed, otherwise
9257 : reduce and destruct it. */
9258 1165 : tree idx = create_tmp_var (size_type_node);
9259 1165 : gimple_seq_add_stmt (end, gimple_build_assign (idx, size_zero_node));
9260 1165 : tree num_thr_sz = create_tmp_var (size_type_node);
9261 1165 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
9262 1165 : tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
9263 1165 : tree lab3 = NULL_TREE, lab7 = NULL_TREE;
9264 1165 : gimple *g;
9265 1165 : if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE)
9266 : {
9267 : /* For worksharing constructs or scope, only perform it in the master
9268 : thread, with the exception of cancelled implicit barriers - then only
9269 : handle the current thread. */
9270 281 : tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
9271 281 : t = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
9272 281 : tree thr_num = create_tmp_var (integer_type_node);
9273 281 : g = gimple_build_call (t, 0);
9274 281 : gimple_call_set_lhs (g, thr_num);
9275 281 : gimple_seq_add_stmt (end, g);
9276 281 : if (cancellable)
9277 : {
9278 14 : tree c;
9279 14 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9280 14 : tree lab6 = create_artificial_label (UNKNOWN_LOCATION);
9281 14 : lab3 = create_artificial_label (UNKNOWN_LOCATION);
9282 14 : if (code == OMP_FOR)
9283 4 : c = gimple_omp_for_clauses (ctx->stmt);
9284 10 : else if (code == OMP_SECTIONS)
9285 0 : c = gimple_omp_sections_clauses (ctx->stmt);
9286 : else /* if (code == OMP_SCOPE) */
9287 10 : c = gimple_omp_scope_clauses (ctx->stmt);
9288 14 : c = OMP_CLAUSE_DECL (omp_find_clause (c, OMP_CLAUSE__REDUCTEMP_));
9289 14 : cancellable = c;
9290 14 : g = gimple_build_cond (NE_EXPR, c, build_zero_cst (TREE_TYPE (c)),
9291 : lab5, lab6);
9292 14 : gimple_seq_add_stmt (end, g);
9293 14 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9294 14 : g = gimple_build_assign (idx, NOP_EXPR, thr_num);
9295 14 : gimple_seq_add_stmt (end, g);
9296 14 : g = gimple_build_assign (num_thr_sz, PLUS_EXPR, idx,
9297 14 : build_one_cst (TREE_TYPE (idx)));
9298 14 : gimple_seq_add_stmt (end, g);
9299 14 : gimple_seq_add_stmt (end, gimple_build_goto (lab3));
9300 14 : gimple_seq_add_stmt (end, gimple_build_label (lab6));
9301 : }
9302 281 : g = gimple_build_cond (NE_EXPR, thr_num, integer_zero_node, lab2, lab4);
9303 281 : gimple_seq_add_stmt (end, g);
9304 281 : gimple_seq_add_stmt (end, gimple_build_label (lab4));
9305 : }
9306 1165 : if (code != OMP_PARALLEL)
9307 : {
9308 1105 : t = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
9309 1105 : tree num_thr = create_tmp_var (integer_type_node);
9310 1105 : g = gimple_build_call (t, 0);
9311 1105 : gimple_call_set_lhs (g, num_thr);
9312 1105 : gimple_seq_add_stmt (end, g);
9313 1105 : g = gimple_build_assign (num_thr_sz, NOP_EXPR, num_thr);
9314 1105 : gimple_seq_add_stmt (end, g);
9315 1105 : if (cancellable)
9316 14 : gimple_seq_add_stmt (end, gimple_build_label (lab3));
9317 : }
9318 : else
9319 : {
9320 60 : tree c = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
9321 : OMP_CLAUSE__REDUCTEMP_);
9322 60 : t = fold_convert (pointer_sized_int_node, OMP_CLAUSE_DECL (c));
9323 60 : t = fold_convert (size_type_node, t);
9324 60 : gimplify_assign (num_thr_sz, t, end);
9325 : }
9326 1165 : t = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (2),
9327 : NULL_TREE, NULL_TREE);
9328 1165 : tree data = create_tmp_var (pointer_sized_int_node);
9329 1165 : gimple_seq_add_stmt (end, gimple_build_assign (data, t));
9330 1165 : if (code == OMP_TASKLOOP)
9331 : {
9332 497 : lab7 = create_artificial_label (UNKNOWN_LOCATION);
9333 497 : g = gimple_build_cond (NE_EXPR, data,
9334 : build_zero_cst (pointer_sized_int_node),
9335 : lab1, lab7);
9336 497 : gimple_seq_add_stmt (end, g);
9337 : }
9338 1165 : gimple_seq_add_stmt (end, gimple_build_label (lab1));
9339 1165 : tree ptr;
9340 1165 : if (TREE_CODE (TYPE_SIZE_UNIT (record_type)) == INTEGER_CST)
9341 1125 : ptr = create_tmp_var (build_pointer_type (record_type));
9342 : else
9343 40 : ptr = create_tmp_var (ptr_type_node);
9344 1165 : gimple_seq_add_stmt (end, gimple_build_assign (ptr, NOP_EXPR, data));
9345 :
9346 1165 : tree field = TYPE_FIELDS (record_type);
9347 1165 : cnt = 0;
9348 1165 : if (cancellable)
9349 14 : field = DECL_CHAIN (DECL_CHAIN (field));
9350 3495 : for (int pass = 0; pass < 2; pass++)
9351 : {
9352 2330 : tree decl, type, next;
9353 2330 : for (tree c = clauses;
9354 3976 : omp_task_reduction_iterate (pass, code, ccode,
9355 1646 : &c, &decl, &type, &next); c = next)
9356 : {
9357 1646 : tree var = decl, ref;
9358 1646 : if (TREE_CODE (decl) == MEM_REF)
9359 : {
9360 485 : var = TREE_OPERAND (var, 0);
9361 485 : if (TREE_CODE (var) == POINTER_PLUS_EXPR)
9362 108 : var = TREE_OPERAND (var, 0);
9363 485 : tree v = var;
9364 485 : if (TREE_CODE (var) == ADDR_EXPR)
9365 310 : var = TREE_OPERAND (var, 0);
9366 175 : else if (INDIRECT_REF_P (var))
9367 4 : var = TREE_OPERAND (var, 0);
9368 485 : tree orig_var = var;
9369 485 : if (is_variable_sized (var))
9370 : {
9371 31 : gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
9372 31 : var = DECL_VALUE_EXPR (var);
9373 31 : gcc_assert (INDIRECT_REF_P (var));
9374 31 : var = TREE_OPERAND (var, 0);
9375 31 : gcc_assert (DECL_P (var));
9376 : }
9377 485 : t = ref = maybe_lookup_decl_in_outer_ctx (var, ctx);
9378 485 : if (orig_var != var)
9379 31 : gcc_assert (TREE_CODE (v) == ADDR_EXPR);
9380 454 : else if (TREE_CODE (v) == ADDR_EXPR)
9381 279 : t = build_fold_addr_expr (t);
9382 175 : else if (INDIRECT_REF_P (v))
9383 4 : t = build_fold_indirect_ref (t);
9384 485 : if (TREE_CODE (TREE_OPERAND (decl, 0)) == POINTER_PLUS_EXPR)
9385 : {
9386 108 : tree b = TREE_OPERAND (TREE_OPERAND (decl, 0), 1);
9387 108 : b = maybe_lookup_decl_in_outer_ctx (b, ctx);
9388 108 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t, b);
9389 : }
9390 485 : if (!integer_zerop (TREE_OPERAND (decl, 1)))
9391 219 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t,
9392 : fold_convert (size_type_node,
9393 : TREE_OPERAND (decl, 1)));
9394 : }
9395 : else
9396 : {
9397 1161 : t = ref = maybe_lookup_decl_in_outer_ctx (var, ctx);
9398 1161 : if (!omp_privatize_by_reference (decl))
9399 1123 : t = build_fold_addr_expr (t);
9400 : }
9401 1646 : t = fold_convert (pointer_sized_int_node, t);
9402 1646 : seq = NULL;
9403 1646 : t = force_gimple_operand (t, &seq, true, NULL_TREE);
9404 1646 : gimple_seq_add_seq (start, seq);
9405 1646 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar,
9406 1646 : size_int (7 + cnt * 3), NULL_TREE, NULL_TREE);
9407 1646 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9408 1646 : t = unshare_expr (byte_position (field));
9409 1646 : t = fold_convert (pointer_sized_int_node, t);
9410 1646 : ctx->task_reduction_map->put (c, cnt);
9411 3292 : ctx->task_reductions.quick_push (TREE_CODE (t) == INTEGER_CST
9412 2018 : ? t : NULL_TREE);
9413 1646 : seq = NULL;
9414 1646 : t = force_gimple_operand (t, &seq, true, NULL_TREE);
9415 1646 : gimple_seq_add_seq (start, seq);
9416 1646 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar,
9417 1646 : size_int (7 + cnt * 3 + 1), NULL_TREE, NULL_TREE);
9418 1646 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9419 :
9420 1646 : tree bfield = DECL_CHAIN (field);
9421 1646 : tree cond;
9422 1646 : if (code == OMP_PARALLEL
9423 1646 : || code == OMP_FOR
9424 : || code == OMP_SECTIONS
9425 : || code == OMP_SCOPE)
9426 : /* In parallel, worksharing or scope all threads unconditionally
9427 : initialize all their task reduction private variables. */
9428 548 : cond = boolean_true_node;
9429 1098 : else if (TREE_TYPE (ptr) == ptr_type_node)
9430 : {
9431 261 : cond = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr,
9432 : unshare_expr (byte_position (bfield)));
9433 261 : seq = NULL;
9434 261 : cond = force_gimple_operand (cond, &seq, true, NULL_TREE);
9435 261 : gimple_seq_add_seq (end, seq);
9436 261 : tree pbool = build_pointer_type (TREE_TYPE (bfield));
9437 261 : cond = build2 (MEM_REF, TREE_TYPE (bfield), cond,
9438 : build_int_cst (pbool, 0));
9439 : }
9440 : else
9441 837 : cond = build3 (COMPONENT_REF, TREE_TYPE (bfield),
9442 : build_simple_mem_ref (ptr), bfield, NULL_TREE);
9443 1646 : tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
9444 1646 : tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
9445 1646 : tree condv = create_tmp_var (boolean_type_node);
9446 1646 : gimple_seq_add_stmt (end, gimple_build_assign (condv, cond));
9447 1646 : g = gimple_build_cond (NE_EXPR, condv, boolean_false_node,
9448 : lab3, lab4);
9449 1646 : gimple_seq_add_stmt (end, g);
9450 1646 : gimple_seq_add_stmt (end, gimple_build_label (lab3));
9451 1646 : if (cancellable && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) == NULL_TREE)
9452 : {
9453 : /* If this reduction doesn't need destruction and parallel
9454 : has been cancelled, there is nothing to do for this
9455 : reduction, so jump around the merge operation. */
9456 18 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9457 18 : g = gimple_build_cond (NE_EXPR, cancellable,
9458 18 : build_zero_cst (TREE_TYPE (cancellable)),
9459 : lab4, lab5);
9460 18 : gimple_seq_add_stmt (end, g);
9461 18 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9462 : }
9463 :
9464 1646 : tree new_var;
9465 1646 : if (TREE_TYPE (ptr) == ptr_type_node)
9466 : {
9467 428 : new_var = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr,
9468 : unshare_expr (byte_position (field)));
9469 428 : seq = NULL;
9470 428 : new_var = force_gimple_operand (new_var, &seq, true, NULL_TREE);
9471 428 : gimple_seq_add_seq (end, seq);
9472 428 : tree pbool = build_pointer_type (TREE_TYPE (field));
9473 428 : new_var = build2 (MEM_REF, TREE_TYPE (field), new_var,
9474 : build_int_cst (pbool, 0));
9475 : }
9476 : else
9477 1218 : new_var = build3 (COMPONENT_REF, TREE_TYPE (field),
9478 : build_simple_mem_ref (ptr), field, NULL_TREE);
9479 :
9480 1646 : enum tree_code rcode = OMP_CLAUSE_REDUCTION_CODE (c);
9481 1646 : if (TREE_CODE (decl) != MEM_REF
9482 1646 : && omp_privatize_by_reference (decl))
9483 38 : ref = build_simple_mem_ref (ref);
9484 : /* reduction(-:var) sums up the partial results, so it acts
9485 : identically to reduction(+:var). */
9486 1646 : if (rcode == MINUS_EXPR)
9487 0 : rcode = PLUS_EXPR;
9488 1646 : if (TREE_CODE (decl) == MEM_REF)
9489 : {
9490 485 : tree type = TREE_TYPE (new_var);
9491 485 : tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
9492 485 : tree i = create_tmp_var (TREE_TYPE (v));
9493 485 : tree ptype = build_pointer_type (TREE_TYPE (type));
9494 485 : if (DECL_P (v))
9495 : {
9496 142 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
9497 142 : tree vv = create_tmp_var (TREE_TYPE (v));
9498 142 : gimplify_assign (vv, v, start);
9499 142 : v = vv;
9500 : }
9501 485 : ref = build4 (ARRAY_REF, pointer_sized_int_node, avar,
9502 485 : size_int (7 + cnt * 3), NULL_TREE, NULL_TREE);
9503 485 : new_var = build_fold_addr_expr (new_var);
9504 485 : new_var = fold_convert (ptype, new_var);
9505 485 : ref = fold_convert (ptype, ref);
9506 485 : tree m = create_tmp_var (ptype);
9507 485 : gimplify_assign (m, new_var, end);
9508 485 : new_var = m;
9509 485 : m = create_tmp_var (ptype);
9510 485 : gimplify_assign (m, ref, end);
9511 485 : ref = m;
9512 485 : gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), end);
9513 485 : tree body = create_artificial_label (UNKNOWN_LOCATION);
9514 485 : tree endl = create_artificial_label (UNKNOWN_LOCATION);
9515 485 : gimple_seq_add_stmt (end, gimple_build_label (body));
9516 485 : tree priv = build_simple_mem_ref (new_var);
9517 485 : tree out = build_simple_mem_ref (ref);
9518 485 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
9519 : {
9520 161 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
9521 161 : tree decl_placeholder
9522 161 : = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
9523 161 : tree lab6 = NULL_TREE;
9524 161 : if (cancellable)
9525 : {
9526 : /* If this reduction needs destruction and parallel
9527 : has been cancelled, jump around the merge operation
9528 : to the destruction. */
9529 4 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9530 4 : lab6 = create_artificial_label (UNKNOWN_LOCATION);
9531 4 : tree zero = build_zero_cst (TREE_TYPE (cancellable));
9532 4 : g = gimple_build_cond (NE_EXPR, cancellable, zero,
9533 : lab6, lab5);
9534 4 : gimple_seq_add_stmt (end, g);
9535 4 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9536 : }
9537 161 : SET_DECL_VALUE_EXPR (placeholder, out);
9538 161 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
9539 161 : SET_DECL_VALUE_EXPR (decl_placeholder, priv);
9540 161 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
9541 161 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
9542 322 : gimple_seq_add_seq (end,
9543 161 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
9544 161 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
9545 161 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
9546 : {
9547 56 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
9548 56 : OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c) = NULL;
9549 : }
9550 161 : if (cancellable)
9551 4 : gimple_seq_add_stmt (end, gimple_build_label (lab6));
9552 161 : tree x = lang_hooks.decls.omp_clause_dtor (c, priv);
9553 161 : if (x)
9554 : {
9555 137 : gimple_seq tseq = NULL;
9556 137 : gimplify_stmt (&x, &tseq);
9557 137 : gimple_seq_add_seq (end, tseq);
9558 : }
9559 : }
9560 : else
9561 : {
9562 324 : tree x = build2 (rcode, TREE_TYPE (out), out, priv);
9563 324 : out = unshare_expr (out);
9564 324 : gimplify_assign (out, x, end);
9565 : }
9566 485 : gimple *g
9567 485 : = gimple_build_assign (new_var, POINTER_PLUS_EXPR, new_var,
9568 485 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
9569 485 : gimple_seq_add_stmt (end, g);
9570 485 : g = gimple_build_assign (ref, POINTER_PLUS_EXPR, ref,
9571 485 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
9572 485 : gimple_seq_add_stmt (end, g);
9573 485 : g = gimple_build_assign (i, PLUS_EXPR, i,
9574 485 : build_int_cst (TREE_TYPE (i), 1));
9575 485 : gimple_seq_add_stmt (end, g);
9576 485 : g = gimple_build_cond (LE_EXPR, i, v, body, endl);
9577 485 : gimple_seq_add_stmt (end, g);
9578 485 : gimple_seq_add_stmt (end, gimple_build_label (endl));
9579 : }
9580 1161 : else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
9581 : {
9582 127 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
9583 127 : tree oldv = NULL_TREE;
9584 127 : tree lab6 = NULL_TREE;
9585 127 : if (cancellable)
9586 : {
9587 : /* If this reduction needs destruction and parallel
9588 : has been cancelled, jump around the merge operation
9589 : to the destruction. */
9590 4 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9591 4 : lab6 = create_artificial_label (UNKNOWN_LOCATION);
9592 4 : tree zero = build_zero_cst (TREE_TYPE (cancellable));
9593 4 : g = gimple_build_cond (NE_EXPR, cancellable, zero,
9594 : lab6, lab5);
9595 4 : gimple_seq_add_stmt (end, g);
9596 4 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9597 : }
9598 127 : if (omp_privatize_by_reference (decl)
9599 137 : && !useless_type_conversion_p (TREE_TYPE (placeholder),
9600 10 : TREE_TYPE (ref)))
9601 0 : ref = build_fold_addr_expr_loc (OMP_CLAUSE_LOCATION (c), ref);
9602 127 : ref = build_fold_addr_expr_loc (OMP_CLAUSE_LOCATION (c), ref);
9603 127 : tree refv = create_tmp_var (TREE_TYPE (ref));
9604 127 : gimplify_assign (refv, ref, end);
9605 127 : ref = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), refv);
9606 127 : SET_DECL_VALUE_EXPR (placeholder, ref);
9607 127 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
9608 127 : tree d = maybe_lookup_decl (decl, ctx);
9609 127 : gcc_assert (d);
9610 127 : if (DECL_HAS_VALUE_EXPR_P (d))
9611 7 : oldv = DECL_VALUE_EXPR (d);
9612 127 : if (omp_privatize_by_reference (var))
9613 : {
9614 10 : tree v = fold_convert (TREE_TYPE (d),
9615 : build_fold_addr_expr (new_var));
9616 10 : SET_DECL_VALUE_EXPR (d, v);
9617 : }
9618 : else
9619 117 : SET_DECL_VALUE_EXPR (d, new_var);
9620 127 : DECL_HAS_VALUE_EXPR_P (d) = 1;
9621 127 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
9622 127 : if (oldv)
9623 7 : SET_DECL_VALUE_EXPR (d, oldv);
9624 : else
9625 : {
9626 120 : SET_DECL_VALUE_EXPR (d, NULL_TREE);
9627 120 : DECL_HAS_VALUE_EXPR_P (d) = 0;
9628 : }
9629 127 : gimple_seq_add_seq (end, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
9630 127 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
9631 127 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
9632 24 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
9633 127 : if (cancellable)
9634 4 : gimple_seq_add_stmt (end, gimple_build_label (lab6));
9635 127 : tree x = lang_hooks.decls.omp_clause_dtor (c, new_var);
9636 127 : if (x)
9637 : {
9638 29 : gimple_seq tseq = NULL;
9639 29 : gimplify_stmt (&x, &tseq);
9640 29 : gimple_seq_add_seq (end, tseq);
9641 : }
9642 : }
9643 : else
9644 : {
9645 1034 : tree x = build2 (rcode, TREE_TYPE (ref), ref, new_var);
9646 1034 : ref = unshare_expr (ref);
9647 1034 : gimplify_assign (ref, x, end);
9648 : }
9649 1646 : gimple_seq_add_stmt (end, gimple_build_label (lab4));
9650 1646 : ++cnt;
9651 1646 : field = DECL_CHAIN (bfield);
9652 : }
9653 : }
9654 :
9655 1165 : if (code == OMP_TASKGROUP)
9656 : {
9657 327 : t = builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_REDUCTION_REGISTER);
9658 327 : g = gimple_build_call (t, 1, build_fold_addr_expr (avar));
9659 327 : gimple_seq_add_stmt (start, g);
9660 : }
9661 : else
9662 : {
9663 838 : tree c;
9664 838 : if (code == OMP_FOR)
9665 227 : c = gimple_omp_for_clauses (ctx->stmt);
9666 611 : else if (code == OMP_SECTIONS)
9667 8 : c = gimple_omp_sections_clauses (ctx->stmt);
9668 603 : else if (code == OMP_SCOPE)
9669 46 : c = gimple_omp_scope_clauses (ctx->stmt);
9670 : else
9671 557 : c = gimple_omp_taskreg_clauses (ctx->stmt);
9672 838 : c = omp_find_clause (c, OMP_CLAUSE__REDUCTEMP_);
9673 838 : t = fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (c)),
9674 : build_fold_addr_expr (avar));
9675 838 : gimplify_assign (OMP_CLAUSE_DECL (c), t, start);
9676 : }
9677 :
9678 1165 : gimple_seq_add_stmt (end, gimple_build_assign (data, PLUS_EXPR, data, sz));
9679 1165 : gimple_seq_add_stmt (end, gimple_build_assign (idx, PLUS_EXPR, idx,
9680 : size_one_node));
9681 1165 : g = gimple_build_cond (NE_EXPR, idx, num_thr_sz, lab1, lab2);
9682 1165 : gimple_seq_add_stmt (end, g);
9683 1165 : gimple_seq_add_stmt (end, gimple_build_label (lab2));
9684 1165 : if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE)
9685 : {
9686 281 : enum built_in_function bfn
9687 : = BUILT_IN_GOMP_WORKSHARE_TASK_REDUCTION_UNREGISTER;
9688 281 : t = builtin_decl_explicit (bfn);
9689 281 : tree c_bool_type = TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (t)));
9690 281 : tree arg;
9691 281 : if (cancellable)
9692 : {
9693 14 : arg = create_tmp_var (c_bool_type);
9694 14 : gimple_seq_add_stmt (end, gimple_build_assign (arg, NOP_EXPR,
9695 : cancellable));
9696 : }
9697 : else
9698 267 : arg = build_int_cst (c_bool_type, 0);
9699 281 : g = gimple_build_call (t, 1, arg);
9700 281 : }
9701 : else
9702 : {
9703 884 : t = builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_REDUCTION_UNREGISTER);
9704 884 : g = gimple_build_call (t, 1, build_fold_addr_expr (avar));
9705 : }
9706 1165 : gimple_seq_add_stmt (end, g);
9707 1165 : if (lab7)
9708 497 : gimple_seq_add_stmt (end, gimple_build_label (lab7));
9709 1165 : t = build_constructor (atype, NULL);
9710 1165 : TREE_THIS_VOLATILE (t) = 1;
9711 1165 : gimple_seq_add_stmt (end, gimple_build_assign (avar, t));
9712 : }
9713 :
9714 : /* Expand code for an OpenMP taskgroup directive. */
9715 :
9716 : static void
9717 536 : lower_omp_taskgroup (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9718 : {
9719 536 : gimple *stmt = gsi_stmt (*gsi_p);
9720 536 : gcall *x;
9721 536 : gbind *bind;
9722 536 : gimple_seq dseq = NULL;
9723 536 : tree block = make_node (BLOCK);
9724 :
9725 536 : bind = gimple_build_bind (NULL, NULL, block);
9726 536 : gsi_replace (gsi_p, bind, true);
9727 536 : gimple_bind_add_stmt (bind, stmt);
9728 :
9729 536 : push_gimplify_context ();
9730 :
9731 536 : x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START),
9732 : 0);
9733 536 : gimple_bind_add_stmt (bind, x);
9734 :
9735 536 : lower_omp_task_reductions (ctx, OMP_TASKGROUP,
9736 : gimple_omp_taskgroup_clauses (stmt),
9737 : gimple_bind_body_ptr (bind), &dseq);
9738 :
9739 536 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
9740 536 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
9741 536 : gimple_omp_set_body (stmt, NULL);
9742 :
9743 536 : gimple_bind_add_seq (bind, dseq);
9744 :
9745 536 : pop_gimplify_context (bind);
9746 :
9747 536 : gimple_bind_append_vars (bind, ctx->block_vars);
9748 536 : BLOCK_VARS (block) = ctx->block_vars;
9749 536 : }
9750 :
9751 :
9752 : /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */
9753 :
9754 : static void
9755 0 : lower_omp_ordered_clauses (gimple_stmt_iterator *gsi_p, gomp_ordered *ord_stmt,
9756 : omp_context *ctx)
9757 : {
9758 0 : struct omp_for_data fd;
9759 0 : if (!ctx->outer || gimple_code (ctx->outer->stmt) != GIMPLE_OMP_FOR)
9760 0 : return;
9761 :
9762 0 : unsigned int len = gimple_omp_for_collapse (ctx->outer->stmt);
9763 0 : struct omp_for_data_loop *loops = XALLOCAVEC (struct omp_for_data_loop, len);
9764 0 : omp_extract_for_data (as_a <gomp_for *> (ctx->outer->stmt), &fd, loops);
9765 0 : if (!fd.ordered)
9766 : return;
9767 :
9768 0 : tree *list_p = gimple_omp_ordered_clauses_ptr (ord_stmt);
9769 0 : tree c = gimple_omp_ordered_clauses (ord_stmt);
9770 0 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS
9771 0 : && OMP_CLAUSE_DOACROSS_KIND (c) == OMP_CLAUSE_DOACROSS_SINK)
9772 : {
9773 : /* Merge depend clauses from multiple adjacent
9774 : #pragma omp ordered depend(sink:...) constructs
9775 : into one #pragma omp ordered depend(sink:...), so that
9776 : we can optimize them together. */
9777 0 : gimple_stmt_iterator gsi = *gsi_p;
9778 0 : gsi_next (&gsi);
9779 0 : while (!gsi_end_p (gsi))
9780 : {
9781 0 : gimple *stmt = gsi_stmt (gsi);
9782 0 : if (is_gimple_debug (stmt)
9783 0 : || gimple_code (stmt) == GIMPLE_NOP)
9784 : {
9785 0 : gsi_next (&gsi);
9786 0 : continue;
9787 : }
9788 0 : if (gimple_code (stmt) != GIMPLE_OMP_ORDERED)
9789 : break;
9790 0 : gomp_ordered *ord_stmt2 = as_a <gomp_ordered *> (stmt);
9791 0 : c = gimple_omp_ordered_clauses (ord_stmt2);
9792 0 : if (c == NULL_TREE
9793 0 : || OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DOACROSS
9794 0 : || OMP_CLAUSE_DOACROSS_KIND (c) != OMP_CLAUSE_DOACROSS_SINK)
9795 : break;
9796 0 : while (*list_p)
9797 0 : list_p = &OMP_CLAUSE_CHAIN (*list_p);
9798 0 : *list_p = c;
9799 0 : gsi_remove (&gsi, true);
9800 : }
9801 : }
9802 :
9803 : /* Canonicalize sink dependence clauses into one folded clause if
9804 : possible.
9805 :
9806 : The basic algorithm is to create a sink vector whose first
9807 : element is the GCD of all the first elements, and whose remaining
9808 : elements are the minimum of the subsequent columns.
9809 :
9810 : We ignore dependence vectors whose first element is zero because
9811 : such dependencies are known to be executed by the same thread.
9812 :
9813 : We take into account the direction of the loop, so a minimum
9814 : becomes a maximum if the loop is iterating forwards. We also
9815 : ignore sink clauses where the loop direction is unknown, or where
9816 : the offsets are clearly invalid because they are not a multiple
9817 : of the loop increment.
9818 :
9819 : For example:
9820 :
9821 : #pragma omp for ordered(2)
9822 : for (i=0; i < N; ++i)
9823 : for (j=0; j < M; ++j)
9824 : {
9825 : #pragma omp ordered \
9826 : depend(sink:i-8,j-2) \
9827 : depend(sink:i,j-1) \ // Completely ignored because i+0.
9828 : depend(sink:i-4,j-3) \
9829 : depend(sink:i-6,j-4)
9830 : #pragma omp ordered depend(source)
9831 : }
9832 :
9833 : Folded clause is:
9834 :
9835 : depend(sink:-gcd(8,4,6),-min(2,3,4))
9836 : -or-
9837 : depend(sink:-2,-2)
9838 : */
9839 :
9840 : /* FIXME: Computing GCD's where the first element is zero is
9841 : non-trivial in the presence of collapsed loops. Do this later. */
9842 0 : if (fd.collapse > 1)
9843 : return;
9844 :
9845 0 : wide_int *folded_deps = XALLOCAVEC (wide_int, 2 * len - 1);
9846 :
9847 : /* wide_int is not a POD so it must be default-constructed. */
9848 0 : for (unsigned i = 0; i != 2 * len - 1; ++i)
9849 0 : new (static_cast<void*>(folded_deps + i)) wide_int ();
9850 :
9851 : tree folded_dep = NULL_TREE;
9852 : /* TRUE if the first dimension's offset is negative. */
9853 : bool neg_offset_p = false;
9854 :
9855 : list_p = gimple_omp_ordered_clauses_ptr (ord_stmt);
9856 : unsigned int i;
9857 0 : while ((c = *list_p) != NULL)
9858 : {
9859 0 : bool remove = false;
9860 :
9861 0 : gcc_assert (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS);
9862 0 : if (OMP_CLAUSE_DOACROSS_KIND (c) != OMP_CLAUSE_DOACROSS_SINK)
9863 0 : goto next_ordered_clause;
9864 :
9865 0 : tree vec;
9866 0 : for (vec = OMP_CLAUSE_DECL (c), i = 0;
9867 0 : vec && TREE_CODE (vec) == TREE_LIST;
9868 0 : vec = TREE_CHAIN (vec), ++i)
9869 : {
9870 0 : gcc_assert (i < len);
9871 :
9872 : /* omp_extract_for_data has canonicalized the condition. */
9873 0 : gcc_assert (fd.loops[i].cond_code == LT_EXPR
9874 : || fd.loops[i].cond_code == GT_EXPR);
9875 0 : bool forward = fd.loops[i].cond_code == LT_EXPR;
9876 0 : bool maybe_lexically_later = true;
9877 :
9878 : /* While the committee makes up its mind, bail if we have any
9879 : non-constant steps. */
9880 0 : if (TREE_CODE (fd.loops[i].step) != INTEGER_CST)
9881 0 : goto lower_omp_ordered_ret;
9882 :
9883 0 : tree itype = TREE_TYPE (TREE_VALUE (vec));
9884 0 : if (POINTER_TYPE_P (itype))
9885 0 : itype = sizetype;
9886 0 : wide_int offset = wide_int::from (wi::to_wide (TREE_PURPOSE (vec)),
9887 0 : TYPE_PRECISION (itype),
9888 0 : TYPE_SIGN (itype));
9889 :
9890 : /* Ignore invalid offsets that are not multiples of the step. */
9891 0 : if (!wi::multiple_of_p (wi::abs (offset),
9892 0 : wi::abs (wi::to_wide (fd.loops[i].step)),
9893 : UNSIGNED))
9894 : {
9895 0 : warning_at (OMP_CLAUSE_LOCATION (c), OPT_Wopenmp,
9896 : "ignoring %<sink%> clause with offset that is not "
9897 : "a multiple of the loop step");
9898 0 : remove = true;
9899 0 : goto next_ordered_clause;
9900 : }
9901 :
9902 : /* Calculate the first dimension. The first dimension of
9903 : the folded dependency vector is the GCD of the first
9904 : elements, while ignoring any first elements whose offset
9905 : is 0. */
9906 0 : if (i == 0)
9907 : {
9908 : /* Ignore dependence vectors whose first dimension is 0. */
9909 0 : if (offset == 0)
9910 : {
9911 0 : remove = true;
9912 0 : goto next_ordered_clause;
9913 : }
9914 : else
9915 : {
9916 0 : if (!TYPE_UNSIGNED (itype) && (forward ^ wi::neg_p (offset)))
9917 : {
9918 0 : error_at (OMP_CLAUSE_LOCATION (c),
9919 : "first offset must be in opposite direction "
9920 : "of loop iterations");
9921 0 : goto lower_omp_ordered_ret;
9922 : }
9923 0 : if (forward)
9924 0 : offset = -offset;
9925 0 : neg_offset_p = forward;
9926 : /* Initialize the first time around. */
9927 0 : if (folded_dep == NULL_TREE)
9928 : {
9929 0 : folded_dep = c;
9930 0 : folded_deps[0] = offset;
9931 : }
9932 : else
9933 0 : folded_deps[0] = wi::gcd (folded_deps[0],
9934 0 : offset, UNSIGNED);
9935 : }
9936 : }
9937 : /* Calculate minimum for the remaining dimensions. */
9938 : else
9939 : {
9940 0 : folded_deps[len + i - 1] = offset;
9941 0 : if (folded_dep == c)
9942 0 : folded_deps[i] = offset;
9943 0 : else if (maybe_lexically_later
9944 0 : && !wi::eq_p (folded_deps[i], offset))
9945 : {
9946 0 : if (forward ^ wi::gts_p (folded_deps[i], offset))
9947 : {
9948 : unsigned int j;
9949 0 : folded_dep = c;
9950 0 : for (j = 1; j <= i; j++)
9951 0 : folded_deps[j] = folded_deps[len + j - 1];
9952 : }
9953 : else
9954 0 : maybe_lexically_later = false;
9955 : }
9956 : }
9957 0 : }
9958 0 : gcc_assert (i == len);
9959 :
9960 : remove = true;
9961 :
9962 0 : next_ordered_clause:
9963 0 : if (remove)
9964 0 : *list_p = OMP_CLAUSE_CHAIN (c);
9965 : else
9966 0 : list_p = &OMP_CLAUSE_CHAIN (c);
9967 : }
9968 :
9969 0 : if (folded_dep)
9970 : {
9971 0 : if (neg_offset_p)
9972 0 : folded_deps[0] = -folded_deps[0];
9973 :
9974 0 : tree itype = TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep)));
9975 0 : if (POINTER_TYPE_P (itype))
9976 0 : itype = sizetype;
9977 :
9978 0 : TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep))
9979 0 : = wide_int_to_tree (itype, folded_deps[0]);
9980 0 : OMP_CLAUSE_CHAIN (folded_dep) = gimple_omp_ordered_clauses (ord_stmt);
9981 0 : *gimple_omp_ordered_clauses_ptr (ord_stmt) = folded_dep;
9982 : }
9983 :
9984 0 : lower_omp_ordered_ret:
9985 :
9986 : /* Ordered without clauses is #pragma omp threads, while we want
9987 : a nop instead if we remove all clauses. */
9988 0 : if (gimple_omp_ordered_clauses (ord_stmt) == NULL_TREE)
9989 0 : gsi_replace (gsi_p, gimple_build_nop (), true);
9990 : }
9991 :
9992 :
9993 : /* Expand code for an OpenMP ordered directive. */
9994 :
9995 : static void
9996 1124 : lower_omp_ordered (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9997 : {
9998 1124 : tree block;
9999 1124 : gimple *stmt = gsi_stmt (*gsi_p), *g;
10000 1124 : gomp_ordered *ord_stmt = as_a <gomp_ordered *> (stmt);
10001 1124 : gcall *x;
10002 1124 : gbind *bind;
10003 1124 : bool simd = omp_find_clause (gimple_omp_ordered_clauses (ord_stmt),
10004 1124 : OMP_CLAUSE_SIMD);
10005 : /* FIXME: this should check presence of OMP_CLAUSE__SIMT_ on the enclosing
10006 : loop. */
10007 1124 : bool maybe_simt
10008 1124 : = simd && omp_maybe_offloaded_ctx (ctx) && omp_max_simt_vf () > 1;
10009 1124 : bool threads = omp_find_clause (gimple_omp_ordered_clauses (ord_stmt),
10010 1124 : OMP_CLAUSE_THREADS);
10011 :
10012 1124 : if (gimple_omp_ordered_standalone_p (ord_stmt))
10013 : {
10014 : /* FIXME: This is needs to be moved to the expansion to verify various
10015 : conditions only testable on cfg with dominators computed, and also
10016 : all the depend clauses to be merged still might need to be available
10017 : for the runtime checks. */
10018 : if (0)
10019 : lower_omp_ordered_clauses (gsi_p, ord_stmt, ctx);
10020 1124 : return;
10021 : }
10022 :
10023 411 : push_gimplify_context ();
10024 :
10025 411 : block = make_node (BLOCK);
10026 411 : bind = gimple_build_bind (NULL, NULL, block);
10027 411 : gsi_replace (gsi_p, bind, true);
10028 411 : gimple_bind_add_stmt (bind, stmt);
10029 :
10030 411 : if (simd)
10031 : {
10032 79 : x = gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START, 1,
10033 79 : build_int_cst (NULL_TREE, threads));
10034 79 : cfun->has_simduid_loops = true;
10035 : }
10036 : else
10037 332 : x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START),
10038 : 0);
10039 411 : gimple_bind_add_stmt (bind, x);
10040 :
10041 411 : tree counter = NULL_TREE, test = NULL_TREE, body = NULL_TREE;
10042 411 : if (maybe_simt)
10043 : {
10044 0 : counter = create_tmp_var (integer_type_node);
10045 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_LANE, 0);
10046 0 : gimple_call_set_lhs (g, counter);
10047 0 : gimple_bind_add_stmt (bind, g);
10048 :
10049 0 : body = create_artificial_label (UNKNOWN_LOCATION);
10050 0 : test = create_artificial_label (UNKNOWN_LOCATION);
10051 0 : gimple_bind_add_stmt (bind, gimple_build_label (body));
10052 :
10053 0 : tree simt_pred = create_tmp_var (integer_type_node);
10054 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_ORDERED_PRED, 1, counter);
10055 0 : gimple_call_set_lhs (g, simt_pred);
10056 0 : gimple_bind_add_stmt (bind, g);
10057 :
10058 0 : tree t = create_artificial_label (UNKNOWN_LOCATION);
10059 0 : g = gimple_build_cond (EQ_EXPR, simt_pred, integer_zero_node, t, test);
10060 0 : gimple_bind_add_stmt (bind, g);
10061 :
10062 0 : gimple_bind_add_stmt (bind, gimple_build_label (t));
10063 : }
10064 411 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
10065 411 : gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
10066 411 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10067 411 : gimple_omp_set_body (stmt, NULL);
10068 :
10069 411 : if (maybe_simt)
10070 : {
10071 0 : gimple_bind_add_stmt (bind, gimple_build_label (test));
10072 0 : g = gimple_build_assign (counter, MINUS_EXPR, counter, integer_one_node);
10073 0 : gimple_bind_add_stmt (bind, g);
10074 :
10075 0 : tree c = build2 (GE_EXPR, boolean_type_node, counter, integer_zero_node);
10076 0 : tree nonneg = create_tmp_var (integer_type_node);
10077 0 : gimple_seq tseq = NULL;
10078 0 : gimplify_assign (nonneg, fold_convert (integer_type_node, c), &tseq);
10079 0 : gimple_bind_add_seq (bind, tseq);
10080 :
10081 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_VOTE_ANY, 1, nonneg);
10082 0 : gimple_call_set_lhs (g, nonneg);
10083 0 : gimple_bind_add_stmt (bind, g);
10084 :
10085 0 : tree end = create_artificial_label (UNKNOWN_LOCATION);
10086 0 : g = gimple_build_cond (NE_EXPR, nonneg, integer_zero_node, body, end);
10087 0 : gimple_bind_add_stmt (bind, g);
10088 :
10089 0 : gimple_bind_add_stmt (bind, gimple_build_label (end));
10090 : }
10091 411 : if (simd)
10092 79 : x = gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END, 1,
10093 79 : build_int_cst (NULL_TREE, threads));
10094 : else
10095 332 : x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END),
10096 : 0);
10097 411 : gimple_bind_add_stmt (bind, x);
10098 :
10099 411 : gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10100 :
10101 411 : pop_gimplify_context (bind);
10102 :
10103 411 : gimple_bind_append_vars (bind, ctx->block_vars);
10104 411 : BLOCK_VARS (block) = gimple_bind_vars (bind);
10105 : }
10106 :
10107 :
10108 : /* Expand code for an OpenMP scan directive and the structured block
10109 : before the scan directive. */
10110 :
10111 : static void
10112 1536 : lower_omp_scan (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10113 : {
10114 1536 : gimple *stmt = gsi_stmt (*gsi_p);
10115 1536 : bool has_clauses
10116 1536 : = gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt)) != NULL;
10117 1536 : tree lane = NULL_TREE;
10118 1536 : gimple_seq before = NULL;
10119 1536 : omp_context *octx = ctx->outer;
10120 1536 : gcc_assert (octx);
10121 1536 : if (octx->scan_exclusive && !has_clauses)
10122 : {
10123 536 : gimple_stmt_iterator gsi2 = *gsi_p;
10124 536 : gsi_next (&gsi2);
10125 536 : gimple *stmt2 = gsi_stmt (gsi2);
10126 : /* For exclusive scan, swap GIMPLE_OMP_SCAN without clauses
10127 : with following GIMPLE_OMP_SCAN with clauses, so that input_phase,
10128 : the one with exclusive clause(s), comes first. */
10129 536 : if (stmt2
10130 274 : && gimple_code (stmt2) == GIMPLE_OMP_SCAN
10131 804 : && gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt2)) != NULL)
10132 : {
10133 268 : gsi_remove (gsi_p, false);
10134 268 : gsi_insert_after (gsi_p, stmt, GSI_SAME_STMT);
10135 268 : ctx = maybe_lookup_ctx (stmt2);
10136 268 : gcc_assert (ctx);
10137 268 : lower_omp_scan (gsi_p, ctx);
10138 268 : return;
10139 : }
10140 : }
10141 :
10142 1268 : bool input_phase = has_clauses ^ octx->scan_inclusive;
10143 1268 : bool is_simd = (gimple_code (octx->stmt) == GIMPLE_OMP_FOR
10144 1268 : && gimple_omp_for_kind (octx->stmt) == GF_OMP_FOR_KIND_SIMD);
10145 1268 : bool is_for = (gimple_code (octx->stmt) == GIMPLE_OMP_FOR
10146 1268 : && gimple_omp_for_kind (octx->stmt) == GF_OMP_FOR_KIND_FOR
10147 1614 : && !gimple_omp_for_combined_p (octx->stmt));
10148 1268 : bool is_for_simd = is_simd && gimple_omp_for_combined_into_p (octx->stmt);
10149 332 : if (is_for_simd && octx->for_simd_scan_phase)
10150 : is_simd = false;
10151 1102 : if (is_simd)
10152 756 : if (tree c = omp_find_clause (gimple_omp_for_clauses (octx->stmt),
10153 : OMP_CLAUSE__SIMDUID_))
10154 : {
10155 368 : tree uid = OMP_CLAUSE__SIMDUID__DECL (c);
10156 368 : lane = create_tmp_var (unsigned_type_node);
10157 368 : tree t = build_int_cst (integer_type_node,
10158 552 : input_phase ? 1
10159 184 : : octx->scan_inclusive ? 2 : 3);
10160 368 : gimple *g
10161 368 : = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 2, uid, t);
10162 368 : gimple_call_set_lhs (g, lane);
10163 368 : gimple_seq_add_stmt (&before, g);
10164 : }
10165 :
10166 1268 : if (is_simd || is_for)
10167 : {
10168 4980 : for (tree c = gimple_omp_for_clauses (octx->stmt);
10169 4980 : c; c = OMP_CLAUSE_CHAIN (c))
10170 4044 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
10171 4044 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
10172 : {
10173 1160 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
10174 1160 : tree var = OMP_CLAUSE_DECL (c);
10175 1160 : tree new_var = lookup_decl (var, octx);
10176 1160 : tree val = new_var;
10177 1160 : tree var2 = NULL_TREE;
10178 1160 : tree var3 = NULL_TREE;
10179 1160 : tree var4 = NULL_TREE;
10180 1160 : tree lane0 = NULL_TREE;
10181 1160 : tree new_vard = new_var;
10182 1160 : if (omp_privatize_by_reference (var))
10183 : {
10184 184 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
10185 184 : val = new_var;
10186 : }
10187 1160 : if (DECL_HAS_VALUE_EXPR_P (new_vard))
10188 : {
10189 464 : val = DECL_VALUE_EXPR (new_vard);
10190 464 : if (new_vard != new_var)
10191 : {
10192 76 : gcc_assert (TREE_CODE (val) == ADDR_EXPR);
10193 76 : val = TREE_OPERAND (val, 0);
10194 : }
10195 464 : if (TREE_CODE (val) == ARRAY_REF
10196 464 : && VAR_P (TREE_OPERAND (val, 0)))
10197 : {
10198 464 : tree v = TREE_OPERAND (val, 0);
10199 464 : if (lookup_attribute ("omp simd array",
10200 464 : DECL_ATTRIBUTES (v)))
10201 : {
10202 464 : val = unshare_expr (val);
10203 464 : lane0 = TREE_OPERAND (val, 1);
10204 464 : TREE_OPERAND (val, 1) = lane;
10205 464 : var2 = lookup_decl (v, octx);
10206 464 : if (octx->scan_exclusive)
10207 228 : var4 = lookup_decl (var2, octx);
10208 464 : if (input_phase
10209 464 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
10210 120 : var3 = maybe_lookup_decl (var4 ? var4 : var2, octx);
10211 464 : if (!input_phase)
10212 : {
10213 232 : var2 = build4 (ARRAY_REF, TREE_TYPE (val),
10214 : var2, lane, NULL_TREE, NULL_TREE);
10215 232 : TREE_THIS_NOTRAP (var2) = 1;
10216 232 : if (octx->scan_exclusive)
10217 : {
10218 114 : var4 = build4 (ARRAY_REF, TREE_TYPE (val),
10219 : var4, lane, NULL_TREE,
10220 : NULL_TREE);
10221 114 : TREE_THIS_NOTRAP (var4) = 1;
10222 : }
10223 : }
10224 : else
10225 : var2 = val;
10226 : }
10227 : }
10228 464 : gcc_assert (var2);
10229 : }
10230 : else
10231 : {
10232 696 : var2 = build_outer_var_ref (var, octx);
10233 696 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
10234 : {
10235 220 : var3 = maybe_lookup_decl (new_vard, octx);
10236 220 : if (var3 == new_vard || var3 == NULL_TREE)
10237 : var3 = NULL_TREE;
10238 108 : else if (is_simd && octx->scan_exclusive && !input_phase)
10239 : {
10240 16 : var4 = maybe_lookup_decl (var3, octx);
10241 16 : if (var4 == var3 || var4 == NULL_TREE)
10242 : {
10243 0 : if (TREE_ADDRESSABLE (TREE_TYPE (new_var)))
10244 : {
10245 : var4 = var3;
10246 : var3 = NULL_TREE;
10247 : }
10248 : else
10249 16 : var4 = NULL_TREE;
10250 : }
10251 : }
10252 : }
10253 664 : if (is_simd
10254 484 : && octx->scan_exclusive
10255 244 : && !input_phase
10256 244 : && var4 == NULL_TREE)
10257 106 : var4 = create_tmp_var (TREE_TYPE (val));
10258 : }
10259 1160 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
10260 : {
10261 376 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
10262 376 : if (input_phase)
10263 : {
10264 188 : if (var3)
10265 : {
10266 : /* If we've added a separate identity element
10267 : variable, copy it over into val. */
10268 92 : tree x = lang_hooks.decls.omp_clause_assign_op (c, val,
10269 : var3);
10270 92 : gimplify_and_add (x, &before);
10271 : }
10272 96 : else if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
10273 : {
10274 : /* Otherwise, assign to it the identity element. */
10275 96 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
10276 96 : if (is_for)
10277 16 : tseq = copy_gimple_seq_and_replace_locals (tseq);
10278 96 : tree ref = build_outer_var_ref (var, octx);
10279 96 : tree x = (DECL_HAS_VALUE_EXPR_P (new_vard)
10280 96 : ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
10281 40 : if (x)
10282 : {
10283 40 : if (new_vard != new_var)
10284 8 : val = build_fold_addr_expr_loc (clause_loc, val);
10285 40 : SET_DECL_VALUE_EXPR (new_vard, val);
10286 : }
10287 96 : SET_DECL_VALUE_EXPR (placeholder, ref);
10288 96 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
10289 96 : lower_omp (&tseq, octx);
10290 96 : if (x)
10291 40 : SET_DECL_VALUE_EXPR (new_vard, x);
10292 96 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
10293 96 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
10294 96 : gimple_seq_add_seq (&before, tseq);
10295 96 : if (is_simd)
10296 80 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
10297 : }
10298 : }
10299 188 : else if (is_simd)
10300 : {
10301 156 : tree x;
10302 156 : if (octx->scan_exclusive)
10303 : {
10304 72 : tree v4 = unshare_expr (var4);
10305 72 : tree v2 = unshare_expr (var2);
10306 72 : x = lang_hooks.decls.omp_clause_assign_op (c, v4, v2);
10307 72 : gimplify_and_add (x, &before);
10308 : }
10309 156 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
10310 156 : x = (DECL_HAS_VALUE_EXPR_P (new_vard)
10311 156 : ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
10312 156 : tree vexpr = val;
10313 156 : if (x && new_vard != new_var)
10314 30 : vexpr = build_fold_addr_expr_loc (clause_loc, val);
10315 156 : if (x)
10316 78 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
10317 156 : SET_DECL_VALUE_EXPR (placeholder, var2);
10318 156 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
10319 156 : lower_omp (&tseq, octx);
10320 156 : gimple_seq_add_seq (&before, tseq);
10321 156 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
10322 156 : if (x)
10323 78 : SET_DECL_VALUE_EXPR (new_vard, x);
10324 156 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
10325 156 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
10326 156 : if (octx->scan_inclusive)
10327 : {
10328 84 : x = lang_hooks.decls.omp_clause_assign_op (c, val,
10329 : var2);
10330 84 : gimplify_and_add (x, &before);
10331 : }
10332 72 : else if (lane0 == NULL_TREE)
10333 : {
10334 36 : x = lang_hooks.decls.omp_clause_assign_op (c, val,
10335 : var4);
10336 36 : gimplify_and_add (x, &before);
10337 : }
10338 : }
10339 : }
10340 : else
10341 : {
10342 784 : if (input_phase)
10343 : {
10344 : /* input phase. Set val to initializer before
10345 : the body. */
10346 392 : tree x = omp_reduction_init (c, TREE_TYPE (new_var));
10347 392 : gimplify_assign (val, x, &before);
10348 : }
10349 392 : else if (is_simd)
10350 : {
10351 : /* scan phase. */
10352 318 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
10353 318 : if (code == MINUS_EXPR)
10354 0 : code = PLUS_EXPR;
10355 :
10356 318 : tree x = build2 (code, TREE_TYPE (var2),
10357 : unshare_expr (var2), unshare_expr (val));
10358 318 : if (octx->scan_inclusive)
10359 : {
10360 154 : gimplify_assign (unshare_expr (var2), x, &before);
10361 154 : gimplify_assign (val, var2, &before);
10362 : }
10363 : else
10364 : {
10365 164 : gimplify_assign (unshare_expr (var4),
10366 : unshare_expr (var2), &before);
10367 164 : gimplify_assign (var2, x, &before);
10368 164 : if (lane0 == NULL_TREE)
10369 86 : gimplify_assign (val, var4, &before);
10370 : }
10371 : }
10372 : }
10373 1160 : if (octx->scan_exclusive && !input_phase && lane0)
10374 : {
10375 114 : tree vexpr = unshare_expr (var4);
10376 114 : TREE_OPERAND (vexpr, 1) = lane0;
10377 114 : if (new_vard != new_var)
10378 16 : vexpr = build_fold_addr_expr_loc (clause_loc, vexpr);
10379 114 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
10380 : }
10381 : }
10382 : }
10383 1268 : if (is_simd && !is_for_simd)
10384 : {
10385 590 : gsi_insert_seq_after (gsi_p, gimple_omp_body (stmt), GSI_SAME_STMT);
10386 590 : gsi_insert_seq_after (gsi_p, before, GSI_SAME_STMT);
10387 590 : gsi_replace (gsi_p, gimple_build_nop (), true);
10388 590 : return;
10389 : }
10390 678 : lower_omp (gimple_omp_body_ptr (stmt), octx);
10391 678 : if (before)
10392 : {
10393 256 : gimple_stmt_iterator gsi = gsi_start (*gimple_omp_body_ptr (stmt));
10394 256 : gsi_insert_seq_before (&gsi, before, GSI_SAME_STMT);
10395 : }
10396 : }
10397 :
10398 :
10399 : /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
10400 : substitution of a couple of function calls. But in the NAMED case,
10401 : requires that languages coordinate a symbol name. It is therefore
10402 : best put here in common code. */
10403 :
10404 : static GTY(()) hash_map<tree, tree> *critical_name_mutexes;
10405 :
10406 : static void
10407 311 : lower_omp_critical (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10408 : {
10409 311 : tree block;
10410 311 : tree name, lock, unlock;
10411 311 : gomp_critical *stmt = as_a <gomp_critical *> (gsi_stmt (*gsi_p));
10412 311 : gbind *bind;
10413 311 : location_t loc = gimple_location (stmt);
10414 311 : gimple_seq tbody;
10415 :
10416 311 : name = gimple_omp_critical_name (stmt);
10417 311 : if (name)
10418 : {
10419 88 : tree decl;
10420 :
10421 88 : if (!critical_name_mutexes)
10422 48 : critical_name_mutexes = hash_map<tree, tree>::create_ggc (10);
10423 :
10424 88 : tree *n = critical_name_mutexes->get (name);
10425 88 : if (n == NULL)
10426 : {
10427 70 : char *new_str;
10428 :
10429 70 : decl = create_tmp_var_raw (ptr_type_node);
10430 :
10431 70 : new_str = ACONCAT ((".gomp_critical_user_",
10432 : IDENTIFIER_POINTER (name), NULL));
10433 70 : DECL_NAME (decl) = get_identifier (new_str);
10434 70 : TREE_PUBLIC (decl) = 1;
10435 70 : TREE_STATIC (decl) = 1;
10436 70 : DECL_COMMON (decl) = 1;
10437 70 : DECL_ARTIFICIAL (decl) = 1;
10438 70 : DECL_IGNORED_P (decl) = 1;
10439 :
10440 70 : varpool_node::finalize_decl (decl);
10441 :
10442 70 : critical_name_mutexes->put (name, decl);
10443 : }
10444 : else
10445 18 : decl = *n;
10446 :
10447 : /* If '#pragma omp critical' is inside offloaded region or
10448 : inside function marked as offloadable, the symbol must be
10449 : marked as offloadable too. */
10450 88 : omp_context *octx;
10451 88 : if (cgraph_node::get (current_function_decl)->offloadable)
10452 1 : varpool_node::get_create (decl)->offloadable = 1;
10453 : else
10454 173 : for (octx = ctx->outer; octx; octx = octx->outer)
10455 87 : if (is_gimple_omp_offloaded (octx->stmt))
10456 : {
10457 1 : varpool_node::get_create (decl)->offloadable = 1;
10458 1 : break;
10459 : }
10460 :
10461 88 : lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START);
10462 88 : lock = build_call_expr_loc (loc, lock, 1,
10463 : build_fold_addr_expr_loc (loc, decl));
10464 :
10465 88 : unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END);
10466 88 : unlock = build_call_expr_loc (loc, unlock, 1,
10467 : build_fold_addr_expr_loc (loc, decl));
10468 : }
10469 : else
10470 : {
10471 223 : lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START);
10472 223 : lock = build_call_expr_loc (loc, lock, 0);
10473 :
10474 223 : unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END);
10475 223 : unlock = build_call_expr_loc (loc, unlock, 0);
10476 : }
10477 :
10478 311 : push_gimplify_context ();
10479 :
10480 311 : block = make_node (BLOCK);
10481 311 : bind = gimple_build_bind (NULL, NULL, block);
10482 311 : gsi_replace (gsi_p, bind, true);
10483 311 : gimple_bind_add_stmt (bind, stmt);
10484 :
10485 311 : tbody = gimple_bind_body (bind);
10486 311 : gimplify_and_add (lock, &tbody);
10487 311 : gimple_bind_set_body (bind, tbody);
10488 :
10489 311 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
10490 311 : gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
10491 311 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10492 311 : gimple_omp_set_body (stmt, NULL);
10493 :
10494 311 : tbody = gimple_bind_body (bind);
10495 311 : gimplify_and_add (unlock, &tbody);
10496 311 : gimple_bind_set_body (bind, tbody);
10497 :
10498 311 : gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10499 :
10500 311 : pop_gimplify_context (bind);
10501 311 : gimple_bind_append_vars (bind, ctx->block_vars);
10502 311 : BLOCK_VARS (block) = gimple_bind_vars (bind);
10503 311 : }
10504 :
10505 : /* A subroutine of lower_omp_for. Generate code to emit the predicate
10506 : for a lastprivate clause. Given a loop control predicate of (V
10507 : cond N2), we gate the clause on (!(V cond N2)). The lowered form
10508 : is appended to *DLIST, iterator initialization is appended to
10509 : *BODY_P. *CLIST is for lastprivate(conditional:) code that needs
10510 : to be emitted in a critical section. */
10511 :
10512 : static void
10513 46890 : lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p,
10514 : gimple_seq *dlist, gimple_seq *clist,
10515 : struct omp_context *ctx)
10516 : {
10517 46890 : tree clauses, cond, vinit;
10518 46890 : enum tree_code cond_code;
10519 46890 : gimple_seq stmts;
10520 :
10521 46890 : cond_code = fd->loop.cond_code;
10522 46890 : cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR;
10523 :
10524 : /* When possible, use a strict equality expression. This can let VRP
10525 : type optimizations deduce the value and remove a copy. */
10526 46890 : if (tree_fits_shwi_p (fd->loop.step))
10527 : {
10528 44651 : HOST_WIDE_INT step = tree_to_shwi (fd->loop.step);
10529 44651 : if (step == 1 || step == -1)
10530 46890 : cond_code = EQ_EXPR;
10531 : }
10532 :
10533 46890 : tree n2 = fd->loop.n2;
10534 46890 : if (fd->collapse > 1
10535 11251 : && TREE_CODE (n2) != INTEGER_CST
10536 53104 : && gimple_omp_for_combined_into_p (fd->for_stmt))
10537 : {
10538 2692 : struct omp_context *taskreg_ctx = NULL;
10539 2692 : if (gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR)
10540 : {
10541 1226 : gomp_for *gfor = as_a <gomp_for *> (ctx->outer->stmt);
10542 1226 : if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_FOR
10543 1226 : || gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_DISTRIBUTE)
10544 : {
10545 1194 : if (gimple_omp_for_combined_into_p (gfor))
10546 : {
10547 672 : gcc_assert (ctx->outer->outer
10548 : && is_parallel_ctx (ctx->outer->outer));
10549 : taskreg_ctx = ctx->outer->outer;
10550 : }
10551 : else
10552 : {
10553 522 : struct omp_for_data outer_fd;
10554 522 : omp_extract_for_data (gfor, &outer_fd, NULL);
10555 522 : n2 = fold_convert (TREE_TYPE (n2), outer_fd.loop.n2);
10556 : }
10557 : }
10558 32 : else if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_TASKLOOP)
10559 32 : taskreg_ctx = ctx->outer->outer;
10560 : }
10561 1466 : else if (is_taskreg_ctx (ctx->outer))
10562 : taskreg_ctx = ctx->outer;
10563 2692 : if (taskreg_ctx)
10564 : {
10565 2170 : int i;
10566 2170 : tree taskreg_clauses
10567 2170 : = gimple_omp_taskreg_clauses (taskreg_ctx->stmt);
10568 2170 : tree innerc = omp_find_clause (taskreg_clauses,
10569 : OMP_CLAUSE__LOOPTEMP_);
10570 2170 : gcc_assert (innerc);
10571 2170 : int count = fd->collapse;
10572 2170 : if (fd->non_rect
10573 24 : && fd->last_nonrect == fd->first_nonrect + 1)
10574 12 : if (tree v = gimple_omp_for_index (fd->for_stmt, fd->last_nonrect))
10575 12 : if (!TYPE_UNSIGNED (TREE_TYPE (v)))
10576 12 : count += 4;
10577 8582 : for (i = 0; i < count; i++)
10578 : {
10579 6412 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
10580 : OMP_CLAUSE__LOOPTEMP_);
10581 6412 : gcc_assert (innerc);
10582 : }
10583 2170 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
10584 : OMP_CLAUSE__LOOPTEMP_);
10585 2170 : if (innerc)
10586 1419 : n2 = fold_convert (TREE_TYPE (n2),
10587 : lookup_decl (OMP_CLAUSE_DECL (innerc),
10588 : taskreg_ctx));
10589 : }
10590 : }
10591 46890 : cond = build2 (cond_code, boolean_type_node, fd->loop.v, n2);
10592 :
10593 46890 : clauses = gimple_omp_for_clauses (fd->for_stmt);
10594 46890 : stmts = NULL;
10595 46890 : lower_lastprivate_clauses (clauses, cond, body_p, &stmts, clist, ctx);
10596 46890 : if (!gimple_seq_empty_p (stmts))
10597 : {
10598 16362 : gimple_seq_add_seq (&stmts, *dlist);
10599 16362 : *dlist = stmts;
10600 :
10601 : /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
10602 16362 : vinit = fd->loop.n1;
10603 16362 : if (cond_code == EQ_EXPR
10604 14375 : && tree_fits_shwi_p (fd->loop.n2)
10605 25923 : && ! integer_zerop (fd->loop.n2))
10606 8345 : vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0);
10607 : else
10608 8017 : vinit = unshare_expr (vinit);
10609 :
10610 : /* Initialize the iterator variable, so that threads that don't execute
10611 : any iterations don't execute the lastprivate clauses by accident. */
10612 16362 : gimplify_assign (fd->loop.v, vinit, body_p);
10613 : }
10614 46890 : }
10615 :
10616 : /* OpenACC privatization.
10617 :
10618 : Or, in other words, *sharing* at the respective OpenACC level of
10619 : parallelism.
10620 :
10621 : From a correctness perspective, a non-addressable variable can't be accessed
10622 : outside the current thread, so it can go in a (faster than shared memory)
10623 : register -- though that register may need to be broadcast in some
10624 : circumstances. A variable can only meaningfully be "shared" across workers
10625 : or vector lanes if its address is taken, e.g. by a call to an atomic
10626 : builtin.
10627 :
10628 : From an optimisation perspective, the answer might be fuzzier: maybe
10629 : sometimes, using shared memory directly would be faster than
10630 : broadcasting. */
10631 :
10632 : static void
10633 18997 : oacc_privatization_begin_diagnose_var (const dump_flags_t l_dump_flags,
10634 : const location_t loc, const tree c,
10635 : const tree decl)
10636 : {
10637 18997 : const dump_user_location_t d_u_loc
10638 18997 : = dump_user_location_t::from_location_t (loc);
10639 : /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */
10640 : #if __GNUC__ >= 10
10641 18997 : # pragma GCC diagnostic push
10642 18997 : # pragma GCC diagnostic ignored "-Wformat"
10643 : #endif
10644 18997 : dump_printf_loc (l_dump_flags, d_u_loc,
10645 : "variable %<%T%> ", decl);
10646 : #if __GNUC__ >= 10
10647 18997 : # pragma GCC diagnostic pop
10648 : #endif
10649 18997 : if (c)
10650 2797 : dump_printf (l_dump_flags,
10651 : "in %qs clause ",
10652 2797 : omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
10653 : else
10654 16200 : dump_printf (l_dump_flags,
10655 : "declared in block ");
10656 18997 : }
10657 :
10658 : static bool
10659 41114 : oacc_privatization_candidate_p (const location_t loc, const tree c,
10660 : const tree decl)
10661 : {
10662 41114 : dump_flags_t l_dump_flags = get_openacc_privatization_dump_flags ();
10663 :
10664 : /* There is some differentiation depending on block vs. clause. */
10665 41114 : bool block = !c;
10666 :
10667 41114 : bool res = true;
10668 :
10669 41114 : if (res && !VAR_P (decl))
10670 : {
10671 : /* A PARM_DECL (appearing in a 'private' clause) is expected to have been
10672 : privatized into a new VAR_DECL. */
10673 712 : gcc_checking_assert (TREE_CODE (decl) != PARM_DECL);
10674 :
10675 712 : res = false;
10676 :
10677 712 : if (dump_enabled_p ())
10678 : {
10679 683 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10680 683 : dump_printf (l_dump_flags,
10681 : "potentially has improper OpenACC privatization level: %qs\n",
10682 683 : get_tree_code_name (TREE_CODE (decl)));
10683 : }
10684 : }
10685 :
10686 41085 : if (res && block && TREE_STATIC (decl))
10687 : {
10688 53 : res = false;
10689 :
10690 53 : if (dump_enabled_p ())
10691 : {
10692 28 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10693 28 : dump_printf (l_dump_flags,
10694 : "isn%'t candidate for adjusting OpenACC privatization level: %s\n",
10695 : "static");
10696 : }
10697 : }
10698 :
10699 41089 : if (res && block && DECL_EXTERNAL (decl))
10700 : {
10701 12 : res = false;
10702 :
10703 12 : if (dump_enabled_p ())
10704 : {
10705 12 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10706 12 : dump_printf (l_dump_flags,
10707 : "isn%'t candidate for adjusting OpenACC privatization level: %s\n",
10708 : "external");
10709 : }
10710 : }
10711 :
10712 41114 : if (res && !TREE_ADDRESSABLE (decl))
10713 : {
10714 39245 : res = false;
10715 :
10716 39245 : if (dump_enabled_p ())
10717 : {
10718 17456 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10719 17456 : dump_printf (l_dump_flags,
10720 : "isn%'t candidate for adjusting OpenACC privatization level: %s\n",
10721 : "not addressable");
10722 : }
10723 : }
10724 :
10725 : /* If an artificial variable has been added to a bind, e.g.
10726 : a compiler-generated temporary structure used by the Fortran front-end, do
10727 : not consider it as a privatization candidate. Note that variables on
10728 : the stack are private per-thread by default: making them "gang-private"
10729 : for OpenACC actually means to share a single instance of a variable
10730 : amongst all workers and threads spawned within each gang.
10731 : At present, no compiler-generated artificial variables require such
10732 : sharing semantics, so this is safe. */
10733 :
10734 18548 : if (res && block && DECL_ARTIFICIAL (decl))
10735 : {
10736 548 : res = false;
10737 :
10738 548 : if (dump_enabled_p ())
10739 : {
10740 360 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10741 360 : dump_printf (l_dump_flags,
10742 : "isn%'t candidate for adjusting OpenACC privatization "
10743 : "level: %s\n", "artificial");
10744 : }
10745 : }
10746 :
10747 40926 : if (res)
10748 : {
10749 544 : if (dump_enabled_p ())
10750 : {
10751 458 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10752 458 : dump_printf (l_dump_flags,
10753 : "is candidate for adjusting OpenACC privatization level\n");
10754 : }
10755 : }
10756 :
10757 41114 : if (dump_file && (dump_flags & TDF_DETAILS))
10758 : {
10759 0 : print_generic_decl (dump_file, decl, dump_flags);
10760 0 : fprintf (dump_file, "\n");
10761 : }
10762 :
10763 41114 : return res;
10764 : }
10765 :
10766 : /* Scan CLAUSES for candidates for adjusting OpenACC privatization level in
10767 : CTX. */
10768 :
10769 : static void
10770 11325 : oacc_privatization_scan_clause_chain (omp_context *ctx, tree clauses)
10771 : {
10772 35223 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
10773 23898 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE)
10774 : {
10775 12986 : tree decl = OMP_CLAUSE_DECL (c);
10776 :
10777 12986 : tree new_decl = lookup_decl (decl, ctx);
10778 :
10779 12986 : if (!oacc_privatization_candidate_p (OMP_CLAUSE_LOCATION (c), c,
10780 : new_decl))
10781 12654 : continue;
10782 :
10783 332 : gcc_checking_assert
10784 : (!ctx->oacc_privatization_candidates.contains (new_decl));
10785 332 : ctx->oacc_privatization_candidates.safe_push (new_decl);
10786 : }
10787 11325 : }
10788 :
10789 : /* Scan DECLS for candidates for adjusting OpenACC privatization level in
10790 : CTX. */
10791 :
10792 : static void
10793 23090 : oacc_privatization_scan_decl_chain (omp_context *ctx, tree decls)
10794 : {
10795 51218 : for (tree decl = decls; decl; decl = DECL_CHAIN (decl))
10796 : {
10797 28128 : tree new_decl = lookup_decl (decl, ctx);
10798 28128 : gcc_checking_assert (new_decl == decl);
10799 :
10800 28128 : if (!oacc_privatization_candidate_p (gimple_location (ctx->stmt), NULL,
10801 : new_decl))
10802 27916 : continue;
10803 :
10804 212 : gcc_checking_assert
10805 : (!ctx->oacc_privatization_candidates.contains (new_decl));
10806 212 : ctx->oacc_privatization_candidates.safe_push (new_decl);
10807 : }
10808 23090 : }
10809 :
10810 : /* Callback for walk_gimple_seq. Find #pragma omp scan statement. */
10811 :
10812 : static tree
10813 2225 : omp_find_scan (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
10814 : struct walk_stmt_info *wi)
10815 : {
10816 2225 : gimple *stmt = gsi_stmt (*gsi_p);
10817 :
10818 2225 : *handled_ops_p = true;
10819 2225 : switch (gimple_code (stmt))
10820 : {
10821 418 : WALK_SUBSTMTS;
10822 :
10823 166 : case GIMPLE_OMP_FOR:
10824 166 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_SIMD
10825 166 : && gimple_omp_for_combined_into_p (stmt))
10826 166 : *handled_ops_p = false;
10827 : break;
10828 :
10829 678 : case GIMPLE_OMP_SCAN:
10830 678 : *(gimple_stmt_iterator *) (wi->info) = *gsi_p;
10831 678 : return integer_zero_node;
10832 : default:
10833 : break;
10834 : }
10835 : return NULL;
10836 : }
10837 :
10838 : /* Helper function for lower_omp_for, add transformations for a worksharing
10839 : loop with scan directives inside of it.
10840 : For worksharing loop not combined with simd, transform:
10841 : #pragma omp for reduction(inscan,+:r) private(i)
10842 : for (i = 0; i < n; i = i + 1)
10843 : {
10844 : {
10845 : update (r);
10846 : }
10847 : #pragma omp scan inclusive(r)
10848 : {
10849 : use (r);
10850 : }
10851 : }
10852 :
10853 : into two worksharing loops + code to merge results:
10854 :
10855 : num_threads = omp_get_num_threads ();
10856 : thread_num = omp_get_thread_num ();
10857 : if (thread_num == 0) goto <D.2099>; else goto <D.2100>;
10858 : <D.2099>:
10859 : var2 = r;
10860 : goto <D.2101>;
10861 : <D.2100>:
10862 : // For UDRs this is UDR init, or if ctors are needed, copy from
10863 : // var3 that has been constructed to contain the neutral element.
10864 : var2 = 0;
10865 : <D.2101>:
10866 : ivar = 0;
10867 : // The _scantemp_ clauses will arrange for rpriva to be initialized to
10868 : // a shared array with num_threads elements and rprivb to a local array
10869 : // number of elements equal to the number of (contiguous) iterations the
10870 : // current thread will perform. controlb and controlp variables are
10871 : // temporaries to handle deallocation of rprivb at the end of second
10872 : // GOMP_FOR.
10873 : #pragma omp for _scantemp_(rpriva) _scantemp_(rprivb) _scantemp_(controlb) \
10874 : _scantemp_(controlp) reduction(inscan,+:r) private(i) nowait
10875 : for (i = 0; i < n; i = i + 1)
10876 : {
10877 : {
10878 : // For UDRs this is UDR init or copy from var3.
10879 : r = 0;
10880 : // This is the input phase from user code.
10881 : update (r);
10882 : }
10883 : {
10884 : // For UDRs this is UDR merge.
10885 : var2 = var2 + r;
10886 : // Rather than handing it over to the user, save to local thread's
10887 : // array.
10888 : rprivb[ivar] = var2;
10889 : // For exclusive scan, the above two statements are swapped.
10890 : ivar = ivar + 1;
10891 : }
10892 : }
10893 : // And remember the final value from this thread's into the shared
10894 : // rpriva array.
10895 : rpriva[(sizetype) thread_num] = var2;
10896 : // If more than one thread, compute using Work-Efficient prefix sum
10897 : // the inclusive parallel scan of the rpriva array.
10898 : if (num_threads > 1) goto <D.2102>; else goto <D.2103>;
10899 : <D.2102>:
10900 : GOMP_barrier ();
10901 : down = 0;
10902 : k = 1;
10903 : num_threadsu = (unsigned int) num_threads;
10904 : thread_numup1 = (unsigned int) thread_num + 1;
10905 : <D.2108>:
10906 : twok = k << 1;
10907 : if (twok > num_threadsu) goto <D.2110>; else goto <D.2111>;
10908 : <D.2110>:
10909 : down = 4294967295;
10910 : k = k >> 1;
10911 : if (k == num_threadsu) goto <D.2112>; else goto <D.2111>;
10912 : <D.2112>:
10913 : k = k >> 1;
10914 : <D.2111>:
10915 : twok = k << 1;
10916 : cplx = .MUL_OVERFLOW (thread_nump1, twok);
10917 : mul = REALPART_EXPR <cplx>;
10918 : ovf = IMAGPART_EXPR <cplx>;
10919 : if (ovf == 0) goto <D.2116>; else goto <D.2117>;
10920 : <D.2116>:
10921 : andv = k & down;
10922 : andvm1 = andv + 4294967295;
10923 : l = mul + andvm1;
10924 : if (l < num_threadsu) goto <D.2120>; else goto <D.2117>;
10925 : <D.2120>:
10926 : // For UDRs this is UDR merge, performed using var2 variable as temporary,
10927 : // i.e. var2 = rpriva[l - k]; UDR merge (var2, rpriva[l]); rpriva[l] = var2;
10928 : rpriva[l] = rpriva[l - k] + rpriva[l];
10929 : <D.2117>:
10930 : if (down == 0) goto <D.2121>; else goto <D.2122>;
10931 : <D.2121>:
10932 : k = k << 1;
10933 : goto <D.2123>;
10934 : <D.2122>:
10935 : k = k >> 1;
10936 : <D.2123>:
10937 : GOMP_barrier ();
10938 : if (k != 0) goto <D.2108>; else goto <D.2103>;
10939 : <D.2103>:
10940 : if (thread_num == 0) goto <D.2124>; else goto <D.2125>;
10941 : <D.2124>:
10942 : // For UDRs this is UDR init or copy from var3.
10943 : var2 = 0;
10944 : goto <D.2126>;
10945 : <D.2125>:
10946 : var2 = rpriva[thread_num - 1];
10947 : <D.2126>:
10948 : ivar = 0;
10949 : #pragma omp for _scantemp_(controlb) _scantemp_(controlp) \
10950 : reduction(inscan,+:r) private(i)
10951 : for (i = 0; i < n; i = i + 1)
10952 : {
10953 : {
10954 : // For UDRs, this is r = var2; UDR merge (r, rprivb[ivar]);
10955 : r = var2 + rprivb[ivar];
10956 : }
10957 : {
10958 : // This is the scan phase from user code.
10959 : use (r);
10960 : // Plus a bump of the iterator.
10961 : ivar = ivar + 1;
10962 : }
10963 : } */
10964 :
10965 : static void
10966 173 : lower_omp_for_scan (gimple_seq *body_p, gimple_seq *dlist, gomp_for *stmt,
10967 : struct omp_for_data *fd, omp_context *ctx)
10968 : {
10969 173 : bool is_for_simd = gimple_omp_for_combined_p (stmt);
10970 173 : gcc_assert (ctx->scan_inclusive || ctx->scan_exclusive);
10971 :
10972 173 : gimple_seq body = gimple_omp_body (stmt);
10973 173 : gimple_stmt_iterator input1_gsi = gsi_none ();
10974 173 : struct walk_stmt_info wi;
10975 173 : memset (&wi, 0, sizeof (wi));
10976 173 : wi.val_only = true;
10977 173 : wi.info = (void *) &input1_gsi;
10978 173 : walk_gimple_seq_mod (&body, omp_find_scan, NULL, &wi);
10979 173 : gcc_assert (!gsi_end_p (input1_gsi));
10980 :
10981 173 : gimple *input_stmt1 = gsi_stmt (input1_gsi);
10982 173 : gimple_stmt_iterator gsi = input1_gsi;
10983 173 : gsi_next (&gsi);
10984 173 : gimple_stmt_iterator scan1_gsi = gsi;
10985 173 : gimple *scan_stmt1 = gsi_stmt (gsi);
10986 173 : gcc_assert (scan_stmt1 && gimple_code (scan_stmt1) == GIMPLE_OMP_SCAN);
10987 :
10988 173 : gimple_seq input_body = gimple_omp_body (input_stmt1);
10989 173 : gimple_seq scan_body = gimple_omp_body (scan_stmt1);
10990 173 : gimple_omp_set_body (input_stmt1, NULL);
10991 173 : gimple_omp_set_body (scan_stmt1, NULL);
10992 173 : gimple_omp_set_body (stmt, NULL);
10993 :
10994 173 : gomp_for *new_stmt = as_a <gomp_for *> (gimple_copy (stmt));
10995 173 : gimple_seq new_body = copy_gimple_seq_and_replace_locals (body);
10996 173 : gimple_omp_set_body (stmt, body);
10997 173 : gimple_omp_set_body (input_stmt1, input_body);
10998 :
10999 173 : gimple_stmt_iterator input2_gsi = gsi_none ();
11000 173 : memset (&wi, 0, sizeof (wi));
11001 173 : wi.val_only = true;
11002 173 : wi.info = (void *) &input2_gsi;
11003 173 : walk_gimple_seq_mod (&new_body, omp_find_scan, NULL, &wi);
11004 173 : gcc_assert (!gsi_end_p (input2_gsi));
11005 :
11006 173 : gimple *input_stmt2 = gsi_stmt (input2_gsi);
11007 173 : gsi = input2_gsi;
11008 173 : gsi_next (&gsi);
11009 173 : gimple_stmt_iterator scan2_gsi = gsi;
11010 173 : gimple *scan_stmt2 = gsi_stmt (gsi);
11011 173 : gcc_assert (scan_stmt2 && gimple_code (scan_stmt2) == GIMPLE_OMP_SCAN);
11012 173 : gimple_omp_set_body (scan_stmt2, scan_body);
11013 :
11014 173 : gimple_stmt_iterator input3_gsi = gsi_none ();
11015 173 : gimple_stmt_iterator scan3_gsi = gsi_none ();
11016 173 : gimple_stmt_iterator input4_gsi = gsi_none ();
11017 173 : gimple_stmt_iterator scan4_gsi = gsi_none ();
11018 173 : gimple *input_stmt3 = NULL, *scan_stmt3 = NULL;
11019 173 : gimple *input_stmt4 = NULL, *scan_stmt4 = NULL;
11020 173 : omp_context *input_simd_ctx = NULL, *scan_simd_ctx = NULL;
11021 173 : if (is_for_simd)
11022 : {
11023 83 : memset (&wi, 0, sizeof (wi));
11024 83 : wi.val_only = true;
11025 83 : wi.info = (void *) &input3_gsi;
11026 83 : walk_gimple_seq_mod (&input_body, omp_find_scan, NULL, &wi);
11027 83 : gcc_assert (!gsi_end_p (input3_gsi));
11028 :
11029 83 : input_stmt3 = gsi_stmt (input3_gsi);
11030 83 : gsi = input3_gsi;
11031 83 : gsi_next (&gsi);
11032 83 : scan3_gsi = gsi;
11033 83 : scan_stmt3 = gsi_stmt (gsi);
11034 83 : gcc_assert (scan_stmt3 && gimple_code (scan_stmt3) == GIMPLE_OMP_SCAN);
11035 :
11036 83 : memset (&wi, 0, sizeof (wi));
11037 83 : wi.val_only = true;
11038 83 : wi.info = (void *) &input4_gsi;
11039 83 : walk_gimple_seq_mod (&scan_body, omp_find_scan, NULL, &wi);
11040 83 : gcc_assert (!gsi_end_p (input4_gsi));
11041 :
11042 83 : input_stmt4 = gsi_stmt (input4_gsi);
11043 83 : gsi = input4_gsi;
11044 83 : gsi_next (&gsi);
11045 83 : scan4_gsi = gsi;
11046 83 : scan_stmt4 = gsi_stmt (gsi);
11047 83 : gcc_assert (scan_stmt4 && gimple_code (scan_stmt4) == GIMPLE_OMP_SCAN);
11048 :
11049 83 : input_simd_ctx = maybe_lookup_ctx (input_stmt3)->outer;
11050 83 : scan_simd_ctx = maybe_lookup_ctx (input_stmt4)->outer;
11051 : }
11052 :
11053 173 : tree num_threads = create_tmp_var (integer_type_node);
11054 173 : tree thread_num = create_tmp_var (integer_type_node);
11055 173 : tree nthreads_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
11056 173 : tree threadnum_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
11057 173 : gimple *g = gimple_build_call (nthreads_decl, 0);
11058 173 : gimple_call_set_lhs (g, num_threads);
11059 173 : gimple_seq_add_stmt (body_p, g);
11060 173 : g = gimple_build_call (threadnum_decl, 0);
11061 173 : gimple_call_set_lhs (g, thread_num);
11062 173 : gimple_seq_add_stmt (body_p, g);
11063 :
11064 173 : tree ivar = create_tmp_var (sizetype);
11065 173 : tree new_clauses1 = NULL_TREE, new_clauses2 = NULL_TREE;
11066 173 : tree *cp1 = &new_clauses1, *cp2 = &new_clauses2;
11067 173 : tree k = create_tmp_var (unsigned_type_node);
11068 173 : tree l = create_tmp_var (unsigned_type_node);
11069 :
11070 173 : gimple_seq clist = NULL, mdlist = NULL;
11071 173 : gimple_seq thr01_list = NULL, thrn1_list = NULL;
11072 173 : gimple_seq thr02_list = NULL, thrn2_list = NULL;
11073 173 : gimple_seq scan1_list = NULL, input2_list = NULL;
11074 173 : gimple_seq last_list = NULL, reduc_list = NULL;
11075 674 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
11076 501 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
11077 501 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
11078 : {
11079 205 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
11080 205 : tree var = OMP_CLAUSE_DECL (c);
11081 205 : tree new_var = lookup_decl (var, ctx);
11082 205 : tree var3 = NULL_TREE;
11083 205 : tree new_vard = new_var;
11084 205 : if (omp_privatize_by_reference (var))
11085 32 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
11086 205 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
11087 : {
11088 64 : var3 = maybe_lookup_decl (new_vard, ctx);
11089 64 : if (var3 == new_vard)
11090 173 : var3 = NULL_TREE;
11091 : }
11092 :
11093 205 : tree ptype = build_pointer_type (TREE_TYPE (new_var));
11094 205 : tree rpriva = create_tmp_var (ptype);
11095 205 : tree nc = build_omp_clause (clause_loc, OMP_CLAUSE__SCANTEMP_);
11096 205 : OMP_CLAUSE_DECL (nc) = rpriva;
11097 205 : *cp1 = nc;
11098 205 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11099 :
11100 205 : tree rprivb = create_tmp_var (ptype);
11101 205 : nc = build_omp_clause (clause_loc, OMP_CLAUSE__SCANTEMP_);
11102 205 : OMP_CLAUSE_DECL (nc) = rprivb;
11103 205 : OMP_CLAUSE__SCANTEMP__ALLOC (nc) = 1;
11104 205 : *cp1 = nc;
11105 205 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11106 :
11107 205 : tree var2 = create_tmp_var_raw (TREE_TYPE (new_var));
11108 205 : if (new_vard != new_var)
11109 32 : TREE_ADDRESSABLE (var2) = 1;
11110 205 : gimple_add_tmp_var (var2);
11111 :
11112 205 : tree x = fold_convert_loc (clause_loc, sizetype, thread_num);
11113 205 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11114 205 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11115 205 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11116 205 : tree rpriva_ref = build_simple_mem_ref_loc (clause_loc, x);
11117 :
11118 205 : x = fold_build2_loc (clause_loc, PLUS_EXPR, integer_type_node,
11119 : thread_num, integer_minus_one_node);
11120 205 : x = fold_convert_loc (clause_loc, sizetype, x);
11121 205 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11122 205 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11123 205 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11124 205 : tree rprivam1_ref = build_simple_mem_ref_loc (clause_loc, x);
11125 :
11126 205 : x = fold_convert_loc (clause_loc, sizetype, l);
11127 205 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11128 205 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11129 205 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11130 205 : tree rprival_ref = build_simple_mem_ref_loc (clause_loc, x);
11131 :
11132 205 : x = fold_build2_loc (clause_loc, MINUS_EXPR, unsigned_type_node, l, k);
11133 205 : x = fold_convert_loc (clause_loc, sizetype, x);
11134 205 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11135 205 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11136 205 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11137 205 : tree rprivalmk_ref = build_simple_mem_ref_loc (clause_loc, x);
11138 :
11139 205 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, ivar,
11140 205 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11141 205 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rprivb), rprivb, x);
11142 205 : tree rprivb_ref = build_simple_mem_ref_loc (clause_loc, x);
11143 :
11144 205 : tree var4 = is_for_simd ? new_var : var2;
11145 99 : tree var5 = NULL_TREE, var6 = NULL_TREE;
11146 99 : if (is_for_simd)
11147 : {
11148 99 : var5 = lookup_decl (var, input_simd_ctx);
11149 99 : var6 = lookup_decl (var, scan_simd_ctx);
11150 99 : if (new_vard != new_var)
11151 : {
11152 16 : var5 = build_simple_mem_ref_loc (clause_loc, var5);
11153 16 : var6 = build_simple_mem_ref_loc (clause_loc, var6);
11154 : }
11155 : }
11156 205 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
11157 : {
11158 64 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
11159 64 : tree val = var2;
11160 :
11161 64 : x = lang_hooks.decls.omp_clause_default_ctor
11162 64 : (c, var2, build_outer_var_ref (var, ctx));
11163 64 : if (x)
11164 32 : gimplify_and_add (x, &clist);
11165 :
11166 64 : x = build_outer_var_ref (var, ctx);
11167 64 : x = lang_hooks.decls.omp_clause_assign_op (c, unshare_expr (var4),
11168 : x);
11169 64 : gimplify_and_add (x, &thr01_list);
11170 :
11171 64 : tree y = (DECL_HAS_VALUE_EXPR_P (new_vard)
11172 64 : ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
11173 64 : if (var3)
11174 : {
11175 32 : x = unshare_expr (var4);
11176 32 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var3);
11177 32 : gimplify_and_add (x, &thrn1_list);
11178 32 : x = unshare_expr (var4);
11179 32 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var3);
11180 32 : gimplify_and_add (x, &thr02_list);
11181 : }
11182 32 : else if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
11183 : {
11184 : /* Otherwise, assign to it the identity element. */
11185 32 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
11186 32 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11187 32 : if (!is_for_simd)
11188 : {
11189 16 : if (new_vard != new_var)
11190 4 : val = build_fold_addr_expr_loc (clause_loc, val);
11191 16 : SET_DECL_VALUE_EXPR (new_vard, val);
11192 16 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
11193 : }
11194 32 : SET_DECL_VALUE_EXPR (placeholder, error_mark_node);
11195 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11196 32 : lower_omp (&tseq, ctx);
11197 32 : gimple_seq_add_seq (&thrn1_list, tseq);
11198 32 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
11199 32 : lower_omp (&tseq, ctx);
11200 32 : gimple_seq_add_seq (&thr02_list, tseq);
11201 32 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
11202 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
11203 32 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
11204 32 : if (y)
11205 0 : SET_DECL_VALUE_EXPR (new_vard, y);
11206 : else
11207 : {
11208 32 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
11209 32 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
11210 : }
11211 : }
11212 :
11213 64 : x = unshare_expr (var4);
11214 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x, rprivam1_ref);
11215 64 : gimplify_and_add (x, &thrn2_list);
11216 :
11217 64 : if (is_for_simd)
11218 : {
11219 32 : x = unshare_expr (rprivb_ref);
11220 32 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var5);
11221 32 : gimplify_and_add (x, &scan1_list);
11222 : }
11223 : else
11224 : {
11225 32 : if (ctx->scan_exclusive)
11226 : {
11227 16 : x = unshare_expr (rprivb_ref);
11228 16 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var2);
11229 16 : gimplify_and_add (x, &scan1_list);
11230 : }
11231 :
11232 32 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
11233 32 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11234 32 : SET_DECL_VALUE_EXPR (placeholder, var2);
11235 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11236 32 : lower_omp (&tseq, ctx);
11237 32 : gimple_seq_add_seq (&scan1_list, tseq);
11238 :
11239 32 : if (ctx->scan_inclusive)
11240 : {
11241 16 : x = unshare_expr (rprivb_ref);
11242 16 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var2);
11243 16 : gimplify_and_add (x, &scan1_list);
11244 : }
11245 : }
11246 :
11247 64 : x = unshare_expr (rpriva_ref);
11248 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x,
11249 : unshare_expr (var4));
11250 64 : gimplify_and_add (x, &mdlist);
11251 :
11252 96 : x = unshare_expr (is_for_simd ? var6 : new_var);
11253 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var4);
11254 64 : gimplify_and_add (x, &input2_list);
11255 :
11256 64 : val = rprivb_ref;
11257 64 : if (new_vard != new_var)
11258 24 : val = build_fold_addr_expr_loc (clause_loc, val);
11259 :
11260 64 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
11261 64 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11262 64 : SET_DECL_VALUE_EXPR (new_vard, val);
11263 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
11264 64 : if (is_for_simd)
11265 : {
11266 32 : SET_DECL_VALUE_EXPR (placeholder, var6);
11267 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11268 : }
11269 : else
11270 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
11271 64 : lower_omp (&tseq, ctx);
11272 64 : if (y)
11273 0 : SET_DECL_VALUE_EXPR (new_vard, y);
11274 : else
11275 : {
11276 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
11277 64 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
11278 : }
11279 64 : if (!is_for_simd)
11280 : {
11281 32 : SET_DECL_VALUE_EXPR (placeholder, new_var);
11282 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11283 32 : lower_omp (&tseq, ctx);
11284 : }
11285 64 : gimple_seq_add_seq (&input2_list, tseq);
11286 :
11287 64 : x = build_outer_var_ref (var, ctx);
11288 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x, rpriva_ref);
11289 64 : gimplify_and_add (x, &last_list);
11290 :
11291 64 : x = lang_hooks.decls.omp_clause_assign_op (c, var2, rprivalmk_ref);
11292 64 : gimplify_and_add (x, &reduc_list);
11293 64 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
11294 64 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11295 64 : val = rprival_ref;
11296 64 : if (new_vard != new_var)
11297 24 : val = build_fold_addr_expr_loc (clause_loc, val);
11298 64 : SET_DECL_VALUE_EXPR (new_vard, val);
11299 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
11300 64 : SET_DECL_VALUE_EXPR (placeholder, var2);
11301 64 : lower_omp (&tseq, ctx);
11302 64 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
11303 64 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
11304 64 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
11305 64 : if (y)
11306 0 : SET_DECL_VALUE_EXPR (new_vard, y);
11307 : else
11308 : {
11309 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
11310 64 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
11311 : }
11312 64 : gimple_seq_add_seq (&reduc_list, tseq);
11313 64 : x = lang_hooks.decls.omp_clause_assign_op (c, rprival_ref, var2);
11314 64 : gimplify_and_add (x, &reduc_list);
11315 :
11316 64 : x = lang_hooks.decls.omp_clause_dtor (c, var2);
11317 64 : if (x)
11318 32 : gimplify_and_add (x, dlist);
11319 : }
11320 : else
11321 : {
11322 141 : x = build_outer_var_ref (var, ctx);
11323 141 : gimplify_assign (unshare_expr (var4), x, &thr01_list);
11324 :
11325 141 : x = omp_reduction_init (c, TREE_TYPE (new_var));
11326 141 : gimplify_assign (unshare_expr (var4), unshare_expr (x),
11327 : &thrn1_list);
11328 141 : gimplify_assign (unshare_expr (var4), x, &thr02_list);
11329 :
11330 141 : gimplify_assign (unshare_expr (var4), rprivam1_ref, &thrn2_list);
11331 :
11332 141 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
11333 141 : if (code == MINUS_EXPR)
11334 0 : code = PLUS_EXPR;
11335 :
11336 141 : if (is_for_simd)
11337 67 : gimplify_assign (unshare_expr (rprivb_ref), var5, &scan1_list);
11338 : else
11339 : {
11340 74 : if (ctx->scan_exclusive)
11341 28 : gimplify_assign (unshare_expr (rprivb_ref), var2,
11342 : &scan1_list);
11343 74 : x = build2 (code, TREE_TYPE (new_var), var2, new_var);
11344 74 : gimplify_assign (var2, x, &scan1_list);
11345 74 : if (ctx->scan_inclusive)
11346 46 : gimplify_assign (unshare_expr (rprivb_ref), var2,
11347 : &scan1_list);
11348 : }
11349 :
11350 141 : gimplify_assign (unshare_expr (rpriva_ref), unshare_expr (var4),
11351 : &mdlist);
11352 :
11353 141 : x = build2 (code, TREE_TYPE (new_var), var4, rprivb_ref);
11354 215 : gimplify_assign (is_for_simd ? var6 : new_var, x, &input2_list);
11355 :
11356 141 : gimplify_assign (build_outer_var_ref (var, ctx), rpriva_ref,
11357 : &last_list);
11358 :
11359 141 : x = build2 (code, TREE_TYPE (new_var), rprivalmk_ref,
11360 : unshare_expr (rprival_ref));
11361 141 : gimplify_assign (rprival_ref, x, &reduc_list);
11362 : }
11363 : }
11364 :
11365 173 : g = gimple_build_assign (ivar, PLUS_EXPR, ivar, size_one_node);
11366 173 : gimple_seq_add_stmt (&scan1_list, g);
11367 173 : g = gimple_build_assign (ivar, PLUS_EXPR, ivar, size_one_node);
11368 263 : gimple_seq_add_stmt (gimple_omp_body_ptr (is_for_simd
11369 : ? scan_stmt4 : scan_stmt2), g);
11370 :
11371 173 : tree controlb = create_tmp_var (boolean_type_node);
11372 173 : tree controlp = create_tmp_var (ptr_type_node);
11373 173 : tree nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11374 173 : OMP_CLAUSE_DECL (nc) = controlb;
11375 173 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11376 173 : *cp1 = nc;
11377 173 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11378 173 : nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11379 173 : OMP_CLAUSE_DECL (nc) = controlp;
11380 173 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11381 173 : *cp1 = nc;
11382 173 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11383 173 : nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11384 173 : OMP_CLAUSE_DECL (nc) = controlb;
11385 173 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11386 173 : *cp2 = nc;
11387 173 : cp2 = &OMP_CLAUSE_CHAIN (nc);
11388 173 : nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11389 173 : OMP_CLAUSE_DECL (nc) = controlp;
11390 173 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11391 173 : *cp2 = nc;
11392 173 : cp2 = &OMP_CLAUSE_CHAIN (nc);
11393 :
11394 173 : *cp1 = gimple_omp_for_clauses (stmt);
11395 173 : gimple_omp_for_set_clauses (stmt, new_clauses1);
11396 173 : *cp2 = gimple_omp_for_clauses (new_stmt);
11397 173 : gimple_omp_for_set_clauses (new_stmt, new_clauses2);
11398 :
11399 173 : if (is_for_simd)
11400 : {
11401 83 : gimple_seq_add_seq (gimple_omp_body_ptr (scan_stmt3), scan1_list);
11402 83 : gimple_seq_add_seq (gimple_omp_body_ptr (input_stmt4), input2_list);
11403 :
11404 83 : gsi_insert_seq_after (&input3_gsi, gimple_omp_body (input_stmt3),
11405 : GSI_SAME_STMT);
11406 83 : gsi_remove (&input3_gsi, true);
11407 83 : gsi_insert_seq_after (&scan3_gsi, gimple_omp_body (scan_stmt3),
11408 : GSI_SAME_STMT);
11409 83 : gsi_remove (&scan3_gsi, true);
11410 83 : gsi_insert_seq_after (&input4_gsi, gimple_omp_body (input_stmt4),
11411 : GSI_SAME_STMT);
11412 83 : gsi_remove (&input4_gsi, true);
11413 83 : gsi_insert_seq_after (&scan4_gsi, gimple_omp_body (scan_stmt4),
11414 : GSI_SAME_STMT);
11415 83 : gsi_remove (&scan4_gsi, true);
11416 : }
11417 : else
11418 : {
11419 90 : gimple_omp_set_body (scan_stmt1, scan1_list);
11420 90 : gimple_omp_set_body (input_stmt2, input2_list);
11421 : }
11422 :
11423 173 : gsi_insert_seq_after (&input1_gsi, gimple_omp_body (input_stmt1),
11424 : GSI_SAME_STMT);
11425 173 : gsi_remove (&input1_gsi, true);
11426 173 : gsi_insert_seq_after (&scan1_gsi, gimple_omp_body (scan_stmt1),
11427 : GSI_SAME_STMT);
11428 173 : gsi_remove (&scan1_gsi, true);
11429 173 : gsi_insert_seq_after (&input2_gsi, gimple_omp_body (input_stmt2),
11430 : GSI_SAME_STMT);
11431 173 : gsi_remove (&input2_gsi, true);
11432 173 : gsi_insert_seq_after (&scan2_gsi, gimple_omp_body (scan_stmt2),
11433 : GSI_SAME_STMT);
11434 173 : gsi_remove (&scan2_gsi, true);
11435 :
11436 173 : gimple_seq_add_seq (body_p, clist);
11437 :
11438 173 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
11439 173 : tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
11440 173 : tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
11441 173 : g = gimple_build_cond (EQ_EXPR, thread_num, integer_zero_node, lab1, lab2);
11442 173 : gimple_seq_add_stmt (body_p, g);
11443 173 : g = gimple_build_label (lab1);
11444 173 : gimple_seq_add_stmt (body_p, g);
11445 173 : gimple_seq_add_seq (body_p, thr01_list);
11446 173 : g = gimple_build_goto (lab3);
11447 173 : gimple_seq_add_stmt (body_p, g);
11448 173 : g = gimple_build_label (lab2);
11449 173 : gimple_seq_add_stmt (body_p, g);
11450 173 : gimple_seq_add_seq (body_p, thrn1_list);
11451 173 : g = gimple_build_label (lab3);
11452 173 : gimple_seq_add_stmt (body_p, g);
11453 :
11454 173 : g = gimple_build_assign (ivar, size_zero_node);
11455 173 : gimple_seq_add_stmt (body_p, g);
11456 :
11457 173 : gimple_seq_add_stmt (body_p, stmt);
11458 173 : gimple_seq_add_seq (body_p, body);
11459 173 : gimple_seq_add_stmt (body_p, gimple_build_omp_continue (fd->loop.v,
11460 : fd->loop.v));
11461 :
11462 173 : g = gimple_build_omp_return (true);
11463 173 : gimple_seq_add_stmt (body_p, g);
11464 173 : gimple_seq_add_seq (body_p, mdlist);
11465 :
11466 173 : lab1 = create_artificial_label (UNKNOWN_LOCATION);
11467 173 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
11468 173 : g = gimple_build_cond (GT_EXPR, num_threads, integer_one_node, lab1, lab2);
11469 173 : gimple_seq_add_stmt (body_p, g);
11470 173 : g = gimple_build_label (lab1);
11471 173 : gimple_seq_add_stmt (body_p, g);
11472 :
11473 173 : g = omp_build_barrier (NULL);
11474 173 : gimple_seq_add_stmt (body_p, g);
11475 :
11476 173 : tree down = create_tmp_var (unsigned_type_node);
11477 173 : g = gimple_build_assign (down, build_zero_cst (unsigned_type_node));
11478 173 : gimple_seq_add_stmt (body_p, g);
11479 :
11480 173 : g = gimple_build_assign (k, build_one_cst (unsigned_type_node));
11481 173 : gimple_seq_add_stmt (body_p, g);
11482 :
11483 173 : tree num_threadsu = create_tmp_var (unsigned_type_node);
11484 173 : g = gimple_build_assign (num_threadsu, NOP_EXPR, num_threads);
11485 173 : gimple_seq_add_stmt (body_p, g);
11486 :
11487 173 : tree thread_numu = create_tmp_var (unsigned_type_node);
11488 173 : g = gimple_build_assign (thread_numu, NOP_EXPR, thread_num);
11489 173 : gimple_seq_add_stmt (body_p, g);
11490 :
11491 173 : tree thread_nump1 = create_tmp_var (unsigned_type_node);
11492 173 : g = gimple_build_assign (thread_nump1, PLUS_EXPR, thread_numu,
11493 : build_int_cst (unsigned_type_node, 1));
11494 173 : gimple_seq_add_stmt (body_p, g);
11495 :
11496 173 : lab3 = create_artificial_label (UNKNOWN_LOCATION);
11497 173 : g = gimple_build_label (lab3);
11498 173 : gimple_seq_add_stmt (body_p, g);
11499 :
11500 173 : tree twok = create_tmp_var (unsigned_type_node);
11501 173 : g = gimple_build_assign (twok, LSHIFT_EXPR, k, integer_one_node);
11502 173 : gimple_seq_add_stmt (body_p, g);
11503 :
11504 173 : tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
11505 173 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
11506 173 : tree lab6 = create_artificial_label (UNKNOWN_LOCATION);
11507 173 : g = gimple_build_cond (GT_EXPR, twok, num_threadsu, lab4, lab5);
11508 173 : gimple_seq_add_stmt (body_p, g);
11509 173 : g = gimple_build_label (lab4);
11510 173 : gimple_seq_add_stmt (body_p, g);
11511 173 : g = gimple_build_assign (down, build_all_ones_cst (unsigned_type_node));
11512 173 : gimple_seq_add_stmt (body_p, g);
11513 173 : g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
11514 173 : gimple_seq_add_stmt (body_p, g);
11515 :
11516 173 : g = gimple_build_cond (EQ_EXPR, k, num_threadsu, lab6, lab5);
11517 173 : gimple_seq_add_stmt (body_p, g);
11518 173 : g = gimple_build_label (lab6);
11519 173 : gimple_seq_add_stmt (body_p, g);
11520 :
11521 173 : g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
11522 173 : gimple_seq_add_stmt (body_p, g);
11523 :
11524 173 : g = gimple_build_label (lab5);
11525 173 : gimple_seq_add_stmt (body_p, g);
11526 :
11527 173 : g = gimple_build_assign (twok, LSHIFT_EXPR, k, integer_one_node);
11528 173 : gimple_seq_add_stmt (body_p, g);
11529 :
11530 173 : tree cplx = create_tmp_var (build_complex_type (unsigned_type_node, false));
11531 173 : g = gimple_build_call_internal (IFN_MUL_OVERFLOW, 2, thread_nump1, twok);
11532 173 : gimple_call_set_lhs (g, cplx);
11533 173 : gimple_seq_add_stmt (body_p, g);
11534 173 : tree mul = create_tmp_var (unsigned_type_node);
11535 173 : g = gimple_build_assign (mul, REALPART_EXPR,
11536 : build1 (REALPART_EXPR, unsigned_type_node, cplx));
11537 173 : gimple_seq_add_stmt (body_p, g);
11538 173 : tree ovf = create_tmp_var (unsigned_type_node);
11539 173 : g = gimple_build_assign (ovf, IMAGPART_EXPR,
11540 : build1 (IMAGPART_EXPR, unsigned_type_node, cplx));
11541 173 : gimple_seq_add_stmt (body_p, g);
11542 :
11543 173 : tree lab7 = create_artificial_label (UNKNOWN_LOCATION);
11544 173 : tree lab8 = create_artificial_label (UNKNOWN_LOCATION);
11545 173 : g = gimple_build_cond (EQ_EXPR, ovf, build_zero_cst (unsigned_type_node),
11546 : lab7, lab8);
11547 173 : gimple_seq_add_stmt (body_p, g);
11548 173 : g = gimple_build_label (lab7);
11549 173 : gimple_seq_add_stmt (body_p, g);
11550 :
11551 173 : tree andv = create_tmp_var (unsigned_type_node);
11552 173 : g = gimple_build_assign (andv, BIT_AND_EXPR, k, down);
11553 173 : gimple_seq_add_stmt (body_p, g);
11554 173 : tree andvm1 = create_tmp_var (unsigned_type_node);
11555 173 : g = gimple_build_assign (andvm1, PLUS_EXPR, andv,
11556 : build_minus_one_cst (unsigned_type_node));
11557 173 : gimple_seq_add_stmt (body_p, g);
11558 :
11559 173 : g = gimple_build_assign (l, PLUS_EXPR, mul, andvm1);
11560 173 : gimple_seq_add_stmt (body_p, g);
11561 :
11562 173 : tree lab9 = create_artificial_label (UNKNOWN_LOCATION);
11563 173 : g = gimple_build_cond (LT_EXPR, l, num_threadsu, lab9, lab8);
11564 173 : gimple_seq_add_stmt (body_p, g);
11565 173 : g = gimple_build_label (lab9);
11566 173 : gimple_seq_add_stmt (body_p, g);
11567 173 : gimple_seq_add_seq (body_p, reduc_list);
11568 173 : g = gimple_build_label (lab8);
11569 173 : gimple_seq_add_stmt (body_p, g);
11570 :
11571 173 : tree lab10 = create_artificial_label (UNKNOWN_LOCATION);
11572 173 : tree lab11 = create_artificial_label (UNKNOWN_LOCATION);
11573 173 : tree lab12 = create_artificial_label (UNKNOWN_LOCATION);
11574 173 : g = gimple_build_cond (EQ_EXPR, down, build_zero_cst (unsigned_type_node),
11575 : lab10, lab11);
11576 173 : gimple_seq_add_stmt (body_p, g);
11577 173 : g = gimple_build_label (lab10);
11578 173 : gimple_seq_add_stmt (body_p, g);
11579 173 : g = gimple_build_assign (k, LSHIFT_EXPR, k, integer_one_node);
11580 173 : gimple_seq_add_stmt (body_p, g);
11581 173 : g = gimple_build_goto (lab12);
11582 173 : gimple_seq_add_stmt (body_p, g);
11583 173 : g = gimple_build_label (lab11);
11584 173 : gimple_seq_add_stmt (body_p, g);
11585 173 : g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
11586 173 : gimple_seq_add_stmt (body_p, g);
11587 173 : g = gimple_build_label (lab12);
11588 173 : gimple_seq_add_stmt (body_p, g);
11589 :
11590 173 : g = omp_build_barrier (NULL);
11591 173 : gimple_seq_add_stmt (body_p, g);
11592 :
11593 173 : g = gimple_build_cond (NE_EXPR, k, build_zero_cst (unsigned_type_node),
11594 : lab3, lab2);
11595 173 : gimple_seq_add_stmt (body_p, g);
11596 :
11597 173 : g = gimple_build_label (lab2);
11598 173 : gimple_seq_add_stmt (body_p, g);
11599 :
11600 173 : lab1 = create_artificial_label (UNKNOWN_LOCATION);
11601 173 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
11602 173 : lab3 = create_artificial_label (UNKNOWN_LOCATION);
11603 173 : g = gimple_build_cond (EQ_EXPR, thread_num, integer_zero_node, lab1, lab2);
11604 173 : gimple_seq_add_stmt (body_p, g);
11605 173 : g = gimple_build_label (lab1);
11606 173 : gimple_seq_add_stmt (body_p, g);
11607 173 : gimple_seq_add_seq (body_p, thr02_list);
11608 173 : g = gimple_build_goto (lab3);
11609 173 : gimple_seq_add_stmt (body_p, g);
11610 173 : g = gimple_build_label (lab2);
11611 173 : gimple_seq_add_stmt (body_p, g);
11612 173 : gimple_seq_add_seq (body_p, thrn2_list);
11613 173 : g = gimple_build_label (lab3);
11614 173 : gimple_seq_add_stmt (body_p, g);
11615 :
11616 173 : g = gimple_build_assign (ivar, size_zero_node);
11617 173 : gimple_seq_add_stmt (body_p, g);
11618 173 : gimple_seq_add_stmt (body_p, new_stmt);
11619 173 : gimple_seq_add_seq (body_p, new_body);
11620 :
11621 173 : gimple_seq new_dlist = NULL;
11622 173 : lab1 = create_artificial_label (UNKNOWN_LOCATION);
11623 173 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
11624 173 : tree num_threadsm1 = create_tmp_var (integer_type_node);
11625 173 : g = gimple_build_assign (num_threadsm1, PLUS_EXPR, num_threads,
11626 : integer_minus_one_node);
11627 173 : gimple_seq_add_stmt (&new_dlist, g);
11628 173 : g = gimple_build_cond (EQ_EXPR, thread_num, num_threadsm1, lab1, lab2);
11629 173 : gimple_seq_add_stmt (&new_dlist, g);
11630 173 : g = gimple_build_label (lab1);
11631 173 : gimple_seq_add_stmt (&new_dlist, g);
11632 173 : gimple_seq_add_seq (&new_dlist, last_list);
11633 173 : g = gimple_build_label (lab2);
11634 173 : gimple_seq_add_stmt (&new_dlist, g);
11635 173 : gimple_seq_add_seq (&new_dlist, *dlist);
11636 173 : *dlist = new_dlist;
11637 173 : }
11638 :
11639 : /* Build an internal UNIQUE function with type IFN_UNIQUE_OACC_PRIVATE listing
11640 : the addresses of variables to be made private at the surrounding
11641 : parallelism level. Such functions appear in the gimple code stream in two
11642 : forms, e.g. for a partitioned loop:
11643 :
11644 : .data_dep.6 = .UNIQUE (OACC_HEAD_MARK, .data_dep.6, 1, 68);
11645 : .data_dep.6 = .UNIQUE (OACC_PRIVATE, .data_dep.6, -1, &w);
11646 : .data_dep.6 = .UNIQUE (OACC_FORK, .data_dep.6, -1);
11647 : .data_dep.6 = .UNIQUE (OACC_HEAD_MARK, .data_dep.6);
11648 :
11649 : or alternatively, OACC_PRIVATE can appear at the top level of a parallel,
11650 : not as part of a HEAD_MARK sequence:
11651 :
11652 : .UNIQUE (OACC_PRIVATE, 0, 0, &w);
11653 :
11654 : For such stand-alone appearances, the 3rd argument is always 0, denoting
11655 : gang partitioning. */
11656 :
11657 : static gcall *
11658 19728 : lower_oacc_private_marker (omp_context *ctx)
11659 : {
11660 19728 : if (ctx->oacc_privatization_candidates.length () == 0)
11661 : return NULL;
11662 :
11663 308 : auto_vec<tree, 5> args;
11664 :
11665 308 : args.quick_push (build_int_cst (integer_type_node, IFN_UNIQUE_OACC_PRIVATE));
11666 308 : args.quick_push (integer_zero_node);
11667 308 : args.quick_push (integer_minus_one_node);
11668 :
11669 308 : int i;
11670 308 : tree decl;
11671 696 : FOR_EACH_VEC_ELT (ctx->oacc_privatization_candidates, i, decl)
11672 : {
11673 388 : gcc_checking_assert (TREE_ADDRESSABLE (decl));
11674 388 : tree addr = build_fold_addr_expr (decl);
11675 388 : args.safe_push (addr);
11676 : }
11677 :
11678 308 : return gimple_build_call_internal_vec (IFN_UNIQUE, args);
11679 308 : }
11680 :
11681 : /* Lower code for an OMP loop directive. */
11682 :
11683 : static void
11684 46890 : lower_omp_for (gimple_stmt_iterator *gsi_p, omp_context *ctx)
11685 : {
11686 46890 : tree *rhs_p, block;
11687 46890 : struct omp_for_data fd, *fdp = NULL;
11688 46890 : gomp_for *stmt = as_a <gomp_for *> (gsi_stmt (*gsi_p));
11689 46890 : gbind *new_stmt;
11690 46890 : gimple_seq omp_for_body, body, dlist, tred_ilist = NULL, tred_dlist = NULL;
11691 46890 : gimple_seq cnt_list = NULL, clist = NULL;
11692 46890 : gimple_seq oacc_head = NULL, oacc_tail = NULL;
11693 46890 : size_t i;
11694 :
11695 46890 : push_gimplify_context ();
11696 :
11697 46890 : if (is_gimple_omp_oacc (ctx->stmt))
11698 11325 : oacc_privatization_scan_clause_chain (ctx, gimple_omp_for_clauses (stmt));
11699 :
11700 46890 : lower_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
11701 :
11702 46890 : block = make_node (BLOCK);
11703 46890 : new_stmt = gimple_build_bind (NULL, NULL, block);
11704 : /* Replace at gsi right away, so that 'stmt' is no member
11705 : of a sequence anymore as we're going to add to a different
11706 : one below. */
11707 46890 : gsi_replace (gsi_p, new_stmt, true);
11708 :
11709 : /* Move declaration of temporaries in the loop body before we make
11710 : it go away. */
11711 46890 : omp_for_body = gimple_omp_body (stmt);
11712 46890 : if (!gimple_seq_empty_p (omp_for_body)
11713 46890 : && gimple_code (gimple_seq_first_stmt (omp_for_body)) == GIMPLE_BIND)
11714 : {
11715 19609 : gbind *inner_bind
11716 19609 : = as_a <gbind *> (gimple_seq_first_stmt (omp_for_body));
11717 19609 : tree vars = gimple_bind_vars (inner_bind);
11718 19609 : if (is_gimple_omp_oacc (ctx->stmt))
11719 5305 : oacc_privatization_scan_decl_chain (ctx, vars);
11720 19609 : gimple_bind_append_vars (new_stmt, vars);
11721 : /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
11722 : keep them on the inner_bind and it's block. */
11723 19609 : gimple_bind_set_vars (inner_bind, NULL_TREE);
11724 19609 : if (gimple_bind_block (inner_bind))
11725 17053 : BLOCK_VARS (gimple_bind_block (inner_bind)) = NULL_TREE;
11726 : }
11727 :
11728 46890 : if (gimple_omp_for_combined_into_p (stmt))
11729 : {
11730 14598 : omp_extract_for_data (stmt, &fd, NULL);
11731 14598 : fdp = &fd;
11732 :
11733 : /* We need two temporaries with fd.loop.v type (istart/iend)
11734 : and then (fd.collapse - 1) temporaries with the same
11735 : type for count2 ... countN-1 vars if not constant. */
11736 14598 : size_t count = 2;
11737 14598 : tree type = fd.iter_type;
11738 14598 : if (fd.collapse > 1
11739 4867 : && TREE_CODE (fd.loop.n2) != INTEGER_CST)
11740 2692 : count += fd.collapse - 1;
11741 14598 : size_t count2 = 0;
11742 14598 : tree type2 = NULL_TREE;
11743 14598 : bool taskreg_for
11744 14598 : = (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR
11745 14598 : || gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_TASKLOOP);
11746 14598 : tree outerc = NULL, *pc = gimple_omp_for_clauses_ptr (stmt);
11747 14598 : tree simtc = NULL;
11748 14598 : tree clauses = *pc;
11749 14598 : if (fd.collapse > 1
11750 4867 : && fd.non_rect
11751 137 : && fd.last_nonrect == fd.first_nonrect + 1
11752 89 : && TREE_CODE (fd.loop.n2) != INTEGER_CST)
11753 64 : if (tree v = gimple_omp_for_index (stmt, fd.last_nonrect))
11754 64 : if (!TYPE_UNSIGNED (TREE_TYPE (v)))
11755 : {
11756 60 : v = gimple_omp_for_index (stmt, fd.first_nonrect);
11757 60 : type2 = TREE_TYPE (v);
11758 60 : count++;
11759 60 : count2 = 3;
11760 : }
11761 14598 : if (taskreg_for)
11762 7594 : outerc
11763 7594 : = omp_find_clause (gimple_omp_taskreg_clauses (ctx->outer->stmt),
11764 : OMP_CLAUSE__LOOPTEMP_);
11765 14598 : if (ctx->simt_stmt)
11766 0 : simtc = omp_find_clause (gimple_omp_for_clauses (ctx->simt_stmt),
11767 : OMP_CLAUSE__LOOPTEMP_);
11768 49203 : for (i = 0; i < count + count2; i++)
11769 : {
11770 34605 : tree temp;
11771 34605 : if (taskreg_for)
11772 : {
11773 18069 : gcc_assert (outerc);
11774 18069 : temp = lookup_decl (OMP_CLAUSE_DECL (outerc), ctx->outer);
11775 18069 : outerc = omp_find_clause (OMP_CLAUSE_CHAIN (outerc),
11776 : OMP_CLAUSE__LOOPTEMP_);
11777 : }
11778 : else
11779 : {
11780 : /* If there are 2 adjacent SIMD stmts, one with _simt_
11781 : clause, another without, make sure they have the same
11782 : decls in _looptemp_ clauses, because the outer stmt
11783 : they are combined into will look up just one inner_stmt. */
11784 16536 : if (ctx->simt_stmt)
11785 0 : temp = OMP_CLAUSE_DECL (simtc);
11786 : else
11787 32928 : temp = create_tmp_var (i >= count ? type2 : type);
11788 16536 : insert_decl_map (&ctx->outer->cb, temp, temp);
11789 : }
11790 34605 : *pc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
11791 34605 : OMP_CLAUSE_DECL (*pc) = temp;
11792 34605 : pc = &OMP_CLAUSE_CHAIN (*pc);
11793 34605 : if (ctx->simt_stmt)
11794 0 : simtc = omp_find_clause (OMP_CLAUSE_CHAIN (simtc),
11795 : OMP_CLAUSE__LOOPTEMP_);
11796 : }
11797 14598 : *pc = clauses;
11798 : }
11799 :
11800 : /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
11801 46890 : dlist = NULL;
11802 46890 : body = NULL;
11803 46890 : tree rclauses
11804 46890 : = omp_task_reductions_find_first (gimple_omp_for_clauses (stmt), OMP_FOR,
11805 : OMP_CLAUSE_REDUCTION);
11806 46890 : tree rtmp = NULL_TREE;
11807 46890 : if (rclauses)
11808 : {
11809 227 : tree type = build_pointer_type (pointer_sized_int_node);
11810 227 : tree temp = create_tmp_var (type);
11811 227 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
11812 227 : OMP_CLAUSE_DECL (c) = temp;
11813 227 : OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (stmt);
11814 227 : gimple_omp_for_set_clauses (stmt, c);
11815 227 : lower_omp_task_reductions (ctx, OMP_FOR,
11816 : gimple_omp_for_clauses (stmt),
11817 : &tred_ilist, &tred_dlist);
11818 227 : rclauses = c;
11819 227 : rtmp = make_ssa_name (type);
11820 227 : gimple_seq_add_stmt (&body, gimple_build_assign (rtmp, temp));
11821 : }
11822 :
11823 46890 : lower_lastprivate_conditional_clauses (gimple_omp_for_clauses_ptr (stmt),
11824 : ctx);
11825 :
11826 46890 : lower_rec_input_clauses (gimple_omp_for_clauses (stmt), &body, &dlist, ctx,
11827 : fdp);
11828 93553 : gimple_seq_add_seq (rclauses ? &tred_ilist : &body,
11829 : gimple_omp_for_pre_body (stmt));
11830 :
11831 46890 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
11832 :
11833 46890 : gcall *private_marker = NULL;
11834 46890 : if (is_gimple_omp_oacc (ctx->stmt)
11835 46890 : && !gimple_seq_empty_p (omp_for_body))
11836 10333 : private_marker = lower_oacc_private_marker (ctx);
11837 :
11838 : /* Lower the header expressions. At this point, we can assume that
11839 : the header is of the form:
11840 :
11841 : #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
11842 :
11843 : We just need to make sure that VAL1, VAL2 and VAL3 are lowered
11844 : using the .omp_data_s mapping, if needed. */
11845 115064 : for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
11846 : {
11847 68174 : rhs_p = gimple_omp_for_initial_ptr (stmt, i);
11848 68174 : if (TREE_CODE (*rhs_p) == TREE_VEC)
11849 : {
11850 639 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 1)))
11851 149 : TREE_VEC_ELT (*rhs_p, 1)
11852 298 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 1), &cnt_list);
11853 639 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 2)))
11854 161 : TREE_VEC_ELT (*rhs_p, 2)
11855 322 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 2), &cnt_list);
11856 : }
11857 67535 : else if (!is_gimple_min_invariant (*rhs_p))
11858 11465 : *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
11859 56070 : else if (TREE_CODE (*rhs_p) == ADDR_EXPR)
11860 4713 : recompute_tree_invariant_for_addr_expr (*rhs_p);
11861 :
11862 68174 : rhs_p = gimple_omp_for_final_ptr (stmt, i);
11863 68174 : if (TREE_CODE (*rhs_p) == TREE_VEC)
11864 : {
11865 519 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 1)))
11866 142 : TREE_VEC_ELT (*rhs_p, 1)
11867 284 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 1), &cnt_list);
11868 519 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 2)))
11869 157 : TREE_VEC_ELT (*rhs_p, 2)
11870 314 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 2), &cnt_list);
11871 : }
11872 67655 : else if (!is_gimple_min_invariant (*rhs_p))
11873 16787 : *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
11874 50868 : else if (TREE_CODE (*rhs_p) == ADDR_EXPR)
11875 4630 : recompute_tree_invariant_for_addr_expr (*rhs_p);
11876 :
11877 68174 : rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1);
11878 68174 : if (!is_gimple_min_invariant (*rhs_p))
11879 6318 : *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
11880 : }
11881 46890 : if (rclauses)
11882 227 : gimple_seq_add_seq (&tred_ilist, cnt_list);
11883 : else
11884 46663 : gimple_seq_add_seq (&body, cnt_list);
11885 :
11886 : /* Once lowered, extract the bounds and clauses. */
11887 46890 : omp_extract_for_data (stmt, &fd, NULL);
11888 :
11889 46890 : if (is_gimple_omp_oacc (ctx->stmt)
11890 46890 : && !ctx_in_oacc_kernels_region (ctx))
11891 9665 : lower_oacc_head_tail (gimple_location (stmt),
11892 : gimple_omp_for_clauses (stmt), private_marker,
11893 : &oacc_head, &oacc_tail, ctx);
11894 :
11895 : /* Add OpenACC partitioning and reduction markers just before the loop. */
11896 46890 : if (oacc_head)
11897 9665 : gimple_seq_add_seq (&body, oacc_head);
11898 :
11899 46890 : lower_omp_for_lastprivate (&fd, &body, &dlist, &clist, ctx);
11900 :
11901 46890 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
11902 85802 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
11903 70230 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
11904 70230 : && !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
11905 : {
11906 227 : OMP_CLAUSE_DECL (c) = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
11907 227 : if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c)))
11908 42 : OMP_CLAUSE_LINEAR_STEP (c)
11909 84 : = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c),
11910 : ctx);
11911 : }
11912 :
11913 46524 : if ((ctx->scan_inclusive || ctx->scan_exclusive)
11914 47158 : && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
11915 173 : lower_omp_for_scan (&body, &dlist, stmt, &fd, ctx);
11916 : else
11917 : {
11918 46717 : gimple_seq_add_stmt (&body, stmt);
11919 46717 : gimple_seq_add_seq (&body, gimple_omp_body (stmt));
11920 : }
11921 :
11922 46890 : gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v,
11923 : fd.loop.v));
11924 :
11925 : /* After the loop, add exit clauses. */
11926 46890 : lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, &clist, ctx);
11927 :
11928 46890 : if (clist)
11929 : {
11930 179 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START);
11931 179 : gcall *g = gimple_build_call (fndecl, 0);
11932 179 : gimple_seq_add_stmt (&body, g);
11933 179 : gimple_seq_add_seq (&body, clist);
11934 179 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END);
11935 179 : g = gimple_build_call (fndecl, 0);
11936 179 : gimple_seq_add_stmt (&body, g);
11937 : }
11938 :
11939 46890 : if (ctx->cancellable)
11940 84 : gimple_seq_add_stmt (&body, gimple_build_label (ctx->cancel_label));
11941 :
11942 46890 : gimple_seq_add_seq (&body, dlist);
11943 :
11944 46890 : if (rclauses)
11945 : {
11946 227 : gimple_seq_add_seq (&tred_ilist, body);
11947 227 : body = tred_ilist;
11948 : }
11949 :
11950 46890 : body = maybe_catch_exception (body);
11951 :
11952 : /* Region exit marker goes at the end of the loop body. */
11953 46890 : gimple *g = gimple_build_omp_return (fd.have_nowait);
11954 46890 : gimple_seq_add_stmt (&body, g);
11955 :
11956 46890 : gimple_seq_add_seq (&body, tred_dlist);
11957 :
11958 46890 : maybe_add_implicit_barrier_cancel (ctx, g, &body);
11959 :
11960 46890 : if (rclauses)
11961 227 : OMP_CLAUSE_DECL (rclauses) = rtmp;
11962 :
11963 : /* Add OpenACC joining and reduction markers just after the loop. */
11964 46890 : if (oacc_tail)
11965 9665 : gimple_seq_add_seq (&body, oacc_tail);
11966 :
11967 46890 : pop_gimplify_context (new_stmt);
11968 :
11969 46890 : gimple_bind_append_vars (new_stmt, ctx->block_vars);
11970 46890 : maybe_remove_omp_member_access_dummy_vars (new_stmt);
11971 46890 : BLOCK_VARS (block) = gimple_bind_vars (new_stmt);
11972 46890 : if (BLOCK_VARS (block))
11973 46247 : TREE_USED (block) = 1;
11974 :
11975 46890 : gimple_bind_set_body (new_stmt, body);
11976 46890 : gimple_omp_set_body (stmt, NULL);
11977 46890 : gimple_omp_for_set_pre_body (stmt, NULL);
11978 46890 : }
11979 :
11980 : /* Callback for walk_stmts. Check if the current statement only contains
11981 : GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
11982 :
11983 : static tree
11984 214162 : check_combined_parallel (gimple_stmt_iterator *gsi_p,
11985 : bool *handled_ops_p,
11986 : struct walk_stmt_info *wi)
11987 : {
11988 214162 : int *info = (int *) wi->info;
11989 214162 : gimple *stmt = gsi_stmt (*gsi_p);
11990 :
11991 214162 : *handled_ops_p = true;
11992 214162 : switch (gimple_code (stmt))
11993 : {
11994 11665 : WALK_SUBSTMTS;
11995 :
11996 : case GIMPLE_DEBUG:
11997 : break;
11998 1901 : case GIMPLE_OMP_FOR:
11999 1901 : case GIMPLE_OMP_SECTIONS:
12000 1901 : *info = *info == 0 ? 1 : -1;
12001 1901 : break;
12002 200557 : default:
12003 200557 : *info = -1;
12004 200557 : break;
12005 : }
12006 214162 : return NULL;
12007 : }
12008 :
12009 : struct omp_taskcopy_context
12010 : {
12011 : /* This field must be at the beginning, as we do "inheritance": Some
12012 : callback functions for tree-inline.cc (e.g., omp_copy_decl)
12013 : receive a copy_body_data pointer that is up-casted to an
12014 : omp_context pointer. */
12015 : copy_body_data cb;
12016 : omp_context *ctx;
12017 : };
12018 :
12019 : static tree
12020 366 : task_copyfn_copy_decl (tree var, copy_body_data *cb)
12021 : {
12022 366 : struct omp_taskcopy_context *tcctx = (struct omp_taskcopy_context *) cb;
12023 :
12024 366 : if (splay_tree_lookup (tcctx->ctx->sfield_map, (splay_tree_key) var))
12025 158 : return create_tmp_var (TREE_TYPE (var));
12026 :
12027 : return var;
12028 : }
12029 :
12030 : static tree
12031 50 : task_copyfn_remap_type (struct omp_taskcopy_context *tcctx, tree orig_type)
12032 : {
12033 50 : tree name, new_fields = NULL, type, f;
12034 :
12035 50 : type = lang_hooks.types.make_type (RECORD_TYPE);
12036 50 : name = DECL_NAME (TYPE_NAME (orig_type));
12037 50 : name = build_decl (gimple_location (tcctx->ctx->stmt),
12038 : TYPE_DECL, name, type);
12039 50 : TYPE_NAME (type) = name;
12040 :
12041 1140 : for (f = TYPE_FIELDS (orig_type); f ; f = TREE_CHAIN (f))
12042 : {
12043 1090 : tree new_f = copy_node (f);
12044 1090 : DECL_CONTEXT (new_f) = type;
12045 1090 : TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &tcctx->cb);
12046 1090 : TREE_CHAIN (new_f) = new_fields;
12047 1090 : walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &tcctx->cb, NULL);
12048 1090 : walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, &tcctx->cb, NULL);
12049 1090 : walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
12050 : &tcctx->cb, NULL);
12051 1090 : new_fields = new_f;
12052 1090 : tcctx->cb.decl_map->put (f, new_f);
12053 : }
12054 50 : TYPE_FIELDS (type) = nreverse (new_fields);
12055 50 : layout_type (type);
12056 50 : return type;
12057 : }
12058 :
12059 : /* Create task copyfn. */
12060 :
12061 : static void
12062 504 : create_task_copyfn (gomp_task *task_stmt, omp_context *ctx)
12063 : {
12064 504 : struct function *child_cfun;
12065 504 : tree child_fn, t, c, src, dst, f, sf, arg, sarg, decl;
12066 504 : tree record_type, srecord_type, bind, list;
12067 504 : bool record_needs_remap = false, srecord_needs_remap = false;
12068 504 : splay_tree_node n;
12069 504 : struct omp_taskcopy_context tcctx;
12070 504 : location_t loc = gimple_location (task_stmt);
12071 504 : size_t looptempno = 0;
12072 :
12073 504 : child_fn = gimple_omp_task_copy_fn (task_stmt);
12074 504 : task_cpyfns.safe_push (task_stmt);
12075 504 : child_cfun = DECL_STRUCT_FUNCTION (child_fn);
12076 504 : gcc_assert (child_cfun->cfg == NULL);
12077 504 : DECL_SAVED_TREE (child_fn) = alloc_stmt_list ();
12078 :
12079 : /* Reset DECL_CONTEXT on function arguments. */
12080 1512 : for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
12081 1008 : DECL_CONTEXT (t) = child_fn;
12082 :
12083 : /* Populate the function. */
12084 504 : push_gimplify_context ();
12085 504 : push_cfun (child_cfun);
12086 :
12087 504 : bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
12088 504 : TREE_SIDE_EFFECTS (bind) = 1;
12089 504 : list = NULL;
12090 504 : DECL_SAVED_TREE (child_fn) = bind;
12091 504 : DECL_SOURCE_LOCATION (child_fn) = gimple_location (task_stmt);
12092 :
12093 : /* Remap src and dst argument types if needed. */
12094 504 : record_type = ctx->record_type;
12095 504 : srecord_type = ctx->srecord_type;
12096 3100 : for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
12097 2621 : if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
12098 : {
12099 : record_needs_remap = true;
12100 : break;
12101 : }
12102 2839 : for (f = TYPE_FIELDS (srecord_type); f ; f = DECL_CHAIN (f))
12103 2360 : if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
12104 : {
12105 : srecord_needs_remap = true;
12106 : break;
12107 : }
12108 :
12109 504 : if (record_needs_remap || srecord_needs_remap)
12110 : {
12111 25 : memset (&tcctx, '\0', sizeof (tcctx));
12112 25 : tcctx.cb.src_fn = ctx->cb.src_fn;
12113 25 : tcctx.cb.dst_fn = child_fn;
12114 25 : tcctx.cb.src_node = cgraph_node::get (tcctx.cb.src_fn);
12115 25 : gcc_checking_assert (tcctx.cb.src_node);
12116 25 : tcctx.cb.dst_node = tcctx.cb.src_node;
12117 25 : tcctx.cb.src_cfun = ctx->cb.src_cfun;
12118 25 : tcctx.cb.copy_decl = task_copyfn_copy_decl;
12119 25 : tcctx.cb.eh_lp_nr = 0;
12120 25 : tcctx.cb.transform_call_graph_edges = CB_CGE_MOVE;
12121 25 : tcctx.cb.decl_map = new hash_map<tree, tree>;
12122 25 : tcctx.ctx = ctx;
12123 :
12124 25 : if (record_needs_remap)
12125 25 : record_type = task_copyfn_remap_type (&tcctx, record_type);
12126 25 : if (srecord_needs_remap)
12127 25 : srecord_type = task_copyfn_remap_type (&tcctx, srecord_type);
12128 : }
12129 : else
12130 479 : tcctx.cb.decl_map = NULL;
12131 :
12132 504 : arg = DECL_ARGUMENTS (child_fn);
12133 504 : TREE_TYPE (arg) = build_pointer_type (record_type);
12134 504 : sarg = DECL_CHAIN (arg);
12135 504 : TREE_TYPE (sarg) = build_pointer_type (srecord_type);
12136 :
12137 : /* First pass: initialize temporaries used in record_type and srecord_type
12138 : sizes and field offsets. */
12139 504 : if (tcctx.cb.decl_map)
12140 631 : for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
12141 606 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
12142 : {
12143 497 : tree *p;
12144 :
12145 497 : decl = OMP_CLAUSE_DECL (c);
12146 497 : p = tcctx.cb.decl_map->get (decl);
12147 497 : if (p == NULL)
12148 339 : continue;
12149 158 : n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
12150 158 : sf = (tree) n->value;
12151 158 : sf = *tcctx.cb.decl_map->get (sf);
12152 158 : src = build_simple_mem_ref_loc (loc, sarg);
12153 158 : src = omp_build_component_ref (src, sf);
12154 158 : t = build2 (MODIFY_EXPR, TREE_TYPE (*p), *p, src);
12155 158 : append_to_statement_list (t, &list);
12156 : }
12157 :
12158 : /* Second pass: copy shared var pointers and copy construct non-VLA
12159 : firstprivate vars. */
12160 5848 : for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
12161 5344 : switch (OMP_CLAUSE_CODE (c))
12162 : {
12163 581 : splay_tree_key key;
12164 581 : case OMP_CLAUSE_SHARED:
12165 581 : decl = OMP_CLAUSE_DECL (c);
12166 581 : key = (splay_tree_key) decl;
12167 581 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
12168 273 : key = (splay_tree_key) &DECL_UID (decl);
12169 581 : n = splay_tree_lookup (ctx->field_map, key);
12170 581 : if (n == NULL)
12171 : break;
12172 147 : f = (tree) n->value;
12173 147 : if (tcctx.cb.decl_map)
12174 12 : f = *tcctx.cb.decl_map->get (f);
12175 147 : n = splay_tree_lookup (ctx->sfield_map, key);
12176 147 : sf = (tree) n->value;
12177 147 : if (tcctx.cb.decl_map)
12178 12 : sf = *tcctx.cb.decl_map->get (sf);
12179 147 : src = build_simple_mem_ref_loc (loc, sarg);
12180 147 : src = omp_build_component_ref (src, sf);
12181 147 : dst = build_simple_mem_ref_loc (loc, arg);
12182 147 : dst = omp_build_component_ref (dst, f);
12183 147 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12184 147 : append_to_statement_list (t, &list);
12185 147 : break;
12186 460 : case OMP_CLAUSE_REDUCTION:
12187 460 : case OMP_CLAUSE_IN_REDUCTION:
12188 460 : decl = OMP_CLAUSE_DECL (c);
12189 460 : if (TREE_CODE (decl) == MEM_REF)
12190 : {
12191 160 : decl = TREE_OPERAND (decl, 0);
12192 160 : if (TREE_CODE (decl) == POINTER_PLUS_EXPR)
12193 38 : decl = TREE_OPERAND (decl, 0);
12194 160 : if (TREE_CODE (decl) == INDIRECT_REF
12195 128 : || TREE_CODE (decl) == ADDR_EXPR)
12196 95 : decl = TREE_OPERAND (decl, 0);
12197 : }
12198 460 : key = (splay_tree_key) decl;
12199 460 : n = splay_tree_lookup (ctx->field_map, key);
12200 460 : if (n == NULL)
12201 : break;
12202 101 : f = (tree) n->value;
12203 101 : if (tcctx.cb.decl_map)
12204 20 : f = *tcctx.cb.decl_map->get (f);
12205 101 : n = splay_tree_lookup (ctx->sfield_map, key);
12206 101 : sf = (tree) n->value;
12207 101 : if (tcctx.cb.decl_map)
12208 20 : sf = *tcctx.cb.decl_map->get (sf);
12209 101 : src = build_simple_mem_ref_loc (loc, sarg);
12210 101 : src = omp_build_component_ref (src, sf);
12211 101 : if (decl != OMP_CLAUSE_DECL (c)
12212 95 : && TREE_CODE (TREE_TYPE (decl)) == REFERENCE_TYPE
12213 173 : && TREE_CODE (TREE_TYPE (TREE_TYPE (decl))) == POINTER_TYPE)
12214 32 : src = build_simple_mem_ref_loc (loc, src);
12215 101 : dst = build_simple_mem_ref_loc (loc, arg);
12216 101 : dst = omp_build_component_ref (dst, f);
12217 101 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12218 101 : append_to_statement_list (t, &list);
12219 101 : break;
12220 754 : case OMP_CLAUSE__LOOPTEMP_:
12221 : /* Fields for first two _looptemp_ clauses are initialized by
12222 : GOMP_taskloop*, the rest are handled like firstprivate. */
12223 754 : if (looptempno < 2)
12224 : {
12225 748 : looptempno++;
12226 748 : break;
12227 : }
12228 : /* FALLTHRU */
12229 1731 : case OMP_CLAUSE__REDUCTEMP_:
12230 1731 : case OMP_CLAUSE_FIRSTPRIVATE:
12231 1731 : decl = OMP_CLAUSE_DECL (c);
12232 1731 : if (is_variable_sized (decl))
12233 : break;
12234 1717 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
12235 1717 : if (n == NULL)
12236 : break;
12237 1717 : f = (tree) n->value;
12238 1717 : if (tcctx.cb.decl_map)
12239 490 : f = *tcctx.cb.decl_map->get (f);
12240 1717 : n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
12241 1717 : if (n != NULL)
12242 : {
12243 1467 : sf = (tree) n->value;
12244 1467 : if (tcctx.cb.decl_map)
12245 490 : sf = *tcctx.cb.decl_map->get (sf);
12246 1467 : src = build_simple_mem_ref_loc (loc, sarg);
12247 1467 : src = omp_build_component_ref (src, sf);
12248 1467 : if (use_pointer_for_field (decl, NULL)
12249 1467 : || omp_privatize_by_reference (decl))
12250 317 : src = build_simple_mem_ref_loc (loc, src);
12251 : }
12252 : else
12253 : src = decl;
12254 1717 : dst = build_simple_mem_ref_loc (loc, arg);
12255 1717 : dst = omp_build_component_ref (dst, f);
12256 1717 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FIRSTPRIVATE)
12257 192 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12258 : else
12259 : {
12260 1525 : if (ctx->allocate_map)
12261 84 : if (tree *allocatorp = ctx->allocate_map->get (decl))
12262 : {
12263 50 : tree allocator = *allocatorp;
12264 50 : HOST_WIDE_INT ialign = 0;
12265 50 : if (TREE_CODE (allocator) == TREE_LIST)
12266 : {
12267 4 : ialign = tree_to_uhwi (TREE_VALUE (allocator));
12268 4 : allocator = TREE_PURPOSE (allocator);
12269 : }
12270 50 : if (TREE_CODE (allocator) != INTEGER_CST)
12271 : {
12272 22 : n = splay_tree_lookup (ctx->sfield_map,
12273 : (splay_tree_key) allocator);
12274 22 : allocator = (tree) n->value;
12275 22 : if (tcctx.cb.decl_map)
12276 0 : allocator = *tcctx.cb.decl_map->get (allocator);
12277 22 : tree a = build_simple_mem_ref_loc (loc, sarg);
12278 22 : allocator = omp_build_component_ref (a, allocator);
12279 : }
12280 50 : allocator = fold_convert (pointer_sized_int_node, allocator);
12281 50 : tree a = builtin_decl_explicit (BUILT_IN_GOMP_ALLOC);
12282 50 : tree align = build_int_cst (size_type_node,
12283 50 : MAX (ialign,
12284 : DECL_ALIGN_UNIT (decl)));
12285 50 : tree sz = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (dst)));
12286 50 : tree ptr = build_call_expr_loc (loc, a, 3, align, sz,
12287 : allocator);
12288 50 : ptr = fold_convert (TREE_TYPE (dst), ptr);
12289 50 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, ptr);
12290 50 : append_to_statement_list (t, &list);
12291 50 : dst = build_simple_mem_ref_loc (loc, dst);
12292 : }
12293 1525 : t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
12294 : }
12295 1717 : append_to_statement_list (t, &list);
12296 1717 : break;
12297 489 : case OMP_CLAUSE_PRIVATE:
12298 489 : if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c))
12299 : break;
12300 12 : decl = OMP_CLAUSE_DECL (c);
12301 12 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
12302 12 : f = (tree) n->value;
12303 12 : if (tcctx.cb.decl_map)
12304 0 : f = *tcctx.cb.decl_map->get (f);
12305 12 : n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
12306 12 : if (n != NULL)
12307 : {
12308 12 : sf = (tree) n->value;
12309 12 : if (tcctx.cb.decl_map)
12310 0 : sf = *tcctx.cb.decl_map->get (sf);
12311 12 : src = build_simple_mem_ref_loc (loc, sarg);
12312 12 : src = omp_build_component_ref (src, sf);
12313 12 : if (use_pointer_for_field (decl, NULL))
12314 12 : src = build_simple_mem_ref_loc (loc, src);
12315 : }
12316 : else
12317 : src = decl;
12318 12 : dst = build_simple_mem_ref_loc (loc, arg);
12319 12 : dst = omp_build_component_ref (dst, f);
12320 12 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12321 12 : append_to_statement_list (t, &list);
12322 12 : break;
12323 : default:
12324 : break;
12325 : }
12326 :
12327 : /* Last pass: handle VLA firstprivates. */
12328 504 : if (tcctx.cb.decl_map)
12329 631 : for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
12330 606 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
12331 : {
12332 497 : tree ind, ptr, df;
12333 :
12334 497 : decl = OMP_CLAUSE_DECL (c);
12335 497 : if (!is_variable_sized (decl))
12336 483 : continue;
12337 14 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
12338 14 : if (n == NULL)
12339 0 : continue;
12340 14 : f = (tree) n->value;
12341 14 : f = *tcctx.cb.decl_map->get (f);
12342 14 : gcc_assert (DECL_HAS_VALUE_EXPR_P (decl));
12343 14 : ind = DECL_VALUE_EXPR (decl);
12344 14 : gcc_assert (TREE_CODE (ind) == INDIRECT_REF);
12345 14 : gcc_assert (DECL_P (TREE_OPERAND (ind, 0)));
12346 42 : n = splay_tree_lookup (ctx->sfield_map,
12347 14 : (splay_tree_key) TREE_OPERAND (ind, 0));
12348 14 : sf = (tree) n->value;
12349 14 : sf = *tcctx.cb.decl_map->get (sf);
12350 14 : src = build_simple_mem_ref_loc (loc, sarg);
12351 14 : src = omp_build_component_ref (src, sf);
12352 14 : src = build_simple_mem_ref_loc (loc, src);
12353 14 : dst = build_simple_mem_ref_loc (loc, arg);
12354 14 : dst = omp_build_component_ref (dst, f);
12355 14 : t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
12356 14 : append_to_statement_list (t, &list);
12357 42 : n = splay_tree_lookup (ctx->field_map,
12358 14 : (splay_tree_key) TREE_OPERAND (ind, 0));
12359 14 : df = (tree) n->value;
12360 14 : df = *tcctx.cb.decl_map->get (df);
12361 14 : ptr = build_simple_mem_ref_loc (loc, arg);
12362 14 : ptr = omp_build_component_ref (ptr, df);
12363 14 : t = build2 (MODIFY_EXPR, TREE_TYPE (ptr), ptr,
12364 : build_fold_addr_expr_loc (loc, dst));
12365 14 : append_to_statement_list (t, &list);
12366 : }
12367 :
12368 504 : t = build1 (RETURN_EXPR, void_type_node, NULL);
12369 504 : append_to_statement_list (t, &list);
12370 :
12371 504 : if (tcctx.cb.decl_map)
12372 25 : delete tcctx.cb.decl_map;
12373 504 : pop_gimplify_context (NULL);
12374 504 : BIND_EXPR_BODY (bind) = list;
12375 504 : pop_cfun ();
12376 504 : }
12377 :
12378 : static void
12379 1553 : lower_depend_clauses (tree *pclauses, gimple_seq *iseq, gimple_seq *oseq)
12380 : {
12381 1553 : tree c, clauses;
12382 1553 : gimple *g;
12383 1553 : size_t cnt[5] = { 0, 0, 0, 0, 0 }, idx = 2, i;
12384 :
12385 1553 : clauses = omp_find_clause (*pclauses, OMP_CLAUSE_DEPEND);
12386 1553 : gcc_assert (clauses);
12387 6314 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
12388 4886 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
12389 1713 : switch (OMP_CLAUSE_DEPEND_KIND (c))
12390 : {
12391 125 : case OMP_CLAUSE_DEPEND_LAST:
12392 : /* Lowering already done at gimplification. */
12393 125 : return;
12394 481 : case OMP_CLAUSE_DEPEND_IN:
12395 481 : cnt[2]++;
12396 481 : break;
12397 969 : case OMP_CLAUSE_DEPEND_OUT:
12398 969 : case OMP_CLAUSE_DEPEND_INOUT:
12399 969 : cnt[0]++;
12400 969 : break;
12401 30 : case OMP_CLAUSE_DEPEND_MUTEXINOUTSET:
12402 30 : cnt[1]++;
12403 30 : break;
12404 76 : case OMP_CLAUSE_DEPEND_DEPOBJ:
12405 76 : cnt[3]++;
12406 76 : break;
12407 32 : case OMP_CLAUSE_DEPEND_INOUTSET:
12408 32 : cnt[4]++;
12409 32 : break;
12410 0 : default:
12411 0 : gcc_unreachable ();
12412 : }
12413 1428 : if (cnt[1] || cnt[3] || cnt[4])
12414 135 : idx = 5;
12415 1428 : size_t total = cnt[0] + cnt[1] + cnt[2] + cnt[3] + cnt[4];
12416 1428 : size_t inoutidx = total + idx;
12417 1428 : tree type = build_array_type_nelts (ptr_type_node, total + idx + 2 * cnt[4]);
12418 1428 : tree array = create_tmp_var (type);
12419 1428 : TREE_ADDRESSABLE (array) = 1;
12420 1428 : tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE,
12421 : NULL_TREE);
12422 1428 : if (idx == 5)
12423 : {
12424 135 : g = gimple_build_assign (r, build_int_cst (ptr_type_node, 0));
12425 135 : gimple_seq_add_stmt (iseq, g);
12426 135 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE,
12427 : NULL_TREE);
12428 : }
12429 1428 : g = gimple_build_assign (r, build_int_cst (ptr_type_node, total));
12430 1428 : gimple_seq_add_stmt (iseq, g);
12431 7140 : for (i = 0; i < (idx == 5 ? 3 : 1); i++)
12432 : {
12433 1698 : r = build4 (ARRAY_REF, ptr_type_node, array,
12434 1698 : size_int (i + 1 + (idx == 5)), NULL_TREE, NULL_TREE);
12435 1698 : g = gimple_build_assign (r, build_int_cst (ptr_type_node, cnt[i]));
12436 1698 : gimple_seq_add_stmt (iseq, g);
12437 : }
12438 8568 : for (i = 0; i < 5; i++)
12439 : {
12440 7140 : if (cnt[i] == 0)
12441 5656 : continue;
12442 6563 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
12443 5079 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DEPEND)
12444 3352 : continue;
12445 : else
12446 : {
12447 1727 : switch (OMP_CLAUSE_DEPEND_KIND (c))
12448 : {
12449 547 : case OMP_CLAUSE_DEPEND_IN:
12450 547 : if (i != 2)
12451 139 : continue;
12452 : break;
12453 1025 : case OMP_CLAUSE_DEPEND_OUT:
12454 1025 : case OMP_CLAUSE_DEPEND_INOUT:
12455 1025 : if (i != 0)
12456 56 : continue;
12457 : break;
12458 44 : case OMP_CLAUSE_DEPEND_MUTEXINOUTSET:
12459 44 : if (i != 1)
12460 14 : continue;
12461 : break;
12462 79 : case OMP_CLAUSE_DEPEND_DEPOBJ:
12463 79 : if (i != 3)
12464 3 : continue;
12465 : break;
12466 32 : case OMP_CLAUSE_DEPEND_INOUTSET:
12467 32 : if (i != 4)
12468 0 : continue;
12469 : break;
12470 0 : default:
12471 0 : gcc_unreachable ();
12472 : }
12473 1588 : tree t = OMP_CLAUSE_DECL (c);
12474 1588 : if (i == 4)
12475 : {
12476 32 : t = build4 (ARRAY_REF, ptr_type_node, array,
12477 32 : size_int (inoutidx), NULL_TREE, NULL_TREE);
12478 32 : t = build_fold_addr_expr (t);
12479 32 : inoutidx += 2;
12480 : }
12481 1588 : t = fold_convert (ptr_type_node, t);
12482 1588 : gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
12483 1588 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
12484 : NULL_TREE, NULL_TREE);
12485 1588 : g = gimple_build_assign (r, t);
12486 1588 : gimple_seq_add_stmt (iseq, g);
12487 : }
12488 : }
12489 1428 : if (cnt[4])
12490 81 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
12491 49 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
12492 49 : && OMP_CLAUSE_DEPEND_KIND (c) == OMP_CLAUSE_DEPEND_INOUTSET)
12493 : {
12494 32 : tree t = OMP_CLAUSE_DECL (c);
12495 32 : t = fold_convert (ptr_type_node, t);
12496 32 : gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
12497 32 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
12498 : NULL_TREE, NULL_TREE);
12499 32 : g = gimple_build_assign (r, t);
12500 32 : gimple_seq_add_stmt (iseq, g);
12501 32 : t = build_int_cst (ptr_type_node, GOMP_DEPEND_INOUTSET);
12502 32 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
12503 : NULL_TREE, NULL_TREE);
12504 32 : g = gimple_build_assign (r, t);
12505 32 : gimple_seq_add_stmt (iseq, g);
12506 : }
12507 :
12508 1428 : c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND);
12509 1428 : OMP_CLAUSE_DEPEND_KIND (c) = OMP_CLAUSE_DEPEND_LAST;
12510 1428 : OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array);
12511 1428 : OMP_CLAUSE_CHAIN (c) = *pclauses;
12512 1428 : *pclauses = c;
12513 1428 : tree clobber = build_clobber (type);
12514 1428 : g = gimple_build_assign (array, clobber);
12515 1428 : gimple_seq_add_stmt (oseq, g);
12516 : }
12517 :
12518 : /* Lower the OpenMP parallel or task directive in the current statement
12519 : in GSI_P. CTX holds context information for the directive. */
12520 :
12521 : static void
12522 22477 : lower_omp_taskreg (gimple_stmt_iterator *gsi_p, omp_context *ctx)
12523 : {
12524 22477 : tree clauses;
12525 22477 : tree child_fn, t;
12526 22477 : gimple *stmt = gsi_stmt (*gsi_p);
12527 22477 : gbind *par_bind, *bind, *dep_bind = NULL;
12528 22477 : gimple_seq par_body;
12529 22477 : location_t loc = gimple_location (stmt);
12530 :
12531 22477 : clauses = gimple_omp_taskreg_clauses (stmt);
12532 22477 : if (gimple_code (stmt) == GIMPLE_OMP_TASK
12533 22477 : && gimple_omp_task_taskwait_p (stmt))
12534 : {
12535 84 : par_bind = NULL;
12536 84 : par_body = NULL;
12537 : }
12538 : else
12539 : {
12540 22393 : par_bind
12541 22393 : = as_a <gbind *> (gimple_seq_first_stmt (gimple_omp_body (stmt)));
12542 22393 : par_body = gimple_bind_body (par_bind);
12543 : }
12544 22477 : child_fn = ctx->cb.dst_fn;
12545 22477 : if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
12546 22477 : && !gimple_omp_parallel_combined_p (stmt))
12547 : {
12548 4207 : struct walk_stmt_info wi;
12549 4207 : int ws_num = 0;
12550 :
12551 4207 : memset (&wi, 0, sizeof (wi));
12552 4207 : wi.info = &ws_num;
12553 4207 : wi.val_only = true;
12554 4207 : walk_gimple_seq (par_body, check_combined_parallel, NULL, &wi);
12555 4207 : if (ws_num == 1)
12556 377 : gimple_omp_parallel_set_combined_p (stmt, true);
12557 : }
12558 22477 : gimple_seq dep_ilist = NULL;
12559 22477 : gimple_seq dep_olist = NULL;
12560 22477 : if (gimple_code (stmt) == GIMPLE_OMP_TASK
12561 22477 : && omp_find_clause (clauses, OMP_CLAUSE_DEPEND))
12562 : {
12563 1189 : push_gimplify_context ();
12564 1189 : dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12565 1189 : lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt),
12566 : &dep_ilist, &dep_olist);
12567 : }
12568 :
12569 22477 : if (gimple_code (stmt) == GIMPLE_OMP_TASK
12570 22477 : && gimple_omp_task_taskwait_p (stmt))
12571 : {
12572 84 : if (dep_bind)
12573 : {
12574 84 : gsi_replace (gsi_p, dep_bind, true);
12575 84 : gimple_bind_add_seq (dep_bind, dep_ilist);
12576 84 : gimple_bind_add_stmt (dep_bind, stmt);
12577 84 : gimple_bind_add_seq (dep_bind, dep_olist);
12578 84 : pop_gimplify_context (dep_bind);
12579 : }
12580 84 : return;
12581 : }
12582 :
12583 22393 : if (ctx->srecord_type)
12584 504 : create_task_copyfn (as_a <gomp_task *> (stmt), ctx);
12585 :
12586 22393 : gimple_seq tskred_ilist = NULL;
12587 22393 : gimple_seq tskred_olist = NULL;
12588 22393 : if ((is_task_ctx (ctx)
12589 3778 : && gimple_omp_task_taskloop_p (ctx->stmt)
12590 1330 : && omp_find_clause (gimple_omp_task_clauses (ctx->stmt),
12591 : OMP_CLAUSE_REDUCTION))
12592 25674 : || (is_parallel_ctx (ctx)
12593 16119 : && omp_find_clause (gimple_omp_parallel_clauses (stmt),
12594 : OMP_CLAUSE__REDUCTEMP_)))
12595 : {
12596 557 : if (dep_bind == NULL)
12597 : {
12598 557 : push_gimplify_context ();
12599 557 : dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12600 : }
12601 617 : lower_omp_task_reductions (ctx, is_task_ctx (ctx) ? OMP_TASKLOOP
12602 : : OMP_PARALLEL,
12603 557 : gimple_omp_taskreg_clauses (ctx->stmt),
12604 : &tskred_ilist, &tskred_olist);
12605 : }
12606 :
12607 22393 : push_gimplify_context ();
12608 :
12609 22393 : gimple_seq par_olist = NULL;
12610 22393 : gimple_seq par_ilist = NULL;
12611 22393 : gimple_seq par_rlist = NULL;
12612 22393 : lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx, NULL);
12613 22389 : lower_omp (&par_body, ctx);
12614 22389 : if (gimple_code (stmt) != GIMPLE_OMP_TASK)
12615 18611 : lower_reduction_clauses (clauses, &par_rlist, NULL, ctx);
12616 :
12617 : /* Declare all the variables created by mapping and the variables
12618 : declared in the scope of the parallel body. */
12619 22389 : record_vars_into (ctx->block_vars, child_fn);
12620 22389 : maybe_remove_omp_member_access_dummy_vars (par_bind);
12621 22389 : record_vars_into (gimple_bind_vars (par_bind), child_fn);
12622 :
12623 22389 : if (ctx->record_type)
12624 : {
12625 17938 : ctx->sender_decl
12626 35372 : = create_tmp_var (ctx->srecord_type ? ctx->srecord_type
12627 : : ctx->record_type, ".omp_data_o");
12628 17938 : DECL_NAMELESS (ctx->sender_decl) = 1;
12629 17938 : TREE_ADDRESSABLE (ctx->sender_decl) = 1;
12630 17938 : gimple_omp_taskreg_set_data_arg (stmt, ctx->sender_decl);
12631 : }
12632 :
12633 22389 : gimple_seq olist = NULL;
12634 22389 : gimple_seq ilist = NULL;
12635 22389 : lower_send_clauses (clauses, &ilist, &olist, ctx);
12636 22389 : lower_send_shared_vars (&ilist, &olist, ctx);
12637 :
12638 22389 : if (ctx->record_type)
12639 : {
12640 17938 : tree clobber = build_clobber (TREE_TYPE (ctx->sender_decl));
12641 17938 : gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
12642 : clobber));
12643 : }
12644 :
12645 : /* Once all the expansions are done, sequence all the different
12646 : fragments inside gimple_omp_body. */
12647 :
12648 22389 : gimple_seq new_body = NULL;
12649 :
12650 22389 : if (ctx->record_type)
12651 : {
12652 17938 : t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
12653 : /* fixup_child_record_type might have changed receiver_decl's type. */
12654 17938 : t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
12655 17938 : gimple_seq_add_stmt (&new_body,
12656 17938 : gimple_build_assign (ctx->receiver_decl, t));
12657 : }
12658 :
12659 22389 : gimple_seq_add_seq (&new_body, par_ilist);
12660 22389 : gimple_seq_add_seq (&new_body, par_body);
12661 22389 : gimple_seq_add_seq (&new_body, par_rlist);
12662 22389 : if (ctx->cancellable)
12663 135 : gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
12664 22389 : gimple_seq_add_seq (&new_body, par_olist);
12665 22389 : new_body = maybe_catch_exception (new_body);
12666 22389 : if (gimple_code (stmt) == GIMPLE_OMP_TASK)
12667 3778 : gimple_seq_add_stmt (&new_body,
12668 3778 : gimple_build_omp_continue (integer_zero_node,
12669 : integer_zero_node));
12670 22389 : gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
12671 22389 : gimple_omp_set_body (stmt, new_body);
12672 :
12673 22389 : if (dep_bind && gimple_bind_block (par_bind) == NULL_TREE)
12674 497 : bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12675 : else
12676 21892 : bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind));
12677 43116 : gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
12678 22389 : gimple_bind_add_seq (bind, ilist);
12679 22389 : gimple_bind_add_stmt (bind, stmt);
12680 22389 : gimple_bind_add_seq (bind, olist);
12681 :
12682 22389 : pop_gimplify_context (NULL);
12683 :
12684 22389 : if (dep_bind)
12685 : {
12686 1662 : gimple_bind_add_seq (dep_bind, dep_ilist);
12687 1662 : gimple_bind_add_seq (dep_bind, tskred_ilist);
12688 1662 : gimple_bind_add_stmt (dep_bind, bind);
12689 1662 : gimple_bind_add_seq (dep_bind, tskred_olist);
12690 1662 : gimple_bind_add_seq (dep_bind, dep_olist);
12691 1662 : pop_gimplify_context (dep_bind);
12692 : }
12693 : }
12694 :
12695 : /* Set EXPR as the hostaddr expression that should result from the clause C
12696 : in the target statement STMT. Returns the tree that should be
12697 : passed as the hostaddr (a pointer to the array containing the expanded
12698 : hostaddrs and sizes of the clause). */
12699 :
12700 : static tree
12701 37305 : lower_omp_map_iterator_expr (tree expr, tree c, gomp_target *stmt)
12702 : {
12703 37305 : if (!OMP_CLAUSE_HAS_ITERATORS (c))
12704 : return expr;
12705 :
12706 215 : tree iterator = OMP_CLAUSE_ITERATORS (c);
12707 215 : tree elems = TREE_VEC_ELT (iterator, 7);
12708 215 : tree index = TREE_VEC_ELT (iterator, 8);
12709 215 : gimple_seq *loop_body_p = enter_omp_iterator_loop_context (c, stmt);
12710 :
12711 : /* IN LOOP BODY: */
12712 : /* elems[idx] = <expr>; */
12713 215 : tree lhs = build4 (ARRAY_REF, ptr_type_node, elems, index,
12714 : NULL_TREE, NULL_TREE);
12715 215 : tree mod_expr = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
12716 : void_type_node, lhs, expr);
12717 215 : gimplify_and_add (mod_expr, loop_body_p);
12718 215 : exit_omp_iterator_loop_context (c);
12719 :
12720 215 : return build_fold_addr_expr_with_type (elems, ptr_type_node);
12721 : }
12722 :
12723 : /* Set SIZE as the size expression that should result from the clause C
12724 : in the target statement STMT. Returns the tree that should be
12725 : passed as the clause size (a size_int with the value SIZE_MAX, indicating
12726 : that the clause uses an iterator). */
12727 :
12728 : static tree
12729 80111 : lower_omp_map_iterator_size (tree size, tree c, gomp_target *stmt)
12730 : {
12731 80111 : if (!OMP_CLAUSE_HAS_ITERATORS (c))
12732 : return size;
12733 :
12734 219 : tree iterator = OMP_CLAUSE_ITERATORS (c);
12735 219 : tree elems = TREE_VEC_ELT (iterator, 7);
12736 219 : tree index = TREE_VEC_ELT (iterator, 8);
12737 219 : gimple_seq *loop_body_p = enter_omp_iterator_loop_context (c, stmt);
12738 :
12739 : /* IN LOOP BODY: */
12740 : /* elems[idx+1] = <size>; */
12741 219 : tree lhs = build4 (ARRAY_REF, ptr_type_node, elems,
12742 : size_binop (PLUS_EXPR, index, size_int (1)),
12743 : NULL_TREE, NULL_TREE);
12744 219 : tree mod_expr = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
12745 : void_type_node, lhs, size);
12746 219 : gimplify_and_add (mod_expr, loop_body_p);
12747 219 : exit_omp_iterator_loop_context (c);
12748 :
12749 219 : return size_int (SIZE_MAX);
12750 : }
12751 :
12752 : /* Lower the GIMPLE_OMP_TARGET in the current statement
12753 : in GSI_P. CTX holds context information for the directive. */
12754 :
12755 : static void
12756 37071 : lower_omp_target (gimple_stmt_iterator *gsi_p, omp_context *ctx)
12757 : {
12758 37071 : tree clauses;
12759 37071 : tree child_fn, t, c;
12760 37071 : gomp_target *stmt = as_a <gomp_target *> (gsi_stmt (*gsi_p));
12761 37071 : gbind *tgt_bind, *bind, *dep_bind = NULL;
12762 37071 : gimple_seq tgt_body, olist, ilist, fplist, new_body;
12763 37071 : location_t loc = gimple_location (stmt);
12764 37071 : bool offloaded, data_region;
12765 37071 : unsigned int map_cnt = 0;
12766 37071 : tree in_reduction_clauses = NULL_TREE;
12767 :
12768 37071 : tree deep_map_cnt = NULL_TREE;
12769 37071 : tree deep_map_data = NULL_TREE;
12770 37071 : tree deep_map_offset_data = NULL_TREE;
12771 37071 : tree deep_map_offset = NULL_TREE;
12772 :
12773 37071 : offloaded = is_gimple_omp_offloaded (stmt);
12774 37071 : switch (gimple_omp_target_kind (stmt))
12775 : {
12776 11716 : case GF_OMP_TARGET_KIND_REGION:
12777 11716 : tree *p, *q;
12778 11716 : q = &in_reduction_clauses;
12779 78988 : for (p = gimple_omp_target_clauses_ptr (stmt); *p; )
12780 67272 : if (OMP_CLAUSE_CODE (*p) == OMP_CLAUSE_IN_REDUCTION)
12781 : {
12782 422 : *q = *p;
12783 422 : q = &OMP_CLAUSE_CHAIN (*q);
12784 422 : *p = OMP_CLAUSE_CHAIN (*p);
12785 : }
12786 : else
12787 66850 : p = &OMP_CLAUSE_CHAIN (*p);
12788 11716 : *q = NULL_TREE;
12789 11716 : *p = in_reduction_clauses;
12790 : /* FALLTHRU */
12791 : case GF_OMP_TARGET_KIND_UPDATE:
12792 : case GF_OMP_TARGET_KIND_ENTER_DATA:
12793 : case GF_OMP_TARGET_KIND_EXIT_DATA:
12794 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
12795 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
12796 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
12797 : case GF_OMP_TARGET_KIND_OACC_UPDATE:
12798 : case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
12799 : case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
12800 : case GF_OMP_TARGET_KIND_OACC_DECLARE:
12801 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
12802 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
12803 : data_region = false;
12804 : break;
12805 : case GF_OMP_TARGET_KIND_DATA:
12806 : case GF_OMP_TARGET_KIND_OACC_DATA:
12807 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
12808 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
12809 : data_region = true;
12810 : break;
12811 0 : default:
12812 0 : gcc_unreachable ();
12813 : }
12814 :
12815 : /* Ensure that requires map is written via output_offload_tables, even if only
12816 : 'target (enter/exit) data' is used in the translation unit. */
12817 37071 : if (ENABLE_OFFLOADING && (omp_requires_mask & OMP_REQUIRES_TARGET_USED))
12818 : g->have_offload = true;
12819 :
12820 37071 : clauses = gimple_omp_target_clauses (stmt);
12821 :
12822 37071 : gimple_seq dep_ilist = NULL;
12823 37071 : gimple_seq dep_olist = NULL;
12824 37071 : bool has_depend = omp_find_clause (clauses, OMP_CLAUSE_DEPEND) != NULL_TREE;
12825 37071 : if (has_depend || in_reduction_clauses)
12826 : {
12827 392 : push_gimplify_context ();
12828 392 : dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12829 392 : if (has_depend)
12830 329 : lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt),
12831 : &dep_ilist, &dep_olist);
12832 392 : if (in_reduction_clauses)
12833 314 : lower_rec_input_clauses (in_reduction_clauses, &dep_ilist, &dep_olist,
12834 : ctx, NULL);
12835 : }
12836 :
12837 37071 : tgt_bind = NULL;
12838 37071 : tgt_body = NULL;
12839 37071 : if (offloaded)
12840 : {
12841 21111 : tgt_bind = gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt));
12842 21111 : tgt_body = gimple_bind_body (tgt_bind);
12843 : }
12844 15960 : else if (data_region)
12845 4021 : tgt_body = gimple_omp_body (stmt);
12846 37071 : child_fn = ctx->cb.dst_fn;
12847 :
12848 37071 : push_gimplify_context ();
12849 37071 : fplist = NULL;
12850 :
12851 37071 : ilist = NULL;
12852 37071 : olist = NULL;
12853 :
12854 37071 : gimple_seq alloc_dlist = NULL;
12855 37071 : hash_map<tree, tree> alloc_map;
12856 74142 : hash_map<tree, gimple_seq> alloc_seq_map;
12857 :
12858 176021 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
12859 138950 : switch (OMP_CLAUSE_CODE (c))
12860 : {
12861 : tree var, x, new_var, allocator, allocate_ptr, size;
12862 : gimple_seq alloc_seq;
12863 : bool by_ref;
12864 :
12865 : default:
12866 92538 : break;
12867 77836 : case OMP_CLAUSE_MAP:
12868 : #if CHECKING_P
12869 : /* First check what we're prepared to handle in the following. */
12870 77836 : switch (OMP_CLAUSE_MAP_KIND (c))
12871 : {
12872 : case GOMP_MAP_ALLOC:
12873 : case GOMP_MAP_TO:
12874 : case GOMP_MAP_FROM:
12875 : case GOMP_MAP_TOFROM:
12876 : case GOMP_MAP_POINTER:
12877 : case GOMP_MAP_TO_PSET:
12878 : case GOMP_MAP_DELETE:
12879 : case GOMP_MAP_RELEASE:
12880 : case GOMP_MAP_ALWAYS_TO:
12881 : case GOMP_MAP_ALWAYS_FROM:
12882 : case GOMP_MAP_ALWAYS_TOFROM:
12883 : case GOMP_MAP_FORCE_PRESENT:
12884 : case GOMP_MAP_ALWAYS_PRESENT_FROM:
12885 : case GOMP_MAP_ALWAYS_PRESENT_TO:
12886 : case GOMP_MAP_ALWAYS_PRESENT_TOFROM:
12887 :
12888 : case GOMP_MAP_FIRSTPRIVATE_POINTER:
12889 : case GOMP_MAP_FIRSTPRIVATE_REFERENCE:
12890 : case GOMP_MAP_STRUCT:
12891 : case GOMP_MAP_STRUCT_UNORD:
12892 : case GOMP_MAP_ALWAYS_POINTER:
12893 : case GOMP_MAP_ATTACH:
12894 : case GOMP_MAP_DETACH:
12895 : case GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION:
12896 : case GOMP_MAP_POINTER_TO_ZERO_LENGTH_ARRAY_SECTION:
12897 : break;
12898 2587 : case GOMP_MAP_IF_PRESENT:
12899 2587 : case GOMP_MAP_FORCE_ALLOC:
12900 2587 : case GOMP_MAP_FORCE_TO:
12901 2587 : case GOMP_MAP_FORCE_FROM:
12902 2587 : case GOMP_MAP_FORCE_TOFROM:
12903 2587 : case GOMP_MAP_FORCE_DEVICEPTR:
12904 2587 : case GOMP_MAP_DEVICE_RESIDENT:
12905 2587 : case GOMP_MAP_LINK:
12906 2587 : case GOMP_MAP_FORCE_DETACH:
12907 2587 : gcc_assert (is_gimple_omp_oacc (stmt));
12908 : break;
12909 0 : default:
12910 0 : gcc_unreachable ();
12911 : }
12912 : #endif
12913 : /* FALLTHRU */
12914 89406 : case OMP_CLAUSE_TO:
12915 89406 : case OMP_CLAUSE_FROM:
12916 89406 : oacc_firstprivate:
12917 89406 : var = OMP_CLAUSE_DECL (c);
12918 89406 : {
12919 89406 : tree extra = lang_hooks.decls.omp_deep_mapping_cnt (stmt, c, &ilist);
12920 89406 : if (extra != NULL_TREE && deep_map_cnt != NULL_TREE)
12921 59 : deep_map_cnt = fold_build2_loc (OMP_CLAUSE_LOCATION (c), PLUS_EXPR,
12922 : size_type_node, deep_map_cnt,
12923 : extra);
12924 89347 : else if (extra != NULL_TREE)
12925 : deep_map_cnt = extra;
12926 : }
12927 :
12928 89262 : if (deep_map_cnt
12929 89628 : && OMP_CLAUSE_HAS_ITERATORS (c))
12930 0 : sorry ("iterators used together with deep mapping are not "
12931 : "supported yet");
12932 :
12933 89406 : if (!DECL_P (var))
12934 : {
12935 36880 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
12936 36880 : || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
12937 35274 : && (OMP_CLAUSE_MAP_KIND (c)
12938 : != GOMP_MAP_FIRSTPRIVATE_POINTER)))
12939 36880 : map_cnt++;
12940 46412 : continue;
12941 : }
12942 :
12943 52526 : if (DECL_SIZE (var)
12944 52526 : && !poly_int_tree_p (DECL_SIZE (var)))
12945 : {
12946 717 : tree var2 = DECL_VALUE_EXPR (var);
12947 717 : gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
12948 717 : var2 = TREE_OPERAND (var2, 0);
12949 717 : gcc_assert (DECL_P (var2));
12950 : var = var2;
12951 : }
12952 :
12953 52526 : if (offloaded
12954 36660 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
12955 85290 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
12956 29529 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE))
12957 : {
12958 3664 : if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
12959 : {
12960 415 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx))
12961 415 : && varpool_node::get_create (var)->offloadable)
12962 2 : continue;
12963 :
12964 413 : tree type = build_pointer_type (TREE_TYPE (var));
12965 413 : tree new_var = lookup_decl (var, ctx);
12966 413 : x = create_tmp_var_raw (type, get_name (new_var));
12967 413 : gimple_add_tmp_var (x);
12968 413 : x = build_simple_mem_ref (x);
12969 413 : SET_DECL_VALUE_EXPR (new_var, x);
12970 413 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
12971 : }
12972 3662 : continue;
12973 3662 : }
12974 :
12975 48862 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
12976 38898 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
12977 38691 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
12978 49123 : && is_omp_target (stmt))
12979 : {
12980 237 : gcc_assert (maybe_lookup_field (c, ctx));
12981 237 : map_cnt++;
12982 237 : continue;
12983 : }
12984 :
12985 48625 : if (!maybe_lookup_field (c, ctx))
12986 5631 : continue;
12987 :
12988 : /* Don't remap compute constructs' reduction variables, because the
12989 : intermediate result must be local to each gang. */
12990 70816 : if (offloaded && !(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
12991 23926 : && is_gimple_omp_oacc (ctx->stmt)
12992 13572 : && OMP_CLAUSE_MAP_IN_REDUCTION (c)))
12993 : {
12994 27042 : x = build_receiver_ref (c, true, ctx);
12995 27042 : tree new_var = lookup_decl (var, ctx);
12996 :
12997 27042 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
12998 23146 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
12999 3014 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
13000 30056 : && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
13001 425 : x = build_simple_mem_ref (x);
13002 27042 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
13003 : {
13004 3896 : gcc_assert (is_gimple_omp_oacc (ctx->stmt));
13005 3896 : if (omp_privatize_by_reference (new_var)
13006 3896 : && (TREE_CODE (TREE_TYPE (new_var)) != POINTER_TYPE
13007 44 : || DECL_BY_REFERENCE (var)))
13008 : {
13009 : /* Create a local object to hold the instance
13010 : value. */
13011 455 : tree type = TREE_TYPE (TREE_TYPE (new_var));
13012 455 : const char *id = IDENTIFIER_POINTER (DECL_NAME (new_var));
13013 455 : tree inst = create_tmp_var (type, id);
13014 455 : gimplify_assign (inst, fold_indirect_ref (x), &fplist);
13015 455 : x = build_fold_addr_expr (inst);
13016 : }
13017 3896 : gimplify_assign (new_var, x, &fplist);
13018 : }
13019 23146 : else if (DECL_P (new_var))
13020 : {
13021 23146 : SET_DECL_VALUE_EXPR (new_var, x);
13022 23146 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13023 : }
13024 : else
13025 0 : gcc_unreachable ();
13026 : }
13027 42994 : map_cnt++;
13028 42994 : break;
13029 :
13030 15296 : case OMP_CLAUSE_FIRSTPRIVATE:
13031 15296 : omp_firstprivate_recv:
13032 15296 : gcc_checking_assert (offloaded);
13033 15296 : if (is_gimple_omp_oacc (ctx->stmt))
13034 : {
13035 : /* No 'firstprivate' clauses on OpenACC 'kernels'. */
13036 3896 : gcc_checking_assert (!is_oacc_kernels (ctx));
13037 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
13038 3896 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
13039 :
13040 3896 : goto oacc_firstprivate;
13041 : }
13042 11400 : map_cnt++;
13043 11400 : var = OMP_CLAUSE_DECL (c);
13044 11400 : new_var = lookup_decl (var, ctx);
13045 11400 : allocator = NULL_TREE;
13046 11400 : allocate_ptr = NULL_TREE;
13047 11400 : size = TREE_TYPE (var);
13048 11400 : by_ref = omp_privatize_by_reference (var);
13049 11400 : if (by_ref)
13050 822 : size = TREE_TYPE (size);
13051 11400 : size = TYPE_SIZE_UNIT (size);
13052 11400 : if (is_variable_sized (var, by_ref))
13053 35 : size = lookup_decl (size, ctx);
13054 11400 : alloc_seq = NULL;
13055 11400 : if (lower_private_allocate (var, new_var, allocator, allocate_ptr,
13056 : &alloc_seq, ctx, by_ref, size))
13057 : {
13058 63 : alloc_map.put (new_var, allocate_ptr);
13059 63 : alloc_seq_map.put (new_var, alloc_seq);
13060 : }
13061 11400 : if (!by_ref && !is_gimple_reg_type (TREE_TYPE (var)))
13062 : {
13063 143 : if (is_variable_sized (var))
13064 : {
13065 6 : tree pvar = DECL_VALUE_EXPR (var);
13066 6 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
13067 6 : pvar = TREE_OPERAND (pvar, 0);
13068 6 : gcc_assert (DECL_P (pvar));
13069 6 : tree new_pvar = lookup_decl (pvar, ctx);
13070 6 : x = build_fold_indirect_ref (new_pvar);
13071 6 : TREE_THIS_NOTRAP (x) = 1;
13072 : }
13073 137 : else if (allocate_ptr)
13074 19 : x = build_fold_indirect_ref (allocate_ptr);
13075 : else
13076 118 : x = build_receiver_ref (var, true, ctx);
13077 143 : SET_DECL_VALUE_EXPR (new_var, x);
13078 143 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13079 : }
13080 : /* Fortran array descriptors: firstprivate of data + attach. */
13081 11400 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
13082 11381 : && lang_hooks.decls.omp_array_data (var, true)
13083 11431 : && lang_hooks.decls.omp_array_data_privatize (var))
13084 23 : map_cnt += 2;
13085 :
13086 11546 : do_dtor:
13087 11546 : if (allocator)
13088 : {
13089 118 : if (!is_gimple_val (allocator))
13090 : {
13091 0 : tree avar = create_tmp_var (TREE_TYPE (allocator));
13092 0 : gimplify_assign (avar, allocator, &alloc_dlist);
13093 0 : allocator = avar;
13094 : }
13095 118 : if (!is_gimple_val (allocate_ptr))
13096 : {
13097 0 : tree apvar = create_tmp_var (TREE_TYPE (allocate_ptr));
13098 0 : gimplify_assign (apvar, allocate_ptr, &alloc_dlist);
13099 0 : allocate_ptr = apvar;
13100 : }
13101 118 : tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
13102 118 : gimple *g = gimple_build_call (f, 2, allocate_ptr, allocator);
13103 118 : gimple_seq_add_stmt (&alloc_dlist, g);
13104 : }
13105 : break;
13106 :
13107 226 : case OMP_CLAUSE_PRIVATE:
13108 226 : gcc_checking_assert (offloaded);
13109 226 : if (is_gimple_omp_oacc (ctx->stmt))
13110 : {
13111 : /* No 'private' clauses on OpenACC 'kernels'. */
13112 80 : gcc_checking_assert (!is_oacc_kernels (ctx));
13113 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
13114 80 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
13115 :
13116 : break;
13117 : }
13118 146 : var = OMP_CLAUSE_DECL (c);
13119 146 : new_var = lookup_decl (var, ctx);
13120 146 : allocator = NULL_TREE;
13121 146 : allocate_ptr = NULL_TREE;
13122 146 : alloc_seq = NULL;
13123 146 : size = TREE_TYPE (var);
13124 146 : by_ref = omp_privatize_by_reference (var);
13125 146 : if (by_ref)
13126 22 : size = TREE_TYPE (size);
13127 146 : size = TYPE_SIZE_UNIT (size);
13128 146 : if (is_variable_sized (var, by_ref))
13129 21 : size = lookup_decl (size, ctx);
13130 146 : lower_private_allocate (var, new_var, allocator, allocate_ptr,
13131 : &alloc_seq, ctx, by_ref, size);
13132 146 : if (allocate_ptr)
13133 : {
13134 55 : alloc_map.put (new_var, allocate_ptr);
13135 55 : alloc_seq_map.put (new_var, alloc_seq);
13136 : }
13137 146 : if (!allocate_ptr && is_variable_sized (var))
13138 : {
13139 1 : tree new_var = lookup_decl (var, ctx);
13140 1 : tree pvar = DECL_VALUE_EXPR (var);
13141 1 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
13142 1 : pvar = TREE_OPERAND (pvar, 0);
13143 1 : gcc_assert (DECL_P (pvar));
13144 1 : tree new_pvar = lookup_decl (pvar, ctx);
13145 1 : x = build_fold_indirect_ref (new_pvar);
13146 1 : TREE_THIS_NOTRAP (x) = 1;
13147 1 : SET_DECL_VALUE_EXPR (new_var, x);
13148 1 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13149 : }
13150 146 : goto do_dtor;
13151 :
13152 2430 : case OMP_CLAUSE_USE_DEVICE_PTR:
13153 2430 : case OMP_CLAUSE_USE_DEVICE_ADDR:
13154 2430 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
13155 2430 : case OMP_CLAUSE_IS_DEVICE_PTR:
13156 2430 : var = OMP_CLAUSE_DECL (c);
13157 2430 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
13158 : {
13159 252 : while (TREE_CODE (var) == INDIRECT_REF
13160 252 : || TREE_CODE (var) == ARRAY_REF)
13161 17 : var = TREE_OPERAND (var, 0);
13162 235 : if (lang_hooks.decls.omp_array_data (var, true))
13163 19 : goto omp_firstprivate_recv;
13164 : }
13165 2411 : map_cnt++;
13166 2411 : if (is_variable_sized (var))
13167 : {
13168 12 : tree new_var = lookup_decl (var, ctx);
13169 12 : tree pvar = DECL_VALUE_EXPR (var);
13170 12 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
13171 12 : pvar = TREE_OPERAND (pvar, 0);
13172 12 : gcc_assert (DECL_P (pvar));
13173 12 : tree new_pvar = lookup_decl (pvar, ctx);
13174 12 : x = build_fold_indirect_ref (new_pvar);
13175 12 : TREE_THIS_NOTRAP (x) = 1;
13176 12 : SET_DECL_VALUE_EXPR (new_var, x);
13177 12 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13178 : }
13179 2399 : else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
13180 542 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
13181 2071 : && !omp_privatize_by_reference (var)
13182 703 : && !omp_is_allocatable_or_ptr (var)
13183 564 : && !lang_hooks.decls.omp_array_data (var, true))
13184 4433 : || TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
13185 : {
13186 389 : tree new_var = lookup_decl (var, ctx);
13187 389 : tree type = build_pointer_type (TREE_TYPE (var));
13188 389 : x = create_tmp_var_raw (type, get_name (new_var));
13189 389 : gimple_add_tmp_var (x);
13190 389 : x = build_simple_mem_ref (x);
13191 389 : SET_DECL_VALUE_EXPR (new_var, x);
13192 389 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13193 : }
13194 : else
13195 : {
13196 2010 : tree new_var = lookup_decl (var, ctx);
13197 2010 : x = create_tmp_var_raw (TREE_TYPE (new_var), get_name (new_var));
13198 2010 : gimple_add_tmp_var (x);
13199 2010 : SET_DECL_VALUE_EXPR (new_var, x);
13200 2010 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13201 : }
13202 : break;
13203 10 : case OMP_CLAUSE_DEVICE_TYPE:
13204 : /* FIXME: Ensure that 'nohost' also has not implied before that
13205 : 'g->have_offload = true' or an implicit declare target. */
13206 10 : if (OMP_CLAUSE_DEVICE_TYPE_KIND (c) != OMP_CLAUSE_DEVICE_TYPE_ANY)
13207 5 : sorry_at (OMP_CLAUSE_LOCATION (c),
13208 : "only the %<device_type(any)%> is supported");
13209 : break;
13210 : }
13211 :
13212 37071 : if (offloaded)
13213 : {
13214 21111 : target_nesting_level++;
13215 21111 : lower_omp (&tgt_body, ctx);
13216 21111 : target_nesting_level--;
13217 : }
13218 15960 : else if (data_region)
13219 4021 : lower_omp (&tgt_body, ctx);
13220 :
13221 37071 : if (offloaded)
13222 : {
13223 : /* Declare all the variables created by mapping and the variables
13224 : declared in the scope of the target body. */
13225 21111 : record_vars_into (ctx->block_vars, child_fn);
13226 21111 : maybe_remove_omp_member_access_dummy_vars (tgt_bind);
13227 21111 : record_vars_into (gimple_bind_vars (tgt_bind), child_fn);
13228 : }
13229 :
13230 37071 : if (ctx->record_type)
13231 : {
13232 32204 : if (deep_map_cnt && TREE_CODE (deep_map_cnt) == INTEGER_CST)
13233 : /* map_cnt = map_cnt + tree_to_hwi (deep_map_cnt); */
13234 : /* deep_map_cnt = NULL_TREE; */
13235 0 : gcc_unreachable ();
13236 144 : else if (deep_map_cnt)
13237 : {
13238 144 : gcc_assert (flexible_array_type_p (ctx->record_type));
13239 144 : tree n = create_tmp_var_raw (size_type_node, "nn_map");
13240 144 : gimple_add_tmp_var (n);
13241 144 : gimplify_assign (n, deep_map_cnt, &ilist);
13242 144 : deep_map_cnt = n;
13243 : }
13244 144 : ctx->sender_decl
13245 32204 : = create_tmp_var (deep_map_cnt ? build_pointer_type (ctx->record_type)
13246 : : ctx->record_type, ".omp_data_arr");
13247 32204 : DECL_NAMELESS (ctx->sender_decl) = 1;
13248 32204 : TREE_ADDRESSABLE (ctx->sender_decl) = 1;
13249 64264 : t = make_tree_vec (deep_map_cnt ? 4 : 3);
13250 32204 : TREE_VEC_ELT (t, 0) = ctx->sender_decl;
13251 32204 : TREE_VEC_ELT (t, 1)
13252 64408 : = create_tmp_var (deep_map_cnt
13253 144 : ? build_pointer_type (size_type_node)
13254 32060 : : build_array_type_nelts (size_type_node, map_cnt),
13255 : ".omp_data_sizes");
13256 32204 : DECL_NAMELESS (TREE_VEC_ELT (t, 1)) = 1;
13257 32204 : TREE_ADDRESSABLE (TREE_VEC_ELT (t, 1)) = 1;
13258 32204 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 1;
13259 32204 : tree tkind_type = short_unsigned_type_node;
13260 32204 : int talign_shift = 8;
13261 32204 : TREE_VEC_ELT (t, 2)
13262 64408 : = create_tmp_var (deep_map_cnt
13263 144 : ? build_pointer_type (tkind_type)
13264 32060 : : build_array_type_nelts (tkind_type, map_cnt),
13265 : ".omp_data_kinds");
13266 32204 : DECL_NAMELESS (TREE_VEC_ELT (t, 2)) = 1;
13267 32204 : TREE_ADDRESSABLE (TREE_VEC_ELT (t, 2)) = 1;
13268 32204 : TREE_STATIC (TREE_VEC_ELT (t, 2)) = 1;
13269 32204 : gimple_omp_target_set_data_arg (stmt, t);
13270 :
13271 32204 : if (deep_map_cnt)
13272 : {
13273 144 : tree tmp, size;
13274 144 : size = create_tmp_var (size_type_node, NULL);
13275 144 : DECL_NAMELESS (size) = 1;
13276 144 : gimplify_assign (size,
13277 : fold_build2_loc (UNKNOWN_LOCATION, PLUS_EXPR,
13278 : size_type_node, deep_map_cnt,
13279 : build_int_cst (size_type_node,
13280 144 : map_cnt)), &ilist);
13281 144 : TREE_VEC_ELT (t, 3) = size;
13282 :
13283 144 : tree call = builtin_decl_explicit (BUILT_IN_MALLOC);
13284 144 : size = fold_build2_loc (UNKNOWN_LOCATION, MULT_EXPR,
13285 : size_type_node, deep_map_cnt,
13286 144 : TYPE_SIZE_UNIT (ptr_type_node));
13287 144 : size = fold_build2_loc (UNKNOWN_LOCATION, PLUS_EXPR,
13288 : size_type_node, size,
13289 144 : TYPE_SIZE_UNIT (ctx->record_type));
13290 144 : tmp = build_call_expr_loc (input_location, call, 1, size);
13291 144 : gimplify_assign (ctx->sender_decl, tmp, &ilist);
13292 :
13293 144 : size = fold_build2_loc (UNKNOWN_LOCATION, MULT_EXPR,
13294 144 : size_type_node, TREE_VEC_ELT (t, 3),
13295 144 : TYPE_SIZE_UNIT (size_type_node));
13296 144 : tmp = build_call_expr_loc (input_location, call, 1, size);
13297 144 : gimplify_assign (TREE_VEC_ELT (t, 1), tmp, &ilist);
13298 :
13299 144 : size = fold_build2_loc (UNKNOWN_LOCATION, MULT_EXPR,
13300 144 : size_type_node, TREE_VEC_ELT (t, 3),
13301 144 : TYPE_SIZE_UNIT (tkind_type));
13302 144 : tmp = build_call_expr_loc (input_location, call, 1, size);
13303 144 : gimplify_assign (TREE_VEC_ELT (t, 2), tmp, &ilist);
13304 144 : tree field = TYPE_FIELDS (TREE_TYPE (TREE_TYPE (ctx->sender_decl)));
13305 595 : for ( ; DECL_CHAIN (field) != NULL_TREE; field = DECL_CHAIN (field))
13306 : ;
13307 144 : gcc_assert (TREE_CODE (TREE_TYPE (field)));
13308 144 : tmp = build_fold_indirect_ref (ctx->sender_decl);
13309 144 : deep_map_data = omp_build_component_ref (tmp, field);
13310 144 : deep_map_offset_data = create_tmp_var_raw (size_type_node,
13311 : "map_offset_data");
13312 144 : deep_map_offset = create_tmp_var_raw (size_type_node, "map_offset");
13313 144 : gimple_add_tmp_var (deep_map_offset_data);
13314 144 : gimple_add_tmp_var (deep_map_offset);
13315 144 : gimplify_assign (deep_map_offset_data, build_int_cst (size_type_node,
13316 : 0), &ilist);
13317 144 : gimplify_assign (deep_map_offset, build_int_cst (size_type_node,
13318 144 : map_cnt), &ilist);
13319 : }
13320 :
13321 32204 : vec<constructor_elt, va_gc> *vsize;
13322 32204 : vec<constructor_elt, va_gc> *vkind;
13323 32204 : vec_alloc (vsize, map_cnt);
13324 32204 : vec_alloc (vkind, map_cnt);
13325 32204 : unsigned int map_idx = 0;
13326 :
13327 161819 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
13328 129615 : switch (OMP_CLAUSE_CODE (c))
13329 : {
13330 : tree ovar, nc, s, purpose, var, x, type;
13331 : unsigned int talign;
13332 :
13333 : default:
13334 125651 : break;
13335 :
13336 87907 : case OMP_CLAUSE_MAP:
13337 87907 : case OMP_CLAUSE_TO:
13338 87907 : case OMP_CLAUSE_FROM:
13339 87907 : oacc_firstprivate_map:
13340 87907 : nc = c;
13341 87907 : ovar = OMP_CLAUSE_DECL (c);
13342 87907 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13343 87907 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
13344 72934 : || (OMP_CLAUSE_MAP_KIND (c)
13345 : == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
13346 : break;
13347 84075 : if (deep_map_cnt)
13348 : {
13349 428 : unsigned HOST_WIDE_INT tkind2;
13350 428 : switch (OMP_CLAUSE_CODE (c))
13351 : {
13352 424 : case OMP_CLAUSE_MAP:
13353 424 : tkind2 = OMP_CLAUSE_MAP_KIND (c);
13354 424 : if (OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (c)
13355 424 : && (((tkind2 & GOMP_MAP_FLAG_SPECIAL_BITS)
13356 : & ~GOMP_MAP_IMPLICIT)
13357 : == 0))
13358 : {
13359 : /* If this is an implicit map, and the GOMP_MAP_IMPLICIT
13360 : bits are not interfered by other special bit
13361 : encodings, then turn the GOMP_IMPLICIT_BIT flag on
13362 : for the runtime to see. */
13363 120 : tkind2 |= GOMP_MAP_IMPLICIT;
13364 : }
13365 : break;
13366 : case OMP_CLAUSE_FIRSTPRIVATE: tkind2 = GOMP_MAP_TO; break;
13367 : case OMP_CLAUSE_TO: tkind2 = GOMP_MAP_TO; break;
13368 2 : case OMP_CLAUSE_FROM: tkind2 = GOMP_MAP_FROM; break;
13369 0 : default: gcc_unreachable ();
13370 : }
13371 428 : lang_hooks.decls.omp_deep_mapping (stmt, c, tkind2,
13372 : deep_map_data,
13373 428 : TREE_VEC_ELT (t, 1),
13374 428 : TREE_VEC_ELT (t, 2),
13375 : deep_map_offset_data,
13376 : deep_map_offset, &ilist);
13377 : }
13378 84075 : if (!DECL_P (ovar))
13379 : {
13380 36880 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13381 36880 : && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
13382 : {
13383 0 : nc = OMP_CLAUSE_CHAIN (c);
13384 0 : gcc_checking_assert (OMP_CLAUSE_DECL (nc)
13385 : == get_base_address (ovar));
13386 0 : ovar = OMP_CLAUSE_DECL (nc);
13387 : }
13388 : else
13389 : {
13390 36880 : tree x = build_sender_ref (c, ctx);
13391 36880 : tree v = ovar;
13392 36880 : if (in_reduction_clauses
13393 163 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13394 37043 : && OMP_CLAUSE_MAP_IN_REDUCTION (c))
13395 : {
13396 104 : v = unshare_expr (v);
13397 104 : tree *p = &v;
13398 104 : while (handled_component_p (*p)
13399 : || TREE_CODE (*p) == INDIRECT_REF
13400 : || TREE_CODE (*p) == ADDR_EXPR
13401 : || TREE_CODE (*p) == MEM_REF
13402 256 : || TREE_CODE (*p) == NON_LVALUE_EXPR)
13403 152 : p = &TREE_OPERAND (*p, 0);
13404 104 : tree d = *p;
13405 104 : if (is_variable_sized (d))
13406 : {
13407 16 : gcc_assert (DECL_HAS_VALUE_EXPR_P (d));
13408 16 : d = DECL_VALUE_EXPR (d);
13409 16 : gcc_assert (TREE_CODE (d) == INDIRECT_REF);
13410 16 : d = TREE_OPERAND (d, 0);
13411 16 : gcc_assert (DECL_P (d));
13412 : }
13413 104 : splay_tree_key key
13414 104 : = (splay_tree_key) &DECL_CONTEXT (d);
13415 104 : tree nd = (tree) splay_tree_lookup (ctx->field_map,
13416 104 : key)->value;
13417 104 : if (d == *p)
13418 88 : *p = nd;
13419 : else
13420 16 : *p = build_fold_indirect_ref (nd);
13421 : }
13422 36880 : v = build_fold_addr_expr_with_type (v, ptr_type_node);
13423 36880 : v = lower_omp_map_iterator_expr (v, c, stmt);
13424 36880 : gimplify_assign (x, v, &ilist);
13425 36880 : nc = NULL_TREE;
13426 : }
13427 : }
13428 : else
13429 : {
13430 47195 : if (DECL_SIZE (ovar)
13431 47195 : && !poly_int_tree_p (DECL_SIZE (ovar)))
13432 : {
13433 7 : tree ovar2 = DECL_VALUE_EXPR (ovar);
13434 7 : gcc_assert (TREE_CODE (ovar2) == INDIRECT_REF);
13435 7 : ovar2 = TREE_OPERAND (ovar2, 0);
13436 7 : gcc_assert (DECL_P (ovar2));
13437 : ovar = ovar2;
13438 : }
13439 47195 : if (!maybe_lookup_field (c, ctx)
13440 51159 : && !(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13441 3964 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
13442 3964 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)))
13443 3964 : continue;
13444 : }
13445 :
13446 80111 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar));
13447 80111 : if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign)
13448 6409 : talign = DECL_ALIGN_UNIT (ovar);
13449 :
13450 80111 : var = NULL_TREE;
13451 80111 : if (nc)
13452 : {
13453 43231 : if (in_reduction_clauses
13454 897 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13455 44128 : && OMP_CLAUSE_MAP_IN_REDUCTION (c))
13456 : {
13457 268 : tree d = ovar;
13458 268 : if (is_variable_sized (d))
13459 : {
13460 0 : gcc_assert (DECL_HAS_VALUE_EXPR_P (d));
13461 0 : d = DECL_VALUE_EXPR (d);
13462 0 : gcc_assert (TREE_CODE (d) == INDIRECT_REF);
13463 0 : d = TREE_OPERAND (d, 0);
13464 0 : gcc_assert (DECL_P (d));
13465 : }
13466 268 : splay_tree_key key
13467 268 : = (splay_tree_key) &DECL_CONTEXT (d);
13468 268 : tree nd = (tree) splay_tree_lookup (ctx->field_map,
13469 268 : key)->value;
13470 268 : if (d == ovar)
13471 : var = nd;
13472 : else
13473 0 : var = build_fold_indirect_ref (nd);
13474 : }
13475 : else
13476 42963 : var = lookup_decl_in_outer_ctx (ovar, ctx);
13477 : }
13478 43231 : if (nc
13479 43231 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13480 33267 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
13481 33060 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
13482 261 : && is_omp_target (stmt))
13483 : {
13484 237 : x = build_sender_ref (c, ctx);
13485 237 : gimplify_assign (x, build_fold_addr_expr (var), &ilist);
13486 : }
13487 79874 : else if (nc)
13488 : {
13489 42994 : x = build_sender_ref (nc, ctx);
13490 :
13491 42994 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13492 33030 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
13493 5184 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
13494 48178 : && TREE_CODE (TREE_TYPE (ovar)) == ARRAY_TYPE)
13495 : {
13496 425 : gcc_assert (offloaded);
13497 425 : tree avar = build_fold_addr_expr (var);
13498 425 : if (!OMP_CLAUSE_ITERATORS (c))
13499 : {
13500 425 : tree tmp = create_tmp_var (TREE_TYPE (TREE_TYPE (x)));
13501 425 : mark_addressable (tmp);
13502 425 : gimplify_assign (tmp, avar, &ilist);
13503 425 : avar = tmp;
13504 : }
13505 425 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (x)));
13506 425 : avar = build_fold_addr_expr (avar);
13507 425 : avar = lower_omp_map_iterator_expr (avar, c, stmt);
13508 425 : gimplify_assign (x, avar, &ilist);
13509 : }
13510 42569 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
13511 : {
13512 3896 : gcc_assert (is_gimple_omp_oacc (ctx->stmt));
13513 3896 : if (!omp_privatize_by_reference (var))
13514 : {
13515 3441 : if (is_gimple_reg (var)
13516 3441 : && OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
13517 2578 : suppress_warning (var);
13518 3441 : var = build_fold_addr_expr (var);
13519 : }
13520 : else
13521 455 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
13522 3896 : gimplify_assign (x, var, &ilist);
13523 : }
13524 38673 : else if (is_gimple_reg (var))
13525 : {
13526 5283 : gcc_assert (offloaded);
13527 5283 : tree avar = create_tmp_var (TREE_TYPE (var));
13528 5283 : mark_addressable (avar);
13529 5283 : enum gomp_map_kind map_kind = OMP_CLAUSE_MAP_KIND (c);
13530 5283 : if (GOMP_MAP_COPY_TO_P (map_kind)
13531 : || map_kind == GOMP_MAP_POINTER
13532 475 : || map_kind == GOMP_MAP_TO_PSET
13533 2 : || map_kind == GOMP_MAP_FORCE_DEVICEPTR)
13534 : {
13535 : /* If we need to initialize a temporary
13536 : with VAR because it is not addressable, and
13537 : the variable hasn't been initialized yet, then
13538 : we'll get a warning for the store to avar.
13539 : Don't warn in that case, the mapping might
13540 : be implicit. */
13541 5281 : suppress_warning (var, OPT_Wuninitialized);
13542 5281 : gimplify_assign (avar, var, &ilist);
13543 : }
13544 5283 : avar = build_fold_addr_expr (avar);
13545 5283 : gimplify_assign (x, avar, &ilist);
13546 5283 : if ((GOMP_MAP_COPY_FROM_P (map_kind)
13547 1486 : || map_kind == GOMP_MAP_FORCE_DEVICEPTR)
13548 9080 : && !TYPE_READONLY (TREE_TYPE (var)))
13549 : {
13550 3797 : x = unshare_expr (x);
13551 3797 : x = build_simple_mem_ref (x);
13552 3797 : gimplify_assign (var, x, &olist);
13553 : }
13554 : }
13555 : else
13556 : {
13557 : /* While MAP is handled explicitly by the FE,
13558 : for 'target update', only the identified is passed. */
13559 33390 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FROM
13560 28266 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TO)
13561 34334 : && (omp_is_allocatable_or_ptr (var)
13562 120 : && omp_check_optional_argument (var, false)))
13563 0 : var = build_fold_indirect_ref (var);
13564 33390 : else if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FROM
13565 28266 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_TO)
13566 34334 : || (!omp_is_allocatable_or_ptr (var)
13567 5948 : && !omp_check_optional_argument (var, false)))
13568 33270 : var = build_fold_addr_expr (var);
13569 33390 : gimplify_assign (x, var, &ilist);
13570 : }
13571 : }
13572 80111 : s = NULL_TREE;
13573 80111 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
13574 : {
13575 3896 : gcc_checking_assert (is_gimple_omp_oacc (ctx->stmt));
13576 3896 : s = TREE_TYPE (ovar);
13577 3896 : if (TREE_CODE (s) == REFERENCE_TYPE
13578 3896 : || omp_check_optional_argument (ovar, false))
13579 449 : s = TREE_TYPE (s);
13580 3896 : s = TYPE_SIZE_UNIT (s);
13581 : }
13582 : else
13583 76215 : s = OMP_CLAUSE_SIZE (c);
13584 80111 : if (s == NULL_TREE)
13585 153 : s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
13586 80111 : s = fold_convert (size_type_node, s);
13587 80111 : s = lower_omp_map_iterator_size (s, c, stmt);
13588 80111 : purpose = size_int (map_idx++);
13589 80111 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
13590 80111 : if (TREE_CODE (s) != INTEGER_CST)
13591 16153 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
13592 :
13593 80111 : unsigned HOST_WIDE_INT tkind, tkind_zero;
13594 80111 : switch (OMP_CLAUSE_CODE (c))
13595 : {
13596 68541 : case OMP_CLAUSE_MAP:
13597 68541 : tkind = OMP_CLAUSE_MAP_KIND (c);
13598 68541 : tkind_zero = tkind;
13599 68541 : if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c))
13600 5276 : switch (tkind)
13601 : {
13602 : case GOMP_MAP_ALLOC:
13603 : case GOMP_MAP_IF_PRESENT:
13604 : case GOMP_MAP_TO:
13605 : case GOMP_MAP_FROM:
13606 : case GOMP_MAP_TOFROM:
13607 : case GOMP_MAP_ALWAYS_TO:
13608 : case GOMP_MAP_ALWAYS_FROM:
13609 : case GOMP_MAP_ALWAYS_TOFROM:
13610 : case GOMP_MAP_ALWAYS_PRESENT_TO:
13611 : case GOMP_MAP_ALWAYS_PRESENT_FROM:
13612 : case GOMP_MAP_ALWAYS_PRESENT_TOFROM:
13613 : case GOMP_MAP_RELEASE:
13614 : case GOMP_MAP_FORCE_TO:
13615 : case GOMP_MAP_FORCE_FROM:
13616 : case GOMP_MAP_FORCE_TOFROM:
13617 : case GOMP_MAP_FORCE_PRESENT:
13618 68541 : tkind_zero = GOMP_MAP_ZERO_LEN_ARRAY_SECTION;
13619 : break;
13620 21 : case GOMP_MAP_DELETE:
13621 21 : tkind_zero = GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION;
13622 : default:
13623 : break;
13624 : }
13625 68541 : if (tkind_zero != tkind)
13626 : {
13627 5276 : if (integer_zerop (s))
13628 : tkind = tkind_zero;
13629 4550 : else if (integer_nonzerop (s))
13630 68541 : tkind_zero = tkind;
13631 : }
13632 68541 : if (tkind_zero == tkind
13633 67080 : && OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (c)
13634 68541 : && (((tkind & GOMP_MAP_FLAG_SPECIAL_BITS)
13635 5231 : & ~GOMP_MAP_IMPLICIT)
13636 : == 0))
13637 : {
13638 : /* If this is an implicit map, and the GOMP_MAP_IMPLICIT
13639 : bits are not interfered by other special bit encodings,
13640 : then turn the GOMP_IMPLICIT_BIT flag on for the runtime
13641 : to see. */
13642 5188 : tkind |= GOMP_MAP_IMPLICIT;
13643 5188 : tkind_zero = tkind;
13644 : }
13645 : break;
13646 3896 : case OMP_CLAUSE_FIRSTPRIVATE:
13647 3896 : gcc_checking_assert (is_gimple_omp_oacc (ctx->stmt));
13648 : tkind = GOMP_MAP_TO;
13649 : tkind_zero = tkind;
13650 : break;
13651 1651 : case OMP_CLAUSE_TO:
13652 1651 : tkind
13653 1651 : = (OMP_CLAUSE_MOTION_PRESENT (c)
13654 1651 : ? GOMP_MAP_ALWAYS_PRESENT_TO : GOMP_MAP_TO);
13655 : tkind_zero = tkind;
13656 : break;
13657 6023 : case OMP_CLAUSE_FROM:
13658 6023 : tkind
13659 6023 : = (OMP_CLAUSE_MOTION_PRESENT (c)
13660 6023 : ? GOMP_MAP_ALWAYS_PRESENT_FROM : GOMP_MAP_FROM);
13661 : tkind_zero = tkind;
13662 : break;
13663 0 : default:
13664 0 : gcc_unreachable ();
13665 : }
13666 68541 : gcc_checking_assert (tkind
13667 : < (HOST_WIDE_INT_C (1U) << talign_shift));
13668 80111 : gcc_checking_assert (tkind_zero
13669 : < (HOST_WIDE_INT_C (1U) << talign_shift));
13670 80111 : talign = ceil_log2 (talign);
13671 80111 : tkind |= talign << talign_shift;
13672 80111 : tkind_zero |= talign << talign_shift;
13673 80111 : gcc_checking_assert (tkind
13674 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
13675 80111 : gcc_checking_assert (tkind_zero
13676 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
13677 80111 : if (tkind == tkind_zero)
13678 78650 : x = build_int_cstu (tkind_type, tkind);
13679 : else
13680 : {
13681 1461 : TREE_STATIC (TREE_VEC_ELT (t, 2)) = 0;
13682 1461 : x = build3 (COND_EXPR, tkind_type,
13683 : fold_build2 (EQ_EXPR, boolean_type_node,
13684 : unshare_expr (s), size_zero_node),
13685 1461 : build_int_cstu (tkind_type, tkind_zero),
13686 1461 : build_int_cstu (tkind_type, tkind));
13687 : }
13688 80111 : CONSTRUCTOR_APPEND_ELT (vkind, purpose, x);
13689 80111 : if (nc && nc != c)
13690 0 : c = nc;
13691 : break;
13692 :
13693 15296 : case OMP_CLAUSE_FIRSTPRIVATE:
13694 15296 : omp_has_device_addr_descr:
13695 15296 : if (is_gimple_omp_oacc (ctx->stmt))
13696 3896 : goto oacc_firstprivate_map;
13697 11400 : ovar = OMP_CLAUSE_DECL (c);
13698 11400 : if (omp_privatize_by_reference (ovar))
13699 822 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
13700 : else
13701 10578 : talign = DECL_ALIGN_UNIT (ovar);
13702 11400 : var = lookup_decl_in_outer_ctx (ovar, ctx);
13703 11400 : x = build_sender_ref (ovar, ctx);
13704 11400 : tkind = GOMP_MAP_FIRSTPRIVATE;
13705 11400 : type = TREE_TYPE (ovar);
13706 11400 : if (omp_privatize_by_reference (ovar))
13707 822 : type = TREE_TYPE (type);
13708 11400 : if ((INTEGRAL_TYPE_P (type)
13709 10998 : && TYPE_PRECISION (type) <= POINTER_SIZE)
13710 11410 : || TREE_CODE (type) == POINTER_TYPE)
13711 : {
13712 11143 : tkind = GOMP_MAP_FIRSTPRIVATE_INT;
13713 11143 : tree t = var;
13714 11143 : if (omp_privatize_by_reference (var))
13715 767 : t = build_simple_mem_ref (var);
13716 10376 : else if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
13717 9312 : suppress_warning (var);
13718 11143 : if (TREE_CODE (type) != POINTER_TYPE)
13719 10988 : t = fold_convert (pointer_sized_int_node, t);
13720 11143 : t = fold_convert (TREE_TYPE (x), t);
13721 11143 : gimplify_assign (x, t, &ilist);
13722 : }
13723 257 : else if (omp_privatize_by_reference (var))
13724 55 : gimplify_assign (x, var, &ilist);
13725 202 : else if (is_gimple_reg (var))
13726 : {
13727 47 : tree avar = create_tmp_var (TREE_TYPE (var));
13728 47 : mark_addressable (avar);
13729 47 : if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
13730 40 : suppress_warning (var);
13731 47 : gimplify_assign (avar, var, &ilist);
13732 47 : avar = build_fold_addr_expr (avar);
13733 47 : gimplify_assign (x, avar, &ilist);
13734 : }
13735 : else
13736 : {
13737 155 : var = build_fold_addr_expr (var);
13738 155 : gimplify_assign (x, var, &ilist);
13739 : }
13740 257 : if (tkind == GOMP_MAP_FIRSTPRIVATE_INT)
13741 11143 : s = size_int (0);
13742 257 : else if (omp_privatize_by_reference (ovar))
13743 55 : s = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
13744 : else
13745 202 : s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
13746 11400 : s = fold_convert (size_type_node, s);
13747 11400 : purpose = size_int (map_idx++);
13748 11400 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
13749 11400 : if (TREE_CODE (s) != INTEGER_CST)
13750 35 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
13751 :
13752 11400 : gcc_checking_assert (tkind
13753 : < (HOST_WIDE_INT_C (1U) << talign_shift));
13754 11400 : talign = ceil_log2 (talign);
13755 11400 : tkind |= talign << talign_shift;
13756 11400 : gcc_checking_assert (tkind
13757 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
13758 11400 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
13759 : build_int_cstu (tkind_type, tkind));
13760 : /* Fortran array descriptors: firstprivate of data + attach. */
13761 11400 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
13762 11381 : && lang_hooks.decls.omp_array_data (ovar, true)
13763 11431 : && lang_hooks.decls.omp_array_data_privatize (ovar))
13764 : {
13765 23 : tree not_null_lb, null_lb, after_lb;
13766 23 : tree var1, var2, size1, size2;
13767 23 : tree present = omp_check_optional_argument (ovar, true);
13768 23 : if (present)
13769 : {
13770 1 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
13771 1 : not_null_lb = create_artificial_label (clause_loc);
13772 1 : null_lb = create_artificial_label (clause_loc);
13773 1 : after_lb = create_artificial_label (clause_loc);
13774 1 : gimple_seq seq = NULL;
13775 1 : present = force_gimple_operand (present, &seq, true,
13776 : NULL_TREE);
13777 1 : gimple_seq_add_seq (&ilist, seq);
13778 1 : gimple_seq_add_stmt (&ilist,
13779 1 : gimple_build_cond_from_tree (present,
13780 : not_null_lb, null_lb));
13781 1 : gimple_seq_add_stmt (&ilist,
13782 1 : gimple_build_label (not_null_lb));
13783 : }
13784 23 : var1 = lang_hooks.decls.omp_array_data (var, false);
13785 23 : size1 = lang_hooks.decls.omp_array_size (var, &ilist);
13786 23 : var2 = build_fold_addr_expr (x);
13787 23 : if (!POINTER_TYPE_P (TREE_TYPE (var)))
13788 0 : var = build_fold_addr_expr (var);
13789 23 : size2 = fold_build2 (POINTER_DIFF_EXPR, ssizetype,
13790 : build_fold_addr_expr (var1), var);
13791 23 : size2 = fold_convert (sizetype, size2);
13792 23 : if (present)
13793 : {
13794 1 : tree tmp = create_tmp_var (TREE_TYPE (var1));
13795 1 : gimplify_assign (tmp, var1, &ilist);
13796 1 : var1 = tmp;
13797 1 : tmp = create_tmp_var (TREE_TYPE (var2));
13798 1 : gimplify_assign (tmp, var2, &ilist);
13799 1 : var2 = tmp;
13800 1 : tmp = create_tmp_var (TREE_TYPE (size1));
13801 1 : gimplify_assign (tmp, size1, &ilist);
13802 1 : size1 = tmp;
13803 1 : tmp = create_tmp_var (TREE_TYPE (size2));
13804 1 : gimplify_assign (tmp, size2, &ilist);
13805 1 : size2 = tmp;
13806 1 : gimple_seq_add_stmt (&ilist, gimple_build_goto (after_lb));
13807 1 : gimple_seq_add_stmt (&ilist, gimple_build_label (null_lb));
13808 1 : gimplify_assign (var1, null_pointer_node, &ilist);
13809 1 : gimplify_assign (var2, null_pointer_node, &ilist);
13810 1 : gimplify_assign (size1, size_zero_node, &ilist);
13811 1 : gimplify_assign (size2, size_zero_node, &ilist);
13812 1 : gimple_seq_add_stmt (&ilist, gimple_build_label (after_lb));
13813 : }
13814 23 : x = build_sender_ref ((splay_tree_key) &DECL_NAME (ovar), ctx);
13815 23 : gimplify_assign (x, var1, &ilist);
13816 23 : tkind = GOMP_MAP_FIRSTPRIVATE;
13817 23 : talign = DECL_ALIGN_UNIT (ovar);
13818 23 : talign = ceil_log2 (talign);
13819 23 : tkind |= talign << talign_shift;
13820 23 : gcc_checking_assert (tkind
13821 : <= tree_to_uhwi (
13822 : TYPE_MAX_VALUE (tkind_type)));
13823 23 : purpose = size_int (map_idx++);
13824 23 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, size1);
13825 23 : if (TREE_CODE (size1) != INTEGER_CST)
13826 23 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
13827 23 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
13828 : build_int_cstu (tkind_type, tkind));
13829 23 : x = build_sender_ref ((splay_tree_key) &DECL_UID (ovar), ctx);
13830 23 : gimplify_assign (x, var2, &ilist);
13831 23 : tkind = GOMP_MAP_ATTACH;
13832 23 : purpose = size_int (map_idx++);
13833 23 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, size2);
13834 23 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
13835 : build_int_cstu (tkind_type, tkind));
13836 : }
13837 : break;
13838 :
13839 2430 : case OMP_CLAUSE_USE_DEVICE_PTR:
13840 2430 : case OMP_CLAUSE_USE_DEVICE_ADDR:
13841 2430 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
13842 2430 : case OMP_CLAUSE_IS_DEVICE_PTR:
13843 2430 : ovar = OMP_CLAUSE_DECL (c);
13844 2430 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
13845 : {
13846 235 : if (lang_hooks.decls.omp_array_data (ovar, true))
13847 19 : goto omp_has_device_addr_descr;
13848 233 : while (TREE_CODE (ovar) == INDIRECT_REF
13849 233 : || TREE_CODE (ovar) == ARRAY_REF)
13850 17 : ovar = TREE_OPERAND (ovar, 0);
13851 : }
13852 2411 : var = lookup_decl_in_outer_ctx (ovar, ctx);
13853 :
13854 2411 : if (lang_hooks.decls.omp_array_data (ovar, true))
13855 : {
13856 601 : tkind = ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
13857 601 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
13858 601 : ? GOMP_MAP_USE_DEVICE_PTR : GOMP_MAP_FIRSTPRIVATE_INT);
13859 601 : x = build_sender_ref ((splay_tree_key) &DECL_NAME (ovar), ctx);
13860 : }
13861 1810 : else if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
13862 1810 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
13863 : {
13864 1453 : tkind = GOMP_MAP_USE_DEVICE_PTR;
13865 1453 : x = build_sender_ref ((splay_tree_key) &DECL_UID (ovar), ctx);
13866 : }
13867 : else
13868 : {
13869 357 : tkind = GOMP_MAP_FIRSTPRIVATE_INT;
13870 357 : x = build_sender_ref (ovar, ctx);
13871 : }
13872 :
13873 2411 : if (is_gimple_omp_oacc (ctx->stmt))
13874 : {
13875 147 : gcc_assert (tkind == GOMP_MAP_USE_DEVICE_PTR);
13876 :
13877 147 : if (OMP_CLAUSE_USE_DEVICE_PTR_IF_PRESENT (c))
13878 57 : tkind = GOMP_MAP_USE_DEVICE_PTR_IF_PRESENT;
13879 : }
13880 :
13881 2411 : type = TREE_TYPE (ovar);
13882 2411 : if (lang_hooks.decls.omp_array_data (ovar, true))
13883 601 : var = lang_hooks.decls.omp_array_data (var, false);
13884 1810 : else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
13885 542 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
13886 1484 : && !omp_privatize_by_reference (ovar)
13887 512 : && !omp_is_allocatable_or_ptr (ovar))
13888 3247 : || TREE_CODE (type) == ARRAY_TYPE)
13889 401 : var = build_fold_addr_expr (var);
13890 : else
13891 : {
13892 1409 : if (omp_privatize_by_reference (ovar)
13893 370 : || omp_check_optional_argument (ovar, false)
13894 1779 : || omp_is_allocatable_or_ptr (ovar))
13895 : {
13896 1212 : type = TREE_TYPE (type);
13897 1212 : if (POINTER_TYPE_P (type)
13898 1212 : && TREE_CODE (type) != ARRAY_TYPE
13899 1571 : && ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_USE_DEVICE_ADDR
13900 36 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
13901 28 : && !omp_is_allocatable_or_ptr (ovar))
13902 337 : || (omp_privatize_by_reference (ovar)
13903 337 : && omp_is_allocatable_or_ptr (ovar))))
13904 357 : var = build_simple_mem_ref (var);
13905 1212 : var = fold_convert (TREE_TYPE (x), var);
13906 : }
13907 : }
13908 2411 : tree present;
13909 2411 : present = omp_check_optional_argument (ovar, true);
13910 2411 : if (present)
13911 : {
13912 878 : tree null_label = create_artificial_label (UNKNOWN_LOCATION);
13913 878 : tree notnull_label = create_artificial_label (UNKNOWN_LOCATION);
13914 878 : tree opt_arg_label = create_artificial_label (UNKNOWN_LOCATION);
13915 878 : tree new_x = unshare_expr (x);
13916 878 : gimplify_expr (&present, &ilist, NULL, is_gimple_val,
13917 : fb_rvalue);
13918 878 : gcond *cond = gimple_build_cond_from_tree (present,
13919 : notnull_label,
13920 : null_label);
13921 878 : gimple_seq_add_stmt (&ilist, cond);
13922 878 : gimple_seq_add_stmt (&ilist, gimple_build_label (null_label));
13923 878 : gimplify_assign (new_x, null_pointer_node, &ilist);
13924 878 : gimple_seq_add_stmt (&ilist, gimple_build_goto (opt_arg_label));
13925 878 : gimple_seq_add_stmt (&ilist,
13926 878 : gimple_build_label (notnull_label));
13927 878 : gimplify_assign (x, var, &ilist);
13928 878 : gimple_seq_add_stmt (&ilist,
13929 878 : gimple_build_label (opt_arg_label));
13930 : }
13931 : else
13932 1533 : gimplify_assign (x, var, &ilist);
13933 2411 : s = size_int (0);
13934 2411 : purpose = size_int (map_idx++);
13935 2411 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
13936 2411 : gcc_checking_assert (tkind
13937 : < (HOST_WIDE_INT_C (1U) << talign_shift));
13938 2411 : gcc_checking_assert (tkind
13939 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
13940 2411 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
13941 : build_int_cstu (tkind_type, tkind));
13942 2411 : break;
13943 : }
13944 :
13945 32204 : gcc_assert (map_idx == map_cnt);
13946 :
13947 32204 : if (!deep_map_cnt)
13948 : {
13949 32060 : DECL_INITIAL (TREE_VEC_ELT (t, 1))
13950 32060 : = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), vsize);
13951 32060 : DECL_INITIAL (TREE_VEC_ELT (t, 2))
13952 64120 : = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 2)), vkind);
13953 : }
13954 96612 : for (int i = 1; i <= 2; i++)
13955 64408 : if (deep_map_cnt || !TREE_STATIC (TREE_VEC_ELT (t, i)))
13956 : {
13957 9656 : tree tmp = TREE_VEC_ELT (t, i);
13958 9656 : if (deep_map_cnt)
13959 : {
13960 288 : const char *prefix = (i == 1 ? ".omp_data_sizes0"
13961 : : ".omp_data_kinds0");
13962 144 : tree type = (i == 1) ? size_type_node : tkind_type;
13963 288 : type = build_array_type_nelts (type, map_cnt);
13964 288 : tree var = create_tmp_var (type, prefix);
13965 288 : DECL_NAMELESS (var) = 1;
13966 288 : TREE_ADDRESSABLE (var) = 1;
13967 288 : TREE_STATIC (var) = TREE_STATIC (tmp);
13968 288 : DECL_INITIAL (var) = build_constructor (type, i == 1
13969 : ? vsize : vkind);
13970 288 : tmp = var;
13971 288 : TREE_STATIC (TREE_VEC_ELT (t, i)) = 0;
13972 : }
13973 :
13974 9656 : gimple_seq initlist = NULL;
13975 9656 : force_gimple_operand (build1 (DECL_EXPR, void_type_node, tmp),
13976 : &initlist, true, NULL_TREE);
13977 9656 : gimple_seq_add_seq (&ilist, initlist);
13978 :
13979 9656 : if (deep_map_cnt)
13980 : {
13981 288 : tree tmp2;
13982 288 : tree call = builtin_decl_explicit (BUILT_IN_MEMCPY);
13983 288 : tmp2 = TYPE_SIZE_UNIT (TREE_TYPE (tmp));
13984 576 : call = build_call_expr_loc (input_location, call, 3,
13985 288 : TREE_VEC_ELT (t, i),
13986 : build_fold_addr_expr (tmp), tmp2);
13987 288 : gimplify_and_add (call, &ilist);
13988 : }
13989 :
13990 9656 : if (!TREE_STATIC (tmp))
13991 : {
13992 9438 : tree clobber = build_clobber (TREE_TYPE (tmp));
13993 9438 : gimple_seq_add_stmt (&olist,
13994 9438 : gimple_build_assign (tmp, clobber));
13995 : }
13996 9656 : if (deep_map_cnt)
13997 : {
13998 288 : tmp = TREE_VEC_ELT (t, i);
13999 288 : tree call = builtin_decl_explicit (BUILT_IN_FREE);
14000 288 : call = build_call_expr_loc (input_location, call, 1, tmp);
14001 288 : gimplify_and_add (call, &olist);
14002 288 : tree clobber = build_clobber (TREE_TYPE (tmp));
14003 288 : gimple_seq_add_stmt (&olist,
14004 288 : gimple_build_assign (tmp, clobber));
14005 : }
14006 : }
14007 54752 : else if (omp_maybe_offloaded_ctx (ctx->outer))
14008 : {
14009 3296 : tree id = get_identifier ("omp declare target");
14010 3296 : tree decl = TREE_VEC_ELT (t, i);
14011 3296 : DECL_ATTRIBUTES (decl)
14012 3296 : = tree_cons (id, NULL_TREE, DECL_ATTRIBUTES (decl));
14013 3296 : varpool_node *node = varpool_node::get (decl);
14014 3296 : if (node)
14015 : {
14016 0 : node->offloadable = 1;
14017 0 : if (ENABLE_OFFLOADING)
14018 : {
14019 : g->have_offload = true;
14020 : vec_safe_push (offload_vars, t);
14021 : }
14022 : }
14023 : }
14024 :
14025 32204 : if (deep_map_cnt)
14026 : {
14027 144 : tree call = builtin_decl_explicit (BUILT_IN_FREE);
14028 144 : call = build_call_expr_loc (input_location, call, 1,
14029 144 : TREE_VEC_ELT (t, 0));
14030 144 : gimplify_and_add (call, &olist);
14031 :
14032 144 : gimplify_expr (&TREE_VEC_ELT (t, 1), &ilist, NULL, is_gimple_val,
14033 : fb_rvalue);
14034 : }
14035 :
14036 32204 : tree clobber = build_clobber (TREE_TYPE (ctx->sender_decl));
14037 32204 : gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
14038 : clobber));
14039 : }
14040 :
14041 : /* Once all the expansions are done, sequence all the different
14042 : fragments inside gimple_omp_body. */
14043 :
14044 37071 : new_body = NULL;
14045 :
14046 37071 : if (offloaded
14047 21111 : && ctx->record_type)
14048 : {
14049 16481 : t = ctx->sender_decl;
14050 16481 : if (!deep_map_cnt)
14051 16374 : t = build_fold_addr_expr_loc (loc, t);
14052 : /* fixup_child_record_type might have changed receiver_decl's type. */
14053 16481 : t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
14054 16481 : if (!AGGREGATE_TYPE_P (TREE_TYPE (ctx->sender_decl)))
14055 107 : gimplify_assign (ctx->receiver_decl, t, &new_body);
14056 : else
14057 16374 : gimple_seq_add_stmt (&new_body,
14058 16374 : gimple_build_assign (ctx->receiver_decl, t));
14059 : }
14060 37071 : gimple_seq_add_seq (&new_body, fplist);
14061 :
14062 37071 : if (offloaded || data_region)
14063 : {
14064 140710 : tree prev = NULL_TREE;
14065 : bool by_ref;
14066 140710 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
14067 115578 : switch (OMP_CLAUSE_CODE (c))
14068 : {
14069 : tree var, x, new_var, *allocate_ptr;
14070 : default:
14071 : break;
14072 15296 : case OMP_CLAUSE_FIRSTPRIVATE:
14073 15296 : omp_firstprivatize_data_region:
14074 15296 : if (is_gimple_omp_oacc (ctx->stmt))
14075 : break;
14076 11400 : var = OMP_CLAUSE_DECL (c);
14077 11400 : new_var = lookup_decl (var, ctx);
14078 11400 : allocate_ptr = alloc_map.get (new_var);
14079 11400 : by_ref = omp_privatize_by_reference (var);
14080 11400 : if (allocate_ptr)
14081 : {
14082 63 : if (is_variable_sized (var, by_ref))
14083 : /* Handle this in the next pass when the size is
14084 : available. */
14085 : break;
14086 :
14087 48 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14088 48 : gcc_assert (allocate_seq);
14089 48 : gimple_seq_add_seq (&new_body, *allocate_seq);
14090 :
14091 48 : if (by_ref)
14092 : {
14093 4 : x = fold_convert (TREE_TYPE (new_var), *allocate_ptr);
14094 4 : gimplify_assign (new_var, x, &new_body);
14095 4 : new_var = build_fold_indirect_ref (new_var);
14096 : }
14097 : else
14098 44 : new_var = build_simple_mem_ref (*allocate_ptr);
14099 : }
14100 11385 : if (by_ref || is_gimple_reg_type (TREE_TYPE (var)))
14101 : {
14102 11244 : tree type;
14103 11244 : type = TREE_TYPE (var);
14104 11244 : if (by_ref)
14105 809 : type = TREE_TYPE (type);
14106 11244 : if ((INTEGRAL_TYPE_P (type)
14107 10998 : && TYPE_PRECISION (type) <= POINTER_SIZE)
14108 11254 : || TREE_CODE (type) == POINTER_TYPE)
14109 : {
14110 11143 : x = build_receiver_ref (var, false, ctx);
14111 11143 : if (TREE_CODE (type) != POINTER_TYPE)
14112 10988 : x = fold_convert (pointer_sized_int_node, x);
14113 11143 : x = fold_convert (type, x);
14114 11143 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14115 : fb_rvalue);
14116 11143 : if (by_ref && !allocate_ptr)
14117 : {
14118 766 : tree v = create_tmp_var_raw (type, get_name (var));
14119 766 : gimple_add_tmp_var (v);
14120 766 : TREE_ADDRESSABLE (v) = 1;
14121 766 : gimple_seq_add_stmt (&new_body,
14122 766 : gimple_build_assign (v, x));
14123 766 : x = build_fold_addr_expr (v);
14124 : }
14125 11143 : gimplify_assign (new_var, x, &new_body);
14126 : }
14127 : else
14128 : {
14129 101 : x = build_receiver_ref (var, allocate_ptr || !by_ref, ctx);
14130 101 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14131 : fb_rvalue);
14132 101 : gimplify_assign (new_var, x, &new_body);
14133 : }
14134 : }
14135 141 : else if (is_variable_sized (var))
14136 : {
14137 4 : tree pvar = DECL_VALUE_EXPR (var);
14138 4 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14139 4 : pvar = TREE_OPERAND (pvar, 0);
14140 4 : gcc_assert (DECL_P (pvar));
14141 4 : tree new_var = lookup_decl (pvar, ctx);
14142 4 : x = build_receiver_ref (var, false, ctx);
14143 4 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14144 4 : gimple_seq_add_stmt (&new_body,
14145 4 : gimple_build_assign (new_var, x));
14146 : }
14147 137 : else if (allocate_ptr)
14148 : {
14149 19 : x = build_receiver_ref (var, true, ctx);
14150 19 : new_var = unshare_expr (new_var);
14151 19 : x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
14152 19 : gimplify_and_add (x, &new_body);
14153 : }
14154 : break;
14155 226 : case OMP_CLAUSE_PRIVATE:
14156 226 : if (is_gimple_omp_oacc (ctx->stmt))
14157 : break;
14158 146 : var = OMP_CLAUSE_DECL (c);
14159 146 : new_var = lookup_decl (var, ctx);
14160 146 : allocate_ptr = alloc_map.get (new_var);
14161 146 : by_ref = omp_privatize_by_reference (var);
14162 146 : if (allocate_ptr)
14163 : {
14164 55 : if (is_variable_sized (var, by_ref))
14165 : /* Handle this in the next pass when the size is
14166 : available. */
14167 : break;
14168 :
14169 43 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14170 43 : gcc_assert (allocate_seq);
14171 43 : gimple_seq_add_seq (&new_body, *allocate_seq);
14172 :
14173 43 : if (!by_ref)
14174 42 : new_var = build_simple_mem_ref (*allocate_ptr);
14175 : }
14176 133 : if (by_ref)
14177 : {
14178 12 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14179 12 : if (!allocate_ptr)
14180 : {
14181 11 : x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
14182 11 : if (TREE_CONSTANT (x))
14183 : {
14184 3 : x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
14185 : get_name (var));
14186 3 : gimple_add_tmp_var (x);
14187 3 : TREE_ADDRESSABLE (x) = 1;
14188 3 : x = build_fold_addr_expr_loc (clause_loc, x);
14189 : }
14190 : else
14191 : break;
14192 :
14193 3 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
14194 3 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14195 : fb_rvalue);
14196 : }
14197 : else
14198 1 : x = *allocate_ptr;
14199 :
14200 4 : gimple_seq_add_stmt (&new_body,
14201 4 : gimple_build_assign (new_var, x));
14202 : }
14203 : break;
14204 2430 : case OMP_CLAUSE_USE_DEVICE_PTR:
14205 2430 : case OMP_CLAUSE_USE_DEVICE_ADDR:
14206 2430 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
14207 2430 : case OMP_CLAUSE_IS_DEVICE_PTR:
14208 2430 : gimple_seq assign_body;
14209 2430 : bool is_array_data;
14210 2430 : bool do_optional_check;
14211 2430 : assign_body = NULL;
14212 2430 : do_optional_check = false;
14213 2430 : var = OMP_CLAUSE_DECL (c);
14214 2430 : is_array_data = lang_hooks.decls.omp_array_data (var, true) != NULL;
14215 2430 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR && is_array_data)
14216 19 : goto omp_firstprivatize_data_region;
14217 :
14218 2411 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
14219 2411 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
14220 4108 : x = build_sender_ref (is_array_data
14221 601 : ? (splay_tree_key) &DECL_NAME (var)
14222 1453 : : (splay_tree_key) &DECL_UID (var), ctx);
14223 : else
14224 : {
14225 357 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
14226 : {
14227 233 : while (TREE_CODE (var) == INDIRECT_REF
14228 233 : || TREE_CODE (var) == ARRAY_REF)
14229 17 : var = TREE_OPERAND (var, 0);
14230 : }
14231 357 : x = build_receiver_ref (var, false, ctx);
14232 : }
14233 :
14234 2411 : if (is_array_data)
14235 : {
14236 601 : bool is_ref = omp_privatize_by_reference (var);
14237 601 : do_optional_check = true;
14238 : /* First, we copy the descriptor data from the host; then
14239 : we update its data to point to the target address. */
14240 601 : new_var = lookup_decl (var, ctx);
14241 601 : new_var = DECL_VALUE_EXPR (new_var);
14242 601 : tree v = new_var;
14243 601 : tree v2 = var;
14244 601 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_PTR
14245 601 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR)
14246 601 : v2 = maybe_lookup_decl_in_outer_ctx (var, ctx);
14247 :
14248 601 : if (is_ref)
14249 : {
14250 396 : v2 = build_fold_indirect_ref (v2);
14251 396 : v = create_tmp_var_raw (TREE_TYPE (v2), get_name (var));
14252 396 : gimple_add_tmp_var (v);
14253 396 : TREE_ADDRESSABLE (v) = 1;
14254 396 : gimplify_assign (v, v2, &assign_body);
14255 396 : tree rhs = build_fold_addr_expr (v);
14256 396 : gimple_seq_add_stmt (&assign_body,
14257 396 : gimple_build_assign (new_var, rhs));
14258 : }
14259 : else
14260 205 : gimplify_assign (new_var, v2, &assign_body);
14261 :
14262 601 : v2 = lang_hooks.decls.omp_array_data (unshare_expr (v), false);
14263 601 : gcc_assert (v2);
14264 601 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14265 1202 : gimple_seq_add_stmt (&assign_body,
14266 601 : gimple_build_assign (v2, x));
14267 : }
14268 1810 : else if (is_variable_sized (var))
14269 : {
14270 12 : tree pvar = DECL_VALUE_EXPR (var);
14271 12 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14272 12 : pvar = TREE_OPERAND (pvar, 0);
14273 12 : gcc_assert (DECL_P (pvar));
14274 12 : new_var = lookup_decl (pvar, ctx);
14275 12 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14276 24 : gimple_seq_add_stmt (&assign_body,
14277 12 : gimple_build_assign (new_var, x));
14278 : }
14279 1798 : else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
14280 536 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
14281 1476 : && !omp_privatize_by_reference (var)
14282 504 : && !omp_is_allocatable_or_ptr (var))
14283 3231 : || TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
14284 : {
14285 389 : new_var = lookup_decl (var, ctx);
14286 389 : new_var = DECL_VALUE_EXPR (new_var);
14287 389 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
14288 389 : new_var = TREE_OPERAND (new_var, 0);
14289 389 : gcc_assert (DECL_P (new_var));
14290 389 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14291 778 : gimple_seq_add_stmt (&assign_body,
14292 389 : gimple_build_assign (new_var, x));
14293 : }
14294 : else
14295 : {
14296 1409 : tree type = TREE_TYPE (var);
14297 1409 : new_var = lookup_decl (var, ctx);
14298 1409 : if (omp_privatize_by_reference (var))
14299 : {
14300 1039 : type = TREE_TYPE (type);
14301 1039 : if (POINTER_TYPE_P (type)
14302 1039 : && TREE_CODE (type) != ARRAY_TYPE
14303 1398 : && ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_USE_DEVICE_ADDR
14304 36 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
14305 331 : || (omp_privatize_by_reference (var)
14306 331 : && omp_is_allocatable_or_ptr (var))))
14307 : {
14308 357 : tree v = create_tmp_var_raw (type, get_name (var));
14309 357 : gimple_add_tmp_var (v);
14310 357 : TREE_ADDRESSABLE (v) = 1;
14311 357 : x = fold_convert (type, x);
14312 357 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val,
14313 : fb_rvalue);
14314 357 : gimple_seq_add_stmt (&assign_body,
14315 357 : gimple_build_assign (v, x));
14316 357 : x = build_fold_addr_expr (v);
14317 357 : do_optional_check = true;
14318 : }
14319 : }
14320 1409 : new_var = DECL_VALUE_EXPR (new_var);
14321 1409 : x = fold_convert (TREE_TYPE (new_var), x);
14322 1409 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14323 1409 : gimple_seq_add_stmt (&assign_body,
14324 1409 : gimple_build_assign (new_var, x));
14325 : }
14326 2411 : tree present;
14327 1002 : present = ((do_optional_check
14328 958 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
14329 952 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR)
14330 2342 : ? omp_check_optional_argument (OMP_CLAUSE_DECL (c), true)
14331 : : NULL_TREE);
14332 2411 : if (present)
14333 : {
14334 480 : tree null_label = create_artificial_label (UNKNOWN_LOCATION);
14335 480 : tree notnull_label = create_artificial_label (UNKNOWN_LOCATION);
14336 480 : tree opt_arg_label = create_artificial_label (UNKNOWN_LOCATION);
14337 480 : glabel *null_glabel = gimple_build_label (null_label);
14338 480 : glabel *notnull_glabel = gimple_build_label (notnull_label);
14339 480 : ggoto *opt_arg_ggoto = gimple_build_goto (opt_arg_label);
14340 480 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14341 : fb_rvalue);
14342 480 : gimplify_expr (&present, &new_body, NULL, is_gimple_val,
14343 : fb_rvalue);
14344 480 : gcond *cond = gimple_build_cond_from_tree (present,
14345 : notnull_label,
14346 : null_label);
14347 480 : gimple_seq_add_stmt (&new_body, cond);
14348 480 : gimple_seq_add_stmt (&new_body, null_glabel);
14349 480 : gimplify_assign (new_var, null_pointer_node, &new_body);
14350 480 : gimple_seq_add_stmt (&new_body, opt_arg_ggoto);
14351 480 : gimple_seq_add_stmt (&new_body, notnull_glabel);
14352 480 : gimple_seq_add_seq (&new_body, assign_body);
14353 480 : gimple_seq_add_stmt (&new_body,
14354 480 : gimple_build_label (opt_arg_label));
14355 : }
14356 : else
14357 1931 : gimple_seq_add_seq (&new_body, assign_body);
14358 : break;
14359 : }
14360 : /* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass,
14361 : so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
14362 : are already handled. Similarly OMP_CLAUSE_PRIVATE for VLAs
14363 : or references to VLAs. */
14364 140710 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
14365 115578 : switch (OMP_CLAUSE_CODE (c))
14366 : {
14367 : tree var, new_var, *allocate_ptr;
14368 : default:
14369 100321 : break;
14370 63499 : case OMP_CLAUSE_MAP:
14371 63499 : if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
14372 63499 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
14373 : {
14374 3664 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14375 3664 : poly_int64 offset = 0;
14376 3664 : gcc_assert (prev);
14377 3664 : var = OMP_CLAUSE_DECL (c);
14378 3664 : if (DECL_P (var)
14379 3664 : && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
14380 1125 : && is_global_var (maybe_lookup_decl_in_outer_ctx (var,
14381 : ctx))
14382 3762 : && varpool_node::get_create (var)->offloadable)
14383 : break;
14384 3662 : if (TREE_CODE (var) == INDIRECT_REF
14385 3662 : && TREE_CODE (TREE_OPERAND (var, 0)) == COMPONENT_REF)
14386 0 : var = TREE_OPERAND (var, 0);
14387 3662 : if (TREE_CODE (var) == COMPONENT_REF)
14388 : {
14389 0 : var = get_addr_base_and_unit_offset (var, &offset);
14390 0 : gcc_assert (var != NULL_TREE && DECL_P (var));
14391 : }
14392 3662 : else if (DECL_SIZE (var)
14393 3662 : && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST)
14394 : {
14395 710 : tree var2 = DECL_VALUE_EXPR (var);
14396 710 : gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
14397 710 : var2 = TREE_OPERAND (var2, 0);
14398 710 : gcc_assert (DECL_P (var2));
14399 : var = var2;
14400 : }
14401 3662 : tree new_var = lookup_decl (var, ctx), x;
14402 3662 : tree type = TREE_TYPE (new_var);
14403 3662 : bool is_ref;
14404 3662 : if (TREE_CODE (OMP_CLAUSE_DECL (c)) == INDIRECT_REF
14405 3662 : && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c), 0))
14406 : == COMPONENT_REF))
14407 : {
14408 0 : type = TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c), 0));
14409 0 : is_ref = true;
14410 0 : new_var = build2 (MEM_REF, type,
14411 : build_fold_addr_expr (new_var),
14412 : build_int_cst (build_pointer_type (type),
14413 : offset));
14414 : }
14415 3662 : else if (TREE_CODE (OMP_CLAUSE_DECL (c)) == COMPONENT_REF)
14416 : {
14417 0 : type = TREE_TYPE (OMP_CLAUSE_DECL (c));
14418 0 : is_ref = TREE_CODE (type) == REFERENCE_TYPE;
14419 0 : new_var = build2 (MEM_REF, type,
14420 : build_fold_addr_expr (new_var),
14421 : build_int_cst (build_pointer_type (type),
14422 : offset));
14423 : }
14424 : else
14425 3662 : is_ref = omp_privatize_by_reference (var);
14426 3662 : if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
14427 : is_ref = false;
14428 6695 : bool ref_to_array = false;
14429 6695 : bool ref_to_ptr = false;
14430 3233 : if (is_ref)
14431 : {
14432 200 : type = TREE_TYPE (type);
14433 200 : if (TREE_CODE (type) == ARRAY_TYPE)
14434 : {
14435 194 : type = build_pointer_type (type);
14436 194 : ref_to_array = true;
14437 : }
14438 : }
14439 3462 : else if (TREE_CODE (type) == ARRAY_TYPE)
14440 : {
14441 413 : tree decl2 = DECL_VALUE_EXPR (new_var);
14442 413 : gcc_assert (TREE_CODE (decl2) == MEM_REF);
14443 413 : decl2 = TREE_OPERAND (decl2, 0);
14444 413 : gcc_assert (DECL_P (decl2));
14445 413 : new_var = decl2;
14446 413 : type = TREE_TYPE (new_var);
14447 : }
14448 3049 : else if (TREE_CODE (type) == REFERENCE_TYPE
14449 3049 : && TREE_CODE (TREE_TYPE (type)) == POINTER_TYPE)
14450 : {
14451 156 : type = TREE_TYPE (type);
14452 156 : ref_to_ptr = true;
14453 : }
14454 3662 : x = build_receiver_ref (prev, false, ctx);
14455 3662 : x = fold_convert_loc (clause_loc, type, x);
14456 3662 : if (!integer_zerop (OMP_CLAUSE_SIZE (c)))
14457 : {
14458 762 : tree bias = OMP_CLAUSE_SIZE (c);
14459 762 : if (DECL_P (bias))
14460 364 : bias = lookup_decl (bias, ctx);
14461 762 : bias = fold_convert_loc (clause_loc, sizetype, bias);
14462 762 : bias = fold_build1_loc (clause_loc, NEGATE_EXPR, sizetype,
14463 : bias);
14464 762 : x = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
14465 762 : TREE_TYPE (x), x, bias);
14466 : }
14467 3662 : if (ref_to_array)
14468 194 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
14469 3662 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14470 3662 : if ((is_ref && !ref_to_array)
14471 3656 : || ref_to_ptr)
14472 : {
14473 162 : tree t = create_tmp_var_raw (type, get_name (var));
14474 162 : gimple_add_tmp_var (t);
14475 162 : TREE_ADDRESSABLE (t) = 1;
14476 162 : gimple_seq_add_stmt (&new_body,
14477 162 : gimple_build_assign (t, x));
14478 162 : x = build_fold_addr_expr_loc (clause_loc, t);
14479 : }
14480 3662 : gimple_seq_add_stmt (&new_body,
14481 3662 : gimple_build_assign (new_var, x));
14482 3662 : prev = NULL_TREE;
14483 : }
14484 59835 : else if (OMP_CLAUSE_CHAIN (c)
14485 43232 : && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c))
14486 : == OMP_CLAUSE_MAP
14487 97645 : && (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c))
14488 : == GOMP_MAP_FIRSTPRIVATE_POINTER
14489 34575 : || (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c))
14490 : == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
14491 : prev = c;
14492 : break;
14493 226 : case OMP_CLAUSE_PRIVATE:
14494 226 : var = OMP_CLAUSE_DECL (c);
14495 226 : by_ref = omp_privatize_by_reference (var);
14496 226 : new_var = lookup_decl (var, ctx);
14497 226 : allocate_ptr = alloc_map.get (new_var);
14498 226 : if (is_variable_sized (var, by_ref)
14499 226 : || (!allocate_ptr && by_ref && !is_gimple_omp_oacc (ctx->stmt)))
14500 : {
14501 24 : if (allocate_ptr)
14502 : {
14503 12 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14504 12 : gcc_assert (allocate_seq);
14505 12 : gimple_seq_add_seq (&new_body, *allocate_seq);
14506 : }
14507 24 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14508 24 : tree pvar = var;
14509 24 : if (!by_ref)
14510 : {
14511 3 : pvar = DECL_VALUE_EXPR (var);
14512 3 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14513 3 : pvar = TREE_OPERAND (pvar, 0);
14514 : }
14515 24 : gcc_assert (DECL_P (pvar));
14516 24 : tree new_pvar = lookup_decl (pvar, ctx);
14517 24 : tree x;
14518 24 : if (!allocate_ptr)
14519 : {
14520 12 : tree atmp = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
14521 12 : tree ty = TREE_TYPE (new_var);
14522 12 : if (by_ref)
14523 11 : ty = TREE_TYPE (ty);
14524 12 : x = TYPE_SIZE_UNIT (ty);
14525 12 : if (TREE_CONSTANT (x))
14526 : break;
14527 9 : tree al = size_int (TYPE_ALIGN (ty));
14528 9 : x = build_call_expr_loc (clause_loc, atmp, 2, x, al);
14529 9 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_pvar), x);
14530 : }
14531 : else
14532 12 : x = *allocate_ptr;
14533 21 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14534 21 : gimple_seq_add_stmt (&new_body,
14535 21 : gimple_build_assign (new_pvar, x));
14536 : }
14537 : break;
14538 15277 : case OMP_CLAUSE_FIRSTPRIVATE:
14539 15277 : var = OMP_CLAUSE_DECL (c);
14540 15277 : by_ref = omp_privatize_by_reference (var);
14541 15277 : if (is_variable_sized (var, by_ref))
14542 : {
14543 35 : tree new_var = lookup_decl (var, ctx);
14544 35 : tree *allocate_ptr = alloc_map.get (new_var);
14545 35 : if (!allocate_ptr)
14546 : break;
14547 15 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14548 15 : gcc_assert (allocate_seq);
14549 15 : gimple_seq_add_seq (&new_body, *allocate_seq);
14550 :
14551 15 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14552 15 : tree pvar = var;
14553 15 : if (!by_ref)
14554 : {
14555 2 : pvar = DECL_VALUE_EXPR (var);
14556 2 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14557 2 : pvar = TREE_OPERAND (pvar, 0);
14558 : }
14559 15 : gcc_assert (DECL_P (pvar));
14560 15 : tree new_pvar = lookup_decl (pvar, ctx);
14561 15 : tree x = fold_convert_loc (clause_loc, TREE_TYPE (new_pvar),
14562 15 : *allocate_ptr);
14563 15 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14564 15 : gimple_seq_add_stmt (&new_body,
14565 15 : gimple_build_assign (new_pvar, x));
14566 :
14567 15 : x = build_receiver_ref (var, true, ctx);
14568 15 : new_var = unshare_expr (new_var);
14569 15 : if (by_ref)
14570 13 : new_var = build_fold_indirect_ref (new_var);
14571 15 : x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
14572 15 : gimplify_and_add (x, &new_body);
14573 : }
14574 : }
14575 :
14576 25132 : gimple_seq fork_seq = NULL;
14577 25132 : gimple_seq join_seq = NULL;
14578 :
14579 25132 : if (offloaded && is_gimple_omp_oacc (ctx->stmt))
14580 : {
14581 : /* If there are reductions on the offloaded region itself, treat
14582 : them as a dummy GANG loop. */
14583 9395 : tree level = build_int_cst (integer_type_node, GOMP_DIM_GANG);
14584 :
14585 9395 : gcall *private_marker = lower_oacc_private_marker (ctx);
14586 :
14587 9395 : if (private_marker)
14588 30 : gimple_call_set_arg (private_marker, 2, level);
14589 :
14590 9395 : lower_oacc_reductions (gimple_location (ctx->stmt), clauses, level,
14591 : false, NULL, private_marker, NULL, &fork_seq,
14592 : &join_seq, ctx);
14593 : }
14594 :
14595 25132 : gimple_seq_add_seq (&new_body, fork_seq);
14596 25132 : gimple_seq_add_seq (&new_body, tgt_body);
14597 25132 : gimple_seq_add_seq (&new_body, join_seq);
14598 25132 : gimple_seq_add_seq (&new_body, alloc_dlist);
14599 :
14600 25132 : if (offloaded)
14601 : {
14602 21111 : new_body = maybe_catch_exception (new_body);
14603 21111 : gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
14604 : }
14605 25132 : gimple_omp_set_body (stmt, new_body);
14606 : }
14607 :
14608 37071 : gsi_insert_seq_before (gsi_p, gimple_omp_target_iterator_loops (stmt),
14609 : GSI_SAME_STMT);
14610 37071 : gimple_omp_target_set_iterator_loops (stmt, NULL);
14611 58182 : bind = gimple_build_bind (NULL, NULL,
14612 21111 : tgt_bind ? gimple_bind_block (tgt_bind)
14613 : : NULL_TREE);
14614 73750 : gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
14615 37071 : gimple_bind_add_seq (bind, ilist);
14616 37071 : gimple_bind_add_stmt (bind, stmt);
14617 37071 : gimple_bind_add_seq (bind, olist);
14618 :
14619 37071 : pop_gimplify_context (NULL);
14620 :
14621 37071 : if (dep_bind)
14622 : {
14623 392 : gimple_bind_add_seq (dep_bind, dep_ilist);
14624 392 : gimple_bind_add_stmt (dep_bind, bind);
14625 392 : gimple_bind_add_seq (dep_bind, dep_olist);
14626 392 : pop_gimplify_context (dep_bind);
14627 : }
14628 37071 : }
14629 :
14630 : /* Generate code to implement the action-clauses (destroy, init, use) of an
14631 : OpenMP interop construct. */
14632 :
14633 : static void
14634 492 : lower_omp_interop_action_clauses (gimple_seq *seq, vec<tree> &objs,
14635 : vec<tree> *interop_types = NULL,
14636 : vec<tree> *prefer_types = NULL)
14637 : {
14638 492 : if (objs.length () == 0)
14639 229 : return;
14640 :
14641 263 : enum omp_clause_code action = OMP_CLAUSE_CODE (objs[0]);
14642 263 : if (action == OMP_CLAUSE_INIT)
14643 402 : gcc_checking_assert (objs.length () == interop_types->length ()
14644 : && objs.length () == prefer_types->length ());
14645 : else
14646 129 : gcc_assert (prefer_types == NULL && interop_types == NULL);
14647 :
14648 263 : tree ret_objs = NULL_TREE, ret_interop_types = NULL_TREE,
14649 263 : ret_prefer_types = NULL_TREE;
14650 :
14651 : /* Build an array of interop objects. */
14652 :
14653 263 : tree type_obj_pref = build_array_type_nelts (ptr_type_node, objs.length ());
14654 263 : ret_objs = create_tmp_var (type_obj_pref, "interopobjs");
14655 :
14656 263 : bool have_pref_type = false;
14657 263 : if (action == OMP_CLAUSE_INIT)
14658 : {
14659 592 : for (tree pref_type : prefer_types)
14660 260 : if (pref_type != NULL_TREE)
14661 : {
14662 : have_pref_type = true;
14663 : break;
14664 : }
14665 134 : tree type_tgtsync
14666 134 : = build_array_type_nelts (integer_type_node, objs.length ());
14667 134 : ret_interop_types = create_tmp_var (type_tgtsync, "tgt_tgtsync");
14668 134 : if (have_pref_type)
14669 70 : ret_prefer_types = create_tmp_var (type_obj_pref, "pref_type");
14670 : else
14671 : {
14672 64 : ret_prefer_types = null_pointer_node;
14673 64 : prefer_types->truncate (0);
14674 : }
14675 : }
14676 :
14677 856 : for (size_t i = 0; !objs.is_empty (); i++)
14678 : {
14679 593 : tree offset = build_int_cst (integer_type_node, i);
14680 593 : tree init = build4 (ARRAY_REF, ptr_type_node, ret_objs, offset, NULL_TREE,
14681 : NULL_TREE);
14682 593 : tree obj = OMP_CLAUSE_DECL (objs.pop ());
14683 593 : if (TREE_CODE (TREE_TYPE (obj)) == REFERENCE_TYPE)
14684 6 : obj = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (obj)), obj);
14685 593 : if (action != OMP_CLAUSE_USE
14686 593 : && TREE_CODE (TREE_TYPE (obj)) != POINTER_TYPE)
14687 : /* For modifying actions, we need a pointer. */
14688 483 : obj = build_fold_addr_expr (obj);
14689 110 : else if (action == OMP_CLAUSE_USE
14690 110 : && TREE_CODE (TREE_TYPE (obj)) == POINTER_TYPE)
14691 : /* For use action, we need the value. */
14692 2 : obj = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (obj)), obj);
14693 593 : init = build2 (MODIFY_EXPR, ptr_type_node, init,
14694 : fold_convert (ptr_type_node, obj));
14695 593 : gimplify_and_add (init, seq);
14696 :
14697 593 : if (action == OMP_CLAUSE_INIT)
14698 : {
14699 384 : init = build4 (ARRAY_REF, integer_type_node, ret_interop_types,
14700 : offset, NULL_TREE, NULL_TREE);
14701 384 : init = build2 (MODIFY_EXPR, integer_type_node, init,
14702 384 : interop_types->pop ());
14703 384 : gimplify_and_add (init, seq);
14704 :
14705 384 : if (have_pref_type)
14706 : {
14707 194 : tree prefer_type = prefer_types->pop ();
14708 194 : tree pref = (prefer_type == NULL_TREE
14709 194 : ? null_pointer_node
14710 194 : : build_fold_addr_expr (prefer_type));
14711 194 : init = build4 (ARRAY_REF, ptr_type_node, ret_prefer_types, offset,
14712 : NULL_TREE, NULL_TREE);
14713 194 : init = build2 (MODIFY_EXPR, ptr_type_node, init, pref);
14714 194 : gimplify_and_add (init, seq);
14715 : }
14716 : }
14717 : }
14718 263 : if (action == OMP_CLAUSE_INIT)
14719 : {
14720 134 : if (have_pref_type)
14721 70 : ret_prefer_types = build_fold_addr_expr (ret_prefer_types);
14722 134 : ret_interop_types = build_fold_addr_expr (ret_interop_types);
14723 : }
14724 263 : ret_objs = build_fold_addr_expr (ret_objs);
14725 :
14726 660 : gcc_assert (objs.is_empty ()
14727 : && (!interop_types || interop_types->is_empty ())
14728 : && (!prefer_types || prefer_types->is_empty ()));
14729 :
14730 263 : objs.safe_push (ret_objs);
14731 263 : if (action == OMP_CLAUSE_INIT)
14732 : {
14733 134 : interop_types->safe_push (ret_interop_types);
14734 134 : prefer_types->safe_push (ret_prefer_types);
14735 : }
14736 : }
14737 :
14738 : /* Lower code for an OpenMP interop directive. */
14739 :
14740 : static void
14741 164 : lower_omp_interop (gimple_stmt_iterator *gsi_p, omp_context *ctx)
14742 : {
14743 164 : push_gimplify_context ();
14744 :
14745 164 : tree block = make_node (BLOCK);
14746 164 : gbind *bind = gimple_build_bind (NULL, NULL, block);
14747 164 : gimple_seq bind_body = NULL;
14748 :
14749 : /* Emit call to GOMP_interop:
14750 : void
14751 : GOMP_interop (int device_num, int n_init, omp_interop_t **init,
14752 : const void *target_targetsync, const void *prefer_type,
14753 : int n_use, omp_interop_t *use, int n_destroy,
14754 : omp_interop_t **destroy, unsigned int flags,
14755 : void **depend) */
14756 :
14757 164 : tree flags = NULL_TREE;
14758 164 : tree depend = null_pointer_node;
14759 164 : tree device_num = NULL_TREE;
14760 :
14761 164 : auto_vec<tree> init_objs, use_objs, destroy_objs, prefer_type,
14762 164 : target_targetsync;
14763 164 : gimple_seq dep_ilist = NULL, dep_olist = NULL;
14764 164 : tree clauses = gimple_omp_interop_clauses (gsi_stmt (*gsi_p));
14765 868 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
14766 : {
14767 704 : switch (OMP_CLAUSE_CODE (c))
14768 : {
14769 384 : case OMP_CLAUSE_INIT:
14770 384 : {
14771 384 : init_objs.safe_push (c);
14772 384 : int target_targetsync_bits = 0;
14773 384 : if (OMP_CLAUSE_INIT_TARGET (c))
14774 229 : target_targetsync_bits |= GOMP_INTEROP_TARGET;
14775 384 : if (OMP_CLAUSE_INIT_TARGETSYNC (c))
14776 204 : target_targetsync_bits |= GOMP_INTEROP_TARGETSYNC;
14777 384 : tree t = build_int_cst (integer_type_node, target_targetsync_bits);
14778 384 : target_targetsync.safe_push (t);
14779 384 : prefer_type.safe_push (OMP_CLAUSE_INIT_PREFER_TYPE (c));
14780 : }
14781 384 : break;
14782 106 : case OMP_CLAUSE_USE:
14783 106 : use_objs.safe_push (c);
14784 106 : break;
14785 103 : case OMP_CLAUSE_DESTROY:
14786 103 : destroy_objs.safe_push (c);
14787 103 : break;
14788 50 : case OMP_CLAUSE_NOWAIT:
14789 50 : flags = build_int_cst (integer_type_node, GOMP_INTEROP_FLAG_NOWAIT);
14790 50 : break;
14791 35 : case OMP_CLAUSE_DEPEND:
14792 35 : {
14793 35 : tree *cp = gimple_omp_interop_clauses_ptr (gsi_stmt (*gsi_p));
14794 35 : lower_depend_clauses (cp, &dep_ilist, &dep_olist);
14795 35 : depend = OMP_CLAUSE_DECL (*cp);
14796 : }
14797 35 : break;
14798 26 : case OMP_CLAUSE_DEVICE:
14799 26 : device_num = OMP_CLAUSE_DEVICE_ID (c);
14800 26 : break;
14801 0 : default:
14802 0 : gcc_unreachable ();
14803 : }
14804 : }
14805 :
14806 164 : if (flags == NULL_TREE)
14807 114 : flags = build_int_cst (integer_type_node, 0);
14808 :
14809 164 : if (device_num == NULL_TREE)
14810 138 : device_num = build_int_cst (integer_type_node, GOMP_DEVICE_DEFAULT_OMP_61);
14811 :
14812 298 : tree n_init = build_int_cst (integer_type_node, init_objs.length ());
14813 234 : tree n_use = build_int_cst (integer_type_node, use_objs.length ());
14814 223 : tree n_destroy = build_int_cst (integer_type_node, destroy_objs.length ());
14815 :
14816 164 : lower_omp_interop_action_clauses (&bind_body, init_objs, &target_targetsync,
14817 : &prefer_type);
14818 164 : lower_omp_interop_action_clauses (&bind_body, use_objs);
14819 164 : lower_omp_interop_action_clauses (&bind_body, destroy_objs);
14820 :
14821 164 : gimple_seq_add_seq (&bind_body, dep_ilist);
14822 164 : tree fn = builtin_decl_explicit (BUILT_IN_GOMP_INTEROP);
14823 298 : tree init_arg = init_objs.length () ? init_objs.pop () : null_pointer_node;
14824 164 : tree target_targetsync_arg = target_targetsync.length ()
14825 134 : ? target_targetsync.pop ()
14826 164 : : null_pointer_node;
14827 164 : tree prefer_type_arg
14828 298 : = prefer_type.length () ? prefer_type.pop () : null_pointer_node;
14829 234 : tree use_arg = use_objs.length () ? use_objs.pop () : null_pointer_node;
14830 164 : tree destroy_arg
14831 223 : = destroy_objs.length () ? destroy_objs.pop () : null_pointer_node;
14832 164 : gcall *call
14833 164 : = gimple_build_call (fn, 11, device_num, n_init, init_arg,
14834 : target_targetsync_arg, prefer_type_arg, n_use, use_arg,
14835 : n_destroy, destroy_arg, flags, depend);
14836 164 : gimple_seq_add_stmt (&bind_body, call);
14837 164 : gimple_seq_add_seq (&bind_body, dep_olist);
14838 :
14839 164 : gsi_replace (gsi_p, bind, true);
14840 164 : gimple_bind_set_body (bind, bind_body);
14841 164 : pop_gimplify_context (bind);
14842 164 : gimple_bind_append_vars (bind, ctx->block_vars);
14843 164 : BLOCK_VARS (block) = ctx->block_vars;
14844 164 : }
14845 :
14846 : /* Expand code for an OpenMP teams directive. */
14847 :
14848 : static void
14849 5917 : lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx)
14850 : {
14851 5917 : gomp_teams *teams_stmt = as_a <gomp_teams *> (gsi_stmt (*gsi_p));
14852 5917 : push_gimplify_context ();
14853 :
14854 5917 : tree block = make_node (BLOCK);
14855 5917 : gbind *bind = gimple_build_bind (NULL, NULL, block);
14856 5917 : gsi_replace (gsi_p, bind, true);
14857 5917 : gimple_seq bind_body = NULL;
14858 5917 : gimple_seq dlist = NULL;
14859 5917 : gimple_seq olist = NULL;
14860 :
14861 5917 : tree num_teams = omp_find_clause (gimple_omp_teams_clauses (teams_stmt),
14862 5917 : OMP_CLAUSE_NUM_TEAMS);
14863 5917 : tree num_teams_lower = NULL_TREE;
14864 5917 : if (num_teams == NULL_TREE)
14865 5300 : num_teams = build_int_cst (unsigned_type_node, 0);
14866 : else
14867 : {
14868 617 : num_teams_lower = OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (num_teams);
14869 617 : if (num_teams_lower)
14870 : {
14871 148 : num_teams_lower = fold_convert (unsigned_type_node, num_teams_lower);
14872 148 : gimplify_expr (&num_teams_lower, &bind_body, NULL, is_gimple_val,
14873 : fb_rvalue);
14874 : }
14875 617 : num_teams = OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (num_teams);
14876 617 : num_teams = fold_convert (unsigned_type_node, num_teams);
14877 617 : gimplify_expr (&num_teams, &bind_body, NULL, is_gimple_val, fb_rvalue);
14878 : }
14879 5917 : if (num_teams_lower == NULL_TREE)
14880 5769 : num_teams_lower = num_teams;
14881 5917 : tree thread_limit = omp_find_clause (gimple_omp_teams_clauses (teams_stmt),
14882 5917 : OMP_CLAUSE_THREAD_LIMIT);
14883 5917 : if (thread_limit == NULL_TREE)
14884 5446 : thread_limit = build_int_cst (unsigned_type_node, 0);
14885 : else
14886 : {
14887 471 : thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit);
14888 471 : thread_limit = fold_convert (unsigned_type_node, thread_limit);
14889 471 : gimplify_expr (&thread_limit, &bind_body, NULL, is_gimple_val,
14890 : fb_rvalue);
14891 : }
14892 5917 : location_t loc = gimple_location (teams_stmt);
14893 5917 : tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS4);
14894 5917 : tree rettype = TREE_TYPE (TREE_TYPE (decl));
14895 5917 : tree first = create_tmp_var (rettype);
14896 5917 : gimple_seq_add_stmt (&bind_body,
14897 5917 : gimple_build_assign (first, build_one_cst (rettype)));
14898 5917 : tree llabel = create_artificial_label (loc);
14899 5917 : gimple_seq_add_stmt (&bind_body, gimple_build_label (llabel));
14900 5917 : gimple *call
14901 5917 : = gimple_build_call (decl, 4, num_teams_lower, num_teams, thread_limit,
14902 : first);
14903 5917 : gimple_set_location (call, loc);
14904 5917 : tree temp = create_tmp_var (rettype);
14905 5917 : gimple_call_set_lhs (call, temp);
14906 5917 : gimple_seq_add_stmt (&bind_body, call);
14907 :
14908 5917 : tree tlabel = create_artificial_label (loc);
14909 5917 : tree flabel = create_artificial_label (loc);
14910 5917 : gimple *cond = gimple_build_cond (NE_EXPR, temp, build_zero_cst (rettype),
14911 : tlabel, flabel);
14912 5917 : gimple_seq_add_stmt (&bind_body, cond);
14913 5917 : gimple_seq_add_stmt (&bind_body, gimple_build_label (tlabel));
14914 5917 : gimple_seq_add_stmt (&bind_body,
14915 5917 : gimple_build_assign (first, build_zero_cst (rettype)));
14916 :
14917 5917 : lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt),
14918 : &bind_body, &dlist, ctx, NULL);
14919 5917 : lower_omp (gimple_omp_body_ptr (teams_stmt), ctx);
14920 5917 : lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt), &olist,
14921 : NULL, ctx);
14922 5917 : gimple_seq_add_stmt (&bind_body, teams_stmt);
14923 :
14924 5917 : gimple_seq_add_seq (&bind_body, gimple_omp_body (teams_stmt));
14925 5917 : gimple_omp_set_body (teams_stmt, NULL);
14926 5917 : gimple_seq_add_seq (&bind_body, olist);
14927 5917 : gimple_seq_add_seq (&bind_body, dlist);
14928 5917 : gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true));
14929 5917 : gimple_seq_add_stmt (&bind_body, gimple_build_goto (llabel));
14930 5917 : gimple_seq_add_stmt (&bind_body, gimple_build_label (flabel));
14931 5917 : gimple_bind_set_body (bind, bind_body);
14932 :
14933 5917 : pop_gimplify_context (bind);
14934 :
14935 5917 : gimple_bind_append_vars (bind, ctx->block_vars);
14936 5917 : BLOCK_VARS (block) = ctx->block_vars;
14937 5917 : if (BLOCK_VARS (block))
14938 687 : TREE_USED (block) = 1;
14939 5917 : }
14940 :
14941 : /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
14942 : regimplified. If DATA is non-NULL, lower_omp_1 is outside
14943 : of OMP context, but with make_addressable_vars set. */
14944 :
14945 : static tree
14946 3209075 : lower_omp_regimplify_p (tree *tp, int *walk_subtrees,
14947 : void *data)
14948 : {
14949 3209075 : tree t = *tp;
14950 :
14951 : /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
14952 3209075 : if ((VAR_P (t) || TREE_CODE (t) == PARM_DECL || TREE_CODE (t) == RESULT_DECL)
14953 1494456 : && data == NULL
14954 1434030 : && DECL_HAS_VALUE_EXPR_P (t))
14955 : return t;
14956 :
14957 3103280 : if (make_addressable_vars
14958 422645 : && DECL_P (t)
14959 3308326 : && bitmap_bit_p (make_addressable_vars, DECL_UID (t)))
14960 : return t;
14961 :
14962 : /* If a global variable has been privatized, TREE_CONSTANT on
14963 : ADDR_EXPR might be wrong. */
14964 3100691 : if (data == NULL && TREE_CODE (t) == ADDR_EXPR)
14965 108640 : recompute_tree_invariant_for_addr_expr (t);
14966 :
14967 3100691 : *walk_subtrees = !IS_TYPE_OR_DECL_P (t);
14968 3100691 : return NULL_TREE;
14969 : }
14970 :
14971 : /* Data to be communicated between lower_omp_regimplify_operands and
14972 : lower_omp_regimplify_operands_p. */
14973 :
14974 : struct lower_omp_regimplify_operands_data
14975 : {
14976 : omp_context *ctx;
14977 : vec<tree> *decls;
14978 : };
14979 :
14980 : /* Helper function for lower_omp_regimplify_operands. Find
14981 : omp_member_access_dummy_var vars and adjust temporarily their
14982 : DECL_VALUE_EXPRs if needed. */
14983 :
14984 : static tree
14985 426784 : lower_omp_regimplify_operands_p (tree *tp, int *walk_subtrees,
14986 : void *data)
14987 : {
14988 426784 : tree t = omp_member_access_dummy_var (*tp);
14989 426784 : if (t)
14990 : {
14991 9 : struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
14992 9 : lower_omp_regimplify_operands_data *ldata
14993 : = (lower_omp_regimplify_operands_data *) wi->info;
14994 9 : tree o = maybe_lookup_decl (t, ldata->ctx);
14995 9 : if (o == NULL_TREE)
14996 5 : o = maybe_lookup_decl_in_outer_ctx (t, ldata->ctx);
14997 9 : if (o != t)
14998 : {
14999 9 : ldata->decls->safe_push (DECL_VALUE_EXPR (*tp));
15000 9 : ldata->decls->safe_push (*tp);
15001 9 : tree v = unshare_and_remap (DECL_VALUE_EXPR (*tp), t, o);
15002 9 : SET_DECL_VALUE_EXPR (*tp, v);
15003 : }
15004 : }
15005 426784 : *walk_subtrees = !IS_TYPE_OR_DECL_P (*tp);
15006 426784 : return NULL_TREE;
15007 : }
15008 :
15009 : /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
15010 : of omp_member_access_dummy_var vars during regimplification. */
15011 :
15012 : static void
15013 108229 : lower_omp_regimplify_operands (omp_context *ctx, gimple *stmt,
15014 : gimple_stmt_iterator *gsi_p)
15015 : {
15016 108229 : auto_vec<tree, 10> decls;
15017 108229 : if (ctx)
15018 : {
15019 105707 : struct walk_stmt_info wi;
15020 105707 : memset (&wi, '\0', sizeof (wi));
15021 105707 : struct lower_omp_regimplify_operands_data data;
15022 105707 : data.ctx = ctx;
15023 105707 : data.decls = &decls;
15024 105707 : wi.info = &data;
15025 105707 : walk_gimple_op (stmt, lower_omp_regimplify_operands_p, &wi);
15026 : }
15027 108229 : gimple_regimplify_operands (stmt, gsi_p);
15028 324696 : while (!decls.is_empty ())
15029 : {
15030 9 : tree t = decls.pop ();
15031 9 : tree v = decls.pop ();
15032 9 : SET_DECL_VALUE_EXPR (t, v);
15033 : }
15034 108229 : }
15035 :
15036 : static void
15037 2505740 : lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx)
15038 : {
15039 2505740 : gimple *stmt = gsi_stmt (*gsi_p);
15040 2505740 : struct walk_stmt_info wi;
15041 2505740 : gcall *call_stmt;
15042 :
15043 2505740 : if (gimple_has_location (stmt))
15044 2058189 : input_location = gimple_location (stmt);
15045 :
15046 2505740 : if (make_addressable_vars)
15047 154096 : memset (&wi, '\0', sizeof (wi));
15048 :
15049 : /* If we have issued syntax errors, avoid doing any heavy lifting.
15050 : Just replace the OMP directives with a NOP to avoid
15051 : confusing RTL expansion. */
15052 2505740 : if (seen_error () && is_gimple_omp (stmt))
15053 : {
15054 11430 : gsi_replace (gsi_p, gimple_build_nop (), true);
15055 11430 : return;
15056 : }
15057 :
15058 2494310 : switch (gimple_code (stmt))
15059 : {
15060 155981 : case GIMPLE_COND:
15061 155981 : {
15062 155981 : gcond *cond_stmt = as_a <gcond *> (stmt);
15063 88668 : if ((ctx || make_addressable_vars)
15064 159589 : && (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
15065 : lower_omp_regimplify_p,
15066 : ctx ? NULL : &wi, NULL)
15067 136291 : || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
15068 : lower_omp_regimplify_p,
15069 : ctx ? NULL : &wi, NULL)))
15070 1245 : lower_omp_regimplify_operands (ctx, cond_stmt, gsi_p);
15071 : }
15072 : break;
15073 20 : case GIMPLE_CATCH:
15074 20 : lower_omp (gimple_catch_handler_ptr (as_a <gcatch *> (stmt)), ctx);
15075 20 : break;
15076 0 : case GIMPLE_EH_FILTER:
15077 0 : lower_omp (gimple_eh_filter_failure_ptr (stmt), ctx);
15078 0 : break;
15079 25917 : case GIMPLE_TRY:
15080 25917 : lower_omp (gimple_try_eval_ptr (stmt), ctx);
15081 25912 : lower_omp (gimple_try_cleanup_ptr (stmt), ctx);
15082 25912 : break;
15083 0 : case GIMPLE_ASSUME:
15084 0 : lower_omp (gimple_assume_body_ptr (stmt), ctx);
15085 0 : break;
15086 5 : case GIMPLE_TRANSACTION:
15087 5 : lower_omp (gimple_transaction_body_ptr (as_a <gtransaction *> (stmt)),
15088 : ctx);
15089 5 : break;
15090 168035 : case GIMPLE_BIND:
15091 168035 : if (ctx && is_gimple_omp_oacc (ctx->stmt))
15092 : {
15093 17785 : tree vars = gimple_bind_vars (as_a <gbind *> (stmt));
15094 17785 : oacc_privatization_scan_decl_chain (ctx, vars);
15095 : }
15096 168035 : lower_omp (gimple_bind_body_ptr (as_a <gbind *> (stmt)), ctx);
15097 168028 : maybe_remove_omp_member_access_dummy_vars (as_a <gbind *> (stmt));
15098 168028 : break;
15099 19981 : case GIMPLE_OMP_PARALLEL:
15100 19981 : case GIMPLE_OMP_TASK:
15101 19981 : ctx = maybe_lookup_ctx (stmt);
15102 19981 : gcc_assert (ctx);
15103 19981 : if (ctx->cancellable)
15104 135 : ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
15105 19981 : lower_omp_taskreg (gsi_p, ctx);
15106 19981 : break;
15107 46890 : case GIMPLE_OMP_FOR:
15108 46890 : ctx = maybe_lookup_ctx (stmt);
15109 46890 : gcc_assert (ctx);
15110 46890 : if (ctx->cancellable)
15111 84 : ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
15112 46890 : lower_omp_for (gsi_p, ctx);
15113 46890 : break;
15114 378 : case GIMPLE_OMP_SECTIONS:
15115 378 : ctx = maybe_lookup_ctx (stmt);
15116 378 : gcc_assert (ctx);
15117 378 : if (ctx->cancellable)
15118 19 : ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
15119 378 : lower_omp_sections (gsi_p, ctx);
15120 378 : break;
15121 133 : case GIMPLE_OMP_SCOPE:
15122 133 : ctx = maybe_lookup_ctx (stmt);
15123 133 : gcc_assert (ctx);
15124 133 : lower_omp_scope (gsi_p, ctx);
15125 133 : break;
15126 659 : case GIMPLE_OMP_DISPATCH:
15127 659 : ctx = maybe_lookup_ctx (stmt);
15128 659 : gcc_assert (ctx);
15129 659 : lower_omp_dispatch (gsi_p, ctx);
15130 659 : break;
15131 164 : case GIMPLE_OMP_INTEROP:
15132 164 : ctx = maybe_lookup_ctx (stmt);
15133 164 : gcc_assert (ctx);
15134 164 : lower_omp_interop (gsi_p, ctx);
15135 164 : break;
15136 1119 : case GIMPLE_OMP_SINGLE:
15137 1119 : ctx = maybe_lookup_ctx (stmt);
15138 1119 : gcc_assert (ctx);
15139 1119 : lower_omp_single (gsi_p, ctx);
15140 1119 : break;
15141 658 : case GIMPLE_OMP_STRUCTURED_BLOCK:
15142 : /* We have already done error checking at this point, so these nodes
15143 : can be completely removed and replaced with their body. */
15144 658 : ctx = maybe_lookup_ctx (stmt);
15145 658 : gcc_assert (ctx);
15146 658 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
15147 658 : gsi_replace_with_seq (gsi_p, gimple_omp_body (stmt), true);
15148 658 : break;
15149 1114 : case GIMPLE_OMP_MASTER:
15150 1114 : case GIMPLE_OMP_MASKED:
15151 1114 : ctx = maybe_lookup_ctx (stmt);
15152 1114 : gcc_assert (ctx);
15153 1114 : lower_omp_master (gsi_p, ctx);
15154 1114 : break;
15155 536 : case GIMPLE_OMP_TASKGROUP:
15156 536 : ctx = maybe_lookup_ctx (stmt);
15157 536 : gcc_assert (ctx);
15158 536 : lower_omp_taskgroup (gsi_p, ctx);
15159 536 : break;
15160 1124 : case GIMPLE_OMP_ORDERED:
15161 1124 : ctx = maybe_lookup_ctx (stmt);
15162 1124 : gcc_assert (ctx);
15163 1124 : lower_omp_ordered (gsi_p, ctx);
15164 1124 : break;
15165 1268 : case GIMPLE_OMP_SCAN:
15166 1268 : ctx = maybe_lookup_ctx (stmt);
15167 1268 : gcc_assert (ctx);
15168 1268 : lower_omp_scan (gsi_p, ctx);
15169 1268 : break;
15170 311 : case GIMPLE_OMP_CRITICAL:
15171 311 : ctx = maybe_lookup_ctx (stmt);
15172 311 : gcc_assert (ctx);
15173 311 : lower_omp_critical (gsi_p, ctx);
15174 311 : break;
15175 2516 : case GIMPLE_OMP_ATOMIC_LOAD:
15176 85 : if ((ctx || make_addressable_vars)
15177 2516 : && walk_tree (gimple_omp_atomic_load_rhs_ptr (
15178 : as_a <gomp_atomic_load *> (stmt)),
15179 : lower_omp_regimplify_p, ctx ? NULL : &wi, NULL))
15180 1425 : lower_omp_regimplify_operands (ctx, stmt, gsi_p);
15181 : break;
15182 37071 : case GIMPLE_OMP_TARGET:
15183 37071 : ctx = maybe_lookup_ctx (stmt);
15184 37071 : gcc_assert (ctx);
15185 37071 : lower_omp_target (gsi_p, ctx);
15186 37071 : break;
15187 8413 : case GIMPLE_OMP_TEAMS:
15188 8413 : ctx = maybe_lookup_ctx (stmt);
15189 8413 : gcc_assert (ctx);
15190 8413 : if (gimple_omp_teams_host (as_a <gomp_teams *> (stmt)))
15191 2496 : lower_omp_taskreg (gsi_p, ctx);
15192 : else
15193 5917 : lower_omp_teams (gsi_p, ctx);
15194 : break;
15195 134497 : case GIMPLE_CALL:
15196 134497 : tree fndecl;
15197 134497 : call_stmt = as_a <gcall *> (stmt);
15198 134497 : fndecl = gimple_call_fndecl (call_stmt);
15199 134497 : if (fndecl
15200 134497 : && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL))
15201 38601 : switch (DECL_FUNCTION_CODE (fndecl))
15202 : {
15203 727 : case BUILT_IN_GOMP_BARRIER:
15204 727 : if (ctx == NULL)
15205 : break;
15206 : /* FALLTHRU */
15207 1037 : case BUILT_IN_GOMP_CANCEL:
15208 1037 : case BUILT_IN_GOMP_CANCELLATION_POINT:
15209 1037 : omp_context *cctx;
15210 1037 : cctx = ctx;
15211 1037 : if (gimple_code (cctx->stmt) == GIMPLE_OMP_SECTION)
15212 43 : cctx = cctx->outer;
15213 1037 : gcc_assert (gimple_call_lhs (call_stmt) == NULL_TREE);
15214 1037 : if (!cctx->cancellable)
15215 : {
15216 679 : if (DECL_FUNCTION_CODE (fndecl)
15217 : == BUILT_IN_GOMP_CANCELLATION_POINT)
15218 : {
15219 0 : stmt = gimple_build_nop ();
15220 0 : gsi_replace (gsi_p, stmt, false);
15221 : }
15222 : break;
15223 : }
15224 358 : if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_GOMP_BARRIER)
15225 : {
15226 14 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL);
15227 14 : gimple_call_set_fndecl (call_stmt, fndecl);
15228 14 : gimple_call_set_fntype (call_stmt, TREE_TYPE (fndecl));
15229 : }
15230 358 : tree lhs;
15231 358 : lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl)));
15232 358 : gimple_call_set_lhs (call_stmt, lhs);
15233 358 : tree fallthru_label;
15234 358 : fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
15235 358 : gimple *g;
15236 358 : g = gimple_build_label (fallthru_label);
15237 358 : gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
15238 358 : g = gimple_build_cond (NE_EXPR, lhs,
15239 358 : fold_convert (TREE_TYPE (lhs),
15240 : boolean_false_node),
15241 : cctx->cancel_label, fallthru_label);
15242 358 : gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
15243 358 : break;
15244 : default:
15245 : break;
15246 : }
15247 134462 : goto regimplify;
15248 :
15249 : case GIMPLE_ASSIGN:
15250 1454247 : for (omp_context *up = ctx; up; up = up->outer)
15251 : {
15252 684666 : if (gimple_code (up->stmt) == GIMPLE_OMP_ORDERED
15253 : || gimple_code (up->stmt) == GIMPLE_OMP_CRITICAL
15254 : || gimple_code (up->stmt) == GIMPLE_OMP_TASKGROUP
15255 : || gimple_code (up->stmt) == GIMPLE_OMP_SCOPE
15256 : || gimple_code (up->stmt) == GIMPLE_OMP_SECTION
15257 : || gimple_code (up->stmt) == GIMPLE_OMP_SCAN
15258 : || (gimple_code (up->stmt) == GIMPLE_OMP_TARGET
15259 229763 : && (gimple_omp_target_kind (up->stmt)
15260 : == GF_OMP_TARGET_KIND_DATA)))
15261 90671 : continue;
15262 593995 : else if (!up->lastprivate_conditional_map)
15263 : break;
15264 4204 : tree lhs = get_base_address (gimple_assign_lhs (stmt));
15265 4204 : if (TREE_CODE (lhs) == MEM_REF
15266 26 : && DECL_P (TREE_OPERAND (lhs, 0))
15267 4227 : && TREE_CODE (TREE_TYPE (TREE_OPERAND (lhs,
15268 : 0))) == REFERENCE_TYPE)
15269 19 : lhs = TREE_OPERAND (lhs, 0);
15270 4204 : if (DECL_P (lhs))
15271 3161 : if (tree *v = up->lastprivate_conditional_map->get (lhs))
15272 : {
15273 382 : tree clauses;
15274 382 : if (up->combined_into_simd_safelen1)
15275 : {
15276 54 : up = up->outer;
15277 54 : if (gimple_code (up->stmt) == GIMPLE_OMP_SCAN)
15278 3 : up = up->outer;
15279 : }
15280 382 : if (gimple_code (up->stmt) == GIMPLE_OMP_FOR)
15281 298 : clauses = gimple_omp_for_clauses (up->stmt);
15282 : else
15283 84 : clauses = gimple_omp_sections_clauses (up->stmt);
15284 382 : tree c = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_);
15285 382 : if (!OMP_CLAUSE__CONDTEMP__ITER (c))
15286 348 : c = omp_find_clause (OMP_CLAUSE_CHAIN (c),
15287 : OMP_CLAUSE__CONDTEMP_);
15288 382 : gcc_assert (OMP_CLAUSE__CONDTEMP__ITER (c));
15289 382 : gimple *g = gimple_build_assign (*v, OMP_CLAUSE_DECL (c));
15290 382 : gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
15291 : }
15292 : }
15293 : /* FALLTHRU */
15294 :
15295 2022017 : default:
15296 2022017 : regimplify:
15297 1065642 : if ((ctx || make_addressable_vars)
15298 2097846 : && walk_gimple_op (stmt, lower_omp_regimplify_p,
15299 : ctx ? NULL : &wi))
15300 : {
15301 : /* Just remove clobbers, this should happen only if we have
15302 : "privatized" local addressable variables in SIMD regions,
15303 : the clobber isn't needed in that case and gimplifying address
15304 : of the ARRAY_REF into a pointer and creating MEM_REF based
15305 : clobber would create worse code than we get with the clobber
15306 : dropped. */
15307 105714 : if (gimple_clobber_p (stmt))
15308 : {
15309 155 : gsi_replace (gsi_p, gimple_build_nop (), true);
15310 155 : break;
15311 : }
15312 105559 : lower_omp_regimplify_operands (ctx, stmt, gsi_p);
15313 : }
15314 : break;
15315 : }
15316 : }
15317 :
15318 : static void
15319 410432 : lower_omp (gimple_seq *body, omp_context *ctx)
15320 : {
15321 410432 : location_t saved_location = input_location;
15322 410432 : gimple_stmt_iterator gsi;
15323 3278447 : for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
15324 2505740 : lower_omp_1 (&gsi, ctx);
15325 : /* During gimplification, we haven't folded statements inside offloading
15326 : or taskreg regions (gimplify.cc:maybe_fold_stmt); do that now. */
15327 410416 : if (target_nesting_level || taskreg_nesting_level)
15328 655351 : for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
15329 442534 : fold_stmt (&gsi);
15330 410416 : input_location = saved_location;
15331 410416 : }
15332 :
15333 : /* Main entry point. */
15334 :
15335 : static unsigned int
15336 2880198 : execute_lower_omp (void)
15337 : {
15338 2880198 : gimple_seq body;
15339 2880198 : int i;
15340 2880198 : omp_context *ctx;
15341 :
15342 : /* This pass always runs, to provide PROP_gimple_lomp.
15343 : But often, there is nothing to do. */
15344 2880198 : if (flag_openacc == 0 && flag_openmp == 0
15345 2825519 : && flag_openmp_simd == 0)
15346 : return 0;
15347 :
15348 56936 : all_contexts = splay_tree_new (splay_tree_compare_pointers, 0,
15349 : delete_omp_context);
15350 :
15351 56936 : body = gimple_body (current_function_decl);
15352 :
15353 56936 : scan_omp (&body, NULL);
15354 56936 : gcc_assert (taskreg_nesting_level == 0);
15355 82950 : FOR_EACH_VEC_ELT (taskreg_contexts, i, ctx)
15356 26014 : finish_taskreg_scan (ctx);
15357 56936 : taskreg_contexts.release ();
15358 :
15359 56936 : if (all_contexts->root)
15360 : {
15361 25700 : if (make_addressable_vars)
15362 620 : push_gimplify_context ();
15363 25700 : lower_omp (&body, NULL);
15364 25696 : if (make_addressable_vars)
15365 620 : pop_gimplify_context (NULL);
15366 : }
15367 :
15368 56932 : if (all_contexts)
15369 : {
15370 56932 : splay_tree_delete (all_contexts);
15371 56932 : all_contexts = NULL;
15372 : }
15373 56932 : BITMAP_FREE (make_addressable_vars);
15374 56932 : BITMAP_FREE (global_nonaddressable_vars);
15375 :
15376 : /* If current function is a method, remove artificial dummy VAR_DECL created
15377 : for non-static data member privatization, they aren't needed for
15378 : debuginfo nor anything else, have been already replaced everywhere in the
15379 : IL and cause problems with LTO. */
15380 56932 : if (DECL_ARGUMENTS (current_function_decl)
15381 32481 : && DECL_ARTIFICIAL (DECL_ARGUMENTS (current_function_decl))
15382 66485 : && (TREE_CODE (TREE_TYPE (DECL_ARGUMENTS (current_function_decl)))
15383 : == POINTER_TYPE))
15384 8991 : remove_member_access_dummy_vars (DECL_INITIAL (current_function_decl));
15385 :
15386 57908 : for (auto task_stmt : task_cpyfns)
15387 504 : finalize_task_copyfn (task_stmt);
15388 56932 : task_cpyfns.release ();
15389 56932 : return 0;
15390 : }
15391 :
15392 : namespace {
15393 :
15394 : const pass_data pass_data_lower_omp =
15395 : {
15396 : GIMPLE_PASS, /* type */
15397 : "omplower", /* name */
15398 : OPTGROUP_OMP, /* optinfo_flags */
15399 : TV_NONE, /* tv_id */
15400 : PROP_gimple_any, /* properties_required */
15401 : PROP_gimple_lomp | PROP_gimple_lomp_dev, /* properties_provided */
15402 : 0, /* properties_destroyed */
15403 : 0, /* todo_flags_start */
15404 : 0, /* todo_flags_finish */
15405 : };
15406 :
15407 : class pass_lower_omp : public gimple_opt_pass
15408 : {
15409 : public:
15410 298828 : pass_lower_omp (gcc::context *ctxt)
15411 597656 : : gimple_opt_pass (pass_data_lower_omp, ctxt)
15412 : {}
15413 :
15414 : /* opt_pass methods: */
15415 2880198 : unsigned int execute (function *) final override
15416 : {
15417 2880198 : return execute_lower_omp ();
15418 : }
15419 :
15420 : }; // class pass_lower_omp
15421 :
15422 : } // anon namespace
15423 :
15424 : gimple_opt_pass *
15425 298828 : make_pass_lower_omp (gcc::context *ctxt)
15426 : {
15427 298828 : return new pass_lower_omp (ctxt);
15428 : }
15429 :
15430 : /* The following is a utility to diagnose structured block violations.
15431 : It is not part of the "omplower" pass, as that's invoked too late. It
15432 : should be invoked by the respective front ends after gimplification. */
15433 :
15434 : static splay_tree all_labels;
15435 :
15436 : /* Check for mismatched contexts and generate an error if needed. Return
15437 : true if an error is detected. */
15438 :
15439 : static bool
15440 560649 : diagnose_sb_0 (gimple_stmt_iterator *gsi_p,
15441 : gimple *branch_ctx, gimple *label_ctx)
15442 : {
15443 560649 : gcc_checking_assert (!branch_ctx || is_gimple_omp (branch_ctx));
15444 560649 : gcc_checking_assert (!label_ctx || is_gimple_omp (label_ctx));
15445 :
15446 560649 : if (label_ctx == branch_ctx)
15447 : return false;
15448 :
15449 180 : const char* kind = NULL;
15450 :
15451 180 : if (flag_openacc)
15452 : {
15453 7 : if ((branch_ctx && is_gimple_omp_oacc (branch_ctx))
15454 16 : || (label_ctx && is_gimple_omp_oacc (label_ctx)))
15455 : {
15456 : gcc_checking_assert (kind == NULL);
15457 : kind = "OpenACC";
15458 : }
15459 : }
15460 : if (kind == NULL)
15461 : {
15462 164 : gcc_checking_assert (flag_openmp || flag_openmp_simd);
15463 : kind = "OpenMP";
15464 : }
15465 :
15466 : /* Previously we kept track of the label's entire context in diagnose_sb_[12]
15467 : so we could traverse it and issue a correct "exit" or "enter" error
15468 : message upon a structured block violation.
15469 :
15470 : We built the context by building a list with tree_cons'ing, but there is
15471 : no easy counterpart in gimple tuples. It seems like far too much work
15472 : for issuing exit/enter error messages. If someone really misses the
15473 : distinct error message... patches welcome. */
15474 :
15475 : #if 0
15476 : /* Try to avoid confusing the user by producing and error message
15477 : with correct "exit" or "enter" verbiage. We prefer "exit"
15478 : unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
15479 : if (branch_ctx == NULL)
15480 : exit_p = false;
15481 : else
15482 : {
15483 : while (label_ctx)
15484 : {
15485 : if (TREE_VALUE (label_ctx) == branch_ctx)
15486 : {
15487 : exit_p = false;
15488 : break;
15489 : }
15490 : label_ctx = TREE_CHAIN (label_ctx);
15491 : }
15492 : }
15493 :
15494 : if (exit_p)
15495 : error ("invalid exit from %s structured block", kind);
15496 : else
15497 : error ("invalid entry to %s structured block", kind);
15498 : #endif
15499 :
15500 : /* If it's obvious we have an invalid entry, be specific about the error. */
15501 180 : if (branch_ctx == NULL)
15502 53 : error ("invalid entry to %s structured block", kind);
15503 : else
15504 : {
15505 : /* Otherwise, be vague and lazy, but efficient. */
15506 127 : error ("invalid branch to/from %s structured block", kind);
15507 : }
15508 :
15509 180 : gsi_replace (gsi_p, gimple_build_nop (), false);
15510 180 : return true;
15511 : }
15512 :
15513 : /* Pass 1: Create a minimal tree of structured blocks, and record
15514 : where each label is found. */
15515 :
15516 : static tree
15517 3347195 : diagnose_sb_1 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
15518 : struct walk_stmt_info *wi)
15519 : {
15520 3347195 : gimple *context = (gimple *) wi->info;
15521 3347195 : gimple *inner_context;
15522 3347195 : gimple *stmt = gsi_stmt (*gsi_p);
15523 :
15524 3347195 : *handled_ops_p = true;
15525 :
15526 3347195 : switch (gimple_code (stmt))
15527 : {
15528 319408 : WALK_SUBSTMTS;
15529 :
15530 83587 : case GIMPLE_OMP_PARALLEL:
15531 83587 : case GIMPLE_OMP_TASK:
15532 83587 : case GIMPLE_OMP_SCOPE:
15533 83587 : case GIMPLE_OMP_SECTIONS:
15534 83587 : case GIMPLE_OMP_SINGLE:
15535 83587 : case GIMPLE_OMP_SECTION:
15536 83587 : case GIMPLE_OMP_STRUCTURED_BLOCK:
15537 83587 : case GIMPLE_OMP_MASTER:
15538 83587 : case GIMPLE_OMP_MASKED:
15539 83587 : case GIMPLE_OMP_ORDERED:
15540 83587 : case GIMPLE_OMP_SCAN:
15541 83587 : case GIMPLE_OMP_CRITICAL:
15542 83587 : case GIMPLE_OMP_TARGET:
15543 83587 : case GIMPLE_OMP_TEAMS:
15544 83587 : case GIMPLE_OMP_TASKGROUP:
15545 : /* The minimal context here is just the current OMP construct. */
15546 83587 : inner_context = stmt;
15547 83587 : wi->info = inner_context;
15548 83587 : walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
15549 83587 : wi->info = context;
15550 83587 : break;
15551 :
15552 52754 : case GIMPLE_OMP_FOR:
15553 52754 : inner_context = stmt;
15554 52754 : wi->info = inner_context;
15555 : /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
15556 : walk them. */
15557 52754 : walk_gimple_seq (gimple_omp_for_pre_body (stmt),
15558 : diagnose_sb_1, NULL, wi);
15559 52754 : walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
15560 52754 : wi->info = context;
15561 52754 : break;
15562 :
15563 557412 : case GIMPLE_LABEL:
15564 1114824 : splay_tree_insert (all_labels,
15565 557412 : (splay_tree_key) gimple_label_label (
15566 557412 : as_a <glabel *> (stmt)),
15567 : (splay_tree_value) context);
15568 557412 : break;
15569 :
15570 : default:
15571 : break;
15572 : }
15573 :
15574 3347195 : return NULL_TREE;
15575 : }
15576 :
15577 : /* Pass 2: Check each branch and see if its context differs from that of
15578 : the destination label's context. */
15579 :
15580 : static tree
15581 3347195 : diagnose_sb_2 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
15582 : struct walk_stmt_info *wi)
15583 : {
15584 3347195 : gimple *context = (gimple *) wi->info;
15585 3347195 : splay_tree_node n;
15586 3347195 : gimple *stmt = gsi_stmt (*gsi_p);
15587 :
15588 3347195 : *handled_ops_p = true;
15589 :
15590 3347195 : switch (gimple_code (stmt))
15591 : {
15592 319408 : WALK_SUBSTMTS;
15593 :
15594 83587 : case GIMPLE_OMP_PARALLEL:
15595 83587 : case GIMPLE_OMP_TASK:
15596 83587 : case GIMPLE_OMP_SCOPE:
15597 83587 : case GIMPLE_OMP_SECTIONS:
15598 83587 : case GIMPLE_OMP_SINGLE:
15599 83587 : case GIMPLE_OMP_SECTION:
15600 83587 : case GIMPLE_OMP_STRUCTURED_BLOCK:
15601 83587 : case GIMPLE_OMP_MASTER:
15602 83587 : case GIMPLE_OMP_MASKED:
15603 83587 : case GIMPLE_OMP_ORDERED:
15604 83587 : case GIMPLE_OMP_SCAN:
15605 83587 : case GIMPLE_OMP_CRITICAL:
15606 83587 : case GIMPLE_OMP_TARGET:
15607 83587 : case GIMPLE_OMP_TEAMS:
15608 83587 : case GIMPLE_OMP_TASKGROUP:
15609 83587 : wi->info = stmt;
15610 83587 : walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
15611 83587 : wi->info = context;
15612 83587 : break;
15613 :
15614 52754 : case GIMPLE_OMP_FOR:
15615 52754 : wi->info = stmt;
15616 : /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
15617 : walk them. */
15618 52754 : walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt),
15619 : diagnose_sb_2, NULL, wi);
15620 52754 : walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
15621 52754 : wi->info = context;
15622 52754 : break;
15623 :
15624 206397 : case GIMPLE_COND:
15625 206397 : {
15626 206397 : gcond *cond_stmt = as_a <gcond *> (stmt);
15627 206397 : tree lab = gimple_cond_true_label (cond_stmt);
15628 206397 : if (lab)
15629 : {
15630 206397 : n = splay_tree_lookup (all_labels,
15631 : (splay_tree_key) lab);
15632 412794 : diagnose_sb_0 (gsi_p, context,
15633 206397 : n ? (gimple *) n->value : NULL);
15634 : }
15635 206397 : lab = gimple_cond_false_label (cond_stmt);
15636 206397 : if (lab)
15637 : {
15638 206397 : n = splay_tree_lookup (all_labels,
15639 : (splay_tree_key) lab);
15640 412794 : diagnose_sb_0 (gsi_p, context,
15641 206397 : n ? (gimple *) n->value : NULL);
15642 : }
15643 : }
15644 : break;
15645 :
15646 112018 : case GIMPLE_GOTO:
15647 112018 : {
15648 112018 : tree lab = gimple_goto_dest (stmt);
15649 112018 : if (TREE_CODE (lab) != LABEL_DECL)
15650 : break;
15651 :
15652 112018 : n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
15653 112018 : diagnose_sb_0 (gsi_p, context, n ? (gimple *) n->value : NULL);
15654 : }
15655 112018 : break;
15656 :
15657 424 : case GIMPLE_SWITCH:
15658 424 : {
15659 424 : gswitch *switch_stmt = as_a <gswitch *> (stmt);
15660 424 : unsigned int i;
15661 1724 : for (i = 0; i < gimple_switch_num_labels (switch_stmt); ++i)
15662 : {
15663 1316 : tree lab = CASE_LABEL (gimple_switch_label (switch_stmt, i));
15664 1316 : n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
15665 1316 : if (n && diagnose_sb_0 (gsi_p, context, (gimple *) n->value))
15666 : break;
15667 : }
15668 : }
15669 : break;
15670 :
15671 742 : case GIMPLE_ASM:
15672 742 : {
15673 742 : gasm *asm_stmt = as_a <gasm *> (stmt);
15674 757 : for (unsigned i = 0; i < gimple_asm_nlabels (asm_stmt); ++i)
15675 : {
15676 21 : tree lab = TREE_VALUE (gimple_asm_label_op (asm_stmt, i));
15677 21 : n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
15678 21 : if (n && diagnose_sb_0 (gsi_p, context, (gimple *) n->value))
15679 : break;
15680 : }
15681 : }
15682 : break;
15683 :
15684 34500 : case GIMPLE_RETURN:
15685 34500 : diagnose_sb_0 (gsi_p, context, NULL);
15686 34500 : break;
15687 :
15688 : default:
15689 : break;
15690 : }
15691 :
15692 3347195 : return NULL_TREE;
15693 : }
15694 :
15695 : static unsigned int
15696 56945 : diagnose_omp_structured_block_errors (void)
15697 : {
15698 56945 : struct walk_stmt_info wi;
15699 56945 : gimple_seq body = gimple_body (current_function_decl);
15700 :
15701 56945 : all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0);
15702 :
15703 56945 : memset (&wi, 0, sizeof (wi));
15704 56945 : walk_gimple_seq (body, diagnose_sb_1, NULL, &wi);
15705 :
15706 56945 : memset (&wi, 0, sizeof (wi));
15707 56945 : wi.want_locations = true;
15708 56945 : walk_gimple_seq_mod (&body, diagnose_sb_2, NULL, &wi);
15709 :
15710 56945 : gimple_set_body (current_function_decl, body);
15711 :
15712 56945 : splay_tree_delete (all_labels);
15713 56945 : all_labels = NULL;
15714 :
15715 56945 : return 0;
15716 : }
15717 :
15718 : namespace {
15719 :
15720 : const pass_data pass_data_diagnose_omp_blocks =
15721 : {
15722 : GIMPLE_PASS, /* type */
15723 : "*diagnose_omp_blocks", /* name */
15724 : OPTGROUP_OMP, /* optinfo_flags */
15725 : TV_NONE, /* tv_id */
15726 : PROP_gimple_any, /* properties_required */
15727 : 0, /* properties_provided */
15728 : 0, /* properties_destroyed */
15729 : 0, /* todo_flags_start */
15730 : 0, /* todo_flags_finish */
15731 : };
15732 :
15733 : class pass_diagnose_omp_blocks : public gimple_opt_pass
15734 : {
15735 : public:
15736 298828 : pass_diagnose_omp_blocks (gcc::context *ctxt)
15737 597656 : : gimple_opt_pass (pass_data_diagnose_omp_blocks, ctxt)
15738 : {}
15739 :
15740 : /* opt_pass methods: */
15741 2880212 : bool gate (function *) final override
15742 : {
15743 2880212 : return flag_openacc || flag_openmp || flag_openmp_simd;
15744 : }
15745 56945 : unsigned int execute (function *) final override
15746 : {
15747 56945 : return diagnose_omp_structured_block_errors ();
15748 : }
15749 :
15750 : }; // class pass_diagnose_omp_blocks
15751 :
15752 : } // anon namespace
15753 :
15754 : gimple_opt_pass *
15755 298828 : make_pass_diagnose_omp_blocks (gcc::context *ctxt)
15756 : {
15757 298828 : return new pass_diagnose_omp_blocks (ctxt);
15758 : }
15759 :
15760 :
15761 : #include "gt-omp-low.h"
|
