Line data Source code
1 : /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 : calls to the runtime library (libgomp), data marshalling to implement data
3 : sharing and copying clauses, offloading to accelerators, and more.
4 :
5 : Contributed by Diego Novillo <dnovillo@redhat.com>
6 :
7 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
8 :
9 : This file is part of GCC.
10 :
11 : GCC is free software; you can redistribute it and/or modify it under
12 : the terms of the GNU General Public License as published by the Free
13 : Software Foundation; either version 3, or (at your option) any later
14 : version.
15 :
16 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 : for more details.
20 :
21 : You should have received a copy of the GNU General Public License
22 : along with GCC; see the file COPYING3. If not see
23 : <http://www.gnu.org/licenses/>. */
24 :
25 : #include "config.h"
26 : #include "system.h"
27 : #include "coretypes.h"
28 : #include "backend.h"
29 : #include "target.h"
30 : #include "tree.h"
31 : #include "gimple.h"
32 : #include "tree-pass.h"
33 : #include "ssa.h"
34 : #include "cgraph.h"
35 : #include "pretty-print.h"
36 : #include "diagnostic-core.h"
37 : #include "fold-const.h"
38 : #include "stor-layout.h"
39 : #include "internal-fn.h"
40 : #include "gimple-iterator.h"
41 : #include "gimple-fold.h"
42 : #include "gimplify.h"
43 : #include "gimplify-me.h"
44 : #include "gimple-walk.h"
45 : #include "tree-iterator.h"
46 : #include "tree-inline.h"
47 : #include "langhooks.h"
48 : #include "tree-dfa.h"
49 : #include "tree-ssa.h"
50 : #include "splay-tree.h"
51 : #include "omp-general.h"
52 : #include "omp-low.h"
53 : #include "gimple-low.h"
54 : #include "alloc-pool.h"
55 : #include "symbol-summary.h"
56 : #include "tree-nested.h"
57 : #include "context.h"
58 : #include "gomp-constants.h"
59 : #include "gimple-pretty-print.h"
60 : #include "stringpool.h"
61 : #include "attribs.h"
62 : #include "omp-offload.h"
63 :
64 : /* Lowering of OMP parallel and workshare constructs proceeds in two
65 : phases. The first phase scans the function looking for OMP statements
66 : and then for variables that must be replaced to satisfy data sharing
67 : clauses. The second phase expands code for the constructs, as well as
68 : re-gimplifying things when variables have been replaced with complex
69 : expressions.
70 :
71 : Final code generation is done by pass_expand_omp. The flowgraph is
72 : scanned for regions which are then moved to a new
73 : function, to be invoked by the thread library, or offloaded. */
74 :
75 : /* Context structure. Used to store information about each parallel
76 : directive in the code. */
77 :
78 : struct omp_context
79 : {
80 : /* This field must be at the beginning, as we do "inheritance": Some
81 : callback functions for tree-inline.cc (e.g., omp_copy_decl)
82 : receive a copy_body_data pointer that is up-casted to an
83 : omp_context pointer. */
84 : copy_body_data cb;
85 :
86 : /* The tree of contexts corresponding to the encountered constructs. */
87 : struct omp_context *outer;
88 : gimple *stmt;
89 :
90 : /* Map variables to fields in a structure that allows communication
91 : between sending and receiving threads. */
92 : splay_tree field_map;
93 : tree record_type;
94 : tree sender_decl;
95 : tree receiver_decl;
96 :
97 : /* These are used just by task contexts, if task firstprivate fn is
98 : needed. srecord_type is used to communicate from the thread
99 : that encountered the task construct to task firstprivate fn,
100 : record_type is allocated by GOMP_task, initialized by task firstprivate
101 : fn and passed to the task body fn. */
102 : splay_tree sfield_map;
103 : tree srecord_type;
104 :
105 : /* A chain of variables to add to the top-level block surrounding the
106 : construct. In the case of a parallel, this is in the child function. */
107 : tree block_vars;
108 :
109 : /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
110 : barriers should jump to during omplower pass. */
111 : tree cancel_label;
112 :
113 : /* The sibling GIMPLE_OMP_FOR simd with _simt_ clause or NULL
114 : otherwise. */
115 : gimple *simt_stmt;
116 :
117 : /* For task reductions registered in this context, a vector containing
118 : the length of the private copies block (if constant, otherwise NULL)
119 : and then offsets (if constant, otherwise NULL) for each entry. */
120 : vec<tree> task_reductions;
121 :
122 : /* A hash map from the reduction clauses to the registered array
123 : elts. */
124 : hash_map<tree, unsigned> *task_reduction_map;
125 :
126 : /* And a hash map from the lastprivate(conditional:) variables to their
127 : corresponding tracking loop iteration variables. */
128 : hash_map<tree, tree> *lastprivate_conditional_map;
129 :
130 : /* And a hash map from the allocate variables to their corresponding
131 : allocators. */
132 : hash_map<tree, tree> *allocate_map;
133 :
134 : /* A tree_list of the reduction clauses in this context. This is
135 : only used for checking the consistency of OpenACC reduction
136 : clauses in scan_omp_for and is not guaranteed to contain a valid
137 : value outside of this function. */
138 : tree local_reduction_clauses;
139 :
140 : /* A tree_list of the reduction clauses in outer contexts. This is
141 : only used for checking the consistency of OpenACC reduction
142 : clauses in scan_omp_for and is not guaranteed to contain a valid
143 : value outside of this function. */
144 : tree outer_reduction_clauses;
145 :
146 : /* Nesting depth of this context. Used to beautify error messages re
147 : invalid gotos. The outermost ctx is depth 1, with depth 0 being
148 : reserved for the main body of the function. */
149 : int depth;
150 :
151 : /* True if this parallel directive is nested within another. */
152 : bool is_nested;
153 :
154 : /* True if this construct can be cancelled. */
155 : bool cancellable;
156 :
157 : /* True if lower_omp_1 should look up lastprivate conditional in parent
158 : context. */
159 : bool combined_into_simd_safelen1;
160 :
161 : /* True if there is nested scan context with inclusive clause. */
162 : bool scan_inclusive;
163 :
164 : /* True if there is nested scan context with exclusive clause. */
165 : bool scan_exclusive;
166 :
167 : /* True in the second simd loop of for simd with inscan reductions. */
168 : bool for_simd_scan_phase;
169 :
170 : /* True if there is order(concurrent) clause on the construct. */
171 : bool order_concurrent;
172 :
173 : /* True if there is bind clause on the construct (i.e. a loop construct). */
174 : bool loop_p;
175 :
176 : /* Only used for omp target contexts. True if a teams construct is
177 : strictly nested in it. */
178 : bool teams_nested_p;
179 :
180 : /* Only used for omp target contexts. True if an OpenMP construct other
181 : than teams is strictly nested in it. */
182 : bool nonteams_nested_p;
183 :
184 : /* Candidates for adjusting OpenACC privatization level. */
185 : vec<tree> oacc_privatization_candidates;
186 : };
187 :
188 : static splay_tree all_contexts;
189 : static int taskreg_nesting_level;
190 : static int target_nesting_level;
191 : static bitmap make_addressable_vars;
192 : static bitmap global_nonaddressable_vars;
193 : static vec<omp_context *> taskreg_contexts;
194 : static vec<gomp_task *> task_cpyfns;
195 :
196 : static void scan_omp (gimple_seq *, omp_context *);
197 : static tree scan_omp_1_op (tree *, int *, void *);
198 : static bool omp_maybe_offloaded_ctx (omp_context *ctx);
199 :
200 : #define WALK_SUBSTMTS \
201 : case GIMPLE_BIND: \
202 : case GIMPLE_TRY: \
203 : case GIMPLE_CATCH: \
204 : case GIMPLE_EH_FILTER: \
205 : case GIMPLE_ASSUME: \
206 : case GIMPLE_TRANSACTION: \
207 : /* The sub-statements for these should be walked. */ \
208 : *handled_ops_p = false; \
209 : break;
210 :
211 : /* Return whether CTX represents an OpenACC 'parallel' or 'serial' construct.
212 : (This doesn't include OpenACC 'kernels' decomposed parts.) */
213 :
214 : static bool
215 33860 : is_oacc_parallel_or_serial (omp_context *ctx)
216 : {
217 33860 : enum gimple_code outer_type = gimple_code (ctx->stmt);
218 33860 : return ((outer_type == GIMPLE_OMP_TARGET)
219 33860 : && ((gimple_omp_target_kind (ctx->stmt)
220 : == GF_OMP_TARGET_KIND_OACC_PARALLEL)
221 15518 : || (gimple_omp_target_kind (ctx->stmt)
222 33860 : == GF_OMP_TARGET_KIND_OACC_SERIAL)));
223 : }
224 :
225 : /* Return whether CTX represents an OpenACC 'kernels' construct.
226 : (This doesn't include OpenACC 'kernels' decomposed parts.) */
227 :
228 : static bool
229 49900 : is_oacc_kernels (omp_context *ctx)
230 : {
231 49900 : enum gimple_code outer_type = gimple_code (ctx->stmt);
232 49900 : return ((outer_type == GIMPLE_OMP_TARGET)
233 49900 : && (gimple_omp_target_kind (ctx->stmt)
234 49900 : == GF_OMP_TARGET_KIND_OACC_KERNELS));
235 : }
236 :
237 : /* Return whether CTX represents an OpenACC 'kernels' decomposed part. */
238 :
239 : static bool
240 21645 : is_oacc_kernels_decomposed_part (omp_context *ctx)
241 : {
242 21645 : enum gimple_code outer_type = gimple_code (ctx->stmt);
243 21645 : return ((outer_type == GIMPLE_OMP_TARGET)
244 21645 : && ((gimple_omp_target_kind (ctx->stmt)
245 : == GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED)
246 14552 : || (gimple_omp_target_kind (ctx->stmt)
247 : == GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE)
248 14552 : || (gimple_omp_target_kind (ctx->stmt)
249 21645 : == GF_OMP_TARGET_KIND_OACC_DATA_KERNELS)));
250 : }
251 :
252 : /* Return true if STMT corresponds to an OpenMP target region. */
253 : static bool
254 43357 : is_omp_target (gimple *stmt)
255 : {
256 43357 : if (gimple_code (stmt) == GIMPLE_OMP_TARGET)
257 : {
258 6523 : int kind = gimple_omp_target_kind (stmt);
259 6523 : return (kind == GF_OMP_TARGET_KIND_REGION
260 6523 : || kind == GF_OMP_TARGET_KIND_DATA
261 6523 : || kind == GF_OMP_TARGET_KIND_ENTER_DATA
262 6523 : || kind == GF_OMP_TARGET_KIND_EXIT_DATA);
263 : }
264 : return false;
265 : }
266 :
267 : /* If DECL is the artificial dummy VAR_DECL created for non-static
268 : data member privatization, return the underlying "this" parameter,
269 : otherwise return NULL. */
270 :
271 : tree
272 544380 : omp_member_access_dummy_var (tree decl)
273 : {
274 544380 : if (!VAR_P (decl)
275 334465 : || !DECL_ARTIFICIAL (decl)
276 164217 : || !DECL_IGNORED_P (decl)
277 158138 : || !DECL_HAS_VALUE_EXPR_P (decl)
278 557152 : || !lang_hooks.decls.omp_disregard_value_expr (decl, false))
279 : return NULL_TREE;
280 :
281 2260 : tree v = DECL_VALUE_EXPR (decl);
282 2260 : if (TREE_CODE (v) != COMPONENT_REF)
283 : return NULL_TREE;
284 :
285 16164 : while (1)
286 9197 : switch (TREE_CODE (v))
287 : {
288 6967 : case COMPONENT_REF:
289 6967 : case MEM_REF:
290 6967 : case INDIRECT_REF:
291 6967 : CASE_CONVERT:
292 6967 : case POINTER_PLUS_EXPR:
293 6967 : v = TREE_OPERAND (v, 0);
294 6967 : continue;
295 2211 : case PARM_DECL:
296 2211 : if (DECL_CONTEXT (v) == current_function_decl
297 2211 : && DECL_ARTIFICIAL (v)
298 4422 : && TREE_CODE (TREE_TYPE (v)) == POINTER_TYPE)
299 2211 : return v;
300 : return NULL_TREE;
301 : default:
302 : return NULL_TREE;
303 : }
304 : }
305 :
306 : /* Helper for unshare_and_remap, called through walk_tree. */
307 :
308 : static tree
309 1579 : unshare_and_remap_1 (tree *tp, int *walk_subtrees, void *data)
310 : {
311 1579 : tree *pair = (tree *) data;
312 1579 : if (*tp == pair[0])
313 : {
314 233 : *tp = unshare_expr (pair[1]);
315 233 : *walk_subtrees = 0;
316 : }
317 1346 : else if (IS_TYPE_OR_DECL_P (*tp))
318 244 : *walk_subtrees = 0;
319 1579 : return NULL_TREE;
320 : }
321 :
322 : /* Return unshare_expr (X) with all occurrences of FROM
323 : replaced with TO. */
324 :
325 : static tree
326 150 : unshare_and_remap (tree x, tree from, tree to)
327 : {
328 150 : tree pair[2] = { from, to };
329 150 : x = unshare_expr (x);
330 150 : walk_tree (&x, unshare_and_remap_1, pair, NULL);
331 150 : return x;
332 : }
333 :
334 : /* Convenience function for calling scan_omp_1_op on tree operands. */
335 :
336 : static inline tree
337 340392 : scan_omp_op (tree *tp, omp_context *ctx)
338 : {
339 340392 : struct walk_stmt_info wi;
340 :
341 340392 : memset (&wi, 0, sizeof (wi));
342 340392 : wi.info = ctx;
343 340392 : wi.want_locations = true;
344 :
345 340392 : return walk_tree (tp, scan_omp_1_op, &wi, NULL);
346 : }
347 :
348 : static void lower_omp (gimple_seq *, omp_context *);
349 : static tree lookup_decl_in_outer_ctx (tree, omp_context *);
350 : static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *);
351 :
352 : /* Return true if CTX is for an omp parallel. */
353 :
354 : static inline bool
355 779973 : is_parallel_ctx (omp_context *ctx)
356 : {
357 61645 : return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL;
358 : }
359 :
360 :
361 : /* Return true if CTX is for an omp task. */
362 :
363 : static inline bool
364 238529 : is_task_ctx (omp_context *ctx)
365 : {
366 39390 : return gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
367 : }
368 :
369 :
370 : /* Return true if CTX is for an omp taskloop. */
371 :
372 : static inline bool
373 8193 : is_taskloop_ctx (omp_context *ctx)
374 : {
375 8193 : return gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
376 8193 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_TASKLOOP;
377 : }
378 :
379 :
380 : /* Return true if CTX is for a host omp teams. */
381 :
382 : static inline bool
383 523363 : is_host_teams_ctx (omp_context *ctx)
384 : {
385 523363 : return gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
386 523363 : && gimple_omp_teams_host (as_a <gomp_teams *> (ctx->stmt));
387 : }
388 :
389 : /* Return true if CTX is for an omp parallel or omp task or host omp teams
390 : (the last one is strictly not a task region in OpenMP speak, but we
391 : need to treat it similarly). */
392 :
393 : static inline bool
394 666635 : is_taskreg_ctx (omp_context *ctx)
395 : {
396 666635 : return is_parallel_ctx (ctx) || is_task_ctx (ctx) || is_host_teams_ctx (ctx);
397 : }
398 :
399 : /* Return true if EXPR is variable sized. If IS_REF is true, then
400 : EXPR is assumed to be a reference and the object it refers
401 : to is checked instead. */
402 :
403 : static inline bool
404 910988 : is_variable_sized (const_tree expr, bool is_ref = false)
405 : {
406 910988 : if (is_ref)
407 2146 : expr = TREE_TYPE (expr);
408 910988 : return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr)));
409 : }
410 :
411 : /* Lookup variables. The "maybe" form
412 : allows for the variable form to not have been entered, otherwise we
413 : assert that the variable must have been entered. */
414 :
415 : static inline tree
416 720060 : lookup_decl (tree var, omp_context *ctx)
417 : {
418 1440013 : tree *n = ctx->cb.decl_map->get (var);
419 720060 : return *n;
420 : }
421 :
422 : static inline tree
423 721448 : maybe_lookup_decl (const_tree var, omp_context *ctx)
424 : {
425 721448 : tree *n = ctx->cb.decl_map->get (const_cast<tree> (var));
426 721448 : return n ? *n : NULL_TREE;
427 : }
428 :
429 : static inline tree
430 128080 : lookup_field (tree var, omp_context *ctx)
431 : {
432 128080 : splay_tree_node n;
433 256160 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
434 128080 : return (tree) n->value;
435 : }
436 :
437 : static inline tree
438 170303 : lookup_sfield (splay_tree_key key, omp_context *ctx)
439 : {
440 170303 : splay_tree_node n;
441 170303 : n = splay_tree_lookup (ctx->sfield_map
442 : ? ctx->sfield_map : ctx->field_map, key);
443 170303 : return (tree) n->value;
444 : }
445 :
446 : static inline tree
447 967 : lookup_sfield (tree var, omp_context *ctx)
448 : {
449 967 : return lookup_sfield ((splay_tree_key) var, ctx);
450 : }
451 :
452 : static inline tree
453 222415 : maybe_lookup_field (splay_tree_key key, omp_context *ctx)
454 : {
455 222415 : splay_tree_node n;
456 222415 : n = splay_tree_lookup (ctx->field_map, key);
457 222415 : return n ? (tree) n->value : NULL_TREE;
458 : }
459 :
460 : static inline tree
461 222415 : maybe_lookup_field (tree var, omp_context *ctx)
462 : {
463 2555 : return maybe_lookup_field ((splay_tree_key) var, ctx);
464 : }
465 :
466 : /* Return true if DECL should be copied by pointer. SHARED_CTX is
467 : the parallel context if DECL is to be shared. */
468 :
469 : static bool
470 234069 : use_pointer_for_field (tree decl, omp_context *shared_ctx)
471 : {
472 456378 : if (AGGREGATE_TYPE_P (TREE_TYPE (decl))
473 217051 : || TYPE_ATOMIC (TREE_TYPE (decl))
474 451120 : || POLY_INT_CST_P (DECL_SIZE (decl)))
475 : return true;
476 :
477 : /* We can only use copy-in/copy-out semantics for shared variables
478 : when we know the value is not accessible from an outer scope. */
479 217021 : if (shared_ctx)
480 : {
481 29703 : gcc_assert (!is_gimple_omp_oacc (shared_ctx->stmt));
482 :
483 : /* ??? Trivially accessible from anywhere. But why would we even
484 : be passing an address in this case? Should we simply assert
485 : this to be false, or should we have a cleanup pass that removes
486 : these from the list of mappings? */
487 29703 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, shared_ctx)))
488 : return true;
489 :
490 : /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
491 : without analyzing the expression whether or not its location
492 : is accessible to anyone else. In the case of nested parallel
493 : regions it certainly may be. */
494 29703 : if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl))
495 : return true;
496 :
497 : /* Do not use copy-in/copy-out for variables that have their
498 : address taken. */
499 28789 : if (is_global_var (decl))
500 : {
501 : /* For file scope vars, track whether we've seen them as
502 : non-addressable initially and in that case, keep the same
503 : answer for the duration of the pass, even when they are made
504 : addressable later on e.g. through reduction expansion. Global
505 : variables which weren't addressable before the pass will not
506 : have their privatized copies address taken. See PR91216. */
507 1235 : if (!TREE_ADDRESSABLE (decl))
508 : {
509 480 : if (!global_nonaddressable_vars)
510 75 : global_nonaddressable_vars = BITMAP_ALLOC (NULL);
511 480 : bitmap_set_bit (global_nonaddressable_vars, DECL_UID (decl));
512 : }
513 755 : else if (!global_nonaddressable_vars
514 990 : || !bitmap_bit_p (global_nonaddressable_vars,
515 235 : DECL_UID (decl)))
516 : return true;
517 : }
518 27554 : else if (TREE_ADDRESSABLE (decl))
519 : return true;
520 :
521 : /* lower_send_shared_vars only uses copy-in, but not copy-out
522 : for these. */
523 18568 : if (TREE_READONLY (decl)
524 18568 : || ((TREE_CODE (decl) == RESULT_DECL
525 11909 : || TREE_CODE (decl) == PARM_DECL)
526 673 : && DECL_BY_REFERENCE (decl)))
527 : return false;
528 :
529 : /* Disallow copy-in/out in nested parallel if
530 : decl is shared in outer parallel, otherwise
531 : each thread could store the shared variable
532 : in its own copy-in location, making the
533 : variable no longer really shared. */
534 11677 : if (shared_ctx->is_nested)
535 : {
536 2476 : omp_context *up;
537 :
538 5985 : for (up = shared_ctx->outer; up; up = up->outer)
539 5711 : if ((is_taskreg_ctx (up)
540 3646 : || (gimple_code (up->stmt) == GIMPLE_OMP_TARGET
541 411 : && is_gimple_omp_offloaded (up->stmt)))
542 6122 : && maybe_lookup_decl (decl, up))
543 : break;
544 :
545 2476 : if (up)
546 : {
547 2202 : tree c;
548 :
549 2202 : if (gimple_code (up->stmt) == GIMPLE_OMP_TARGET)
550 : {
551 309 : for (c = gimple_omp_target_clauses (up->stmt);
552 2031 : c; c = OMP_CLAUSE_CHAIN (c))
553 1932 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
554 1932 : && OMP_CLAUSE_DECL (c) == decl)
555 : break;
556 : }
557 : else
558 1893 : for (c = gimple_omp_taskreg_clauses (up->stmt);
559 6925 : c; c = OMP_CLAUSE_CHAIN (c))
560 6468 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
561 6468 : && OMP_CLAUSE_DECL (c) == decl)
562 : break;
563 :
564 2202 : if (c)
565 1646 : goto maybe_mark_addressable_and_ret;
566 : }
567 : }
568 :
569 : /* For tasks avoid using copy-in/out. As tasks can be
570 : deferred or executed in different thread, when GOMP_task
571 : returns, the task hasn't necessarily terminated. */
572 10031 : if (is_task_ctx (shared_ctx))
573 : {
574 2201 : tree outer;
575 555 : maybe_mark_addressable_and_ret:
576 2201 : outer = maybe_lookup_decl_in_outer_ctx (decl, shared_ctx);
577 2201 : if (is_gimple_reg (outer) && !omp_member_access_dummy_var (outer))
578 : {
579 : /* Taking address of OUTER in lower_send_shared_vars
580 : might need regimplification of everything that uses the
581 : variable. */
582 986 : if (!make_addressable_vars)
583 493 : make_addressable_vars = BITMAP_ALLOC (NULL);
584 986 : bitmap_set_bit (make_addressable_vars, DECL_UID (outer));
585 986 : TREE_ADDRESSABLE (outer) = 1;
586 : }
587 : return true;
588 : }
589 : }
590 :
591 : return false;
592 : }
593 :
594 : /* Construct a new automatic decl similar to VAR. */
595 :
596 : static tree
597 231043 : omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx)
598 : {
599 231043 : tree copy = copy_var_decl (var, name, type);
600 :
601 231043 : DECL_CONTEXT (copy) = current_function_decl;
602 :
603 231043 : if (ctx)
604 : {
605 230762 : DECL_CHAIN (copy) = ctx->block_vars;
606 230762 : ctx->block_vars = copy;
607 : }
608 : else
609 281 : record_vars (copy);
610 :
611 : /* If VAR is listed in make_addressable_vars, it wasn't
612 : originally addressable, but was only later made so.
613 : We don't need to take address of privatizations
614 : from that var. */
615 231043 : if (TREE_ADDRESSABLE (var)
616 231043 : && ((make_addressable_vars
617 3470 : && bitmap_bit_p (make_addressable_vars, DECL_UID (var)))
618 40028 : || (global_nonaddressable_vars
619 515 : && bitmap_bit_p (global_nonaddressable_vars, DECL_UID (var)))))
620 2079 : TREE_ADDRESSABLE (copy) = 0;
621 :
622 231043 : return copy;
623 : }
624 :
625 : static tree
626 231043 : omp_copy_decl_1 (tree var, omp_context *ctx)
627 : {
628 231043 : return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx);
629 : }
630 :
631 : /* Build tree nodes to access the field for VAR on the receiver side. */
632 :
633 : static tree
634 123620 : build_receiver_ref (tree var, bool by_ref, omp_context *ctx)
635 : {
636 123620 : tree x, field = lookup_field (var, ctx);
637 :
638 : /* If the receiver record type was remapped in the child function,
639 : remap the field into the new record type. */
640 123620 : x = maybe_lookup_field (field, ctx);
641 123620 : if (x != NULL)
642 11295 : field = x;
643 :
644 123620 : x = build_simple_mem_ref (ctx->receiver_decl);
645 123620 : TREE_THIS_NOTRAP (x) = 1;
646 123620 : x = omp_build_component_ref (x, field);
647 123620 : if (by_ref)
648 : {
649 36338 : x = build_simple_mem_ref (x);
650 36338 : TREE_THIS_NOTRAP (x) = 1;
651 : }
652 :
653 123620 : return x;
654 : }
655 :
656 : /* Build tree nodes to access VAR in the scope outer to CTX. In the case
657 : of a parallel, this is a component reference; for workshare constructs
658 : this is some variable. */
659 :
660 : static tree
661 114907 : build_outer_var_ref (tree var, omp_context *ctx,
662 : enum omp_clause_code code = OMP_CLAUSE_ERROR)
663 : {
664 114907 : tree x;
665 114907 : omp_context *outer = ctx->outer;
666 115296 : for (; outer; outer = outer->outer)
667 : {
668 82750 : if (gimple_code (outer->stmt) == GIMPLE_OMP_TASKGROUP)
669 373 : continue;
670 82393 : if (gimple_code (outer->stmt) == GIMPLE_OMP_SCOPE
671 82377 : && !maybe_lookup_decl (var, outer))
672 16 : continue;
673 : break;
674 : }
675 :
676 114907 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx)))
677 : x = var;
678 106762 : else if (is_variable_sized (var))
679 : {
680 51 : x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0);
681 51 : x = build_outer_var_ref (x, ctx, code);
682 51 : x = build_simple_mem_ref (x);
683 : }
684 106711 : else if (is_taskreg_ctx (ctx))
685 : {
686 64067 : bool by_ref = use_pointer_for_field (var, NULL);
687 64067 : x = build_receiver_ref (var, by_ref, ctx);
688 : }
689 42644 : else if ((gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
690 41271 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
691 14150 : || ctx->loop_p
692 13377 : || code == OMP_CLAUSE_ALLOCATE
693 55765 : || (code == OMP_CLAUSE_PRIVATE
694 40 : && (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
695 : || gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS
696 : || gimple_code (ctx->stmt) == GIMPLE_OMP_SINGLE)))
697 : {
698 : /* #pragma omp simd isn't a worksharing construct, and can reference
699 : even private vars in its linear etc. clauses.
700 : Similarly for OMP_CLAUSE_PRIVATE with outer ref, that can refer
701 : to private vars in all worksharing constructs. */
702 29543 : x = NULL_TREE;
703 29543 : if (outer && is_taskreg_ctx (outer))
704 1002 : x = lookup_decl (var, outer);
705 28541 : else if (outer)
706 22343 : x = maybe_lookup_decl_in_outer_ctx (var, ctx);
707 23345 : if (x == NULL_TREE)
708 : x = var;
709 : }
710 13101 : else if (code == OMP_CLAUSE_LASTPRIVATE && is_taskloop_ctx (ctx))
711 : {
712 689 : gcc_assert (outer);
713 689 : splay_tree_node n
714 1378 : = splay_tree_lookup (outer->field_map,
715 689 : (splay_tree_key) &DECL_UID (var));
716 689 : if (n == NULL)
717 : {
718 610 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, outer)))
719 : x = var;
720 : else
721 384 : x = lookup_decl (var, outer);
722 : }
723 : else
724 : {
725 79 : tree field = (tree) n->value;
726 : /* If the receiver record type was remapped in the child function,
727 : remap the field into the new record type. */
728 79 : x = maybe_lookup_field (field, outer);
729 79 : if (x != NULL)
730 0 : field = x;
731 :
732 79 : x = build_simple_mem_ref (outer->receiver_decl);
733 79 : x = omp_build_component_ref (x, field);
734 79 : if (use_pointer_for_field (var, outer))
735 59 : x = build_simple_mem_ref (x);
736 : }
737 : }
738 12412 : else if (outer)
739 12220 : x = lookup_decl (var, outer);
740 192 : else if (omp_privatize_by_reference (var))
741 : /* This can happen with orphaned constructs. If var is reference, it is
742 : possible it is shared and as such valid. */
743 : x = var;
744 94 : else if (omp_member_access_dummy_var (var))
745 : x = var;
746 : else
747 0 : gcc_unreachable ();
748 :
749 114907 : if (x == var)
750 : {
751 16338 : tree t = omp_member_access_dummy_var (var);
752 16338 : if (t)
753 : {
754 220 : x = DECL_VALUE_EXPR (var);
755 220 : tree o = maybe_lookup_decl_in_outer_ctx (t, ctx);
756 220 : if (o != t)
757 54 : x = unshare_and_remap (x, t, o);
758 : else
759 166 : x = unshare_expr (x);
760 : }
761 : }
762 :
763 114907 : if (omp_privatize_by_reference (var))
764 3146 : x = build_simple_mem_ref (x);
765 :
766 114907 : return x;
767 : }
768 :
769 : /* Build tree nodes to access the field for VAR on the sender side. */
770 :
771 : static tree
772 169257 : build_sender_ref (splay_tree_key key, omp_context *ctx)
773 : {
774 169257 : tree field = lookup_sfield (key, ctx);
775 169257 : tree tmp = ctx->sender_decl;
776 169257 : if (POINTER_TYPE_P (TREE_TYPE (tmp)))
777 451 : tmp = build_fold_indirect_ref (tmp);
778 169257 : return omp_build_component_ref (tmp, field);
779 : }
780 :
781 : static tree
782 165103 : build_sender_ref (tree var, omp_context *ctx)
783 : {
784 0 : return build_sender_ref ((splay_tree_key) var, ctx);
785 : }
786 :
787 : /* Add a new field for VAR inside the structure CTX. If BY_REF is true,
788 : use a pointer to the VAR rather than VAR itself.
789 : MASK is a bit mask of other options. Bits are interpreted as:
790 : 1: Install VAR in ctx->field_map.
791 : 2: Install VAR in ctx->sfield_map.
792 : 4: VAR is an array, convert it to a pointer.
793 : 8: Use DECL_UID (VAR) instead of VAR as key.
794 : 16: Use DECL_NAME (VAR) instead of VAR as key.
795 : 32: Don't dereference omp_is_reference types.
796 : KEY_EXPR allows specifying something other than VAR as the lookup key.
797 : If specified, it also overrides the 8 and 16 MASK bits. */
798 :
799 : static void
800 132132 : install_var_field (tree var, bool by_ref, int mask, omp_context *ctx,
801 : tree key_expr = NULL_TREE)
802 : {
803 132132 : tree field, type, sfield = NULL_TREE;
804 132132 : splay_tree_key key = (splay_tree_key) var;
805 :
806 132132 : if (key_expr)
807 : /* Allow caller to explicitly set the expression used as the key. */
808 44824 : key = (splay_tree_key) key_expr;
809 : else
810 : {
811 87308 : if ((mask & 16) != 0)
812 : {
813 624 : key = (splay_tree_key) &DECL_NAME (var);
814 624 : gcc_checking_assert (key != (splay_tree_key) var);
815 : }
816 87308 : if ((mask & 8) != 0)
817 : {
818 1619 : key = (splay_tree_key) &DECL_UID (var);
819 1619 : gcc_checking_assert (key != (splay_tree_key) var);
820 : }
821 : }
822 132132 : gcc_assert ((mask & 1) == 0
823 : || !splay_tree_lookup (ctx->field_map, key));
824 132132 : gcc_assert ((mask & 2) == 0 || !ctx->sfield_map
825 : || !splay_tree_lookup (ctx->sfield_map, key));
826 132132 : gcc_assert ((mask & 3) == 3
827 : || !is_gimple_omp_oacc (ctx->stmt));
828 :
829 132132 : type = TREE_TYPE (var);
830 132132 : if ((mask & 16) != 0)
831 624 : type = lang_hooks.decls.omp_array_data (var, true);
832 :
833 : /* Prevent redeclaring the var in the split-off function with a restrict
834 : pointer type. Note that we only clear type itself, restrict qualifiers in
835 : the pointed-to type will be ignored by points-to analysis. */
836 132132 : if (POINTER_TYPE_P (type)
837 132132 : && TYPE_RESTRICT (type))
838 7022 : type = build_qualified_type (type,
839 7022 : TYPE_QUALS (type)
840 : .without (TYPE_QUAL_RESTRICT));
841 :
842 132132 : if (mask & 4)
843 : {
844 425 : gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
845 425 : type = build_pointer_type (build_pointer_type (type));
846 : }
847 131707 : else if (by_ref)
848 64492 : type = build_pointer_type (type);
849 67215 : else if ((mask & (32 | 3)) == 1
850 67215 : && omp_privatize_by_reference (var))
851 162 : type = TREE_TYPE (type);
852 :
853 132132 : field = build_decl (DECL_SOURCE_LOCATION (var),
854 132132 : FIELD_DECL, DECL_NAME (var), type);
855 :
856 : /* Remember what variable this field was created for. This does have a
857 : side effect of making dwarf2out ignore this member, so for helpful
858 : debugging we clear it later in delete_omp_context. */
859 132132 : DECL_ABSTRACT_ORIGIN (field) = var;
860 132132 : if ((mask & 16) == 0 && type == TREE_TYPE (var))
861 : {
862 62701 : SET_DECL_ALIGN (field, DECL_ALIGN (var));
863 62701 : DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
864 62701 : TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
865 : }
866 : else
867 69431 : SET_DECL_ALIGN (field, TYPE_ALIGN (type));
868 :
869 132132 : if ((mask & 3) == 3)
870 : {
871 131068 : insert_field_into_struct (ctx->record_type, field);
872 131068 : if (ctx->srecord_type)
873 : {
874 682 : sfield = build_decl (DECL_SOURCE_LOCATION (var),
875 682 : FIELD_DECL, DECL_NAME (var), type);
876 682 : DECL_ABSTRACT_ORIGIN (sfield) = var;
877 682 : SET_DECL_ALIGN (sfield, DECL_ALIGN (field));
878 682 : DECL_USER_ALIGN (sfield) = DECL_USER_ALIGN (field);
879 682 : TREE_THIS_VOLATILE (sfield) = TREE_THIS_VOLATILE (field);
880 682 : insert_field_into_struct (ctx->srecord_type, sfield);
881 : }
882 : }
883 : else
884 : {
885 1064 : if (ctx->srecord_type == NULL_TREE)
886 : {
887 550 : tree t;
888 :
889 550 : ctx->srecord_type = lang_hooks.types.make_type (RECORD_TYPE);
890 550 : ctx->sfield_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
891 2033 : for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t))
892 : {
893 1483 : sfield = build_decl (DECL_SOURCE_LOCATION (t),
894 1483 : FIELD_DECL, DECL_NAME (t), TREE_TYPE (t));
895 1483 : DECL_ABSTRACT_ORIGIN (sfield) = DECL_ABSTRACT_ORIGIN (t);
896 1483 : insert_field_into_struct (ctx->srecord_type, sfield);
897 2966 : splay_tree_insert (ctx->sfield_map,
898 1483 : (splay_tree_key) DECL_ABSTRACT_ORIGIN (t),
899 : (splay_tree_value) sfield);
900 : }
901 : }
902 1064 : sfield = field;
903 1064 : insert_field_into_struct ((mask & 1) ? ctx->record_type
904 : : ctx->srecord_type, field);
905 : }
906 :
907 132132 : if (mask & 1)
908 131739 : splay_tree_insert (ctx->field_map, key, (splay_tree_value) field);
909 132132 : if ((mask & 2) && ctx->sfield_map)
910 1075 : splay_tree_insert (ctx->sfield_map, key, (splay_tree_value) sfield);
911 132132 : }
912 :
913 : static tree
914 230612 : install_var_local (tree var, omp_context *ctx)
915 : {
916 230612 : tree new_var = omp_copy_decl_1 (var, ctx);
917 230612 : insert_decl_map (&ctx->cb, var, new_var);
918 230612 : return new_var;
919 : }
920 :
921 : /* Adjust the replacement for DECL in CTX for the new context. This means
922 : copying the DECL_VALUE_EXPR, and fixing up the type. */
923 :
924 : static void
925 205587 : fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug)
926 : {
927 205587 : tree new_decl, size;
928 :
929 205587 : new_decl = lookup_decl (decl, ctx);
930 :
931 205587 : TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), &ctx->cb);
932 :
933 410030 : if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug)
934 205776 : && DECL_HAS_VALUE_EXPR_P (decl))
935 : {
936 1333 : tree ve = DECL_VALUE_EXPR (decl);
937 1333 : walk_tree (&ve, copy_tree_body_r, &ctx->cb, NULL);
938 1333 : SET_DECL_VALUE_EXPR (new_decl, ve);
939 1333 : DECL_HAS_VALUE_EXPR_P (new_decl) = 1;
940 : }
941 :
942 205587 : if (!TREE_CONSTANT (DECL_SIZE (new_decl)))
943 : {
944 1144 : size = remap_decl (DECL_SIZE (decl), &ctx->cb);
945 1144 : if (size == error_mark_node)
946 321 : size = TYPE_SIZE (TREE_TYPE (new_decl));
947 1144 : DECL_SIZE (new_decl) = size;
948 :
949 1144 : size = remap_decl (DECL_SIZE_UNIT (decl), &ctx->cb);
950 1144 : if (size == error_mark_node)
951 308 : size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl));
952 1144 : DECL_SIZE_UNIT (new_decl) = size;
953 : }
954 205587 : }
955 :
956 : /* The callback for remap_decl. Search all containing contexts for a
957 : mapping of the variable; this avoids having to duplicate the splay
958 : tree ahead of time. We know a mapping doesn't already exist in the
959 : given context. Create new mappings to implement default semantics. */
960 :
961 : static tree
962 466309 : omp_copy_decl (tree var, copy_body_data *cb)
963 : {
964 466309 : omp_context *ctx = (omp_context *) cb;
965 466309 : tree new_var;
966 :
967 466309 : if (TREE_CODE (var) == LABEL_DECL)
968 : {
969 189902 : if (FORCED_LABEL (var) || DECL_NONLOCAL (var))
970 : return var;
971 189896 : new_var = create_artificial_label (DECL_SOURCE_LOCATION (var));
972 189896 : DECL_CONTEXT (new_var) = current_function_decl;
973 189896 : insert_decl_map (&ctx->cb, var, new_var);
974 189896 : return new_var;
975 : }
976 :
977 358347 : while (!is_taskreg_ctx (ctx))
978 : {
979 339023 : ctx = ctx->outer;
980 339023 : if (ctx == NULL)
981 : return var;
982 280248 : new_var = maybe_lookup_decl (var, ctx);
983 280248 : if (new_var)
984 : return new_var;
985 : }
986 :
987 19324 : if (is_global_var (var) || decl_function_context (var) != ctx->cb.src_fn)
988 : return var;
989 :
990 5876 : return error_mark_node;
991 : }
992 :
993 : /* Create a new context, with OUTER_CTX being the surrounding context. */
994 :
995 : static omp_context *
996 138952 : new_omp_context (gimple *stmt, omp_context *outer_ctx)
997 : {
998 138952 : omp_context *ctx = XCNEW (omp_context);
999 :
1000 138952 : splay_tree_insert (all_contexts, (splay_tree_key) stmt,
1001 : (splay_tree_value) ctx);
1002 138952 : ctx->stmt = stmt;
1003 :
1004 138952 : if (outer_ctx)
1005 : {
1006 74905 : ctx->outer = outer_ctx;
1007 74905 : ctx->cb = outer_ctx->cb;
1008 74905 : ctx->cb.block = NULL;
1009 74905 : ctx->depth = outer_ctx->depth + 1;
1010 : }
1011 : else
1012 : {
1013 64047 : ctx->cb.src_fn = current_function_decl;
1014 64047 : ctx->cb.dst_fn = current_function_decl;
1015 64047 : ctx->cb.src_node = cgraph_node::get (current_function_decl);
1016 64047 : gcc_checking_assert (ctx->cb.src_node);
1017 64047 : ctx->cb.dst_node = ctx->cb.src_node;
1018 64047 : ctx->cb.src_cfun = cfun;
1019 64047 : ctx->cb.copy_decl = omp_copy_decl;
1020 64047 : ctx->cb.eh_lp_nr = 0;
1021 64047 : ctx->cb.transform_call_graph_edges = CB_CGE_MOVE;
1022 64047 : ctx->cb.adjust_array_error_bounds = true;
1023 64047 : ctx->cb.dont_remap_vla_if_no_change = true;
1024 64047 : ctx->depth = 1;
1025 : }
1026 :
1027 138952 : ctx->cb.decl_map = new hash_map<tree, tree>;
1028 :
1029 138952 : return ctx;
1030 : }
1031 :
1032 : static gimple_seq maybe_catch_exception (gimple_seq);
1033 :
1034 : /* Finalize task copyfn. */
1035 :
1036 : static void
1037 513 : finalize_task_copyfn (gomp_task *task_stmt)
1038 : {
1039 513 : struct function *child_cfun;
1040 513 : tree child_fn;
1041 513 : gimple_seq seq = NULL, new_seq;
1042 513 : gbind *bind;
1043 :
1044 513 : child_fn = gimple_omp_task_copy_fn (task_stmt);
1045 513 : if (child_fn == NULL_TREE)
1046 0 : return;
1047 :
1048 513 : child_cfun = DECL_STRUCT_FUNCTION (child_fn);
1049 513 : DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
1050 :
1051 513 : push_cfun (child_cfun);
1052 513 : bind = gimplify_body (child_fn, false);
1053 513 : gimple_seq_add_stmt (&seq, bind);
1054 513 : new_seq = maybe_catch_exception (seq);
1055 513 : if (new_seq != seq)
1056 : {
1057 341 : bind = gimple_build_bind (NULL, new_seq, NULL);
1058 341 : seq = NULL;
1059 341 : gimple_seq_add_stmt (&seq, bind);
1060 : }
1061 513 : gimple_set_body (child_fn, seq);
1062 513 : pop_cfun ();
1063 :
1064 : /* Inform the callgraph about the new function. */
1065 513 : cgraph_node *node = cgraph_node::get_create (child_fn);
1066 513 : node->parallelized_function = 1;
1067 513 : cgraph_node::add_new_function (child_fn, false);
1068 : }
1069 :
1070 : /* Destroy a omp_context data structures. Called through the splay tree
1071 : value delete callback. */
1072 :
1073 : static void
1074 138946 : delete_omp_context (splay_tree_value value)
1075 : {
1076 138946 : omp_context *ctx = (omp_context *) value;
1077 :
1078 277892 : delete ctx->cb.decl_map;
1079 :
1080 138946 : if (ctx->field_map)
1081 69934 : splay_tree_delete (ctx->field_map);
1082 138946 : if (ctx->sfield_map)
1083 550 : splay_tree_delete (ctx->sfield_map);
1084 :
1085 : /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1086 : it produces corrupt debug information. */
1087 138946 : if (ctx->record_type)
1088 : {
1089 53880 : tree t;
1090 223958 : for (t = TYPE_FIELDS (ctx->record_type); t ; t = DECL_CHAIN (t))
1091 170078 : DECL_ABSTRACT_ORIGIN (t) = NULL;
1092 : }
1093 138946 : if (ctx->srecord_type)
1094 : {
1095 550 : tree t;
1096 3108 : for (t = TYPE_FIELDS (ctx->srecord_type); t ; t = DECL_CHAIN (t))
1097 2558 : DECL_ABSTRACT_ORIGIN (t) = NULL;
1098 : }
1099 :
1100 138946 : if (ctx->task_reduction_map)
1101 : {
1102 1176 : ctx->task_reductions.release ();
1103 2352 : delete ctx->task_reduction_map;
1104 : }
1105 :
1106 139219 : delete ctx->lastprivate_conditional_map;
1107 140454 : delete ctx->allocate_map;
1108 :
1109 138946 : XDELETE (ctx);
1110 138946 : }
1111 :
1112 : /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1113 : context. */
1114 :
1115 : static void
1116 37176 : fixup_child_record_type (omp_context *ctx)
1117 : {
1118 37176 : tree f, type = ctx->record_type;
1119 :
1120 37176 : if (!ctx->receiver_decl)
1121 : return;
1122 : /* ??? It isn't sufficient to just call remap_type here, because
1123 : variably_modified_type_p doesn't work the way we expect for
1124 : record types. Testing each field for whether it needs remapping
1125 : and creating a new record by hand works, however. */
1126 162843 : for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
1127 126958 : if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
1128 : break;
1129 37176 : if (f)
1130 : {
1131 1291 : tree name, new_fields = NULL;
1132 :
1133 1291 : type = lang_hooks.types.make_type (RECORD_TYPE);
1134 1291 : name = DECL_NAME (TYPE_NAME (ctx->record_type));
1135 1291 : name = build_decl (DECL_SOURCE_LOCATION (ctx->receiver_decl),
1136 : TYPE_DECL, name, type);
1137 1291 : TYPE_NAME (type) = name;
1138 :
1139 13576 : for (f = TYPE_FIELDS (ctx->record_type); f ; f = DECL_CHAIN (f))
1140 : {
1141 12285 : tree new_f = copy_node (f);
1142 12285 : DECL_CONTEXT (new_f) = type;
1143 12285 : TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &ctx->cb);
1144 12285 : DECL_CHAIN (new_f) = new_fields;
1145 12285 : walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &ctx->cb, NULL);
1146 12285 : walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r,
1147 : &ctx->cb, NULL);
1148 12285 : walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
1149 : &ctx->cb, NULL);
1150 12285 : new_fields = new_f;
1151 :
1152 : /* Arrange to be able to look up the receiver field
1153 : given the sender field. */
1154 12285 : splay_tree_insert (ctx->field_map, (splay_tree_key) f,
1155 : (splay_tree_value) new_f);
1156 : }
1157 1291 : TYPE_FIELDS (type) = nreverse (new_fields);
1158 1291 : layout_type (type);
1159 : }
1160 :
1161 : /* In a target region we never modify any of the pointers in *.omp_data_i,
1162 : so attempt to help the optimizers. */
1163 37176 : if (is_gimple_omp_offloaded (ctx->stmt))
1164 17720 : type = build_qualified_type (type, TYPE_QUAL_CONST);
1165 :
1166 37176 : TREE_TYPE (ctx->receiver_decl)
1167 111528 : = build_qualified_type (flexible_array_type_p (type)
1168 107 : ? build_pointer_type (type)
1169 37069 : : build_reference_type (type), TYPE_QUAL_RESTRICT);
1170 : }
1171 :
1172 : /* Instantiate decls as necessary in CTX to satisfy the data sharing
1173 : specified by CLAUSES. */
1174 :
1175 : static void
1176 131041 : scan_sharing_clauses (tree clauses, omp_context *ctx)
1177 : {
1178 131041 : tree c, decl;
1179 131041 : bool scan_array_reductions = false;
1180 131041 : bool flex_array_ptr = false;
1181 :
1182 553751 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1183 422710 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_ALLOCATE
1184 422710 : && (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) == NULL_TREE
1185 : /* omp_default_mem_alloc is 1 */
1186 1297 : || !integer_onep (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c))
1187 572 : || OMP_CLAUSE_ALLOCATE_ALIGN (c) != NULL_TREE))
1188 : {
1189 : /* The allocate clauses that appear on a target construct or on
1190 : constructs in a target region must specify an allocator expression
1191 : unless a requires directive with the dynamic_allocators clause
1192 : is present in the same compilation unit. */
1193 1985 : if (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) == NULL_TREE
1194 1216 : && ((omp_requires_mask & OMP_REQUIRES_DYNAMIC_ALLOCATORS) == 0)
1195 3189 : && omp_maybe_offloaded_ctx (ctx))
1196 14 : error_at (OMP_CLAUSE_LOCATION (c), "%<allocate%> clause must"
1197 : " specify an allocator here");
1198 1985 : if ((omp_requires_mask & OMP_REQUIRES_DYNAMIC_ALLOCATORS) == 0
1199 1973 : && OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) != NULL_TREE
1200 769 : && DECL_P (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c))
1201 2494 : && !DECL_ARTIFICIAL (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c)))
1202 : {
1203 279 : tree alloc2 = OMP_CLAUSE_ALLOCATE_ALLOCATOR (c);
1204 279 : if (TREE_CODE (alloc2) == MEM_REF
1205 279 : || TREE_CODE (alloc2) == INDIRECT_REF)
1206 0 : alloc2 = TREE_OPERAND (alloc2, 0);
1207 279 : omp_context *ctx2 = ctx;
1208 740 : for (; ctx2; ctx2 = ctx2->outer)
1209 480 : if (is_gimple_omp_offloaded (ctx2->stmt))
1210 : break;
1211 279 : if (ctx2 != NULL)
1212 : {
1213 19 : tree c2 = gimple_omp_target_clauses (ctx2->stmt);
1214 108 : for (; c2; c2 = OMP_CLAUSE_CHAIN (c2))
1215 106 : if (OMP_CLAUSE_CODE (c2) == OMP_CLAUSE_USES_ALLOCATORS
1216 123 : && operand_equal_p (
1217 17 : alloc2, OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c2)))
1218 : break;
1219 19 : if (c2 == NULL_TREE)
1220 4 : error_at (EXPR_LOC_OR_LOC (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c),
1221 : OMP_CLAUSE_LOCATION (c)),
1222 : "allocator %qE in %<allocate%> clause inside a "
1223 : "target region must be specified in an "
1224 : "%<uses_allocators%> clause on the %<target%> "
1225 : "directive", alloc2);
1226 : }
1227 : }
1228 1985 : if (ctx->allocate_map == NULL)
1229 1508 : ctx->allocate_map = new hash_map<tree, tree>;
1230 1985 : tree val = integer_zero_node;
1231 1985 : if (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c))
1232 769 : val = OMP_CLAUSE_ALLOCATE_ALLOCATOR (c);
1233 1985 : if (OMP_CLAUSE_ALLOCATE_ALIGN (c))
1234 295 : val = build_tree_list (val, OMP_CLAUSE_ALLOCATE_ALIGN (c));
1235 1985 : ctx->allocate_map->put (OMP_CLAUSE_DECL (c), val);
1236 : }
1237 :
1238 553751 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1239 : {
1240 422710 : bool by_ref;
1241 :
1242 422710 : switch (OMP_CLAUSE_CODE (c))
1243 : {
1244 73503 : case OMP_CLAUSE_PRIVATE:
1245 73503 : decl = OMP_CLAUSE_DECL (c);
1246 73503 : if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
1247 165 : goto do_private;
1248 73338 : else if (!is_variable_sized (decl))
1249 72974 : install_var_local (decl, ctx);
1250 : break;
1251 :
1252 46994 : case OMP_CLAUSE_SHARED:
1253 46994 : decl = OMP_CLAUSE_DECL (c);
1254 46994 : if (ctx->allocate_map && ctx->allocate_map->get (decl))
1255 37 : ctx->allocate_map->remove (decl);
1256 : /* Ignore shared directives in teams construct inside of
1257 : target construct. */
1258 46994 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
1259 46994 : && !is_host_teams_ctx (ctx))
1260 : {
1261 : /* Global variables don't need to be copied,
1262 : the receiver side will use them directly. */
1263 11864 : tree odecl = maybe_lookup_decl_in_outer_ctx (decl, ctx);
1264 11864 : if (is_global_var (odecl))
1265 : break;
1266 8135 : insert_decl_map (&ctx->cb, decl, odecl);
1267 8135 : break;
1268 : }
1269 35130 : gcc_assert (is_taskreg_ctx (ctx));
1270 35130 : gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl))
1271 : || !is_variable_sized (decl));
1272 : /* Global variables don't need to be copied,
1273 : the receiver side will use them directly. */
1274 35130 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1275 : break;
1276 29052 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
1277 : {
1278 86 : use_pointer_for_field (decl, ctx);
1279 86 : break;
1280 : }
1281 28966 : by_ref = use_pointer_for_field (decl, NULL);
1282 55433 : if ((! TREE_READONLY (decl) && !OMP_CLAUSE_SHARED_READONLY (c))
1283 19367 : || TREE_ADDRESSABLE (decl)
1284 19010 : || by_ref
1285 47976 : || omp_privatize_by_reference (decl))
1286 : {
1287 12101 : by_ref = use_pointer_for_field (decl, ctx);
1288 12101 : install_var_field (decl, by_ref, 3, ctx);
1289 12101 : install_var_local (decl, ctx);
1290 12101 : break;
1291 : }
1292 : /* We don't need to copy const scalar vars back. */
1293 16865 : OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE);
1294 16865 : goto do_private;
1295 :
1296 14692 : case OMP_CLAUSE_REDUCTION:
1297 : /* Collect 'reduction' clauses on OpenACC compute construct. */
1298 14692 : if (is_gimple_omp_oacc (ctx->stmt)
1299 14692 : && is_gimple_omp_offloaded (ctx->stmt))
1300 : {
1301 : /* No 'reduction' clauses on OpenACC 'kernels'. */
1302 790 : gcc_checking_assert (!is_oacc_kernels (ctx));
1303 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
1304 790 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
1305 :
1306 790 : ctx->local_reduction_clauses
1307 790 : = tree_cons (NULL, c, ctx->local_reduction_clauses);
1308 : }
1309 : /* FALLTHRU */
1310 :
1311 16791 : case OMP_CLAUSE_IN_REDUCTION:
1312 16791 : decl = OMP_CLAUSE_DECL (c);
1313 16791 : if (ctx->allocate_map
1314 16791 : && ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1315 473 : && (OMP_CLAUSE_REDUCTION_INSCAN (c)
1316 473 : || OMP_CLAUSE_REDUCTION_TASK (c)))
1317 481 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
1318 473 : || is_task_ctx (ctx)))
1319 : {
1320 : /* For now. */
1321 17 : if (ctx->allocate_map->get (decl))
1322 17 : ctx->allocate_map->remove (decl);
1323 : }
1324 16791 : if (TREE_CODE (decl) == MEM_REF)
1325 : {
1326 2441 : tree t = TREE_OPERAND (decl, 0);
1327 2441 : if (TREE_CODE (t) == POINTER_PLUS_EXPR)
1328 436 : t = TREE_OPERAND (t, 0);
1329 2441 : if (INDIRECT_REF_P (t)
1330 2329 : || TREE_CODE (t) == ADDR_EXPR)
1331 1414 : t = TREE_OPERAND (t, 0);
1332 2441 : if (is_omp_target (ctx->stmt))
1333 : {
1334 96 : if (is_variable_sized (t))
1335 : {
1336 32 : gcc_assert (DECL_HAS_VALUE_EXPR_P (t));
1337 32 : t = DECL_VALUE_EXPR (t);
1338 32 : gcc_assert (INDIRECT_REF_P (t));
1339 32 : t = TREE_OPERAND (t, 0);
1340 32 : gcc_assert (DECL_P (t));
1341 : }
1342 96 : tree at = t;
1343 96 : if (ctx->outer)
1344 96 : scan_omp_op (&at, ctx->outer);
1345 96 : tree nt = omp_copy_decl_1 (at, ctx->outer);
1346 192 : splay_tree_insert (ctx->field_map,
1347 96 : (splay_tree_key) &DECL_CONTEXT (t),
1348 : (splay_tree_value) nt);
1349 96 : if (at != t)
1350 96 : splay_tree_insert (ctx->field_map,
1351 48 : (splay_tree_key) &DECL_CONTEXT (at),
1352 : (splay_tree_value) nt);
1353 96 : break;
1354 : }
1355 2345 : install_var_local (t, ctx);
1356 2345 : if (is_taskreg_ctx (ctx)
1357 1897 : && (!is_global_var (maybe_lookup_decl_in_outer_ctx (t, ctx))
1358 387 : || (is_task_ctx (ctx)
1359 310 : && (TREE_CODE (TREE_TYPE (t)) == POINTER_TYPE
1360 234 : || (TREE_CODE (TREE_TYPE (t)) == REFERENCE_TYPE
1361 0 : && (TREE_CODE (TREE_TYPE (TREE_TYPE (t)))
1362 : == POINTER_TYPE)))))
1363 1586 : && !is_variable_sized (t)
1364 3872 : && (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
1365 692 : || (!OMP_CLAUSE_REDUCTION_TASK (c)
1366 653 : && !is_task_ctx (ctx))))
1367 : {
1368 1410 : by_ref = use_pointer_for_field (t, NULL);
1369 1410 : if (is_task_ctx (ctx)
1370 835 : && TREE_CODE (TREE_TYPE (t)) == REFERENCE_TYPE
1371 1570 : && TREE_CODE (TREE_TYPE (TREE_TYPE (t))) == POINTER_TYPE)
1372 : {
1373 32 : install_var_field (t, false, 1, ctx);
1374 32 : install_var_field (t, by_ref, 2, ctx);
1375 : }
1376 : else
1377 1378 : install_var_field (t, by_ref, 3, ctx);
1378 : }
1379 : break;
1380 : }
1381 14350 : if (is_omp_target (ctx->stmt))
1382 : {
1383 335 : tree at = decl;
1384 335 : if (ctx->outer)
1385 54 : scan_omp_op (&at, ctx->outer);
1386 335 : tree nt = omp_copy_decl_1 (at, ctx->outer);
1387 670 : splay_tree_insert (ctx->field_map,
1388 335 : (splay_tree_key) &DECL_CONTEXT (decl),
1389 : (splay_tree_value) nt);
1390 335 : if (at != decl)
1391 24 : splay_tree_insert (ctx->field_map,
1392 12 : (splay_tree_key) &DECL_CONTEXT (at),
1393 : (splay_tree_value) nt);
1394 335 : break;
1395 : }
1396 14015 : if (is_task_ctx (ctx)
1397 26935 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1398 12920 : && OMP_CLAUSE_REDUCTION_TASK (c)
1399 390 : && is_parallel_ctx (ctx)))
1400 : {
1401 : /* Global variables don't need to be copied,
1402 : the receiver side will use them directly. */
1403 1183 : if (!is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1404 : {
1405 487 : by_ref = use_pointer_for_field (decl, ctx);
1406 487 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION)
1407 325 : install_var_field (decl, by_ref, 3, ctx);
1408 : }
1409 1183 : install_var_local (decl, ctx);
1410 1183 : break;
1411 : }
1412 12832 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1413 12832 : && OMP_CLAUSE_REDUCTION_TASK (c))
1414 : {
1415 302 : install_var_local (decl, ctx);
1416 302 : break;
1417 : }
1418 12530 : goto do_private;
1419 :
1420 22774 : case OMP_CLAUSE_LASTPRIVATE:
1421 : /* Let the corresponding firstprivate clause create
1422 : the variable. */
1423 22774 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1424 : break;
1425 : /* FALLTHRU */
1426 :
1427 59809 : case OMP_CLAUSE_FIRSTPRIVATE:
1428 59809 : case OMP_CLAUSE_LINEAR:
1429 59809 : decl = OMP_CLAUSE_DECL (c);
1430 89758 : do_private:
1431 89758 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1432 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR
1433 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1434 46133 : && is_gimple_omp_offloaded (ctx->stmt))
1435 : {
1436 16458 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1437 16458 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR
1438 240 : && lang_hooks.decls.omp_array_data (decl, true)))
1439 : {
1440 : /* OpenACC firstprivate clauses are later processed with same
1441 : code path as map clauses in lower_omp_target, so follow
1442 : the same convention of using the whole clause expression
1443 : as splay-tree key. */
1444 16088 : tree k = (is_oacc_parallel_or_serial (ctx) ? c : NULL_TREE);
1445 16088 : by_ref = !omp_privatize_by_reference (decl);
1446 16088 : install_var_field (decl, by_ref, 3, ctx, k);
1447 : }
1448 370 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1449 : {
1450 221 : if (INDIRECT_REF_P (decl))
1451 0 : decl = TREE_OPERAND (decl, 0);
1452 221 : install_var_field (decl, true, 3, ctx);
1453 : }
1454 149 : else if (TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1455 8 : install_var_field (decl, true, 3, ctx);
1456 : else
1457 141 : install_var_field (decl, false, 3, ctx);
1458 : }
1459 89758 : if (is_variable_sized (decl))
1460 : {
1461 68 : if (is_task_ctx (ctx))
1462 : {
1463 14 : if (ctx->allocate_map
1464 14 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
1465 : {
1466 : /* For now. */
1467 0 : if (ctx->allocate_map->get (decl))
1468 0 : ctx->allocate_map->remove (decl);
1469 : }
1470 14 : install_var_field (decl, false, 1, ctx);
1471 : }
1472 : break;
1473 : }
1474 89690 : else if (is_taskreg_ctx (ctx))
1475 : {
1476 38545 : bool global
1477 38545 : = is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx));
1478 38545 : by_ref = use_pointer_for_field (decl, NULL);
1479 :
1480 38545 : if (is_task_ctx (ctx)
1481 38545 : && (global || by_ref || omp_privatize_by_reference (decl)))
1482 : {
1483 625 : if (ctx->allocate_map
1484 625 : && ctx->allocate_map->get (decl))
1485 59 : install_var_field (decl, by_ref, 32 | 1, ctx);
1486 : else
1487 566 : install_var_field (decl, false, 1, ctx);
1488 625 : if (!global)
1489 361 : install_var_field (decl, by_ref, 2, ctx);
1490 : }
1491 37920 : else if (!global)
1492 36647 : install_var_field (decl, by_ref, 3, ctx);
1493 : }
1494 89690 : install_var_local (decl, ctx);
1495 : /* For descr arrays on target: firstprivatize data + attach ptr. */
1496 89690 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1497 45689 : && is_gimple_omp_offloaded (ctx->stmt)
1498 16059 : && !is_gimple_omp_oacc (ctx->stmt)
1499 11811 : && lang_hooks.decls.omp_array_data (decl, true)
1500 89721 : && lang_hooks.decls.omp_array_data_privatize (decl))
1501 : {
1502 23 : install_var_field (decl, false, 16 | 3, ctx);
1503 23 : install_var_field (decl, true, 8 | 3, ctx);
1504 : }
1505 : break;
1506 :
1507 2118 : case OMP_CLAUSE_USE_DEVICE_PTR:
1508 2118 : case OMP_CLAUSE_USE_DEVICE_ADDR:
1509 2118 : decl = OMP_CLAUSE_DECL (c);
1510 :
1511 : /* Fortran array descriptors. */
1512 2118 : if (lang_hooks.decls.omp_array_data (decl, true))
1513 601 : install_var_field (decl, false, 19, ctx);
1514 1517 : else if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
1515 1312 : && !omp_privatize_by_reference (decl)
1516 513 : && !omp_is_allocatable_or_ptr (decl))
1517 2454 : || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1518 407 : install_var_field (decl, true, 11, ctx);
1519 : else
1520 1110 : install_var_field (decl, false, 11, ctx);
1521 2118 : if (DECL_SIZE (decl)
1522 2118 : && !poly_int_tree_p (DECL_SIZE (decl)))
1523 : {
1524 6 : tree decl2 = DECL_VALUE_EXPR (decl);
1525 6 : gcc_assert (INDIRECT_REF_P (decl2));
1526 6 : decl2 = TREE_OPERAND (decl2, 0);
1527 6 : gcc_assert (DECL_P (decl2));
1528 6 : install_var_local (decl2, ctx);
1529 : }
1530 2118 : install_var_local (decl, ctx);
1531 2118 : break;
1532 :
1533 240 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
1534 240 : decl = OMP_CLAUSE_DECL (c);
1535 240 : while (INDIRECT_REF_P (decl)
1536 257 : || TREE_CODE (decl) == ARRAY_REF)
1537 17 : decl = TREE_OPERAND (decl, 0);
1538 240 : goto do_private;
1539 :
1540 149 : case OMP_CLAUSE_IS_DEVICE_PTR:
1541 149 : decl = OMP_CLAUSE_DECL (c);
1542 149 : goto do_private;
1543 :
1544 20211 : case OMP_CLAUSE__LOOPTEMP_:
1545 20211 : case OMP_CLAUSE__REDUCTEMP_:
1546 20211 : gcc_assert (is_taskreg_ctx (ctx));
1547 20211 : decl = OMP_CLAUSE_DECL (c);
1548 20211 : install_var_field (decl, false, 3, ctx);
1549 20211 : install_var_local (decl, ctx);
1550 20211 : break;
1551 :
1552 899 : case OMP_CLAUSE_COPYPRIVATE:
1553 899 : case OMP_CLAUSE_COPYIN:
1554 899 : decl = OMP_CLAUSE_DECL (c);
1555 899 : by_ref = use_pointer_for_field (decl, NULL);
1556 899 : install_var_field (decl, by_ref, 3, ctx);
1557 899 : break;
1558 :
1559 55715 : case OMP_CLAUSE_FINAL:
1560 55715 : case OMP_CLAUSE_IF:
1561 55715 : case OMP_CLAUSE_SELF:
1562 55715 : case OMP_CLAUSE_NUM_THREADS:
1563 55715 : case OMP_CLAUSE_NUM_TEAMS:
1564 55715 : case OMP_CLAUSE_THREAD_LIMIT:
1565 55715 : case OMP_CLAUSE_DEVICE:
1566 55715 : case OMP_CLAUSE_SCHEDULE:
1567 55715 : case OMP_CLAUSE_DIST_SCHEDULE:
1568 55715 : case OMP_CLAUSE_DEPEND:
1569 55715 : case OMP_CLAUSE_PRIORITY:
1570 55715 : case OMP_CLAUSE_GRAINSIZE:
1571 55715 : case OMP_CLAUSE_NUM_TASKS:
1572 55715 : case OMP_CLAUSE_NUM_GANGS:
1573 55715 : case OMP_CLAUSE_NUM_WORKERS:
1574 55715 : case OMP_CLAUSE_VECTOR_LENGTH:
1575 55715 : case OMP_CLAUSE_DETACH:
1576 55715 : case OMP_CLAUSE_FILTER:
1577 55715 : if (ctx->outer)
1578 17667 : scan_omp_op (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer);
1579 : break;
1580 :
1581 88546 : case OMP_CLAUSE_TO:
1582 88546 : case OMP_CLAUSE_FROM:
1583 88546 : case OMP_CLAUSE_MAP:
1584 88546 : if (ctx->outer)
1585 15339 : scan_omp_op (&OMP_CLAUSE_SIZE (c), ctx->outer);
1586 88546 : decl = OMP_CLAUSE_DECL (c);
1587 : /* If requested, make 'decl' addressable. */
1588 88546 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1589 88546 : && OMP_CLAUSE_MAP_DECL_MAKE_ADDRESSABLE (c))
1590 : {
1591 433 : gcc_checking_assert (DECL_P (decl));
1592 :
1593 433 : bool decl_addressable = TREE_ADDRESSABLE (decl);
1594 433 : if (!decl_addressable)
1595 : {
1596 219 : if (!make_addressable_vars)
1597 131 : make_addressable_vars = BITMAP_ALLOC (NULL);
1598 219 : bitmap_set_bit (make_addressable_vars, DECL_UID (decl));
1599 219 : TREE_ADDRESSABLE (decl) = 1;
1600 : }
1601 :
1602 433 : if (dump_enabled_p ())
1603 : {
1604 421 : location_t loc = OMP_CLAUSE_LOCATION (c);
1605 421 : const dump_user_location_t d_u_loc
1606 421 : = dump_user_location_t::from_location_t (loc);
1607 : /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */
1608 : #if __GNUC__ >= 10
1609 421 : # pragma GCC diagnostic push
1610 421 : # pragma GCC diagnostic ignored "-Wformat"
1611 : #endif
1612 421 : if (!decl_addressable)
1613 207 : dump_printf_loc (MSG_NOTE, d_u_loc,
1614 : "variable %<%T%>"
1615 : " made addressable\n",
1616 : decl);
1617 : else
1618 214 : dump_printf_loc (MSG_NOTE, d_u_loc,
1619 : "variable %<%T%>"
1620 : " already made addressable\n",
1621 : decl);
1622 : #if __GNUC__ >= 10
1623 421 : # pragma GCC diagnostic pop
1624 : #endif
1625 : }
1626 :
1627 : /* Done. */
1628 433 : OMP_CLAUSE_MAP_DECL_MAKE_ADDRESSABLE (c) = 0;
1629 : }
1630 : /* Global variables with "omp declare target" attribute
1631 : don't need to be copied, the receiver side will use them
1632 : directly. However, global variables with "omp declare target link"
1633 : attribute need to be copied. Or when ALWAYS modifier is used. */
1634 88546 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1635 80730 : && DECL_P (decl)
1636 44429 : && ((OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER
1637 40604 : && (OMP_CLAUSE_MAP_KIND (c)
1638 : != GOMP_MAP_FIRSTPRIVATE_REFERENCE)
1639 40175 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ATTACH
1640 39937 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_DETACH)
1641 4557 : || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1642 41077 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_TO
1643 40940 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_FROM
1644 40907 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_TOFROM
1645 40415 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_TO
1646 40385 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_FROM
1647 40363 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_TOFROM
1648 40349 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_TO_PSET
1649 36358 : && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1650 7582 : && varpool_node::get_create (decl)->offloadable
1651 93834 : && !lookup_attribute ("omp declare target link",
1652 5288 : DECL_ATTRIBUTES (decl)))
1653 : break;
1654 83299 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1655 83299 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER)
1656 : {
1657 : /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
1658 : not offloaded; there is nothing to map for those. */
1659 10478 : if (!is_gimple_omp_offloaded (ctx->stmt)
1660 4979 : && !POINTER_TYPE_P (TREE_TYPE (decl))
1661 10742 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
1662 : break;
1663 : }
1664 83035 : if (!flex_array_ptr)
1665 82807 : flex_array_ptr = lang_hooks.decls.omp_deep_mapping_p (ctx->stmt, c);
1666 83035 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1667 75219 : && DECL_P (decl)
1668 38918 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
1669 38680 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
1670 83338 : && is_omp_target (ctx->stmt))
1671 : {
1672 : /* If this is an offloaded region, an attach operation should
1673 : only exist when the pointer variable is mapped in a prior
1674 : clause. An exception is if we have a reference (to pointer):
1675 : in that case we should have mapped "*decl" in a previous
1676 : mapping instead of "decl". Skip the assertion in that case.
1677 : If we had an error, we may not have attempted to sort clauses
1678 : properly, so avoid the test. */
1679 265 : if (TREE_CODE (TREE_TYPE (decl)) != REFERENCE_TYPE
1680 233 : && is_gimple_omp_offloaded (ctx->stmt)
1681 335 : && !seen_error ())
1682 62 : gcc_assert
1683 : (maybe_lookup_decl (decl, ctx)
1684 : || (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1685 : && lookup_attribute ("omp declare target",
1686 : DECL_ATTRIBUTES (decl))));
1687 :
1688 : /* By itself, attach/detach is generated as part of pointer
1689 : variable mapping and should not create new variables in the
1690 : offloaded region, however sender refs for it must be created
1691 : for its address to be passed to the runtime. */
1692 265 : tree field
1693 265 : = build_decl (OMP_CLAUSE_LOCATION (c),
1694 : FIELD_DECL, NULL_TREE, ptr_type_node);
1695 265 : SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
1696 265 : insert_field_into_struct (ctx->record_type, field);
1697 : /* To not clash with a map of the pointer variable itself,
1698 : attach/detach maps have their field looked up by the *clause*
1699 : tree expression, not the decl. */
1700 265 : gcc_assert (!splay_tree_lookup (ctx->field_map,
1701 : (splay_tree_key) c));
1702 265 : splay_tree_insert (ctx->field_map, (splay_tree_key) c,
1703 : (splay_tree_value) field);
1704 265 : break;
1705 : }
1706 82770 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1707 82770 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
1708 71131 : || (OMP_CLAUSE_MAP_KIND (c)
1709 : == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
1710 : {
1711 4252 : if (TREE_CODE (decl) == COMPONENT_REF
1712 4252 : || (INDIRECT_REF_P (decl)
1713 0 : && TREE_CODE (TREE_OPERAND (decl, 0)) == COMPONENT_REF
1714 0 : && (((TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0)))
1715 : == REFERENCE_TYPE)
1716 0 : || (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0)))
1717 : == POINTER_TYPE)))))
1718 : break;
1719 4252 : if (DECL_SIZE (decl)
1720 4252 : && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
1721 : {
1722 712 : tree decl2 = DECL_VALUE_EXPR (decl);
1723 712 : gcc_assert (INDIRECT_REF_P (decl2));
1724 712 : decl2 = TREE_OPERAND (decl2, 0);
1725 712 : gcc_assert (DECL_P (decl2));
1726 712 : install_var_local (decl2, ctx);
1727 : }
1728 4252 : install_var_local (decl, ctx);
1729 4252 : break;
1730 : }
1731 78518 : if (DECL_P (decl))
1732 : {
1733 40576 : if (DECL_SIZE (decl)
1734 40576 : && !poly_int_tree_p (DECL_SIZE (decl)))
1735 : {
1736 0 : tree decl2 = DECL_VALUE_EXPR (decl);
1737 0 : gcc_assert (INDIRECT_REF_P (decl2));
1738 0 : decl2 = TREE_OPERAND (decl2, 0);
1739 0 : gcc_assert (DECL_P (decl2));
1740 0 : install_var_field (decl2, true, 3, ctx, c);
1741 0 : install_var_local (decl2, ctx);
1742 0 : install_var_local (decl, ctx);
1743 : }
1744 : else
1745 : {
1746 40576 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1747 34401 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
1748 5193 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
1749 45769 : && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1750 425 : install_var_field (decl, true, 7, ctx, c);
1751 : else
1752 40151 : install_var_field (decl, true, 3, ctx, c);
1753 40576 : if (is_gimple_omp_offloaded (ctx->stmt)
1754 40576 : && !(is_gimple_omp_oacc (ctx->stmt)
1755 13857 : && OMP_CLAUSE_MAP_IN_REDUCTION (c)))
1756 23829 : install_var_local (decl, ctx);
1757 : }
1758 : }
1759 : else
1760 : {
1761 37942 : tree base = get_base_address (decl);
1762 37942 : tree nc = OMP_CLAUSE_CHAIN (c);
1763 37942 : if (DECL_P (base)
1764 9358 : && nc != NULL_TREE
1765 6737 : && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP
1766 5573 : && OMP_CLAUSE_DECL (nc) == base
1767 1184 : && OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_POINTER
1768 38610 : && integer_zerop (OMP_CLAUSE_SIZE (nc)))
1769 : {
1770 0 : OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) = 1;
1771 0 : OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc) = 1;
1772 : }
1773 : else
1774 : {
1775 37942 : if (ctx->outer)
1776 : {
1777 7152 : scan_omp_op (&OMP_CLAUSE_DECL (c), ctx->outer);
1778 7152 : decl = OMP_CLAUSE_DECL (c);
1779 : }
1780 37942 : gcc_assert (!splay_tree_lookup (ctx->field_map,
1781 : (splay_tree_key) decl));
1782 37942 : tree field
1783 37942 : = build_decl (OMP_CLAUSE_LOCATION (c),
1784 : FIELD_DECL, NULL_TREE, ptr_type_node);
1785 37942 : SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
1786 37942 : insert_field_into_struct (ctx->record_type, field);
1787 37942 : splay_tree_insert (ctx->field_map, (splay_tree_key) c,
1788 : (splay_tree_value) field);
1789 : }
1790 : }
1791 : break;
1792 :
1793 3988 : case OMP_CLAUSE_ORDER:
1794 3988 : ctx->order_concurrent = true;
1795 3988 : break;
1796 :
1797 1922 : case OMP_CLAUSE_BIND:
1798 1922 : ctx->loop_p = true;
1799 1922 : break;
1800 :
1801 : case OMP_CLAUSE_NOWAIT:
1802 : case OMP_CLAUSE_ORDERED:
1803 : case OMP_CLAUSE_COLLAPSE:
1804 : case OMP_CLAUSE_UNTIED:
1805 : case OMP_CLAUSE_MERGEABLE:
1806 : case OMP_CLAUSE_PROC_BIND:
1807 : case OMP_CLAUSE_SAFELEN:
1808 : case OMP_CLAUSE_SIMDLEN:
1809 : case OMP_CLAUSE_THREADS:
1810 : case OMP_CLAUSE_SIMD:
1811 : case OMP_CLAUSE_NOGROUP:
1812 : case OMP_CLAUSE_DEFAULTMAP:
1813 : case OMP_CLAUSE_ASYNC:
1814 : case OMP_CLAUSE_WAIT:
1815 : case OMP_CLAUSE_GANG:
1816 : case OMP_CLAUSE_WORKER:
1817 : case OMP_CLAUSE_VECTOR:
1818 : case OMP_CLAUSE_INDEPENDENT:
1819 : case OMP_CLAUSE_AUTO:
1820 : case OMP_CLAUSE_SEQ:
1821 : case OMP_CLAUSE_TILE:
1822 : case OMP_CLAUSE__SIMT_:
1823 : case OMP_CLAUSE_DEFAULT:
1824 : case OMP_CLAUSE_NONTEMPORAL:
1825 : case OMP_CLAUSE_IF_PRESENT:
1826 : case OMP_CLAUSE_FINALIZE:
1827 : case OMP_CLAUSE_TASK_REDUCTION:
1828 : case OMP_CLAUSE_ALLOCATE:
1829 : case OMP_CLAUSE_DEVICE_TYPE:
1830 : break;
1831 :
1832 112 : case OMP_CLAUSE_USES_ALLOCATORS:
1833 112 : decl = OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c);
1834 112 : gcc_assert (DECL_P (decl));
1835 112 : gcc_assert (is_gimple_omp_offloaded (ctx->stmt));
1836 112 : install_var_field (decl, false, 3, ctx);
1837 112 : install_var_local (decl, ctx);
1838 112 : break;
1839 :
1840 162 : case OMP_CLAUSE_ALIGNED:
1841 162 : decl = OMP_CLAUSE_DECL (c);
1842 162 : if (is_global_var (decl)
1843 162 : && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1844 86 : install_var_local (decl, ctx);
1845 : break;
1846 :
1847 361 : case OMP_CLAUSE__CONDTEMP_:
1848 361 : decl = OMP_CLAUSE_DECL (c);
1849 361 : if (is_parallel_ctx (ctx))
1850 : {
1851 118 : install_var_field (decl, false, 3, ctx);
1852 118 : install_var_local (decl, ctx);
1853 : }
1854 243 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
1855 243 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
1856 368 : && !OMP_CLAUSE__CONDTEMP__ITER (c))
1857 125 : install_var_local (decl, ctx);
1858 : break;
1859 :
1860 : case OMP_CLAUSE_INIT:
1861 : case OMP_CLAUSE_USE:
1862 : case OMP_CLAUSE_DESTROY:
1863 : case OMP_CLAUSE_MESSAGE:
1864 : break;
1865 :
1866 0 : case OMP_CLAUSE__CACHE_:
1867 0 : case OMP_CLAUSE_NOHOST:
1868 0 : default:
1869 0 : gcc_unreachable ();
1870 : }
1871 : }
1872 :
1873 553751 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1874 : {
1875 422710 : switch (OMP_CLAUSE_CODE (c))
1876 : {
1877 22774 : case OMP_CLAUSE_LASTPRIVATE:
1878 : /* Let the corresponding firstprivate clause create
1879 : the variable. */
1880 22774 : if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
1881 7245 : scan_array_reductions = true;
1882 22774 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1883 : break;
1884 : /* FALLTHRU */
1885 :
1886 150566 : case OMP_CLAUSE_FIRSTPRIVATE:
1887 150566 : case OMP_CLAUSE_PRIVATE:
1888 150566 : case OMP_CLAUSE_LINEAR:
1889 150566 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
1890 150566 : case OMP_CLAUSE_IS_DEVICE_PTR:
1891 150566 : decl = OMP_CLAUSE_DECL (c);
1892 150566 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1893 : {
1894 257 : while (INDIRECT_REF_P (decl)
1895 257 : || TREE_CODE (decl) == ARRAY_REF)
1896 17 : decl = TREE_OPERAND (decl, 0);
1897 : }
1898 :
1899 150566 : if (is_variable_sized (decl))
1900 : {
1901 432 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1902 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR
1903 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1904 61 : && is_gimple_omp_offloaded (ctx->stmt))
1905 : {
1906 16 : tree decl2 = DECL_VALUE_EXPR (decl);
1907 16 : gcc_assert (INDIRECT_REF_P (decl2));
1908 16 : decl2 = TREE_OPERAND (decl2, 0);
1909 16 : gcc_assert (DECL_P (decl2));
1910 16 : install_var_local (decl2, ctx);
1911 16 : fixup_remapped_decl (decl2, ctx, false);
1912 : }
1913 432 : install_var_local (decl, ctx);
1914 : }
1915 301132 : fixup_remapped_decl (decl, ctx,
1916 150566 : OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE
1917 73503 : && OMP_CLAUSE_PRIVATE_DEBUG (c));
1918 150566 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
1919 150566 : && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
1920 : scan_array_reductions = true;
1921 : break;
1922 :
1923 16791 : case OMP_CLAUSE_REDUCTION:
1924 16791 : case OMP_CLAUSE_IN_REDUCTION:
1925 16791 : decl = OMP_CLAUSE_DECL (c);
1926 16791 : if (TREE_CODE (decl) != MEM_REF && !is_omp_target (ctx->stmt))
1927 : {
1928 14015 : if (is_variable_sized (decl))
1929 0 : install_var_local (decl, ctx);
1930 14015 : fixup_remapped_decl (decl, ctx, false);
1931 : }
1932 16791 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1933 422710 : scan_array_reductions = true;
1934 : break;
1935 :
1936 530 : case OMP_CLAUSE_TASK_REDUCTION:
1937 530 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1938 422710 : scan_array_reductions = true;
1939 : break;
1940 :
1941 30129 : case OMP_CLAUSE_SHARED:
1942 : /* Ignore shared directives in teams construct inside of
1943 : target construct. */
1944 30129 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
1945 30129 : && !is_host_teams_ctx (ctx))
1946 : break;
1947 18265 : decl = OMP_CLAUSE_DECL (c);
1948 18265 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1949 : break;
1950 12187 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
1951 : {
1952 86 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl,
1953 : ctx->outer)))
1954 : break;
1955 79 : bool by_ref = use_pointer_for_field (decl, ctx);
1956 79 : install_var_field (decl, by_ref, 11, ctx);
1957 79 : break;
1958 : }
1959 12101 : fixup_remapped_decl (decl, ctx, false);
1960 12101 : break;
1961 :
1962 80730 : case OMP_CLAUSE_MAP:
1963 80730 : if (!is_gimple_omp_offloaded (ctx->stmt))
1964 : break;
1965 53898 : decl = OMP_CLAUSE_DECL (c);
1966 53898 : if (DECL_P (decl)
1967 33905 : && ((OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER
1968 30248 : && (OMP_CLAUSE_MAP_KIND (c)
1969 : != GOMP_MAP_FIRSTPRIVATE_REFERENCE))
1970 4086 : || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1971 31024 : && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1972 61303 : && varpool_node::get_create (decl)->offloadable)
1973 : break;
1974 48194 : if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
1975 45982 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
1976 2212 : && is_omp_target (ctx->stmt)
1977 50197 : && !is_gimple_omp_offloaded (ctx->stmt))
1978 : break;
1979 48194 : if (DECL_P (decl))
1980 : {
1981 28201 : if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
1982 25160 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER)
1983 6696 : && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
1984 29828 : && !COMPLETE_TYPE_P (TREE_TYPE (decl)))
1985 : {
1986 24 : tree new_decl = lookup_decl (decl, ctx);
1987 24 : TREE_TYPE (new_decl)
1988 48 : = remap_type (TREE_TYPE (decl), &ctx->cb);
1989 : }
1990 28177 : else if (DECL_SIZE (decl)
1991 28177 : && !poly_int_tree_p (DECL_SIZE (decl)))
1992 : {
1993 712 : tree decl2 = DECL_VALUE_EXPR (decl);
1994 712 : gcc_assert (INDIRECT_REF_P (decl2));
1995 712 : decl2 = TREE_OPERAND (decl2, 0);
1996 712 : gcc_assert (DECL_P (decl2));
1997 712 : fixup_remapped_decl (decl2, ctx, false);
1998 712 : fixup_remapped_decl (decl, ctx, true);
1999 : }
2000 : else
2001 27465 : fixup_remapped_decl (decl, ctx, false);
2002 : }
2003 : break;
2004 :
2005 : case OMP_CLAUSE_COPYPRIVATE:
2006 : case OMP_CLAUSE_COPYIN:
2007 : case OMP_CLAUSE_DEFAULT:
2008 : case OMP_CLAUSE_IF:
2009 : case OMP_CLAUSE_SELF:
2010 : case OMP_CLAUSE_NUM_THREADS:
2011 : case OMP_CLAUSE_NUM_TEAMS:
2012 : case OMP_CLAUSE_THREAD_LIMIT:
2013 : case OMP_CLAUSE_DEVICE:
2014 : case OMP_CLAUSE_SCHEDULE:
2015 : case OMP_CLAUSE_DIST_SCHEDULE:
2016 : case OMP_CLAUSE_NOWAIT:
2017 : case OMP_CLAUSE_ORDERED:
2018 : case OMP_CLAUSE_COLLAPSE:
2019 : case OMP_CLAUSE_UNTIED:
2020 : case OMP_CLAUSE_FINAL:
2021 : case OMP_CLAUSE_MERGEABLE:
2022 : case OMP_CLAUSE_PROC_BIND:
2023 : case OMP_CLAUSE_SAFELEN:
2024 : case OMP_CLAUSE_SIMDLEN:
2025 : case OMP_CLAUSE_ALIGNED:
2026 : case OMP_CLAUSE_DEPEND:
2027 : case OMP_CLAUSE_DETACH:
2028 : case OMP_CLAUSE_ALLOCATE:
2029 : case OMP_CLAUSE__LOOPTEMP_:
2030 : case OMP_CLAUSE__REDUCTEMP_:
2031 : case OMP_CLAUSE_TO:
2032 : case OMP_CLAUSE_FROM:
2033 : case OMP_CLAUSE_PRIORITY:
2034 : case OMP_CLAUSE_GRAINSIZE:
2035 : case OMP_CLAUSE_NUM_TASKS:
2036 : case OMP_CLAUSE_THREADS:
2037 : case OMP_CLAUSE_SIMD:
2038 : case OMP_CLAUSE_NOGROUP:
2039 : case OMP_CLAUSE_DEFAULTMAP:
2040 : case OMP_CLAUSE_ORDER:
2041 : case OMP_CLAUSE_BIND:
2042 : case OMP_CLAUSE_USE_DEVICE_PTR:
2043 : case OMP_CLAUSE_USE_DEVICE_ADDR:
2044 : case OMP_CLAUSE_NONTEMPORAL:
2045 : case OMP_CLAUSE_ASYNC:
2046 : case OMP_CLAUSE_WAIT:
2047 : case OMP_CLAUSE_NUM_GANGS:
2048 : case OMP_CLAUSE_NUM_WORKERS:
2049 : case OMP_CLAUSE_VECTOR_LENGTH:
2050 : case OMP_CLAUSE_GANG:
2051 : case OMP_CLAUSE_WORKER:
2052 : case OMP_CLAUSE_VECTOR:
2053 : case OMP_CLAUSE_INDEPENDENT:
2054 : case OMP_CLAUSE_AUTO:
2055 : case OMP_CLAUSE_SEQ:
2056 : case OMP_CLAUSE_TILE:
2057 : case OMP_CLAUSE__SIMT_:
2058 : case OMP_CLAUSE_IF_PRESENT:
2059 : case OMP_CLAUSE_FINALIZE:
2060 : case OMP_CLAUSE_FILTER:
2061 : case OMP_CLAUSE__CONDTEMP_:
2062 : case OMP_CLAUSE_INIT:
2063 : case OMP_CLAUSE_USE:
2064 : case OMP_CLAUSE_DESTROY:
2065 : case OMP_CLAUSE_DEVICE_TYPE:
2066 : case OMP_CLAUSE_USES_ALLOCATORS:
2067 : case OMP_CLAUSE_MESSAGE:
2068 : break;
2069 :
2070 0 : case OMP_CLAUSE__CACHE_:
2071 0 : case OMP_CLAUSE_NOHOST:
2072 0 : default:
2073 0 : gcc_unreachable ();
2074 : }
2075 : }
2076 :
2077 131041 : gcc_checking_assert (!scan_array_reductions
2078 : || !is_gimple_omp_oacc (ctx->stmt));
2079 : if (scan_array_reductions)
2080 : {
2081 34699 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2082 29488 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
2083 27085 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
2084 26437 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
2085 30216 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
2086 : {
2087 2229 : omp_context *rctx = ctx;
2088 2229 : if (is_omp_target (ctx->stmt))
2089 60 : rctx = ctx->outer;
2090 2229 : scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), rctx);
2091 2229 : scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), rctx);
2092 : }
2093 27259 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
2094 27259 : && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
2095 7245 : scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
2096 20014 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
2097 20014 : && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
2098 656 : scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
2099 : }
2100 131041 : if (flex_array_ptr)
2101 : {
2102 144 : tree field = build_range_type (size_type_node,
2103 : build_int_cstu (size_type_node, 0),
2104 : NULL_TREE);
2105 144 : field = build_array_type (ptr_type_node, field);
2106 144 : field = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE, field);
2107 144 : SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
2108 144 : DECL_CONTEXT (field) = ctx->record_type;
2109 144 : DECL_CHAIN (field) = TYPE_FIELDS (ctx->record_type);
2110 144 : TYPE_FIELDS (ctx->record_type) = field;
2111 : }
2112 131041 : }
2113 :
2114 : /* Create a new name for omp child function. Returns an identifier. */
2115 :
2116 : static tree
2117 52123 : create_omp_child_function_name (bool task_copy)
2118 : {
2119 103696 : return clone_function_name_numbered (current_function_decl,
2120 52123 : task_copy ? "_omp_cpyfn" : "_omp_fn");
2121 : }
2122 :
2123 : /* Return true if CTX may belong to offloaded code: either if current function
2124 : is offloaded, or any enclosing context corresponds to a target region. */
2125 :
2126 : static bool
2127 124211 : omp_maybe_offloaded_ctx (omp_context *ctx)
2128 : {
2129 124211 : if (cgraph_node::get (current_function_decl)->offloadable)
2130 : return true;
2131 205784 : for (; ctx; ctx = ctx->outer)
2132 121527 : if (is_gimple_omp_offloaded (ctx->stmt))
2133 : return true;
2134 : return false;
2135 : }
2136 :
2137 : /* Build a decl for the omp child function. It'll not contain a body
2138 : yet, just the bare decl. If HOST_ONLY, do not create a device version. */
2139 :
2140 : static void
2141 52123 : create_omp_child_function (omp_context *ctx, bool task_copy, bool host_only)
2142 : {
2143 52123 : tree decl, type, name, t;
2144 :
2145 52123 : name = create_omp_child_function_name (task_copy);
2146 52123 : if (task_copy)
2147 550 : type = build_function_type_list (void_type_node, ptr_type_node,
2148 : ptr_type_node, NULL_TREE);
2149 : else
2150 51573 : type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE);
2151 :
2152 52123 : decl = build_decl (gimple_location (ctx->stmt), FUNCTION_DECL, name, type);
2153 :
2154 52123 : gcc_checking_assert (!is_gimple_omp_oacc (ctx->stmt)
2155 : || !task_copy);
2156 40737 : if (!task_copy)
2157 51573 : ctx->cb.dst_fn = decl;
2158 : else
2159 550 : gimple_omp_task_set_copy_fn (ctx->stmt, decl);
2160 :
2161 52123 : TREE_STATIC (decl) = 1;
2162 52123 : TREE_USED (decl) = 1;
2163 52123 : DECL_ARTIFICIAL (decl) = 1;
2164 52123 : DECL_IGNORED_P (decl) = 0;
2165 52123 : TREE_PUBLIC (decl) = 0;
2166 52123 : DECL_UNINLINABLE (decl) = 1;
2167 52123 : DECL_EXTERNAL (decl) = 0;
2168 52123 : DECL_CONTEXT (decl) = NULL_TREE;
2169 52123 : DECL_INITIAL (decl) = make_node (BLOCK);
2170 52123 : BLOCK_SUPERCONTEXT (DECL_INITIAL (decl)) = decl;
2171 52123 : DECL_ATTRIBUTES (decl) = DECL_ATTRIBUTES (current_function_decl);
2172 : /* Remove omp declare simd attribute from the new attributes. */
2173 52123 : if (tree a = lookup_attribute ("omp declare simd", DECL_ATTRIBUTES (decl)))
2174 : {
2175 9 : while (tree a2 = lookup_attribute ("omp declare simd", TREE_CHAIN (a)))
2176 : a = a2;
2177 9 : a = TREE_CHAIN (a);
2178 22 : for (tree *p = &DECL_ATTRIBUTES (decl); *p != a;)
2179 13 : if (is_attribute_p ("omp declare simd", get_attribute_name (*p)))
2180 9 : *p = TREE_CHAIN (*p);
2181 : else
2182 : {
2183 4 : tree chain = TREE_CHAIN (*p);
2184 4 : *p = copy_node (*p);
2185 4 : p = &TREE_CHAIN (*p);
2186 4 : *p = chain;
2187 : }
2188 : }
2189 104246 : DECL_FUNCTION_SPECIFIC_OPTIMIZATION (decl)
2190 52123 : = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (current_function_decl);
2191 104246 : DECL_FUNCTION_SPECIFIC_TARGET (decl)
2192 52123 : = DECL_FUNCTION_SPECIFIC_TARGET (current_function_decl);
2193 104246 : DECL_FUNCTION_VERSIONED (decl)
2194 52123 : = DECL_FUNCTION_VERSIONED (current_function_decl);
2195 :
2196 52123 : if (omp_maybe_offloaded_ctx (ctx) && !host_only)
2197 : {
2198 31199 : cgraph_node::get_create (decl)->offloadable = 1;
2199 31199 : if (ENABLE_OFFLOADING)
2200 : g->have_offload = true;
2201 : }
2202 :
2203 52123 : if (cgraph_node::get_create (decl)->offloadable)
2204 : {
2205 31199 : const char *target_attr = (is_gimple_omp_offloaded (ctx->stmt)
2206 31199 : ? "omp target entrypoint"
2207 6242 : : "omp declare target");
2208 31199 : if (lookup_attribute ("omp declare target",
2209 31199 : DECL_ATTRIBUTES (current_function_decl)))
2210 : {
2211 3032 : if (is_gimple_omp_offloaded (ctx->stmt))
2212 1620 : DECL_ATTRIBUTES (decl)
2213 1620 : = remove_attribute ("omp declare target",
2214 1620 : copy_list (DECL_ATTRIBUTES (decl)));
2215 : else
2216 : target_attr = NULL;
2217 : }
2218 1620 : if (target_attr
2219 29787 : && is_gimple_omp_offloaded (ctx->stmt)
2220 24957 : && lookup_attribute ("noclone", DECL_ATTRIBUTES (decl)) == NULL_TREE)
2221 19450 : DECL_ATTRIBUTES (decl) = tree_cons (get_identifier ("noclone"),
2222 19450 : NULL_TREE, DECL_ATTRIBUTES (decl));
2223 31199 : if (target_attr)
2224 29787 : DECL_ATTRIBUTES (decl)
2225 59574 : = tree_cons (get_identifier (target_attr),
2226 29787 : NULL_TREE, DECL_ATTRIBUTES (decl));
2227 : }
2228 :
2229 52123 : t = build_decl (DECL_SOURCE_LOCATION (decl),
2230 : RESULT_DECL, NULL_TREE, void_type_node);
2231 52123 : DECL_ARTIFICIAL (t) = 1;
2232 52123 : DECL_IGNORED_P (t) = 1;
2233 52123 : DECL_CONTEXT (t) = decl;
2234 52123 : DECL_RESULT (decl) = t;
2235 :
2236 52123 : tree data_name = get_identifier (".omp_data_i");
2237 52123 : t = build_decl (DECL_SOURCE_LOCATION (decl), PARM_DECL, data_name,
2238 : ptr_type_node);
2239 52123 : DECL_ARTIFICIAL (t) = 1;
2240 52123 : DECL_NAMELESS (t) = 1;
2241 52123 : DECL_ARG_TYPE (t) = ptr_type_node;
2242 52123 : DECL_CONTEXT (t) = current_function_decl;
2243 52123 : TREE_USED (t) = 1;
2244 52123 : TREE_READONLY (t) = 1;
2245 52123 : DECL_ARGUMENTS (decl) = t;
2246 52123 : if (!task_copy)
2247 51573 : ctx->receiver_decl = t;
2248 : else
2249 : {
2250 550 : t = build_decl (DECL_SOURCE_LOCATION (decl),
2251 : PARM_DECL, get_identifier (".omp_data_o"),
2252 : ptr_type_node);
2253 550 : DECL_ARTIFICIAL (t) = 1;
2254 550 : DECL_NAMELESS (t) = 1;
2255 550 : DECL_ARG_TYPE (t) = ptr_type_node;
2256 550 : DECL_CONTEXT (t) = current_function_decl;
2257 550 : TREE_USED (t) = 1;
2258 550 : TREE_ADDRESSABLE (t) = 1;
2259 550 : DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2260 550 : DECL_ARGUMENTS (decl) = t;
2261 : }
2262 :
2263 : /* Allocate memory for the function structure. The call to
2264 : allocate_struct_function clobbers CFUN, so we need to restore
2265 : it afterward. */
2266 52123 : push_struct_function (decl);
2267 52123 : cfun->function_end_locus = gimple_location (ctx->stmt);
2268 52123 : init_tree_ssa (cfun);
2269 52123 : pop_cfun ();
2270 52123 : }
2271 :
2272 : /* Callback for walk_gimple_seq. Check if combined parallel
2273 : contains gimple_omp_for_combined_into_p OMP_FOR. */
2274 :
2275 : tree
2276 35863 : omp_find_combined_for (gimple_stmt_iterator *gsi_p,
2277 : bool *handled_ops_p,
2278 : struct walk_stmt_info *wi)
2279 : {
2280 35863 : gimple *stmt = gsi_stmt (*gsi_p);
2281 :
2282 35863 : *handled_ops_p = true;
2283 35863 : switch (gimple_code (stmt))
2284 : {
2285 20118 : WALK_SUBSTMTS;
2286 :
2287 13574 : case GIMPLE_OMP_FOR:
2288 13574 : if (gimple_omp_for_combined_into_p (stmt)
2289 13574 : && gimple_omp_for_kind (stmt)
2290 8009 : == *(const enum gf_mask *) (wi->info))
2291 : {
2292 8009 : wi->info = stmt;
2293 8009 : return integer_zero_node;
2294 : }
2295 : break;
2296 : default:
2297 : break;
2298 : }
2299 : return NULL;
2300 : }
2301 :
2302 : /* Add _LOOPTEMP_/_REDUCTEMP_ clauses on OpenMP parallel or task. */
2303 :
2304 : static void
2305 14159 : add_taskreg_looptemp_clauses (enum gf_mask msk, gimple *stmt,
2306 : omp_context *outer_ctx)
2307 : {
2308 14159 : struct walk_stmt_info wi;
2309 :
2310 14159 : memset (&wi, 0, sizeof (wi));
2311 14159 : wi.val_only = true;
2312 14159 : wi.info = (void *) &msk;
2313 14159 : walk_gimple_seq (gimple_omp_body (stmt), omp_find_combined_for, NULL, &wi);
2314 14159 : if (wi.info != (void *) &msk)
2315 : {
2316 8009 : gomp_for *for_stmt = as_a <gomp_for *> ((gimple *) wi.info);
2317 8009 : struct omp_for_data fd;
2318 8009 : omp_extract_for_data (for_stmt, &fd, NULL);
2319 : /* We need two temporaries with fd.loop.v type (istart/iend)
2320 : and then (fd.collapse - 1) temporaries with the same
2321 : type for count2 ... countN-1 vars if not constant. */
2322 8009 : size_t count = 2, i;
2323 8009 : tree type = fd.iter_type;
2324 8009 : if (fd.collapse > 1
2325 2648 : && TREE_CODE (fd.loop.n2) != INTEGER_CST)
2326 : {
2327 1466 : count += fd.collapse - 1;
2328 : /* If there are lastprivate clauses on the inner
2329 : GIMPLE_OMP_FOR, add one more temporaries for the total number
2330 : of iterations (product of count1 ... countN-1). */
2331 1466 : if (omp_find_clause (gimple_omp_for_clauses (for_stmt),
2332 : OMP_CLAUSE_LASTPRIVATE)
2333 1466 : || (msk == GF_OMP_FOR_KIND_FOR
2334 1335 : && omp_find_clause (gimple_omp_parallel_clauses (stmt),
2335 : OMP_CLAUSE_LASTPRIVATE)))
2336 : {
2337 726 : tree temp = create_tmp_var (type);
2338 726 : tree c = build_omp_clause (UNKNOWN_LOCATION,
2339 : OMP_CLAUSE__LOOPTEMP_);
2340 726 : insert_decl_map (&outer_ctx->cb, temp, temp);
2341 726 : OMP_CLAUSE_DECL (c) = temp;
2342 726 : OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
2343 726 : gimple_omp_taskreg_set_clauses (stmt, c);
2344 : }
2345 1466 : if (fd.non_rect
2346 24 : && fd.last_nonrect == fd.first_nonrect + 1)
2347 12 : if (tree v = gimple_omp_for_index (for_stmt, fd.last_nonrect))
2348 12 : if (!TYPE_UNSIGNED (TREE_TYPE (v)))
2349 : {
2350 12 : v = gimple_omp_for_index (for_stmt, fd.first_nonrect);
2351 12 : tree type2 = TREE_TYPE (v);
2352 12 : count++;
2353 48 : for (i = 0; i < 3; i++)
2354 : {
2355 36 : tree temp = create_tmp_var (type2);
2356 36 : tree c = build_omp_clause (UNKNOWN_LOCATION,
2357 : OMP_CLAUSE__LOOPTEMP_);
2358 36 : insert_decl_map (&outer_ctx->cb, temp, temp);
2359 36 : OMP_CLAUSE_DECL (c) = temp;
2360 36 : OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
2361 36 : gimple_omp_taskreg_set_clauses (stmt, c);
2362 : }
2363 : }
2364 : }
2365 26872 : for (i = 0; i < count; i++)
2366 : {
2367 18863 : tree temp = create_tmp_var (type);
2368 18863 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
2369 18863 : insert_decl_map (&outer_ctx->cb, temp, temp);
2370 18863 : OMP_CLAUSE_DECL (c) = temp;
2371 18863 : OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
2372 18863 : gimple_omp_taskreg_set_clauses (stmt, c);
2373 : }
2374 : }
2375 14159 : if (msk == GF_OMP_FOR_KIND_TASKLOOP
2376 15726 : && omp_find_clause (gimple_omp_task_clauses (stmt),
2377 : OMP_CLAUSE_REDUCTION))
2378 : {
2379 517 : tree type = build_pointer_type (pointer_sized_int_node);
2380 517 : tree temp = create_tmp_var (type);
2381 517 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
2382 517 : insert_decl_map (&outer_ctx->cb, temp, temp);
2383 517 : OMP_CLAUSE_DECL (c) = temp;
2384 517 : OMP_CLAUSE_CHAIN (c) = gimple_omp_task_clauses (stmt);
2385 517 : gimple_omp_task_set_clauses (stmt, c);
2386 : }
2387 14159 : }
2388 :
2389 : /* Scan an OpenMP parallel directive. */
2390 :
2391 : static void
2392 18393 : scan_omp_parallel (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2393 : {
2394 18393 : omp_context *ctx;
2395 18393 : tree name;
2396 18393 : gomp_parallel *stmt = as_a <gomp_parallel *> (gsi_stmt (*gsi));
2397 :
2398 : /* Ignore parallel directives with empty bodies, unless there
2399 : are copyin clauses. */
2400 18393 : if (optimize > 0
2401 12590 : && empty_body_p (gimple_omp_body (stmt))
2402 18440 : && omp_find_clause (gimple_omp_parallel_clauses (stmt),
2403 : OMP_CLAUSE_COPYIN) == NULL)
2404 : {
2405 41 : gsi_replace (gsi, gimple_build_nop (), false);
2406 41 : return;
2407 : }
2408 :
2409 18352 : if (gimple_omp_parallel_combined_p (stmt))
2410 12592 : add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR, stmt, outer_ctx);
2411 18352 : for (tree c = omp_find_clause (gimple_omp_parallel_clauses (stmt),
2412 : OMP_CLAUSE_REDUCTION);
2413 21221 : c; c = omp_find_clause (OMP_CLAUSE_CHAIN (c), OMP_CLAUSE_REDUCTION))
2414 3986 : if (OMP_CLAUSE_REDUCTION_TASK (c))
2415 : {
2416 69 : tree type = build_pointer_type (pointer_sized_int_node);
2417 69 : tree temp = create_tmp_var (type);
2418 69 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
2419 69 : if (outer_ctx)
2420 18 : insert_decl_map (&outer_ctx->cb, temp, temp);
2421 69 : OMP_CLAUSE_DECL (c) = temp;
2422 69 : OMP_CLAUSE_CHAIN (c) = gimple_omp_parallel_clauses (stmt);
2423 69 : gimple_omp_parallel_set_clauses (stmt, c);
2424 69 : break;
2425 : }
2426 3917 : else if (OMP_CLAUSE_CHAIN (c) == NULL_TREE)
2427 : break;
2428 :
2429 18352 : ctx = new_omp_context (stmt, outer_ctx);
2430 18352 : taskreg_contexts.safe_push (ctx);
2431 18352 : if (taskreg_nesting_level > 1)
2432 6287 : ctx->is_nested = true;
2433 18352 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2434 18352 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2435 18352 : name = create_tmp_var_name (".omp_data_s");
2436 18352 : name = build_decl (gimple_location (stmt),
2437 : TYPE_DECL, name, ctx->record_type);
2438 18352 : DECL_ARTIFICIAL (name) = 1;
2439 18352 : DECL_NAMELESS (name) = 1;
2440 18352 : TYPE_NAME (ctx->record_type) = name;
2441 18352 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
2442 18352 : create_omp_child_function (ctx, false, false);
2443 18352 : gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn);
2444 :
2445 18352 : scan_sharing_clauses (gimple_omp_parallel_clauses (stmt), ctx);
2446 18352 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
2447 :
2448 18352 : if (TYPE_FIELDS (ctx->record_type) == NULL)
2449 3617 : ctx->record_type = ctx->receiver_decl = NULL;
2450 : }
2451 :
2452 : /* Scan an OpenMP task directive. */
2453 :
2454 : static void
2455 5651 : scan_omp_task (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2456 : {
2457 5651 : omp_context *ctx;
2458 5651 : tree name, t;
2459 5651 : gomp_task *stmt = as_a <gomp_task *> (gsi_stmt (*gsi));
2460 :
2461 : /* Ignore task directives with empty bodies, unless they have depend
2462 : clause. */
2463 5651 : if (optimize > 0
2464 2575 : && gimple_omp_body (stmt)
2465 2532 : && empty_body_p (gimple_omp_body (stmt))
2466 5707 : && !omp_find_clause (gimple_omp_task_clauses (stmt), OMP_CLAUSE_DEPEND))
2467 : {
2468 21 : gsi_replace (gsi, gimple_build_nop (), false);
2469 135 : return;
2470 : }
2471 :
2472 5630 : if (gimple_omp_task_taskloop_p (stmt))
2473 1567 : add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP, stmt, outer_ctx);
2474 :
2475 5630 : ctx = new_omp_context (stmt, outer_ctx);
2476 :
2477 5630 : if (gimple_omp_task_taskwait_p (stmt))
2478 : {
2479 93 : scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
2480 93 : return;
2481 : }
2482 :
2483 5537 : taskreg_contexts.safe_push (ctx);
2484 5537 : if (taskreg_nesting_level > 1)
2485 2342 : ctx->is_nested = true;
2486 5537 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2487 5537 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2488 5537 : name = create_tmp_var_name (".omp_data_s");
2489 5537 : name = build_decl (gimple_location (stmt),
2490 : TYPE_DECL, name, ctx->record_type);
2491 5537 : DECL_ARTIFICIAL (name) = 1;
2492 5537 : DECL_NAMELESS (name) = 1;
2493 5537 : TYPE_NAME (ctx->record_type) = name;
2494 5537 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
2495 5537 : create_omp_child_function (ctx, false, false);
2496 5537 : gimple_omp_task_set_child_fn (stmt, ctx->cb.dst_fn);
2497 :
2498 5537 : scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
2499 :
2500 5537 : if (ctx->srecord_type)
2501 : {
2502 550 : name = create_tmp_var_name (".omp_data_a");
2503 550 : name = build_decl (gimple_location (stmt),
2504 : TYPE_DECL, name, ctx->srecord_type);
2505 550 : DECL_ARTIFICIAL (name) = 1;
2506 550 : DECL_NAMELESS (name) = 1;
2507 550 : TYPE_NAME (ctx->srecord_type) = name;
2508 550 : TYPE_ARTIFICIAL (ctx->srecord_type) = 1;
2509 550 : create_omp_child_function (ctx, true, false);
2510 : }
2511 :
2512 5537 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
2513 :
2514 5537 : if (TYPE_FIELDS (ctx->record_type) == NULL)
2515 : {
2516 2082 : ctx->record_type = ctx->receiver_decl = NULL;
2517 2082 : t = build_int_cst (long_integer_type_node, 0);
2518 2082 : gimple_omp_task_set_arg_size (stmt, t);
2519 2082 : t = build_int_cst (long_integer_type_node, 1);
2520 2082 : gimple_omp_task_set_arg_align (stmt, t);
2521 : }
2522 : }
2523 :
2524 : /* Helper function for finish_taskreg_scan, called through walk_tree.
2525 : If maybe_lookup_decl_in_outer_context returns non-NULL for some
2526 : tree, replace it in the expression. */
2527 :
2528 : static tree
2529 276 : finish_taskreg_remap (tree *tp, int *walk_subtrees, void *data)
2530 : {
2531 276 : if (VAR_P (*tp))
2532 : {
2533 96 : omp_context *ctx = (omp_context *) data;
2534 96 : tree t = maybe_lookup_decl_in_outer_ctx (*tp, ctx);
2535 96 : if (t != *tp)
2536 : {
2537 9 : if (DECL_HAS_VALUE_EXPR_P (t))
2538 0 : t = unshare_expr (DECL_VALUE_EXPR (t));
2539 9 : *tp = t;
2540 : }
2541 96 : *walk_subtrees = 0;
2542 : }
2543 180 : else if (IS_TYPE_OR_DECL_P (*tp))
2544 0 : *walk_subtrees = 0;
2545 276 : return NULL_TREE;
2546 : }
2547 :
2548 : /* If any decls have been made addressable during scan_omp,
2549 : adjust their fields if needed, and layout record types
2550 : of parallel/task constructs. */
2551 :
2552 : static void
2553 26610 : finish_taskreg_scan (omp_context *ctx)
2554 : {
2555 26610 : if (ctx->record_type == NULL_TREE)
2556 : return;
2557 :
2558 : /* If any make_addressable_vars were needed, verify all
2559 : OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK,TEAMS}
2560 : statements if use_pointer_for_field hasn't changed
2561 : because of that. If it did, update field types now. */
2562 19456 : if (make_addressable_vars)
2563 : {
2564 2216 : tree c;
2565 :
2566 12369 : for (c = gimple_omp_taskreg_clauses (ctx->stmt);
2567 12369 : c; c = OMP_CLAUSE_CHAIN (c))
2568 10153 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
2569 10153 : && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
2570 : {
2571 2617 : tree decl = OMP_CLAUSE_DECL (c);
2572 :
2573 : /* Global variables don't need to be copied,
2574 : the receiver side will use them directly. */
2575 2617 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
2576 282 : continue;
2577 2335 : if (!bitmap_bit_p (make_addressable_vars, DECL_UID (decl))
2578 2335 : || !use_pointer_for_field (decl, ctx))
2579 1784 : continue;
2580 551 : tree field = lookup_field (decl, ctx);
2581 551 : if (TREE_CODE (TREE_TYPE (field)) == POINTER_TYPE
2582 551 : && TREE_TYPE (TREE_TYPE (field)) == TREE_TYPE (decl))
2583 497 : continue;
2584 54 : TREE_TYPE (field) = build_pointer_type (TREE_TYPE (decl));
2585 54 : TREE_THIS_VOLATILE (field) = 0;
2586 54 : DECL_USER_ALIGN (field) = 0;
2587 54 : SET_DECL_ALIGN (field, TYPE_ALIGN (TREE_TYPE (field)));
2588 54 : if (TYPE_ALIGN (ctx->record_type) < DECL_ALIGN (field))
2589 16 : SET_TYPE_ALIGN (ctx->record_type, DECL_ALIGN (field));
2590 54 : if (ctx->srecord_type)
2591 : {
2592 0 : tree sfield = lookup_sfield (decl, ctx);
2593 0 : TREE_TYPE (sfield) = TREE_TYPE (field);
2594 0 : TREE_THIS_VOLATILE (sfield) = 0;
2595 0 : DECL_USER_ALIGN (sfield) = 0;
2596 0 : SET_DECL_ALIGN (sfield, DECL_ALIGN (field));
2597 0 : if (TYPE_ALIGN (ctx->srecord_type) < DECL_ALIGN (sfield))
2598 0 : SET_TYPE_ALIGN (ctx->srecord_type, DECL_ALIGN (sfield));
2599 : }
2600 : }
2601 : }
2602 :
2603 19456 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL)
2604 : {
2605 14735 : tree clauses = gimple_omp_parallel_clauses (ctx->stmt);
2606 14735 : tree c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
2607 14735 : if (c)
2608 : {
2609 : /* Move the _reductemp_ clause first. GOMP_parallel_reductions
2610 : expects to find it at the start of data. */
2611 69 : tree f = lookup_field (OMP_CLAUSE_DECL (c), ctx);
2612 69 : tree *p = &TYPE_FIELDS (ctx->record_type);
2613 155 : while (*p)
2614 155 : if (*p == f)
2615 : {
2616 69 : *p = DECL_CHAIN (*p);
2617 69 : break;
2618 : }
2619 : else
2620 86 : p = &DECL_CHAIN (*p);
2621 69 : DECL_CHAIN (f) = TYPE_FIELDS (ctx->record_type);
2622 69 : TYPE_FIELDS (ctx->record_type) = f;
2623 : }
2624 14735 : layout_type (ctx->record_type);
2625 14735 : fixup_child_record_type (ctx);
2626 : }
2627 4721 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
2628 : {
2629 1266 : layout_type (ctx->record_type);
2630 1266 : fixup_child_record_type (ctx);
2631 : }
2632 : else
2633 : {
2634 3455 : location_t loc = gimple_location (ctx->stmt);
2635 3455 : tree *p, vla_fields = NULL_TREE, *q = &vla_fields;
2636 3455 : tree detach_clause
2637 3455 : = omp_find_clause (gimple_omp_task_clauses (ctx->stmt),
2638 : OMP_CLAUSE_DETACH);
2639 : /* Move VLA fields to the end. */
2640 3455 : p = &TYPE_FIELDS (ctx->record_type);
2641 15410 : while (*p)
2642 11955 : if (!TYPE_SIZE_UNIT (TREE_TYPE (*p))
2643 11955 : || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p))))
2644 : {
2645 96 : *q = *p;
2646 96 : *p = TREE_CHAIN (*p);
2647 96 : TREE_CHAIN (*q) = NULL_TREE;
2648 96 : q = &TREE_CHAIN (*q);
2649 : }
2650 : else
2651 11859 : p = &DECL_CHAIN (*p);
2652 3455 : *p = vla_fields;
2653 3455 : if (gimple_omp_task_taskloop_p (ctx->stmt))
2654 : {
2655 : /* Move fields corresponding to first and second _looptemp_
2656 : clause first. There are filled by GOMP_taskloop
2657 : and thus need to be in specific positions. */
2658 1567 : tree clauses = gimple_omp_task_clauses (ctx->stmt);
2659 1567 : tree c1 = omp_find_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
2660 1567 : tree c2 = omp_find_clause (OMP_CLAUSE_CHAIN (c1),
2661 : OMP_CLAUSE__LOOPTEMP_);
2662 1567 : tree c3 = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
2663 1567 : tree f1 = lookup_field (OMP_CLAUSE_DECL (c1), ctx);
2664 1567 : tree f2 = lookup_field (OMP_CLAUSE_DECL (c2), ctx);
2665 2084 : tree f3 = c3 ? lookup_field (OMP_CLAUSE_DECL (c3), ctx) : NULL_TREE;
2666 1567 : p = &TYPE_FIELDS (ctx->record_type);
2667 8873 : while (*p)
2668 7306 : if (*p == f1 || *p == f2 || *p == f3)
2669 3651 : *p = DECL_CHAIN (*p);
2670 : else
2671 3655 : p = &DECL_CHAIN (*p);
2672 1567 : DECL_CHAIN (f1) = f2;
2673 1567 : if (c3)
2674 : {
2675 517 : DECL_CHAIN (f2) = f3;
2676 517 : DECL_CHAIN (f3) = TYPE_FIELDS (ctx->record_type);
2677 : }
2678 : else
2679 1050 : DECL_CHAIN (f2) = TYPE_FIELDS (ctx->record_type);
2680 1567 : TYPE_FIELDS (ctx->record_type) = f1;
2681 1567 : if (ctx->srecord_type)
2682 : {
2683 389 : f1 = lookup_sfield (OMP_CLAUSE_DECL (c1), ctx);
2684 389 : f2 = lookup_sfield (OMP_CLAUSE_DECL (c2), ctx);
2685 389 : if (c3)
2686 186 : f3 = lookup_sfield (OMP_CLAUSE_DECL (c3), ctx);
2687 389 : p = &TYPE_FIELDS (ctx->srecord_type);
2688 2099 : while (*p)
2689 1710 : if (*p == f1 || *p == f2 || *p == f3)
2690 964 : *p = DECL_CHAIN (*p);
2691 : else
2692 746 : p = &DECL_CHAIN (*p);
2693 389 : DECL_CHAIN (f1) = f2;
2694 389 : DECL_CHAIN (f2) = TYPE_FIELDS (ctx->srecord_type);
2695 389 : if (c3)
2696 : {
2697 186 : DECL_CHAIN (f2) = f3;
2698 186 : DECL_CHAIN (f3) = TYPE_FIELDS (ctx->srecord_type);
2699 : }
2700 : else
2701 203 : DECL_CHAIN (f2) = TYPE_FIELDS (ctx->srecord_type);
2702 389 : TYPE_FIELDS (ctx->srecord_type) = f1;
2703 : }
2704 : }
2705 3455 : if (detach_clause)
2706 : {
2707 189 : tree c, field;
2708 :
2709 : /* Look for a firstprivate clause with the detach event handle. */
2710 540 : for (c = gimple_omp_taskreg_clauses (ctx->stmt);
2711 540 : c; c = OMP_CLAUSE_CHAIN (c))
2712 : {
2713 540 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FIRSTPRIVATE)
2714 347 : continue;
2715 193 : if (maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_DECL (c), ctx)
2716 193 : == OMP_CLAUSE_DECL (detach_clause))
2717 : break;
2718 : }
2719 :
2720 189 : gcc_assert (c);
2721 189 : field = lookup_field (OMP_CLAUSE_DECL (c), ctx);
2722 :
2723 : /* Move field corresponding to the detach clause first.
2724 : This is filled by GOMP_task and needs to be in a
2725 : specific position. */
2726 189 : p = &TYPE_FIELDS (ctx->record_type);
2727 518 : while (*p)
2728 329 : if (*p == field)
2729 189 : *p = DECL_CHAIN (*p);
2730 : else
2731 140 : p = &DECL_CHAIN (*p);
2732 189 : DECL_CHAIN (field) = TYPE_FIELDS (ctx->record_type);
2733 189 : TYPE_FIELDS (ctx->record_type) = field;
2734 189 : if (ctx->srecord_type)
2735 : {
2736 3 : field = lookup_sfield (OMP_CLAUSE_DECL (c), ctx);
2737 3 : p = &TYPE_FIELDS (ctx->srecord_type);
2738 9 : while (*p)
2739 6 : if (*p == field)
2740 3 : *p = DECL_CHAIN (*p);
2741 : else
2742 3 : p = &DECL_CHAIN (*p);
2743 3 : DECL_CHAIN (field) = TYPE_FIELDS (ctx->srecord_type);
2744 3 : TYPE_FIELDS (ctx->srecord_type) = field;
2745 : }
2746 : }
2747 3455 : layout_type (ctx->record_type);
2748 3455 : fixup_child_record_type (ctx);
2749 3455 : if (ctx->srecord_type)
2750 550 : layout_type (ctx->srecord_type);
2751 3455 : tree t = fold_convert_loc (loc, long_integer_type_node,
2752 3455 : TYPE_SIZE_UNIT (ctx->record_type));
2753 3455 : if (TREE_CODE (t) != INTEGER_CST)
2754 : {
2755 21 : t = unshare_expr (t);
2756 21 : walk_tree (&t, finish_taskreg_remap, ctx, NULL);
2757 : }
2758 3455 : gimple_omp_task_set_arg_size (ctx->stmt, t);
2759 3455 : t = build_int_cst (long_integer_type_node,
2760 3455 : TYPE_ALIGN_UNIT (ctx->record_type));
2761 3455 : gimple_omp_task_set_arg_align (ctx->stmt, t);
2762 : }
2763 : }
2764 :
2765 : /* Find the enclosing offload context. */
2766 :
2767 : static omp_context *
2768 9665 : enclosing_target_ctx (omp_context *ctx)
2769 : {
2770 43359 : for (; ctx; ctx = ctx->outer)
2771 41745 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET)
2772 : break;
2773 :
2774 22111 : return ctx;
2775 : }
2776 :
2777 : /* Return whether CTX's parent compute construct is an OpenACC 'kernels'
2778 : construct.
2779 : (This doesn't include OpenACC 'kernels' decomposed parts.) */
2780 :
2781 : static bool
2782 11325 : ctx_in_oacc_kernels_region (omp_context *ctx)
2783 : {
2784 39137 : for (;ctx != NULL; ctx = ctx->outer)
2785 : {
2786 29472 : gimple *stmt = ctx->stmt;
2787 29472 : if (gimple_code (stmt) == GIMPLE_OMP_TARGET
2788 29472 : && gimple_omp_target_kind (stmt) == GF_OMP_TARGET_KIND_OACC_KERNELS)
2789 : return true;
2790 : }
2791 :
2792 : return false;
2793 : }
2794 :
2795 : /* Check the parallelism clauses inside a OpenACC 'kernels' region.
2796 : (This doesn't include OpenACC 'kernels' decomposed parts.)
2797 : Until kernels handling moves to use the same loop indirection
2798 : scheme as parallel, we need to do this checking early. */
2799 :
2800 : static unsigned
2801 7151 : check_oacc_kernel_gwv (gomp_for *stmt, omp_context *ctx)
2802 : {
2803 7151 : bool checking = true;
2804 7151 : unsigned outer_mask = 0;
2805 7151 : unsigned this_mask = 0;
2806 7151 : bool has_seq = false, has_auto = false;
2807 :
2808 7151 : if (ctx->outer)
2809 4819 : outer_mask = check_oacc_kernel_gwv (NULL, ctx->outer);
2810 7151 : if (!stmt)
2811 : {
2812 4819 : checking = false;
2813 4819 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR)
2814 : return outer_mask;
2815 1730 : stmt = as_a <gomp_for *> (ctx->stmt);
2816 : }
2817 :
2818 11330 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
2819 : {
2820 7268 : switch (OMP_CLAUSE_CODE (c))
2821 : {
2822 645 : case OMP_CLAUSE_GANG:
2823 645 : this_mask |= GOMP_DIM_MASK (GOMP_DIM_GANG);
2824 645 : break;
2825 439 : case OMP_CLAUSE_WORKER:
2826 439 : this_mask |= GOMP_DIM_MASK (GOMP_DIM_WORKER);
2827 439 : break;
2828 342 : case OMP_CLAUSE_VECTOR:
2829 342 : this_mask |= GOMP_DIM_MASK (GOMP_DIM_VECTOR);
2830 342 : break;
2831 63 : case OMP_CLAUSE_SEQ:
2832 63 : has_seq = true;
2833 63 : break;
2834 94 : case OMP_CLAUSE_AUTO:
2835 94 : has_auto = true;
2836 94 : break;
2837 : default:
2838 : break;
2839 : }
2840 : }
2841 :
2842 4062 : if (checking)
2843 : {
2844 2332 : if (has_seq && (this_mask || has_auto))
2845 36 : error_at (gimple_location (stmt), "%<seq%> overrides other"
2846 : " OpenACC loop specifiers");
2847 2296 : else if (has_auto && this_mask)
2848 30 : error_at (gimple_location (stmt), "%<auto%> conflicts with other"
2849 : " OpenACC loop specifiers");
2850 :
2851 2332 : if (this_mask & outer_mask)
2852 15 : error_at (gimple_location (stmt), "inner loop uses same"
2853 : " OpenACC parallelism as containing loop");
2854 : }
2855 :
2856 4062 : return outer_mask | this_mask;
2857 : }
2858 :
2859 : /* Scan a GIMPLE_OMP_FOR. */
2860 :
2861 : static omp_context *
2862 52983 : scan_omp_for (gomp_for *stmt, omp_context *outer_ctx)
2863 : {
2864 52983 : omp_context *ctx;
2865 52983 : size_t i;
2866 52983 : tree clauses = gimple_omp_for_clauses (stmt);
2867 :
2868 52983 : ctx = new_omp_context (stmt, outer_ctx);
2869 :
2870 52983 : if (is_gimple_omp_oacc (stmt))
2871 : {
2872 12446 : omp_context *tgt = enclosing_target_ctx (outer_ctx);
2873 :
2874 12446 : if (!(tgt && is_oacc_kernels (tgt)))
2875 32251 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2876 : {
2877 22137 : tree c_op0;
2878 22137 : switch (OMP_CLAUSE_CODE (c))
2879 : {
2880 1952 : case OMP_CLAUSE_GANG:
2881 1952 : c_op0 = OMP_CLAUSE_GANG_EXPR (c);
2882 1952 : break;
2883 :
2884 1426 : case OMP_CLAUSE_WORKER:
2885 1426 : c_op0 = OMP_CLAUSE_WORKER_EXPR (c);
2886 1426 : break;
2887 :
2888 1627 : case OMP_CLAUSE_VECTOR:
2889 1627 : c_op0 = OMP_CLAUSE_VECTOR_EXPR (c);
2890 1627 : break;
2891 :
2892 17132 : default:
2893 17132 : continue;
2894 : }
2895 :
2896 5005 : if (c_op0)
2897 : {
2898 : /* By construction, this is impossible for OpenACC 'kernels'
2899 : decomposed parts. */
2900 210 : gcc_assert (!(tgt && is_oacc_kernels_decomposed_part (tgt)));
2901 :
2902 210 : error_at (OMP_CLAUSE_LOCATION (c),
2903 : "argument not permitted on %qs clause",
2904 210 : omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
2905 210 : if (tgt)
2906 180 : inform (gimple_location (tgt->stmt),
2907 : "enclosing parent compute construct");
2908 30 : else if (oacc_get_fn_attrib (current_function_decl))
2909 30 : inform (DECL_SOURCE_LOCATION (current_function_decl),
2910 : "enclosing routine");
2911 : else
2912 0 : gcc_unreachable ();
2913 : }
2914 : }
2915 :
2916 12446 : if (tgt && is_oacc_kernels (tgt))
2917 2332 : check_oacc_kernel_gwv (stmt, ctx);
2918 :
2919 : /* Collect all variables named in reductions on this loop. Ensure
2920 : that, if this loop has a reduction on some variable v, and there is
2921 : a reduction on v somewhere in an outer context, then there is a
2922 : reduction on v on all intervening loops as well. */
2923 12446 : tree local_reduction_clauses = NULL;
2924 39374 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
2925 : {
2926 26928 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
2927 4712 : local_reduction_clauses
2928 4712 : = tree_cons (NULL, c, local_reduction_clauses);
2929 : }
2930 12446 : if (ctx->outer_reduction_clauses == NULL && ctx->outer != NULL)
2931 11920 : ctx->outer_reduction_clauses
2932 11920 : = chainon (unshare_expr (ctx->outer->local_reduction_clauses),
2933 : ctx->outer->outer_reduction_clauses);
2934 12446 : tree outer_reduction_clauses = ctx->outer_reduction_clauses;
2935 12446 : tree local_iter = local_reduction_clauses;
2936 17158 : for (; local_iter; local_iter = TREE_CHAIN (local_iter))
2937 : {
2938 4712 : tree local_clause = TREE_VALUE (local_iter);
2939 4712 : tree local_var = OMP_CLAUSE_DECL (local_clause);
2940 4712 : tree_code local_op = OMP_CLAUSE_REDUCTION_CODE (local_clause);
2941 4712 : bool have_outer_reduction = false;
2942 4712 : tree ctx_iter = outer_reduction_clauses;
2943 5826 : for (; ctx_iter; ctx_iter = TREE_CHAIN (ctx_iter))
2944 : {
2945 3325 : tree outer_clause = TREE_VALUE (ctx_iter);
2946 3325 : tree outer_var = OMP_CLAUSE_DECL (outer_clause);
2947 3325 : tree_code outer_op = OMP_CLAUSE_REDUCTION_CODE (outer_clause);
2948 3325 : if (outer_var == local_var && outer_op != local_op)
2949 : {
2950 535 : if (warning_at (OMP_CLAUSE_LOCATION (local_clause),
2951 535 : OPT_Wopenmp, "conflicting reduction "
2952 : "operations for %qE",
2953 : local_var))
2954 535 : inform (OMP_CLAUSE_LOCATION (outer_clause),
2955 : "location of the previous reduction for %qE",
2956 : outer_var);
2957 : }
2958 3325 : if (outer_var == local_var)
2959 : {
2960 : have_outer_reduction = true;
2961 : break;
2962 : }
2963 : }
2964 4712 : if (have_outer_reduction)
2965 : {
2966 : /* There is a reduction on outer_var both on this loop and on
2967 : some enclosing loop. Walk up the context tree until such a
2968 : loop with a reduction on outer_var is found, and complain
2969 : about all intervening loops that do not have such a
2970 : reduction. */
2971 2211 : struct omp_context *curr_loop = ctx->outer;
2972 2211 : bool found = false;
2973 2826 : while (curr_loop != NULL)
2974 : {
2975 2826 : tree curr_iter = curr_loop->local_reduction_clauses;
2976 3460 : for (; curr_iter; curr_iter = TREE_CHAIN (curr_iter))
2977 : {
2978 2845 : tree curr_clause = TREE_VALUE (curr_iter);
2979 2845 : tree curr_var = OMP_CLAUSE_DECL (curr_clause);
2980 2845 : if (curr_var == local_var)
2981 : {
2982 : found = true;
2983 : break;
2984 : }
2985 : }
2986 2826 : if (!found)
2987 615 : warning_at (gimple_location (curr_loop->stmt), OPT_Wopenmp,
2988 : "nested loop in reduction needs "
2989 : "reduction clause for %qE",
2990 : local_var);
2991 : else
2992 : break;
2993 615 : curr_loop = curr_loop->outer;
2994 : }
2995 : }
2996 : }
2997 12446 : ctx->local_reduction_clauses = local_reduction_clauses;
2998 12446 : ctx->outer_reduction_clauses
2999 12446 : = chainon (unshare_expr (ctx->local_reduction_clauses),
3000 : ctx->outer_reduction_clauses);
3001 :
3002 12446 : if (tgt && is_oacc_kernels (tgt))
3003 : {
3004 : /* Strip out reductions, as they are not handled yet. */
3005 : tree *prev_ptr = &clauses;
3006 :
3007 7123 : while (tree probe = *prev_ptr)
3008 : {
3009 4791 : tree *next_ptr = &OMP_CLAUSE_CHAIN (probe);
3010 :
3011 4791 : if (OMP_CLAUSE_CODE (probe) == OMP_CLAUSE_REDUCTION)
3012 718 : *prev_ptr = *next_ptr;
3013 : else
3014 : prev_ptr = next_ptr;
3015 : }
3016 :
3017 2332 : gimple_omp_for_set_clauses (stmt, clauses);
3018 : }
3019 : }
3020 :
3021 52983 : scan_sharing_clauses (clauses, ctx);
3022 :
3023 52983 : scan_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
3024 180987 : for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
3025 : {
3026 75021 : scan_omp_op (gimple_omp_for_index_ptr (stmt, i), ctx);
3027 75021 : scan_omp_op (gimple_omp_for_initial_ptr (stmt, i), ctx);
3028 75021 : scan_omp_op (gimple_omp_for_final_ptr (stmt, i), ctx);
3029 75021 : scan_omp_op (gimple_omp_for_incr_ptr (stmt, i), ctx);
3030 : }
3031 52983 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3032 52983 : return ctx;
3033 : }
3034 :
3035 : /* Duplicate #pragma omp simd, one for SIMT, another one for SIMD. */
3036 :
3037 : static void
3038 0 : scan_omp_simd (gimple_stmt_iterator *gsi, gomp_for *stmt,
3039 : omp_context *outer_ctx)
3040 : {
3041 0 : gbind *bind = gimple_build_bind (NULL, NULL, NULL);
3042 0 : gsi_replace (gsi, bind, false);
3043 0 : gimple_seq seq = NULL;
3044 0 : gimple *g = gimple_build_call_internal (IFN_GOMP_USE_SIMT, 0);
3045 0 : tree cond = create_tmp_var_raw (integer_type_node);
3046 0 : DECL_CONTEXT (cond) = current_function_decl;
3047 0 : DECL_SEEN_IN_BIND_EXPR_P (cond) = 1;
3048 0 : gimple_bind_set_vars (bind, cond);
3049 0 : gimple_call_set_lhs (g, cond);
3050 0 : gimple_seq_add_stmt (&seq, g);
3051 0 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
3052 0 : tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
3053 0 : tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
3054 0 : g = gimple_build_cond (NE_EXPR, cond, integer_zero_node, lab1, lab2);
3055 0 : gimple_seq_add_stmt (&seq, g);
3056 0 : g = gimple_build_label (lab1);
3057 0 : gimple_seq_add_stmt (&seq, g);
3058 0 : gimple_seq new_seq = copy_gimple_seq_and_replace_locals (stmt);
3059 0 : gomp_for *new_stmt = as_a <gomp_for *> (new_seq);
3060 0 : tree clause = build_omp_clause (gimple_location (stmt), OMP_CLAUSE__SIMT_);
3061 0 : OMP_CLAUSE_CHAIN (clause) = gimple_omp_for_clauses (new_stmt);
3062 0 : gimple_omp_for_set_clauses (new_stmt, clause);
3063 0 : gimple_seq_add_stmt (&seq, new_stmt);
3064 0 : g = gimple_build_goto (lab3);
3065 0 : gimple_seq_add_stmt (&seq, g);
3066 0 : g = gimple_build_label (lab2);
3067 0 : gimple_seq_add_stmt (&seq, g);
3068 0 : gimple_seq_add_stmt (&seq, stmt);
3069 0 : g = gimple_build_label (lab3);
3070 0 : gimple_seq_add_stmt (&seq, g);
3071 0 : gimple_bind_set_body (bind, seq);
3072 0 : update_stmt (bind);
3073 0 : scan_omp_for (new_stmt, outer_ctx);
3074 0 : scan_omp_for (stmt, outer_ctx)->simt_stmt = new_stmt;
3075 0 : }
3076 :
3077 : static tree omp_find_scan (gimple_stmt_iterator *, bool *,
3078 : struct walk_stmt_info *);
3079 : static omp_context *maybe_lookup_ctx (gimple *);
3080 :
3081 : /* Duplicate #pragma omp simd, one for the scan input phase loop and one
3082 : for scan phase loop. */
3083 :
3084 : static void
3085 91 : scan_omp_simd_scan (gimple_stmt_iterator *gsi, gomp_for *stmt,
3086 : omp_context *outer_ctx)
3087 : {
3088 : /* The only change between inclusive and exclusive scan will be
3089 : within the first simd loop, so just use inclusive in the
3090 : worksharing loop. */
3091 91 : outer_ctx->scan_inclusive = true;
3092 91 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_INCLUSIVE);
3093 91 : OMP_CLAUSE_DECL (c) = integer_zero_node;
3094 :
3095 91 : gomp_scan *input_stmt = gimple_build_omp_scan (NULL, NULL_TREE);
3096 91 : gomp_scan *scan_stmt = gimple_build_omp_scan (NULL, c);
3097 91 : gsi_replace (gsi, input_stmt, false);
3098 91 : gimple_seq input_body = NULL;
3099 91 : gimple_seq_add_stmt (&input_body, stmt);
3100 91 : gsi_insert_after (gsi, scan_stmt, GSI_NEW_STMT);
3101 :
3102 91 : gimple_stmt_iterator input1_gsi = gsi_none ();
3103 91 : struct walk_stmt_info wi;
3104 91 : memset (&wi, 0, sizeof (wi));
3105 91 : wi.val_only = true;
3106 91 : wi.info = (void *) &input1_gsi;
3107 91 : walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), omp_find_scan, NULL, &wi);
3108 91 : gcc_assert (!gsi_end_p (input1_gsi));
3109 :
3110 91 : gimple *input_stmt1 = gsi_stmt (input1_gsi);
3111 91 : gsi_next (&input1_gsi);
3112 91 : gimple *scan_stmt1 = gsi_stmt (input1_gsi);
3113 91 : gcc_assert (scan_stmt1 && gimple_code (scan_stmt1) == GIMPLE_OMP_SCAN);
3114 91 : c = gimple_omp_scan_clauses (as_a <gomp_scan *> (scan_stmt1));
3115 182 : if (c && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_EXCLUSIVE)
3116 : std::swap (input_stmt1, scan_stmt1);
3117 :
3118 91 : gimple_seq input_body1 = gimple_omp_body (input_stmt1);
3119 91 : gimple_omp_set_body (input_stmt1, NULL);
3120 :
3121 91 : gimple_seq scan_body = copy_gimple_seq_and_replace_locals (stmt);
3122 91 : gomp_for *new_stmt = as_a <gomp_for *> (scan_body);
3123 :
3124 91 : gimple_omp_set_body (input_stmt1, input_body1);
3125 91 : gimple_omp_set_body (scan_stmt1, NULL);
3126 :
3127 91 : gimple_stmt_iterator input2_gsi = gsi_none ();
3128 91 : memset (&wi, 0, sizeof (wi));
3129 91 : wi.val_only = true;
3130 91 : wi.info = (void *) &input2_gsi;
3131 91 : walk_gimple_seq_mod (gimple_omp_body_ptr (new_stmt), omp_find_scan,
3132 : NULL, &wi);
3133 91 : gcc_assert (!gsi_end_p (input2_gsi));
3134 :
3135 91 : gimple *input_stmt2 = gsi_stmt (input2_gsi);
3136 91 : gsi_next (&input2_gsi);
3137 91 : gimple *scan_stmt2 = gsi_stmt (input2_gsi);
3138 91 : gcc_assert (scan_stmt2 && gimple_code (scan_stmt2) == GIMPLE_OMP_SCAN);
3139 182 : if (c && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_EXCLUSIVE)
3140 : std::swap (input_stmt2, scan_stmt2);
3141 :
3142 91 : gimple_omp_set_body (input_stmt2, NULL);
3143 :
3144 91 : gimple_omp_set_body (input_stmt, input_body);
3145 91 : gimple_omp_set_body (scan_stmt, scan_body);
3146 :
3147 91 : omp_context *ctx = new_omp_context (input_stmt, outer_ctx);
3148 91 : scan_omp (gimple_omp_body_ptr (input_stmt), ctx);
3149 :
3150 91 : ctx = new_omp_context (scan_stmt, outer_ctx);
3151 91 : scan_omp (gimple_omp_body_ptr (scan_stmt), ctx);
3152 :
3153 91 : maybe_lookup_ctx (new_stmt)->for_simd_scan_phase = true;
3154 91 : }
3155 :
3156 : /* Scan an OpenMP sections directive. */
3157 :
3158 : static void
3159 589 : scan_omp_sections (gomp_sections *stmt, omp_context *outer_ctx)
3160 : {
3161 589 : omp_context *ctx;
3162 :
3163 589 : ctx = new_omp_context (stmt, outer_ctx);
3164 589 : scan_sharing_clauses (gimple_omp_sections_clauses (stmt), ctx);
3165 589 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3166 589 : }
3167 :
3168 : /* Scan an OpenMP single directive. */
3169 :
3170 : static void
3171 1252 : scan_omp_single (gomp_single *stmt, omp_context *outer_ctx)
3172 : {
3173 1252 : omp_context *ctx;
3174 1252 : tree name;
3175 :
3176 1252 : ctx = new_omp_context (stmt, outer_ctx);
3177 1252 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
3178 1252 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
3179 1252 : name = create_tmp_var_name (".omp_copy_s");
3180 1252 : name = build_decl (gimple_location (stmt),
3181 : TYPE_DECL, name, ctx->record_type);
3182 1252 : TYPE_NAME (ctx->record_type) = name;
3183 :
3184 1252 : scan_sharing_clauses (gimple_omp_single_clauses (stmt), ctx);
3185 1252 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3186 :
3187 1252 : if (TYPE_FIELDS (ctx->record_type) == NULL)
3188 1102 : ctx->record_type = NULL;
3189 : else
3190 150 : layout_type (ctx->record_type);
3191 1252 : }
3192 :
3193 : /* Scan a GIMPLE_OMP_TARGET. */
3194 :
3195 : static void
3196 42078 : scan_omp_target (gomp_target *stmt, omp_context *outer_ctx)
3197 : {
3198 42078 : omp_context *ctx;
3199 42078 : tree name;
3200 42078 : bool offloaded = is_gimple_omp_offloaded (stmt);
3201 42078 : tree clauses = gimple_omp_target_clauses (stmt);
3202 :
3203 42078 : ctx = new_omp_context (stmt, outer_ctx);
3204 42078 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
3205 42078 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
3206 42078 : name = create_tmp_var_name (".omp_data_t");
3207 42078 : name = build_decl (gimple_location (stmt),
3208 : TYPE_DECL, name, ctx->record_type);
3209 42078 : DECL_ARTIFICIAL (name) = 1;
3210 42078 : DECL_NAMELESS (name) = 1;
3211 42078 : TYPE_NAME (ctx->record_type) = name;
3212 42078 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
3213 :
3214 42078 : if (offloaded)
3215 : {
3216 24963 : tree c = omp_find_clause (clauses, OMP_CLAUSE_DEVICE_TYPE);
3217 24963 : bool host_only
3218 24963 : = c && OMP_CLAUSE_DEVICE_TYPE_KIND (c) == OMP_CLAUSE_DEVICE_TYPE_HOST;
3219 24963 : create_omp_child_function (ctx, false, host_only);
3220 24963 : gimple_omp_target_set_child_fn (stmt, ctx->cb.dst_fn);
3221 : }
3222 :
3223 42078 : scan_sharing_clauses (clauses, ctx);
3224 42078 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3225 :
3226 42078 : if (TYPE_FIELDS (ctx->record_type) == NULL)
3227 7798 : ctx->record_type = ctx->receiver_decl = NULL;
3228 : else
3229 : {
3230 34280 : TYPE_FIELDS (ctx->record_type)
3231 34280 : = nreverse (TYPE_FIELDS (ctx->record_type));
3232 34280 : if (flag_checking)
3233 : {
3234 34280 : unsigned int align = DECL_ALIGN (TYPE_FIELDS (ctx->record_type));
3235 34280 : for (tree field = TYPE_FIELDS (ctx->record_type);
3236 131941 : field;
3237 97661 : field = DECL_CHAIN (field))
3238 97661 : gcc_assert (DECL_ALIGN (field) == align);
3239 : }
3240 34280 : layout_type (ctx->record_type);
3241 34280 : if (offloaded)
3242 17720 : fixup_child_record_type (ctx);
3243 : }
3244 :
3245 42078 : if (ctx->teams_nested_p && ctx->nonteams_nested_p)
3246 : {
3247 4 : error_at (gimple_location (stmt),
3248 : "%<target%> construct with nested %<teams%> construct "
3249 : "contains directives outside of the %<teams%> construct");
3250 4 : gimple_omp_set_body (stmt, gimple_build_bind (NULL, NULL, NULL));
3251 : }
3252 42078 : }
3253 :
3254 : /* Scan an OpenMP teams directive. */
3255 :
3256 : static void
3257 8878 : scan_omp_teams (gomp_teams *stmt, omp_context *outer_ctx)
3258 : {
3259 8878 : omp_context *ctx = new_omp_context (stmt, outer_ctx);
3260 :
3261 8878 : if (!gimple_omp_teams_host (stmt))
3262 : {
3263 6157 : scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
3264 6157 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3265 6157 : return;
3266 : }
3267 2721 : taskreg_contexts.safe_push (ctx);
3268 2721 : gcc_assert (taskreg_nesting_level == 1);
3269 2721 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
3270 2721 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
3271 2721 : tree name = create_tmp_var_name (".omp_data_s");
3272 2721 : name = build_decl (gimple_location (stmt),
3273 : TYPE_DECL, name, ctx->record_type);
3274 2721 : DECL_ARTIFICIAL (name) = 1;
3275 2721 : DECL_NAMELESS (name) = 1;
3276 2721 : TYPE_NAME (ctx->record_type) = name;
3277 2721 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
3278 2721 : create_omp_child_function (ctx, false, false);
3279 2721 : gimple_omp_teams_set_child_fn (stmt, ctx->cb.dst_fn);
3280 :
3281 2721 : scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
3282 2721 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3283 :
3284 2721 : if (TYPE_FIELDS (ctx->record_type) == NULL)
3285 1455 : ctx->record_type = ctx->receiver_decl = NULL;
3286 : }
3287 :
3288 : /* Check nesting restrictions. */
3289 : static bool
3290 158419 : check_omp_nesting_restrictions (gimple *stmt, omp_context *ctx)
3291 : {
3292 158419 : tree c;
3293 :
3294 : /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
3295 : inside an OpenACC CTX. */
3296 158419 : if (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
3297 158419 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE)
3298 : /* ..., except for the atomic codes that OpenACC shares with OpenMP. */
3299 : ;
3300 284632 : else if (!(is_gimple_omp (stmt)
3301 140187 : && is_gimple_omp_oacc (stmt)))
3302 : {
3303 114181 : if (oacc_get_fn_attrib (cfun->decl) != NULL)
3304 : {
3305 48 : error_at (gimple_location (stmt),
3306 : "non-OpenACC construct inside of OpenACC routine");
3307 48 : return false;
3308 : }
3309 : else
3310 238484 : for (omp_context *octx = ctx; octx != NULL; octx = octx->outer)
3311 248878 : if (is_gimple_omp (octx->stmt)
3312 124527 : && is_gimple_omp_oacc (octx->stmt))
3313 : {
3314 176 : error_at (gimple_location (stmt),
3315 : "non-OpenACC construct inside of OpenACC region");
3316 176 : return false;
3317 : }
3318 : }
3319 :
3320 158195 : if (ctx != NULL)
3321 : {
3322 84713 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
3323 84713 : && gimple_omp_target_kind (ctx->stmt) == GF_OMP_TARGET_KIND_REGION)
3324 : {
3325 8381 : c = omp_find_clause (gimple_omp_target_clauses (ctx->stmt),
3326 : OMP_CLAUSE_DEVICE);
3327 8381 : if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
3328 : {
3329 9 : error_at (gimple_location (stmt),
3330 : "OpenMP constructs are not allowed in target region "
3331 : "with %<ancestor%>");
3332 9 : return false;
3333 : }
3334 :
3335 8372 : if (gimple_code (stmt) == GIMPLE_OMP_TEAMS && !ctx->teams_nested_p)
3336 6153 : ctx->teams_nested_p = true;
3337 : else
3338 2219 : ctx->nonteams_nested_p = true;
3339 : }
3340 84704 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_SCAN
3341 188 : && ctx->outer
3342 84892 : && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR)
3343 : ctx = ctx->outer;
3344 84704 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3345 28449 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
3346 86547 : && !ctx->loop_p)
3347 : {
3348 1623 : c = NULL_TREE;
3349 1623 : if (ctx->order_concurrent
3350 1623 : && (gimple_code (stmt) == GIMPLE_OMP_ORDERED
3351 280 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
3352 190 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE))
3353 : {
3354 210 : error_at (gimple_location (stmt),
3355 : "OpenMP constructs other than %<parallel%>, %<loop%>"
3356 : " or %<simd%> may not be nested inside a region with"
3357 : " the %<order(concurrent)%> clause");
3358 210 : return false;
3359 : }
3360 1413 : if (gimple_code (stmt) == GIMPLE_OMP_ORDERED)
3361 : {
3362 133 : c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
3363 133 : if (omp_find_clause (c, OMP_CLAUSE_SIMD))
3364 : {
3365 133 : if (omp_find_clause (c, OMP_CLAUSE_THREADS)
3366 133 : && (ctx->outer == NULL
3367 29 : || !gimple_omp_for_combined_into_p (ctx->stmt)
3368 25 : || gimple_code (ctx->outer->stmt) != GIMPLE_OMP_FOR
3369 25 : || (gimple_omp_for_kind (ctx->outer->stmt)
3370 : != GF_OMP_FOR_KIND_FOR)
3371 25 : || !gimple_omp_for_combined_p (ctx->outer->stmt)))
3372 : {
3373 8 : error_at (gimple_location (stmt),
3374 : "%<ordered simd threads%> must be closely "
3375 : "nested inside of %<%s simd%> region",
3376 8 : lang_GNU_Fortran () ? "do" : "for");
3377 8 : return false;
3378 : }
3379 : return true;
3380 : }
3381 : }
3382 : else if (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
3383 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE
3384 : || gimple_code (stmt) == GIMPLE_OMP_SCAN
3385 : || gimple_code (stmt) == GIMPLE_OMP_STRUCTURED_BLOCK)
3386 : return true;
3387 : else if (gimple_code (stmt) == GIMPLE_OMP_FOR
3388 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
3389 : return true;
3390 60 : error_at (gimple_location (stmt),
3391 : "OpenMP constructs other than "
3392 : "%<ordered simd%>, %<simd%>, %<loop%> or %<atomic%> may "
3393 : "not be nested inside %<simd%> region");
3394 60 : return false;
3395 : }
3396 83081 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
3397 : {
3398 5938 : if ((gimple_code (stmt) != GIMPLE_OMP_FOR
3399 5614 : || (gimple_omp_for_kind (stmt) != GF_OMP_FOR_KIND_DISTRIBUTE
3400 25 : && omp_find_clause (gimple_omp_for_clauses (stmt),
3401 : OMP_CLAUSE_BIND) == NULL_TREE))
3402 5954 : && gimple_code (stmt) != GIMPLE_OMP_PARALLEL)
3403 : {
3404 132 : error_at (gimple_location (stmt),
3405 : "only %<distribute%>, %<parallel%> or %<loop%> "
3406 : "regions are allowed to be strictly nested inside "
3407 : "%<teams%> region");
3408 132 : return false;
3409 : }
3410 : }
3411 77143 : else if (ctx->order_concurrent
3412 1953 : && gimple_code (stmt) != GIMPLE_OMP_PARALLEL
3413 1470 : && (gimple_code (stmt) != GIMPLE_OMP_FOR
3414 1196 : || gimple_omp_for_kind (stmt) != GF_OMP_FOR_KIND_SIMD)
3415 289 : && gimple_code (stmt) != GIMPLE_OMP_SCAN
3416 77432 : && gimple_code (stmt) != GIMPLE_OMP_STRUCTURED_BLOCK)
3417 : {
3418 285 : if (ctx->loop_p)
3419 135 : error_at (gimple_location (stmt),
3420 : "OpenMP constructs other than %<parallel%>, %<loop%> or "
3421 : "%<simd%> may not be nested inside a %<loop%> region");
3422 : else
3423 150 : error_at (gimple_location (stmt),
3424 : "OpenMP constructs other than %<parallel%>, %<loop%> or "
3425 : "%<simd%> may not be nested inside a region with "
3426 : "the %<order(concurrent)%> clause");
3427 : return false;
3428 : }
3429 : }
3430 156146 : switch (gimple_code (stmt))
3431 : {
3432 53108 : case GIMPLE_OMP_FOR:
3433 53108 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_SIMD)
3434 : return true;
3435 42216 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
3436 : {
3437 8306 : if (ctx != NULL && gimple_code (ctx->stmt) != GIMPLE_OMP_TEAMS)
3438 : {
3439 0 : error_at (gimple_location (stmt),
3440 : "%<distribute%> region must be strictly nested "
3441 : "inside %<teams%> construct");
3442 0 : return false;
3443 : }
3444 : return true;
3445 : }
3446 : /* We split taskloop into task and nested taskloop in it. */
3447 33910 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_TASKLOOP)
3448 : return true;
3449 : /* For now, hope this will change and loop bind(parallel) will not
3450 : be allowed in lots of contexts. */
3451 30776 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR
3452 49041 : && omp_find_clause (gimple_omp_for_clauses (stmt), OMP_CLAUSE_BIND))
3453 : return true;
3454 30142 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
3455 : {
3456 12511 : bool ok = false;
3457 :
3458 12511 : if (ctx)
3459 11964 : switch (gimple_code (ctx->stmt))
3460 : {
3461 4049 : case GIMPLE_OMP_FOR:
3462 4049 : ok = (gimple_omp_for_kind (ctx->stmt)
3463 : == GF_OMP_FOR_KIND_OACC_LOOP);
3464 4049 : break;
3465 :
3466 7887 : case GIMPLE_OMP_TARGET:
3467 7887 : switch (gimple_omp_target_kind (ctx->stmt))
3468 : {
3469 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
3470 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
3471 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
3472 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
3473 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
3474 : ok = true;
3475 : break;
3476 :
3477 : default:
3478 : break;
3479 : }
3480 :
3481 : default:
3482 : break;
3483 : }
3484 547 : else if (oacc_get_fn_attrib (current_function_decl))
3485 : ok = true;
3486 4049 : if (!ok)
3487 : {
3488 65 : error_at (gimple_location (stmt),
3489 : "OpenACC loop directive must be associated with"
3490 : " an OpenACC compute region");
3491 65 : return false;
3492 : }
3493 : }
3494 : /* FALLTHRU */
3495 34275 : case GIMPLE_CALL:
3496 34275 : if (is_gimple_call (stmt)
3497 34275 : && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3498 : == BUILT_IN_GOMP_CANCEL
3499 3081 : || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3500 : == BUILT_IN_GOMP_CANCELLATION_POINT))
3501 : {
3502 1925 : const char *bad = NULL;
3503 1925 : const char *kind = NULL;
3504 1925 : const char *construct
3505 1925 : = (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3506 : == BUILT_IN_GOMP_CANCEL)
3507 1925 : ? "cancel"
3508 808 : : "cancellation point";
3509 1925 : if (ctx == NULL)
3510 : {
3511 41 : error_at (gimple_location (stmt), "orphaned %qs construct",
3512 : construct);
3513 41 : return false;
3514 : }
3515 1884 : switch (tree_fits_shwi_p (gimple_call_arg (stmt, 0))
3516 1884 : ? tree_to_shwi (gimple_call_arg (stmt, 0))
3517 : : 0)
3518 : {
3519 546 : case 1:
3520 546 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_PARALLEL)
3521 : bad = "parallel";
3522 226 : else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3523 : == BUILT_IN_GOMP_CANCEL
3524 226 : && !integer_zerop (gimple_call_arg (stmt, 1)))
3525 177 : ctx->cancellable = true;
3526 : kind = "parallel";
3527 : break;
3528 427 : case 2:
3529 427 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
3530 427 : || gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR)
3531 : bad = "for";
3532 117 : else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3533 : == BUILT_IN_GOMP_CANCEL
3534 117 : && !integer_zerop (gimple_call_arg (stmt, 1)))
3535 : {
3536 106 : ctx->cancellable = true;
3537 106 : if (omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3538 : OMP_CLAUSE_NOWAIT))
3539 5 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3540 : "%<cancel for%> inside "
3541 : "%<nowait%> for construct");
3542 106 : if (omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3543 : OMP_CLAUSE_ORDERED))
3544 5 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3545 : "%<cancel for%> inside "
3546 : "%<ordered%> for construct");
3547 : }
3548 : kind = "for";
3549 : break;
3550 383 : case 4:
3551 383 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_SECTIONS
3552 383 : && gimple_code (ctx->stmt) != GIMPLE_OMP_SECTION)
3553 : bad = "sections";
3554 93 : else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3555 : == BUILT_IN_GOMP_CANCEL
3556 93 : && !integer_zerop (gimple_call_arg (stmt, 1)))
3557 : {
3558 73 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
3559 : {
3560 0 : ctx->cancellable = true;
3561 0 : if (omp_find_clause (gimple_omp_sections_clauses
3562 0 : (ctx->stmt),
3563 : OMP_CLAUSE_NOWAIT))
3564 0 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3565 : "%<cancel sections%> inside "
3566 : "%<nowait%> sections construct");
3567 : }
3568 : else
3569 : {
3570 73 : gcc_assert (ctx->outer
3571 : && gimple_code (ctx->outer->stmt)
3572 : == GIMPLE_OMP_SECTIONS);
3573 73 : ctx->outer->cancellable = true;
3574 73 : if (omp_find_clause (gimple_omp_sections_clauses
3575 73 : (ctx->outer->stmt),
3576 : OMP_CLAUSE_NOWAIT))
3577 10 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3578 : "%<cancel sections%> inside "
3579 : "%<nowait%> sections construct");
3580 : }
3581 : }
3582 : kind = "sections";
3583 : break;
3584 528 : case 8:
3585 528 : if (!is_task_ctx (ctx)
3586 528 : && (!is_taskloop_ctx (ctx)
3587 54 : || ctx->outer == NULL
3588 54 : || !is_task_ctx (ctx->outer)))
3589 : bad = "task";
3590 : else
3591 : {
3592 248 : for (omp_context *octx = ctx->outer;
3593 439 : octx; octx = octx->outer)
3594 : {
3595 425 : switch (gimple_code (octx->stmt))
3596 : {
3597 : case GIMPLE_OMP_TASKGROUP:
3598 : break;
3599 40 : case GIMPLE_OMP_TARGET:
3600 40 : if (gimple_omp_target_kind (octx->stmt)
3601 : != GF_OMP_TARGET_KIND_REGION)
3602 20 : continue;
3603 : /* FALLTHRU */
3604 99 : case GIMPLE_OMP_PARALLEL:
3605 99 : case GIMPLE_OMP_TEAMS:
3606 99 : error_at (gimple_location (stmt),
3607 : "%<%s taskgroup%> construct not closely "
3608 : "nested inside of %<taskgroup%> region",
3609 : construct);
3610 99 : return false;
3611 104 : case GIMPLE_OMP_TASK:
3612 104 : if (gimple_omp_task_taskloop_p (octx->stmt)
3613 94 : && octx->outer
3614 198 : && is_taskloop_ctx (octx->outer))
3615 : {
3616 94 : tree clauses
3617 94 : = gimple_omp_for_clauses (octx->outer->stmt);
3618 94 : if (!omp_find_clause (clauses, OMP_CLAUSE_NOGROUP))
3619 : break;
3620 : }
3621 60 : continue;
3622 111 : default:
3623 111 : continue;
3624 171 : }
3625 : break;
3626 : }
3627 149 : ctx->cancellable = true;
3628 : }
3629 : kind = "taskgroup";
3630 : break;
3631 0 : default:
3632 0 : error_at (gimple_location (stmt), "invalid arguments");
3633 0 : return false;
3634 : }
3635 341 : if (bad)
3636 : {
3637 1200 : error_at (gimple_location (stmt),
3638 : "%<%s %s%> construct not closely nested inside of %qs",
3639 : construct, kind, bad);
3640 1200 : return false;
3641 : }
3642 : }
3643 : /* FALLTHRU */
3644 : case GIMPLE_OMP_SECTIONS:
3645 : case GIMPLE_OMP_SINGLE:
3646 55767 : for (; ctx != NULL; ctx = ctx->outer)
3647 37636 : switch (gimple_code (ctx->stmt))
3648 : {
3649 6873 : case GIMPLE_OMP_FOR:
3650 6873 : if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
3651 6873 : && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
3652 : break;
3653 : /* FALLTHRU */
3654 778 : case GIMPLE_OMP_SECTIONS:
3655 778 : case GIMPLE_OMP_SINGLE:
3656 778 : case GIMPLE_OMP_ORDERED:
3657 778 : case GIMPLE_OMP_MASTER:
3658 778 : case GIMPLE_OMP_MASKED:
3659 778 : case GIMPLE_OMP_TASK:
3660 778 : case GIMPLE_OMP_CRITICAL:
3661 778 : if (is_gimple_call (stmt))
3662 : {
3663 707 : if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3664 : != BUILT_IN_GOMP_BARRIER)
3665 : return true;
3666 23 : error_at (gimple_location (stmt),
3667 : "barrier region may not be closely nested inside "
3668 : "of work-sharing, %<loop%>, %<critical%>, "
3669 : "%<ordered%>, %<master%>, %<masked%>, explicit "
3670 : "%<task%> or %<taskloop%> region");
3671 23 : return false;
3672 : }
3673 71 : error_at (gimple_location (stmt),
3674 : "work-sharing region may not be closely nested inside "
3675 : "of work-sharing, %<loop%>, %<critical%>, %<ordered%>, "
3676 : "%<master%>, %<masked%>, explicit %<task%> or "
3677 : "%<taskloop%> region");
3678 71 : return false;
3679 : case GIMPLE_OMP_PARALLEL:
3680 : case GIMPLE_OMP_TEAMS:
3681 : return true;
3682 13437 : case GIMPLE_OMP_TARGET:
3683 13437 : if (gimple_omp_target_kind (ctx->stmt)
3684 : == GF_OMP_TARGET_KIND_REGION)
3685 : return true;
3686 : break;
3687 : default:
3688 : break;
3689 : }
3690 : break;
3691 : case GIMPLE_OMP_MASTER:
3692 : case GIMPLE_OMP_MASKED:
3693 1526 : for (; ctx != NULL; ctx = ctx->outer)
3694 1017 : switch (gimple_code (ctx->stmt))
3695 : {
3696 14 : case GIMPLE_OMP_FOR:
3697 14 : if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
3698 14 : && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
3699 : break;
3700 : /* FALLTHRU */
3701 30 : case GIMPLE_OMP_SECTIONS:
3702 30 : case GIMPLE_OMP_SINGLE:
3703 30 : case GIMPLE_OMP_TASK:
3704 60 : error_at (gimple_location (stmt),
3705 : "%qs region may not be closely nested inside "
3706 : "of work-sharing, %<loop%>, explicit %<task%> or "
3707 : "%<taskloop%> region",
3708 30 : gimple_code (stmt) == GIMPLE_OMP_MASTER
3709 : ? "master" : "masked");
3710 30 : return false;
3711 : case GIMPLE_OMP_PARALLEL:
3712 : case GIMPLE_OMP_TEAMS:
3713 : return true;
3714 10 : case GIMPLE_OMP_TARGET:
3715 10 : if (gimple_omp_target_kind (ctx->stmt)
3716 : == GF_OMP_TARGET_KIND_REGION)
3717 : return true;
3718 : break;
3719 : default:
3720 : break;
3721 : }
3722 : break;
3723 : case GIMPLE_OMP_SCOPE:
3724 302 : for (; ctx != NULL; ctx = ctx->outer)
3725 183 : switch (gimple_code (ctx->stmt))
3726 : {
3727 7 : case GIMPLE_OMP_FOR:
3728 7 : if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
3729 7 : && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
3730 : break;
3731 : /* FALLTHRU */
3732 25 : case GIMPLE_OMP_SECTIONS:
3733 25 : case GIMPLE_OMP_SINGLE:
3734 25 : case GIMPLE_OMP_TASK:
3735 25 : case GIMPLE_OMP_CRITICAL:
3736 25 : case GIMPLE_OMP_ORDERED:
3737 25 : case GIMPLE_OMP_MASTER:
3738 25 : case GIMPLE_OMP_MASKED:
3739 25 : error_at (gimple_location (stmt),
3740 : "%<scope%> region may not be closely nested inside "
3741 : "of work-sharing, %<loop%>, explicit %<task%>, "
3742 : "%<taskloop%>, %<critical%>, %<ordered%>, %<master%>, "
3743 : "or %<masked%> region");
3744 25 : return false;
3745 : case GIMPLE_OMP_PARALLEL:
3746 : case GIMPLE_OMP_TEAMS:
3747 : return true;
3748 5 : case GIMPLE_OMP_TARGET:
3749 5 : if (gimple_omp_target_kind (ctx->stmt)
3750 : == GF_OMP_TARGET_KIND_REGION)
3751 : return true;
3752 : break;
3753 : default:
3754 : break;
3755 : }
3756 : break;
3757 5655 : case GIMPLE_OMP_TASK:
3758 23275 : for (c = gimple_omp_task_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
3759 17624 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS)
3760 : {
3761 4 : enum omp_clause_doacross_kind kind = OMP_CLAUSE_DOACROSS_KIND (c);
3762 4 : error_at (OMP_CLAUSE_LOCATION (c),
3763 : "%<%s(%s)%> is only allowed in %<omp ordered%>",
3764 4 : OMP_CLAUSE_DOACROSS_DEPEND (c) ? "depend" : "doacross",
3765 : kind == OMP_CLAUSE_DOACROSS_SOURCE ? "source" : "sink");
3766 4 : return false;
3767 : }
3768 : break;
3769 1635 : case GIMPLE_OMP_ORDERED:
3770 2881 : for (c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
3771 2881 : c; c = OMP_CLAUSE_CHAIN (c))
3772 : {
3773 1262 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DOACROSS)
3774 : {
3775 173 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
3776 : {
3777 4 : error_at (OMP_CLAUSE_LOCATION (c),
3778 : "invalid depend kind in omp %<ordered%> %<depend%>");
3779 4 : return false;
3780 : }
3781 169 : gcc_assert (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_THREADS
3782 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SIMD);
3783 169 : continue;
3784 : }
3785 :
3786 1089 : tree oclause;
3787 : /* Look for containing ordered(N) loop. */
3788 1089 : if (ctx == NULL
3789 1077 : || gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
3790 1089 : || (oclause
3791 1077 : = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3792 : OMP_CLAUSE_ORDERED)) == NULL_TREE)
3793 : {
3794 12 : error_at (OMP_CLAUSE_LOCATION (c),
3795 : "%<ordered%> construct with %<depend%> clause "
3796 : "must be closely nested inside an %<ordered%> loop");
3797 12 : return false;
3798 : }
3799 : }
3800 1619 : c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
3801 1619 : if (omp_find_clause (c, OMP_CLAUSE_SIMD))
3802 : {
3803 : /* ordered simd must be closely nested inside of simd region,
3804 : and simd region must not encounter constructs other than
3805 : ordered simd, therefore ordered simd may be either orphaned,
3806 : or ctx->stmt must be simd. The latter case is handled already
3807 : earlier. */
3808 36 : if (ctx != NULL)
3809 : {
3810 10 : error_at (gimple_location (stmt),
3811 : "%<ordered%> %<simd%> must be closely nested inside "
3812 : "%<simd%> region");
3813 10 : return false;
3814 : }
3815 : }
3816 1609 : for (; ctx != NULL; ctx = ctx->outer)
3817 1474 : switch (gimple_code (ctx->stmt))
3818 : {
3819 38 : case GIMPLE_OMP_CRITICAL:
3820 38 : case GIMPLE_OMP_TASK:
3821 38 : case GIMPLE_OMP_ORDERED:
3822 38 : ordered_in_taskloop:
3823 38 : error_at (gimple_location (stmt),
3824 : "%<ordered%> region may not be closely nested inside "
3825 : "of %<critical%>, %<ordered%>, explicit %<task%> or "
3826 : "%<taskloop%> region");
3827 38 : return false;
3828 1416 : case GIMPLE_OMP_FOR:
3829 1416 : if (gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_TASKLOOP)
3830 10 : goto ordered_in_taskloop;
3831 1406 : tree o;
3832 1406 : o = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3833 : OMP_CLAUSE_ORDERED);
3834 1406 : if (o == NULL)
3835 : {
3836 22 : error_at (gimple_location (stmt),
3837 : "%<ordered%> region must be closely nested inside "
3838 : "a loop region with an %<ordered%> clause");
3839 22 : return false;
3840 : }
3841 1384 : if (!gimple_omp_ordered_standalone_p (stmt))
3842 : {
3843 375 : if (OMP_CLAUSE_ORDERED_DOACROSS (o))
3844 : {
3845 10 : error_at (gimple_location (stmt),
3846 : "%<ordered%> construct without %<doacross%> or "
3847 : "%<depend%> clauses must not have the same "
3848 : "binding region as %<ordered%> construct with "
3849 : "those clauses");
3850 10 : return false;
3851 : }
3852 365 : else if (OMP_CLAUSE_ORDERED_EXPR (o))
3853 : {
3854 23 : tree co
3855 23 : = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3856 : OMP_CLAUSE_COLLAPSE);
3857 23 : HOST_WIDE_INT
3858 23 : o_n = tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (o));
3859 23 : HOST_WIDE_INT c_n = 1;
3860 23 : if (co)
3861 5 : c_n = tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (co));
3862 23 : if (o_n != c_n)
3863 : {
3864 10 : error_at (gimple_location (stmt),
3865 : "%<ordered%> construct without %<doacross%> "
3866 : "or %<depend%> clauses binds to loop where "
3867 : "%<collapse%> argument %wd is different from "
3868 : "%<ordered%> argument %wd", c_n, o_n);
3869 10 : return false;
3870 : }
3871 : }
3872 : }
3873 : return true;
3874 10 : case GIMPLE_OMP_TARGET:
3875 10 : if (gimple_omp_target_kind (ctx->stmt)
3876 : != GF_OMP_TARGET_KIND_REGION)
3877 : break;
3878 : /* FALLTHRU */
3879 30 : case GIMPLE_OMP_PARALLEL:
3880 30 : case GIMPLE_OMP_TEAMS:
3881 30 : error_at (gimple_location (stmt),
3882 : "%<ordered%> region must be closely nested inside "
3883 : "a loop region with an %<ordered%> clause");
3884 30 : return false;
3885 : default:
3886 : break;
3887 : }
3888 : break;
3889 468 : case GIMPLE_OMP_CRITICAL:
3890 468 : {
3891 468 : tree this_stmt_name
3892 468 : = gimple_omp_critical_name (as_a <gomp_critical *> (stmt));
3893 905 : for (; ctx != NULL; ctx = ctx->outer)
3894 443 : if (gomp_critical *other_crit
3895 462 : = dyn_cast <gomp_critical *> (ctx->stmt))
3896 25 : if (this_stmt_name == gimple_omp_critical_name (other_crit))
3897 : {
3898 6 : error_at (gimple_location (stmt),
3899 : "%<critical%> region may not be nested inside "
3900 : "a %<critical%> region with the same name");
3901 6 : return false;
3902 : }
3903 : }
3904 : break;
3905 8922 : case GIMPLE_OMP_TEAMS:
3906 8922 : if (ctx == NULL)
3907 : break;
3908 6201 : else if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET
3909 6201 : || (gimple_omp_target_kind (ctx->stmt)
3910 : != GF_OMP_TARGET_KIND_REGION))
3911 : {
3912 : /* Teams construct can appear either strictly nested inside of
3913 : target construct with no intervening stmts, or can be encountered
3914 : only by initial task (so must not appear inside any OpenMP
3915 : construct. */
3916 44 : error_at (gimple_location (stmt),
3917 : "%<teams%> construct must be closely nested inside of "
3918 : "%<target%> construct or not nested in any OpenMP "
3919 : "construct");
3920 44 : return false;
3921 : }
3922 : break;
3923 42461 : case GIMPLE_OMP_TARGET:
3924 190310 : for (c = gimple_omp_target_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
3925 147853 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS)
3926 : {
3927 4 : enum omp_clause_doacross_kind kind = OMP_CLAUSE_DOACROSS_KIND (c);
3928 4 : error_at (OMP_CLAUSE_LOCATION (c),
3929 : "%<depend(%s)%> is only allowed in %<omp ordered%>",
3930 : kind == OMP_CLAUSE_DOACROSS_SOURCE ? "source" : "sink");
3931 4 : return false;
3932 : }
3933 42457 : if (is_gimple_omp_offloaded (stmt)
3934 42457 : && oacc_get_fn_attrib (cfun->decl) != NULL)
3935 : {
3936 4 : error_at (gimple_location (stmt),
3937 : "OpenACC region inside of OpenACC routine, nested "
3938 : "parallelism not supported yet");
3939 4 : return false;
3940 : }
3941 49869 : for (; ctx != NULL; ctx = ctx->outer)
3942 : {
3943 7822 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET)
3944 : {
3945 768 : if (is_gimple_omp (stmt)
3946 768 : && is_gimple_omp_oacc (stmt)
3947 242 : && is_gimple_omp (ctx->stmt))
3948 : {
3949 242 : error_at (gimple_location (stmt),
3950 : "OpenACC construct inside of non-OpenACC region");
3951 242 : return false;
3952 : }
3953 526 : continue;
3954 : }
3955 :
3956 7054 : const char *stmt_name, *ctx_stmt_name;
3957 7054 : switch (gimple_omp_target_kind (stmt))
3958 : {
3959 : case GF_OMP_TARGET_KIND_REGION: stmt_name = "target"; break;
3960 399 : case GF_OMP_TARGET_KIND_DATA: stmt_name = "target data"; break;
3961 1688 : case GF_OMP_TARGET_KIND_UPDATE: stmt_name = "target update"; break;
3962 12 : case GF_OMP_TARGET_KIND_ENTER_DATA:
3963 12 : stmt_name = "target enter data"; break;
3964 18 : case GF_OMP_TARGET_KIND_EXIT_DATA:
3965 18 : stmt_name = "target exit data"; break;
3966 1442 : case GF_OMP_TARGET_KIND_OACC_PARALLEL: stmt_name = "parallel"; break;
3967 1621 : case GF_OMP_TARGET_KIND_OACC_KERNELS: stmt_name = "kernels"; break;
3968 183 : case GF_OMP_TARGET_KIND_OACC_SERIAL: stmt_name = "serial"; break;
3969 386 : case GF_OMP_TARGET_KIND_OACC_DATA: stmt_name = "data"; break;
3970 256 : case GF_OMP_TARGET_KIND_OACC_UPDATE: stmt_name = "update"; break;
3971 94 : case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
3972 94 : stmt_name = "enter data"; break;
3973 86 : case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
3974 86 : stmt_name = "exit data"; break;
3975 64 : case GF_OMP_TARGET_KIND_OACC_DECLARE: stmt_name = "declare"; break;
3976 128 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA: stmt_name = "host_data";
3977 128 : break;
3978 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
3979 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
3980 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
3981 : /* OpenACC 'kernels' decomposed parts. */
3982 1621 : stmt_name = "kernels"; break;
3983 0 : default: gcc_unreachable ();
3984 : }
3985 7054 : switch (gimple_omp_target_kind (ctx->stmt))
3986 : {
3987 : case GF_OMP_TARGET_KIND_REGION: ctx_stmt_name = "target"; break;
3988 2714 : case GF_OMP_TARGET_KIND_DATA: ctx_stmt_name = "target data"; break;
3989 35 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
3990 35 : ctx_stmt_name = "parallel"; break;
3991 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
3992 1037 : ctx_stmt_name = "kernels"; break;
3993 35 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
3994 35 : ctx_stmt_name = "serial"; break;
3995 3117 : case GF_OMP_TARGET_KIND_OACC_DATA: ctx_stmt_name = "data"; break;
3996 8 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
3997 8 : ctx_stmt_name = "host_data"; break;
3998 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
3999 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
4000 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
4001 : /* OpenACC 'kernels' decomposed parts. */
4002 1037 : ctx_stmt_name = "kernels"; break;
4003 0 : default: gcc_unreachable ();
4004 : }
4005 :
4006 : /* OpenACC/OpenMP mismatch? */
4007 14108 : if (is_gimple_omp_oacc (stmt)
4008 7054 : != is_gimple_omp_oacc (ctx->stmt))
4009 : {
4010 56 : error_at (gimple_location (stmt),
4011 : "%s %qs construct inside of %s %qs region",
4012 : (is_gimple_omp_oacc (stmt)
4013 : ? "OpenACC" : "OpenMP"), stmt_name,
4014 : (is_gimple_omp_oacc (ctx->stmt)
4015 : ? "OpenACC" : "OpenMP"), ctx_stmt_name);
4016 28 : return false;
4017 : }
4018 7026 : if (is_gimple_omp_offloaded (ctx->stmt))
4019 : {
4020 : /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
4021 185 : if (is_gimple_omp_oacc (ctx->stmt))
4022 : {
4023 105 : error_at (gimple_location (stmt),
4024 : "%qs construct inside of %qs region",
4025 : stmt_name, ctx_stmt_name);
4026 105 : return false;
4027 : }
4028 : else
4029 : {
4030 80 : if ((gimple_omp_target_kind (ctx->stmt)
4031 : == GF_OMP_TARGET_KIND_REGION)
4032 80 : && (gimple_omp_target_kind (stmt)
4033 : == GF_OMP_TARGET_KIND_REGION))
4034 : {
4035 58 : c = omp_find_clause (gimple_omp_target_clauses (stmt),
4036 : OMP_CLAUSE_DEVICE);
4037 58 : if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
4038 : break;
4039 : }
4040 49 : warning_at (gimple_location (stmt), OPT_Wopenmp,
4041 : "%qs construct inside of %qs region",
4042 : stmt_name, ctx_stmt_name);
4043 : }
4044 : }
4045 : }
4046 : break;
4047 : default:
4048 : break;
4049 : }
4050 : return true;
4051 : }
4052 :
4053 :
4054 : /* Helper function scan_omp.
4055 :
4056 : Callback for walk_tree or operators in walk_gimple_stmt used to
4057 : scan for OMP directives in TP. */
4058 :
4059 : static tree
4060 9930159 : scan_omp_1_op (tree *tp, int *walk_subtrees, void *data)
4061 : {
4062 9930159 : struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
4063 9930159 : omp_context *ctx = (omp_context *) wi->info;
4064 9930159 : tree t = *tp;
4065 9930159 : tree tmp;
4066 :
4067 9930159 : switch (TREE_CODE (t))
4068 : {
4069 4203310 : case VAR_DECL:
4070 4203310 : case PARM_DECL:
4071 4203310 : case LABEL_DECL:
4072 4203310 : case RESULT_DECL:
4073 4203310 : if (ctx)
4074 : {
4075 2139957 : tmp = NULL_TREE;
4076 2139957 : if (TREE_CODE (t) == VAR_DECL
4077 3833479 : && (tmp = lookup_attribute ("omp allocate var",
4078 1693522 : DECL_ATTRIBUTES (t))) != NULL_TREE)
4079 105 : t = TREE_VALUE (TREE_VALUE (tmp));
4080 2139957 : tree repl = remap_decl (t, &ctx->cb);
4081 2139957 : gcc_checking_assert (TREE_CODE (repl) != ERROR_MARK);
4082 2139957 : if (tmp != NULL_TREE && t != repl)
4083 51 : *tp = build_fold_addr_expr (repl);
4084 2139906 : else if (tmp == NULL_TREE)
4085 2139852 : *tp = repl;
4086 : }
4087 : break;
4088 :
4089 197459 : case INDIRECT_REF:
4090 197459 : case MEM_REF:
4091 197459 : if (ctx
4092 78466 : && TREE_CODE (TREE_OPERAND (t, 0)) == VAR_DECL
4093 256827 : && ((tmp = lookup_attribute ("omp allocate var",
4094 59368 : DECL_ATTRIBUTES (TREE_OPERAND (t, 0))))
4095 : != NULL_TREE))
4096 : {
4097 261 : tmp = TREE_VALUE (TREE_VALUE (tmp));
4098 261 : tree repl = remap_decl (tmp, &ctx->cb);
4099 261 : gcc_checking_assert (TREE_CODE (repl) != ERROR_MARK);
4100 261 : if (tmp != repl)
4101 245 : *tp = repl;
4102 : break;
4103 : }
4104 5726588 : gcc_fallthrough ();
4105 :
4106 5726588 : default:
4107 5726588 : if (ctx && TYPE_P (t))
4108 5 : *tp = remap_type (t, &ctx->cb);
4109 5726583 : else if (!DECL_P (t))
4110 : {
4111 4750902 : *walk_subtrees = 1;
4112 4750902 : if (ctx)
4113 : {
4114 1436695 : tree tem = remap_type (TREE_TYPE (t), &ctx->cb);
4115 1436695 : if (tem != TREE_TYPE (t))
4116 : {
4117 9946 : if (TREE_CODE (t) == INTEGER_CST)
4118 4202 : *tp = wide_int_to_tree (tem, wi::to_wide (t));
4119 : else
4120 5744 : TREE_TYPE (t) = tem;
4121 : }
4122 : }
4123 : }
4124 : break;
4125 : }
4126 :
4127 9930159 : return NULL_TREE;
4128 : }
4129 :
4130 : /* Return true if FNDECL is a setjmp or a longjmp. */
4131 :
4132 : static bool
4133 3086 : setjmp_or_longjmp_p (const_tree fndecl)
4134 : {
4135 3086 : if (fndecl_built_in_p (fndecl, BUILT_IN_SETJMP, BUILT_IN_LONGJMP))
4136 : return true;
4137 :
4138 3086 : tree declname = DECL_NAME (fndecl);
4139 3086 : if (!declname
4140 3086 : || (DECL_CONTEXT (fndecl) != NULL_TREE
4141 2391 : && TREE_CODE (DECL_CONTEXT (fndecl)) != TRANSLATION_UNIT_DECL)
4142 5913 : || !TREE_PUBLIC (fndecl))
4143 : return false;
4144 :
4145 2665 : const char *name = IDENTIFIER_POINTER (declname);
4146 2665 : return !strcmp (name, "setjmp") || !strcmp (name, "longjmp");
4147 : }
4148 :
4149 : /* Helper function for scan_omp.
4150 :
4151 : Callback for walk_gimple_stmt used to scan for OMP directives in
4152 : the current statement in GSI. */
4153 :
4154 : static tree
4155 3458031 : scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
4156 : struct walk_stmt_info *wi)
4157 : {
4158 3458031 : gimple *stmt = gsi_stmt (*gsi);
4159 3458031 : omp_context *ctx = (omp_context *) wi->info;
4160 :
4161 3458031 : if (gimple_has_location (stmt))
4162 2819100 : input_location = gimple_location (stmt);
4163 :
4164 : /* Check the nesting restrictions. */
4165 3458031 : bool remove = false;
4166 3458031 : if (is_gimple_omp (stmt))
4167 154161 : remove = !check_omp_nesting_restrictions (stmt, ctx);
4168 3303870 : else if (is_gimple_call (stmt))
4169 : {
4170 239866 : tree fndecl = gimple_call_fndecl (stmt);
4171 239866 : if (fndecl)
4172 : {
4173 222752 : if (ctx
4174 68869 : && gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
4175 14776 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
4176 3086 : && setjmp_or_longjmp_p (fndecl)
4177 222760 : && !ctx->loop_p)
4178 : {
4179 4 : remove = true;
4180 4 : error_at (gimple_location (stmt),
4181 : "setjmp/longjmp inside %<simd%> construct");
4182 : }
4183 222748 : else if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
4184 66498 : switch (DECL_FUNCTION_CODE (fndecl))
4185 : {
4186 4258 : case BUILT_IN_GOMP_BARRIER:
4187 4258 : case BUILT_IN_GOMP_CANCEL:
4188 4258 : case BUILT_IN_GOMP_CANCELLATION_POINT:
4189 4258 : case BUILT_IN_GOMP_TASKYIELD:
4190 4258 : case BUILT_IN_GOMP_TASKWAIT:
4191 4258 : case BUILT_IN_GOMP_TASKGROUP_START:
4192 4258 : case BUILT_IN_GOMP_TASKGROUP_END:
4193 4258 : remove = !check_omp_nesting_restrictions (stmt, ctx);
4194 4258 : break;
4195 : default:
4196 : break;
4197 : }
4198 156250 : else if (ctx)
4199 : {
4200 46682 : omp_context *octx = ctx;
4201 46682 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_SCAN && ctx->outer)
4202 46682 : octx = ctx->outer;
4203 46682 : if (octx->order_concurrent && omp_runtime_api_call (fndecl))
4204 : {
4205 240 : remove = true;
4206 240 : error_at (gimple_location (stmt),
4207 : "OpenMP runtime API call %qD in a region with "
4208 : "%<order(concurrent)%> clause", fndecl);
4209 : }
4210 46682 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
4211 3343 : && omp_runtime_api_call (fndecl)
4212 342 : && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
4213 : != strlen ("omp_get_num_teams"))
4214 188 : || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
4215 : "omp_get_num_teams") != 0)
4216 169 : && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
4217 : != strlen ("omp_get_num_teams_dim"))
4218 6 : || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
4219 : "omp_get_num_teams_dim") != 0)
4220 163 : && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
4221 : != strlen ("omp_get_team_num"))
4222 124 : || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
4223 : "omp_get_team_num") != 0)
4224 46721 : && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
4225 : != strlen ("omp_get_team_num_dim"))
4226 12 : || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
4227 : "omp_get_team_num_dim") != 0))
4228 : {
4229 27 : remove = true;
4230 27 : error_at (gimple_location (stmt),
4231 : "OpenMP runtime API call %qD strictly nested in a "
4232 : "%<teams%> region", fndecl);
4233 : }
4234 46682 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
4235 13440 : && (gimple_omp_target_kind (ctx->stmt)
4236 : == GF_OMP_TARGET_KIND_REGION)
4237 51735 : && omp_runtime_api_call (fndecl))
4238 : {
4239 396 : tree tgt_clauses = gimple_omp_target_clauses (ctx->stmt);
4240 396 : tree c = omp_find_clause (tgt_clauses, OMP_CLAUSE_DEVICE);
4241 396 : if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
4242 5 : error_at (gimple_location (stmt),
4243 : "OpenMP runtime API call %qD in a region with "
4244 : "%<device(ancestor)%> clause", fndecl);
4245 : }
4246 : }
4247 : }
4248 : }
4249 205105 : if (remove)
4250 : {
4251 3326 : stmt = gimple_build_nop ();
4252 3326 : gsi_replace (gsi, stmt, false);
4253 : }
4254 :
4255 3458031 : *handled_ops_p = true;
4256 :
4257 3458031 : switch (gimple_code (stmt))
4258 : {
4259 18393 : case GIMPLE_OMP_PARALLEL:
4260 18393 : taskreg_nesting_level++;
4261 18393 : scan_omp_parallel (gsi, ctx);
4262 18393 : taskreg_nesting_level--;
4263 18393 : break;
4264 :
4265 5651 : case GIMPLE_OMP_TASK:
4266 5651 : taskreg_nesting_level++;
4267 5651 : scan_omp_task (gsi, ctx);
4268 5651 : taskreg_nesting_level--;
4269 5651 : break;
4270 :
4271 53074 : case GIMPLE_OMP_FOR:
4272 53074 : if ((gimple_omp_for_kind (as_a <gomp_for *> (stmt))
4273 : == GF_OMP_FOR_KIND_SIMD)
4274 10946 : && gimple_omp_for_combined_into_p (stmt)
4275 60809 : && gimple_code (ctx->stmt) != GIMPLE_OMP_SCAN)
4276 : {
4277 7553 : tree clauses = gimple_omp_for_clauses (as_a <gomp_for *> (stmt));
4278 7553 : tree c = omp_find_clause (clauses, OMP_CLAUSE_REDUCTION);
4279 7553 : if (c && OMP_CLAUSE_REDUCTION_INSCAN (c) && !seen_error ())
4280 : {
4281 91 : scan_omp_simd_scan (gsi, as_a <gomp_for *> (stmt), ctx);
4282 91 : break;
4283 : }
4284 : }
4285 52983 : if ((gimple_omp_for_kind (as_a <gomp_for *> (stmt))
4286 : == GF_OMP_FOR_KIND_SIMD)
4287 10855 : && omp_maybe_offloaded_ctx (ctx)
4288 3656 : && omp_max_simt_vf ()
4289 52983 : && gimple_omp_for_collapse (stmt) == 1)
4290 0 : scan_omp_simd (gsi, as_a <gomp_for *> (stmt), ctx);
4291 : else
4292 52983 : scan_omp_for (as_a <gomp_for *> (stmt), ctx);
4293 : break;
4294 :
4295 212 : case GIMPLE_OMP_SCOPE:
4296 212 : ctx = new_omp_context (stmt, ctx);
4297 212 : scan_sharing_clauses (gimple_omp_scope_clauses (stmt), ctx);
4298 212 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4299 212 : break;
4300 :
4301 852 : case GIMPLE_OMP_DISPATCH:
4302 852 : ctx = new_omp_context (stmt, ctx);
4303 852 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4304 852 : break;
4305 :
4306 612 : case GIMPLE_OMP_INTEROP:
4307 612 : ctx = new_omp_context (stmt, ctx);
4308 612 : break;
4309 :
4310 589 : case GIMPLE_OMP_SECTIONS:
4311 589 : scan_omp_sections (as_a <gomp_sections *> (stmt), ctx);
4312 589 : break;
4313 :
4314 1252 : case GIMPLE_OMP_SINGLE:
4315 1252 : scan_omp_single (as_a <gomp_single *> (stmt), ctx);
4316 1252 : break;
4317 :
4318 1324 : case GIMPLE_OMP_SCAN:
4319 1324 : if (tree clauses = gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt)))
4320 : {
4321 626 : if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_INCLUSIVE)
4322 324 : ctx->scan_inclusive = true;
4323 302 : else if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_EXCLUSIVE)
4324 302 : ctx->scan_exclusive = true;
4325 : }
4326 : /* FALLTHRU */
4327 6265 : case GIMPLE_OMP_SECTION:
4328 6265 : case GIMPLE_OMP_STRUCTURED_BLOCK:
4329 6265 : case GIMPLE_OMP_MASTER:
4330 6265 : case GIMPLE_OMP_ORDERED:
4331 6265 : case GIMPLE_OMP_CRITICAL:
4332 6265 : ctx = new_omp_context (stmt, ctx);
4333 6265 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4334 6265 : break;
4335 :
4336 460 : case GIMPLE_OMP_MASKED:
4337 460 : ctx = new_omp_context (stmt, ctx);
4338 460 : scan_sharing_clauses (gimple_omp_masked_clauses (stmt), ctx);
4339 460 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4340 460 : break;
4341 :
4342 607 : case GIMPLE_OMP_TASKGROUP:
4343 607 : ctx = new_omp_context (stmt, ctx);
4344 607 : scan_sharing_clauses (gimple_omp_taskgroup_clauses (stmt), ctx);
4345 607 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4346 607 : break;
4347 :
4348 42078 : case GIMPLE_OMP_TARGET:
4349 42078 : if (is_gimple_omp_offloaded (stmt))
4350 : {
4351 24963 : taskreg_nesting_level++;
4352 24963 : scan_omp_target (as_a <gomp_target *> (stmt), ctx);
4353 24963 : taskreg_nesting_level--;
4354 : }
4355 : else
4356 17115 : scan_omp_target (as_a <gomp_target *> (stmt), ctx);
4357 : break;
4358 :
4359 8878 : case GIMPLE_OMP_TEAMS:
4360 8878 : if (gimple_omp_teams_host (as_a <gomp_teams *> (stmt)))
4361 : {
4362 2721 : taskreg_nesting_level++;
4363 2721 : scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
4364 2721 : taskreg_nesting_level--;
4365 : }
4366 : else
4367 6157 : scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
4368 : break;
4369 :
4370 288094 : case GIMPLE_BIND:
4371 288094 : {
4372 288094 : tree var;
4373 :
4374 288094 : *handled_ops_p = false;
4375 288094 : if (ctx)
4376 154097 : for (var = gimple_bind_vars (as_a <gbind *> (stmt));
4377 582779 : var ;
4378 428682 : var = DECL_CHAIN (var))
4379 428682 : insert_decl_map (&ctx->cb, var, var);
4380 : }
4381 : break;
4382 3031014 : default:
4383 3031014 : *handled_ops_p = false;
4384 3031014 : break;
4385 : }
4386 :
4387 3458031 : return NULL_TREE;
4388 : }
4389 :
4390 :
4391 : /* Scan all the statements starting at the current statement. CTX
4392 : contains context information about the OMP directives and
4393 : clauses found during the scan. */
4394 :
4395 : static void
4396 263142 : scan_omp (gimple_seq *body_p, omp_context *ctx)
4397 : {
4398 263142 : location_t saved_location;
4399 263142 : struct walk_stmt_info wi;
4400 :
4401 263142 : memset (&wi, 0, sizeof (wi));
4402 263142 : wi.info = ctx;
4403 263142 : wi.want_locations = true;
4404 :
4405 263142 : saved_location = input_location;
4406 263142 : walk_gimple_seq_mod (body_p, scan_omp_1_stmt, scan_omp_1_op, &wi);
4407 263142 : input_location = saved_location;
4408 263142 : }
4409 :
4410 : /* Re-gimplification and code generation routines. */
4411 :
4412 : /* Remove omp_member_access_dummy_var variables from gimple_bind_vars
4413 : of BIND if in a method. */
4414 :
4415 : static void
4416 260605 : maybe_remove_omp_member_access_dummy_vars (gbind *bind)
4417 : {
4418 260605 : if (DECL_ARGUMENTS (current_function_decl)
4419 98671 : && DECL_ARTIFICIAL (DECL_ARGUMENTS (current_function_decl))
4420 271359 : && (TREE_CODE (TREE_TYPE (DECL_ARGUMENTS (current_function_decl)))
4421 : == POINTER_TYPE))
4422 : {
4423 3520 : tree vars = gimple_bind_vars (bind);
4424 19484 : for (tree *pvar = &vars; *pvar; )
4425 15964 : if (omp_member_access_dummy_var (*pvar))
4426 719 : *pvar = DECL_CHAIN (*pvar);
4427 : else
4428 15245 : pvar = &DECL_CHAIN (*pvar);
4429 3520 : gimple_bind_set_vars (bind, vars);
4430 : }
4431 260605 : }
4432 :
4433 : /* Remove omp_member_access_dummy_var variables from BLOCK_VARS of
4434 : block and its subblocks. */
4435 :
4436 : static void
4437 14494 : remove_member_access_dummy_vars (tree block)
4438 : {
4439 25745 : for (tree *pvar = &BLOCK_VARS (block); *pvar; )
4440 11251 : if (omp_member_access_dummy_var (*pvar))
4441 108 : *pvar = DECL_CHAIN (*pvar);
4442 : else
4443 11143 : pvar = &DECL_CHAIN (*pvar);
4444 :
4445 19150 : for (block = BLOCK_SUBBLOCKS (block); block; block = BLOCK_CHAIN (block))
4446 4656 : remove_member_access_dummy_vars (block);
4447 14494 : }
4448 :
4449 : /* If a context was created for STMT when it was scanned, return it. */
4450 :
4451 : static omp_context *
4452 122167 : maybe_lookup_ctx (gimple *stmt)
4453 : {
4454 122167 : splay_tree_node n;
4455 122167 : n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt);
4456 122167 : return n ? (omp_context *) n->value : NULL;
4457 : }
4458 :
4459 :
4460 : /* Find the mapping for DECL in CTX or the immediately enclosing
4461 : context that has a mapping for DECL.
4462 :
4463 : If CTX is a nested parallel directive, we may have to use the decl
4464 : mappings created in CTX's parent context. Suppose that we have the
4465 : following parallel nesting (variable UIDs showed for clarity):
4466 :
4467 : iD.1562 = 0;
4468 : #omp parallel shared(iD.1562) -> outer parallel
4469 : iD.1562 = iD.1562 + 1;
4470 :
4471 : #omp parallel shared (iD.1562) -> inner parallel
4472 : iD.1562 = iD.1562 - 1;
4473 :
4474 : Each parallel structure will create a distinct .omp_data_s structure
4475 : for copying iD.1562 in/out of the directive:
4476 :
4477 : outer parallel .omp_data_s.1.i -> iD.1562
4478 : inner parallel .omp_data_s.2.i -> iD.1562
4479 :
4480 : A shared variable mapping will produce a copy-out operation before
4481 : the parallel directive and a copy-in operation after it. So, in
4482 : this case we would have:
4483 :
4484 : iD.1562 = 0;
4485 : .omp_data_o.1.i = iD.1562;
4486 : #omp parallel shared(iD.1562) -> outer parallel
4487 : .omp_data_i.1 = &.omp_data_o.1
4488 : .omp_data_i.1->i = .omp_data_i.1->i + 1;
4489 :
4490 : .omp_data_o.2.i = iD.1562; -> **
4491 : #omp parallel shared(iD.1562) -> inner parallel
4492 : .omp_data_i.2 = &.omp_data_o.2
4493 : .omp_data_i.2->i = .omp_data_i.2->i - 1;
4494 :
4495 :
4496 : ** This is a problem. The symbol iD.1562 cannot be referenced
4497 : inside the body of the outer parallel region. But since we are
4498 : emitting this copy operation while expanding the inner parallel
4499 : directive, we need to access the CTX structure of the outer
4500 : parallel directive to get the correct mapping:
4501 :
4502 : .omp_data_o.2.i = .omp_data_i.1->i
4503 :
4504 : Since there may be other workshare or parallel directives enclosing
4505 : the parallel directive, it may be necessary to walk up the context
4506 : parent chain. This is not a problem in general because nested
4507 : parallelism happens only rarely. */
4508 :
4509 : static tree
4510 126935 : lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
4511 : {
4512 126935 : tree t;
4513 126935 : omp_context *up;
4514 :
4515 183920 : for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
4516 56985 : t = maybe_lookup_decl (decl, up);
4517 :
4518 126935 : gcc_assert (!ctx->is_nested || t || is_global_var (decl));
4519 :
4520 98436 : return t ? t : decl;
4521 : }
4522 :
4523 :
4524 : /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
4525 : in outer contexts. */
4526 :
4527 : static tree
4528 358704 : maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
4529 : {
4530 358704 : tree t = NULL;
4531 358704 : omp_context *up;
4532 :
4533 630595 : for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
4534 271891 : t = maybe_lookup_decl (decl, up);
4535 :
4536 358704 : return t ? t : decl;
4537 : }
4538 :
4539 :
4540 : /* Construct the initialization value for reduction operation OP. */
4541 :
4542 : tree
4543 13271 : omp_reduction_init_op (location_t loc, enum tree_code op, tree type)
4544 : {
4545 13271 : switch (op)
4546 : {
4547 10937 : case PLUS_EXPR:
4548 10937 : case MINUS_EXPR:
4549 10937 : case BIT_IOR_EXPR:
4550 10937 : case BIT_XOR_EXPR:
4551 10937 : case TRUTH_OR_EXPR:
4552 10937 : case TRUTH_ORIF_EXPR:
4553 10937 : case TRUTH_XOR_EXPR:
4554 10937 : case NE_EXPR:
4555 10937 : return build_zero_cst (type);
4556 :
4557 1430 : case MULT_EXPR:
4558 1430 : case TRUTH_AND_EXPR:
4559 1430 : case TRUTH_ANDIF_EXPR:
4560 1430 : case EQ_EXPR:
4561 1430 : return fold_convert_loc (loc, type, integer_one_node);
4562 :
4563 208 : case BIT_AND_EXPR:
4564 208 : return fold_convert_loc (loc, type, integer_minus_one_node);
4565 :
4566 388 : case MAX_EXPR:
4567 388 : if (SCALAR_FLOAT_TYPE_P (type))
4568 : {
4569 158 : REAL_VALUE_TYPE min;
4570 158 : if (HONOR_INFINITIES (type))
4571 158 : real_arithmetic (&min, NEGATE_EXPR, &dconstinf, NULL);
4572 : else
4573 0 : real_maxval (&min, 1, TYPE_MODE (type));
4574 158 : return build_real (type, min);
4575 : }
4576 230 : else if (POINTER_TYPE_P (type))
4577 : {
4578 2 : wide_int min
4579 2 : = wi::min_value (TYPE_PRECISION (type), TYPE_SIGN (type));
4580 2 : return wide_int_to_tree (type, min);
4581 2 : }
4582 : else
4583 : {
4584 228 : gcc_assert (INTEGRAL_TYPE_P (type));
4585 228 : return TYPE_MIN_VALUE (type);
4586 : }
4587 :
4588 308 : case MIN_EXPR:
4589 308 : if (SCALAR_FLOAT_TYPE_P (type))
4590 : {
4591 108 : REAL_VALUE_TYPE max;
4592 108 : if (HONOR_INFINITIES (type))
4593 108 : max = dconstinf;
4594 : else
4595 0 : real_maxval (&max, 0, TYPE_MODE (type));
4596 108 : return build_real (type, max);
4597 : }
4598 200 : else if (POINTER_TYPE_P (type))
4599 : {
4600 2 : wide_int max
4601 2 : = wi::max_value (TYPE_PRECISION (type), TYPE_SIGN (type));
4602 2 : return wide_int_to_tree (type, max);
4603 2 : }
4604 : else
4605 : {
4606 198 : gcc_assert (INTEGRAL_TYPE_P (type));
4607 198 : return TYPE_MAX_VALUE (type);
4608 : }
4609 :
4610 0 : default:
4611 0 : gcc_unreachable ();
4612 : }
4613 : }
4614 :
4615 : /* Construct the initialization value for reduction CLAUSE. */
4616 :
4617 : tree
4618 10651 : omp_reduction_init (tree clause, tree type)
4619 : {
4620 10651 : return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause),
4621 10651 : OMP_CLAUSE_REDUCTION_CODE (clause), type);
4622 : }
4623 :
4624 : /* Return alignment to be assumed for var in CLAUSE, which should be
4625 : OMP_CLAUSE_ALIGNED. */
4626 :
4627 : static tree
4628 137 : omp_clause_aligned_alignment (tree clause)
4629 : {
4630 137 : if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause))
4631 120 : return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause);
4632 :
4633 : /* Otherwise return implementation defined alignment. */
4634 17 : unsigned int al = 1;
4635 17 : opt_scalar_mode mode_iter;
4636 17 : auto_vector_modes modes;
4637 17 : targetm.vectorize.autovectorize_vector_modes (&modes, true);
4638 17 : static enum mode_class classes[]
4639 : = { MODE_INT, MODE_VECTOR_INT, MODE_FLOAT, MODE_VECTOR_FLOAT };
4640 68 : for (int i = 0; i < 4; i += 2)
4641 : /* The for loop above dictates that we only walk through scalar classes. */
4642 255 : FOR_EACH_MODE_IN_CLASS (mode_iter, classes[i])
4643 : {
4644 221 : scalar_mode mode = mode_iter.require ();
4645 221 : machine_mode vmode = targetm.vectorize.preferred_simd_mode (mode);
4646 221 : if (GET_MODE_CLASS (vmode) != classes[i + 1])
4647 102 : continue;
4648 : machine_mode alt_vmode;
4649 476 : for (unsigned int j = 0; j < modes.length (); ++j)
4650 527 : if (related_vector_mode (modes[j], mode).exists (&alt_vmode)
4651 578 : && known_ge (GET_MODE_SIZE (alt_vmode), GET_MODE_SIZE (vmode)))
4652 119 : vmode = alt_vmode;
4653 :
4654 119 : tree type = lang_hooks.types.type_for_mode (mode, 1);
4655 119 : if (type == NULL_TREE || TYPE_MODE (type) != mode)
4656 17 : continue;
4657 102 : type = build_vector_type_for_mode (type, vmode);
4658 102 : if (TYPE_MODE (type) != vmode)
4659 0 : continue;
4660 102 : if (TYPE_ALIGN_UNIT (type) > al)
4661 17 : al = TYPE_ALIGN_UNIT (type);
4662 : }
4663 17 : return build_int_cst (integer_type_node, al);
4664 17 : }
4665 :
4666 :
4667 : /* This structure is part of the interface between lower_rec_simd_input_clauses
4668 : and lower_rec_input_clauses. */
4669 :
4670 : class omplow_simd_context {
4671 : public:
4672 77766 : omplow_simd_context () { memset (this, 0, sizeof (*this)); }
4673 : tree idx;
4674 : tree lane;
4675 : tree lastlane;
4676 : vec<tree, va_heap> simt_eargs;
4677 : gimple_seq simt_dlist;
4678 : poly_uint64 max_vf;
4679 : bool is_simt;
4680 : };
4681 :
4682 : /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
4683 : privatization. */
4684 :
4685 : static bool
4686 10247 : lower_rec_simd_input_clauses (tree new_var, omp_context *ctx,
4687 : omplow_simd_context *sctx, tree &ivar,
4688 : tree &lvar, tree *rvar = NULL,
4689 : tree *rvar2 = NULL)
4690 : {
4691 10247 : if (known_eq (sctx->max_vf, 0U))
4692 : {
4693 4809 : sctx->max_vf = (sctx->is_simt ? omp_max_simt_vf ()
4694 4809 : : omp_max_vf (omp_maybe_offloaded_ctx (ctx)));
4695 4809 : if (maybe_gt (sctx->max_vf, 1U))
4696 : {
4697 3455 : tree c = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
4698 : OMP_CLAUSE_SAFELEN);
4699 3455 : if (c)
4700 : {
4701 88 : poly_uint64 safe_len;
4702 88 : if (!poly_int_tree_p (OMP_CLAUSE_SAFELEN_EXPR (c), &safe_len)
4703 88 : || maybe_lt (safe_len, 1U))
4704 0 : sctx->max_vf = 1;
4705 : else
4706 88 : sctx->max_vf = lower_bound (sctx->max_vf, safe_len);
4707 : }
4708 : }
4709 4809 : if (sctx->is_simt && !known_eq (sctx->max_vf, 1U))
4710 : {
4711 0 : for (tree c = gimple_omp_for_clauses (ctx->stmt); c;
4712 0 : c = OMP_CLAUSE_CHAIN (c))
4713 : {
4714 0 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
4715 0 : continue;
4716 :
4717 0 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
4718 : {
4719 : /* UDR reductions are not supported yet for SIMT, disable
4720 : SIMT. */
4721 0 : sctx->max_vf = 1;
4722 0 : break;
4723 : }
4724 :
4725 0 : if (truth_value_p (OMP_CLAUSE_REDUCTION_CODE (c))
4726 0 : && !INTEGRAL_TYPE_P (TREE_TYPE (new_var)))
4727 : {
4728 : /* Doing boolean operations on non-integral types is
4729 : for conformance only, it's not worth supporting this
4730 : for SIMT. */
4731 0 : sctx->max_vf = 1;
4732 0 : break;
4733 : }
4734 : }
4735 : }
4736 4809 : if (maybe_gt (sctx->max_vf, 1U))
4737 : {
4738 3442 : sctx->idx = create_tmp_var (unsigned_type_node);
4739 3442 : sctx->lane = create_tmp_var (unsigned_type_node);
4740 : }
4741 : }
4742 10247 : if (known_eq (sctx->max_vf, 1U))
4743 : return false;
4744 :
4745 7183 : if (sctx->is_simt)
4746 : {
4747 0 : if (is_gimple_reg (new_var))
4748 : {
4749 0 : ivar = lvar = new_var;
4750 0 : return true;
4751 : }
4752 0 : tree type = TREE_TYPE (new_var), ptype = build_pointer_type (type);
4753 0 : ivar = lvar = create_tmp_var (type);
4754 0 : TREE_ADDRESSABLE (ivar) = 1;
4755 0 : DECL_ATTRIBUTES (ivar) = tree_cons (get_identifier ("omp simt private"),
4756 0 : NULL, DECL_ATTRIBUTES (ivar));
4757 0 : sctx->simt_eargs.safe_push (build1 (ADDR_EXPR, ptype, ivar));
4758 0 : tree clobber = build_clobber (type);
4759 0 : gimple *g = gimple_build_assign (ivar, clobber);
4760 0 : gimple_seq_add_stmt (&sctx->simt_dlist, g);
4761 : }
4762 : else
4763 : {
4764 7183 : tree atype = build_array_type_nelts (TREE_TYPE (new_var), sctx->max_vf);
4765 7183 : tree avar = create_tmp_var_raw (atype);
4766 7183 : if (TREE_ADDRESSABLE (new_var))
4767 920 : TREE_ADDRESSABLE (avar) = 1;
4768 7183 : DECL_ATTRIBUTES (avar)
4769 7183 : = tree_cons (get_identifier ("omp simd array"), NULL,
4770 7183 : DECL_ATTRIBUTES (avar));
4771 7183 : gimple_add_tmp_var (avar);
4772 7183 : tree iavar = avar;
4773 7183 : if (rvar && !ctx->for_simd_scan_phase)
4774 : {
4775 : /* For inscan reductions, create another array temporary,
4776 : which will hold the reduced value. */
4777 232 : iavar = create_tmp_var_raw (atype);
4778 232 : if (TREE_ADDRESSABLE (new_var))
4779 33 : TREE_ADDRESSABLE (iavar) = 1;
4780 232 : DECL_ATTRIBUTES (iavar)
4781 232 : = tree_cons (get_identifier ("omp simd array"), NULL,
4782 : tree_cons (get_identifier ("omp simd inscan"), NULL,
4783 232 : DECL_ATTRIBUTES (iavar)));
4784 232 : gimple_add_tmp_var (iavar);
4785 232 : ctx->cb.decl_map->put (avar, iavar);
4786 232 : if (sctx->lastlane == NULL_TREE)
4787 184 : sctx->lastlane = create_tmp_var (unsigned_type_node);
4788 232 : *rvar = build4 (ARRAY_REF, TREE_TYPE (new_var), iavar,
4789 : sctx->lastlane, NULL_TREE, NULL_TREE);
4790 232 : TREE_THIS_NOTRAP (*rvar) = 1;
4791 :
4792 232 : if (ctx->scan_exclusive)
4793 : {
4794 : /* And for exclusive scan yet another one, which will
4795 : hold the value during the scan phase. */
4796 114 : tree savar = create_tmp_var_raw (atype);
4797 114 : if (TREE_ADDRESSABLE (new_var))
4798 17 : TREE_ADDRESSABLE (savar) = 1;
4799 114 : DECL_ATTRIBUTES (savar)
4800 114 : = tree_cons (get_identifier ("omp simd array"), NULL,
4801 : tree_cons (get_identifier ("omp simd inscan "
4802 : "exclusive"), NULL,
4803 114 : DECL_ATTRIBUTES (savar)));
4804 114 : gimple_add_tmp_var (savar);
4805 114 : ctx->cb.decl_map->put (iavar, savar);
4806 114 : *rvar2 = build4 (ARRAY_REF, TREE_TYPE (new_var), savar,
4807 : sctx->idx, NULL_TREE, NULL_TREE);
4808 114 : TREE_THIS_NOTRAP (*rvar2) = 1;
4809 : }
4810 : }
4811 7183 : ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), iavar, sctx->idx,
4812 : NULL_TREE, NULL_TREE);
4813 7183 : lvar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, sctx->lane,
4814 : NULL_TREE, NULL_TREE);
4815 7183 : TREE_THIS_NOTRAP (ivar) = 1;
4816 7183 : TREE_THIS_NOTRAP (lvar) = 1;
4817 : }
4818 7183 : if (DECL_P (new_var))
4819 : {
4820 7029 : SET_DECL_VALUE_EXPR (new_var, lvar);
4821 7029 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
4822 : }
4823 : return true;
4824 : }
4825 :
4826 : /* Helper function of lower_rec_input_clauses. For a reference
4827 : in simd reduction, add an underlying variable it will reference. */
4828 :
4829 : static void
4830 64 : handle_simd_reference (location_t loc, tree new_vard, gimple_seq *ilist)
4831 : {
4832 64 : tree z = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard)));
4833 64 : if (TREE_CONSTANT (z))
4834 : {
4835 64 : z = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard)),
4836 : get_name (new_vard));
4837 64 : gimple_add_tmp_var (z);
4838 64 : TREE_ADDRESSABLE (z) = 1;
4839 64 : z = build_fold_addr_expr_loc (loc, z);
4840 64 : gimplify_assign (new_vard, z, ilist);
4841 : }
4842 64 : }
4843 :
4844 : /* Helper function for lower_rec_input_clauses. Emit into ilist sequence
4845 : code to emit (type) (tskred_temp[idx]). */
4846 :
4847 : static tree
4848 1133 : task_reduction_read (gimple_seq *ilist, tree tskred_temp, tree type,
4849 : unsigned idx)
4850 : {
4851 1133 : unsigned HOST_WIDE_INT sz
4852 1133 : = tree_to_uhwi (TYPE_SIZE_UNIT (pointer_sized_int_node));
4853 1133 : tree r = build2 (MEM_REF, pointer_sized_int_node,
4854 1133 : tskred_temp, build_int_cst (TREE_TYPE (tskred_temp),
4855 1133 : idx * sz));
4856 1133 : tree v = create_tmp_var (pointer_sized_int_node);
4857 1133 : gimple *g = gimple_build_assign (v, r);
4858 1133 : gimple_seq_add_stmt (ilist, g);
4859 1133 : if (!useless_type_conversion_p (type, pointer_sized_int_node))
4860 : {
4861 885 : v = create_tmp_var (type);
4862 885 : g = gimple_build_assign (v, NOP_EXPR, gimple_assign_lhs (g));
4863 885 : gimple_seq_add_stmt (ilist, g);
4864 : }
4865 1133 : return v;
4866 : }
4867 :
4868 : /* Lower early initialization of privatized variable NEW_VAR
4869 : if it needs an allocator (has allocate clause). */
4870 :
4871 : static bool
4872 145317 : lower_private_allocate (tree var, tree new_var, tree &allocator,
4873 : tree &allocate_ptr, gimple_seq *ilist,
4874 : omp_context *ctx, bool is_ref, tree size)
4875 : {
4876 145317 : if (allocator)
4877 : return false;
4878 145214 : gcc_assert (allocate_ptr == NULL_TREE);
4879 145214 : if (ctx->allocate_map
4880 3240 : && (DECL_P (new_var) || (TYPE_P (new_var) && size)))
4881 3237 : if (tree *allocatorp = ctx->allocate_map->get (var))
4882 1257 : allocator = *allocatorp;
4883 145214 : if (allocator == NULL_TREE)
4884 : return false;
4885 1257 : if (!is_ref && omp_privatize_by_reference (var))
4886 : {
4887 0 : allocator = NULL_TREE;
4888 0 : return false;
4889 : }
4890 :
4891 1257 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET && DECL_P (allocator))
4892 : {
4893 4 : allocator = lookup_decl (allocator, ctx);
4894 4 : gcc_checking_assert (allocator != NULL_TREE);
4895 : }
4896 :
4897 1257 : unsigned HOST_WIDE_INT ialign = 0;
4898 1257 : if (TREE_CODE (allocator) == TREE_LIST)
4899 : {
4900 248 : ialign = tree_to_uhwi (TREE_VALUE (allocator));
4901 248 : allocator = TREE_PURPOSE (allocator);
4902 : }
4903 1257 : if (TREE_CODE (allocator) != INTEGER_CST)
4904 443 : allocator = build_outer_var_ref (allocator, ctx, OMP_CLAUSE_ALLOCATE);
4905 1257 : allocator = fold_convert (pointer_sized_int_node, allocator);
4906 1257 : if (TREE_CODE (allocator) != INTEGER_CST)
4907 : {
4908 443 : tree var = create_tmp_var (TREE_TYPE (allocator));
4909 443 : gimplify_assign (var, allocator, ilist);
4910 443 : allocator = var;
4911 : }
4912 :
4913 1257 : tree ptr_type, align, sz = size;
4914 1257 : if (TYPE_P (new_var))
4915 : {
4916 300 : ptr_type = build_pointer_type (new_var);
4917 300 : ialign = MAX (ialign, TYPE_ALIGN_UNIT (new_var));
4918 : }
4919 957 : else if (is_ref)
4920 : {
4921 123 : ptr_type = build_pointer_type (TREE_TYPE (TREE_TYPE (new_var)));
4922 123 : ialign = MAX (ialign, TYPE_ALIGN_UNIT (TREE_TYPE (ptr_type)));
4923 : }
4924 : else
4925 : {
4926 834 : ptr_type = build_pointer_type (TREE_TYPE (new_var));
4927 834 : ialign = MAX (ialign, DECL_ALIGN_UNIT (new_var));
4928 834 : if (sz == NULL_TREE)
4929 729 : sz = fold_convert (size_type_node, DECL_SIZE_UNIT (new_var));
4930 : }
4931 1257 : align = build_int_cst (size_type_node, ialign);
4932 1257 : if (TREE_CODE (sz) != INTEGER_CST)
4933 : {
4934 54 : tree szvar = create_tmp_var (size_type_node);
4935 54 : gimplify_assign (szvar, sz, ilist);
4936 54 : sz = szvar;
4937 : }
4938 1257 : allocate_ptr = create_tmp_var (ptr_type);
4939 1257 : tree a = builtin_decl_explicit (BUILT_IN_GOMP_ALLOC);
4940 1257 : gimple *g = gimple_build_call (a, 3, align, sz, allocator);
4941 1257 : gimple_call_set_lhs (g, allocate_ptr);
4942 1257 : gimple_seq_add_stmt (ilist, g);
4943 1257 : if (!is_ref)
4944 : {
4945 834 : tree x = build_simple_mem_ref (allocate_ptr);
4946 834 : TREE_THIS_NOTRAP (x) = 1;
4947 834 : SET_DECL_VALUE_EXPR (new_var, x);
4948 834 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
4949 : }
4950 : return true;
4951 : }
4952 :
4953 : /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
4954 : from the receiver (aka child) side and initializers for REFERENCE_TYPE
4955 : private variables. Initialization statements go in ILIST, while calls
4956 : to destructors go in DLIST. */
4957 :
4958 : static void
4959 77766 : lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist,
4960 : omp_context *ctx, struct omp_for_data *fd)
4961 : {
4962 77766 : tree c, copyin_seq, x, ptr;
4963 77766 : bool copyin_by_ref = false;
4964 77766 : bool lastprivate_firstprivate = false;
4965 77766 : bool reduction_omp_orig_ref = false;
4966 77766 : int pass;
4967 77766 : bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
4968 77766 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD);
4969 77766 : omplow_simd_context sctx = omplow_simd_context ();
4970 77766 : tree simt_lane = NULL_TREE, simtrec = NULL_TREE;
4971 77766 : tree ivar = NULL_TREE, lvar = NULL_TREE, uid = NULL_TREE;
4972 77766 : gimple_seq llist[4] = { };
4973 77766 : tree nonconst_simd_if = NULL_TREE;
4974 :
4975 77766 : copyin_seq = NULL;
4976 77766 : sctx.is_simt = is_simd && omp_find_clause (clauses, OMP_CLAUSE__SIMT_);
4977 :
4978 : /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
4979 : with data sharing clauses referencing variable sized vars. That
4980 : is unnecessarily hard to support and very unlikely to result in
4981 : vectorized code anyway. */
4982 9390 : if (is_simd)
4983 61587 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
4984 52197 : switch (OMP_CLAUSE_CODE (c))
4985 : {
4986 7613 : case OMP_CLAUSE_LINEAR:
4987 7613 : if (OMP_CLAUSE_LINEAR_ARRAY (c))
4988 216 : sctx.max_vf = 1;
4989 : /* FALLTHRU */
4990 25196 : case OMP_CLAUSE_PRIVATE:
4991 25196 : case OMP_CLAUSE_FIRSTPRIVATE:
4992 25196 : case OMP_CLAUSE_LASTPRIVATE:
4993 25196 : if (is_variable_sized (OMP_CLAUSE_DECL (c)))
4994 0 : sctx.max_vf = 1;
4995 25196 : else if (omp_privatize_by_reference (OMP_CLAUSE_DECL (c)))
4996 : {
4997 279 : tree rtype = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
4998 279 : if (!TREE_CONSTANT (TYPE_SIZE_UNIT (rtype)))
4999 96 : sctx.max_vf = 1;
5000 : }
5001 : break;
5002 2595 : case OMP_CLAUSE_REDUCTION:
5003 2595 : case OMP_CLAUSE_IN_REDUCTION:
5004 2595 : if (TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF
5005 2595 : || is_variable_sized (OMP_CLAUSE_DECL (c)))
5006 180 : sctx.max_vf = 1;
5007 2415 : else if (omp_privatize_by_reference (OMP_CLAUSE_DECL (c)))
5008 : {
5009 144 : tree rtype = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
5010 144 : if (!TREE_CONSTANT (TYPE_SIZE_UNIT (rtype)))
5011 0 : sctx.max_vf = 1;
5012 : }
5013 : break;
5014 759 : case OMP_CLAUSE_IF:
5015 759 : if (integer_zerop (OMP_CLAUSE_IF_EXPR (c)))
5016 124 : sctx.max_vf = 1;
5017 635 : else if (TREE_CODE (OMP_CLAUSE_IF_EXPR (c)) != INTEGER_CST)
5018 625 : nonconst_simd_if = OMP_CLAUSE_IF_EXPR (c);
5019 : break;
5020 623 : case OMP_CLAUSE_SIMDLEN:
5021 623 : if (integer_onep (OMP_CLAUSE_SIMDLEN_EXPR (c)))
5022 110 : sctx.max_vf = 1;
5023 : break;
5024 184 : case OMP_CLAUSE__CONDTEMP_:
5025 : /* FIXME: lastprivate(conditional:) not handled for SIMT yet. */
5026 184 : if (sctx.is_simt)
5027 0 : sctx.max_vf = 1;
5028 : break;
5029 22840 : default:
5030 22840 : continue;
5031 22840 : }
5032 :
5033 : /* Add a placeholder for simduid. */
5034 77766 : if (sctx.is_simt && maybe_ne (sctx.max_vf, 1U))
5035 0 : sctx.simt_eargs.safe_push (NULL_TREE);
5036 :
5037 77766 : unsigned task_reduction_cnt = 0;
5038 77766 : unsigned task_reduction_cntorig = 0;
5039 77766 : unsigned task_reduction_cnt_full = 0;
5040 77766 : unsigned task_reduction_cntorig_full = 0;
5041 77766 : unsigned task_reduction_other_cnt = 0;
5042 77766 : tree tskred_atype = NULL_TREE, tskred_avar = NULL_TREE;
5043 77766 : tree tskred_base = NULL_TREE, tskred_temp = NULL_TREE;
5044 : /* Do all the fixed sized types in the first pass, and the variable sized
5045 : types in the second pass. This makes sure that the scalar arguments to
5046 : the variable sized types are processed before we use them in the
5047 : variable sized operations. For task reductions we use 4 passes, in the
5048 : first two we ignore them, in the third one gather arguments for
5049 : GOMP_task_reduction_remap call and in the last pass actually handle
5050 : the task reductions. */
5051 472778 : for (pass = 0; pass < ((task_reduction_cnt || task_reduction_other_cnt)
5052 236389 : ? 4 : 2); ++pass)
5053 : {
5054 158627 : if (pass == 2 && task_reduction_cnt)
5055 : {
5056 878 : tskred_atype
5057 878 : = build_array_type_nelts (ptr_type_node, task_reduction_cnt
5058 878 : + task_reduction_cntorig);
5059 878 : tskred_avar = create_tmp_var_raw (tskred_atype);
5060 878 : gimple_add_tmp_var (tskred_avar);
5061 878 : TREE_ADDRESSABLE (tskred_avar) = 1;
5062 878 : task_reduction_cnt_full = task_reduction_cnt;
5063 878 : task_reduction_cntorig_full = task_reduction_cntorig;
5064 : }
5065 157749 : else if (pass == 3 && task_reduction_cnt)
5066 : {
5067 878 : x = builtin_decl_explicit (BUILT_IN_GOMP_TASK_REDUCTION_REMAP);
5068 878 : gimple *g
5069 878 : = gimple_build_call (x, 3, size_int (task_reduction_cnt),
5070 878 : size_int (task_reduction_cntorig),
5071 : build_fold_addr_expr (tskred_avar));
5072 878 : gimple_seq_add_stmt (ilist, g);
5073 : }
5074 158627 : if (pass == 3 && task_reduction_other_cnt)
5075 : {
5076 : /* For reduction clauses, build
5077 : tskred_base = (void *) tskred_temp[2]
5078 : + omp_get_thread_num () * tskred_temp[1]
5079 : or if tskred_temp[1] is known to be constant, that constant
5080 : directly. This is the start of the private reduction copy block
5081 : for the current thread. */
5082 849 : tree v = create_tmp_var (integer_type_node);
5083 849 : x = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
5084 849 : gimple *g = gimple_build_call (x, 0);
5085 849 : gimple_call_set_lhs (g, v);
5086 849 : gimple_seq_add_stmt (ilist, g);
5087 849 : c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
5088 849 : tskred_temp = OMP_CLAUSE_DECL (c);
5089 849 : if (is_taskreg_ctx (ctx))
5090 559 : tskred_temp = lookup_decl (tskred_temp, ctx);
5091 849 : tree v2 = create_tmp_var (sizetype);
5092 849 : g = gimple_build_assign (v2, NOP_EXPR, v);
5093 849 : gimple_seq_add_stmt (ilist, g);
5094 849 : if (ctx->task_reductions[0])
5095 832 : v = fold_convert (sizetype, ctx->task_reductions[0]);
5096 : else
5097 17 : v = task_reduction_read (ilist, tskred_temp, sizetype, 1);
5098 849 : tree v3 = create_tmp_var (sizetype);
5099 849 : g = gimple_build_assign (v3, MULT_EXPR, v2, v);
5100 849 : gimple_seq_add_stmt (ilist, g);
5101 849 : v = task_reduction_read (ilist, tskred_temp, ptr_type_node, 2);
5102 849 : tskred_base = create_tmp_var (ptr_type_node);
5103 849 : g = gimple_build_assign (tskred_base, POINTER_PLUS_EXPR, v, v3);
5104 849 : gimple_seq_add_stmt (ilist, g);
5105 : }
5106 158627 : task_reduction_cnt = 0;
5107 158627 : task_reduction_cntorig = 0;
5108 158627 : task_reduction_other_cnt = 0;
5109 765452 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
5110 : {
5111 606829 : enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c);
5112 606829 : tree var, new_var;
5113 606829 : bool by_ref;
5114 606829 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
5115 606829 : bool task_reduction_p = false;
5116 606829 : bool task_reduction_needs_orig_p = false;
5117 606829 : tree cond = NULL_TREE;
5118 606829 : tree allocator, allocate_ptr;
5119 :
5120 606829 : switch (c_kind)
5121 : {
5122 136504 : case OMP_CLAUSE_PRIVATE:
5123 136504 : if (OMP_CLAUSE_PRIVATE_DEBUG (c))
5124 436719 : continue;
5125 : break;
5126 60228 : case OMP_CLAUSE_SHARED:
5127 : /* Ignore shared directives in teams construct inside
5128 : of target construct. */
5129 60228 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
5130 60228 : && !is_host_teams_ctx (ctx))
5131 23462 : continue;
5132 36766 : if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL)
5133 : {
5134 12080 : gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c)
5135 : || is_global_var (OMP_CLAUSE_DECL (c)));
5136 12080 : continue;
5137 : }
5138 : case OMP_CLAUSE_FIRSTPRIVATE:
5139 : case OMP_CLAUSE_COPYIN:
5140 : break;
5141 15680 : case OMP_CLAUSE_LINEAR:
5142 15680 : if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c)
5143 15680 : && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
5144 : lastprivate_firstprivate = true;
5145 : break;
5146 37676 : case OMP_CLAUSE_REDUCTION:
5147 37676 : case OMP_CLAUSE_IN_REDUCTION:
5148 37676 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
5149 29328 : || is_task_ctx (ctx)
5150 64664 : || OMP_CLAUSE_REDUCTION_TASK (c))
5151 : {
5152 12948 : task_reduction_p = true;
5153 12948 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
5154 : {
5155 4600 : task_reduction_other_cnt++;
5156 4600 : if (pass == 2)
5157 1150 : continue;
5158 : }
5159 : else
5160 8348 : task_reduction_cnt++;
5161 11798 : if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5162 : {
5163 756 : var = OMP_CLAUSE_DECL (c);
5164 : /* If var is a global variable that isn't privatized
5165 : in outer contexts, we don't need to look up the
5166 : original address, it is always the address of the
5167 : global variable itself. */
5168 756 : if (!DECL_P (var)
5169 332 : || omp_privatize_by_reference (var)
5170 1013 : || !is_global_var
5171 257 : (maybe_lookup_decl_in_outer_ctx (var, ctx)))
5172 : {
5173 640 : task_reduction_needs_orig_p = true;
5174 640 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5175 532 : task_reduction_cntorig++;
5176 : }
5177 : }
5178 : }
5179 24728 : else if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5180 359292 : reduction_omp_orig_ref = true;
5181 : break;
5182 3396 : case OMP_CLAUSE__REDUCTEMP_:
5183 3396 : if (!is_taskreg_ctx (ctx))
5184 1160 : continue;
5185 : /* FALLTHRU */
5186 111856 : case OMP_CLAUSE__LOOPTEMP_:
5187 : /* Handle _looptemp_/_reductemp_ clauses only on
5188 : parallel/task. */
5189 111856 : if (fd)
5190 69442 : continue;
5191 : break;
5192 43724 : case OMP_CLAUSE_LASTPRIVATE:
5193 43724 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
5194 : {
5195 2066 : lastprivate_firstprivate = true;
5196 2066 : if (pass != 0 || is_taskloop_ctx (ctx))
5197 1347 : continue;
5198 : }
5199 : /* Even without corresponding firstprivate, if
5200 : decl is Fortran allocatable, it needs outer var
5201 : reference. */
5202 41658 : else if (pass == 0
5203 62451 : && lang_hooks.decls.omp_private_outer_ref
5204 20793 : (OMP_CLAUSE_DECL (c)))
5205 : lastprivate_firstprivate = true;
5206 : break;
5207 274 : case OMP_CLAUSE_ALIGNED:
5208 274 : if (pass != 1)
5209 137 : continue;
5210 137 : var = OMP_CLAUSE_DECL (c);
5211 137 : if (TREE_CODE (TREE_TYPE (var)) == POINTER_TYPE
5212 137 : && !is_global_var (var))
5213 : {
5214 73 : new_var = maybe_lookup_decl (var, ctx);
5215 73 : if (new_var == NULL_TREE)
5216 6 : new_var = maybe_lookup_decl_in_outer_ctx (var, ctx);
5217 73 : x = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
5218 73 : tree alarg = omp_clause_aligned_alignment (c);
5219 73 : alarg = fold_convert_loc (clause_loc, size_type_node, alarg);
5220 73 : x = build_call_expr_loc (clause_loc, x, 2, new_var, alarg);
5221 73 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
5222 73 : x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
5223 73 : gimplify_and_add (x, ilist);
5224 : }
5225 64 : else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
5226 64 : && is_global_var (var))
5227 : {
5228 64 : tree ptype = build_pointer_type (TREE_TYPE (var)), t, t2;
5229 64 : new_var = lookup_decl (var, ctx);
5230 64 : t = maybe_lookup_decl_in_outer_ctx (var, ctx);
5231 64 : t = build_fold_addr_expr_loc (clause_loc, t);
5232 64 : t2 = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
5233 64 : tree alarg = omp_clause_aligned_alignment (c);
5234 64 : alarg = fold_convert_loc (clause_loc, size_type_node, alarg);
5235 64 : t = build_call_expr_loc (clause_loc, t2, 2, t, alarg);
5236 64 : t = fold_convert_loc (clause_loc, ptype, t);
5237 64 : x = create_tmp_var (ptype);
5238 64 : t = build2 (MODIFY_EXPR, ptype, x, t);
5239 64 : gimplify_and_add (t, ilist);
5240 64 : t = build_simple_mem_ref_loc (clause_loc, x);
5241 64 : SET_DECL_VALUE_EXPR (new_var, t);
5242 64 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5243 : }
5244 137 : continue;
5245 1514 : case OMP_CLAUSE__CONDTEMP_:
5246 1514 : if (is_parallel_ctx (ctx)
5247 1514 : || (is_simd && !OMP_CLAUSE__CONDTEMP__ITER (c)))
5248 : break;
5249 1094 : continue;
5250 136604 : default:
5251 136604 : continue;
5252 137835 : }
5253 :
5254 359292 : if (task_reduction_p != (pass >= 2))
5255 15708 : continue;
5256 :
5257 343584 : allocator = NULL_TREE;
5258 343584 : allocate_ptr = NULL_TREE;
5259 343584 : new_var = var = OMP_CLAUSE_DECL (c);
5260 343584 : if ((c_kind == OMP_CLAUSE_REDUCTION
5261 343584 : || c_kind == OMP_CLAUSE_IN_REDUCTION)
5262 29998 : && TREE_CODE (var) == MEM_REF)
5263 : {
5264 4506 : var = TREE_OPERAND (var, 0);
5265 4506 : if (TREE_CODE (var) == POINTER_PLUS_EXPR)
5266 773 : var = TREE_OPERAND (var, 0);
5267 4506 : if (TREE_CODE (var) == INDIRECT_REF
5268 4282 : || TREE_CODE (var) == ADDR_EXPR)
5269 2578 : var = TREE_OPERAND (var, 0);
5270 4506 : if (is_variable_sized (var))
5271 : {
5272 179 : gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
5273 179 : var = DECL_VALUE_EXPR (var);
5274 179 : gcc_assert (TREE_CODE (var) == INDIRECT_REF);
5275 179 : var = TREE_OPERAND (var, 0);
5276 179 : gcc_assert (DECL_P (var));
5277 : }
5278 4506 : new_var = var;
5279 : }
5280 29998 : if (c_kind == OMP_CLAUSE_IN_REDUCTION && is_omp_target (ctx->stmt))
5281 : {
5282 844 : splay_tree_key key = (splay_tree_key) &DECL_CONTEXT (var);
5283 844 : new_var = (tree) splay_tree_lookup (ctx->field_map, key)->value;
5284 : }
5285 342740 : else if (c_kind != OMP_CLAUSE_COPYIN)
5286 341712 : new_var = lookup_decl (var, ctx);
5287 :
5288 343584 : if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN)
5289 : {
5290 25038 : if (pass != 0)
5291 12518 : continue;
5292 : }
5293 : /* C/C++ array section reductions. */
5294 318546 : else if ((c_kind == OMP_CLAUSE_REDUCTION
5295 : || c_kind == OMP_CLAUSE_IN_REDUCTION)
5296 318546 : && var != OMP_CLAUSE_DECL (c))
5297 : {
5298 4506 : if (pass == 0)
5299 948 : continue;
5300 :
5301 3558 : tree bias = TREE_OPERAND (OMP_CLAUSE_DECL (c), 1);
5302 3558 : tree orig_var = TREE_OPERAND (OMP_CLAUSE_DECL (c), 0);
5303 :
5304 3558 : if (TREE_CODE (orig_var) == POINTER_PLUS_EXPR)
5305 : {
5306 723 : tree b = TREE_OPERAND (orig_var, 1);
5307 723 : if (is_omp_target (ctx->stmt))
5308 : b = NULL_TREE;
5309 : else
5310 723 : b = maybe_lookup_decl (b, ctx);
5311 723 : if (b == NULL)
5312 : {
5313 17 : b = TREE_OPERAND (orig_var, 1);
5314 17 : b = maybe_lookup_decl_in_outer_ctx (b, ctx);
5315 : }
5316 723 : if (integer_zerop (bias))
5317 : bias = b;
5318 : else
5319 : {
5320 0 : bias = fold_convert_loc (clause_loc,
5321 0 : TREE_TYPE (b), bias);
5322 0 : bias = fold_build2_loc (clause_loc, PLUS_EXPR,
5323 0 : TREE_TYPE (b), b, bias);
5324 : }
5325 723 : orig_var = TREE_OPERAND (orig_var, 0);
5326 : }
5327 3558 : if (pass == 2)
5328 : {
5329 1149 : tree out = maybe_lookup_decl_in_outer_ctx (var, ctx);
5330 1149 : if (is_global_var (out)
5331 241 : && TREE_CODE (TREE_TYPE (out)) != POINTER_TYPE
5332 1336 : && (TREE_CODE (TREE_TYPE (out)) != REFERENCE_TYPE
5333 0 : || (TREE_CODE (TREE_TYPE (TREE_TYPE (out)))
5334 : != POINTER_TYPE)))
5335 : x = var;
5336 962 : else if (is_omp_target (ctx->stmt))
5337 : x = out;
5338 : else
5339 : {
5340 866 : bool by_ref = use_pointer_for_field (var, NULL);
5341 866 : x = build_receiver_ref (var, by_ref, ctx);
5342 866 : if (TREE_CODE (TREE_TYPE (var)) == REFERENCE_TYPE
5343 866 : && (TREE_CODE (TREE_TYPE (TREE_TYPE (var)))
5344 : == POINTER_TYPE))
5345 32 : x = build_fold_addr_expr (x);
5346 : }
5347 1149 : if (TREE_CODE (orig_var) == INDIRECT_REF)
5348 40 : x = build_simple_mem_ref (x);
5349 1109 : else if (TREE_CODE (orig_var) == ADDR_EXPR)
5350 : {
5351 646 : if (var == TREE_OPERAND (orig_var, 0))
5352 583 : x = build_fold_addr_expr (x);
5353 : }
5354 1149 : bias = fold_convert (sizetype, bias);
5355 1149 : x = fold_convert (ptr_type_node, x);
5356 1149 : x = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
5357 1149 : TREE_TYPE (x), x, bias);
5358 1149 : unsigned cnt = task_reduction_cnt - 1;
5359 1149 : if (!task_reduction_needs_orig_p)
5360 1061 : cnt += (task_reduction_cntorig_full
5361 1061 : - task_reduction_cntorig);
5362 : else
5363 88 : cnt = task_reduction_cntorig - 1;
5364 1149 : tree r = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5365 1149 : size_int (cnt), NULL_TREE, NULL_TREE);
5366 1149 : gimplify_assign (r, x, ilist);
5367 1149 : continue;
5368 1149 : }
5369 :
5370 2409 : if (TREE_CODE (orig_var) == INDIRECT_REF
5371 2297 : || TREE_CODE (orig_var) == ADDR_EXPR)
5372 1398 : orig_var = TREE_OPERAND (orig_var, 0);
5373 2409 : tree d = OMP_CLAUSE_DECL (c);
5374 2409 : tree type = TREE_TYPE (d);
5375 2409 : gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
5376 2409 : tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
5377 2409 : tree sz = v;
5378 2409 : const char *name = get_name (orig_var);
5379 2409 : if (pass != 3 && !TREE_CONSTANT (v))
5380 : {
5381 93 : tree t;
5382 93 : if (is_omp_target (ctx->stmt))
5383 : t = NULL_TREE;
5384 : else
5385 93 : t = maybe_lookup_decl (v, ctx);
5386 93 : if (t)
5387 88 : v = t;
5388 : else
5389 5 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
5390 93 : gimplify_expr (&v, ilist, NULL, is_gimple_val, fb_rvalue);
5391 186 : t = fold_build2_loc (clause_loc, PLUS_EXPR,
5392 93 : TREE_TYPE (v), v,
5393 93 : build_int_cst (TREE_TYPE (v), 1));
5394 93 : sz = fold_build2_loc (clause_loc, MULT_EXPR,
5395 93 : TREE_TYPE (v), t,
5396 93 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5397 : }
5398 2409 : if (pass == 3)
5399 : {
5400 1461 : tree xv = create_tmp_var (ptr_type_node);
5401 1461 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5402 : {
5403 1149 : unsigned cnt = task_reduction_cnt - 1;
5404 1149 : if (!task_reduction_needs_orig_p)
5405 1061 : cnt += (task_reduction_cntorig_full
5406 1061 : - task_reduction_cntorig);
5407 : else
5408 88 : cnt = task_reduction_cntorig - 1;
5409 1149 : x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5410 1149 : size_int (cnt), NULL_TREE, NULL_TREE);
5411 :
5412 1149 : gimple *g = gimple_build_assign (xv, x);
5413 1149 : gimple_seq_add_stmt (ilist, g);
5414 : }
5415 : else
5416 : {
5417 312 : unsigned int idx = *ctx->task_reduction_map->get (c);
5418 312 : tree off;
5419 312 : if (ctx->task_reductions[1 + idx])
5420 81 : off = fold_convert (sizetype,
5421 : ctx->task_reductions[1 + idx]);
5422 : else
5423 231 : off = task_reduction_read (ilist, tskred_temp, sizetype,
5424 231 : 7 + 3 * idx + 1);
5425 312 : gimple *g = gimple_build_assign (xv, POINTER_PLUS_EXPR,
5426 : tskred_base, off);
5427 312 : gimple_seq_add_stmt (ilist, g);
5428 : }
5429 1461 : x = fold_convert (build_pointer_type (boolean_type_node),
5430 : xv);
5431 1461 : if (TREE_CONSTANT (v))
5432 1065 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (x), x,
5433 : TYPE_SIZE_UNIT (type));
5434 : else
5435 : {
5436 396 : tree t;
5437 396 : if (is_omp_target (ctx->stmt))
5438 : t = NULL_TREE;
5439 : else
5440 336 : t = maybe_lookup_decl (v, ctx);
5441 336 : if (t)
5442 316 : v = t;
5443 : else
5444 80 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
5445 396 : gimplify_expr (&v, ilist, NULL, is_gimple_val,
5446 : fb_rvalue);
5447 792 : t = fold_build2_loc (clause_loc, PLUS_EXPR,
5448 396 : TREE_TYPE (v), v,
5449 396 : build_int_cst (TREE_TYPE (v), 1));
5450 396 : t = fold_build2_loc (clause_loc, MULT_EXPR,
5451 396 : TREE_TYPE (v), t,
5452 396 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5453 396 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (x), x, t);
5454 : }
5455 1461 : cond = create_tmp_var (TREE_TYPE (x));
5456 1461 : gimplify_assign (cond, x, ilist);
5457 1461 : x = xv;
5458 : }
5459 948 : else if (lower_private_allocate (var, type, allocator,
5460 : allocate_ptr, ilist, ctx,
5461 : true,
5462 948 : TREE_CONSTANT (v)
5463 855 : ? TYPE_SIZE_UNIT (type)
5464 : : sz))
5465 300 : x = allocate_ptr;
5466 648 : else if (TREE_CONSTANT (v))
5467 : {
5468 560 : x = create_tmp_var_raw (type, name);
5469 560 : gimple_add_tmp_var (x);
5470 560 : TREE_ADDRESSABLE (x) = 1;
5471 560 : x = build_fold_addr_expr_loc (clause_loc, x);
5472 : }
5473 : else
5474 : {
5475 88 : tree atmp
5476 88 : = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
5477 88 : tree al = size_int (TYPE_ALIGN (TREE_TYPE (type)));
5478 88 : x = build_call_expr_loc (clause_loc, atmp, 2, sz, al);
5479 : }
5480 :
5481 2409 : tree ptype = build_pointer_type (TREE_TYPE (type));
5482 2409 : x = fold_convert_loc (clause_loc, ptype, x);
5483 2409 : tree y = create_tmp_var (ptype, name);
5484 2409 : gimplify_assign (y, x, ilist);
5485 2409 : x = y;
5486 2409 : tree yb = y;
5487 :
5488 2409 : if (!integer_zerop (bias))
5489 : {
5490 1542 : bias = fold_convert_loc (clause_loc, pointer_sized_int_node,
5491 : bias);
5492 1542 : yb = fold_convert_loc (clause_loc, pointer_sized_int_node,
5493 : x);
5494 1542 : yb = fold_build2_loc (clause_loc, MINUS_EXPR,
5495 : pointer_sized_int_node, yb, bias);
5496 1542 : x = fold_convert_loc (clause_loc, TREE_TYPE (x), yb);
5497 1542 : yb = create_tmp_var (ptype, name);
5498 1542 : gimplify_assign (yb, x, ilist);
5499 1542 : x = yb;
5500 : }
5501 :
5502 2409 : d = TREE_OPERAND (d, 0);
5503 2409 : if (TREE_CODE (d) == POINTER_PLUS_EXPR)
5504 420 : d = TREE_OPERAND (d, 0);
5505 2409 : if (TREE_CODE (d) == ADDR_EXPR)
5506 : {
5507 1286 : if (orig_var != var)
5508 : {
5509 91 : gcc_assert (is_variable_sized (orig_var));
5510 91 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var),
5511 : x);
5512 91 : gimplify_assign (new_var, x, ilist);
5513 91 : tree new_orig_var = lookup_decl (orig_var, ctx);
5514 91 : tree t = build_fold_indirect_ref (new_var);
5515 91 : DECL_IGNORED_P (new_var) = 0;
5516 91 : TREE_THIS_NOTRAP (t) = 1;
5517 91 : SET_DECL_VALUE_EXPR (new_orig_var, t);
5518 91 : DECL_HAS_VALUE_EXPR_P (new_orig_var) = 1;
5519 : }
5520 : else
5521 : {
5522 1195 : x = build2 (MEM_REF, TREE_TYPE (new_var), x,
5523 : build_int_cst (ptype, 0));
5524 1195 : SET_DECL_VALUE_EXPR (new_var, x);
5525 1195 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5526 : }
5527 : }
5528 : else
5529 : {
5530 1123 : gcc_assert (orig_var == var);
5531 1123 : if (TREE_CODE (d) == INDIRECT_REF)
5532 : {
5533 112 : x = create_tmp_var (ptype, name);
5534 112 : TREE_ADDRESSABLE (x) = 1;
5535 112 : gimplify_assign (x, yb, ilist);
5536 112 : x = build_fold_addr_expr_loc (clause_loc, x);
5537 : }
5538 1123 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
5539 1123 : gimplify_assign (new_var, x, ilist);
5540 : }
5541 : /* GOMP_taskgroup_reduction_register memsets the whole
5542 : array to zero. If the initializer is zero, we don't
5543 : need to initialize it again, just mark it as ever
5544 : used unconditionally, i.e. cond = true. */
5545 2409 : if (cond
5546 1461 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) == NULL_TREE
5547 3342 : && initializer_zerop (omp_reduction_init (c,
5548 933 : TREE_TYPE (type))))
5549 : {
5550 624 : gimple *g = gimple_build_assign (build_simple_mem_ref (cond),
5551 : boolean_true_node);
5552 624 : gimple_seq_add_stmt (ilist, g);
5553 624 : continue;
5554 624 : }
5555 1785 : tree end = create_artificial_label (UNKNOWN_LOCATION);
5556 1785 : if (cond)
5557 : {
5558 837 : gimple *g;
5559 837 : if (!is_parallel_ctx (ctx))
5560 : {
5561 786 : tree condv = create_tmp_var (boolean_type_node);
5562 786 : g = gimple_build_assign (condv,
5563 : build_simple_mem_ref (cond));
5564 786 : gimple_seq_add_stmt (ilist, g);
5565 786 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
5566 786 : g = gimple_build_cond (NE_EXPR, condv,
5567 : boolean_false_node, end, lab1);
5568 786 : gimple_seq_add_stmt (ilist, g);
5569 786 : gimple_seq_add_stmt (ilist, gimple_build_label (lab1));
5570 : }
5571 837 : g = gimple_build_assign (build_simple_mem_ref (cond),
5572 : boolean_true_node);
5573 837 : gimple_seq_add_stmt (ilist, g);
5574 : }
5575 :
5576 1785 : tree y1 = create_tmp_var (ptype);
5577 1785 : gimplify_assign (y1, y, ilist);
5578 1785 : tree i2 = NULL_TREE, y2 = NULL_TREE;
5579 1785 : tree body2 = NULL_TREE, end2 = NULL_TREE;
5580 1785 : tree y3 = NULL_TREE, y4 = NULL_TREE;
5581 1785 : if (task_reduction_needs_orig_p)
5582 : {
5583 112 : y3 = create_tmp_var (ptype);
5584 112 : tree ref;
5585 112 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5586 88 : ref = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5587 88 : size_int (task_reduction_cnt_full
5588 : + task_reduction_cntorig - 1),
5589 : NULL_TREE, NULL_TREE);
5590 : else
5591 : {
5592 24 : unsigned int idx = *ctx->task_reduction_map->get (c);
5593 24 : ref = task_reduction_read (ilist, tskred_temp, ptype,
5594 24 : 7 + 3 * idx);
5595 : }
5596 112 : gimplify_assign (y3, ref, ilist);
5597 : }
5598 1673 : else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) || is_simd)
5599 : {
5600 668 : if (pass != 3)
5601 : {
5602 252 : y2 = create_tmp_var (ptype);
5603 252 : gimplify_assign (y2, y, ilist);
5604 : }
5605 668 : if (is_simd || OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5606 : {
5607 188 : tree ref = build_outer_var_ref (var, ctx);
5608 : /* For ref build_outer_var_ref already performs this. */
5609 188 : if (TREE_CODE (d) == INDIRECT_REF)
5610 0 : gcc_assert (omp_privatize_by_reference (var));
5611 188 : else if (TREE_CODE (d) == ADDR_EXPR)
5612 96 : ref = build_fold_addr_expr (ref);
5613 92 : else if (omp_privatize_by_reference (var))
5614 8 : ref = build_fold_addr_expr (ref);
5615 188 : ref = fold_convert_loc (clause_loc, ptype, ref);
5616 188 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)
5617 188 : && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5618 : {
5619 8 : y3 = create_tmp_var (ptype);
5620 8 : gimplify_assign (y3, unshare_expr (ref), ilist);
5621 : }
5622 188 : if (is_simd)
5623 : {
5624 180 : y4 = create_tmp_var (ptype);
5625 180 : gimplify_assign (y4, ref, dlist);
5626 : }
5627 : }
5628 : }
5629 1785 : tree i = create_tmp_var (TREE_TYPE (v));
5630 1785 : gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), ilist);
5631 1785 : tree body = create_artificial_label (UNKNOWN_LOCATION);
5632 1785 : gimple_seq_add_stmt (ilist, gimple_build_label (body));
5633 1785 : if (y2)
5634 : {
5635 252 : i2 = create_tmp_var (TREE_TYPE (v));
5636 252 : gimplify_assign (i2, build_int_cst (TREE_TYPE (v), 0), dlist);
5637 252 : body2 = create_artificial_label (UNKNOWN_LOCATION);
5638 252 : end2 = create_artificial_label (UNKNOWN_LOCATION);
5639 252 : gimple_seq_add_stmt (dlist, gimple_build_label (body2));
5640 : }
5641 1785 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
5642 : {
5643 600 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
5644 600 : tree decl_placeholder
5645 600 : = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
5646 600 : SET_DECL_VALUE_EXPR (decl_placeholder,
5647 : build_simple_mem_ref (y1));
5648 600 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
5649 600 : SET_DECL_VALUE_EXPR (placeholder,
5650 : y3 ? build_simple_mem_ref (y3)
5651 : : error_mark_node);
5652 600 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
5653 600 : x = lang_hooks.decls.omp_clause_default_ctor
5654 720 : (c, build_simple_mem_ref (y1),
5655 120 : y3 ? build_simple_mem_ref (y3) : NULL_TREE);
5656 600 : if (x)
5657 152 : gimplify_and_add (x, ilist);
5658 600 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
5659 : {
5660 576 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
5661 576 : lower_omp (&tseq, ctx);
5662 576 : gimple_seq_add_seq (ilist, tseq);
5663 : }
5664 600 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
5665 600 : if (is_simd)
5666 : {
5667 0 : SET_DECL_VALUE_EXPR (decl_placeholder,
5668 : build_simple_mem_ref (y2));
5669 0 : SET_DECL_VALUE_EXPR (placeholder,
5670 : build_simple_mem_ref (y4));
5671 0 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
5672 0 : lower_omp (&tseq, ctx);
5673 0 : gimple_seq_add_seq (dlist, tseq);
5674 0 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
5675 : }
5676 600 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
5677 600 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 0;
5678 600 : if (y2)
5679 : {
5680 72 : x = lang_hooks.decls.omp_clause_dtor
5681 72 : (c, build_simple_mem_ref (y2));
5682 72 : if (x)
5683 40 : gimplify_and_add (x, dlist);
5684 : }
5685 : }
5686 : else
5687 : {
5688 1185 : x = omp_reduction_init (c, TREE_TYPE (type));
5689 1185 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
5690 :
5691 : /* reduction(-:var) sums up the partial results, so it
5692 : acts identically to reduction(+:var). */
5693 1185 : if (code == MINUS_EXPR)
5694 0 : code = PLUS_EXPR;
5695 :
5696 1185 : gimplify_assign (build_simple_mem_ref (y1), x, ilist);
5697 1185 : if (is_simd)
5698 : {
5699 180 : x = build2 (code, TREE_TYPE (type),
5700 : build_simple_mem_ref (y4),
5701 : build_simple_mem_ref (y2));
5702 180 : gimplify_assign (build_simple_mem_ref (y4), x, dlist);
5703 : }
5704 : }
5705 1785 : gimple *g
5706 1785 : = gimple_build_assign (y1, POINTER_PLUS_EXPR, y1,
5707 1785 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5708 1785 : gimple_seq_add_stmt (ilist, g);
5709 1785 : if (y3)
5710 : {
5711 120 : g = gimple_build_assign (y3, POINTER_PLUS_EXPR, y3,
5712 120 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5713 120 : gimple_seq_add_stmt (ilist, g);
5714 : }
5715 1785 : g = gimple_build_assign (i, PLUS_EXPR, i,
5716 1785 : build_int_cst (TREE_TYPE (i), 1));
5717 1785 : gimple_seq_add_stmt (ilist, g);
5718 1785 : g = gimple_build_cond (LE_EXPR, i, v, body, end);
5719 1785 : gimple_seq_add_stmt (ilist, g);
5720 1785 : gimple_seq_add_stmt (ilist, gimple_build_label (end));
5721 1785 : if (y2)
5722 : {
5723 252 : g = gimple_build_assign (y2, POINTER_PLUS_EXPR, y2,
5724 252 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5725 252 : gimple_seq_add_stmt (dlist, g);
5726 252 : if (y4)
5727 : {
5728 180 : g = gimple_build_assign
5729 180 : (y4, POINTER_PLUS_EXPR, y4,
5730 180 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5731 180 : gimple_seq_add_stmt (dlist, g);
5732 : }
5733 252 : g = gimple_build_assign (i2, PLUS_EXPR, i2,
5734 252 : build_int_cst (TREE_TYPE (i2), 1));
5735 252 : gimple_seq_add_stmt (dlist, g);
5736 252 : g = gimple_build_cond (LE_EXPR, i2, v, body2, end2);
5737 252 : gimple_seq_add_stmt (dlist, g);
5738 252 : gimple_seq_add_stmt (dlist, gimple_build_label (end2));
5739 : }
5740 1785 : if (allocator)
5741 : {
5742 300 : tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
5743 300 : g = gimple_build_call (f, 2, allocate_ptr, allocator);
5744 300 : gimple_seq_add_stmt (dlist, g);
5745 : }
5746 1785 : continue;
5747 1785 : }
5748 314040 : else if (pass == 2)
5749 : {
5750 938 : tree out = maybe_lookup_decl_in_outer_ctx (var, ctx);
5751 938 : if (is_global_var (out))
5752 : x = var;
5753 382 : else if (is_omp_target (ctx->stmt))
5754 : x = out;
5755 : else
5756 : {
5757 322 : bool by_ref = use_pointer_for_field (var, ctx);
5758 322 : x = build_receiver_ref (var, by_ref, ctx);
5759 : }
5760 938 : if (!omp_privatize_by_reference (var))
5761 836 : x = build_fold_addr_expr (x);
5762 938 : x = fold_convert (ptr_type_node, x);
5763 938 : unsigned cnt = task_reduction_cnt - 1;
5764 938 : if (!task_reduction_needs_orig_p)
5765 893 : cnt += task_reduction_cntorig_full - task_reduction_cntorig;
5766 : else
5767 45 : cnt = task_reduction_cntorig - 1;
5768 938 : tree r = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5769 938 : size_int (cnt), NULL_TREE, NULL_TREE);
5770 938 : gimplify_assign (r, x, ilist);
5771 938 : continue;
5772 938 : }
5773 313102 : else if (pass == 3)
5774 : {
5775 1776 : tree type = TREE_TYPE (new_var);
5776 1776 : if (!omp_privatize_by_reference (var))
5777 1654 : type = build_pointer_type (type);
5778 1776 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5779 : {
5780 938 : unsigned cnt = task_reduction_cnt - 1;
5781 938 : if (!task_reduction_needs_orig_p)
5782 893 : cnt += (task_reduction_cntorig_full
5783 893 : - task_reduction_cntorig);
5784 : else
5785 45 : cnt = task_reduction_cntorig - 1;
5786 938 : x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5787 938 : size_int (cnt), NULL_TREE, NULL_TREE);
5788 : }
5789 : else
5790 : {
5791 838 : unsigned int idx = *ctx->task_reduction_map->get (c);
5792 838 : tree off;
5793 838 : if (ctx->task_reductions[1 + idx])
5794 838 : off = fold_convert (sizetype,
5795 : ctx->task_reductions[1 + idx]);
5796 : else
5797 0 : off = task_reduction_read (ilist, tskred_temp, sizetype,
5798 0 : 7 + 3 * idx + 1);
5799 838 : x = fold_build2 (POINTER_PLUS_EXPR, ptr_type_node,
5800 : tskred_base, off);
5801 : }
5802 1776 : x = fold_convert (type, x);
5803 1776 : tree t;
5804 1776 : if (omp_privatize_by_reference (var))
5805 : {
5806 122 : gimplify_assign (new_var, x, ilist);
5807 122 : t = new_var;
5808 122 : new_var = build_simple_mem_ref (new_var);
5809 : }
5810 : else
5811 : {
5812 1654 : t = create_tmp_var (type);
5813 1654 : gimplify_assign (t, x, ilist);
5814 1654 : SET_DECL_VALUE_EXPR (new_var, build_simple_mem_ref (t));
5815 1654 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5816 : }
5817 1776 : t = fold_convert (build_pointer_type (boolean_type_node), t);
5818 1776 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t,
5819 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5820 1776 : cond = create_tmp_var (TREE_TYPE (t));
5821 1776 : gimplify_assign (cond, t, ilist);
5822 : }
5823 311326 : else if (is_variable_sized (var))
5824 : {
5825 : /* For variable sized types, we need to allocate the
5826 : actual storage here. Call alloca and store the
5827 : result in the pointer decl that we created elsewhere. */
5828 270 : if (pass == 0)
5829 138 : continue;
5830 :
5831 132 : if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx))
5832 : {
5833 118 : tree tmp;
5834 :
5835 118 : ptr = DECL_VALUE_EXPR (new_var);
5836 118 : gcc_assert (TREE_CODE (ptr) == INDIRECT_REF);
5837 118 : ptr = TREE_OPERAND (ptr, 0);
5838 118 : gcc_assert (DECL_P (ptr));
5839 118 : x = TYPE_SIZE_UNIT (TREE_TYPE (new_var));
5840 :
5841 118 : if (lower_private_allocate (var, new_var, allocator,
5842 : allocate_ptr, ilist, ctx,
5843 : false, x))
5844 8 : tmp = allocate_ptr;
5845 : else
5846 : {
5847 : /* void *tmp = __builtin_alloca */
5848 110 : tree atmp
5849 110 : = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
5850 110 : gcall *stmt
5851 110 : = gimple_build_call (atmp, 2, x,
5852 110 : size_int (DECL_ALIGN (var)));
5853 110 : cfun->calls_alloca = 1;
5854 110 : tmp = create_tmp_var_raw (ptr_type_node);
5855 110 : gimple_add_tmp_var (tmp);
5856 110 : gimple_call_set_lhs (stmt, tmp);
5857 :
5858 110 : gimple_seq_add_stmt (ilist, stmt);
5859 : }
5860 :
5861 118 : x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp);
5862 118 : gimplify_assign (ptr, x, ilist);
5863 : }
5864 : }
5865 311056 : else if (omp_privatize_by_reference (var)
5866 311056 : && (c_kind != OMP_CLAUSE_FIRSTPRIVATE
5867 2016 : || !OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c)))
5868 : {
5869 : /* For references that are being privatized for Fortran,
5870 : allocate new backing storage for the new pointer
5871 : variable. This allows us to avoid changing all the
5872 : code that expects a pointer to something that expects
5873 : a direct variable. */
5874 4913 : if (pass == 0)
5875 2559 : continue;
5876 :
5877 2354 : x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
5878 2354 : if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx))
5879 : {
5880 146 : x = build_receiver_ref (var, false, ctx);
5881 146 : if (ctx->allocate_map)
5882 16 : if (tree *allocatep = ctx->allocate_map->get (var))
5883 : {
5884 16 : allocator = *allocatep;
5885 16 : if (TREE_CODE (allocator) == TREE_LIST)
5886 0 : allocator = TREE_PURPOSE (allocator);
5887 16 : if (TREE_CODE (allocator) != INTEGER_CST)
5888 8 : allocator = build_outer_var_ref (allocator, ctx);
5889 16 : allocator = fold_convert (pointer_sized_int_node,
5890 : allocator);
5891 16 : allocate_ptr = unshare_expr (x);
5892 : }
5893 146 : if (allocator == NULL_TREE)
5894 130 : x = build_fold_addr_expr_loc (clause_loc, x);
5895 : }
5896 2208 : else if (lower_private_allocate (var, new_var, allocator,
5897 : allocate_ptr,
5898 : ilist, ctx, true, x))
5899 95 : x = allocate_ptr;
5900 2113 : else if (TREE_CONSTANT (x))
5901 : {
5902 : /* For reduction in SIMD loop, defer adding the
5903 : initialization of the reference, because if we decide
5904 : to use SIMD array for it, the initialization could cause
5905 : expansion ICE. Ditto for other privatization clauses. */
5906 1404 : if (is_simd)
5907 : x = NULL_TREE;
5908 : else
5909 : {
5910 1077 : x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
5911 : get_name (var));
5912 1077 : gimple_add_tmp_var (x);
5913 1077 : TREE_ADDRESSABLE (x) = 1;
5914 1077 : x = build_fold_addr_expr_loc (clause_loc, x);
5915 : }
5916 : }
5917 : else
5918 : {
5919 709 : tree atmp
5920 709 : = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
5921 709 : tree rtype = TREE_TYPE (TREE_TYPE (new_var));
5922 709 : tree al = size_int (TYPE_ALIGN (rtype));
5923 709 : x = build_call_expr_loc (clause_loc, atmp, 2, x, al);
5924 : }
5925 :
5926 2027 : if (x)
5927 : {
5928 2027 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
5929 2027 : gimplify_assign (new_var, x, ilist);
5930 : }
5931 :
5932 2354 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
5933 : }
5934 306143 : else if ((c_kind == OMP_CLAUSE_REDUCTION
5935 : || c_kind == OMP_CLAUSE_IN_REDUCTION)
5936 306143 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
5937 : {
5938 1982 : if (pass == 0)
5939 991 : continue;
5940 : }
5941 304161 : else if (pass != 0)
5942 151824 : continue;
5943 :
5944 170110 : switch (OMP_CLAUSE_CODE (c))
5945 : {
5946 12006 : case OMP_CLAUSE_SHARED:
5947 : /* Ignore shared directives in teams construct inside
5948 : target construct. */
5949 12006 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
5950 12006 : && !is_host_teams_ctx (ctx))
5951 0 : continue;
5952 : /* Shared global vars are just accessed directly. */
5953 12006 : if (is_global_var (new_var))
5954 : break;
5955 : /* For taskloop firstprivate/lastprivate, represented
5956 : as firstprivate and shared clause on the task, new_var
5957 : is the firstprivate var. */
5958 11839 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
5959 : break;
5960 : /* Set up the DECL_VALUE_EXPR for shared variables now. This
5961 : needs to be delayed until after fixup_child_record_type so
5962 : that we get the correct type during the dereference. */
5963 11527 : by_ref = use_pointer_for_field (var, ctx);
5964 11527 : x = build_receiver_ref (var, by_ref, ctx);
5965 11527 : SET_DECL_VALUE_EXPR (new_var, x);
5966 11527 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5967 :
5968 : /* ??? If VAR is not passed by reference, and the variable
5969 : hasn't been initialized yet, then we'll get a warning for
5970 : the store into the omp_data_s structure. Ideally, we'd be
5971 : able to notice this and not store anything at all, but
5972 : we're generating code too early. Suppress the warning. */
5973 11527 : if (!by_ref)
5974 5243 : suppress_warning (var, OPT_Wuninitialized);
5975 : break;
5976 :
5977 210 : case OMP_CLAUSE__CONDTEMP_:
5978 210 : if (is_parallel_ctx (ctx))
5979 : {
5980 103 : x = build_receiver_ref (var, false, ctx);
5981 103 : SET_DECL_VALUE_EXPR (new_var, x);
5982 103 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5983 : }
5984 107 : else if (is_simd && !OMP_CLAUSE__CONDTEMP__ITER (c))
5985 : {
5986 107 : x = build_zero_cst (TREE_TYPE (var));
5987 107 : goto do_private;
5988 : }
5989 : break;
5990 :
5991 21301 : case OMP_CLAUSE_LASTPRIVATE:
5992 21301 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
5993 : break;
5994 : /* FALLTHRU */
5995 :
5996 87639 : case OMP_CLAUSE_PRIVATE:
5997 87639 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_PRIVATE)
5998 20793 : x = build_outer_var_ref (var, ctx);
5999 66846 : else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
6000 : {
6001 163 : if (is_task_ctx (ctx))
6002 12 : x = build_receiver_ref (var, false, ctx);
6003 : else
6004 151 : x = build_outer_var_ref (var, ctx, OMP_CLAUSE_PRIVATE);
6005 : }
6006 : else
6007 : x = NULL;
6008 94317 : do_private:
6009 94317 : tree nx;
6010 94317 : bool copy_ctor;
6011 94317 : copy_ctor = false;
6012 94317 : lower_private_allocate (var, new_var, allocator, allocate_ptr,
6013 : ilist, ctx, false, NULL_TREE);
6014 94317 : nx = unshare_expr (new_var);
6015 94317 : if (is_simd
6016 24240 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6017 101703 : && OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c))
6018 22 : copy_ctor = true;
6019 94317 : if (copy_ctor)
6020 22 : nx = lang_hooks.decls.omp_clause_copy_ctor (c, nx, x);
6021 : else
6022 94295 : nx = lang_hooks.decls.omp_clause_default_ctor (c, nx, x);
6023 94317 : if (is_simd)
6024 : {
6025 24240 : tree y = lang_hooks.decls.omp_clause_dtor (c, new_var);
6026 23556 : if ((TREE_ADDRESSABLE (new_var) || nx || y
6027 23540 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6028 7010 : && (gimple_omp_for_collapse (ctx->stmt) != 1
6029 1375 : || (gimple_omp_for_index (ctx->stmt, 0)
6030 : != new_var)))
6031 16832 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE__CONDTEMP_
6032 16725 : || omp_privatize_by_reference (var))
6033 31105 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
6034 : ivar, lvar))
6035 : {
6036 5983 : if (omp_privatize_by_reference (var))
6037 : {
6038 52 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6039 52 : tree new_vard = TREE_OPERAND (new_var, 0);
6040 52 : gcc_assert (DECL_P (new_vard));
6041 52 : SET_DECL_VALUE_EXPR (new_vard,
6042 : build_fold_addr_expr (lvar));
6043 52 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6044 : }
6045 :
6046 5983 : if (nx)
6047 : {
6048 30 : tree iv = unshare_expr (ivar);
6049 30 : if (copy_ctor)
6050 22 : x = lang_hooks.decls.omp_clause_copy_ctor (c, iv,
6051 : x);
6052 : else
6053 8 : x = lang_hooks.decls.omp_clause_default_ctor (c,
6054 : iv,
6055 : x);
6056 : }
6057 5953 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__CONDTEMP_)
6058 : {
6059 37 : x = build2 (MODIFY_EXPR, TREE_TYPE (ivar),
6060 : unshare_expr (ivar), x);
6061 37 : nx = x;
6062 : }
6063 5983 : if (nx && x)
6064 67 : gimplify_and_add (x, &llist[0]);
6065 5983 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6066 5983 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
6067 : {
6068 37 : tree v = new_var;
6069 37 : if (!DECL_P (v))
6070 : {
6071 4 : gcc_assert (TREE_CODE (v) == MEM_REF);
6072 4 : v = TREE_OPERAND (v, 0);
6073 4 : gcc_assert (DECL_P (v));
6074 : }
6075 37 : v = *ctx->lastprivate_conditional_map->get (v);
6076 37 : tree t = create_tmp_var (TREE_TYPE (v));
6077 37 : tree z = build_zero_cst (TREE_TYPE (v));
6078 37 : tree orig_v
6079 37 : = build_outer_var_ref (var, ctx,
6080 : OMP_CLAUSE_LASTPRIVATE);
6081 37 : gimple_seq_add_stmt (dlist,
6082 37 : gimple_build_assign (t, z));
6083 37 : gcc_assert (DECL_HAS_VALUE_EXPR_P (v));
6084 37 : tree civar = DECL_VALUE_EXPR (v);
6085 37 : gcc_assert (TREE_CODE (civar) == ARRAY_REF);
6086 37 : civar = unshare_expr (civar);
6087 37 : TREE_OPERAND (civar, 1) = sctx.idx;
6088 37 : x = build2 (MODIFY_EXPR, TREE_TYPE (t), t,
6089 : unshare_expr (civar));
6090 74 : x = build2 (COMPOUND_EXPR, TREE_TYPE (orig_v), x,
6091 37 : build2 (MODIFY_EXPR, TREE_TYPE (orig_v),
6092 : orig_v, unshare_expr (ivar)));
6093 37 : tree cond = build2 (LT_EXPR, boolean_type_node, t,
6094 : civar);
6095 37 : x = build3 (COND_EXPR, void_type_node, cond, x,
6096 : void_node);
6097 37 : gimple_seq tseq = NULL;
6098 37 : gimplify_and_add (x, &tseq);
6099 37 : if (ctx->outer)
6100 27 : lower_omp (&tseq, ctx->outer);
6101 37 : gimple_seq_add_seq (&llist[1], tseq);
6102 : }
6103 5983 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6104 5983 : && ctx->for_simd_scan_phase)
6105 : {
6106 9 : x = unshare_expr (ivar);
6107 9 : tree orig_v
6108 9 : = build_outer_var_ref (var, ctx,
6109 : OMP_CLAUSE_LASTPRIVATE);
6110 9 : x = lang_hooks.decls.omp_clause_assign_op (c, x,
6111 : orig_v);
6112 9 : gimplify_and_add (x, &llist[0]);
6113 : }
6114 5983 : if (y)
6115 : {
6116 30 : y = lang_hooks.decls.omp_clause_dtor (c, ivar);
6117 30 : if (y)
6118 30 : gimplify_and_add (y, &llist[1]);
6119 : }
6120 : break;
6121 : }
6122 18257 : if (omp_privatize_by_reference (var))
6123 : {
6124 28 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6125 28 : tree new_vard = TREE_OPERAND (new_var, 0);
6126 28 : gcc_assert (DECL_P (new_vard));
6127 28 : tree type = TREE_TYPE (TREE_TYPE (new_vard));
6128 28 : x = TYPE_SIZE_UNIT (type);
6129 28 : if (TREE_CONSTANT (x))
6130 : {
6131 28 : x = create_tmp_var_raw (type, get_name (var));
6132 28 : gimple_add_tmp_var (x);
6133 28 : TREE_ADDRESSABLE (x) = 1;
6134 28 : x = build_fold_addr_expr_loc (clause_loc, x);
6135 28 : x = fold_convert_loc (clause_loc,
6136 28 : TREE_TYPE (new_vard), x);
6137 28 : gimplify_assign (new_vard, x, ilist);
6138 : }
6139 : }
6140 : }
6141 88334 : if (nx)
6142 488 : gimplify_and_add (nx, ilist);
6143 88334 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6144 15170 : && is_simd
6145 90097 : && ctx->for_simd_scan_phase)
6146 : {
6147 5 : tree orig_v = build_outer_var_ref (var, ctx,
6148 : OMP_CLAUSE_LASTPRIVATE);
6149 5 : x = lang_hooks.decls.omp_clause_assign_op (c, new_var,
6150 : orig_v);
6151 5 : gimplify_and_add (x, ilist);
6152 : }
6153 : /* FALLTHRU */
6154 :
6155 119300 : do_dtor:
6156 119300 : x = lang_hooks.decls.omp_clause_dtor (c, new_var);
6157 119300 : if (x)
6158 1134 : gimplify_and_add (x, dlist);
6159 119300 : if (allocator)
6160 : {
6161 891 : if (!is_gimple_val (allocator))
6162 : {
6163 22 : tree avar = create_tmp_var (TREE_TYPE (allocator));
6164 22 : gimplify_assign (avar, allocator, dlist);
6165 22 : allocator = avar;
6166 : }
6167 891 : if (!is_gimple_val (allocate_ptr))
6168 : {
6169 59 : tree apvar = create_tmp_var (TREE_TYPE (allocate_ptr));
6170 59 : gimplify_assign (apvar, allocate_ptr, dlist);
6171 59 : allocate_ptr = apvar;
6172 : }
6173 891 : tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
6174 891 : gimple *g
6175 891 : = gimple_build_call (f, 2, allocate_ptr, allocator);
6176 891 : gimple_seq_add_stmt (dlist, g);
6177 : }
6178 : break;
6179 :
6180 7840 : case OMP_CLAUSE_LINEAR:
6181 7840 : if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c))
6182 1269 : goto do_firstprivate;
6183 6571 : if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
6184 : x = NULL;
6185 : else
6186 3605 : x = build_outer_var_ref (var, ctx);
6187 6571 : goto do_private;
6188 :
6189 28794 : case OMP_CLAUSE_FIRSTPRIVATE:
6190 28794 : if (is_task_ctx (ctx))
6191 : {
6192 4645 : if ((omp_privatize_by_reference (var)
6193 146 : && !OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c))
6194 4645 : || is_variable_sized (var))
6195 160 : goto do_dtor;
6196 4485 : else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var,
6197 : ctx))
6198 4485 : || use_pointer_for_field (var, NULL))
6199 : {
6200 430 : x = build_receiver_ref (var, false, ctx);
6201 430 : if (ctx->allocate_map)
6202 43 : if (tree *allocatep = ctx->allocate_map->get (var))
6203 : {
6204 43 : allocator = *allocatep;
6205 43 : if (TREE_CODE (allocator) == TREE_LIST)
6206 4 : allocator = TREE_PURPOSE (allocator);
6207 43 : if (TREE_CODE (allocator) != INTEGER_CST)
6208 14 : allocator = build_outer_var_ref (allocator, ctx);
6209 43 : allocator = fold_convert (pointer_sized_int_node,
6210 : allocator);
6211 43 : allocate_ptr = unshare_expr (x);
6212 43 : x = build_simple_mem_ref (x);
6213 43 : TREE_THIS_NOTRAP (x) = 1;
6214 : }
6215 430 : SET_DECL_VALUE_EXPR (new_var, x);
6216 430 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
6217 430 : goto do_dtor;
6218 : }
6219 : }
6220 28204 : if (OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c)
6221 28204 : && omp_privatize_by_reference (var))
6222 : {
6223 0 : x = build_outer_var_ref (var, ctx);
6224 0 : gcc_assert (TREE_CODE (x) == MEM_REF
6225 : && integer_zerop (TREE_OPERAND (x, 1)));
6226 0 : x = TREE_OPERAND (x, 0);
6227 0 : x = lang_hooks.decls.omp_clause_copy_ctor
6228 0 : (c, unshare_expr (new_var), x);
6229 0 : gimplify_and_add (x, ilist);
6230 0 : goto do_dtor;
6231 : }
6232 29473 : do_firstprivate:
6233 29473 : lower_private_allocate (var, new_var, allocator, allocate_ptr,
6234 : ilist, ctx, false, NULL_TREE);
6235 29473 : x = build_outer_var_ref (var, ctx);
6236 29473 : if (is_simd)
6237 : {
6238 1063 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
6239 1063 : && gimple_omp_for_combined_into_p (ctx->stmt))
6240 : {
6241 668 : tree t = OMP_CLAUSE_LINEAR_STEP (c);
6242 668 : if (DECL_P (t))
6243 114 : t = build_outer_var_ref (t, ctx);
6244 668 : tree stept = TREE_TYPE (t);
6245 668 : tree ct = omp_find_clause (clauses,
6246 : OMP_CLAUSE__LOOPTEMP_);
6247 668 : gcc_assert (ct);
6248 668 : tree l = OMP_CLAUSE_DECL (ct);
6249 668 : tree n1 = fd->loop.n1;
6250 668 : tree step = fd->loop.step;
6251 668 : tree itype = TREE_TYPE (l);
6252 668 : if (POINTER_TYPE_P (itype))
6253 0 : itype = signed_type_for (itype);
6254 668 : l = fold_build2 (MINUS_EXPR, itype, l, n1);
6255 668 : if (TYPE_UNSIGNED (itype)
6256 668 : && fd->loop.cond_code == GT_EXPR)
6257 0 : l = fold_build2 (TRUNC_DIV_EXPR, itype,
6258 : fold_build1 (NEGATE_EXPR, itype, l),
6259 : fold_build1 (NEGATE_EXPR,
6260 : itype, step));
6261 : else
6262 668 : l = fold_build2 (TRUNC_DIV_EXPR, itype, l, step);
6263 668 : t = fold_build2 (MULT_EXPR, stept,
6264 : fold_convert (stept, l), t);
6265 :
6266 668 : if (OMP_CLAUSE_LINEAR_ARRAY (c))
6267 : {
6268 108 : if (omp_privatize_by_reference (var))
6269 : {
6270 72 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6271 72 : tree new_vard = TREE_OPERAND (new_var, 0);
6272 72 : gcc_assert (DECL_P (new_vard));
6273 72 : tree type = TREE_TYPE (TREE_TYPE (new_vard));
6274 72 : nx = TYPE_SIZE_UNIT (type);
6275 72 : if (TREE_CONSTANT (nx))
6276 : {
6277 24 : nx = create_tmp_var_raw (type,
6278 : get_name (var));
6279 24 : gimple_add_tmp_var (nx);
6280 24 : TREE_ADDRESSABLE (nx) = 1;
6281 24 : nx = build_fold_addr_expr_loc (clause_loc,
6282 : nx);
6283 24 : nx = fold_convert_loc (clause_loc,
6284 24 : TREE_TYPE (new_vard),
6285 : nx);
6286 24 : gimplify_assign (new_vard, nx, ilist);
6287 : }
6288 : }
6289 :
6290 108 : x = lang_hooks.decls.omp_clause_linear_ctor
6291 108 : (c, new_var, x, t);
6292 108 : gimplify_and_add (x, ilist);
6293 108 : goto do_dtor;
6294 : }
6295 :
6296 560 : if (POINTER_TYPE_P (TREE_TYPE (x)))
6297 11 : x = fold_build_pointer_plus (x, t);
6298 : else
6299 549 : x = fold_build2 (PLUS_EXPR, TREE_TYPE (x), x,
6300 : fold_convert (TREE_TYPE (x), t));
6301 : }
6302 :
6303 955 : if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LINEAR
6304 934 : || TREE_ADDRESSABLE (new_var)
6305 799 : || omp_privatize_by_reference (var))
6306 1217 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
6307 : ivar, lvar))
6308 : {
6309 152 : if (omp_privatize_by_reference (var))
6310 : {
6311 22 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6312 22 : tree new_vard = TREE_OPERAND (new_var, 0);
6313 22 : gcc_assert (DECL_P (new_vard));
6314 22 : SET_DECL_VALUE_EXPR (new_vard,
6315 : build_fold_addr_expr (lvar));
6316 22 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6317 : }
6318 152 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR)
6319 : {
6320 131 : tree iv = create_tmp_var (TREE_TYPE (new_var));
6321 131 : x = lang_hooks.decls.omp_clause_copy_ctor (c, iv, x);
6322 131 : gimplify_and_add (x, ilist);
6323 131 : gimple_stmt_iterator gsi
6324 131 : = gsi_start (*gimple_omp_body_ptr (ctx->stmt));
6325 131 : gassign *g
6326 131 : = gimple_build_assign (unshare_expr (lvar), iv);
6327 131 : gsi_insert_before_without_update (&gsi, g,
6328 : GSI_SAME_STMT);
6329 131 : tree t = OMP_CLAUSE_LINEAR_STEP (c);
6330 131 : enum tree_code code = PLUS_EXPR;
6331 131 : if (POINTER_TYPE_P (TREE_TYPE (new_var)))
6332 : code = POINTER_PLUS_EXPR;
6333 131 : g = gimple_build_assign (iv, code, iv, t);
6334 131 : gsi_insert_before_without_update (&gsi, g,
6335 : GSI_SAME_STMT);
6336 131 : break;
6337 : }
6338 21 : x = lang_hooks.decls.omp_clause_copy_ctor
6339 21 : (c, unshare_expr (ivar), x);
6340 21 : gimplify_and_add (x, &llist[0]);
6341 21 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6342 21 : if (x)
6343 21 : gimplify_and_add (x, &llist[1]);
6344 : break;
6345 : }
6346 803 : if (omp_privatize_by_reference (var))
6347 : {
6348 105 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6349 105 : tree new_vard = TREE_OPERAND (new_var, 0);
6350 105 : gcc_assert (DECL_P (new_vard));
6351 105 : tree type = TREE_TYPE (TREE_TYPE (new_vard));
6352 105 : nx = TYPE_SIZE_UNIT (type);
6353 105 : if (TREE_CONSTANT (nx))
6354 : {
6355 57 : nx = create_tmp_var_raw (type, get_name (var));
6356 57 : gimple_add_tmp_var (nx);
6357 57 : TREE_ADDRESSABLE (nx) = 1;
6358 57 : nx = build_fold_addr_expr_loc (clause_loc, nx);
6359 57 : nx = fold_convert_loc (clause_loc,
6360 57 : TREE_TYPE (new_vard), nx);
6361 57 : gimplify_assign (new_vard, nx, ilist);
6362 : }
6363 : }
6364 : }
6365 29213 : x = lang_hooks.decls.omp_clause_copy_ctor
6366 29213 : (c, unshare_expr (new_var), x);
6367 29209 : gimplify_and_add (x, ilist);
6368 29209 : goto do_dtor;
6369 :
6370 19434 : case OMP_CLAUSE__LOOPTEMP_:
6371 19434 : case OMP_CLAUSE__REDUCTEMP_:
6372 19434 : gcc_assert (is_taskreg_ctx (ctx));
6373 19434 : x = build_outer_var_ref (var, ctx);
6374 19434 : x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
6375 19434 : gimplify_and_add (x, ilist);
6376 19434 : break;
6377 :
6378 514 : case OMP_CLAUSE_COPYIN:
6379 514 : by_ref = use_pointer_for_field (var, NULL);
6380 514 : x = build_receiver_ref (var, by_ref, ctx);
6381 514 : x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x);
6382 514 : append_to_statement_list (x, ©in_seq);
6383 514 : copyin_by_ref |= by_ref;
6384 514 : break;
6385 :
6386 13165 : case OMP_CLAUSE_REDUCTION:
6387 13165 : case OMP_CLAUSE_IN_REDUCTION:
6388 : /* OpenACC reductions are initialized using the
6389 : GOACC_REDUCTION internal function. */
6390 13165 : if (is_gimple_omp_oacc (ctx->stmt))
6391 : break;
6392 9183 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
6393 : {
6394 1491 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
6395 1491 : gimple *tseq;
6396 1491 : tree ptype = TREE_TYPE (placeholder);
6397 1491 : if (cond)
6398 : {
6399 327 : x = error_mark_node;
6400 327 : if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)
6401 327 : && !task_reduction_needs_orig_p)
6402 32 : x = var;
6403 295 : else if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
6404 : {
6405 57 : tree pptype = build_pointer_type (ptype);
6406 57 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
6407 45 : x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
6408 45 : size_int (task_reduction_cnt_full
6409 : + task_reduction_cntorig - 1),
6410 : NULL_TREE, NULL_TREE);
6411 : else
6412 : {
6413 12 : unsigned int idx
6414 12 : = *ctx->task_reduction_map->get (c);
6415 12 : x = task_reduction_read (ilist, tskred_temp,
6416 12 : pptype, 7 + 3 * idx);
6417 : }
6418 57 : x = fold_convert (pptype, x);
6419 57 : x = build_simple_mem_ref (x);
6420 : }
6421 : }
6422 : else
6423 : {
6424 1164 : lower_private_allocate (var, new_var, allocator,
6425 : allocate_ptr, ilist, ctx, false,
6426 : NULL_TREE);
6427 1164 : x = build_outer_var_ref (var, ctx);
6428 :
6429 1164 : if (omp_privatize_by_reference (var)
6430 1164 : && !useless_type_conversion_p (ptype, TREE_TYPE (x)))
6431 49 : x = build_fold_addr_expr_loc (clause_loc, x);
6432 : }
6433 1491 : SET_DECL_VALUE_EXPR (placeholder, x);
6434 1491 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
6435 1491 : tree new_vard = new_var;
6436 1491 : if (omp_privatize_by_reference (var))
6437 : {
6438 219 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6439 219 : new_vard = TREE_OPERAND (new_var, 0);
6440 219 : gcc_assert (DECL_P (new_vard));
6441 : }
6442 1491 : tree rvar = NULL_TREE, *rvarp = NULL, rvar2 = NULL_TREE;
6443 1491 : if (is_simd
6444 312 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6445 1803 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6446 : rvarp = &rvar;
6447 1491 : if (is_simd
6448 1491 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
6449 : ivar, lvar, rvarp,
6450 : &rvar2))
6451 : {
6452 175 : if (new_vard == new_var)
6453 : {
6454 129 : gcc_assert (DECL_VALUE_EXPR (new_var) == lvar);
6455 129 : SET_DECL_VALUE_EXPR (new_var, ivar);
6456 : }
6457 : else
6458 : {
6459 46 : SET_DECL_VALUE_EXPR (new_vard,
6460 : build_fold_addr_expr (ivar));
6461 46 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6462 : }
6463 175 : x = lang_hooks.decls.omp_clause_default_ctor
6464 175 : (c, unshare_expr (ivar),
6465 : build_outer_var_ref (var, ctx));
6466 175 : if (rvarp && ctx->for_simd_scan_phase)
6467 : {
6468 16 : if (x)
6469 8 : gimplify_and_add (x, &llist[0]);
6470 16 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6471 16 : if (x)
6472 8 : gimplify_and_add (x, &llist[1]);
6473 502 : break;
6474 : }
6475 78 : else if (rvarp)
6476 : {
6477 78 : if (x)
6478 : {
6479 38 : gimplify_and_add (x, &llist[0]);
6480 :
6481 38 : tree ivar2 = unshare_expr (lvar);
6482 38 : TREE_OPERAND (ivar2, 1) = sctx.idx;
6483 38 : x = lang_hooks.decls.omp_clause_default_ctor
6484 38 : (c, ivar2, build_outer_var_ref (var, ctx));
6485 38 : gimplify_and_add (x, &llist[0]);
6486 :
6487 38 : if (rvar2)
6488 : {
6489 16 : x = lang_hooks.decls.omp_clause_default_ctor
6490 16 : (c, unshare_expr (rvar2),
6491 : build_outer_var_ref (var, ctx));
6492 16 : gimplify_and_add (x, &llist[0]);
6493 : }
6494 :
6495 : /* For types that need construction, add another
6496 : private var which will be default constructed
6497 : and optionally initialized with
6498 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT, as in the
6499 : loop we want to assign this value instead of
6500 : constructing and destructing it in each
6501 : iteration. */
6502 38 : tree nv = create_tmp_var_raw (TREE_TYPE (ivar));
6503 38 : gimple_add_tmp_var (nv);
6504 60 : ctx->cb.decl_map->put (TREE_OPERAND (rvar2
6505 : ? rvar2
6506 : : ivar, 0),
6507 : nv);
6508 38 : x = lang_hooks.decls.omp_clause_default_ctor
6509 38 : (c, nv, build_outer_var_ref (var, ctx));
6510 38 : gimplify_and_add (x, ilist);
6511 :
6512 38 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6513 : {
6514 19 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6515 19 : x = DECL_VALUE_EXPR (new_vard);
6516 19 : tree vexpr = nv;
6517 19 : if (new_vard != new_var)
6518 11 : vexpr = build_fold_addr_expr (nv);
6519 19 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
6520 19 : lower_omp (&tseq, ctx);
6521 19 : SET_DECL_VALUE_EXPR (new_vard, x);
6522 19 : gimple_seq_add_seq (ilist, tseq);
6523 19 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6524 : }
6525 :
6526 38 : x = lang_hooks.decls.omp_clause_dtor (c, nv);
6527 38 : if (x)
6528 38 : gimplify_and_add (x, dlist);
6529 : }
6530 :
6531 78 : tree ref = build_outer_var_ref (var, ctx);
6532 78 : x = unshare_expr (ivar);
6533 78 : x = lang_hooks.decls.omp_clause_assign_op (c, x,
6534 : ref);
6535 78 : gimplify_and_add (x, &llist[0]);
6536 :
6537 78 : ref = build_outer_var_ref (var, ctx);
6538 78 : x = lang_hooks.decls.omp_clause_assign_op (c, ref,
6539 : rvar);
6540 78 : gimplify_and_add (x, &llist[3]);
6541 :
6542 78 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6543 78 : if (new_vard == new_var)
6544 48 : SET_DECL_VALUE_EXPR (new_var, lvar);
6545 : else
6546 30 : SET_DECL_VALUE_EXPR (new_vard,
6547 : build_fold_addr_expr (lvar));
6548 :
6549 78 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6550 78 : if (x)
6551 38 : gimplify_and_add (x, &llist[1]);
6552 :
6553 78 : tree ivar2 = unshare_expr (lvar);
6554 78 : TREE_OPERAND (ivar2, 1) = sctx.idx;
6555 78 : x = lang_hooks.decls.omp_clause_dtor (c, ivar2);
6556 78 : if (x)
6557 38 : gimplify_and_add (x, &llist[1]);
6558 :
6559 78 : if (rvar2)
6560 : {
6561 36 : x = lang_hooks.decls.omp_clause_dtor (c, rvar2);
6562 36 : if (x)
6563 16 : gimplify_and_add (x, &llist[1]);
6564 : }
6565 : break;
6566 : }
6567 81 : if (x)
6568 34 : gimplify_and_add (x, &llist[0]);
6569 81 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6570 : {
6571 77 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6572 77 : lower_omp (&tseq, ctx);
6573 77 : gimple_seq_add_seq (&llist[0], tseq);
6574 : }
6575 81 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6576 81 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
6577 81 : lower_omp (&tseq, ctx);
6578 81 : gimple_seq_add_seq (&llist[1], tseq);
6579 81 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
6580 81 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6581 81 : if (new_vard == new_var)
6582 71 : SET_DECL_VALUE_EXPR (new_var, lvar);
6583 : else
6584 10 : SET_DECL_VALUE_EXPR (new_vard,
6585 : build_fold_addr_expr (lvar));
6586 81 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6587 81 : if (x)
6588 38 : gimplify_and_add (x, &llist[1]);
6589 : break;
6590 : }
6591 : /* If this is a reference to constant size reduction var
6592 : with placeholder, we haven't emitted the initializer
6593 : for it because it is undesirable if SIMD arrays are used.
6594 : But if they aren't used, we need to emit the deferred
6595 : initialization now. */
6596 1316 : else if (omp_privatize_by_reference (var) && is_simd)
6597 46 : handle_simd_reference (clause_loc, new_vard, ilist);
6598 :
6599 1316 : tree lab2 = NULL_TREE;
6600 1316 : if (cond)
6601 : {
6602 327 : gimple *g;
6603 327 : if (!is_parallel_ctx (ctx))
6604 : {
6605 302 : tree condv = create_tmp_var (boolean_type_node);
6606 302 : tree m = build_simple_mem_ref (cond);
6607 302 : g = gimple_build_assign (condv, m);
6608 302 : gimple_seq_add_stmt (ilist, g);
6609 302 : tree lab1
6610 302 : = create_artificial_label (UNKNOWN_LOCATION);
6611 302 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
6612 302 : g = gimple_build_cond (NE_EXPR, condv,
6613 : boolean_false_node,
6614 : lab2, lab1);
6615 302 : gimple_seq_add_stmt (ilist, g);
6616 302 : gimple_seq_add_stmt (ilist,
6617 302 : gimple_build_label (lab1));
6618 : }
6619 327 : g = gimple_build_assign (build_simple_mem_ref (cond),
6620 : boolean_true_node);
6621 327 : gimple_seq_add_stmt (ilist, g);
6622 : }
6623 327 : x = lang_hooks.decls.omp_clause_default_ctor
6624 1316 : (c, unshare_expr (new_var),
6625 : cond ? NULL_TREE
6626 989 : : build_outer_var_ref (var, ctx));
6627 1316 : if (x)
6628 302 : gimplify_and_add (x, ilist);
6629 :
6630 1316 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6631 1316 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6632 : {
6633 158 : if (ctx->for_simd_scan_phase)
6634 989 : goto do_dtor;
6635 142 : if (x || (!is_simd
6636 32 : && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)))
6637 : {
6638 70 : tree nv = create_tmp_var_raw (TREE_TYPE (new_var));
6639 70 : gimple_add_tmp_var (nv);
6640 70 : ctx->cb.decl_map->put (new_vard, nv);
6641 70 : x = lang_hooks.decls.omp_clause_default_ctor
6642 70 : (c, nv, build_outer_var_ref (var, ctx));
6643 70 : if (x)
6644 70 : gimplify_and_add (x, ilist);
6645 70 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6646 : {
6647 35 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6648 35 : tree vexpr = nv;
6649 35 : if (new_vard != new_var)
6650 19 : vexpr = build_fold_addr_expr (nv);
6651 35 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
6652 35 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6653 35 : lower_omp (&tseq, ctx);
6654 35 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
6655 35 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
6656 35 : gimple_seq_add_seq (ilist, tseq);
6657 : }
6658 70 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6659 70 : if (is_simd && ctx->scan_exclusive)
6660 : {
6661 16 : tree nv2
6662 16 : = create_tmp_var_raw (TREE_TYPE (new_var));
6663 16 : gimple_add_tmp_var (nv2);
6664 16 : ctx->cb.decl_map->put (nv, nv2);
6665 16 : x = lang_hooks.decls.omp_clause_default_ctor
6666 16 : (c, nv2, build_outer_var_ref (var, ctx));
6667 16 : gimplify_and_add (x, ilist);
6668 16 : x = lang_hooks.decls.omp_clause_dtor (c, nv2);
6669 16 : if (x)
6670 16 : gimplify_and_add (x, dlist);
6671 : }
6672 70 : x = lang_hooks.decls.omp_clause_dtor (c, nv);
6673 70 : if (x)
6674 70 : gimplify_and_add (x, dlist);
6675 : }
6676 72 : else if (is_simd
6677 40 : && ctx->scan_exclusive
6678 92 : && TREE_ADDRESSABLE (TREE_TYPE (new_var)))
6679 : {
6680 0 : tree nv2 = create_tmp_var_raw (TREE_TYPE (new_var));
6681 0 : gimple_add_tmp_var (nv2);
6682 0 : ctx->cb.decl_map->put (new_vard, nv2);
6683 0 : x = lang_hooks.decls.omp_clause_dtor (c, nv2);
6684 0 : if (x)
6685 0 : gimplify_and_add (x, dlist);
6686 : }
6687 142 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6688 142 : goto do_dtor;
6689 : }
6690 :
6691 1158 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6692 : {
6693 1110 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6694 1110 : if (c_kind == OMP_CLAUSE_IN_REDUCTION
6695 1110 : && is_omp_target (ctx->stmt))
6696 : {
6697 12 : tree d = maybe_lookup_decl_in_outer_ctx (var, ctx);
6698 12 : tree oldv = NULL_TREE;
6699 12 : gcc_assert (d);
6700 12 : if (DECL_HAS_VALUE_EXPR_P (d))
6701 6 : oldv = DECL_VALUE_EXPR (d);
6702 12 : SET_DECL_VALUE_EXPR (d, new_vard);
6703 12 : DECL_HAS_VALUE_EXPR_P (d) = 1;
6704 12 : lower_omp (&tseq, ctx);
6705 12 : if (oldv)
6706 6 : SET_DECL_VALUE_EXPR (d, oldv);
6707 : else
6708 : {
6709 6 : SET_DECL_VALUE_EXPR (d, NULL_TREE);
6710 6 : DECL_HAS_VALUE_EXPR_P (d) = 0;
6711 : }
6712 : }
6713 : else
6714 1098 : lower_omp (&tseq, ctx);
6715 1110 : gimple_seq_add_seq (ilist, tseq);
6716 : }
6717 1158 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6718 1158 : if (is_simd)
6719 : {
6720 43 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
6721 43 : lower_omp (&tseq, ctx);
6722 43 : gimple_seq_add_seq (dlist, tseq);
6723 43 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
6724 : }
6725 1158 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6726 1158 : if (cond)
6727 : {
6728 327 : if (lab2)
6729 302 : gimple_seq_add_stmt (ilist, gimple_build_label (lab2));
6730 : break;
6731 : }
6732 831 : goto do_dtor;
6733 : }
6734 : else
6735 : {
6736 7692 : x = omp_reduction_init (c, TREE_TYPE (new_var));
6737 7692 : gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE);
6738 7692 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
6739 :
6740 7692 : if (cond)
6741 : {
6742 1449 : gimple *g;
6743 1449 : tree lab2 = NULL_TREE;
6744 : /* GOMP_taskgroup_reduction_register memsets the whole
6745 : array to zero. If the initializer is zero, we don't
6746 : need to initialize it again, just mark it as ever
6747 : used unconditionally, i.e. cond = true. */
6748 1449 : if (initializer_zerop (x))
6749 : {
6750 1268 : g = gimple_build_assign (build_simple_mem_ref (cond),
6751 : boolean_true_node);
6752 1268 : gimple_seq_add_stmt (ilist, g);
6753 3271 : break;
6754 : }
6755 :
6756 : /* Otherwise, emit
6757 : if (!cond) { cond = true; new_var = x; } */
6758 181 : if (!is_parallel_ctx (ctx))
6759 : {
6760 177 : tree condv = create_tmp_var (boolean_type_node);
6761 177 : tree m = build_simple_mem_ref (cond);
6762 177 : g = gimple_build_assign (condv, m);
6763 177 : gimple_seq_add_stmt (ilist, g);
6764 177 : tree lab1
6765 177 : = create_artificial_label (UNKNOWN_LOCATION);
6766 177 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
6767 177 : g = gimple_build_cond (NE_EXPR, condv,
6768 : boolean_false_node,
6769 : lab2, lab1);
6770 177 : gimple_seq_add_stmt (ilist, g);
6771 177 : gimple_seq_add_stmt (ilist,
6772 177 : gimple_build_label (lab1));
6773 : }
6774 181 : g = gimple_build_assign (build_simple_mem_ref (cond),
6775 : boolean_true_node);
6776 181 : gimple_seq_add_stmt (ilist, g);
6777 181 : gimplify_assign (new_var, x, ilist);
6778 181 : if (lab2)
6779 177 : gimple_seq_add_stmt (ilist, gimple_build_label (lab2));
6780 : break;
6781 : }
6782 :
6783 : /* reduction(-:var) sums up the partial results, so it
6784 : acts identically to reduction(+:var). */
6785 6243 : if (code == MINUS_EXPR)
6786 33 : code = PLUS_EXPR;
6787 :
6788 6243 : bool is_truth_op
6789 6243 : = (code == TRUTH_ANDIF_EXPR || code == TRUTH_ORIF_EXPR);
6790 6243 : tree new_vard = new_var;
6791 6243 : if (is_simd && omp_privatize_by_reference (var))
6792 : {
6793 52 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6794 52 : new_vard = TREE_OPERAND (new_var, 0);
6795 52 : gcc_assert (DECL_P (new_vard));
6796 : }
6797 6243 : tree rvar = NULL_TREE, *rvarp = NULL, rvar2 = NULL_TREE;
6798 6243 : if (is_simd
6799 2103 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6800 8346 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6801 : rvarp = &rvar;
6802 6243 : if (is_simd
6803 6243 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
6804 : ivar, lvar, rvarp,
6805 : &rvar2))
6806 : {
6807 873 : if (new_vard != new_var)
6808 : {
6809 34 : SET_DECL_VALUE_EXPR (new_vard,
6810 : build_fold_addr_expr (lvar));
6811 34 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6812 : }
6813 :
6814 873 : tree ref = build_outer_var_ref (var, ctx);
6815 :
6816 873 : if (rvarp)
6817 : {
6818 193 : if (ctx->for_simd_scan_phase)
6819 : break;
6820 154 : gimplify_assign (ivar, ref, &llist[0]);
6821 154 : ref = build_outer_var_ref (var, ctx);
6822 154 : gimplify_assign (ref, rvar, &llist[3]);
6823 154 : break;
6824 : }
6825 :
6826 680 : gimplify_assign (unshare_expr (ivar), x, &llist[0]);
6827 :
6828 680 : if (sctx.is_simt)
6829 : {
6830 0 : if (!simt_lane)
6831 0 : simt_lane = create_tmp_var (unsigned_type_node);
6832 0 : x = build_call_expr_internal_loc
6833 0 : (UNKNOWN_LOCATION, IFN_GOMP_SIMT_XCHG_BFLY,
6834 0 : TREE_TYPE (ivar), 2, ivar, simt_lane);
6835 : /* Make sure x is evaluated unconditionally. */
6836 0 : tree bfly_var = create_tmp_var (TREE_TYPE (ivar));
6837 0 : gimplify_assign (bfly_var, x, &llist[2]);
6838 0 : x = build2 (code, TREE_TYPE (ivar), ivar, bfly_var);
6839 0 : gimplify_assign (ivar, x, &llist[2]);
6840 : }
6841 680 : tree ivar2 = ivar;
6842 680 : tree ref2 = ref;
6843 680 : if (is_truth_op)
6844 : {
6845 99 : tree zero = build_zero_cst (TREE_TYPE (ivar));
6846 99 : ivar2 = fold_build2_loc (clause_loc, NE_EXPR,
6847 : boolean_type_node, ivar,
6848 : zero);
6849 99 : ref2 = fold_build2_loc (clause_loc, NE_EXPR,
6850 : boolean_type_node, ref,
6851 : zero);
6852 : }
6853 680 : x = build2 (code, TREE_TYPE (ref), ref2, ivar2);
6854 680 : if (is_truth_op)
6855 99 : x = fold_convert (TREE_TYPE (ref), x);
6856 680 : ref = build_outer_var_ref (var, ctx);
6857 680 : gimplify_assign (ref, x, &llist[1]);
6858 :
6859 : }
6860 : else
6861 : {
6862 5370 : lower_private_allocate (var, new_var, allocator,
6863 : allocate_ptr, ilist, ctx,
6864 : false, NULL_TREE);
6865 5370 : if (omp_privatize_by_reference (var) && is_simd)
6866 18 : handle_simd_reference (clause_loc, new_vard, ilist);
6867 5370 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6868 5370 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6869 : break;
6870 5009 : gimplify_assign (new_var, x, ilist);
6871 5009 : if (is_simd)
6872 : {
6873 1022 : tree ref = build_outer_var_ref (var, ctx);
6874 1022 : tree new_var2 = new_var;
6875 1022 : tree ref2 = ref;
6876 1022 : if (is_truth_op)
6877 : {
6878 24 : tree zero = build_zero_cst (TREE_TYPE (new_var));
6879 24 : new_var2
6880 24 : = fold_build2_loc (clause_loc, NE_EXPR,
6881 : boolean_type_node, new_var,
6882 : zero);
6883 24 : ref2 = fold_build2_loc (clause_loc, NE_EXPR,
6884 : boolean_type_node, ref,
6885 : zero);
6886 : }
6887 1022 : x = build2 (code, TREE_TYPE (ref2), ref2, new_var2);
6888 1022 : if (is_truth_op)
6889 24 : x = fold_convert (TREE_TYPE (new_var), x);
6890 1022 : ref = build_outer_var_ref (var, ctx);
6891 1022 : gimplify_assign (ref, x, dlist);
6892 : }
6893 5009 : if (allocator)
6894 70 : goto do_dtor;
6895 : }
6896 : }
6897 5619 : break;
6898 :
6899 0 : default:
6900 0 : gcc_unreachable ();
6901 : }
6902 : }
6903 : }
6904 77762 : if (tskred_avar)
6905 : {
6906 878 : tree clobber = build_clobber (TREE_TYPE (tskred_avar));
6907 878 : gimple_seq_add_stmt (ilist, gimple_build_assign (tskred_avar, clobber));
6908 : }
6909 :
6910 77762 : if (known_eq (sctx.max_vf, 1U))
6911 : {
6912 1789 : sctx.is_simt = false;
6913 1789 : if (ctx->lastprivate_conditional_map)
6914 : {
6915 52 : if (gimple_omp_for_combined_into_p (ctx->stmt))
6916 : {
6917 : /* Signal to lower_omp_1 that it should use parent context. */
6918 43 : ctx->combined_into_simd_safelen1 = true;
6919 370 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
6920 327 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6921 327 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
6922 : {
6923 57 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
6924 57 : omp_context *outer = ctx->outer;
6925 57 : if (gimple_code (outer->stmt) == GIMPLE_OMP_SCAN)
6926 6 : outer = outer->outer;
6927 57 : tree *v = ctx->lastprivate_conditional_map->get (o);
6928 57 : tree po = lookup_decl (OMP_CLAUSE_DECL (c), outer);
6929 57 : tree *pv = outer->lastprivate_conditional_map->get (po);
6930 57 : *v = *pv;
6931 : }
6932 : }
6933 : else
6934 : {
6935 : /* When not vectorized, treat lastprivate(conditional:) like
6936 : normal lastprivate, as there will be just one simd lane
6937 : writing the privatized variable. */
6938 9 : delete ctx->lastprivate_conditional_map;
6939 9 : ctx->lastprivate_conditional_map = NULL;
6940 : }
6941 : }
6942 : }
6943 :
6944 77762 : if (nonconst_simd_if)
6945 : {
6946 625 : if (sctx.lane == NULL_TREE)
6947 : {
6948 565 : sctx.idx = create_tmp_var (unsigned_type_node);
6949 565 : sctx.lane = create_tmp_var (unsigned_type_node);
6950 : }
6951 : /* FIXME: For now. */
6952 625 : sctx.is_simt = false;
6953 : }
6954 :
6955 77762 : if (sctx.lane || sctx.is_simt)
6956 : {
6957 4007 : uid = create_tmp_var (ptr_type_node, "simduid");
6958 : /* Don't want uninit warnings on simduid, it is always uninitialized,
6959 : but we use it not for the value, but for the DECL_UID only. */
6960 4007 : suppress_warning (uid, OPT_Wuninitialized);
6961 4007 : c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SIMDUID_);
6962 4007 : OMP_CLAUSE__SIMDUID__DECL (c) = uid;
6963 4007 : OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
6964 4007 : gimple_omp_for_set_clauses (ctx->stmt, c);
6965 : }
6966 : /* Emit calls denoting privatized variables and initializing a pointer to
6967 : structure that holds private variables as fields after ompdevlow pass. */
6968 77762 : if (sctx.is_simt)
6969 : {
6970 0 : sctx.simt_eargs[0] = uid;
6971 0 : gimple *g
6972 0 : = gimple_build_call_internal_vec (IFN_GOMP_SIMT_ENTER, sctx.simt_eargs);
6973 0 : gimple_call_set_lhs (g, uid);
6974 0 : gimple_seq_add_stmt (ilist, g);
6975 0 : sctx.simt_eargs.release ();
6976 :
6977 0 : simtrec = create_tmp_var (ptr_type_node, ".omp_simt");
6978 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_ENTER_ALLOC, 1, uid);
6979 0 : gimple_call_set_lhs (g, simtrec);
6980 0 : gimple_seq_add_stmt (ilist, g);
6981 : }
6982 77762 : if (sctx.lane)
6983 : {
6984 7389 : gimple *g = gimple_build_call_internal (IFN_GOMP_SIMD_LANE,
6985 : 2 + (nonconst_simd_if != NULL),
6986 : uid, integer_zero_node,
6987 : nonconst_simd_if);
6988 4007 : gimple_call_set_lhs (g, sctx.lane);
6989 4007 : gimple_stmt_iterator gsi = gsi_start (*gimple_omp_body_ptr (ctx->stmt));
6990 4007 : gsi_insert_before_without_update (&gsi, g, GSI_SAME_STMT);
6991 4007 : g = gimple_build_assign (sctx.lane, INTEGER_CST,
6992 : build_int_cst (unsigned_type_node, 0));
6993 4007 : gimple_seq_add_stmt (ilist, g);
6994 4007 : if (sctx.lastlane)
6995 : {
6996 184 : g = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
6997 : 2, uid, sctx.lane);
6998 184 : gimple_call_set_lhs (g, sctx.lastlane);
6999 184 : gimple_seq_add_stmt (dlist, g);
7000 184 : gimple_seq_add_seq (dlist, llist[3]);
7001 : }
7002 : /* Emit reductions across SIMT lanes in log_2(simt_vf) steps. */
7003 4007 : if (llist[2])
7004 : {
7005 0 : tree simt_vf = create_tmp_var (unsigned_type_node);
7006 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_VF, 0);
7007 0 : gimple_call_set_lhs (g, simt_vf);
7008 0 : gimple_seq_add_stmt (dlist, g);
7009 :
7010 0 : tree t = build_int_cst (unsigned_type_node, 1);
7011 0 : g = gimple_build_assign (simt_lane, INTEGER_CST, t);
7012 0 : gimple_seq_add_stmt (dlist, g);
7013 :
7014 0 : t = build_int_cst (unsigned_type_node, 0);
7015 0 : g = gimple_build_assign (sctx.idx, INTEGER_CST, t);
7016 0 : gimple_seq_add_stmt (dlist, g);
7017 :
7018 0 : tree body = create_artificial_label (UNKNOWN_LOCATION);
7019 0 : tree header = create_artificial_label (UNKNOWN_LOCATION);
7020 0 : tree end = create_artificial_label (UNKNOWN_LOCATION);
7021 0 : gimple_seq_add_stmt (dlist, gimple_build_goto (header));
7022 0 : gimple_seq_add_stmt (dlist, gimple_build_label (body));
7023 :
7024 0 : gimple_seq_add_seq (dlist, llist[2]);
7025 :
7026 0 : g = gimple_build_assign (simt_lane, LSHIFT_EXPR, simt_lane, integer_one_node);
7027 0 : gimple_seq_add_stmt (dlist, g);
7028 :
7029 0 : gimple_seq_add_stmt (dlist, gimple_build_label (header));
7030 0 : g = gimple_build_cond (LT_EXPR, simt_lane, simt_vf, body, end);
7031 0 : gimple_seq_add_stmt (dlist, g);
7032 :
7033 0 : gimple_seq_add_stmt (dlist, gimple_build_label (end));
7034 : }
7035 12021 : for (int i = 0; i < 2; i++)
7036 8014 : if (llist[i])
7037 : {
7038 1730 : tree vf = create_tmp_var (unsigned_type_node);
7039 1730 : g = gimple_build_call_internal (IFN_GOMP_SIMD_VF, 1, uid);
7040 1730 : gimple_call_set_lhs (g, vf);
7041 1730 : gimple_seq *seq = i == 0 ? ilist : dlist;
7042 1730 : gimple_seq_add_stmt (seq, g);
7043 1730 : tree t = build_int_cst (unsigned_type_node, 0);
7044 1730 : g = gimple_build_assign (sctx.idx, INTEGER_CST, t);
7045 1730 : gimple_seq_add_stmt (seq, g);
7046 1730 : tree body = create_artificial_label (UNKNOWN_LOCATION);
7047 1730 : tree header = create_artificial_label (UNKNOWN_LOCATION);
7048 1730 : tree end = create_artificial_label (UNKNOWN_LOCATION);
7049 1730 : gimple_seq_add_stmt (seq, gimple_build_goto (header));
7050 1730 : gimple_seq_add_stmt (seq, gimple_build_label (body));
7051 1730 : gimple_seq_add_seq (seq, llist[i]);
7052 1730 : t = build_int_cst (unsigned_type_node, 1);
7053 1730 : g = gimple_build_assign (sctx.idx, PLUS_EXPR, sctx.idx, t);
7054 1730 : gimple_seq_add_stmt (seq, g);
7055 1730 : gimple_seq_add_stmt (seq, gimple_build_label (header));
7056 1730 : g = gimple_build_cond (LT_EXPR, sctx.idx, vf, body, end);
7057 1730 : gimple_seq_add_stmt (seq, g);
7058 1730 : gimple_seq_add_stmt (seq, gimple_build_label (end));
7059 : }
7060 : }
7061 77762 : if (sctx.is_simt)
7062 : {
7063 0 : gimple_seq_add_seq (dlist, sctx.simt_dlist);
7064 0 : gimple *g
7065 0 : = gimple_build_call_internal (IFN_GOMP_SIMT_EXIT, 1, simtrec);
7066 0 : gimple_seq_add_stmt (dlist, g);
7067 : }
7068 :
7069 : /* The copyin sequence is not to be executed by the main thread, since
7070 : that would result in self-copies. Perhaps not visible to scalars,
7071 : but it certainly is to C++ operator=. */
7072 77762 : if (copyin_seq)
7073 : {
7074 440 : x = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM),
7075 : 0);
7076 440 : x = build2 (NE_EXPR, boolean_type_node, x,
7077 440 : build_int_cst (TREE_TYPE (x), 0));
7078 440 : x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL);
7079 440 : gimplify_and_add (x, ilist);
7080 : }
7081 :
7082 : /* If any copyin variable is passed by reference, we must ensure the
7083 : master thread doesn't modify it before it is copied over in all
7084 : threads. Similarly for variables in both firstprivate and
7085 : lastprivate clauses we need to ensure the lastprivate copying
7086 : happens after firstprivate copying in all threads. And similarly
7087 : for UDRs if initializer expression refers to omp_orig. */
7088 77762 : if (copyin_by_ref || lastprivate_firstprivate
7089 75822 : || (reduction_omp_orig_ref
7090 101 : && !ctx->scan_inclusive
7091 101 : && !ctx->scan_exclusive))
7092 : {
7093 : /* Don't add any barrier for #pragma omp simd or
7094 : #pragma omp distribute. */
7095 2041 : if (!is_task_ctx (ctx)
7096 2041 : && (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
7097 1486 : || gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_FOR))
7098 : {
7099 904 : int barrier_kind;
7100 904 : switch (gimple_code (ctx->stmt))
7101 : {
7102 : case GIMPLE_OMP_SECTIONS:
7103 : case GIMPLE_OMP_SCOPE:
7104 : case GIMPLE_OMP_FOR:
7105 : barrier_kind = GOMP_BARRIER_IMPLICIT_WORKSHARE;
7106 : break;
7107 540 : case GIMPLE_OMP_PARALLEL:
7108 540 : case GIMPLE_OMP_TEAMS:
7109 540 : barrier_kind = GOMP_BARRIER_IMPLICIT_PARALLEL;
7110 540 : break;
7111 0 : default:
7112 0 : gcc_unreachable ();
7113 : }
7114 904 : gimple_seq_add_stmt (ilist,
7115 : omp_build_barrier (NULL_TREE, barrier_kind));
7116 : }
7117 : }
7118 :
7119 : /* If max_vf is non-zero, then we can use only a vectorization factor
7120 : up to the max_vf we chose. So stick it into the safelen clause. */
7121 77762 : if (maybe_ne (sctx.max_vf, 0U))
7122 : {
7123 5231 : tree c = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
7124 : OMP_CLAUSE_SAFELEN);
7125 5231 : poly_uint64 safe_len;
7126 5231 : if (c == NULL_TREE
7127 5231 : || (poly_int_tree_p (OMP_CLAUSE_SAFELEN_EXPR (c), &safe_len)
7128 618 : && maybe_gt (safe_len, sctx.max_vf)))
7129 : {
7130 5148 : c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_SAFELEN);
7131 5148 : OMP_CLAUSE_SAFELEN_EXPR (c) = build_int_cst (integer_type_node,
7132 5148 : sctx.max_vf);
7133 5148 : OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
7134 5148 : gimple_omp_for_set_clauses (ctx->stmt, c);
7135 : }
7136 : }
7137 77762 : }
7138 :
7139 : /* Create temporary variables for lastprivate(conditional:) implementation
7140 : in context CTX with CLAUSES. */
7141 :
7142 : static void
7143 47488 : lower_lastprivate_conditional_clauses (tree *clauses, omp_context *ctx)
7144 : {
7145 47488 : tree iter_type = NULL_TREE;
7146 47488 : tree cond_ptr = NULL_TREE;
7147 47488 : tree iter_var = NULL_TREE;
7148 47488 : bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
7149 47488 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD);
7150 47488 : tree next = *clauses;
7151 227790 : for (tree c = *clauses; c; c = OMP_CLAUSE_CHAIN (c))
7152 180302 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7153 180302 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
7154 : {
7155 386 : if (is_simd)
7156 : {
7157 107 : tree cc = omp_find_clause (next, OMP_CLAUSE__CONDTEMP_);
7158 107 : gcc_assert (cc);
7159 107 : if (iter_type == NULL_TREE)
7160 : {
7161 77 : iter_type = TREE_TYPE (OMP_CLAUSE_DECL (cc));
7162 77 : iter_var = create_tmp_var_raw (iter_type);
7163 77 : DECL_CONTEXT (iter_var) = current_function_decl;
7164 77 : DECL_SEEN_IN_BIND_EXPR_P (iter_var) = 1;
7165 77 : DECL_CHAIN (iter_var) = ctx->block_vars;
7166 77 : ctx->block_vars = iter_var;
7167 77 : tree c3
7168 77 : = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__CONDTEMP_);
7169 77 : OMP_CLAUSE__CONDTEMP__ITER (c3) = 1;
7170 77 : OMP_CLAUSE_DECL (c3) = iter_var;
7171 77 : OMP_CLAUSE_CHAIN (c3) = *clauses;
7172 77 : *clauses = c3;
7173 77 : ctx->lastprivate_conditional_map = new hash_map<tree, tree>;
7174 : }
7175 107 : next = OMP_CLAUSE_CHAIN (cc);
7176 107 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
7177 107 : tree v = lookup_decl (OMP_CLAUSE_DECL (cc), ctx);
7178 107 : ctx->lastprivate_conditional_map->put (o, v);
7179 107 : continue;
7180 107 : }
7181 279 : if (iter_type == NULL)
7182 : {
7183 205 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR)
7184 : {
7185 193 : struct omp_for_data fd;
7186 193 : omp_extract_for_data (as_a <gomp_for *> (ctx->stmt), &fd,
7187 : NULL);
7188 193 : iter_type = unsigned_type_for (fd.iter_type);
7189 : }
7190 12 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
7191 12 : iter_type = unsigned_type_node;
7192 205 : tree c2 = omp_find_clause (*clauses, OMP_CLAUSE__CONDTEMP_);
7193 205 : if (c2)
7194 : {
7195 103 : cond_ptr
7196 103 : = lookup_decl_in_outer_ctx (OMP_CLAUSE_DECL (c2), ctx);
7197 103 : OMP_CLAUSE_DECL (c2) = cond_ptr;
7198 : }
7199 : else
7200 : {
7201 102 : cond_ptr = create_tmp_var_raw (build_pointer_type (iter_type));
7202 102 : DECL_CONTEXT (cond_ptr) = current_function_decl;
7203 102 : DECL_SEEN_IN_BIND_EXPR_P (cond_ptr) = 1;
7204 102 : DECL_CHAIN (cond_ptr) = ctx->block_vars;
7205 102 : ctx->block_vars = cond_ptr;
7206 102 : c2 = build_omp_clause (UNKNOWN_LOCATION,
7207 : OMP_CLAUSE__CONDTEMP_);
7208 102 : OMP_CLAUSE_DECL (c2) = cond_ptr;
7209 102 : OMP_CLAUSE_CHAIN (c2) = *clauses;
7210 102 : *clauses = c2;
7211 : }
7212 205 : iter_var = create_tmp_var_raw (iter_type);
7213 205 : DECL_CONTEXT (iter_var) = current_function_decl;
7214 205 : DECL_SEEN_IN_BIND_EXPR_P (iter_var) = 1;
7215 205 : DECL_CHAIN (iter_var) = ctx->block_vars;
7216 205 : ctx->block_vars = iter_var;
7217 205 : tree c3
7218 205 : = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__CONDTEMP_);
7219 205 : OMP_CLAUSE__CONDTEMP__ITER (c3) = 1;
7220 205 : OMP_CLAUSE_DECL (c3) = iter_var;
7221 205 : OMP_CLAUSE_CHAIN (c3) = OMP_CLAUSE_CHAIN (c2);
7222 205 : OMP_CLAUSE_CHAIN (c2) = c3;
7223 205 : ctx->lastprivate_conditional_map = new hash_map<tree, tree>;
7224 : }
7225 279 : tree v = create_tmp_var_raw (iter_type);
7226 279 : DECL_CONTEXT (v) = current_function_decl;
7227 279 : DECL_SEEN_IN_BIND_EXPR_P (v) = 1;
7228 279 : DECL_CHAIN (v) = ctx->block_vars;
7229 279 : ctx->block_vars = v;
7230 279 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
7231 279 : ctx->lastprivate_conditional_map->put (o, v);
7232 : }
7233 47488 : }
7234 :
7235 :
7236 : /* Generate code to implement the LASTPRIVATE clauses. This is used for
7237 : both parallel and workshare constructs. PREDICATE may be NULL if it's
7238 : always true. BODY_P is the sequence to insert early initialization
7239 : if needed, STMT_LIST is where the non-conditional lastprivate handling
7240 : goes into and CSTMT_LIST is a sequence that needs to be run in a critical
7241 : section. */
7242 :
7243 : static void
7244 47488 : lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *body_p,
7245 : gimple_seq *stmt_list, gimple_seq *cstmt_list,
7246 : omp_context *ctx)
7247 : {
7248 47488 : tree x, c, label = NULL, orig_clauses = clauses;
7249 47488 : bool par_clauses = false;
7250 47488 : tree simduid = NULL, lastlane = NULL, simtcond = NULL, simtlast = NULL;
7251 47488 : unsigned HOST_WIDE_INT conditional_off = 0;
7252 47488 : gimple_seq post_stmt_list = NULL;
7253 :
7254 : /* Early exit if there are no lastprivate or linear clauses. */
7255 197051 : for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
7256 162059 : if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LASTPRIVATE
7257 162059 : || (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LINEAR
7258 6797 : && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses)))
7259 : break;
7260 47488 : if (clauses == NULL)
7261 : {
7262 : /* If this was a workshare clause, see if it had been combined
7263 : with its parallel. In that case, look for the clauses on the
7264 : parallel statement itself. */
7265 34992 : if (is_parallel_ctx (ctx))
7266 30989 : return;
7267 :
7268 34992 : ctx = ctx->outer;
7269 34992 : if (ctx == NULL || !is_parallel_ctx (ctx))
7270 : return;
7271 :
7272 12312 : clauses = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
7273 : OMP_CLAUSE_LASTPRIVATE);
7274 12312 : if (clauses == NULL)
7275 : return;
7276 : par_clauses = true;
7277 : }
7278 :
7279 16499 : bool maybe_simt = false;
7280 16499 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
7281 16499 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
7282 : {
7283 7431 : maybe_simt = omp_find_clause (orig_clauses, OMP_CLAUSE__SIMT_);
7284 7431 : simduid = omp_find_clause (orig_clauses, OMP_CLAUSE__SIMDUID_);
7285 7431 : if (simduid)
7286 3552 : simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
7287 : }
7288 :
7289 16499 : if (predicate)
7290 : {
7291 16376 : gcond *stmt;
7292 16376 : tree label_true, arm1, arm2;
7293 16376 : enum tree_code pred_code = TREE_CODE (predicate);
7294 :
7295 16376 : label = create_artificial_label (UNKNOWN_LOCATION);
7296 16376 : label_true = create_artificial_label (UNKNOWN_LOCATION);
7297 16376 : if (TREE_CODE_CLASS (pred_code) == tcc_comparison)
7298 : {
7299 16376 : arm1 = TREE_OPERAND (predicate, 0);
7300 16376 : arm2 = TREE_OPERAND (predicate, 1);
7301 16376 : gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
7302 16376 : gimplify_expr (&arm2, stmt_list, NULL, is_gimple_val, fb_rvalue);
7303 : }
7304 : else
7305 : {
7306 0 : arm1 = predicate;
7307 0 : gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
7308 0 : arm2 = boolean_false_node;
7309 0 : pred_code = NE_EXPR;
7310 : }
7311 16376 : if (maybe_simt)
7312 : {
7313 0 : c = build2 (pred_code, boolean_type_node, arm1, arm2);
7314 0 : c = fold_convert (integer_type_node, c);
7315 0 : simtcond = create_tmp_var (integer_type_node);
7316 0 : gimplify_assign (simtcond, c, stmt_list);
7317 0 : gcall *g = gimple_build_call_internal (IFN_GOMP_SIMT_VOTE_ANY,
7318 : 1, simtcond);
7319 0 : c = create_tmp_var (integer_type_node);
7320 0 : gimple_call_set_lhs (g, c);
7321 0 : gimple_seq_add_stmt (stmt_list, g);
7322 0 : stmt = gimple_build_cond (NE_EXPR, c, integer_zero_node,
7323 : label_true, label);
7324 : }
7325 : else
7326 16376 : stmt = gimple_build_cond (pred_code, arm1, arm2, label_true, label);
7327 16376 : gimple_seq_add_stmt (stmt_list, stmt);
7328 16376 : gimple_seq_add_stmt (stmt_list, gimple_build_label (label_true));
7329 : }
7330 :
7331 16499 : tree cond_ptr = NULL_TREE;
7332 61608 : for (c = clauses; c ;)
7333 : {
7334 56376 : tree var, new_var;
7335 56376 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
7336 56376 : gimple_seq *this_stmt_list = stmt_list;
7337 56376 : tree lab2 = NULL_TREE;
7338 :
7339 56376 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7340 21824 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c)
7341 386 : && ctx->lastprivate_conditional_map
7342 56749 : && !ctx->combined_into_simd_safelen1)
7343 : {
7344 316 : gcc_assert (body_p);
7345 316 : if (simduid)
7346 37 : goto next;
7347 279 : if (cond_ptr == NULL_TREE)
7348 : {
7349 205 : cond_ptr = omp_find_clause (orig_clauses, OMP_CLAUSE__CONDTEMP_);
7350 205 : cond_ptr = OMP_CLAUSE_DECL (cond_ptr);
7351 : }
7352 279 : tree type = TREE_TYPE (TREE_TYPE (cond_ptr));
7353 279 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
7354 279 : tree v = *ctx->lastprivate_conditional_map->get (o);
7355 279 : gimplify_assign (v, build_zero_cst (type), body_p);
7356 279 : this_stmt_list = cstmt_list;
7357 279 : tree mem;
7358 279 : if (POINTER_TYPE_P (TREE_TYPE (cond_ptr)))
7359 : {
7360 136 : mem = build2 (MEM_REF, type, cond_ptr,
7361 136 : build_int_cst (TREE_TYPE (cond_ptr),
7362 136 : conditional_off));
7363 136 : conditional_off += tree_to_uhwi (TYPE_SIZE_UNIT (type));
7364 : }
7365 : else
7366 143 : mem = build4 (ARRAY_REF, type, cond_ptr,
7367 143 : size_int (conditional_off++), NULL_TREE, NULL_TREE);
7368 279 : tree mem2 = copy_node (mem);
7369 279 : gimple_seq seq = NULL;
7370 279 : mem = force_gimple_operand (mem, &seq, true, NULL_TREE);
7371 279 : gimple_seq_add_seq (this_stmt_list, seq);
7372 279 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
7373 279 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
7374 279 : gimple *g = gimple_build_cond (GT_EXPR, v, mem, lab1, lab2);
7375 279 : gimple_seq_add_stmt (this_stmt_list, g);
7376 279 : gimple_seq_add_stmt (this_stmt_list, gimple_build_label (lab1));
7377 279 : gimplify_assign (mem2, v, this_stmt_list);
7378 : }
7379 56060 : else if (predicate
7380 55814 : && ctx->combined_into_simd_safelen1
7381 116 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7382 57 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c)
7383 56117 : && ctx->lastprivate_conditional_map)
7384 : this_stmt_list = &post_stmt_list;
7385 :
7386 56339 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7387 56339 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
7388 4874 : && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c)))
7389 : {
7390 26661 : var = OMP_CLAUSE_DECL (c);
7391 26661 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7392 21787 : && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)
7393 27692 : && is_taskloop_ctx (ctx))
7394 : {
7395 312 : gcc_checking_assert (ctx->outer && is_task_ctx (ctx->outer));
7396 312 : new_var = lookup_decl (var, ctx->outer);
7397 : }
7398 : else
7399 : {
7400 26349 : new_var = lookup_decl (var, ctx);
7401 : /* Avoid uninitialized warnings for lastprivate and
7402 : for linear iterators. */
7403 26349 : if (predicate
7404 52505 : && (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7405 4874 : || OMP_CLAUSE_LINEAR_NO_COPYIN (c)))
7406 24887 : suppress_warning (new_var, OPT_Wuninitialized);
7407 : }
7408 :
7409 26661 : if (!maybe_simt && simduid && DECL_HAS_VALUE_EXPR_P (new_var))
7410 : {
7411 5864 : tree val = DECL_VALUE_EXPR (new_var);
7412 5864 : if (TREE_CODE (val) == ARRAY_REF
7413 5816 : && VAR_P (TREE_OPERAND (val, 0))
7414 11680 : && lookup_attribute ("omp simd array",
7415 5816 : DECL_ATTRIBUTES (TREE_OPERAND (val,
7416 : 0))))
7417 : {
7418 5816 : if (lastlane == NULL)
7419 : {
7420 2759 : lastlane = create_tmp_var (unsigned_type_node);
7421 2759 : gcall *g
7422 2759 : = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
7423 : 2, simduid,
7424 2759 : TREE_OPERAND (val, 1));
7425 2759 : gimple_call_set_lhs (g, lastlane);
7426 2759 : gimple_seq_add_stmt (this_stmt_list, g);
7427 : }
7428 5816 : new_var = build4 (ARRAY_REF, TREE_TYPE (val),
7429 5816 : TREE_OPERAND (val, 0), lastlane,
7430 : NULL_TREE, NULL_TREE);
7431 5816 : TREE_THIS_NOTRAP (new_var) = 1;
7432 : }
7433 : }
7434 20797 : else if (maybe_simt)
7435 : {
7436 0 : tree val = (DECL_HAS_VALUE_EXPR_P (new_var)
7437 0 : ? DECL_VALUE_EXPR (new_var)
7438 : : new_var);
7439 0 : if (simtlast == NULL)
7440 : {
7441 0 : simtlast = create_tmp_var (unsigned_type_node);
7442 0 : gcall *g = gimple_build_call_internal
7443 0 : (IFN_GOMP_SIMT_LAST_LANE, 1, simtcond);
7444 0 : gimple_call_set_lhs (g, simtlast);
7445 0 : gimple_seq_add_stmt (this_stmt_list, g);
7446 : }
7447 0 : x = build_call_expr_internal_loc
7448 0 : (UNKNOWN_LOCATION, IFN_GOMP_SIMT_XCHG_IDX,
7449 0 : TREE_TYPE (val), 2, val, simtlast);
7450 0 : new_var = unshare_expr (new_var);
7451 0 : gimplify_assign (new_var, x, this_stmt_list);
7452 0 : new_var = unshare_expr (new_var);
7453 : }
7454 :
7455 26661 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7456 26661 : && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
7457 : {
7458 7159 : lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
7459 7159 : gimple_seq_add_seq (this_stmt_list,
7460 7159 : OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c));
7461 7159 : OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL;
7462 : }
7463 19502 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
7464 19502 : && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
7465 : {
7466 426 : lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
7467 426 : gimple_seq_add_seq (this_stmt_list,
7468 426 : OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c));
7469 426 : OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c) = NULL;
7470 : }
7471 :
7472 26661 : x = NULL_TREE;
7473 26661 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7474 21787 : && OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c)
7475 26721 : && is_taskloop_ctx (ctx))
7476 : {
7477 76 : tree ovar = maybe_lookup_decl_in_outer_ctx (var,
7478 38 : ctx->outer->outer);
7479 38 : if (is_global_var (ovar))
7480 7 : x = ovar;
7481 : }
7482 7 : if (!x)
7483 26654 : x = build_outer_var_ref (var, ctx, OMP_CLAUSE_LASTPRIVATE);
7484 26661 : if (omp_privatize_by_reference (var))
7485 657 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
7486 26661 : x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var);
7487 26661 : gimplify_and_add (x, this_stmt_list);
7488 :
7489 26661 : if (lab2)
7490 279 : gimple_seq_add_stmt (this_stmt_list, gimple_build_label (lab2));
7491 : }
7492 :
7493 56376 : next:
7494 56376 : c = OMP_CLAUSE_CHAIN (c);
7495 56376 : if (c == NULL && !par_clauses)
7496 : {
7497 : /* If this was a workshare clause, see if it had been combined
7498 : with its parallel. In that case, continue looking for the
7499 : clauses also on the parallel statement itself. */
7500 12496 : if (is_parallel_ctx (ctx))
7501 : break;
7502 :
7503 12496 : ctx = ctx->outer;
7504 12496 : if (ctx == NULL || !is_parallel_ctx (ctx))
7505 : break;
7506 :
7507 1229 : c = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
7508 : OMP_CLAUSE_LASTPRIVATE);
7509 1229 : par_clauses = true;
7510 : }
7511 : }
7512 :
7513 16499 : if (label)
7514 16376 : gimple_seq_add_stmt (stmt_list, gimple_build_label (label));
7515 16499 : gimple_seq_add_seq (stmt_list, post_stmt_list);
7516 : }
7517 :
7518 : /* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL
7519 : (which might be a placeholder). INNER is true if this is an inner
7520 : axis of a multi-axis loop. FORK and JOIN are (optional) fork and
7521 : join markers. Generate the before-loop forking sequence in
7522 : FORK_SEQ and the after-loop joining sequence to JOIN_SEQ. The
7523 : general form of these sequences is
7524 :
7525 : GOACC_REDUCTION_SETUP
7526 : GOACC_FORK
7527 : GOACC_REDUCTION_INIT
7528 : ...
7529 : GOACC_REDUCTION_FINI
7530 : GOACC_JOIN
7531 : GOACC_REDUCTION_TEARDOWN. */
7532 :
7533 : static void
7534 25946 : lower_oacc_reductions (location_t loc, tree clauses, tree level, bool inner,
7535 : gcall *fork, gcall *private_marker, gcall *join,
7536 : gimple_seq *fork_seq, gimple_seq *join_seq,
7537 : omp_context *ctx)
7538 : {
7539 25946 : gimple_seq before_fork = NULL;
7540 25946 : gimple_seq after_fork = NULL;
7541 25946 : gimple_seq before_join = NULL;
7542 25946 : gimple_seq after_join = NULL;
7543 25946 : tree init_code = NULL_TREE, fini_code = NULL_TREE,
7544 25946 : setup_code = NULL_TREE, teardown_code = NULL_TREE;
7545 25946 : unsigned offset = 0;
7546 :
7547 96516 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
7548 70570 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
7549 : {
7550 : /* No 'reduction' clauses on OpenACC 'kernels'. */
7551 7751 : gcc_checking_assert (!is_oacc_kernels (ctx));
7552 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
7553 7751 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
7554 :
7555 7751 : tree orig = OMP_CLAUSE_DECL (c);
7556 7751 : tree orig_clause;
7557 7751 : tree var = maybe_lookup_decl (orig, ctx);
7558 7751 : tree ref_to_res = NULL_TREE;
7559 7751 : tree incoming, outgoing, v1, v2, v3;
7560 7751 : bool is_private = false;
7561 :
7562 7751 : enum tree_code rcode = OMP_CLAUSE_REDUCTION_CODE (c);
7563 7751 : if (rcode == MINUS_EXPR)
7564 : rcode = PLUS_EXPR;
7565 6609 : else if (rcode == TRUTH_ANDIF_EXPR)
7566 : rcode = BIT_AND_EXPR;
7567 6431 : else if (rcode == TRUTH_ORIF_EXPR)
7568 433 : rcode = BIT_IOR_EXPR;
7569 7751 : tree op = build_int_cst (unsigned_type_node, rcode);
7570 :
7571 7751 : if (!var)
7572 4 : var = orig;
7573 :
7574 7751 : incoming = outgoing = var;
7575 :
7576 7751 : if (!inner)
7577 : {
7578 : /* See if an outer construct also reduces this variable. */
7579 : omp_context *outer = ctx;
7580 :
7581 7385 : while (omp_context *probe = outer->outer)
7582 : {
7583 4708 : enum gimple_code type = gimple_code (probe->stmt);
7584 4708 : tree cls;
7585 :
7586 4708 : switch (type)
7587 : {
7588 2220 : case GIMPLE_OMP_FOR:
7589 2220 : cls = gimple_omp_for_clauses (probe->stmt);
7590 2220 : break;
7591 :
7592 2488 : case GIMPLE_OMP_TARGET:
7593 : /* No 'reduction' clauses inside OpenACC 'kernels'
7594 : regions. */
7595 2488 : gcc_checking_assert (!is_oacc_kernels (probe));
7596 :
7597 2488 : if (!is_gimple_omp_offloaded (probe->stmt))
7598 26 : goto do_lookup;
7599 :
7600 2462 : cls = gimple_omp_target_clauses (probe->stmt);
7601 2462 : break;
7602 :
7603 0 : default:
7604 0 : goto do_lookup;
7605 : }
7606 :
7607 4682 : outer = probe;
7608 14249 : for (; cls; cls = OMP_CLAUSE_CHAIN (cls))
7609 11632 : if (OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_REDUCTION
7610 11632 : && orig == OMP_CLAUSE_DECL (cls))
7611 : {
7612 1772 : incoming = outgoing = lookup_decl (orig, probe);
7613 1772 : goto has_outer_reduction;
7614 : }
7615 9860 : else if ((OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_FIRSTPRIVATE
7616 8971 : || OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_PRIVATE)
7617 11164 : && orig == OMP_CLAUSE_DECL (cls))
7618 : {
7619 293 : is_private = true;
7620 293 : goto do_lookup;
7621 : }
7622 : }
7623 :
7624 2677 : do_lookup:
7625 : /* This is the outermost construct with this reduction,
7626 : see if there's a mapping for it. */
7627 2996 : orig_clause = NULL_TREE;
7628 2996 : if (gimple_code (outer->stmt) == GIMPLE_OMP_TARGET)
7629 7557 : for (tree cls = gimple_omp_target_clauses (outer->stmt);
7630 7557 : cls; cls = OMP_CLAUSE_CHAIN (cls))
7631 7250 : if (OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_MAP
7632 4823 : && orig == OMP_CLAUSE_DECL (cls)
7633 9805 : && maybe_lookup_field (cls, outer))
7634 : {
7635 : orig_clause = cls;
7636 : break;
7637 : }
7638 2996 : if (orig_clause != NULL_TREE && !is_private)
7639 : {
7640 2553 : ref_to_res = build_receiver_ref (orig_clause, false, outer);
7641 2553 : if (omp_privatize_by_reference (orig))
7642 79 : ref_to_res = build_simple_mem_ref (ref_to_res);
7643 :
7644 2553 : tree type = TREE_TYPE (var);
7645 2553 : if (POINTER_TYPE_P (type))
7646 79 : type = TREE_TYPE (type);
7647 :
7648 2553 : outgoing = var;
7649 2553 : incoming = omp_reduction_init_op (loc, rcode, type);
7650 : }
7651 : else
7652 : {
7653 : /* Try to look at enclosing contexts for reduction var,
7654 : use original if no mapping found. */
7655 443 : tree t = NULL_TREE;
7656 443 : omp_context *c = ctx->outer;
7657 895 : while (c && !t)
7658 : {
7659 452 : t = maybe_lookup_decl (orig, c);
7660 452 : c = c->outer;
7661 : }
7662 443 : incoming = outgoing = (t ? t : orig);
7663 : }
7664 :
7665 2553 : has_outer_reduction:;
7666 : }
7667 :
7668 4325 : if (!ref_to_res)
7669 5198 : ref_to_res = integer_zero_node;
7670 :
7671 7751 : if (omp_privatize_by_reference (orig))
7672 : {
7673 159 : tree type = TREE_TYPE (var);
7674 159 : const char *id = IDENTIFIER_POINTER (DECL_NAME (var));
7675 :
7676 159 : if (!inner)
7677 : {
7678 110 : tree x = create_tmp_var (TREE_TYPE (type), id);
7679 110 : gimplify_assign (var, build_fold_addr_expr (x), fork_seq);
7680 : }
7681 :
7682 159 : v1 = create_tmp_var (type, id);
7683 159 : v2 = create_tmp_var (type, id);
7684 159 : v3 = create_tmp_var (type, id);
7685 :
7686 159 : gimplify_assign (v1, var, fork_seq);
7687 159 : gimplify_assign (v2, var, fork_seq);
7688 159 : gimplify_assign (v3, var, fork_seq);
7689 :
7690 159 : var = build_simple_mem_ref (var);
7691 159 : v1 = build_simple_mem_ref (v1);
7692 159 : v2 = build_simple_mem_ref (v2);
7693 159 : v3 = build_simple_mem_ref (v3);
7694 159 : outgoing = build_simple_mem_ref (outgoing);
7695 :
7696 159 : if (!TREE_CONSTANT (incoming))
7697 80 : incoming = build_simple_mem_ref (incoming);
7698 : }
7699 : else
7700 : /* Note that 'var' might be a mem ref. */
7701 : v1 = v2 = v3 = var;
7702 :
7703 : /* Determine position in reduction buffer, which may be used
7704 : by target. The parser has ensured that this is not a
7705 : variable-sized type. */
7706 7751 : fixed_size_mode mode
7707 7751 : = as_a <fixed_size_mode> (TYPE_MODE (TREE_TYPE (var)));
7708 7751 : unsigned align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT;
7709 7751 : offset = (offset + align - 1) & ~(align - 1);
7710 7751 : tree off = build_int_cst (sizetype, offset);
7711 7751 : offset += GET_MODE_SIZE (mode);
7712 :
7713 7751 : if (!init_code)
7714 : {
7715 6969 : init_code = build_int_cst (integer_type_node,
7716 : IFN_GOACC_REDUCTION_INIT);
7717 6969 : fini_code = build_int_cst (integer_type_node,
7718 : IFN_GOACC_REDUCTION_FINI);
7719 6969 : setup_code = build_int_cst (integer_type_node,
7720 : IFN_GOACC_REDUCTION_SETUP);
7721 6969 : teardown_code = build_int_cst (integer_type_node,
7722 : IFN_GOACC_REDUCTION_TEARDOWN);
7723 : }
7724 :
7725 7751 : tree setup_call
7726 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7727 7751 : TREE_TYPE (var), 6, setup_code,
7728 : unshare_expr (ref_to_res),
7729 : unshare_expr (incoming),
7730 : level, op, off);
7731 7751 : tree init_call
7732 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7733 7751 : TREE_TYPE (var), 6, init_code,
7734 : unshare_expr (ref_to_res),
7735 : unshare_expr (v1), level, op, off);
7736 7751 : tree fini_call
7737 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7738 7751 : TREE_TYPE (var), 6, fini_code,
7739 : unshare_expr (ref_to_res),
7740 : unshare_expr (v2), level, op, off);
7741 7751 : tree teardown_call
7742 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7743 7751 : TREE_TYPE (var), 6, teardown_code,
7744 : ref_to_res, unshare_expr (v3),
7745 : level, op, off);
7746 :
7747 7751 : gimplify_assign (unshare_expr (v1), setup_call, &before_fork);
7748 7751 : gimplify_assign (unshare_expr (v2), init_call, &after_fork);
7749 7751 : gimplify_assign (unshare_expr (v3), fini_call, &before_join);
7750 7751 : gimplify_assign (unshare_expr (outgoing), teardown_call, &after_join);
7751 : }
7752 :
7753 : /* Now stitch things together. */
7754 25946 : gimple_seq_add_seq (fork_seq, before_fork);
7755 25946 : if (private_marker)
7756 259 : gimple_seq_add_stmt (fork_seq, private_marker);
7757 25946 : if (fork)
7758 16551 : gimple_seq_add_stmt (fork_seq, fork);
7759 25946 : gimple_seq_add_seq (fork_seq, after_fork);
7760 :
7761 25946 : gimple_seq_add_seq (join_seq, before_join);
7762 25946 : if (join)
7763 16551 : gimple_seq_add_stmt (join_seq, join);
7764 25946 : gimple_seq_add_seq (join_seq, after_join);
7765 25946 : }
7766 :
7767 : /* Generate code to implement the REDUCTION clauses, append it
7768 : to STMT_SEQP. CLIST if non-NULL is a pointer to a sequence
7769 : that should be emitted also inside of the critical section,
7770 : in that case clear *CLIST afterwards, otherwise leave it as is
7771 : and let the caller emit it itself. */
7772 :
7773 : static void
7774 72466 : lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp,
7775 : gimple_seq *clist, omp_context *ctx)
7776 : {
7777 72466 : gimple_seq sub_seq = NULL;
7778 72466 : gimple *stmt;
7779 72466 : tree x, c;
7780 72466 : int count = 0;
7781 :
7782 : /* OpenACC loop reductions are handled elsewhere. */
7783 72466 : if (is_gimple_omp_oacc (ctx->stmt))
7784 71332 : return;
7785 :
7786 : /* SIMD reductions are handled in lower_rec_input_clauses. */
7787 61141 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
7788 61141 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
7789 : return;
7790 :
7791 : /* inscan reductions are handled elsewhere. */
7792 51751 : if (ctx->scan_inclusive || ctx->scan_exclusive)
7793 : return;
7794 :
7795 : /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
7796 : update in that case, otherwise use a lock. */
7797 243639 : for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c))
7798 193010 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
7799 193010 : && !OMP_CLAUSE_REDUCTION_TASK (c))
7800 : {
7801 4727 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)
7802 4727 : || TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF)
7803 : {
7804 : /* Never use OMP_ATOMIC for array reductions or UDRs. */
7805 : count = -1;
7806 : break;
7807 : }
7808 3790 : count++;
7809 : }
7810 :
7811 51566 : if (count == 0)
7812 : return;
7813 :
7814 15410 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
7815 : {
7816 14276 : tree var, ref, new_var, orig_var;
7817 14276 : enum tree_code code;
7818 14276 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
7819 :
7820 23013 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
7821 14276 : || OMP_CLAUSE_REDUCTION_TASK (c))
7822 8737 : continue;
7823 :
7824 5539 : enum omp_clause_code ccode = OMP_CLAUSE_REDUCTION;
7825 5539 : orig_var = var = OMP_CLAUSE_DECL (c);
7826 5539 : if (TREE_CODE (var) == MEM_REF)
7827 : {
7828 764 : var = TREE_OPERAND (var, 0);
7829 764 : if (TREE_CODE (var) == POINTER_PLUS_EXPR)
7830 50 : var = TREE_OPERAND (var, 0);
7831 764 : if (TREE_CODE (var) == ADDR_EXPR)
7832 322 : var = TREE_OPERAND (var, 0);
7833 : else
7834 : {
7835 : /* If this is a pointer or referenced based array
7836 : section, the var could be private in the outer
7837 : context e.g. on orphaned loop construct. Pretend this
7838 : is private variable's outer reference. */
7839 442 : ccode = OMP_CLAUSE_PRIVATE;
7840 442 : if (INDIRECT_REF_P (var))
7841 72 : var = TREE_OPERAND (var, 0);
7842 : }
7843 764 : orig_var = var;
7844 764 : if (is_variable_sized (var))
7845 : {
7846 25 : gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
7847 25 : var = DECL_VALUE_EXPR (var);
7848 25 : gcc_assert (INDIRECT_REF_P (var));
7849 25 : var = TREE_OPERAND (var, 0);
7850 25 : gcc_assert (DECL_P (var));
7851 : }
7852 : }
7853 5539 : new_var = lookup_decl (var, ctx);
7854 5539 : if (var == OMP_CLAUSE_DECL (c)
7855 5539 : && omp_privatize_by_reference (var))
7856 161 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
7857 5539 : ref = build_outer_var_ref (var, ctx, ccode);
7858 5539 : code = OMP_CLAUSE_REDUCTION_CODE (c);
7859 :
7860 : /* reduction(-:var) sums up the partial results, so it acts
7861 : identically to reduction(+:var). */
7862 5539 : if (code == MINUS_EXPR)
7863 57 : code = PLUS_EXPR;
7864 :
7865 5539 : bool is_truth_op = (code == TRUTH_ANDIF_EXPR || code == TRUTH_ORIF_EXPR);
7866 5539 : if (count == 1)
7867 : {
7868 3341 : tree addr = build_fold_addr_expr_loc (clause_loc, ref);
7869 :
7870 3341 : addr = save_expr (addr);
7871 3341 : ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr);
7872 3341 : tree new_var2 = new_var;
7873 3341 : tree ref2 = ref;
7874 3341 : if (is_truth_op)
7875 : {
7876 784 : tree zero = build_zero_cst (TREE_TYPE (new_var));
7877 784 : new_var2 = fold_build2_loc (clause_loc, NE_EXPR,
7878 : boolean_type_node, new_var, zero);
7879 784 : ref2 = fold_build2_loc (clause_loc, NE_EXPR, boolean_type_node,
7880 : ref, zero);
7881 : }
7882 3341 : x = fold_build2_loc (clause_loc, code, TREE_TYPE (new_var2), ref2,
7883 : new_var2);
7884 3341 : if (is_truth_op)
7885 784 : x = fold_convert (TREE_TYPE (new_var), x);
7886 3341 : x = build2 (OMP_ATOMIC, void_type_node, addr, x);
7887 3341 : OMP_ATOMIC_MEMORY_ORDER (x) = OMP_MEMORY_ORDER_RELAXED;
7888 3341 : gimplify_and_add (x, stmt_seqp);
7889 3341 : return;
7890 : }
7891 2198 : else if (TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF)
7892 : {
7893 764 : tree d = OMP_CLAUSE_DECL (c);
7894 764 : tree type = TREE_TYPE (d);
7895 764 : tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
7896 764 : tree i = create_tmp_var (TREE_TYPE (v));
7897 764 : tree ptype = build_pointer_type (TREE_TYPE (type));
7898 764 : tree bias = TREE_OPERAND (d, 1);
7899 764 : d = TREE_OPERAND (d, 0);
7900 764 : if (TREE_CODE (d) == POINTER_PLUS_EXPR)
7901 : {
7902 50 : tree b = TREE_OPERAND (d, 1);
7903 50 : b = maybe_lookup_decl (b, ctx);
7904 50 : if (b == NULL)
7905 : {
7906 2 : b = TREE_OPERAND (d, 1);
7907 2 : b = maybe_lookup_decl_in_outer_ctx (b, ctx);
7908 : }
7909 50 : if (integer_zerop (bias))
7910 : bias = b;
7911 : else
7912 : {
7913 0 : bias = fold_convert_loc (clause_loc, TREE_TYPE (b), bias);
7914 0 : bias = fold_build2_loc (clause_loc, PLUS_EXPR,
7915 0 : TREE_TYPE (b), b, bias);
7916 : }
7917 50 : d = TREE_OPERAND (d, 0);
7918 : }
7919 : /* For ref build_outer_var_ref already performs this, so
7920 : only new_var needs a dereference. */
7921 764 : if (INDIRECT_REF_P (d))
7922 : {
7923 72 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
7924 72 : gcc_assert (omp_privatize_by_reference (var)
7925 : && var == orig_var);
7926 : }
7927 692 : else if (TREE_CODE (d) == ADDR_EXPR)
7928 : {
7929 322 : if (orig_var == var)
7930 : {
7931 297 : new_var = build_fold_addr_expr (new_var);
7932 297 : ref = build_fold_addr_expr (ref);
7933 : }
7934 : }
7935 : else
7936 : {
7937 370 : gcc_assert (orig_var == var);
7938 370 : if (omp_privatize_by_reference (var))
7939 106 : ref = build_fold_addr_expr (ref);
7940 : }
7941 764 : if (DECL_P (v))
7942 : {
7943 93 : tree t = maybe_lookup_decl (v, ctx);
7944 93 : if (t)
7945 : v = t;
7946 : else
7947 5 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
7948 93 : gimplify_expr (&v, stmt_seqp, NULL, is_gimple_val, fb_rvalue);
7949 : }
7950 764 : if (!integer_zerop (bias))
7951 : {
7952 432 : bias = fold_convert_loc (clause_loc, sizetype, bias);
7953 864 : new_var = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
7954 432 : TREE_TYPE (new_var), new_var,
7955 : unshare_expr (bias));
7956 432 : ref = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
7957 432 : TREE_TYPE (ref), ref, bias);
7958 : }
7959 764 : new_var = fold_convert_loc (clause_loc, ptype, new_var);
7960 764 : ref = fold_convert_loc (clause_loc, ptype, ref);
7961 764 : tree m = create_tmp_var (ptype);
7962 764 : gimplify_assign (m, new_var, stmt_seqp);
7963 764 : new_var = m;
7964 764 : m = create_tmp_var (ptype);
7965 764 : gimplify_assign (m, ref, stmt_seqp);
7966 764 : ref = m;
7967 764 : gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), stmt_seqp);
7968 764 : tree body = create_artificial_label (UNKNOWN_LOCATION);
7969 764 : tree end = create_artificial_label (UNKNOWN_LOCATION);
7970 764 : gimple_seq_add_stmt (&sub_seq, gimple_build_label (body));
7971 764 : tree priv = build_simple_mem_ref_loc (clause_loc, new_var);
7972 764 : tree out = build_simple_mem_ref_loc (clause_loc, ref);
7973 764 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
7974 : {
7975 72 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
7976 72 : tree decl_placeholder
7977 72 : = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
7978 72 : SET_DECL_VALUE_EXPR (placeholder, out);
7979 72 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
7980 72 : SET_DECL_VALUE_EXPR (decl_placeholder, priv);
7981 72 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
7982 72 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
7983 72 : gimple_seq_add_seq (&sub_seq,
7984 72 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
7985 72 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
7986 72 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
7987 72 : OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c) = NULL;
7988 : }
7989 : else
7990 : {
7991 692 : tree out2 = out;
7992 692 : tree priv2 = priv;
7993 692 : if (is_truth_op)
7994 : {
7995 0 : tree zero = build_zero_cst (TREE_TYPE (out));
7996 0 : out2 = fold_build2_loc (clause_loc, NE_EXPR,
7997 : boolean_type_node, out, zero);
7998 0 : priv2 = fold_build2_loc (clause_loc, NE_EXPR,
7999 : boolean_type_node, priv, zero);
8000 : }
8001 692 : x = build2 (code, TREE_TYPE (out2), out2, priv2);
8002 692 : if (is_truth_op)
8003 0 : x = fold_convert (TREE_TYPE (out), x);
8004 692 : out = unshare_expr (out);
8005 692 : gimplify_assign (out, x, &sub_seq);
8006 : }
8007 764 : gimple *g = gimple_build_assign (new_var, POINTER_PLUS_EXPR, new_var,
8008 764 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
8009 764 : gimple_seq_add_stmt (&sub_seq, g);
8010 764 : g = gimple_build_assign (ref, POINTER_PLUS_EXPR, ref,
8011 764 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
8012 764 : gimple_seq_add_stmt (&sub_seq, g);
8013 764 : g = gimple_build_assign (i, PLUS_EXPR, i,
8014 764 : build_int_cst (TREE_TYPE (i), 1));
8015 764 : gimple_seq_add_stmt (&sub_seq, g);
8016 764 : g = gimple_build_cond (LE_EXPR, i, v, body, end);
8017 764 : gimple_seq_add_stmt (&sub_seq, g);
8018 764 : gimple_seq_add_stmt (&sub_seq, gimple_build_label (end));
8019 : }
8020 1434 : else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
8021 : {
8022 788 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
8023 :
8024 788 : if (omp_privatize_by_reference (var)
8025 845 : && !useless_type_conversion_p (TREE_TYPE (placeholder),
8026 57 : TREE_TYPE (ref)))
8027 49 : ref = build_fold_addr_expr_loc (clause_loc, ref);
8028 788 : SET_DECL_VALUE_EXPR (placeholder, ref);
8029 788 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
8030 788 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
8031 788 : gimple_seq_add_seq (&sub_seq, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
8032 788 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
8033 788 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
8034 : }
8035 : else
8036 : {
8037 646 : tree new_var2 = new_var;
8038 646 : tree ref2 = ref;
8039 646 : if (is_truth_op)
8040 : {
8041 78 : tree zero = build_zero_cst (TREE_TYPE (new_var));
8042 78 : new_var2 = fold_build2_loc (clause_loc, NE_EXPR,
8043 : boolean_type_node, new_var, zero);
8044 78 : ref2 = fold_build2_loc (clause_loc, NE_EXPR, boolean_type_node,
8045 : ref, zero);
8046 : }
8047 646 : x = build2 (code, TREE_TYPE (ref), ref2, new_var2);
8048 646 : if (is_truth_op)
8049 78 : x = fold_convert (TREE_TYPE (new_var), x);
8050 646 : ref = build_outer_var_ref (var, ctx);
8051 646 : gimplify_assign (ref, x, &sub_seq);
8052 : }
8053 : }
8054 :
8055 1134 : stmt
8056 1134 : = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_START),
8057 : 0);
8058 1134 : gimple_seq_add_stmt (stmt_seqp, stmt);
8059 :
8060 1134 : gimple_seq_add_seq (stmt_seqp, sub_seq);
8061 :
8062 1134 : if (clist)
8063 : {
8064 173 : gimple_seq_add_seq (stmt_seqp, *clist);
8065 173 : *clist = NULL;
8066 : }
8067 :
8068 1134 : stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_END),
8069 : 0);
8070 1134 : gimple_seq_add_stmt (stmt_seqp, stmt);
8071 : }
8072 :
8073 :
8074 : /* Generate code to implement the COPYPRIVATE clauses. */
8075 :
8076 : static void
8077 118 : lower_copyprivate_clauses (tree clauses, gimple_seq *slist, gimple_seq *rlist,
8078 : omp_context *ctx)
8079 : {
8080 118 : tree c;
8081 :
8082 469 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
8083 : {
8084 351 : tree var, new_var, ref, x;
8085 351 : bool by_ref;
8086 351 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
8087 :
8088 351 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE)
8089 6 : continue;
8090 :
8091 345 : var = OMP_CLAUSE_DECL (c);
8092 345 : by_ref = use_pointer_for_field (var, NULL);
8093 :
8094 345 : ref = build_sender_ref (var, ctx);
8095 345 : x = new_var = lookup_decl_in_outer_ctx (var, ctx);
8096 345 : if (by_ref)
8097 : {
8098 74 : x = build_fold_addr_expr_loc (clause_loc, new_var);
8099 74 : x = fold_convert_loc (clause_loc, TREE_TYPE (ref), x);
8100 : }
8101 345 : gimplify_assign (ref, x, slist);
8102 :
8103 345 : ref = build_receiver_ref (var, false, ctx);
8104 345 : if (by_ref)
8105 : {
8106 74 : ref = fold_convert_loc (clause_loc,
8107 74 : build_pointer_type (TREE_TYPE (new_var)),
8108 : ref);
8109 74 : ref = build_fold_indirect_ref_loc (clause_loc, ref);
8110 : }
8111 345 : if (omp_privatize_by_reference (var))
8112 : {
8113 171 : ref = fold_convert_loc (clause_loc, TREE_TYPE (new_var), ref);
8114 171 : ref = build_simple_mem_ref_loc (clause_loc, ref);
8115 171 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
8116 : }
8117 345 : x = lang_hooks.decls.omp_clause_assign_op (c, new_var, ref);
8118 345 : gimplify_and_add (x, rlist);
8119 : }
8120 118 : }
8121 :
8122 :
8123 : /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
8124 : and REDUCTION from the sender (aka parent) side. */
8125 :
8126 : static void
8127 22721 : lower_send_clauses (tree clauses, gimple_seq *ilist, gimple_seq *olist,
8128 : omp_context *ctx)
8129 : {
8130 22721 : tree c, t;
8131 22721 : int ignored_looptemp = 0;
8132 22721 : bool is_taskloop = false;
8133 :
8134 : /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
8135 : by GOMP_taskloop. */
8136 22721 : if (is_task_ctx (ctx) && gimple_omp_task_taskloop_p (ctx->stmt))
8137 : {
8138 : ignored_looptemp = 2;
8139 : is_taskloop = true;
8140 : }
8141 :
8142 118024 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
8143 : {
8144 95303 : tree val, ref, x, var;
8145 95303 : bool by_ref, do_in = false, do_out = false;
8146 95303 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
8147 :
8148 95303 : switch (OMP_CLAUSE_CODE (c))
8149 : {
8150 4721 : case OMP_CLAUSE_PRIVATE:
8151 4721 : if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
8152 : break;
8153 4566 : continue;
8154 : case OMP_CLAUSE_FIRSTPRIVATE:
8155 : case OMP_CLAUSE_COPYIN:
8156 : case OMP_CLAUSE_LASTPRIVATE:
8157 : case OMP_CLAUSE_IN_REDUCTION:
8158 : case OMP_CLAUSE__REDUCTEMP_:
8159 : break;
8160 4910 : case OMP_CLAUSE_REDUCTION:
8161 4910 : if (is_task_ctx (ctx) || OMP_CLAUSE_REDUCTION_TASK (c))
8162 753 : continue;
8163 : break;
8164 17670 : case OMP_CLAUSE_SHARED:
8165 17670 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
8166 : break;
8167 17358 : continue;
8168 18875 : case OMP_CLAUSE__LOOPTEMP_:
8169 18875 : if (ignored_looptemp)
8170 : {
8171 2672 : ignored_looptemp--;
8172 2672 : continue;
8173 : }
8174 : break;
8175 12132 : default:
8176 12132 : continue;
8177 : }
8178 :
8179 57822 : val = OMP_CLAUSE_DECL (c);
8180 57822 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
8181 53665 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION)
8182 59487 : && TREE_CODE (val) == MEM_REF)
8183 : {
8184 1653 : val = TREE_OPERAND (val, 0);
8185 1653 : if (TREE_CODE (val) == POINTER_PLUS_EXPR)
8186 351 : val = TREE_OPERAND (val, 0);
8187 1653 : if (INDIRECT_REF_P (val)
8188 1551 : || TREE_CODE (val) == ADDR_EXPR)
8189 942 : val = TREE_OPERAND (val, 0);
8190 1653 : if (is_variable_sized (val))
8191 56 : continue;
8192 : }
8193 :
8194 : /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
8195 : outer taskloop region. */
8196 57766 : omp_context *ctx_for_o = ctx;
8197 57766 : if (is_taskloop
8198 3917 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
8199 58078 : && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
8200 312 : ctx_for_o = ctx->outer;
8201 :
8202 57766 : var = lookup_decl_in_outer_ctx (val, ctx_for_o);
8203 :
8204 57766 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN
8205 57252 : && is_global_var (var)
8206 60079 : && (val == OMP_CLAUSE_DECL (c)
8207 273 : || !is_task_ctx (ctx)
8208 241 : || (TREE_CODE (TREE_TYPE (val)) != POINTER_TYPE
8209 187 : && (TREE_CODE (TREE_TYPE (val)) != REFERENCE_TYPE
8210 0 : || (TREE_CODE (TREE_TYPE (TREE_TYPE (val)))
8211 : != POINTER_TYPE)))))
8212 2259 : continue;
8213 :
8214 55507 : t = omp_member_access_dummy_var (var);
8215 55507 : if (t)
8216 : {
8217 249 : var = DECL_VALUE_EXPR (var);
8218 249 : tree o = maybe_lookup_decl_in_outer_ctx (t, ctx_for_o);
8219 249 : if (o != t)
8220 69 : var = unshare_and_remap (var, t, o);
8221 : else
8222 180 : var = unshare_expr (var);
8223 : }
8224 :
8225 55507 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED)
8226 : {
8227 : /* Handle taskloop firstprivate/lastprivate, where the
8228 : lastprivate on GIMPLE_OMP_TASK is represented as
8229 : OMP_CLAUSE_SHARED_FIRSTPRIVATE. */
8230 79 : tree f = lookup_sfield ((splay_tree_key) &DECL_UID (val), ctx);
8231 79 : x = omp_build_component_ref (ctx->sender_decl, f);
8232 79 : if (use_pointer_for_field (val, ctx))
8233 59 : var = build_fold_addr_expr (var);
8234 79 : gimplify_assign (x, var, ilist);
8235 79 : DECL_ABSTRACT_ORIGIN (f) = NULL;
8236 79 : continue;
8237 79 : }
8238 :
8239 55486 : if (((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
8240 52016 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IN_REDUCTION)
8241 4569 : || val == OMP_CLAUSE_DECL (c))
8242 109478 : && is_variable_sized (val))
8243 58 : continue;
8244 55370 : by_ref = use_pointer_for_field (val, NULL);
8245 :
8246 55370 : switch (OMP_CLAUSE_CODE (c))
8247 : {
8248 26904 : case OMP_CLAUSE_FIRSTPRIVATE:
8249 26904 : if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c)
8250 7277 : && !by_ref
8251 34127 : && is_task_ctx (ctx))
8252 2331 : suppress_warning (var);
8253 : do_in = true;
8254 : break;
8255 :
8256 : case OMP_CLAUSE_PRIVATE:
8257 : case OMP_CLAUSE_COPYIN:
8258 : case OMP_CLAUSE__LOOPTEMP_:
8259 : case OMP_CLAUSE__REDUCTEMP_:
8260 : do_in = true;
8261 : break;
8262 :
8263 6467 : case OMP_CLAUSE_LASTPRIVATE:
8264 6467 : if (by_ref || omp_privatize_by_reference (val))
8265 : {
8266 384 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
8267 239 : continue;
8268 : do_in = true;
8269 : }
8270 : else
8271 : {
8272 6083 : do_out = true;
8273 6083 : if (lang_hooks.decls.omp_private_outer_ref (val))
8274 : do_in = true;
8275 : }
8276 : break;
8277 :
8278 4569 : case OMP_CLAUSE_REDUCTION:
8279 4569 : case OMP_CLAUSE_IN_REDUCTION:
8280 4569 : do_in = true;
8281 4569 : if (val == OMP_CLAUSE_DECL (c))
8282 : {
8283 3191 : if (is_task_ctx (ctx))
8284 322 : by_ref = use_pointer_for_field (val, ctx);
8285 : else
8286 2869 : do_out = !(by_ref || omp_privatize_by_reference (val));
8287 : }
8288 : else
8289 1378 : by_ref = TREE_CODE (TREE_TYPE (val)) == ARRAY_TYPE;
8290 : break;
8291 :
8292 0 : default:
8293 0 : gcc_unreachable ();
8294 : }
8295 :
8296 4031 : if (do_in)
8297 : {
8298 49072 : ref = build_sender_ref (val, ctx);
8299 49072 : x = by_ref ? build_fold_addr_expr_loc (clause_loc, var) : var;
8300 49072 : gimplify_assign (ref, x, ilist);
8301 49072 : if (is_task_ctx (ctx))
8302 6267 : DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref, 1)) = NULL;
8303 : }
8304 :
8305 55131 : if (do_out)
8306 : {
8307 8400 : ref = build_sender_ref (val, ctx);
8308 8400 : gimplify_assign (var, ref, olist);
8309 : }
8310 : }
8311 22721 : }
8312 :
8313 : /* Generate code to implement SHARED from the sender (aka parent)
8314 : side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
8315 : list things that got automatically shared. */
8316 :
8317 : static void
8318 22721 : lower_send_shared_vars (gimple_seq *ilist, gimple_seq *olist, omp_context *ctx)
8319 : {
8320 22721 : tree var, ovar, nvar, t, f, x, record_type;
8321 :
8322 22721 : if (ctx->record_type == NULL)
8323 4540 : return;
8324 :
8325 18181 : record_type = ctx->srecord_type ? ctx->srecord_type : ctx->record_type;
8326 87438 : for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
8327 : {
8328 69257 : ovar = DECL_ABSTRACT_ORIGIN (f);
8329 69257 : if (!ovar || TREE_CODE (ovar) == FIELD_DECL)
8330 6346 : continue;
8331 :
8332 62911 : nvar = maybe_lookup_decl (ovar, ctx);
8333 114091 : if (!nvar
8334 62498 : || !DECL_HAS_VALUE_EXPR_P (nvar)
8335 74863 : || (ctx->allocate_map
8336 317 : && ctx->allocate_map->get (ovar)))
8337 51180 : continue;
8338 :
8339 : /* If CTX is a nested parallel directive. Find the immediately
8340 : enclosing parallel or workshare construct that contains a
8341 : mapping for OVAR. */
8342 11731 : var = lookup_decl_in_outer_ctx (ovar, ctx);
8343 :
8344 11731 : t = omp_member_access_dummy_var (var);
8345 11731 : if (t)
8346 : {
8347 49 : var = DECL_VALUE_EXPR (var);
8348 49 : tree o = maybe_lookup_decl_in_outer_ctx (t, ctx);
8349 49 : if (o != t)
8350 18 : var = unshare_and_remap (var, t, o);
8351 : else
8352 31 : var = unshare_expr (var);
8353 : }
8354 :
8355 11731 : if (use_pointer_for_field (ovar, ctx))
8356 : {
8357 6490 : x = build_sender_ref (ovar, ctx);
8358 6490 : if (TREE_CODE (TREE_TYPE (f)) == ARRAY_TYPE
8359 6490 : && TREE_TYPE (f) == TREE_TYPE (ovar))
8360 : {
8361 103 : gcc_assert (is_parallel_ctx (ctx)
8362 : && DECL_ARTIFICIAL (ovar));
8363 : /* _condtemp_ clause. */
8364 103 : var = build_constructor (TREE_TYPE (x), NULL);
8365 : }
8366 : else
8367 6387 : var = build_fold_addr_expr (var);
8368 6490 : gimplify_assign (x, var, ilist);
8369 : }
8370 : else
8371 : {
8372 5241 : x = build_sender_ref (ovar, ctx);
8373 5241 : gimplify_assign (x, var, ilist);
8374 :
8375 5241 : if (!TREE_READONLY (var)
8376 : /* We don't need to receive a new reference to a result
8377 : or parm decl. In fact we may not store to it as we will
8378 : invalidate any pending RSO and generate wrong gimple
8379 : during inlining. */
8380 5241 : && !((TREE_CODE (var) == RESULT_DECL
8381 3348 : || TREE_CODE (var) == PARM_DECL)
8382 182 : && DECL_BY_REFERENCE (var)))
8383 : {
8384 3277 : x = build_sender_ref (ovar, ctx);
8385 3277 : gimplify_assign (var, x, olist);
8386 : }
8387 : }
8388 : }
8389 : }
8390 :
8391 : /* Emit an OpenACC head marker call, encapulating the partitioning and
8392 : other information that must be processed by the target compiler.
8393 : Return the maximum number of dimensions the associated loop might
8394 : be partitioned over. */
8395 :
8396 : static unsigned
8397 9665 : lower_oacc_head_mark (location_t loc, tree ddvar, tree clauses,
8398 : gimple_seq *seq, omp_context *ctx)
8399 : {
8400 9665 : unsigned levels = 0;
8401 9665 : unsigned tag = 0;
8402 9665 : tree gang_static = NULL_TREE;
8403 9665 : auto_vec<tree, 5> args;
8404 :
8405 19330 : args.quick_push (build_int_cst
8406 9665 : (integer_type_node, IFN_UNIQUE_OACC_HEAD_MARK));
8407 9665 : args.quick_push (ddvar);
8408 40519 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
8409 : {
8410 21189 : switch (OMP_CLAUSE_CODE (c))
8411 : {
8412 1856 : case OMP_CLAUSE_GANG:
8413 1856 : tag |= OLF_DIM_GANG;
8414 1856 : gang_static = OMP_CLAUSE_GANG_STATIC_EXPR (c);
8415 : /* static:* is represented by -1, and we can ignore it, as
8416 : scheduling is always static. */
8417 1856 : if (gang_static && integer_minus_onep (gang_static))
8418 70 : gang_static = NULL_TREE;
8419 1856 : levels++;
8420 1856 : break;
8421 :
8422 1330 : case OMP_CLAUSE_WORKER:
8423 1330 : tag |= OLF_DIM_WORKER;
8424 1330 : levels++;
8425 1330 : break;
8426 :
8427 1523 : case OMP_CLAUSE_VECTOR:
8428 1523 : tag |= OLF_DIM_VECTOR;
8429 1523 : levels++;
8430 1523 : break;
8431 :
8432 314 : case OMP_CLAUSE_SEQ:
8433 314 : tag |= OLF_SEQ;
8434 314 : break;
8435 :
8436 409 : case OMP_CLAUSE_AUTO:
8437 409 : tag |= OLF_AUTO;
8438 409 : break;
8439 :
8440 260 : case OMP_CLAUSE_INDEPENDENT:
8441 260 : tag |= OLF_INDEPENDENT;
8442 260 : break;
8443 :
8444 136 : case OMP_CLAUSE_TILE:
8445 136 : tag |= OLF_TILE;
8446 136 : break;
8447 :
8448 3986 : case OMP_CLAUSE_REDUCTION:
8449 3986 : tag |= OLF_REDUCTION;
8450 3986 : break;
8451 :
8452 11375 : default:
8453 11375 : continue;
8454 : }
8455 : }
8456 :
8457 9665 : if (gang_static)
8458 : {
8459 146 : if (DECL_P (gang_static))
8460 16 : gang_static = build_outer_var_ref (gang_static, ctx);
8461 146 : tag |= OLF_GANG_STATIC;
8462 : }
8463 :
8464 9665 : omp_context *tgt = enclosing_target_ctx (ctx);
8465 9665 : if (!tgt || is_oacc_parallel_or_serial (tgt))
8466 : ;
8467 62 : else if (is_oacc_kernels (tgt))
8468 : /* Not using this loops handling inside OpenACC 'kernels' regions. */
8469 0 : gcc_unreachable ();
8470 62 : else if (is_oacc_kernels_decomposed_part (tgt))
8471 : ;
8472 : else
8473 0 : gcc_unreachable ();
8474 :
8475 : /* In a parallel region, loops are implicitly INDEPENDENT. */
8476 9665 : if (!tgt || is_oacc_parallel_or_serial (tgt))
8477 9603 : tag |= OLF_INDEPENDENT;
8478 :
8479 : /* Loops inside OpenACC 'kernels' decomposed parts' regions are expected to
8480 : have an explicit 'seq' or 'independent' clause, and no 'auto' clause. */
8481 9665 : if (tgt && is_oacc_kernels_decomposed_part (tgt))
8482 : {
8483 62 : gcc_assert (tag & (OLF_SEQ | OLF_INDEPENDENT));
8484 62 : gcc_assert (!(tag & OLF_AUTO));
8485 : }
8486 :
8487 9665 : if (tag & OLF_TILE)
8488 : /* Tiling could use all 3 levels. */
8489 : levels = 3;
8490 : else
8491 : {
8492 : /* A loop lacking SEQ, GANG, WORKER and/or VECTOR could be AUTO.
8493 : Ensure at least one level, or 2 for possible auto
8494 : partitioning */
8495 9529 : bool maybe_auto = !(tag & (((GOMP_DIM_MASK (GOMP_DIM_MAX) - 1)
8496 : << OLF_DIM_BASE) | OLF_SEQ));
8497 :
8498 9529 : if (levels < 1u + maybe_auto)
8499 : levels = 1u + maybe_auto;
8500 : }
8501 :
8502 9665 : args.quick_push (build_int_cst (integer_type_node, levels));
8503 9665 : args.quick_push (build_int_cst (integer_type_node, tag));
8504 9665 : if (gang_static)
8505 146 : args.quick_push (gang_static);
8506 :
8507 9665 : gcall *call = gimple_build_call_internal_vec (IFN_UNIQUE, args);
8508 9665 : gimple_set_location (call, loc);
8509 9665 : gimple_set_lhs (call, ddvar);
8510 9665 : gimple_seq_add_stmt (seq, call);
8511 :
8512 9665 : return levels;
8513 9665 : }
8514 :
8515 : /* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the
8516 : partitioning level of the enclosed region. */
8517 :
8518 : static void
8519 42767 : lower_oacc_loop_marker (location_t loc, tree ddvar, bool head,
8520 : tree tofollow, gimple_seq *seq)
8521 : {
8522 42767 : int marker_kind = (head ? IFN_UNIQUE_OACC_HEAD_MARK
8523 : : IFN_UNIQUE_OACC_TAIL_MARK);
8524 42767 : tree marker = build_int_cst (integer_type_node, marker_kind);
8525 42767 : int nargs = 2 + (tofollow != NULL_TREE);
8526 42767 : gcall *call = gimple_build_call_internal (IFN_UNIQUE, nargs,
8527 : marker, ddvar, tofollow);
8528 42767 : gimple_set_location (call, loc);
8529 42767 : gimple_set_lhs (call, ddvar);
8530 42767 : gimple_seq_add_stmt (seq, call);
8531 42767 : }
8532 :
8533 : /* Generate the before and after OpenACC loop sequences. CLAUSES are
8534 : the loop clauses, from which we extract reductions. Initialize
8535 : HEAD and TAIL. */
8536 :
8537 : static void
8538 9665 : lower_oacc_head_tail (location_t loc, tree clauses, gcall *private_marker,
8539 : gimple_seq *head, gimple_seq *tail, omp_context *ctx)
8540 : {
8541 9665 : bool inner = false;
8542 9665 : tree ddvar = create_tmp_var (integer_type_node, ".data_dep");
8543 9665 : gimple_seq_add_stmt (head, gimple_build_assign (ddvar, integer_zero_node));
8544 :
8545 9665 : unsigned count = lower_oacc_head_mark (loc, ddvar, clauses, head, ctx);
8546 :
8547 9665 : if (private_marker)
8548 : {
8549 229 : gimple_set_location (private_marker, loc);
8550 229 : gimple_call_set_lhs (private_marker, ddvar);
8551 229 : gimple_call_set_arg (private_marker, 1, ddvar);
8552 : }
8553 :
8554 9665 : tree fork_kind = build_int_cst (unsigned_type_node, IFN_UNIQUE_OACC_FORK);
8555 9665 : tree join_kind = build_int_cst (unsigned_type_node, IFN_UNIQUE_OACC_JOIN);
8556 :
8557 9665 : gcc_assert (count);
8558 26216 : for (unsigned done = 1; count; count--, done++)
8559 : {
8560 16551 : gimple_seq fork_seq = NULL;
8561 16551 : gimple_seq join_seq = NULL;
8562 :
8563 16551 : tree place = build_int_cst (integer_type_node, -1);
8564 16551 : gcall *fork = gimple_build_call_internal (IFN_UNIQUE, 3,
8565 : fork_kind, ddvar, place);
8566 16551 : gimple_set_location (fork, loc);
8567 16551 : gimple_set_lhs (fork, ddvar);
8568 :
8569 16551 : gcall *join = gimple_build_call_internal (IFN_UNIQUE, 3,
8570 : join_kind, ddvar, place);
8571 16551 : gimple_set_location (join, loc);
8572 16551 : gimple_set_lhs (join, ddvar);
8573 :
8574 : /* Mark the beginning of this level sequence. */
8575 16551 : if (inner)
8576 6886 : lower_oacc_loop_marker (loc, ddvar, true,
8577 6886 : build_int_cst (integer_type_node, count),
8578 : &fork_seq);
8579 16551 : lower_oacc_loop_marker (loc, ddvar, false,
8580 16551 : build_int_cst (integer_type_node, done),
8581 : &join_seq);
8582 :
8583 23437 : lower_oacc_reductions (loc, clauses, place, inner,
8584 : fork, (count == 1) ? private_marker : NULL,
8585 : join, &fork_seq, &join_seq, ctx);
8586 :
8587 : /* Append this level to head. */
8588 16551 : gimple_seq_add_seq (head, fork_seq);
8589 : /* Prepend it to tail. */
8590 16551 : gimple_seq_add_seq (&join_seq, *tail);
8591 16551 : *tail = join_seq;
8592 :
8593 16551 : inner = true;
8594 : }
8595 :
8596 : /* Mark the end of the sequence. */
8597 9665 : lower_oacc_loop_marker (loc, ddvar, true, NULL_TREE, head);
8598 9665 : lower_oacc_loop_marker (loc, ddvar, false, NULL_TREE, tail);
8599 9665 : }
8600 :
8601 : /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
8602 : catch handler and return it. This prevents programs from violating the
8603 : structured block semantics with throws. */
8604 :
8605 : static gimple_seq
8606 95852 : maybe_catch_exception (gimple_seq body)
8607 : {
8608 95852 : gimple *g;
8609 95852 : tree decl;
8610 :
8611 95852 : if (!flag_exceptions)
8612 : return body;
8613 :
8614 43023 : if (lang_hooks.eh_protect_cleanup_actions != NULL)
8615 42995 : decl = lang_hooks.eh_protect_cleanup_actions ();
8616 : else
8617 28 : decl = builtin_decl_explicit (BUILT_IN_TRAP);
8618 :
8619 43023 : g = gimple_build_eh_must_not_throw (decl);
8620 43023 : g = gimple_build_try (body, gimple_seq_alloc_with_stmt (g),
8621 : GIMPLE_TRY_CATCH);
8622 :
8623 43023 : return gimple_seq_alloc_with_stmt (g);
8624 : }
8625 :
8626 :
8627 : /* Routines to lower OMP directives into OMP-GIMPLE. */
8628 :
8629 : /* If ctx is a worksharing context inside of a cancellable parallel
8630 : region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8631 : and conditional branch to parallel's cancel_label to handle
8632 : cancellation in the implicit barrier. */
8633 :
8634 : static void
8635 48789 : maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple *omp_return,
8636 : gimple_seq *body)
8637 : {
8638 48789 : gcc_assert (gimple_code (omp_return) == GIMPLE_OMP_RETURN);
8639 48789 : if (gimple_omp_return_nowait_p (omp_return))
8640 : return;
8641 19679 : for (omp_context *outer = ctx->outer; outer; outer = outer->outer)
8642 16046 : if (gimple_code (outer->stmt) == GIMPLE_OMP_PARALLEL
8643 16046 : && outer->cancellable)
8644 : {
8645 65 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_CANCEL);
8646 65 : tree c_bool_type = TREE_TYPE (TREE_TYPE (fndecl));
8647 65 : tree lhs = create_tmp_var (c_bool_type);
8648 65 : gimple_omp_return_set_lhs (omp_return, lhs);
8649 65 : tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
8650 65 : gimple *g = gimple_build_cond (NE_EXPR, lhs,
8651 : fold_convert (c_bool_type,
8652 : boolean_false_node),
8653 : outer->cancel_label, fallthru_label);
8654 65 : gimple_seq_add_stmt (body, g);
8655 65 : gimple_seq_add_stmt (body, gimple_build_label (fallthru_label));
8656 : }
8657 15981 : else if (gimple_code (outer->stmt) != GIMPLE_OMP_TASKGROUP
8658 15981 : && gimple_code (outer->stmt) != GIMPLE_OMP_SCOPE)
8659 : return;
8660 : }
8661 :
8662 : /* Find the first task_reduction or reduction clause or return NULL
8663 : if there are none. */
8664 :
8665 : static inline tree
8666 49034 : omp_task_reductions_find_first (tree clauses, enum tree_code code,
8667 : enum omp_clause_code ccode)
8668 : {
8669 64468 : while (1)
8670 : {
8671 56751 : clauses = omp_find_clause (clauses, ccode);
8672 56751 : if (clauses == NULL_TREE)
8673 : return NULL_TREE;
8674 9183 : if (ccode != OMP_CLAUSE_REDUCTION
8675 9183 : || code == OMP_TASKLOOP
8676 9183 : || OMP_CLAUSE_REDUCTION_TASK (clauses))
8677 : return clauses;
8678 7717 : clauses = OMP_CLAUSE_CHAIN (clauses);
8679 : }
8680 : }
8681 :
8682 : static void lower_omp_task_reductions (omp_context *, enum tree_code, tree,
8683 : gimple_seq *, gimple_seq *);
8684 :
8685 : /* Lower the OpenMP sections directive in the current statement in GSI_P.
8686 : CTX is the enclosing OMP context for the current statement. */
8687 :
8688 : static void
8689 386 : lower_omp_sections (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8690 : {
8691 386 : tree block, control;
8692 386 : gimple_stmt_iterator tgsi;
8693 386 : gomp_sections *stmt;
8694 386 : gimple *t;
8695 386 : gbind *new_stmt, *bind;
8696 386 : gimple_seq ilist, dlist, olist, tred_dlist = NULL, clist = NULL, new_body;
8697 :
8698 386 : stmt = as_a <gomp_sections *> (gsi_stmt (*gsi_p));
8699 :
8700 386 : push_gimplify_context ();
8701 :
8702 386 : dlist = NULL;
8703 386 : ilist = NULL;
8704 :
8705 386 : tree rclauses
8706 386 : = omp_task_reductions_find_first (gimple_omp_sections_clauses (stmt),
8707 : OMP_SECTIONS, OMP_CLAUSE_REDUCTION);
8708 386 : tree rtmp = NULL_TREE;
8709 386 : if (rclauses)
8710 : {
8711 8 : tree type = build_pointer_type (pointer_sized_int_node);
8712 8 : tree temp = create_tmp_var (type);
8713 8 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
8714 8 : OMP_CLAUSE_DECL (c) = temp;
8715 8 : OMP_CLAUSE_CHAIN (c) = gimple_omp_sections_clauses (stmt);
8716 8 : gimple_omp_sections_set_clauses (stmt, c);
8717 8 : lower_omp_task_reductions (ctx, OMP_SECTIONS,
8718 : gimple_omp_sections_clauses (stmt),
8719 : &ilist, &tred_dlist);
8720 8 : rclauses = c;
8721 8 : rtmp = make_ssa_name (type);
8722 8 : gimple_seq_add_stmt (&ilist, gimple_build_assign (rtmp, temp));
8723 : }
8724 :
8725 386 : tree *clauses_ptr = gimple_omp_sections_clauses_ptr (stmt);
8726 386 : lower_lastprivate_conditional_clauses (clauses_ptr, ctx);
8727 :
8728 386 : lower_rec_input_clauses (gimple_omp_sections_clauses (stmt),
8729 : &ilist, &dlist, ctx, NULL);
8730 :
8731 386 : control = create_tmp_var (unsigned_type_node, ".section");
8732 386 : gimple_omp_sections_set_control (stmt, control);
8733 :
8734 386 : new_body = gimple_omp_body (stmt);
8735 386 : gimple_omp_set_body (stmt, NULL);
8736 386 : tgsi = gsi_start (new_body);
8737 1249 : for (; !gsi_end_p (tgsi); gsi_next (&tgsi))
8738 : {
8739 863 : omp_context *sctx;
8740 863 : gimple *sec_start;
8741 :
8742 863 : sec_start = gsi_stmt (tgsi);
8743 863 : sctx = maybe_lookup_ctx (sec_start);
8744 863 : gcc_assert (sctx);
8745 :
8746 863 : lower_omp (gimple_omp_body_ptr (sec_start), sctx);
8747 863 : gsi_insert_seq_after (&tgsi, gimple_omp_body (sec_start),
8748 : GSI_CONTINUE_LINKING);
8749 863 : gimple_omp_set_body (sec_start, NULL);
8750 :
8751 863 : if (gsi_one_before_end_p (tgsi))
8752 : {
8753 386 : gimple_seq l = NULL;
8754 386 : lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt), NULL,
8755 : &ilist, &l, &clist, ctx);
8756 386 : gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING);
8757 386 : gimple_omp_section_set_last (sec_start);
8758 : }
8759 :
8760 863 : gsi_insert_after (&tgsi, gimple_build_omp_return (false),
8761 : GSI_CONTINUE_LINKING);
8762 : }
8763 :
8764 386 : block = make_node (BLOCK);
8765 386 : bind = gimple_build_bind (NULL, new_body, block);
8766 :
8767 386 : olist = NULL;
8768 386 : lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist,
8769 : &clist, ctx);
8770 386 : if (clist)
8771 : {
8772 10 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_START);
8773 10 : gcall *g = gimple_build_call (fndecl, 0);
8774 10 : gimple_seq_add_stmt (&olist, g);
8775 10 : gimple_seq_add_seq (&olist, clist);
8776 10 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_END);
8777 10 : g = gimple_build_call (fndecl, 0);
8778 10 : gimple_seq_add_stmt (&olist, g);
8779 : }
8780 :
8781 386 : block = make_node (BLOCK);
8782 386 : new_stmt = gimple_build_bind (NULL, NULL, block);
8783 386 : gsi_replace (gsi_p, new_stmt, true);
8784 :
8785 386 : pop_gimplify_context (new_stmt);
8786 386 : gimple_bind_append_vars (new_stmt, ctx->block_vars);
8787 386 : BLOCK_VARS (block) = gimple_bind_vars (bind);
8788 386 : if (BLOCK_VARS (block))
8789 0 : TREE_USED (block) = 1;
8790 :
8791 386 : new_body = NULL;
8792 386 : gimple_seq_add_seq (&new_body, ilist);
8793 386 : gimple_seq_add_stmt (&new_body, stmt);
8794 386 : gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ());
8795 386 : gimple_seq_add_stmt (&new_body, bind);
8796 :
8797 386 : t = gimple_build_omp_continue (control, control);
8798 386 : gimple_seq_add_stmt (&new_body, t);
8799 :
8800 386 : gimple_seq_add_seq (&new_body, olist);
8801 386 : if (ctx->cancellable)
8802 19 : gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
8803 386 : gimple_seq_add_seq (&new_body, dlist);
8804 :
8805 386 : new_body = maybe_catch_exception (new_body);
8806 :
8807 386 : bool nowait = omp_find_clause (gimple_omp_sections_clauses (stmt),
8808 386 : OMP_CLAUSE_NOWAIT) != NULL_TREE;
8809 386 : t = gimple_build_omp_return (nowait);
8810 386 : gimple_seq_add_stmt (&new_body, t);
8811 386 : gimple_seq_add_seq (&new_body, tred_dlist);
8812 386 : maybe_add_implicit_barrier_cancel (ctx, t, &new_body);
8813 :
8814 386 : if (rclauses)
8815 8 : OMP_CLAUSE_DECL (rclauses) = rtmp;
8816 :
8817 386 : gimple_bind_set_body (new_stmt, new_body);
8818 386 : }
8819 :
8820 :
8821 : /* A subroutine of lower_omp_single. Expand the simple form of
8822 : a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8823 :
8824 : if (GOMP_single_start ())
8825 : BODY;
8826 : [ GOMP_barrier (); ] -> unless 'nowait' is present.
8827 :
8828 : FIXME. It may be better to delay expanding the logic of this until
8829 : pass_expand_omp. The expanded logic may make the job more difficult
8830 : to a synchronization analysis pass. */
8831 :
8832 : static void
8833 1022 : lower_omp_single_simple (gomp_single *single_stmt, gimple_seq *pre_p)
8834 : {
8835 1022 : location_t loc = gimple_location (single_stmt);
8836 1022 : tree tlabel = create_artificial_label (loc);
8837 1022 : tree flabel = create_artificial_label (loc);
8838 1022 : gimple *call, *cond;
8839 1022 : tree lhs, decl;
8840 :
8841 2040 : decl = builtin_decl_explicit (flag_openmp_ompt
8842 : ? BUILT_IN_GOMP_SINGLE_START_WITH_END
8843 : : BUILT_IN_GOMP_SINGLE_START);
8844 1022 : lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)));
8845 1022 : call = gimple_build_call (decl, 0);
8846 1022 : gimple_call_set_lhs (call, lhs);
8847 1022 : gimple_seq_add_stmt (pre_p, call);
8848 :
8849 1022 : cond = gimple_build_cond (EQ_EXPR, lhs,
8850 1022 : fold_convert_loc (loc, TREE_TYPE (lhs),
8851 : boolean_true_node),
8852 : tlabel, flabel);
8853 1022 : gimple_seq_add_stmt (pre_p, cond);
8854 1022 : gimple_seq_add_stmt (pre_p, gimple_build_label (tlabel));
8855 1022 : gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
8856 1022 : gimple_seq_add_stmt (pre_p, gimple_build_label (flabel));
8857 1022 : }
8858 :
8859 :
8860 : /* A subroutine of lower_omp_single. Expand the simple form of
8861 : a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8862 :
8863 : #pragma omp single copyprivate (a, b, c)
8864 :
8865 : Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
8866 :
8867 : {
8868 : if ((copyout_p = GOMP_single_copy_start ()) == NULL)
8869 : {
8870 : BODY;
8871 : copyout.a = a;
8872 : copyout.b = b;
8873 : copyout.c = c;
8874 : GOMP_single_copy_end (©out);
8875 : }
8876 : else
8877 : {
8878 : a = copyout_p->a;
8879 : b = copyout_p->b;
8880 : c = copyout_p->c;
8881 : }
8882 : GOMP_barrier ();
8883 : }
8884 :
8885 : FIXME. It may be better to delay expanding the logic of this until
8886 : pass_expand_omp. The expanded logic may make the job more difficult
8887 : to a synchronization analysis pass. */
8888 :
8889 : static void
8890 118 : lower_omp_single_copy (gomp_single *single_stmt, gimple_seq *pre_p,
8891 : omp_context *ctx)
8892 : {
8893 118 : tree ptr_type, t, l0, l1, l2, bfn_decl;
8894 118 : gimple_seq copyin_seq;
8895 118 : location_t loc = gimple_location (single_stmt);
8896 :
8897 118 : ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o");
8898 :
8899 118 : ptr_type = build_pointer_type (ctx->record_type);
8900 118 : ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i");
8901 :
8902 118 : l0 = create_artificial_label (loc);
8903 118 : l1 = create_artificial_label (loc);
8904 118 : l2 = create_artificial_label (loc);
8905 :
8906 118 : bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START);
8907 118 : t = build_call_expr_loc (loc, bfn_decl, 0);
8908 118 : t = fold_convert_loc (loc, ptr_type, t);
8909 118 : gimplify_assign (ctx->receiver_decl, t, pre_p);
8910 :
8911 118 : t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl,
8912 : build_int_cst (ptr_type, 0));
8913 118 : t = build3 (COND_EXPR, void_type_node, t,
8914 : build_and_jump (&l0), build_and_jump (&l1));
8915 118 : gimplify_and_add (t, pre_p);
8916 :
8917 118 : gimple_seq_add_stmt (pre_p, gimple_build_label (l0));
8918 :
8919 118 : gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
8920 :
8921 118 : copyin_seq = NULL;
8922 118 : lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt), pre_p,
8923 : ©in_seq, ctx);
8924 :
8925 118 : t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
8926 118 : bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END);
8927 118 : t = build_call_expr_loc (loc, bfn_decl, 1, t);
8928 118 : gimplify_and_add (t, pre_p);
8929 :
8930 118 : t = build_and_jump (&l2);
8931 118 : gimplify_and_add (t, pre_p);
8932 :
8933 118 : gimple_seq_add_stmt (pre_p, gimple_build_label (l1));
8934 :
8935 118 : gimple_seq_add_seq (pre_p, copyin_seq);
8936 :
8937 118 : gimple_seq_add_stmt (pre_p, gimple_build_label (l2));
8938 118 : }
8939 :
8940 :
8941 : /* Expand code for an OpenMP single directive. */
8942 :
8943 : static void
8944 1140 : lower_omp_single (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8945 : {
8946 1140 : tree block;
8947 1140 : gomp_single *single_stmt = as_a <gomp_single *> (gsi_stmt (*gsi_p));
8948 1140 : gbind *bind;
8949 1140 : gimple_seq bind_body, bind_body_tail = NULL, dlist;
8950 :
8951 1140 : push_gimplify_context ();
8952 :
8953 1140 : block = make_node (BLOCK);
8954 1140 : bind = gimple_build_bind (NULL, NULL, block);
8955 1140 : gsi_replace (gsi_p, bind, true);
8956 1140 : bind_body = NULL;
8957 1140 : dlist = NULL;
8958 1140 : lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt),
8959 : &bind_body, &dlist, ctx, NULL);
8960 1140 : lower_omp (gimple_omp_body_ptr (single_stmt), ctx);
8961 :
8962 1140 : gimple_seq_add_stmt (&bind_body, single_stmt);
8963 :
8964 1140 : if (ctx->record_type)
8965 118 : lower_omp_single_copy (single_stmt, &bind_body, ctx);
8966 : else
8967 1022 : lower_omp_single_simple (single_stmt, &bind_body);
8968 :
8969 1140 : gimple_omp_set_body (single_stmt, NULL);
8970 :
8971 1140 : gimple_seq_add_seq (&bind_body, dlist);
8972 :
8973 1140 : bind_body = maybe_catch_exception (bind_body);
8974 :
8975 1140 : bool nowait = omp_find_clause (gimple_omp_single_clauses (single_stmt),
8976 1140 : OMP_CLAUSE_NOWAIT) != NULL_TREE;
8977 1140 : gimple *g = gimple_build_omp_return (nowait);
8978 1140 : gimple_seq_add_stmt (&bind_body_tail, g);
8979 1140 : maybe_add_implicit_barrier_cancel (ctx, g, &bind_body_tail);
8980 :
8981 1140 : if (flag_openmp_ompt && !ctx->record_type)
8982 : {
8983 : /* Insert call to GOMP_single_end. */
8984 4 : tree decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_END);
8985 4 : gimple *stmt = gimple_build_call (decl, 0);
8986 4 : gimple_seq_add_stmt (&bind_body_tail, stmt);
8987 : }
8988 :
8989 1140 : if (ctx->record_type)
8990 : {
8991 118 : gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
8992 118 : tree clobber = build_clobber (ctx->record_type);
8993 118 : gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
8994 : clobber), GSI_SAME_STMT);
8995 : }
8996 1140 : gimple_seq_add_seq (&bind_body, bind_body_tail);
8997 1140 : gimple_bind_set_body (bind, bind_body);
8998 :
8999 1140 : pop_gimplify_context (bind);
9000 :
9001 1140 : gimple_bind_append_vars (bind, ctx->block_vars);
9002 1140 : BLOCK_VARS (block) = ctx->block_vars;
9003 1140 : if (BLOCK_VARS (block))
9004 26 : TREE_USED (block) = 1;
9005 1140 : }
9006 :
9007 :
9008 : /* Lower code for an OMP scope directive. */
9009 :
9010 : static void
9011 161 : lower_omp_scope (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9012 : {
9013 161 : tree block;
9014 161 : gimple *scope_stmt = gsi_stmt (*gsi_p);
9015 161 : gbind *bind;
9016 161 : gimple_seq bind_body, bind_body_tail = NULL, dlist;
9017 161 : gimple_seq tred_dlist = NULL;
9018 :
9019 161 : push_gimplify_context ();
9020 :
9021 161 : block = make_node (BLOCK);
9022 161 : bind = gimple_build_bind (NULL, NULL, block);
9023 161 : gsi_replace (gsi_p, bind, true);
9024 161 : bind_body = NULL;
9025 161 : dlist = NULL;
9026 :
9027 161 : tree rclauses
9028 161 : = omp_task_reductions_find_first (gimple_omp_scope_clauses (scope_stmt),
9029 : OMP_SCOPE, OMP_CLAUSE_REDUCTION);
9030 161 : if (rclauses)
9031 : {
9032 54 : tree type = build_pointer_type (pointer_sized_int_node);
9033 54 : tree temp = create_tmp_var (type);
9034 54 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
9035 54 : OMP_CLAUSE_DECL (c) = temp;
9036 54 : OMP_CLAUSE_CHAIN (c) = gimple_omp_scope_clauses (scope_stmt);
9037 54 : gimple_omp_scope_set_clauses (scope_stmt, c);
9038 54 : lower_omp_task_reductions (ctx, OMP_SCOPE,
9039 : gimple_omp_scope_clauses (scope_stmt),
9040 : &bind_body, &tred_dlist);
9041 54 : rclauses = c;
9042 54 : tree fndecl = builtin_decl_explicit (
9043 54 : flag_openmp_ompt ? BUILT_IN_GOMP_SCOPE_START_WITH_END
9044 : : BUILT_IN_GOMP_SCOPE_START);
9045 54 : gimple *stmt = gimple_build_call (fndecl, 1, temp);
9046 54 : gimple_seq_add_stmt (&bind_body, stmt);
9047 : }
9048 107 : else if (flag_openmp_ompt)
9049 : {
9050 4 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_SCOPE_START_WITH_END);
9051 4 : gimple *stmt = gimple_build_call (fndecl, 1, null_pointer_node);
9052 4 : gimple_seq_add_stmt (&bind_body, stmt);
9053 : }
9054 :
9055 161 : lower_rec_input_clauses (gimple_omp_scope_clauses (scope_stmt),
9056 : &bind_body, &dlist, ctx, NULL);
9057 161 : lower_omp (gimple_omp_body_ptr (scope_stmt), ctx);
9058 :
9059 161 : gimple_seq_add_stmt (&bind_body, scope_stmt);
9060 :
9061 161 : gimple_seq_add_seq (&bind_body, gimple_omp_body (scope_stmt));
9062 :
9063 161 : gimple_omp_set_body (scope_stmt, NULL);
9064 :
9065 161 : gimple_seq clist = NULL;
9066 161 : lower_reduction_clauses (gimple_omp_scope_clauses (scope_stmt),
9067 : &bind_body, &clist, ctx);
9068 161 : if (clist)
9069 : {
9070 0 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_START);
9071 0 : gcall *g = gimple_build_call (fndecl, 0);
9072 0 : gimple_seq_add_stmt (&bind_body, g);
9073 0 : gimple_seq_add_seq (&bind_body, clist);
9074 0 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_END);
9075 0 : g = gimple_build_call (fndecl, 0);
9076 0 : gimple_seq_add_stmt (&bind_body, g);
9077 : }
9078 :
9079 161 : gimple_seq_add_seq (&bind_body, dlist);
9080 :
9081 161 : bind_body = maybe_catch_exception (bind_body);
9082 :
9083 161 : if (flag_openmp_ompt)
9084 : {
9085 8 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_SCOPE_END);
9086 8 : gcall *g = gimple_build_call (fndecl, 0);
9087 8 : gimple_seq_add_stmt (&bind_body_tail, g);
9088 : }
9089 :
9090 161 : bool nowait = omp_find_clause (gimple_omp_scope_clauses (scope_stmt),
9091 161 : OMP_CLAUSE_NOWAIT) != NULL_TREE;
9092 161 : gimple *g = gimple_build_omp_return (nowait);
9093 161 : gimple_seq_add_stmt (&bind_body_tail, g);
9094 161 : gimple_seq_add_seq (&bind_body_tail, tred_dlist);
9095 161 : maybe_add_implicit_barrier_cancel (ctx, g, &bind_body_tail);
9096 161 : if (ctx->record_type)
9097 : {
9098 0 : gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
9099 0 : tree clobber = build_clobber (ctx->record_type);
9100 0 : gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
9101 : clobber), GSI_SAME_STMT);
9102 : }
9103 161 : gimple_seq_add_seq (&bind_body, bind_body_tail);
9104 :
9105 161 : gimple_bind_set_body (bind, bind_body);
9106 :
9107 161 : pop_gimplify_context (bind);
9108 :
9109 161 : gimple_bind_append_vars (bind, ctx->block_vars);
9110 161 : BLOCK_VARS (block) = ctx->block_vars;
9111 161 : if (BLOCK_VARS (block))
9112 121 : TREE_USED (block) = 1;
9113 161 : }
9114 :
9115 : /* Lower code for an OMP dispatch directive. */
9116 :
9117 : static void
9118 659 : lower_omp_dispatch (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9119 : {
9120 659 : tree block;
9121 659 : gimple *stmt = gsi_stmt (*gsi_p);
9122 659 : gbind *bind;
9123 :
9124 659 : push_gimplify_context ();
9125 :
9126 659 : block = make_node (BLOCK);
9127 659 : bind = gimple_build_bind (NULL, NULL, block);
9128 659 : gsi_replace (gsi_p, bind, true);
9129 :
9130 659 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
9131 659 : gimple_bind_set_body (bind, maybe_catch_exception (gimple_omp_body (stmt)));
9132 :
9133 659 : pop_gimplify_context (bind);
9134 :
9135 659 : gimple_bind_append_vars (bind, ctx->block_vars);
9136 659 : BLOCK_VARS (block) = ctx->block_vars;
9137 659 : }
9138 :
9139 : /* Expand code for an OpenMP master or masked directive. */
9140 :
9141 : static void
9142 1127 : lower_omp_master (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9143 : {
9144 1127 : tree block, lab = NULL, x, bfn_decl;
9145 1127 : gimple *stmt = gsi_stmt (*gsi_p);
9146 1127 : gbind *bind;
9147 1127 : location_t loc = gimple_location (stmt);
9148 1127 : gimple_seq tseq;
9149 1127 : tree filter = integer_zero_node;
9150 :
9151 1127 : push_gimplify_context ();
9152 :
9153 1127 : if (gimple_code (stmt) == GIMPLE_OMP_MASKED)
9154 : {
9155 393 : filter = omp_find_clause (gimple_omp_masked_clauses (stmt),
9156 : OMP_CLAUSE_FILTER);
9157 393 : if (filter)
9158 252 : filter = fold_convert (integer_type_node,
9159 : OMP_CLAUSE_FILTER_EXPR (filter));
9160 : else
9161 141 : filter = integer_zero_node;
9162 : }
9163 1127 : block = make_node (BLOCK);
9164 1127 : bind = gimple_build_bind (NULL, NULL, block);
9165 1127 : gsi_replace (gsi_p, bind, true);
9166 1127 : gimple_bind_add_stmt (bind, stmt);
9167 :
9168 1127 : bfn_decl = builtin_decl_explicit (
9169 1127 : flag_openmp_ompt ? BUILT_IN_GOMP_HAS_MASKED_THREAD_NUM_WITH_END
9170 : : BUILT_IN_GOMP_HAS_MASKED_THREAD_NUM);
9171 1127 : x = build_call_expr_loc (loc, bfn_decl, 1, filter);
9172 1127 : x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab));
9173 1127 : tseq = NULL;
9174 1127 : gimplify_and_add (x, &tseq);
9175 1127 : gimple_bind_add_seq (bind, tseq);
9176 :
9177 1127 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
9178 1127 : gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
9179 1127 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
9180 1127 : gimple_omp_set_body (stmt, NULL);
9181 :
9182 1127 : if (flag_openmp_ompt)
9183 : {
9184 : /* Insert call to GOMP_masked_end at the end of the body. */
9185 4 : tree decl = builtin_decl_explicit (BUILT_IN_GOMP_MASKED_END);
9186 4 : gcall *g = gimple_build_call (decl, 0);
9187 4 : gimple_bind_add_stmt (bind, g);
9188 : }
9189 :
9190 1127 : gimple_bind_add_stmt (bind, gimple_build_label (lab));
9191 :
9192 1127 : gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
9193 :
9194 1127 : pop_gimplify_context (bind);
9195 :
9196 1127 : gimple_bind_append_vars (bind, ctx->block_vars);
9197 1127 : BLOCK_VARS (block) = ctx->block_vars;
9198 1127 : }
9199 :
9200 : /* Helper function for lower_omp_task_reductions. For a specific PASS
9201 : find out the current clause it should be processed, or return false
9202 : if all have been processed already. */
9203 :
9204 : static inline bool
9205 8030 : omp_task_reduction_iterate (int pass, enum tree_code code,
9206 : enum omp_clause_code ccode, tree *c, tree *decl,
9207 : tree *type, tree *next)
9208 : {
9209 11412 : for (; *c; *c = omp_find_clause (OMP_CLAUSE_CHAIN (*c), ccode))
9210 : {
9211 6764 : if (ccode == OMP_CLAUSE_REDUCTION
9212 6708 : && code != OMP_TASKLOOP
9213 6708 : && !OMP_CLAUSE_REDUCTION_TASK (*c))
9214 56 : continue;
9215 6652 : *decl = OMP_CLAUSE_DECL (*c);
9216 6652 : *type = TREE_TYPE (*decl);
9217 6652 : if (TREE_CODE (*decl) == MEM_REF)
9218 : {
9219 1940 : if (pass != 1)
9220 970 : continue;
9221 : }
9222 : else
9223 : {
9224 4712 : if (omp_privatize_by_reference (*decl))
9225 168 : *type = TREE_TYPE (*type);
9226 4712 : if (pass != (!TREE_CONSTANT (TYPE_SIZE_UNIT (*type))))
9227 2356 : continue;
9228 : }
9229 3326 : *next = omp_find_clause (OMP_CLAUSE_CHAIN (*c), ccode);
9230 3326 : return true;
9231 : }
9232 4704 : *decl = NULL_TREE;
9233 4704 : *type = NULL_TREE;
9234 4704 : *next = NULL_TREE;
9235 4704 : return false;
9236 : }
9237 :
9238 : /* Lower task_reduction and reduction clauses (the latter unless CODE is
9239 : OMP_TASKGROUP only with task modifier). Register mapping of those in
9240 : START sequence and reducing them and unregister them in the END sequence. */
9241 :
9242 : static void
9243 1385 : lower_omp_task_reductions (omp_context *ctx, enum tree_code code, tree clauses,
9244 : gimple_seq *start, gimple_seq *end)
9245 : {
9246 849 : enum omp_clause_code ccode
9247 : = (code == OMP_TASKGROUP
9248 1385 : ? OMP_CLAUSE_TASK_REDUCTION : OMP_CLAUSE_REDUCTION);
9249 1385 : tree cancellable = NULL_TREE;
9250 1385 : clauses = omp_task_reductions_find_first (clauses, code, ccode);
9251 1385 : if (clauses == NULL_TREE)
9252 209 : return;
9253 1176 : if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE)
9254 : {
9255 325 : for (omp_context *outer = ctx->outer; outer; outer = outer->outer)
9256 239 : if (gimple_code (outer->stmt) == GIMPLE_OMP_PARALLEL
9257 239 : && outer->cancellable)
9258 : {
9259 14 : cancellable = error_mark_node;
9260 14 : break;
9261 : }
9262 225 : else if (gimple_code (outer->stmt) != GIMPLE_OMP_TASKGROUP
9263 225 : && gimple_code (outer->stmt) != GIMPLE_OMP_SCOPE)
9264 : break;
9265 : }
9266 1176 : tree record_type = lang_hooks.types.make_type (RECORD_TYPE);
9267 1176 : tree *last = &TYPE_FIELDS (record_type);
9268 1176 : unsigned cnt = 0;
9269 1176 : if (cancellable)
9270 : {
9271 14 : tree field = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE,
9272 : ptr_type_node);
9273 14 : tree ifield = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE,
9274 : integer_type_node);
9275 14 : *last = field;
9276 14 : DECL_CHAIN (field) = ifield;
9277 14 : last = &DECL_CHAIN (ifield);
9278 14 : DECL_CONTEXT (field) = record_type;
9279 14 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (field))
9280 0 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (field));
9281 14 : DECL_CONTEXT (ifield) = record_type;
9282 14 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (ifield))
9283 0 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (ifield));
9284 : }
9285 3528 : for (int pass = 0; pass < 2; pass++)
9286 : {
9287 2352 : tree decl, type, next;
9288 2352 : for (tree c = clauses;
9289 4015 : omp_task_reduction_iterate (pass, code, ccode,
9290 1663 : &c, &decl, &type, &next); c = next)
9291 : {
9292 1663 : ++cnt;
9293 1663 : tree new_type = type;
9294 1663 : if (ctx->outer)
9295 1166 : new_type = remap_type (type, &ctx->outer->cb);
9296 1663 : tree field
9297 1663 : = build_decl (OMP_CLAUSE_LOCATION (c), FIELD_DECL,
9298 1663 : DECL_P (decl) ? DECL_NAME (decl) : NULL_TREE,
9299 : new_type);
9300 1663 : if (DECL_P (decl) && type == TREE_TYPE (decl))
9301 : {
9302 1136 : SET_DECL_ALIGN (field, DECL_ALIGN (decl));
9303 1136 : DECL_USER_ALIGN (field) = DECL_USER_ALIGN (decl);
9304 1136 : TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (decl);
9305 : }
9306 : else
9307 527 : SET_DECL_ALIGN (field, TYPE_ALIGN (type));
9308 1663 : DECL_CONTEXT (field) = record_type;
9309 1663 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (field))
9310 1200 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (field));
9311 1663 : *last = field;
9312 1663 : last = &DECL_CHAIN (field);
9313 1663 : tree bfield
9314 1663 : = build_decl (OMP_CLAUSE_LOCATION (c), FIELD_DECL, NULL_TREE,
9315 : boolean_type_node);
9316 1663 : DECL_CONTEXT (bfield) = record_type;
9317 1663 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (bfield))
9318 0 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (bfield));
9319 1663 : *last = bfield;
9320 1663 : last = &DECL_CHAIN (bfield);
9321 : }
9322 : }
9323 1176 : *last = NULL_TREE;
9324 1176 : layout_type (record_type);
9325 :
9326 : /* Build up an array which registers with the runtime all the reductions
9327 : and deregisters them at the end. Format documented in libgomp/task.c. */
9328 1176 : tree atype = build_array_type_nelts (pointer_sized_int_node, 7 + cnt * 3);
9329 1176 : tree avar = create_tmp_var_raw (atype);
9330 1176 : gimple_add_tmp_var (avar);
9331 1176 : TREE_ADDRESSABLE (avar) = 1;
9332 1176 : tree r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_zero_node,
9333 : NULL_TREE, NULL_TREE);
9334 1176 : tree t = build_int_cst (pointer_sized_int_node, cnt);
9335 1176 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9336 1176 : gimple_seq seq = NULL;
9337 1176 : tree sz = fold_convert (pointer_sized_int_node,
9338 : TYPE_SIZE_UNIT (record_type));
9339 1176 : int cachesz = 64;
9340 1176 : sz = fold_build2 (PLUS_EXPR, pointer_sized_int_node, sz,
9341 : build_int_cst (pointer_sized_int_node, cachesz - 1));
9342 1176 : sz = fold_build2 (BIT_AND_EXPR, pointer_sized_int_node, sz,
9343 : build_int_cst (pointer_sized_int_node, ~(cachesz - 1)));
9344 1176 : ctx->task_reductions.create (1 + cnt);
9345 1176 : ctx->task_reduction_map = new hash_map<tree, unsigned>;
9346 2352 : ctx->task_reductions.quick_push (TREE_CODE (sz) == INTEGER_CST
9347 1216 : ? sz : NULL_TREE);
9348 1176 : sz = force_gimple_operand (sz, &seq, true, NULL_TREE);
9349 1176 : gimple_seq_add_seq (start, seq);
9350 1176 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_one_node,
9351 : NULL_TREE, NULL_TREE);
9352 1176 : gimple_seq_add_stmt (start, gimple_build_assign (r, sz));
9353 1176 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (2),
9354 : NULL_TREE, NULL_TREE);
9355 1176 : t = build_int_cst (pointer_sized_int_node,
9356 1176 : MAX (TYPE_ALIGN_UNIT (record_type), (unsigned) cachesz));
9357 1176 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9358 1176 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (3),
9359 : NULL_TREE, NULL_TREE);
9360 1176 : t = build_int_cst (pointer_sized_int_node, -1);
9361 1176 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9362 1176 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (4),
9363 : NULL_TREE, NULL_TREE);
9364 1176 : t = build_int_cst (pointer_sized_int_node, 0);
9365 1176 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9366 :
9367 : /* In end, build a loop that iterates from 0 to < omp_get_num_threads ()
9368 : and for each task reduction checks a bool right after the private variable
9369 : within that thread's chunk; if the bool is clear, it hasn't been
9370 : initialized and thus isn't going to be reduced nor destructed, otherwise
9371 : reduce and destruct it. */
9372 1176 : tree idx = create_tmp_var (size_type_node);
9373 1176 : gimple_seq_add_stmt (end, gimple_build_assign (idx, size_zero_node));
9374 1176 : tree num_thr_sz = create_tmp_var (size_type_node);
9375 1176 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
9376 1176 : tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
9377 1176 : tree lab3 = NULL_TREE, lab7 = NULL_TREE;
9378 1176 : gimple *g;
9379 1176 : if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE)
9380 : {
9381 : /* For worksharing constructs or scope, only perform it in the master
9382 : thread, with the exception of cancelled implicit barriers - then only
9383 : handle the current thread. */
9384 290 : tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
9385 290 : t = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
9386 290 : tree thr_num = create_tmp_var (integer_type_node);
9387 290 : g = gimple_build_call (t, 0);
9388 290 : gimple_call_set_lhs (g, thr_num);
9389 290 : gimple_seq_add_stmt (end, g);
9390 290 : if (cancellable)
9391 : {
9392 14 : tree c;
9393 14 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9394 14 : tree lab6 = create_artificial_label (UNKNOWN_LOCATION);
9395 14 : lab3 = create_artificial_label (UNKNOWN_LOCATION);
9396 14 : if (code == OMP_FOR)
9397 4 : c = gimple_omp_for_clauses (ctx->stmt);
9398 10 : else if (code == OMP_SECTIONS)
9399 0 : c = gimple_omp_sections_clauses (ctx->stmt);
9400 : else /* if (code == OMP_SCOPE) */
9401 10 : c = gimple_omp_scope_clauses (ctx->stmt);
9402 14 : c = OMP_CLAUSE_DECL (omp_find_clause (c, OMP_CLAUSE__REDUCTEMP_));
9403 14 : cancellable = c;
9404 14 : g = gimple_build_cond (NE_EXPR, c, build_zero_cst (TREE_TYPE (c)),
9405 : lab5, lab6);
9406 14 : gimple_seq_add_stmt (end, g);
9407 14 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9408 14 : g = gimple_build_assign (idx, NOP_EXPR, thr_num);
9409 14 : gimple_seq_add_stmt (end, g);
9410 14 : g = gimple_build_assign (num_thr_sz, PLUS_EXPR, idx,
9411 14 : build_one_cst (TREE_TYPE (idx)));
9412 14 : gimple_seq_add_stmt (end, g);
9413 14 : gimple_seq_add_stmt (end, gimple_build_goto (lab3));
9414 14 : gimple_seq_add_stmt (end, gimple_build_label (lab6));
9415 : }
9416 290 : g = gimple_build_cond (NE_EXPR, thr_num, integer_zero_node, lab2, lab4);
9417 290 : gimple_seq_add_stmt (end, g);
9418 290 : gimple_seq_add_stmt (end, gimple_build_label (lab4));
9419 : }
9420 1176 : if (code != OMP_PARALLEL)
9421 : {
9422 1114 : t = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
9423 1114 : tree num_thr = create_tmp_var (integer_type_node);
9424 1114 : g = gimple_build_call (t, 0);
9425 1114 : gimple_call_set_lhs (g, num_thr);
9426 1114 : gimple_seq_add_stmt (end, g);
9427 1114 : g = gimple_build_assign (num_thr_sz, NOP_EXPR, num_thr);
9428 1114 : gimple_seq_add_stmt (end, g);
9429 1114 : if (cancellable)
9430 14 : gimple_seq_add_stmt (end, gimple_build_label (lab3));
9431 : }
9432 : else
9433 : {
9434 62 : tree c = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
9435 : OMP_CLAUSE__REDUCTEMP_);
9436 62 : t = fold_convert (pointer_sized_int_node, OMP_CLAUSE_DECL (c));
9437 62 : t = fold_convert (size_type_node, t);
9438 62 : gimplify_assign (num_thr_sz, t, end);
9439 : }
9440 1176 : t = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (2),
9441 : NULL_TREE, NULL_TREE);
9442 1176 : tree data = create_tmp_var (pointer_sized_int_node);
9443 1176 : gimple_seq_add_stmt (end, gimple_build_assign (data, t));
9444 1176 : if (code == OMP_TASKLOOP)
9445 : {
9446 497 : lab7 = create_artificial_label (UNKNOWN_LOCATION);
9447 497 : g = gimple_build_cond (NE_EXPR, data,
9448 : build_zero_cst (pointer_sized_int_node),
9449 : lab1, lab7);
9450 497 : gimple_seq_add_stmt (end, g);
9451 : }
9452 1176 : gimple_seq_add_stmt (end, gimple_build_label (lab1));
9453 1176 : tree ptr;
9454 1176 : if (TREE_CODE (TYPE_SIZE_UNIT (record_type)) == INTEGER_CST)
9455 1136 : ptr = create_tmp_var (build_pointer_type (record_type));
9456 : else
9457 40 : ptr = create_tmp_var (ptr_type_node);
9458 1176 : gimple_seq_add_stmt (end, gimple_build_assign (ptr, NOP_EXPR, data));
9459 :
9460 1176 : tree field = TYPE_FIELDS (record_type);
9461 1176 : cnt = 0;
9462 1176 : if (cancellable)
9463 14 : field = DECL_CHAIN (DECL_CHAIN (field));
9464 3528 : for (int pass = 0; pass < 2; pass++)
9465 : {
9466 2352 : tree decl, type, next;
9467 2352 : for (tree c = clauses;
9468 4015 : omp_task_reduction_iterate (pass, code, ccode,
9469 1663 : &c, &decl, &type, &next); c = next)
9470 : {
9471 1663 : tree var = decl, ref;
9472 1663 : if (TREE_CODE (decl) == MEM_REF)
9473 : {
9474 485 : var = TREE_OPERAND (var, 0);
9475 485 : if (TREE_CODE (var) == POINTER_PLUS_EXPR)
9476 108 : var = TREE_OPERAND (var, 0);
9477 485 : tree v = var;
9478 485 : if (TREE_CODE (var) == ADDR_EXPR)
9479 310 : var = TREE_OPERAND (var, 0);
9480 175 : else if (INDIRECT_REF_P (var))
9481 4 : var = TREE_OPERAND (var, 0);
9482 485 : tree orig_var = var;
9483 485 : if (is_variable_sized (var))
9484 : {
9485 31 : gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
9486 31 : var = DECL_VALUE_EXPR (var);
9487 31 : gcc_assert (INDIRECT_REF_P (var));
9488 31 : var = TREE_OPERAND (var, 0);
9489 31 : gcc_assert (DECL_P (var));
9490 : }
9491 485 : t = ref = maybe_lookup_decl_in_outer_ctx (var, ctx);
9492 485 : if (orig_var != var)
9493 31 : gcc_assert (TREE_CODE (v) == ADDR_EXPR);
9494 454 : else if (TREE_CODE (v) == ADDR_EXPR)
9495 279 : t = build_fold_addr_expr (t);
9496 175 : else if (INDIRECT_REF_P (v))
9497 4 : t = build_fold_indirect_ref (t);
9498 485 : if (TREE_CODE (TREE_OPERAND (decl, 0)) == POINTER_PLUS_EXPR)
9499 : {
9500 108 : tree b = TREE_OPERAND (TREE_OPERAND (decl, 0), 1);
9501 108 : b = maybe_lookup_decl_in_outer_ctx (b, ctx);
9502 108 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t, b);
9503 : }
9504 485 : if (!integer_zerop (TREE_OPERAND (decl, 1)))
9505 219 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t,
9506 : fold_convert (size_type_node,
9507 : TREE_OPERAND (decl, 1)));
9508 : }
9509 : else
9510 : {
9511 1178 : t = ref = maybe_lookup_decl_in_outer_ctx (var, ctx);
9512 1178 : if (!omp_privatize_by_reference (decl))
9513 1136 : t = build_fold_addr_expr (t);
9514 : }
9515 1663 : t = fold_convert (pointer_sized_int_node, t);
9516 1663 : seq = NULL;
9517 1663 : t = force_gimple_operand (t, &seq, true, NULL_TREE);
9518 1663 : gimple_seq_add_seq (start, seq);
9519 1663 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar,
9520 1663 : size_int (7 + cnt * 3), NULL_TREE, NULL_TREE);
9521 1663 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9522 1663 : t = unshare_expr (byte_position (field));
9523 1663 : t = fold_convert (pointer_sized_int_node, t);
9524 1663 : ctx->task_reduction_map->put (c, cnt);
9525 3326 : ctx->task_reductions.quick_push (TREE_CODE (t) == INTEGER_CST
9526 2035 : ? t : NULL_TREE);
9527 1663 : seq = NULL;
9528 1663 : t = force_gimple_operand (t, &seq, true, NULL_TREE);
9529 1663 : gimple_seq_add_seq (start, seq);
9530 1663 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar,
9531 1663 : size_int (7 + cnt * 3 + 1), NULL_TREE, NULL_TREE);
9532 1663 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9533 :
9534 1663 : tree bfield = DECL_CHAIN (field);
9535 1663 : tree cond;
9536 1663 : if (code == OMP_PARALLEL
9537 1663 : || code == OMP_FOR
9538 : || code == OMP_SECTIONS
9539 : || code == OMP_SCOPE)
9540 : /* In parallel, worksharing or scope all threads unconditionally
9541 : initialize all their task reduction private variables. */
9542 565 : cond = boolean_true_node;
9543 1098 : else if (TREE_TYPE (ptr) == ptr_type_node)
9544 : {
9545 261 : cond = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr,
9546 : unshare_expr (byte_position (bfield)));
9547 261 : seq = NULL;
9548 261 : cond = force_gimple_operand (cond, &seq, true, NULL_TREE);
9549 261 : gimple_seq_add_seq (end, seq);
9550 261 : tree pbool = build_pointer_type (TREE_TYPE (bfield));
9551 261 : cond = build2 (MEM_REF, TREE_TYPE (bfield), cond,
9552 : build_int_cst (pbool, 0));
9553 : }
9554 : else
9555 837 : cond = build3 (COMPONENT_REF, TREE_TYPE (bfield),
9556 : build_simple_mem_ref (ptr), bfield, NULL_TREE);
9557 1663 : tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
9558 1663 : tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
9559 1663 : tree condv = create_tmp_var (boolean_type_node);
9560 1663 : gimple_seq_add_stmt (end, gimple_build_assign (condv, cond));
9561 1663 : g = gimple_build_cond (NE_EXPR, condv, boolean_false_node,
9562 : lab3, lab4);
9563 1663 : gimple_seq_add_stmt (end, g);
9564 1663 : gimple_seq_add_stmt (end, gimple_build_label (lab3));
9565 1663 : if (cancellable && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) == NULL_TREE)
9566 : {
9567 : /* If this reduction doesn't need destruction and parallel
9568 : has been cancelled, there is nothing to do for this
9569 : reduction, so jump around the merge operation. */
9570 18 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9571 18 : g = gimple_build_cond (NE_EXPR, cancellable,
9572 18 : build_zero_cst (TREE_TYPE (cancellable)),
9573 : lab4, lab5);
9574 18 : gimple_seq_add_stmt (end, g);
9575 18 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9576 : }
9577 :
9578 1663 : tree new_var;
9579 1663 : if (TREE_TYPE (ptr) == ptr_type_node)
9580 : {
9581 428 : new_var = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr,
9582 : unshare_expr (byte_position (field)));
9583 428 : seq = NULL;
9584 428 : new_var = force_gimple_operand (new_var, &seq, true, NULL_TREE);
9585 428 : gimple_seq_add_seq (end, seq);
9586 428 : tree pbool = build_pointer_type (TREE_TYPE (field));
9587 428 : new_var = build2 (MEM_REF, TREE_TYPE (field), new_var,
9588 : build_int_cst (pbool, 0));
9589 : }
9590 : else
9591 1235 : new_var = build3 (COMPONENT_REF, TREE_TYPE (field),
9592 : build_simple_mem_ref (ptr), field, NULL_TREE);
9593 :
9594 1663 : enum tree_code rcode = OMP_CLAUSE_REDUCTION_CODE (c);
9595 1663 : if (TREE_CODE (decl) != MEM_REF
9596 1663 : && omp_privatize_by_reference (decl))
9597 42 : ref = build_simple_mem_ref (ref);
9598 : /* reduction(-:var) sums up the partial results, so it acts
9599 : identically to reduction(+:var). */
9600 1663 : if (rcode == MINUS_EXPR)
9601 0 : rcode = PLUS_EXPR;
9602 1663 : if (TREE_CODE (decl) == MEM_REF)
9603 : {
9604 485 : tree type = TREE_TYPE (new_var);
9605 485 : tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
9606 485 : tree i = create_tmp_var (TREE_TYPE (v));
9607 485 : tree ptype = build_pointer_type (TREE_TYPE (type));
9608 485 : if (DECL_P (v))
9609 : {
9610 142 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
9611 142 : tree vv = create_tmp_var (TREE_TYPE (v));
9612 142 : gimplify_assign (vv, v, start);
9613 142 : v = vv;
9614 : }
9615 485 : ref = build4 (ARRAY_REF, pointer_sized_int_node, avar,
9616 485 : size_int (7 + cnt * 3), NULL_TREE, NULL_TREE);
9617 485 : new_var = build_fold_addr_expr (new_var);
9618 485 : new_var = fold_convert (ptype, new_var);
9619 485 : ref = fold_convert (ptype, ref);
9620 485 : tree m = create_tmp_var (ptype);
9621 485 : gimplify_assign (m, new_var, end);
9622 485 : new_var = m;
9623 485 : m = create_tmp_var (ptype);
9624 485 : gimplify_assign (m, ref, end);
9625 485 : ref = m;
9626 485 : gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), end);
9627 485 : tree body = create_artificial_label (UNKNOWN_LOCATION);
9628 485 : tree endl = create_artificial_label (UNKNOWN_LOCATION);
9629 485 : gimple_seq_add_stmt (end, gimple_build_label (body));
9630 485 : tree priv = build_simple_mem_ref (new_var);
9631 485 : tree out = build_simple_mem_ref (ref);
9632 485 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
9633 : {
9634 161 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
9635 161 : tree decl_placeholder
9636 161 : = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
9637 161 : tree lab6 = NULL_TREE;
9638 161 : if (cancellable)
9639 : {
9640 : /* If this reduction needs destruction and parallel
9641 : has been cancelled, jump around the merge operation
9642 : to the destruction. */
9643 4 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9644 4 : lab6 = create_artificial_label (UNKNOWN_LOCATION);
9645 4 : tree zero = build_zero_cst (TREE_TYPE (cancellable));
9646 4 : g = gimple_build_cond (NE_EXPR, cancellable, zero,
9647 : lab6, lab5);
9648 4 : gimple_seq_add_stmt (end, g);
9649 4 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9650 : }
9651 161 : SET_DECL_VALUE_EXPR (placeholder, out);
9652 161 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
9653 161 : SET_DECL_VALUE_EXPR (decl_placeholder, priv);
9654 161 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
9655 161 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
9656 322 : gimple_seq_add_seq (end,
9657 161 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
9658 161 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
9659 161 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
9660 : {
9661 56 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
9662 56 : OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c) = NULL;
9663 : }
9664 161 : if (cancellable)
9665 4 : gimple_seq_add_stmt (end, gimple_build_label (lab6));
9666 161 : tree x = lang_hooks.decls.omp_clause_dtor (c, priv);
9667 161 : if (x)
9668 : {
9669 137 : gimple_seq tseq = NULL;
9670 137 : gimplify_stmt (&x, &tseq);
9671 137 : gimple_seq_add_seq (end, tseq);
9672 : }
9673 : }
9674 : else
9675 : {
9676 324 : tree x = build2 (rcode, TREE_TYPE (out), out, priv);
9677 324 : out = unshare_expr (out);
9678 324 : gimplify_assign (out, x, end);
9679 : }
9680 485 : gimple *g
9681 485 : = gimple_build_assign (new_var, POINTER_PLUS_EXPR, new_var,
9682 485 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
9683 485 : gimple_seq_add_stmt (end, g);
9684 485 : g = gimple_build_assign (ref, POINTER_PLUS_EXPR, ref,
9685 485 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
9686 485 : gimple_seq_add_stmt (end, g);
9687 485 : g = gimple_build_assign (i, PLUS_EXPR, i,
9688 485 : build_int_cst (TREE_TYPE (i), 1));
9689 485 : gimple_seq_add_stmt (end, g);
9690 485 : g = gimple_build_cond (LE_EXPR, i, v, body, endl);
9691 485 : gimple_seq_add_stmt (end, g);
9692 485 : gimple_seq_add_stmt (end, gimple_build_label (endl));
9693 : }
9694 1178 : else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
9695 : {
9696 135 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
9697 135 : tree oldv = NULL_TREE;
9698 135 : tree lab6 = NULL_TREE;
9699 135 : if (cancellable)
9700 : {
9701 : /* If this reduction needs destruction and parallel
9702 : has been cancelled, jump around the merge operation
9703 : to the destruction. */
9704 4 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9705 4 : lab6 = create_artificial_label (UNKNOWN_LOCATION);
9706 4 : tree zero = build_zero_cst (TREE_TYPE (cancellable));
9707 4 : g = gimple_build_cond (NE_EXPR, cancellable, zero,
9708 : lab6, lab5);
9709 4 : gimple_seq_add_stmt (end, g);
9710 4 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9711 : }
9712 135 : if (omp_privatize_by_reference (decl)
9713 149 : && !useless_type_conversion_p (TREE_TYPE (placeholder),
9714 14 : TREE_TYPE (ref)))
9715 0 : ref = build_fold_addr_expr_loc (OMP_CLAUSE_LOCATION (c), ref);
9716 135 : ref = build_fold_addr_expr_loc (OMP_CLAUSE_LOCATION (c), ref);
9717 135 : tree refv = create_tmp_var (TREE_TYPE (ref));
9718 135 : gimplify_assign (refv, ref, end);
9719 135 : ref = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), refv);
9720 135 : SET_DECL_VALUE_EXPR (placeholder, ref);
9721 135 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
9722 135 : tree d = maybe_lookup_decl (decl, ctx);
9723 135 : gcc_assert (d);
9724 135 : if (DECL_HAS_VALUE_EXPR_P (d))
9725 7 : oldv = DECL_VALUE_EXPR (d);
9726 135 : if (omp_privatize_by_reference (var))
9727 : {
9728 14 : tree v = fold_convert (TREE_TYPE (d),
9729 : build_fold_addr_expr (new_var));
9730 14 : SET_DECL_VALUE_EXPR (d, v);
9731 : }
9732 : else
9733 121 : SET_DECL_VALUE_EXPR (d, new_var);
9734 135 : DECL_HAS_VALUE_EXPR_P (d) = 1;
9735 135 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
9736 135 : if (oldv)
9737 7 : SET_DECL_VALUE_EXPR (d, oldv);
9738 : else
9739 : {
9740 128 : SET_DECL_VALUE_EXPR (d, NULL_TREE);
9741 128 : DECL_HAS_VALUE_EXPR_P (d) = 0;
9742 : }
9743 135 : gimple_seq_add_seq (end, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
9744 135 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
9745 135 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
9746 24 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
9747 135 : if (cancellable)
9748 4 : gimple_seq_add_stmt (end, gimple_build_label (lab6));
9749 135 : tree x = lang_hooks.decls.omp_clause_dtor (c, new_var);
9750 135 : if (x)
9751 : {
9752 37 : gimple_seq tseq = NULL;
9753 37 : gimplify_stmt (&x, &tseq);
9754 37 : gimple_seq_add_seq (end, tseq);
9755 : }
9756 : }
9757 : else
9758 : {
9759 1043 : tree x = build2 (rcode, TREE_TYPE (ref), ref, new_var);
9760 1043 : ref = unshare_expr (ref);
9761 1043 : gimplify_assign (ref, x, end);
9762 : }
9763 1663 : gimple_seq_add_stmt (end, gimple_build_label (lab4));
9764 1663 : ++cnt;
9765 1663 : field = DECL_CHAIN (bfield);
9766 : }
9767 : }
9768 :
9769 1176 : if (code == OMP_TASKGROUP)
9770 : {
9771 327 : t = builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_REDUCTION_REGISTER);
9772 327 : g = gimple_build_call (t, 1, build_fold_addr_expr (avar));
9773 327 : gimple_seq_add_stmt (start, g);
9774 : }
9775 : else
9776 : {
9777 849 : tree c;
9778 849 : if (code == OMP_FOR)
9779 228 : c = gimple_omp_for_clauses (ctx->stmt);
9780 621 : else if (code == OMP_SECTIONS)
9781 8 : c = gimple_omp_sections_clauses (ctx->stmt);
9782 613 : else if (code == OMP_SCOPE)
9783 54 : c = gimple_omp_scope_clauses (ctx->stmt);
9784 : else
9785 559 : c = gimple_omp_taskreg_clauses (ctx->stmt);
9786 849 : c = omp_find_clause (c, OMP_CLAUSE__REDUCTEMP_);
9787 849 : t = fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (c)),
9788 : build_fold_addr_expr (avar));
9789 849 : gimplify_assign (OMP_CLAUSE_DECL (c), t, start);
9790 : }
9791 :
9792 1176 : gimple_seq_add_stmt (end, gimple_build_assign (data, PLUS_EXPR, data, sz));
9793 1176 : gimple_seq_add_stmt (end, gimple_build_assign (idx, PLUS_EXPR, idx,
9794 : size_one_node));
9795 1176 : g = gimple_build_cond (NE_EXPR, idx, num_thr_sz, lab1, lab2);
9796 1176 : gimple_seq_add_stmt (end, g);
9797 1176 : gimple_seq_add_stmt (end, gimple_build_label (lab2));
9798 1176 : if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE)
9799 : {
9800 290 : enum built_in_function bfn
9801 : = BUILT_IN_GOMP_WORKSHARE_TASK_REDUCTION_UNREGISTER;
9802 290 : t = builtin_decl_explicit (bfn);
9803 290 : tree c_bool_type = TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (t)));
9804 290 : tree arg;
9805 290 : if (cancellable)
9806 : {
9807 14 : arg = create_tmp_var (c_bool_type);
9808 14 : gimple_seq_add_stmt (end, gimple_build_assign (arg, NOP_EXPR,
9809 : cancellable));
9810 : }
9811 : else
9812 276 : arg = build_int_cst (c_bool_type, 0);
9813 290 : g = gimple_build_call (t, 1, arg);
9814 290 : }
9815 : else
9816 : {
9817 886 : t = builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_REDUCTION_UNREGISTER);
9818 886 : g = gimple_build_call (t, 1, build_fold_addr_expr (avar));
9819 : }
9820 1176 : gimple_seq_add_stmt (end, g);
9821 1176 : if (lab7)
9822 497 : gimple_seq_add_stmt (end, gimple_build_label (lab7));
9823 1176 : t = build_constructor (atype, NULL);
9824 1176 : TREE_THIS_VOLATILE (t) = 1;
9825 1176 : gimple_seq_add_stmt (end, gimple_build_assign (avar, t));
9826 : }
9827 :
9828 : /* Expand code for an OpenMP taskgroup directive. */
9829 :
9830 : static void
9831 536 : lower_omp_taskgroup (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9832 : {
9833 536 : gimple *stmt = gsi_stmt (*gsi_p);
9834 536 : gcall *x;
9835 536 : gbind *bind;
9836 536 : gimple_seq dseq = NULL;
9837 536 : tree block = make_node (BLOCK);
9838 :
9839 536 : bind = gimple_build_bind (NULL, NULL, block);
9840 536 : gsi_replace (gsi_p, bind, true);
9841 536 : gimple_bind_add_stmt (bind, stmt);
9842 :
9843 536 : push_gimplify_context ();
9844 :
9845 536 : x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START),
9846 : 0);
9847 536 : gimple_bind_add_stmt (bind, x);
9848 :
9849 536 : lower_omp_task_reductions (ctx, OMP_TASKGROUP,
9850 : gimple_omp_taskgroup_clauses (stmt),
9851 : gimple_bind_body_ptr (bind), &dseq);
9852 :
9853 536 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
9854 536 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
9855 536 : gimple_omp_set_body (stmt, NULL);
9856 :
9857 536 : gimple_bind_add_seq (bind, dseq);
9858 :
9859 536 : pop_gimplify_context (bind);
9860 :
9861 536 : gimple_bind_append_vars (bind, ctx->block_vars);
9862 536 : BLOCK_VARS (block) = ctx->block_vars;
9863 536 : }
9864 :
9865 :
9866 : /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */
9867 :
9868 : static void
9869 0 : lower_omp_ordered_clauses (gimple_stmt_iterator *gsi_p, gomp_ordered *ord_stmt,
9870 : omp_context *ctx)
9871 : {
9872 0 : struct omp_for_data fd;
9873 0 : if (!ctx->outer || gimple_code (ctx->outer->stmt) != GIMPLE_OMP_FOR)
9874 0 : return;
9875 :
9876 0 : unsigned int len = gimple_omp_for_collapse (ctx->outer->stmt);
9877 0 : struct omp_for_data_loop *loops = XALLOCAVEC (struct omp_for_data_loop, len);
9878 0 : omp_extract_for_data (as_a <gomp_for *> (ctx->outer->stmt), &fd, loops);
9879 0 : if (!fd.ordered)
9880 : return;
9881 :
9882 0 : tree *list_p = gimple_omp_ordered_clauses_ptr (ord_stmt);
9883 0 : tree c = gimple_omp_ordered_clauses (ord_stmt);
9884 0 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS
9885 0 : && OMP_CLAUSE_DOACROSS_KIND (c) == OMP_CLAUSE_DOACROSS_SINK)
9886 : {
9887 : /* Merge depend clauses from multiple adjacent
9888 : #pragma omp ordered depend(sink:...) constructs
9889 : into one #pragma omp ordered depend(sink:...), so that
9890 : we can optimize them together. */
9891 0 : gimple_stmt_iterator gsi = *gsi_p;
9892 0 : gsi_next (&gsi);
9893 0 : while (!gsi_end_p (gsi))
9894 : {
9895 0 : gimple *stmt = gsi_stmt (gsi);
9896 0 : if (is_gimple_debug (stmt)
9897 0 : || gimple_code (stmt) == GIMPLE_NOP)
9898 : {
9899 0 : gsi_next (&gsi);
9900 0 : continue;
9901 : }
9902 0 : if (gimple_code (stmt) != GIMPLE_OMP_ORDERED)
9903 : break;
9904 0 : gomp_ordered *ord_stmt2 = as_a <gomp_ordered *> (stmt);
9905 0 : c = gimple_omp_ordered_clauses (ord_stmt2);
9906 0 : if (c == NULL_TREE
9907 0 : || OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DOACROSS
9908 0 : || OMP_CLAUSE_DOACROSS_KIND (c) != OMP_CLAUSE_DOACROSS_SINK)
9909 : break;
9910 0 : while (*list_p)
9911 0 : list_p = &OMP_CLAUSE_CHAIN (*list_p);
9912 0 : *list_p = c;
9913 0 : gsi_remove (&gsi, true);
9914 : }
9915 : }
9916 :
9917 : /* Canonicalize sink dependence clauses into one folded clause if
9918 : possible.
9919 :
9920 : The basic algorithm is to create a sink vector whose first
9921 : element is the GCD of all the first elements, and whose remaining
9922 : elements are the minimum of the subsequent columns.
9923 :
9924 : We ignore dependence vectors whose first element is zero because
9925 : such dependencies are known to be executed by the same thread.
9926 :
9927 : We take into account the direction of the loop, so a minimum
9928 : becomes a maximum if the loop is iterating forwards. We also
9929 : ignore sink clauses where the loop direction is unknown, or where
9930 : the offsets are clearly invalid because they are not a multiple
9931 : of the loop increment.
9932 :
9933 : For example:
9934 :
9935 : #pragma omp for ordered(2)
9936 : for (i=0; i < N; ++i)
9937 : for (j=0; j < M; ++j)
9938 : {
9939 : #pragma omp ordered \
9940 : depend(sink:i-8,j-2) \
9941 : depend(sink:i,j-1) \ // Completely ignored because i+0.
9942 : depend(sink:i-4,j-3) \
9943 : depend(sink:i-6,j-4)
9944 : #pragma omp ordered depend(source)
9945 : }
9946 :
9947 : Folded clause is:
9948 :
9949 : depend(sink:-gcd(8,4,6),-min(2,3,4))
9950 : -or-
9951 : depend(sink:-2,-2)
9952 : */
9953 :
9954 : /* FIXME: Computing GCD's where the first element is zero is
9955 : non-trivial in the presence of collapsed loops. Do this later. */
9956 0 : if (fd.collapse > 1)
9957 : return;
9958 :
9959 0 : wide_int *folded_deps = XALLOCAVEC (wide_int, 2 * len - 1);
9960 :
9961 : /* wide_int is not a POD so it must be default-constructed. */
9962 0 : for (unsigned i = 0; i != 2 * len - 1; ++i)
9963 0 : new (static_cast<void*>(folded_deps + i)) wide_int ();
9964 :
9965 : tree folded_dep = NULL_TREE;
9966 : /* TRUE if the first dimension's offset is negative. */
9967 : bool neg_offset_p = false;
9968 :
9969 : list_p = gimple_omp_ordered_clauses_ptr (ord_stmt);
9970 : unsigned int i;
9971 0 : while ((c = *list_p) != NULL)
9972 : {
9973 0 : bool remove = false;
9974 :
9975 0 : gcc_assert (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS);
9976 0 : if (OMP_CLAUSE_DOACROSS_KIND (c) != OMP_CLAUSE_DOACROSS_SINK)
9977 0 : goto next_ordered_clause;
9978 :
9979 0 : tree vec;
9980 0 : for (vec = OMP_CLAUSE_DECL (c), i = 0;
9981 0 : vec && TREE_CODE (vec) == TREE_LIST;
9982 0 : vec = TREE_CHAIN (vec), ++i)
9983 : {
9984 0 : gcc_assert (i < len);
9985 :
9986 : /* omp_extract_for_data has canonicalized the condition. */
9987 0 : gcc_assert (fd.loops[i].cond_code == LT_EXPR
9988 : || fd.loops[i].cond_code == GT_EXPR);
9989 0 : bool forward = fd.loops[i].cond_code == LT_EXPR;
9990 0 : bool maybe_lexically_later = true;
9991 :
9992 : /* While the committee makes up its mind, bail if we have any
9993 : non-constant steps. */
9994 0 : if (TREE_CODE (fd.loops[i].step) != INTEGER_CST)
9995 0 : goto lower_omp_ordered_ret;
9996 :
9997 0 : tree itype = TREE_TYPE (TREE_VALUE (vec));
9998 0 : if (POINTER_TYPE_P (itype))
9999 0 : itype = sizetype;
10000 0 : wide_int offset = wide_int::from (wi::to_wide (TREE_PURPOSE (vec)),
10001 0 : TYPE_PRECISION (itype),
10002 0 : TYPE_SIGN (itype));
10003 :
10004 : /* Ignore invalid offsets that are not multiples of the step. */
10005 0 : if (!wi::multiple_of_p (wi::abs (offset),
10006 0 : wi::abs (wi::to_wide (fd.loops[i].step)),
10007 : UNSIGNED))
10008 : {
10009 0 : warning_at (OMP_CLAUSE_LOCATION (c), OPT_Wopenmp,
10010 : "ignoring %<sink%> clause with offset that is not "
10011 : "a multiple of the loop step");
10012 0 : remove = true;
10013 0 : goto next_ordered_clause;
10014 : }
10015 :
10016 : /* Calculate the first dimension. The first dimension of
10017 : the folded dependency vector is the GCD of the first
10018 : elements, while ignoring any first elements whose offset
10019 : is 0. */
10020 0 : if (i == 0)
10021 : {
10022 : /* Ignore dependence vectors whose first dimension is 0. */
10023 0 : if (offset == 0)
10024 : {
10025 0 : remove = true;
10026 0 : goto next_ordered_clause;
10027 : }
10028 : else
10029 : {
10030 0 : if (!TYPE_UNSIGNED (itype) && (forward ^ wi::neg_p (offset)))
10031 : {
10032 0 : error_at (OMP_CLAUSE_LOCATION (c),
10033 : "first offset must be in opposite direction "
10034 : "of loop iterations");
10035 0 : goto lower_omp_ordered_ret;
10036 : }
10037 0 : if (forward)
10038 0 : offset = -offset;
10039 0 : neg_offset_p = forward;
10040 : /* Initialize the first time around. */
10041 0 : if (folded_dep == NULL_TREE)
10042 : {
10043 0 : folded_dep = c;
10044 0 : folded_deps[0] = offset;
10045 : }
10046 : else
10047 0 : folded_deps[0] = wi::gcd (folded_deps[0],
10048 0 : offset, UNSIGNED);
10049 : }
10050 : }
10051 : /* Calculate minimum for the remaining dimensions. */
10052 : else
10053 : {
10054 0 : folded_deps[len + i - 1] = offset;
10055 0 : if (folded_dep == c)
10056 0 : folded_deps[i] = offset;
10057 0 : else if (maybe_lexically_later
10058 0 : && !wi::eq_p (folded_deps[i], offset))
10059 : {
10060 0 : if (forward ^ wi::gts_p (folded_deps[i], offset))
10061 : {
10062 : unsigned int j;
10063 0 : folded_dep = c;
10064 0 : for (j = 1; j <= i; j++)
10065 0 : folded_deps[j] = folded_deps[len + j - 1];
10066 : }
10067 : else
10068 0 : maybe_lexically_later = false;
10069 : }
10070 : }
10071 0 : }
10072 0 : gcc_assert (i == len);
10073 :
10074 : remove = true;
10075 :
10076 0 : next_ordered_clause:
10077 0 : if (remove)
10078 0 : *list_p = OMP_CLAUSE_CHAIN (c);
10079 : else
10080 0 : list_p = &OMP_CLAUSE_CHAIN (c);
10081 : }
10082 :
10083 0 : if (folded_dep)
10084 : {
10085 0 : if (neg_offset_p)
10086 0 : folded_deps[0] = -folded_deps[0];
10087 :
10088 0 : tree itype = TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep)));
10089 0 : if (POINTER_TYPE_P (itype))
10090 0 : itype = sizetype;
10091 :
10092 0 : TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep))
10093 0 : = wide_int_to_tree (itype, folded_deps[0]);
10094 0 : OMP_CLAUSE_CHAIN (folded_dep) = gimple_omp_ordered_clauses (ord_stmt);
10095 0 : *gimple_omp_ordered_clauses_ptr (ord_stmt) = folded_dep;
10096 : }
10097 :
10098 0 : lower_omp_ordered_ret:
10099 :
10100 : /* Ordered without clauses is #pragma omp threads, while we want
10101 : a nop instead if we remove all clauses. */
10102 0 : if (gimple_omp_ordered_clauses (ord_stmt) == NULL_TREE)
10103 0 : gsi_replace (gsi_p, gimple_build_nop (), true);
10104 : }
10105 :
10106 :
10107 : /* Expand code for an OpenMP ordered directive. */
10108 :
10109 : static void
10110 1128 : lower_omp_ordered (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10111 : {
10112 1128 : tree block;
10113 1128 : gimple *stmt = gsi_stmt (*gsi_p), *g;
10114 1128 : gomp_ordered *ord_stmt = as_a <gomp_ordered *> (stmt);
10115 1128 : gcall *x;
10116 1128 : gbind *bind;
10117 1128 : bool simd = omp_find_clause (gimple_omp_ordered_clauses (ord_stmt),
10118 1128 : OMP_CLAUSE_SIMD);
10119 : /* FIXME: this should check presence of OMP_CLAUSE__SIMT_ on the enclosing
10120 : loop. */
10121 1128 : bool maybe_simt
10122 1128 : = simd && omp_maybe_offloaded_ctx (ctx) && omp_max_simt_vf () > 1;
10123 1128 : bool threads = omp_find_clause (gimple_omp_ordered_clauses (ord_stmt),
10124 1128 : OMP_CLAUSE_THREADS);
10125 :
10126 1128 : if (gimple_omp_ordered_standalone_p (ord_stmt))
10127 : {
10128 : /* FIXME: This is needs to be moved to the expansion to verify various
10129 : conditions only testable on cfg with dominators computed, and also
10130 : all the depend clauses to be merged still might need to be available
10131 : for the runtime checks. */
10132 : if (0)
10133 : lower_omp_ordered_clauses (gsi_p, ord_stmt, ctx);
10134 1128 : return;
10135 : }
10136 :
10137 415 : push_gimplify_context ();
10138 :
10139 415 : block = make_node (BLOCK);
10140 415 : bind = gimple_build_bind (NULL, NULL, block);
10141 415 : gsi_replace (gsi_p, bind, true);
10142 415 : gimple_bind_add_stmt (bind, stmt);
10143 :
10144 415 : if (simd)
10145 : {
10146 81 : x = gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START, 1,
10147 81 : build_int_cst (integer_type_node, threads));
10148 81 : cfun->has_simduid_loops = true;
10149 : }
10150 : else
10151 334 : x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START),
10152 : 0);
10153 415 : gimple_bind_add_stmt (bind, x);
10154 :
10155 415 : tree counter = NULL_TREE, test = NULL_TREE, body = NULL_TREE;
10156 415 : if (maybe_simt)
10157 : {
10158 0 : counter = create_tmp_var (integer_type_node);
10159 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_LANE, 0);
10160 0 : gimple_call_set_lhs (g, counter);
10161 0 : gimple_bind_add_stmt (bind, g);
10162 :
10163 0 : body = create_artificial_label (UNKNOWN_LOCATION);
10164 0 : test = create_artificial_label (UNKNOWN_LOCATION);
10165 0 : gimple_bind_add_stmt (bind, gimple_build_label (body));
10166 :
10167 0 : tree simt_pred = create_tmp_var (integer_type_node);
10168 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_ORDERED_PRED, 1, counter);
10169 0 : gimple_call_set_lhs (g, simt_pred);
10170 0 : gimple_bind_add_stmt (bind, g);
10171 :
10172 0 : tree t = create_artificial_label (UNKNOWN_LOCATION);
10173 0 : g = gimple_build_cond (EQ_EXPR, simt_pred, integer_zero_node, t, test);
10174 0 : gimple_bind_add_stmt (bind, g);
10175 :
10176 0 : gimple_bind_add_stmt (bind, gimple_build_label (t));
10177 : }
10178 415 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
10179 415 : gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
10180 415 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10181 415 : gimple_omp_set_body (stmt, NULL);
10182 :
10183 415 : if (maybe_simt)
10184 : {
10185 0 : gimple_bind_add_stmt (bind, gimple_build_label (test));
10186 0 : g = gimple_build_assign (counter, MINUS_EXPR, counter, integer_one_node);
10187 0 : gimple_bind_add_stmt (bind, g);
10188 :
10189 0 : tree c = build2 (GE_EXPR, boolean_type_node, counter, integer_zero_node);
10190 0 : tree nonneg = create_tmp_var (integer_type_node);
10191 0 : gimple_seq tseq = NULL;
10192 0 : gimplify_assign (nonneg, fold_convert (integer_type_node, c), &tseq);
10193 0 : gimple_bind_add_seq (bind, tseq);
10194 :
10195 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_VOTE_ANY, 1, nonneg);
10196 0 : gimple_call_set_lhs (g, nonneg);
10197 0 : gimple_bind_add_stmt (bind, g);
10198 :
10199 0 : tree end = create_artificial_label (UNKNOWN_LOCATION);
10200 0 : g = gimple_build_cond (NE_EXPR, nonneg, integer_zero_node, body, end);
10201 0 : gimple_bind_add_stmt (bind, g);
10202 :
10203 0 : gimple_bind_add_stmt (bind, gimple_build_label (end));
10204 : }
10205 415 : if (simd)
10206 81 : x = gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END, 1,
10207 81 : build_int_cst (integer_type_node, threads));
10208 : else
10209 334 : x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END),
10210 : 0);
10211 415 : gimple_bind_add_stmt (bind, x);
10212 :
10213 415 : gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10214 :
10215 415 : pop_gimplify_context (bind);
10216 :
10217 415 : gimple_bind_append_vars (bind, ctx->block_vars);
10218 415 : BLOCK_VARS (block) = gimple_bind_vars (bind);
10219 : }
10220 :
10221 :
10222 : /* Expand code for an OpenMP scan directive and the structured block
10223 : before the scan directive. */
10224 :
10225 : static void
10226 1592 : lower_omp_scan (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10227 : {
10228 1592 : gimple *stmt = gsi_stmt (*gsi_p);
10229 1592 : bool has_clauses
10230 1592 : = gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt)) != NULL;
10231 1592 : tree lane = NULL_TREE;
10232 1592 : gimple_seq before = NULL;
10233 1592 : omp_context *octx = ctx->outer;
10234 1592 : gcc_assert (octx);
10235 1592 : if (octx->scan_exclusive && !has_clauses)
10236 : {
10237 536 : gimple_stmt_iterator gsi2 = *gsi_p;
10238 536 : gsi_next (&gsi2);
10239 536 : gimple *stmt2 = gsi_stmt (gsi2);
10240 : /* For exclusive scan, swap GIMPLE_OMP_SCAN without clauses
10241 : with following GIMPLE_OMP_SCAN with clauses, so that input_phase,
10242 : the one with exclusive clause(s), comes first. */
10243 536 : if (stmt2
10244 274 : && gimple_code (stmt2) == GIMPLE_OMP_SCAN
10245 804 : && gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt2)) != NULL)
10246 : {
10247 268 : gsi_remove (gsi_p, false);
10248 268 : gsi_insert_after (gsi_p, stmt, GSI_SAME_STMT);
10249 268 : ctx = maybe_lookup_ctx (stmt2);
10250 268 : gcc_assert (ctx);
10251 268 : lower_omp_scan (gsi_p, ctx);
10252 268 : return;
10253 : }
10254 : }
10255 :
10256 1324 : bool input_phase = has_clauses ^ octx->scan_inclusive;
10257 1324 : bool is_simd = (gimple_code (octx->stmt) == GIMPLE_OMP_FOR
10258 1324 : && gimple_omp_for_kind (octx->stmt) == GF_OMP_FOR_KIND_SIMD);
10259 1324 : bool is_for = (gimple_code (octx->stmt) == GIMPLE_OMP_FOR
10260 1324 : && gimple_omp_for_kind (octx->stmt) == GF_OMP_FOR_KIND_FOR
10261 1694 : && !gimple_omp_for_combined_p (octx->stmt));
10262 1324 : bool is_for_simd = is_simd && gimple_omp_for_combined_into_p (octx->stmt);
10263 364 : if (is_for_simd && octx->for_simd_scan_phase)
10264 : is_simd = false;
10265 1142 : if (is_simd)
10266 772 : if (tree c = omp_find_clause (gimple_omp_for_clauses (octx->stmt),
10267 : OMP_CLAUSE__SIMDUID_))
10268 : {
10269 368 : tree uid = OMP_CLAUSE__SIMDUID__DECL (c);
10270 368 : lane = create_tmp_var (unsigned_type_node);
10271 368 : tree t = build_int_cst (integer_type_node,
10272 552 : input_phase ? 1
10273 184 : : octx->scan_inclusive ? 2 : 3);
10274 368 : gimple *g
10275 368 : = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 2, uid, t);
10276 368 : gimple_call_set_lhs (g, lane);
10277 368 : gimple_seq_add_stmt (&before, g);
10278 : }
10279 :
10280 1324 : if (is_simd || is_for)
10281 : {
10282 5100 : for (tree c = gimple_omp_for_clauses (octx->stmt);
10283 5100 : c; c = OMP_CLAUSE_CHAIN (c))
10284 4140 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
10285 4140 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
10286 : {
10287 1184 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
10288 1184 : tree var = OMP_CLAUSE_DECL (c);
10289 1184 : tree new_var = lookup_decl (var, octx);
10290 1184 : tree val = new_var;
10291 1184 : tree var2 = NULL_TREE;
10292 1184 : tree var3 = NULL_TREE;
10293 1184 : tree var4 = NULL_TREE;
10294 1184 : tree lane0 = NULL_TREE;
10295 1184 : tree new_vard = new_var;
10296 1184 : if (omp_privatize_by_reference (var))
10297 : {
10298 184 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
10299 184 : val = new_var;
10300 : }
10301 1184 : if (DECL_HAS_VALUE_EXPR_P (new_vard))
10302 : {
10303 464 : val = DECL_VALUE_EXPR (new_vard);
10304 464 : if (new_vard != new_var)
10305 : {
10306 76 : gcc_assert (TREE_CODE (val) == ADDR_EXPR);
10307 76 : val = TREE_OPERAND (val, 0);
10308 : }
10309 464 : if (TREE_CODE (val) == ARRAY_REF
10310 464 : && VAR_P (TREE_OPERAND (val, 0)))
10311 : {
10312 464 : tree v = TREE_OPERAND (val, 0);
10313 464 : if (lookup_attribute ("omp simd array",
10314 464 : DECL_ATTRIBUTES (v)))
10315 : {
10316 464 : val = unshare_expr (val);
10317 464 : lane0 = TREE_OPERAND (val, 1);
10318 464 : TREE_OPERAND (val, 1) = lane;
10319 464 : var2 = lookup_decl (v, octx);
10320 464 : if (octx->scan_exclusive)
10321 228 : var4 = lookup_decl (var2, octx);
10322 464 : if (input_phase
10323 464 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
10324 120 : var3 = maybe_lookup_decl (var4 ? var4 : var2, octx);
10325 464 : if (!input_phase)
10326 : {
10327 232 : var2 = build4 (ARRAY_REF, TREE_TYPE (val),
10328 : var2, lane, NULL_TREE, NULL_TREE);
10329 232 : TREE_THIS_NOTRAP (var2) = 1;
10330 232 : if (octx->scan_exclusive)
10331 : {
10332 114 : var4 = build4 (ARRAY_REF, TREE_TYPE (val),
10333 : var4, lane, NULL_TREE,
10334 : NULL_TREE);
10335 114 : TREE_THIS_NOTRAP (var4) = 1;
10336 : }
10337 : }
10338 : else
10339 : var2 = val;
10340 : }
10341 : }
10342 464 : gcc_assert (var2);
10343 : }
10344 : else
10345 : {
10346 720 : var2 = build_outer_var_ref (var, octx);
10347 720 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
10348 : {
10349 220 : var3 = maybe_lookup_decl (new_vard, octx);
10350 220 : if (var3 == new_vard || var3 == NULL_TREE)
10351 : var3 = NULL_TREE;
10352 108 : else if (is_simd && octx->scan_exclusive && !input_phase)
10353 : {
10354 16 : var4 = maybe_lookup_decl (var3, octx);
10355 16 : if (var4 == var3 || var4 == NULL_TREE)
10356 : {
10357 0 : if (TREE_ADDRESSABLE (TREE_TYPE (new_var)))
10358 : {
10359 : var4 = var3;
10360 : var3 = NULL_TREE;
10361 : }
10362 : else
10363 16 : var4 = NULL_TREE;
10364 : }
10365 : }
10366 : }
10367 688 : if (is_simd
10368 500 : && octx->scan_exclusive
10369 244 : && !input_phase
10370 244 : && var4 == NULL_TREE)
10371 106 : var4 = create_tmp_var (TREE_TYPE (val));
10372 : }
10373 1184 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
10374 : {
10375 376 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
10376 376 : if (input_phase)
10377 : {
10378 188 : if (var3)
10379 : {
10380 : /* If we've added a separate identity element
10381 : variable, copy it over into val. */
10382 92 : tree x = lang_hooks.decls.omp_clause_assign_op (c, val,
10383 : var3);
10384 92 : gimplify_and_add (x, &before);
10385 : }
10386 96 : else if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
10387 : {
10388 : /* Otherwise, assign to it the identity element. */
10389 96 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
10390 96 : if (is_for)
10391 16 : tseq = copy_gimple_seq_and_replace_locals (tseq);
10392 96 : tree ref = build_outer_var_ref (var, octx);
10393 96 : tree x = (DECL_HAS_VALUE_EXPR_P (new_vard)
10394 96 : ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
10395 40 : if (x)
10396 : {
10397 40 : if (new_vard != new_var)
10398 8 : val = build_fold_addr_expr_loc (clause_loc, val);
10399 40 : SET_DECL_VALUE_EXPR (new_vard, val);
10400 : }
10401 96 : SET_DECL_VALUE_EXPR (placeholder, ref);
10402 96 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
10403 96 : lower_omp (&tseq, octx);
10404 96 : if (x)
10405 40 : SET_DECL_VALUE_EXPR (new_vard, x);
10406 96 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
10407 96 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
10408 96 : gimple_seq_add_seq (&before, tseq);
10409 96 : if (is_simd)
10410 80 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
10411 : }
10412 : }
10413 188 : else if (is_simd)
10414 : {
10415 156 : tree x;
10416 156 : if (octx->scan_exclusive)
10417 : {
10418 72 : tree v4 = unshare_expr (var4);
10419 72 : tree v2 = unshare_expr (var2);
10420 72 : x = lang_hooks.decls.omp_clause_assign_op (c, v4, v2);
10421 72 : gimplify_and_add (x, &before);
10422 : }
10423 156 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
10424 156 : x = (DECL_HAS_VALUE_EXPR_P (new_vard)
10425 156 : ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
10426 156 : tree vexpr = val;
10427 156 : if (x && new_vard != new_var)
10428 30 : vexpr = build_fold_addr_expr_loc (clause_loc, val);
10429 156 : if (x)
10430 78 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
10431 156 : SET_DECL_VALUE_EXPR (placeholder, var2);
10432 156 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
10433 156 : lower_omp (&tseq, octx);
10434 156 : gimple_seq_add_seq (&before, tseq);
10435 156 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
10436 156 : if (x)
10437 78 : SET_DECL_VALUE_EXPR (new_vard, x);
10438 156 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
10439 156 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
10440 156 : if (octx->scan_inclusive)
10441 : {
10442 84 : x = lang_hooks.decls.omp_clause_assign_op (c, val,
10443 : var2);
10444 84 : gimplify_and_add (x, &before);
10445 : }
10446 72 : else if (lane0 == NULL_TREE)
10447 : {
10448 36 : x = lang_hooks.decls.omp_clause_assign_op (c, val,
10449 : var4);
10450 36 : gimplify_and_add (x, &before);
10451 : }
10452 : }
10453 : }
10454 : else
10455 : {
10456 808 : if (input_phase)
10457 : {
10458 : /* input phase. Set val to initializer before
10459 : the body. */
10460 404 : tree x = omp_reduction_init (c, TREE_TYPE (new_var));
10461 404 : gimplify_assign (val, x, &before);
10462 : }
10463 404 : else if (is_simd)
10464 : {
10465 : /* scan phase. */
10466 326 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
10467 326 : if (code == MINUS_EXPR)
10468 0 : code = PLUS_EXPR;
10469 :
10470 326 : tree x = build2 (code, TREE_TYPE (var2),
10471 : unshare_expr (var2), unshare_expr (val));
10472 326 : if (octx->scan_inclusive)
10473 : {
10474 162 : gimplify_assign (unshare_expr (var2), x, &before);
10475 162 : gimplify_assign (val, var2, &before);
10476 : }
10477 : else
10478 : {
10479 164 : gimplify_assign (unshare_expr (var4),
10480 : unshare_expr (var2), &before);
10481 164 : gimplify_assign (var2, x, &before);
10482 164 : if (lane0 == NULL_TREE)
10483 86 : gimplify_assign (val, var4, &before);
10484 : }
10485 : }
10486 : }
10487 1184 : if (octx->scan_exclusive && !input_phase && lane0)
10488 : {
10489 114 : tree vexpr = unshare_expr (var4);
10490 114 : TREE_OPERAND (vexpr, 1) = lane0;
10491 114 : if (new_vard != new_var)
10492 16 : vexpr = build_fold_addr_expr_loc (clause_loc, vexpr);
10493 114 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
10494 : }
10495 : }
10496 : }
10497 1324 : if (is_simd && !is_for_simd)
10498 : {
10499 590 : gsi_insert_seq_after (gsi_p, gimple_omp_body (stmt), GSI_SAME_STMT);
10500 590 : gsi_insert_seq_after (gsi_p, before, GSI_SAME_STMT);
10501 590 : gsi_replace (gsi_p, gimple_build_nop (), true);
10502 590 : return;
10503 : }
10504 734 : lower_omp (gimple_omp_body_ptr (stmt), octx);
10505 734 : if (before)
10506 : {
10507 276 : gimple_stmt_iterator gsi = gsi_start (*gimple_omp_body_ptr (stmt));
10508 276 : gsi_insert_seq_before (&gsi, before, GSI_SAME_STMT);
10509 : }
10510 : }
10511 :
10512 :
10513 : /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
10514 : substitution of a couple of function calls. But in the NAMED case,
10515 : requires that languages coordinate a symbol name. It is therefore
10516 : best put here in common code. */
10517 :
10518 : static GTY(()) hash_map<tree, tree> *critical_name_mutexes;
10519 :
10520 : static void
10521 317 : lower_omp_critical (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10522 : {
10523 317 : tree block;
10524 317 : tree name, lock, unlock;
10525 317 : gomp_critical *stmt = as_a <gomp_critical *> (gsi_stmt (*gsi_p));
10526 317 : gbind *bind;
10527 317 : location_t loc = gimple_location (stmt);
10528 317 : gimple_seq tbody;
10529 :
10530 317 : name = gimple_omp_critical_name (stmt);
10531 317 : if (name)
10532 : {
10533 92 : tree decl;
10534 :
10535 92 : if (!critical_name_mutexes)
10536 50 : critical_name_mutexes = hash_map<tree, tree>::create_ggc (10);
10537 :
10538 92 : tree *n = critical_name_mutexes->get (name);
10539 92 : if (n == NULL)
10540 : {
10541 74 : char *new_str;
10542 :
10543 74 : decl = create_tmp_var_raw (ptr_type_node);
10544 :
10545 74 : new_str = ACONCAT ((".gomp_critical_user_",
10546 : IDENTIFIER_POINTER (name), NULL));
10547 74 : DECL_NAME (decl) = get_identifier (new_str);
10548 74 : TREE_PUBLIC (decl) = 1;
10549 74 : TREE_STATIC (decl) = 1;
10550 74 : DECL_COMMON (decl) = 1;
10551 74 : DECL_ARTIFICIAL (decl) = 1;
10552 74 : DECL_IGNORED_P (decl) = 1;
10553 :
10554 74 : varpool_node::finalize_decl (decl);
10555 :
10556 74 : critical_name_mutexes->put (name, decl);
10557 : }
10558 : else
10559 18 : decl = *n;
10560 :
10561 : /* If '#pragma omp critical' is inside offloaded region or
10562 : inside function marked as offloadable, the symbol must be
10563 : marked as offloadable too. */
10564 92 : omp_context *octx;
10565 92 : if (cgraph_node::get (current_function_decl)->offloadable)
10566 1 : varpool_node::get_create (decl)->offloadable = 1;
10567 : else
10568 181 : for (octx = ctx->outer; octx; octx = octx->outer)
10569 94 : if (is_gimple_omp_offloaded (octx->stmt))
10570 : {
10571 4 : varpool_node::get_create (decl)->offloadable = 1;
10572 4 : break;
10573 : }
10574 :
10575 92 : lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START);
10576 92 : lock = build_call_expr_loc (loc, lock, 1,
10577 : build_fold_addr_expr_loc (loc, decl));
10578 :
10579 92 : unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END);
10580 92 : unlock = build_call_expr_loc (loc, unlock, 1,
10581 : build_fold_addr_expr_loc (loc, decl));
10582 : }
10583 : else
10584 : {
10585 225 : lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START);
10586 225 : lock = build_call_expr_loc (loc, lock, 0);
10587 :
10588 225 : unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END);
10589 225 : unlock = build_call_expr_loc (loc, unlock, 0);
10590 : }
10591 :
10592 317 : push_gimplify_context ();
10593 :
10594 317 : block = make_node (BLOCK);
10595 317 : bind = gimple_build_bind (NULL, NULL, block);
10596 317 : gsi_replace (gsi_p, bind, true);
10597 317 : gimple_bind_add_stmt (bind, stmt);
10598 :
10599 317 : tbody = gimple_bind_body (bind);
10600 317 : gimplify_and_add (lock, &tbody);
10601 317 : gimple_bind_set_body (bind, tbody);
10602 :
10603 317 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
10604 317 : gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
10605 317 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10606 317 : gimple_omp_set_body (stmt, NULL);
10607 :
10608 317 : tbody = gimple_bind_body (bind);
10609 317 : gimplify_and_add (unlock, &tbody);
10610 317 : gimple_bind_set_body (bind, tbody);
10611 :
10612 317 : gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10613 :
10614 317 : pop_gimplify_context (bind);
10615 317 : gimple_bind_append_vars (bind, ctx->block_vars);
10616 317 : BLOCK_VARS (block) = gimple_bind_vars (bind);
10617 317 : }
10618 :
10619 : /* A subroutine of lower_omp_for. Generate code to emit the predicate
10620 : for a lastprivate clause. Given a loop control predicate of (V
10621 : cond N2), we gate the clause on (!(V cond N2)). The lowered form
10622 : is appended to *DLIST, iterator initialization is appended to
10623 : *BODY_P. *CLIST is for lastprivate(conditional:) code that needs
10624 : to be emitted in a critical section. */
10625 :
10626 : static void
10627 47102 : lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p,
10628 : gimple_seq *dlist, gimple_seq *clist,
10629 : struct omp_context *ctx)
10630 : {
10631 47102 : tree clauses, cond, vinit;
10632 47102 : enum tree_code cond_code;
10633 47102 : gimple_seq stmts;
10634 :
10635 47102 : cond_code = fd->loop.cond_code;
10636 47102 : cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR;
10637 :
10638 : /* When possible, use a strict equality expression. This can let VRP
10639 : type optimizations deduce the value and remove a copy. */
10640 47102 : if (tree_fits_shwi_p (fd->loop.step))
10641 : {
10642 44863 : HOST_WIDE_INT step = tree_to_shwi (fd->loop.step);
10643 44863 : if (step == 1 || step == -1)
10644 47102 : cond_code = EQ_EXPR;
10645 : }
10646 :
10647 47102 : tree n2 = fd->loop.n2;
10648 47102 : if (fd->collapse > 1
10649 11251 : && TREE_CODE (n2) != INTEGER_CST
10650 53316 : && gimple_omp_for_combined_into_p (fd->for_stmt))
10651 : {
10652 2692 : struct omp_context *taskreg_ctx = NULL;
10653 2692 : if (gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR)
10654 : {
10655 1226 : gomp_for *gfor = as_a <gomp_for *> (ctx->outer->stmt);
10656 1226 : if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_FOR
10657 1226 : || gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_DISTRIBUTE)
10658 : {
10659 1194 : if (gimple_omp_for_combined_into_p (gfor))
10660 : {
10661 672 : gcc_assert (ctx->outer->outer
10662 : && is_parallel_ctx (ctx->outer->outer));
10663 : taskreg_ctx = ctx->outer->outer;
10664 : }
10665 : else
10666 : {
10667 522 : struct omp_for_data outer_fd;
10668 522 : omp_extract_for_data (gfor, &outer_fd, NULL);
10669 522 : n2 = fold_convert (TREE_TYPE (n2), outer_fd.loop.n2);
10670 : }
10671 : }
10672 32 : else if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_TASKLOOP)
10673 32 : taskreg_ctx = ctx->outer->outer;
10674 : }
10675 1466 : else if (is_taskreg_ctx (ctx->outer))
10676 : taskreg_ctx = ctx->outer;
10677 2692 : if (taskreg_ctx)
10678 : {
10679 2170 : int i;
10680 2170 : tree taskreg_clauses
10681 2170 : = gimple_omp_taskreg_clauses (taskreg_ctx->stmt);
10682 2170 : tree innerc = omp_find_clause (taskreg_clauses,
10683 : OMP_CLAUSE__LOOPTEMP_);
10684 2170 : gcc_assert (innerc);
10685 2170 : int count = fd->collapse;
10686 2170 : if (fd->non_rect
10687 24 : && fd->last_nonrect == fd->first_nonrect + 1)
10688 12 : if (tree v = gimple_omp_for_index (fd->for_stmt, fd->last_nonrect))
10689 12 : if (!TYPE_UNSIGNED (TREE_TYPE (v)))
10690 12 : count += 4;
10691 8582 : for (i = 0; i < count; i++)
10692 : {
10693 6412 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
10694 : OMP_CLAUSE__LOOPTEMP_);
10695 6412 : gcc_assert (innerc);
10696 : }
10697 2170 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
10698 : OMP_CLAUSE__LOOPTEMP_);
10699 2170 : if (innerc)
10700 1419 : n2 = fold_convert (TREE_TYPE (n2),
10701 : lookup_decl (OMP_CLAUSE_DECL (innerc),
10702 : taskreg_ctx));
10703 : }
10704 : }
10705 47102 : cond = build2 (cond_code, boolean_type_node, fd->loop.v, n2);
10706 :
10707 47102 : clauses = gimple_omp_for_clauses (fd->for_stmt);
10708 47102 : stmts = NULL;
10709 47102 : lower_lastprivate_clauses (clauses, cond, body_p, &stmts, clist, ctx);
10710 47102 : if (!gimple_seq_empty_p (stmts))
10711 : {
10712 16376 : gimple_seq_add_seq (&stmts, *dlist);
10713 16376 : *dlist = stmts;
10714 :
10715 : /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
10716 16376 : vinit = fd->loop.n1;
10717 16376 : if (cond_code == EQ_EXPR
10718 14389 : && tree_fits_shwi_p (fd->loop.n2)
10719 25951 : && ! integer_zerop (fd->loop.n2))
10720 8359 : vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0);
10721 : else
10722 8017 : vinit = unshare_expr (vinit);
10723 :
10724 : /* Initialize the iterator variable, so that threads that don't execute
10725 : any iterations don't execute the lastprivate clauses by accident. */
10726 16376 : gimplify_assign (fd->loop.v, vinit, body_p);
10727 : }
10728 47102 : }
10729 :
10730 : /* OpenACC privatization.
10731 :
10732 : Or, in other words, *sharing* at the respective OpenACC level of
10733 : parallelism.
10734 :
10735 : From a correctness perspective, a non-addressable variable can't be accessed
10736 : outside the current thread, so it can go in a (faster than shared memory)
10737 : register -- though that register may need to be broadcast in some
10738 : circumstances. A variable can only meaningfully be "shared" across workers
10739 : or vector lanes if its address is taken, e.g. by a call to an atomic
10740 : builtin.
10741 :
10742 : From an optimisation perspective, the answer might be fuzzier: maybe
10743 : sometimes, using shared memory directly would be faster than
10744 : broadcasting. */
10745 :
10746 : static void
10747 18997 : oacc_privatization_begin_diagnose_var (const dump_flags_t l_dump_flags,
10748 : const location_t loc, const tree c,
10749 : const tree decl)
10750 : {
10751 18997 : const dump_user_location_t d_u_loc
10752 18997 : = dump_user_location_t::from_location_t (loc);
10753 : /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */
10754 : #if __GNUC__ >= 10
10755 18997 : # pragma GCC diagnostic push
10756 18997 : # pragma GCC diagnostic ignored "-Wformat"
10757 : #endif
10758 18997 : dump_printf_loc (l_dump_flags, d_u_loc,
10759 : "variable %<%T%> ", decl);
10760 : #if __GNUC__ >= 10
10761 18997 : # pragma GCC diagnostic pop
10762 : #endif
10763 18997 : if (c)
10764 2797 : dump_printf (l_dump_flags,
10765 : "in %qs clause ",
10766 2797 : omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
10767 : else
10768 16200 : dump_printf (l_dump_flags,
10769 : "declared in block ");
10770 18997 : }
10771 :
10772 : static bool
10773 41210 : oacc_privatization_candidate_p (const location_t loc, const tree c,
10774 : const tree decl)
10775 : {
10776 41210 : dump_flags_t l_dump_flags = get_openacc_privatization_dump_flags ();
10777 :
10778 : /* There is some differentiation depending on block vs. clause. */
10779 41210 : bool block = !c;
10780 :
10781 41210 : bool res = true;
10782 :
10783 41210 : if (res && !VAR_P (decl))
10784 : {
10785 : /* A PARM_DECL (appearing in a 'private' clause) is expected to have been
10786 : privatized into a new VAR_DECL. */
10787 712 : gcc_checking_assert (TREE_CODE (decl) != PARM_DECL);
10788 :
10789 712 : res = false;
10790 :
10791 712 : if (dump_enabled_p ())
10792 : {
10793 683 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10794 683 : dump_printf (l_dump_flags,
10795 : "potentially has improper OpenACC privatization level: %qs\n",
10796 683 : get_tree_code_name (TREE_CODE (decl)));
10797 : }
10798 : }
10799 :
10800 41181 : if (res && block && TREE_STATIC (decl))
10801 : {
10802 53 : res = false;
10803 :
10804 53 : if (dump_enabled_p ())
10805 : {
10806 28 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10807 28 : dump_printf (l_dump_flags,
10808 : "isn%'t candidate for adjusting OpenACC privatization level: %s\n",
10809 : "static");
10810 : }
10811 : }
10812 :
10813 41185 : if (res && block && DECL_EXTERNAL (decl))
10814 : {
10815 12 : res = false;
10816 :
10817 12 : if (dump_enabled_p ())
10818 : {
10819 12 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10820 12 : dump_printf (l_dump_flags,
10821 : "isn%'t candidate for adjusting OpenACC privatization level: %s\n",
10822 : "external");
10823 : }
10824 : }
10825 :
10826 41210 : if (res && !TREE_ADDRESSABLE (decl))
10827 : {
10828 39341 : res = false;
10829 :
10830 39341 : if (dump_enabled_p ())
10831 : {
10832 17456 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10833 17456 : dump_printf (l_dump_flags,
10834 : "isn%'t candidate for adjusting OpenACC privatization level: %s\n",
10835 : "not addressable");
10836 : }
10837 : }
10838 :
10839 : /* If an artificial variable has been added to a bind, e.g.
10840 : a compiler-generated temporary structure used by the Fortran front-end, do
10841 : not consider it as a privatization candidate. Note that variables on
10842 : the stack are private per-thread by default: making them "gang-private"
10843 : for OpenACC actually means to share a single instance of a variable
10844 : amongst all workers and threads spawned within each gang.
10845 : At present, no compiler-generated artificial variables require such
10846 : sharing semantics, so this is safe. */
10847 :
10848 18548 : if (res && block && DECL_ARTIFICIAL (decl))
10849 : {
10850 548 : res = false;
10851 :
10852 548 : if (dump_enabled_p ())
10853 : {
10854 360 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10855 360 : dump_printf (l_dump_flags,
10856 : "isn%'t candidate for adjusting OpenACC privatization "
10857 : "level: %s\n", "artificial");
10858 : }
10859 : }
10860 :
10861 41022 : if (res)
10862 : {
10863 544 : if (dump_enabled_p ())
10864 : {
10865 458 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10866 458 : dump_printf (l_dump_flags,
10867 : "is candidate for adjusting OpenACC privatization level\n");
10868 : }
10869 : }
10870 :
10871 41210 : if (dump_file && (dump_flags & TDF_DETAILS))
10872 : {
10873 0 : print_generic_decl (dump_file, decl, dump_flags);
10874 0 : fprintf (dump_file, "\n");
10875 : }
10876 :
10877 41210 : return res;
10878 : }
10879 :
10880 : /* Scan CLAUSES for candidates for adjusting OpenACC privatization level in
10881 : CTX. */
10882 :
10883 : static void
10884 11325 : oacc_privatization_scan_clause_chain (omp_context *ctx, tree clauses)
10885 : {
10886 35223 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
10887 23898 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE)
10888 : {
10889 12986 : tree decl = OMP_CLAUSE_DECL (c);
10890 :
10891 12986 : tree new_decl = lookup_decl (decl, ctx);
10892 :
10893 12986 : if (!oacc_privatization_candidate_p (OMP_CLAUSE_LOCATION (c), c,
10894 : new_decl))
10895 12654 : continue;
10896 :
10897 332 : gcc_checking_assert
10898 : (!ctx->oacc_privatization_candidates.contains (new_decl));
10899 332 : ctx->oacc_privatization_candidates.safe_push (new_decl);
10900 : }
10901 11325 : }
10902 :
10903 : /* Scan DECLS for candidates for adjusting OpenACC privatization level in
10904 : CTX. */
10905 :
10906 : static void
10907 23090 : oacc_privatization_scan_decl_chain (omp_context *ctx, tree decls)
10908 : {
10909 51314 : for (tree decl = decls; decl; decl = DECL_CHAIN (decl))
10910 : {
10911 28224 : tree new_decl = lookup_decl (decl, ctx);
10912 28224 : gcc_checking_assert (new_decl == decl);
10913 :
10914 28224 : if (!oacc_privatization_candidate_p (gimple_location (ctx->stmt), NULL,
10915 : new_decl))
10916 28012 : continue;
10917 :
10918 212 : gcc_checking_assert
10919 : (!ctx->oacc_privatization_candidates.contains (new_decl));
10920 212 : ctx->oacc_privatization_candidates.safe_push (new_decl);
10921 : }
10922 23090 : }
10923 :
10924 : /* Callback for walk_gimple_seq. Find #pragma omp scan statement. */
10925 :
10926 : static tree
10927 2341 : omp_find_scan (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
10928 : struct walk_stmt_info *wi)
10929 : {
10930 2341 : gimple *stmt = gsi_stmt (*gsi_p);
10931 :
10932 2341 : *handled_ops_p = true;
10933 2341 : switch (gimple_code (stmt))
10934 : {
10935 462 : WALK_SUBSTMTS;
10936 :
10937 182 : case GIMPLE_OMP_FOR:
10938 182 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_SIMD
10939 182 : && gimple_omp_for_combined_into_p (stmt))
10940 182 : *handled_ops_p = false;
10941 : break;
10942 :
10943 734 : case GIMPLE_OMP_SCAN:
10944 734 : *(gimple_stmt_iterator *) (wi->info) = *gsi_p;
10945 734 : return integer_zero_node;
10946 : default:
10947 : break;
10948 : }
10949 : return NULL;
10950 : }
10951 :
10952 : /* Helper function for lower_omp_for, add transformations for a worksharing
10953 : loop with scan directives inside of it.
10954 : For worksharing loop not combined with simd, transform:
10955 : #pragma omp for reduction(inscan,+:r) private(i)
10956 : for (i = 0; i < n; i = i + 1)
10957 : {
10958 : {
10959 : update (r);
10960 : }
10961 : #pragma omp scan inclusive(r)
10962 : {
10963 : use (r);
10964 : }
10965 : }
10966 :
10967 : into two worksharing loops + code to merge results:
10968 :
10969 : num_threads = omp_get_num_threads ();
10970 : thread_num = omp_get_thread_num ();
10971 : if (thread_num == 0) goto <D.2099>; else goto <D.2100>;
10972 : <D.2099>:
10973 : var2 = r;
10974 : goto <D.2101>;
10975 : <D.2100>:
10976 : // For UDRs this is UDR init, or if ctors are needed, copy from
10977 : // var3 that has been constructed to contain the neutral element.
10978 : var2 = 0;
10979 : <D.2101>:
10980 : ivar = 0;
10981 : // The _scantemp_ clauses will arrange for rpriva to be initialized to
10982 : // a shared array with num_threads elements and rprivb to a local array
10983 : // number of elements equal to the number of (contiguous) iterations the
10984 : // current thread will perform. controlb and controlp variables are
10985 : // temporaries to handle deallocation of rprivb at the end of second
10986 : // GOMP_FOR.
10987 : #pragma omp for _scantemp_(rpriva) _scantemp_(rprivb) _scantemp_(controlb) \
10988 : _scantemp_(controlp) reduction(inscan,+:r) private(i) nowait
10989 : for (i = 0; i < n; i = i + 1)
10990 : {
10991 : {
10992 : // For UDRs this is UDR init or copy from var3.
10993 : r = 0;
10994 : // This is the input phase from user code.
10995 : update (r);
10996 : }
10997 : {
10998 : // For UDRs this is UDR merge.
10999 : var2 = var2 + r;
11000 : // Rather than handing it over to the user, save to local thread's
11001 : // array.
11002 : rprivb[ivar] = var2;
11003 : // For exclusive scan, the above two statements are swapped.
11004 : ivar = ivar + 1;
11005 : }
11006 : }
11007 : // And remember the final value from this thread's into the shared
11008 : // rpriva array.
11009 : rpriva[(sizetype) thread_num] = var2;
11010 : // If more than one thread, compute using Work-Efficient prefix sum
11011 : // the inclusive parallel scan of the rpriva array.
11012 : if (num_threads > 1) goto <D.2102>; else goto <D.2103>;
11013 : <D.2102>:
11014 : GOMP_barrier ();
11015 : down = 0;
11016 : k = 1;
11017 : num_threadsu = (unsigned int) num_threads;
11018 : thread_numup1 = (unsigned int) thread_num + 1;
11019 : <D.2108>:
11020 : twok = k << 1;
11021 : if (twok > num_threadsu) goto <D.2110>; else goto <D.2111>;
11022 : <D.2110>:
11023 : down = 4294967295;
11024 : k = k >> 1;
11025 : if (k == num_threadsu) goto <D.2112>; else goto <D.2111>;
11026 : <D.2112>:
11027 : k = k >> 1;
11028 : <D.2111>:
11029 : twok = k << 1;
11030 : cplx = .MUL_OVERFLOW (thread_nump1, twok);
11031 : mul = REALPART_EXPR <cplx>;
11032 : ovf = IMAGPART_EXPR <cplx>;
11033 : if (ovf == 0) goto <D.2116>; else goto <D.2117>;
11034 : <D.2116>:
11035 : andv = k & down;
11036 : andvm1 = andv + 4294967295;
11037 : l = mul + andvm1;
11038 : if (l < num_threadsu) goto <D.2120>; else goto <D.2117>;
11039 : <D.2120>:
11040 : // For UDRs this is UDR merge, performed using var2 variable as temporary,
11041 : // i.e. var2 = rpriva[l - k]; UDR merge (var2, rpriva[l]); rpriva[l] = var2;
11042 : rpriva[l] = rpriva[l - k] + rpriva[l];
11043 : <D.2117>:
11044 : if (down == 0) goto <D.2121>; else goto <D.2122>;
11045 : <D.2121>:
11046 : k = k << 1;
11047 : goto <D.2123>;
11048 : <D.2122>:
11049 : k = k >> 1;
11050 : <D.2123>:
11051 : GOMP_barrier ();
11052 : if (k != 0) goto <D.2108>; else goto <D.2103>;
11053 : <D.2103>:
11054 : if (thread_num == 0) goto <D.2124>; else goto <D.2125>;
11055 : <D.2124>:
11056 : // For UDRs this is UDR init or copy from var3.
11057 : var2 = 0;
11058 : goto <D.2126>;
11059 : <D.2125>:
11060 : var2 = rpriva[thread_num - 1];
11061 : <D.2126>:
11062 : ivar = 0;
11063 : #pragma omp for _scantemp_(controlb) _scantemp_(controlp) \
11064 : reduction(inscan,+:r) private(i)
11065 : for (i = 0; i < n; i = i + 1)
11066 : {
11067 : {
11068 : // For UDRs, this is r = var2; UDR merge (r, rprivb[ivar]);
11069 : r = var2 + rprivb[ivar];
11070 : }
11071 : {
11072 : // This is the scan phase from user code.
11073 : use (r);
11074 : // Plus a bump of the iterator.
11075 : ivar = ivar + 1;
11076 : }
11077 : } */
11078 :
11079 : static void
11080 185 : lower_omp_for_scan (gimple_seq *body_p, gimple_seq *dlist, gomp_for *stmt,
11081 : struct omp_for_data *fd, omp_context *ctx)
11082 : {
11083 185 : bool is_for_simd = gimple_omp_for_combined_p (stmt);
11084 185 : gcc_assert (ctx->scan_inclusive || ctx->scan_exclusive);
11085 :
11086 185 : gimple_seq body = gimple_omp_body (stmt);
11087 185 : gimple_stmt_iterator input1_gsi = gsi_none ();
11088 185 : struct walk_stmt_info wi;
11089 185 : memset (&wi, 0, sizeof (wi));
11090 185 : wi.val_only = true;
11091 185 : wi.info = (void *) &input1_gsi;
11092 185 : walk_gimple_seq_mod (&body, omp_find_scan, NULL, &wi);
11093 185 : gcc_assert (!gsi_end_p (input1_gsi));
11094 :
11095 185 : gimple *input_stmt1 = gsi_stmt (input1_gsi);
11096 185 : gimple_stmt_iterator gsi = input1_gsi;
11097 185 : gsi_next (&gsi);
11098 185 : gimple_stmt_iterator scan1_gsi = gsi;
11099 185 : gimple *scan_stmt1 = gsi_stmt (gsi);
11100 185 : gcc_assert (scan_stmt1 && gimple_code (scan_stmt1) == GIMPLE_OMP_SCAN);
11101 :
11102 185 : gimple_seq input_body = gimple_omp_body (input_stmt1);
11103 185 : gimple_seq scan_body = gimple_omp_body (scan_stmt1);
11104 185 : gimple_omp_set_body (input_stmt1, NULL);
11105 185 : gimple_omp_set_body (scan_stmt1, NULL);
11106 185 : gimple_omp_set_body (stmt, NULL);
11107 :
11108 185 : gomp_for *new_stmt = as_a <gomp_for *> (gimple_copy (stmt));
11109 185 : gimple_seq new_body = copy_gimple_seq_and_replace_locals (body);
11110 185 : gimple_omp_set_body (stmt, body);
11111 185 : gimple_omp_set_body (input_stmt1, input_body);
11112 :
11113 185 : gimple_stmt_iterator input2_gsi = gsi_none ();
11114 185 : memset (&wi, 0, sizeof (wi));
11115 185 : wi.val_only = true;
11116 185 : wi.info = (void *) &input2_gsi;
11117 185 : walk_gimple_seq_mod (&new_body, omp_find_scan, NULL, &wi);
11118 185 : gcc_assert (!gsi_end_p (input2_gsi));
11119 :
11120 185 : gimple *input_stmt2 = gsi_stmt (input2_gsi);
11121 185 : gsi = input2_gsi;
11122 185 : gsi_next (&gsi);
11123 185 : gimple_stmt_iterator scan2_gsi = gsi;
11124 185 : gimple *scan_stmt2 = gsi_stmt (gsi);
11125 185 : gcc_assert (scan_stmt2 && gimple_code (scan_stmt2) == GIMPLE_OMP_SCAN);
11126 185 : gimple_omp_set_body (scan_stmt2, scan_body);
11127 :
11128 185 : gimple_stmt_iterator input3_gsi = gsi_none ();
11129 185 : gimple_stmt_iterator scan3_gsi = gsi_none ();
11130 185 : gimple_stmt_iterator input4_gsi = gsi_none ();
11131 185 : gimple_stmt_iterator scan4_gsi = gsi_none ();
11132 185 : gimple *input_stmt3 = NULL, *scan_stmt3 = NULL;
11133 185 : gimple *input_stmt4 = NULL, *scan_stmt4 = NULL;
11134 185 : omp_context *input_simd_ctx = NULL, *scan_simd_ctx = NULL;
11135 185 : if (is_for_simd)
11136 : {
11137 91 : memset (&wi, 0, sizeof (wi));
11138 91 : wi.val_only = true;
11139 91 : wi.info = (void *) &input3_gsi;
11140 91 : walk_gimple_seq_mod (&input_body, omp_find_scan, NULL, &wi);
11141 91 : gcc_assert (!gsi_end_p (input3_gsi));
11142 :
11143 91 : input_stmt3 = gsi_stmt (input3_gsi);
11144 91 : gsi = input3_gsi;
11145 91 : gsi_next (&gsi);
11146 91 : scan3_gsi = gsi;
11147 91 : scan_stmt3 = gsi_stmt (gsi);
11148 91 : gcc_assert (scan_stmt3 && gimple_code (scan_stmt3) == GIMPLE_OMP_SCAN);
11149 :
11150 91 : memset (&wi, 0, sizeof (wi));
11151 91 : wi.val_only = true;
11152 91 : wi.info = (void *) &input4_gsi;
11153 91 : walk_gimple_seq_mod (&scan_body, omp_find_scan, NULL, &wi);
11154 91 : gcc_assert (!gsi_end_p (input4_gsi));
11155 :
11156 91 : input_stmt4 = gsi_stmt (input4_gsi);
11157 91 : gsi = input4_gsi;
11158 91 : gsi_next (&gsi);
11159 91 : scan4_gsi = gsi;
11160 91 : scan_stmt4 = gsi_stmt (gsi);
11161 91 : gcc_assert (scan_stmt4 && gimple_code (scan_stmt4) == GIMPLE_OMP_SCAN);
11162 :
11163 91 : input_simd_ctx = maybe_lookup_ctx (input_stmt3)->outer;
11164 91 : scan_simd_ctx = maybe_lookup_ctx (input_stmt4)->outer;
11165 : }
11166 :
11167 185 : tree num_threads = create_tmp_var (integer_type_node);
11168 185 : tree thread_num = create_tmp_var (integer_type_node);
11169 185 : tree nthreads_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
11170 185 : tree threadnum_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
11171 185 : gimple *g = gimple_build_call (nthreads_decl, 0);
11172 185 : gimple_call_set_lhs (g, num_threads);
11173 185 : gimple_seq_add_stmt (body_p, g);
11174 185 : g = gimple_build_call (threadnum_decl, 0);
11175 185 : gimple_call_set_lhs (g, thread_num);
11176 185 : gimple_seq_add_stmt (body_p, g);
11177 :
11178 185 : tree ivar = create_tmp_var (sizetype);
11179 185 : tree new_clauses1 = NULL_TREE, new_clauses2 = NULL_TREE;
11180 185 : tree *cp1 = &new_clauses1, *cp2 = &new_clauses2;
11181 185 : tree k = create_tmp_var (unsigned_type_node);
11182 185 : tree l = create_tmp_var (unsigned_type_node);
11183 :
11184 185 : gimple_seq clist = NULL, mdlist = NULL;
11185 185 : gimple_seq thr01_list = NULL, thrn1_list = NULL;
11186 185 : gimple_seq thr02_list = NULL, thrn2_list = NULL;
11187 185 : gimple_seq scan1_list = NULL, input2_list = NULL;
11188 185 : gimple_seq last_list = NULL, reduc_list = NULL;
11189 718 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
11190 533 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
11191 533 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
11192 : {
11193 217 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
11194 217 : tree var = OMP_CLAUSE_DECL (c);
11195 217 : tree new_var = lookup_decl (var, ctx);
11196 217 : tree var3 = NULL_TREE;
11197 217 : tree new_vard = new_var;
11198 217 : if (omp_privatize_by_reference (var))
11199 32 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
11200 217 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
11201 : {
11202 64 : var3 = maybe_lookup_decl (new_vard, ctx);
11203 64 : if (var3 == new_vard)
11204 185 : var3 = NULL_TREE;
11205 : }
11206 :
11207 217 : tree ptype = build_pointer_type (TREE_TYPE (new_var));
11208 217 : tree rpriva = create_tmp_var (ptype);
11209 217 : tree nc = build_omp_clause (clause_loc, OMP_CLAUSE__SCANTEMP_);
11210 217 : OMP_CLAUSE_DECL (nc) = rpriva;
11211 217 : *cp1 = nc;
11212 217 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11213 :
11214 217 : tree rprivb = create_tmp_var (ptype);
11215 217 : nc = build_omp_clause (clause_loc, OMP_CLAUSE__SCANTEMP_);
11216 217 : OMP_CLAUSE_DECL (nc) = rprivb;
11217 217 : OMP_CLAUSE__SCANTEMP__ALLOC (nc) = 1;
11218 217 : *cp1 = nc;
11219 217 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11220 :
11221 217 : tree var2 = create_tmp_var_raw (TREE_TYPE (new_var));
11222 217 : if (new_vard != new_var)
11223 32 : TREE_ADDRESSABLE (var2) = 1;
11224 217 : gimple_add_tmp_var (var2);
11225 :
11226 217 : tree x = fold_convert_loc (clause_loc, sizetype, thread_num);
11227 217 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11228 217 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11229 217 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11230 217 : tree rpriva_ref = build_simple_mem_ref_loc (clause_loc, x);
11231 :
11232 217 : x = fold_build2_loc (clause_loc, PLUS_EXPR, integer_type_node,
11233 : thread_num, integer_minus_one_node);
11234 217 : x = fold_convert_loc (clause_loc, sizetype, x);
11235 217 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11236 217 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11237 217 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11238 217 : tree rprivam1_ref = build_simple_mem_ref_loc (clause_loc, x);
11239 :
11240 217 : x = fold_convert_loc (clause_loc, sizetype, l);
11241 217 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11242 217 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11243 217 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11244 217 : tree rprival_ref = build_simple_mem_ref_loc (clause_loc, x);
11245 :
11246 217 : x = fold_build2_loc (clause_loc, MINUS_EXPR, unsigned_type_node, l, k);
11247 217 : x = fold_convert_loc (clause_loc, sizetype, x);
11248 217 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11249 217 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11250 217 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11251 217 : tree rprivalmk_ref = build_simple_mem_ref_loc (clause_loc, x);
11252 :
11253 217 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, ivar,
11254 217 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11255 217 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rprivb), rprivb, x);
11256 217 : tree rprivb_ref = build_simple_mem_ref_loc (clause_loc, x);
11257 :
11258 217 : tree var4 = is_for_simd ? new_var : var2;
11259 107 : tree var5 = NULL_TREE, var6 = NULL_TREE;
11260 107 : if (is_for_simd)
11261 : {
11262 107 : var5 = lookup_decl (var, input_simd_ctx);
11263 107 : var6 = lookup_decl (var, scan_simd_ctx);
11264 107 : if (new_vard != new_var)
11265 : {
11266 16 : var5 = build_simple_mem_ref_loc (clause_loc, var5);
11267 16 : var6 = build_simple_mem_ref_loc (clause_loc, var6);
11268 : }
11269 : }
11270 217 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
11271 : {
11272 64 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
11273 64 : tree val = var2;
11274 :
11275 64 : x = lang_hooks.decls.omp_clause_default_ctor
11276 64 : (c, var2, build_outer_var_ref (var, ctx));
11277 64 : if (x)
11278 32 : gimplify_and_add (x, &clist);
11279 :
11280 64 : x = build_outer_var_ref (var, ctx);
11281 64 : x = lang_hooks.decls.omp_clause_assign_op (c, unshare_expr (var4),
11282 : x);
11283 64 : gimplify_and_add (x, &thr01_list);
11284 :
11285 64 : tree y = (DECL_HAS_VALUE_EXPR_P (new_vard)
11286 64 : ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
11287 64 : if (var3)
11288 : {
11289 32 : x = unshare_expr (var4);
11290 32 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var3);
11291 32 : gimplify_and_add (x, &thrn1_list);
11292 32 : x = unshare_expr (var4);
11293 32 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var3);
11294 32 : gimplify_and_add (x, &thr02_list);
11295 : }
11296 32 : else if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
11297 : {
11298 : /* Otherwise, assign to it the identity element. */
11299 32 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
11300 32 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11301 32 : if (!is_for_simd)
11302 : {
11303 16 : if (new_vard != new_var)
11304 4 : val = build_fold_addr_expr_loc (clause_loc, val);
11305 16 : SET_DECL_VALUE_EXPR (new_vard, val);
11306 16 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
11307 : }
11308 32 : SET_DECL_VALUE_EXPR (placeholder, error_mark_node);
11309 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11310 32 : lower_omp (&tseq, ctx);
11311 32 : gimple_seq_add_seq (&thrn1_list, tseq);
11312 32 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
11313 32 : lower_omp (&tseq, ctx);
11314 32 : gimple_seq_add_seq (&thr02_list, tseq);
11315 32 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
11316 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
11317 32 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
11318 32 : if (y)
11319 0 : SET_DECL_VALUE_EXPR (new_vard, y);
11320 : else
11321 : {
11322 32 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
11323 32 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
11324 : }
11325 : }
11326 :
11327 64 : x = unshare_expr (var4);
11328 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x, rprivam1_ref);
11329 64 : gimplify_and_add (x, &thrn2_list);
11330 :
11331 64 : if (is_for_simd)
11332 : {
11333 32 : x = unshare_expr (rprivb_ref);
11334 32 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var5);
11335 32 : gimplify_and_add (x, &scan1_list);
11336 : }
11337 : else
11338 : {
11339 32 : if (ctx->scan_exclusive)
11340 : {
11341 16 : x = unshare_expr (rprivb_ref);
11342 16 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var2);
11343 16 : gimplify_and_add (x, &scan1_list);
11344 : }
11345 :
11346 32 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
11347 32 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11348 32 : SET_DECL_VALUE_EXPR (placeholder, var2);
11349 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11350 32 : lower_omp (&tseq, ctx);
11351 32 : gimple_seq_add_seq (&scan1_list, tseq);
11352 :
11353 32 : if (ctx->scan_inclusive)
11354 : {
11355 16 : x = unshare_expr (rprivb_ref);
11356 16 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var2);
11357 16 : gimplify_and_add (x, &scan1_list);
11358 : }
11359 : }
11360 :
11361 64 : x = unshare_expr (rpriva_ref);
11362 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x,
11363 : unshare_expr (var4));
11364 64 : gimplify_and_add (x, &mdlist);
11365 :
11366 96 : x = unshare_expr (is_for_simd ? var6 : new_var);
11367 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var4);
11368 64 : gimplify_and_add (x, &input2_list);
11369 :
11370 64 : val = rprivb_ref;
11371 64 : if (new_vard != new_var)
11372 24 : val = build_fold_addr_expr_loc (clause_loc, val);
11373 :
11374 64 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
11375 64 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11376 64 : SET_DECL_VALUE_EXPR (new_vard, val);
11377 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
11378 64 : if (is_for_simd)
11379 : {
11380 32 : SET_DECL_VALUE_EXPR (placeholder, var6);
11381 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11382 : }
11383 : else
11384 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
11385 64 : lower_omp (&tseq, ctx);
11386 64 : if (y)
11387 0 : SET_DECL_VALUE_EXPR (new_vard, y);
11388 : else
11389 : {
11390 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
11391 64 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
11392 : }
11393 64 : if (!is_for_simd)
11394 : {
11395 32 : SET_DECL_VALUE_EXPR (placeholder, new_var);
11396 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11397 32 : lower_omp (&tseq, ctx);
11398 : }
11399 64 : gimple_seq_add_seq (&input2_list, tseq);
11400 :
11401 64 : x = build_outer_var_ref (var, ctx);
11402 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x, rpriva_ref);
11403 64 : gimplify_and_add (x, &last_list);
11404 :
11405 64 : x = lang_hooks.decls.omp_clause_assign_op (c, var2, rprivalmk_ref);
11406 64 : gimplify_and_add (x, &reduc_list);
11407 64 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
11408 64 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11409 64 : val = rprival_ref;
11410 64 : if (new_vard != new_var)
11411 24 : val = build_fold_addr_expr_loc (clause_loc, val);
11412 64 : SET_DECL_VALUE_EXPR (new_vard, val);
11413 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
11414 64 : SET_DECL_VALUE_EXPR (placeholder, var2);
11415 64 : lower_omp (&tseq, ctx);
11416 64 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
11417 64 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
11418 64 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
11419 64 : if (y)
11420 0 : SET_DECL_VALUE_EXPR (new_vard, y);
11421 : else
11422 : {
11423 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
11424 64 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
11425 : }
11426 64 : gimple_seq_add_seq (&reduc_list, tseq);
11427 64 : x = lang_hooks.decls.omp_clause_assign_op (c, rprival_ref, var2);
11428 64 : gimplify_and_add (x, &reduc_list);
11429 :
11430 64 : x = lang_hooks.decls.omp_clause_dtor (c, var2);
11431 64 : if (x)
11432 32 : gimplify_and_add (x, dlist);
11433 : }
11434 : else
11435 : {
11436 153 : x = build_outer_var_ref (var, ctx);
11437 153 : gimplify_assign (unshare_expr (var4), x, &thr01_list);
11438 :
11439 153 : x = omp_reduction_init (c, TREE_TYPE (new_var));
11440 153 : gimplify_assign (unshare_expr (var4), unshare_expr (x),
11441 : &thrn1_list);
11442 153 : gimplify_assign (unshare_expr (var4), x, &thr02_list);
11443 :
11444 153 : gimplify_assign (unshare_expr (var4), rprivam1_ref, &thrn2_list);
11445 :
11446 153 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
11447 153 : if (code == MINUS_EXPR)
11448 0 : code = PLUS_EXPR;
11449 :
11450 153 : if (is_for_simd)
11451 75 : gimplify_assign (unshare_expr (rprivb_ref), var5, &scan1_list);
11452 : else
11453 : {
11454 78 : if (ctx->scan_exclusive)
11455 28 : gimplify_assign (unshare_expr (rprivb_ref), var2,
11456 : &scan1_list);
11457 78 : x = build2 (code, TREE_TYPE (new_var), var2, new_var);
11458 78 : gimplify_assign (var2, x, &scan1_list);
11459 78 : if (ctx->scan_inclusive)
11460 50 : gimplify_assign (unshare_expr (rprivb_ref), var2,
11461 : &scan1_list);
11462 : }
11463 :
11464 153 : gimplify_assign (unshare_expr (rpriva_ref), unshare_expr (var4),
11465 : &mdlist);
11466 :
11467 153 : x = build2 (code, TREE_TYPE (new_var), var4, rprivb_ref);
11468 231 : gimplify_assign (is_for_simd ? var6 : new_var, x, &input2_list);
11469 :
11470 153 : gimplify_assign (build_outer_var_ref (var, ctx), rpriva_ref,
11471 : &last_list);
11472 :
11473 153 : x = build2 (code, TREE_TYPE (new_var), rprivalmk_ref,
11474 : unshare_expr (rprival_ref));
11475 153 : gimplify_assign (rprival_ref, x, &reduc_list);
11476 : }
11477 : }
11478 :
11479 185 : g = gimple_build_assign (ivar, PLUS_EXPR, ivar, size_one_node);
11480 185 : gimple_seq_add_stmt (&scan1_list, g);
11481 185 : g = gimple_build_assign (ivar, PLUS_EXPR, ivar, size_one_node);
11482 279 : gimple_seq_add_stmt (gimple_omp_body_ptr (is_for_simd
11483 : ? scan_stmt4 : scan_stmt2), g);
11484 :
11485 185 : tree controlb = create_tmp_var (boolean_type_node);
11486 185 : tree controlp = create_tmp_var (ptr_type_node);
11487 185 : tree nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11488 185 : OMP_CLAUSE_DECL (nc) = controlb;
11489 185 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11490 185 : *cp1 = nc;
11491 185 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11492 185 : nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11493 185 : OMP_CLAUSE_DECL (nc) = controlp;
11494 185 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11495 185 : *cp1 = nc;
11496 185 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11497 185 : nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11498 185 : OMP_CLAUSE_DECL (nc) = controlb;
11499 185 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11500 185 : *cp2 = nc;
11501 185 : cp2 = &OMP_CLAUSE_CHAIN (nc);
11502 185 : nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11503 185 : OMP_CLAUSE_DECL (nc) = controlp;
11504 185 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11505 185 : *cp2 = nc;
11506 185 : cp2 = &OMP_CLAUSE_CHAIN (nc);
11507 :
11508 185 : *cp1 = gimple_omp_for_clauses (stmt);
11509 185 : gimple_omp_for_set_clauses (stmt, new_clauses1);
11510 185 : *cp2 = gimple_omp_for_clauses (new_stmt);
11511 185 : gimple_omp_for_set_clauses (new_stmt, new_clauses2);
11512 :
11513 185 : if (is_for_simd)
11514 : {
11515 91 : gimple_seq_add_seq (gimple_omp_body_ptr (scan_stmt3), scan1_list);
11516 91 : gimple_seq_add_seq (gimple_omp_body_ptr (input_stmt4), input2_list);
11517 :
11518 91 : gsi_insert_seq_after (&input3_gsi, gimple_omp_body (input_stmt3),
11519 : GSI_SAME_STMT);
11520 91 : gsi_remove (&input3_gsi, true);
11521 91 : gsi_insert_seq_after (&scan3_gsi, gimple_omp_body (scan_stmt3),
11522 : GSI_SAME_STMT);
11523 91 : gsi_remove (&scan3_gsi, true);
11524 91 : gsi_insert_seq_after (&input4_gsi, gimple_omp_body (input_stmt4),
11525 : GSI_SAME_STMT);
11526 91 : gsi_remove (&input4_gsi, true);
11527 91 : gsi_insert_seq_after (&scan4_gsi, gimple_omp_body (scan_stmt4),
11528 : GSI_SAME_STMT);
11529 91 : gsi_remove (&scan4_gsi, true);
11530 : }
11531 : else
11532 : {
11533 94 : gimple_omp_set_body (scan_stmt1, scan1_list);
11534 94 : gimple_omp_set_body (input_stmt2, input2_list);
11535 : }
11536 :
11537 185 : gsi_insert_seq_after (&input1_gsi, gimple_omp_body (input_stmt1),
11538 : GSI_SAME_STMT);
11539 185 : gsi_remove (&input1_gsi, true);
11540 185 : gsi_insert_seq_after (&scan1_gsi, gimple_omp_body (scan_stmt1),
11541 : GSI_SAME_STMT);
11542 185 : gsi_remove (&scan1_gsi, true);
11543 185 : gsi_insert_seq_after (&input2_gsi, gimple_omp_body (input_stmt2),
11544 : GSI_SAME_STMT);
11545 185 : gsi_remove (&input2_gsi, true);
11546 185 : gsi_insert_seq_after (&scan2_gsi, gimple_omp_body (scan_stmt2),
11547 : GSI_SAME_STMT);
11548 185 : gsi_remove (&scan2_gsi, true);
11549 :
11550 185 : gimple_seq_add_seq (body_p, clist);
11551 :
11552 185 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
11553 185 : tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
11554 185 : tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
11555 185 : g = gimple_build_cond (EQ_EXPR, thread_num, integer_zero_node, lab1, lab2);
11556 185 : gimple_seq_add_stmt (body_p, g);
11557 185 : g = gimple_build_label (lab1);
11558 185 : gimple_seq_add_stmt (body_p, g);
11559 185 : gimple_seq_add_seq (body_p, thr01_list);
11560 185 : g = gimple_build_goto (lab3);
11561 185 : gimple_seq_add_stmt (body_p, g);
11562 185 : g = gimple_build_label (lab2);
11563 185 : gimple_seq_add_stmt (body_p, g);
11564 185 : gimple_seq_add_seq (body_p, thrn1_list);
11565 185 : g = gimple_build_label (lab3);
11566 185 : gimple_seq_add_stmt (body_p, g);
11567 :
11568 185 : g = gimple_build_assign (ivar, size_zero_node);
11569 185 : gimple_seq_add_stmt (body_p, g);
11570 :
11571 185 : gimple_seq_add_stmt (body_p, stmt);
11572 185 : gimple_seq_add_seq (body_p, body);
11573 185 : gimple_seq_add_stmt (body_p, gimple_build_omp_continue (fd->loop.v,
11574 : fd->loop.v));
11575 :
11576 185 : g = gimple_build_omp_return (true);
11577 185 : gimple_seq_add_stmt (body_p, g);
11578 185 : gimple_seq_add_seq (body_p, mdlist);
11579 :
11580 185 : lab1 = create_artificial_label (UNKNOWN_LOCATION);
11581 185 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
11582 185 : g = gimple_build_cond (GT_EXPR, num_threads, integer_one_node, lab1, lab2);
11583 185 : gimple_seq_add_stmt (body_p, g);
11584 185 : g = gimple_build_label (lab1);
11585 185 : gimple_seq_add_stmt (body_p, g);
11586 :
11587 185 : g = omp_build_barrier (NULL, GOMP_BARRIER_IMPLICIT_WORKSHARE);
11588 185 : gimple_seq_add_stmt (body_p, g);
11589 :
11590 185 : tree down = create_tmp_var (unsigned_type_node);
11591 185 : g = gimple_build_assign (down, build_zero_cst (unsigned_type_node));
11592 185 : gimple_seq_add_stmt (body_p, g);
11593 :
11594 185 : g = gimple_build_assign (k, build_one_cst (unsigned_type_node));
11595 185 : gimple_seq_add_stmt (body_p, g);
11596 :
11597 185 : tree num_threadsu = create_tmp_var (unsigned_type_node);
11598 185 : g = gimple_build_assign (num_threadsu, NOP_EXPR, num_threads);
11599 185 : gimple_seq_add_stmt (body_p, g);
11600 :
11601 185 : tree thread_numu = create_tmp_var (unsigned_type_node);
11602 185 : g = gimple_build_assign (thread_numu, NOP_EXPR, thread_num);
11603 185 : gimple_seq_add_stmt (body_p, g);
11604 :
11605 185 : tree thread_nump1 = create_tmp_var (unsigned_type_node);
11606 185 : g = gimple_build_assign (thread_nump1, PLUS_EXPR, thread_numu,
11607 : build_int_cst (unsigned_type_node, 1));
11608 185 : gimple_seq_add_stmt (body_p, g);
11609 :
11610 185 : lab3 = create_artificial_label (UNKNOWN_LOCATION);
11611 185 : g = gimple_build_label (lab3);
11612 185 : gimple_seq_add_stmt (body_p, g);
11613 :
11614 185 : tree twok = create_tmp_var (unsigned_type_node);
11615 185 : g = gimple_build_assign (twok, LSHIFT_EXPR, k, integer_one_node);
11616 185 : gimple_seq_add_stmt (body_p, g);
11617 :
11618 185 : tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
11619 185 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
11620 185 : tree lab6 = create_artificial_label (UNKNOWN_LOCATION);
11621 185 : g = gimple_build_cond (GT_EXPR, twok, num_threadsu, lab4, lab5);
11622 185 : gimple_seq_add_stmt (body_p, g);
11623 185 : g = gimple_build_label (lab4);
11624 185 : gimple_seq_add_stmt (body_p, g);
11625 185 : g = gimple_build_assign (down, build_all_ones_cst (unsigned_type_node));
11626 185 : gimple_seq_add_stmt (body_p, g);
11627 185 : g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
11628 185 : gimple_seq_add_stmt (body_p, g);
11629 :
11630 185 : g = gimple_build_cond (EQ_EXPR, k, num_threadsu, lab6, lab5);
11631 185 : gimple_seq_add_stmt (body_p, g);
11632 185 : g = gimple_build_label (lab6);
11633 185 : gimple_seq_add_stmt (body_p, g);
11634 :
11635 185 : g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
11636 185 : gimple_seq_add_stmt (body_p, g);
11637 :
11638 185 : g = gimple_build_label (lab5);
11639 185 : gimple_seq_add_stmt (body_p, g);
11640 :
11641 185 : g = gimple_build_assign (twok, LSHIFT_EXPR, k, integer_one_node);
11642 185 : gimple_seq_add_stmt (body_p, g);
11643 :
11644 185 : tree cplx = create_tmp_var (build_complex_type (unsigned_type_node, false));
11645 185 : g = gimple_build_call_internal (IFN_MUL_OVERFLOW, 2, thread_nump1, twok);
11646 185 : gimple_call_set_lhs (g, cplx);
11647 185 : gimple_seq_add_stmt (body_p, g);
11648 185 : tree mul = create_tmp_var (unsigned_type_node);
11649 185 : g = gimple_build_assign (mul, REALPART_EXPR,
11650 : build1 (REALPART_EXPR, unsigned_type_node, cplx));
11651 185 : gimple_seq_add_stmt (body_p, g);
11652 185 : tree ovf = create_tmp_var (unsigned_type_node);
11653 185 : g = gimple_build_assign (ovf, IMAGPART_EXPR,
11654 : build1 (IMAGPART_EXPR, unsigned_type_node, cplx));
11655 185 : gimple_seq_add_stmt (body_p, g);
11656 :
11657 185 : tree lab7 = create_artificial_label (UNKNOWN_LOCATION);
11658 185 : tree lab8 = create_artificial_label (UNKNOWN_LOCATION);
11659 185 : g = gimple_build_cond (EQ_EXPR, ovf, build_zero_cst (unsigned_type_node),
11660 : lab7, lab8);
11661 185 : gimple_seq_add_stmt (body_p, g);
11662 185 : g = gimple_build_label (lab7);
11663 185 : gimple_seq_add_stmt (body_p, g);
11664 :
11665 185 : tree andv = create_tmp_var (unsigned_type_node);
11666 185 : g = gimple_build_assign (andv, BIT_AND_EXPR, k, down);
11667 185 : gimple_seq_add_stmt (body_p, g);
11668 185 : tree andvm1 = create_tmp_var (unsigned_type_node);
11669 185 : g = gimple_build_assign (andvm1, PLUS_EXPR, andv,
11670 : build_minus_one_cst (unsigned_type_node));
11671 185 : gimple_seq_add_stmt (body_p, g);
11672 :
11673 185 : g = gimple_build_assign (l, PLUS_EXPR, mul, andvm1);
11674 185 : gimple_seq_add_stmt (body_p, g);
11675 :
11676 185 : tree lab9 = create_artificial_label (UNKNOWN_LOCATION);
11677 185 : g = gimple_build_cond (LT_EXPR, l, num_threadsu, lab9, lab8);
11678 185 : gimple_seq_add_stmt (body_p, g);
11679 185 : g = gimple_build_label (lab9);
11680 185 : gimple_seq_add_stmt (body_p, g);
11681 185 : gimple_seq_add_seq (body_p, reduc_list);
11682 185 : g = gimple_build_label (lab8);
11683 185 : gimple_seq_add_stmt (body_p, g);
11684 :
11685 185 : tree lab10 = create_artificial_label (UNKNOWN_LOCATION);
11686 185 : tree lab11 = create_artificial_label (UNKNOWN_LOCATION);
11687 185 : tree lab12 = create_artificial_label (UNKNOWN_LOCATION);
11688 185 : g = gimple_build_cond (EQ_EXPR, down, build_zero_cst (unsigned_type_node),
11689 : lab10, lab11);
11690 185 : gimple_seq_add_stmt (body_p, g);
11691 185 : g = gimple_build_label (lab10);
11692 185 : gimple_seq_add_stmt (body_p, g);
11693 185 : g = gimple_build_assign (k, LSHIFT_EXPR, k, integer_one_node);
11694 185 : gimple_seq_add_stmt (body_p, g);
11695 185 : g = gimple_build_goto (lab12);
11696 185 : gimple_seq_add_stmt (body_p, g);
11697 185 : g = gimple_build_label (lab11);
11698 185 : gimple_seq_add_stmt (body_p, g);
11699 185 : g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
11700 185 : gimple_seq_add_stmt (body_p, g);
11701 185 : g = gimple_build_label (lab12);
11702 185 : gimple_seq_add_stmt (body_p, g);
11703 :
11704 185 : g = omp_build_barrier (NULL, GOMP_BARRIER_IMPLICIT_WORKSHARE);
11705 185 : gimple_seq_add_stmt (body_p, g);
11706 :
11707 185 : g = gimple_build_cond (NE_EXPR, k, build_zero_cst (unsigned_type_node),
11708 : lab3, lab2);
11709 185 : gimple_seq_add_stmt (body_p, g);
11710 :
11711 185 : g = gimple_build_label (lab2);
11712 185 : gimple_seq_add_stmt (body_p, g);
11713 :
11714 185 : lab1 = create_artificial_label (UNKNOWN_LOCATION);
11715 185 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
11716 185 : lab3 = create_artificial_label (UNKNOWN_LOCATION);
11717 185 : g = gimple_build_cond (EQ_EXPR, thread_num, integer_zero_node, lab1, lab2);
11718 185 : gimple_seq_add_stmt (body_p, g);
11719 185 : g = gimple_build_label (lab1);
11720 185 : gimple_seq_add_stmt (body_p, g);
11721 185 : gimple_seq_add_seq (body_p, thr02_list);
11722 185 : g = gimple_build_goto (lab3);
11723 185 : gimple_seq_add_stmt (body_p, g);
11724 185 : g = gimple_build_label (lab2);
11725 185 : gimple_seq_add_stmt (body_p, g);
11726 185 : gimple_seq_add_seq (body_p, thrn2_list);
11727 185 : g = gimple_build_label (lab3);
11728 185 : gimple_seq_add_stmt (body_p, g);
11729 :
11730 185 : g = gimple_build_assign (ivar, size_zero_node);
11731 185 : gimple_seq_add_stmt (body_p, g);
11732 185 : gimple_seq_add_stmt (body_p, new_stmt);
11733 185 : gimple_seq_add_seq (body_p, new_body);
11734 :
11735 185 : gimple_seq new_dlist = NULL;
11736 185 : lab1 = create_artificial_label (UNKNOWN_LOCATION);
11737 185 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
11738 185 : tree num_threadsm1 = create_tmp_var (integer_type_node);
11739 185 : g = gimple_build_assign (num_threadsm1, PLUS_EXPR, num_threads,
11740 : integer_minus_one_node);
11741 185 : gimple_seq_add_stmt (&new_dlist, g);
11742 185 : g = gimple_build_cond (EQ_EXPR, thread_num, num_threadsm1, lab1, lab2);
11743 185 : gimple_seq_add_stmt (&new_dlist, g);
11744 185 : g = gimple_build_label (lab1);
11745 185 : gimple_seq_add_stmt (&new_dlist, g);
11746 185 : gimple_seq_add_seq (&new_dlist, last_list);
11747 185 : g = gimple_build_label (lab2);
11748 185 : gimple_seq_add_stmt (&new_dlist, g);
11749 185 : gimple_seq_add_seq (&new_dlist, *dlist);
11750 185 : *dlist = new_dlist;
11751 185 : }
11752 :
11753 : /* Build an internal UNIQUE function with type IFN_UNIQUE_OACC_PRIVATE listing
11754 : the addresses of variables to be made private at the surrounding
11755 : parallelism level. Such functions appear in the gimple code stream in two
11756 : forms, e.g. for a partitioned loop:
11757 :
11758 : .data_dep.6 = .UNIQUE (OACC_HEAD_MARK, .data_dep.6, 1, 68);
11759 : .data_dep.6 = .UNIQUE (OACC_PRIVATE, .data_dep.6, -1, &w);
11760 : .data_dep.6 = .UNIQUE (OACC_FORK, .data_dep.6, -1);
11761 : .data_dep.6 = .UNIQUE (OACC_HEAD_MARK, .data_dep.6);
11762 :
11763 : or alternatively, OACC_PRIVATE can appear at the top level of a parallel,
11764 : not as part of a HEAD_MARK sequence:
11765 :
11766 : .UNIQUE (OACC_PRIVATE, 0, 0, &w);
11767 :
11768 : For such stand-alone appearances, the 3rd argument is always 0, denoting
11769 : gang partitioning. */
11770 :
11771 : static gcall *
11772 19728 : lower_oacc_private_marker (omp_context *ctx)
11773 : {
11774 19728 : if (ctx->oacc_privatization_candidates.length () == 0)
11775 : return NULL;
11776 :
11777 308 : auto_vec<tree, 5> args;
11778 :
11779 308 : args.quick_push (build_int_cst (integer_type_node, IFN_UNIQUE_OACC_PRIVATE));
11780 308 : args.quick_push (integer_zero_node);
11781 308 : args.quick_push (integer_minus_one_node);
11782 :
11783 308 : int i;
11784 308 : tree decl;
11785 1004 : FOR_EACH_VEC_ELT (ctx->oacc_privatization_candidates, i, decl)
11786 : {
11787 388 : gcc_checking_assert (TREE_ADDRESSABLE (decl));
11788 388 : tree addr = build_fold_addr_expr (decl);
11789 388 : args.safe_push (addr);
11790 : }
11791 :
11792 308 : return gimple_build_call_internal_vec (IFN_UNIQUE, args);
11793 308 : }
11794 :
11795 : /* Lower code for an OMP loop directive. */
11796 :
11797 : static void
11798 47102 : lower_omp_for (gimple_stmt_iterator *gsi_p, omp_context *ctx)
11799 : {
11800 47102 : tree *rhs_p, block;
11801 47102 : struct omp_for_data fd, *fdp = NULL;
11802 47102 : gomp_for *stmt = as_a <gomp_for *> (gsi_stmt (*gsi_p));
11803 47102 : gbind *new_stmt;
11804 47102 : gimple_seq omp_for_body, body, dlist, tred_ilist = NULL, tred_dlist = NULL;
11805 47102 : gimple_seq cnt_list = NULL, clist = NULL;
11806 47102 : gimple_seq oacc_head = NULL, oacc_tail = NULL;
11807 47102 : size_t i;
11808 :
11809 47102 : push_gimplify_context ();
11810 :
11811 47102 : if (is_gimple_omp_oacc (ctx->stmt))
11812 11325 : oacc_privatization_scan_clause_chain (ctx, gimple_omp_for_clauses (stmt));
11813 :
11814 47102 : lower_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
11815 :
11816 47102 : block = make_node (BLOCK);
11817 47102 : new_stmt = gimple_build_bind (NULL, NULL, block);
11818 : /* Replace at gsi right away, so that 'stmt' is no member
11819 : of a sequence anymore as we're going to add to a different
11820 : one below. */
11821 47102 : gsi_replace (gsi_p, new_stmt, true);
11822 :
11823 : /* Move declaration of temporaries in the loop body before we make
11824 : it go away. */
11825 47102 : omp_for_body = gimple_omp_body (stmt);
11826 47102 : if (!gimple_seq_empty_p (omp_for_body)
11827 47102 : && gimple_code (gimple_seq_first_stmt (omp_for_body)) == GIMPLE_BIND)
11828 : {
11829 19656 : gbind *inner_bind
11830 19656 : = as_a <gbind *> (gimple_seq_first_stmt (omp_for_body));
11831 19656 : tree vars = gimple_bind_vars (inner_bind);
11832 19656 : if (is_gimple_omp_oacc (ctx->stmt))
11833 5305 : oacc_privatization_scan_decl_chain (ctx, vars);
11834 19656 : gimple_bind_append_vars (new_stmt, vars);
11835 : /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
11836 : keep them on the inner_bind and it's block. */
11837 19656 : gimple_bind_set_vars (inner_bind, NULL_TREE);
11838 19656 : if (gimple_bind_block (inner_bind))
11839 17087 : BLOCK_VARS (gimple_bind_block (inner_bind)) = NULL_TREE;
11840 : }
11841 :
11842 47102 : if (gimple_omp_for_combined_into_p (stmt))
11843 : {
11844 14656 : omp_extract_for_data (stmt, &fd, NULL);
11845 14656 : fdp = &fd;
11846 :
11847 : /* We need two temporaries with fd.loop.v type (istart/iend)
11848 : and then (fd.collapse - 1) temporaries with the same
11849 : type for count2 ... countN-1 vars if not constant. */
11850 14656 : size_t count = 2;
11851 14656 : tree type = fd.iter_type;
11852 14656 : if (fd.collapse > 1
11853 4867 : && TREE_CODE (fd.loop.n2) != INTEGER_CST)
11854 2692 : count += fd.collapse - 1;
11855 14656 : size_t count2 = 0;
11856 14656 : tree type2 = NULL_TREE;
11857 14656 : bool taskreg_for
11858 14656 : = (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR
11859 14656 : || gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_TASKLOOP);
11860 14656 : tree outerc = NULL, *pc = gimple_omp_for_clauses_ptr (stmt);
11861 14656 : tree simtc = NULL;
11862 14656 : tree clauses = *pc;
11863 14656 : if (fd.collapse > 1
11864 4867 : && fd.non_rect
11865 137 : && fd.last_nonrect == fd.first_nonrect + 1
11866 89 : && TREE_CODE (fd.loop.n2) != INTEGER_CST)
11867 64 : if (tree v = gimple_omp_for_index (stmt, fd.last_nonrect))
11868 64 : if (!TYPE_UNSIGNED (TREE_TYPE (v)))
11869 : {
11870 60 : v = gimple_omp_for_index (stmt, fd.first_nonrect);
11871 60 : type2 = TREE_TYPE (v);
11872 60 : count++;
11873 60 : count2 = 3;
11874 : }
11875 14656 : if (taskreg_for)
11876 7634 : outerc
11877 7634 : = omp_find_clause (gimple_omp_taskreg_clauses (ctx->outer->stmt),
11878 : OMP_CLAUSE__LOOPTEMP_);
11879 14656 : if (ctx->simt_stmt)
11880 0 : simtc = omp_find_clause (gimple_omp_for_clauses (ctx->simt_stmt),
11881 : OMP_CLAUSE__LOOPTEMP_);
11882 49377 : for (i = 0; i < count + count2; i++)
11883 : {
11884 34721 : tree temp;
11885 34721 : if (taskreg_for)
11886 : {
11887 18149 : gcc_assert (outerc);
11888 18149 : temp = lookup_decl (OMP_CLAUSE_DECL (outerc), ctx->outer);
11889 18149 : outerc = omp_find_clause (OMP_CLAUSE_CHAIN (outerc),
11890 : OMP_CLAUSE__LOOPTEMP_);
11891 : }
11892 : else
11893 : {
11894 : /* If there are 2 adjacent SIMD stmts, one with _simt_
11895 : clause, another without, make sure they have the same
11896 : decls in _looptemp_ clauses, because the outer stmt
11897 : they are combined into will look up just one inner_stmt. */
11898 16572 : if (ctx->simt_stmt)
11899 0 : temp = OMP_CLAUSE_DECL (simtc);
11900 : else
11901 33000 : temp = create_tmp_var (i >= count ? type2 : type);
11902 16572 : insert_decl_map (&ctx->outer->cb, temp, temp);
11903 : }
11904 34721 : *pc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
11905 34721 : OMP_CLAUSE_DECL (*pc) = temp;
11906 34721 : pc = &OMP_CLAUSE_CHAIN (*pc);
11907 34721 : if (ctx->simt_stmt)
11908 0 : simtc = omp_find_clause (OMP_CLAUSE_CHAIN (simtc),
11909 : OMP_CLAUSE__LOOPTEMP_);
11910 : }
11911 14656 : *pc = clauses;
11912 : }
11913 :
11914 : /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
11915 47102 : dlist = NULL;
11916 47102 : body = NULL;
11917 47102 : tree rclauses
11918 47102 : = omp_task_reductions_find_first (gimple_omp_for_clauses (stmt), OMP_FOR,
11919 : OMP_CLAUSE_REDUCTION);
11920 47102 : tree rtmp = NULL_TREE;
11921 47102 : if (rclauses)
11922 : {
11923 228 : tree type = build_pointer_type (pointer_sized_int_node);
11924 228 : tree temp = create_tmp_var (type);
11925 228 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
11926 228 : OMP_CLAUSE_DECL (c) = temp;
11927 228 : OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (stmt);
11928 228 : gimple_omp_for_set_clauses (stmt, c);
11929 228 : lower_omp_task_reductions (ctx, OMP_FOR,
11930 : gimple_omp_for_clauses (stmt),
11931 : &tred_ilist, &tred_dlist);
11932 228 : rclauses = c;
11933 228 : rtmp = make_ssa_name (type);
11934 228 : gimple_seq_add_stmt (&body, gimple_build_assign (rtmp, temp));
11935 : }
11936 :
11937 47102 : lower_lastprivate_conditional_clauses (gimple_omp_for_clauses_ptr (stmt),
11938 : ctx);
11939 :
11940 47102 : lower_rec_input_clauses (gimple_omp_for_clauses (stmt), &body, &dlist, ctx,
11941 : fdp);
11942 93976 : gimple_seq_add_seq (rclauses ? &tred_ilist : &body,
11943 : gimple_omp_for_pre_body (stmt));
11944 :
11945 47102 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
11946 :
11947 47102 : gcall *private_marker = NULL;
11948 47102 : if (is_gimple_omp_oacc (ctx->stmt)
11949 47102 : && !gimple_seq_empty_p (omp_for_body))
11950 10333 : private_marker = lower_oacc_private_marker (ctx);
11951 :
11952 : /* Lower the header expressions. At this point, we can assume that
11953 : the header is of the form:
11954 :
11955 : #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
11956 :
11957 : We just need to make sure that VAL1, VAL2 and VAL3 are lowered
11958 : using the .omp_data_s mapping, if needed. */
11959 115488 : for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
11960 : {
11961 68386 : rhs_p = gimple_omp_for_initial_ptr (stmt, i);
11962 68386 : if (TREE_CODE (*rhs_p) == TREE_VEC)
11963 : {
11964 639 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 1)))
11965 149 : TREE_VEC_ELT (*rhs_p, 1)
11966 298 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 1), &cnt_list);
11967 639 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 2)))
11968 161 : TREE_VEC_ELT (*rhs_p, 2)
11969 322 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 2), &cnt_list);
11970 : }
11971 67747 : else if (!is_gimple_min_invariant (*rhs_p))
11972 11465 : *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
11973 56282 : else if (TREE_CODE (*rhs_p) == ADDR_EXPR)
11974 4713 : recompute_tree_invariant_for_addr_expr (*rhs_p);
11975 :
11976 68386 : rhs_p = gimple_omp_for_final_ptr (stmt, i);
11977 68386 : if (TREE_CODE (*rhs_p) == TREE_VEC)
11978 : {
11979 519 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 1)))
11980 142 : TREE_VEC_ELT (*rhs_p, 1)
11981 284 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 1), &cnt_list);
11982 519 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 2)))
11983 157 : TREE_VEC_ELT (*rhs_p, 2)
11984 314 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 2), &cnt_list);
11985 : }
11986 67867 : else if (!is_gimple_min_invariant (*rhs_p))
11987 16893 : *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
11988 50974 : else if (TREE_CODE (*rhs_p) == ADDR_EXPR)
11989 4630 : recompute_tree_invariant_for_addr_expr (*rhs_p);
11990 :
11991 68386 : rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1);
11992 68386 : if (!is_gimple_min_invariant (*rhs_p))
11993 6318 : *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
11994 : }
11995 47102 : if (rclauses)
11996 228 : gimple_seq_add_seq (&tred_ilist, cnt_list);
11997 : else
11998 46874 : gimple_seq_add_seq (&body, cnt_list);
11999 :
12000 : /* Once lowered, extract the bounds and clauses. */
12001 47102 : omp_extract_for_data (stmt, &fd, NULL);
12002 :
12003 47102 : if (is_gimple_omp_oacc (ctx->stmt)
12004 47102 : && !ctx_in_oacc_kernels_region (ctx))
12005 9665 : lower_oacc_head_tail (gimple_location (stmt),
12006 : gimple_omp_for_clauses (stmt), private_marker,
12007 : &oacc_head, &oacc_tail, ctx);
12008 :
12009 : /* Add OpenACC partitioning and reduction markers just before the loop. */
12010 47102 : if (oacc_head)
12011 9665 : gimple_seq_add_seq (&body, oacc_head);
12012 :
12013 47102 : lower_omp_for_lastprivate (&fd, &body, &dlist, &clist, ctx);
12014 :
12015 47102 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
12016 86180 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
12017 70509 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
12018 70509 : && !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
12019 : {
12020 227 : OMP_CLAUSE_DECL (c) = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
12021 227 : if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c)))
12022 42 : OMP_CLAUSE_LINEAR_STEP (c)
12023 84 : = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c),
12024 : ctx);
12025 : }
12026 :
12027 46708 : if ((ctx->scan_inclusive || ctx->scan_exclusive)
12028 47370 : && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
12029 185 : lower_omp_for_scan (&body, &dlist, stmt, &fd, ctx);
12030 : else
12031 : {
12032 46917 : gimple_seq_add_stmt (&body, stmt);
12033 46917 : gimple_seq_add_seq (&body, gimple_omp_body (stmt));
12034 : }
12035 :
12036 47102 : gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v,
12037 : fd.loop.v));
12038 :
12039 : /* After the loop, add exit clauses. */
12040 47102 : lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, &clist, ctx);
12041 :
12042 47102 : if (clist)
12043 : {
12044 179 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_START);
12045 179 : gcall *g = gimple_build_call (fndecl, 0);
12046 179 : gimple_seq_add_stmt (&body, g);
12047 179 : gimple_seq_add_seq (&body, clist);
12048 179 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_END);
12049 179 : g = gimple_build_call (fndecl, 0);
12050 179 : gimple_seq_add_stmt (&body, g);
12051 : }
12052 :
12053 47102 : if (ctx->cancellable)
12054 84 : gimple_seq_add_stmt (&body, gimple_build_label (ctx->cancel_label));
12055 :
12056 47102 : gimple_seq_add_seq (&body, dlist);
12057 :
12058 47102 : if (rclauses)
12059 : {
12060 228 : gimple_seq_add_seq (&tred_ilist, body);
12061 228 : body = tred_ilist;
12062 : }
12063 :
12064 47102 : body = maybe_catch_exception (body);
12065 :
12066 : /* Region exit marker goes at the end of the loop body. */
12067 47102 : gimple *g = gimple_build_omp_return (fd.have_nowait);
12068 47102 : gimple_seq_add_stmt (&body, g);
12069 :
12070 47102 : gimple_seq_add_seq (&body, tred_dlist);
12071 :
12072 47102 : maybe_add_implicit_barrier_cancel (ctx, g, &body);
12073 :
12074 47102 : if (rclauses)
12075 228 : OMP_CLAUSE_DECL (rclauses) = rtmp;
12076 :
12077 : /* Add OpenACC joining and reduction markers just after the loop. */
12078 47102 : if (oacc_tail)
12079 9665 : gimple_seq_add_seq (&body, oacc_tail);
12080 :
12081 47102 : pop_gimplify_context (new_stmt);
12082 :
12083 47102 : gimple_bind_append_vars (new_stmt, ctx->block_vars);
12084 47102 : maybe_remove_omp_member_access_dummy_vars (new_stmt);
12085 47102 : BLOCK_VARS (block) = gimple_bind_vars (new_stmt);
12086 47102 : if (BLOCK_VARS (block))
12087 46453 : TREE_USED (block) = 1;
12088 :
12089 47102 : gimple_bind_set_body (new_stmt, body);
12090 47102 : gimple_omp_set_body (stmt, NULL);
12091 47102 : gimple_omp_for_set_pre_body (stmt, NULL);
12092 47102 : }
12093 :
12094 : /* Callback for walk_stmts. Check if the current statement only contains
12095 : GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
12096 :
12097 : static tree
12098 217552 : check_combined_parallel (gimple_stmt_iterator *gsi_p,
12099 : bool *handled_ops_p,
12100 : struct walk_stmt_info *wi)
12101 : {
12102 217552 : int *info = (int *) wi->info;
12103 217552 : gimple *stmt = gsi_stmt (*gsi_p);
12104 :
12105 217552 : *handled_ops_p = true;
12106 217552 : switch (gimple_code (stmt))
12107 : {
12108 12030 : WALK_SUBSTMTS;
12109 :
12110 : case GIMPLE_DEBUG:
12111 : break;
12112 1914 : case GIMPLE_OMP_FOR:
12113 1914 : case GIMPLE_OMP_SECTIONS:
12114 1914 : *info = *info == 0 ? 1 : -1;
12115 1914 : break;
12116 203569 : default:
12117 203569 : *info = -1;
12118 203569 : break;
12119 : }
12120 217552 : return NULL;
12121 : }
12122 :
12123 : struct omp_taskcopy_context
12124 : {
12125 : /* This field must be at the beginning, as we do "inheritance": Some
12126 : callback functions for tree-inline.cc (e.g., omp_copy_decl)
12127 : receive a copy_body_data pointer that is up-casted to an
12128 : omp_context pointer. */
12129 : copy_body_data cb;
12130 : omp_context *ctx;
12131 : };
12132 :
12133 : static tree
12134 366 : task_copyfn_copy_decl (tree var, copy_body_data *cb)
12135 : {
12136 366 : struct omp_taskcopy_context *tcctx = (struct omp_taskcopy_context *) cb;
12137 :
12138 366 : if (splay_tree_lookup (tcctx->ctx->sfield_map, (splay_tree_key) var))
12139 158 : return create_tmp_var (TREE_TYPE (var));
12140 :
12141 : return var;
12142 : }
12143 :
12144 : static tree
12145 50 : task_copyfn_remap_type (struct omp_taskcopy_context *tcctx, tree orig_type)
12146 : {
12147 50 : tree name, new_fields = NULL, type, f;
12148 :
12149 50 : type = lang_hooks.types.make_type (RECORD_TYPE);
12150 50 : name = DECL_NAME (TYPE_NAME (orig_type));
12151 50 : name = build_decl (gimple_location (tcctx->ctx->stmt),
12152 : TYPE_DECL, name, type);
12153 50 : TYPE_NAME (type) = name;
12154 :
12155 1140 : for (f = TYPE_FIELDS (orig_type); f ; f = TREE_CHAIN (f))
12156 : {
12157 1090 : tree new_f = copy_node (f);
12158 1090 : DECL_CONTEXT (new_f) = type;
12159 1090 : TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &tcctx->cb);
12160 1090 : TREE_CHAIN (new_f) = new_fields;
12161 1090 : walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &tcctx->cb, NULL);
12162 1090 : walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, &tcctx->cb, NULL);
12163 1090 : walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
12164 : &tcctx->cb, NULL);
12165 1090 : new_fields = new_f;
12166 1090 : tcctx->cb.decl_map->put (f, new_f);
12167 : }
12168 50 : TYPE_FIELDS (type) = nreverse (new_fields);
12169 50 : layout_type (type);
12170 50 : return type;
12171 : }
12172 :
12173 : /* Create task copyfn. */
12174 :
12175 : static void
12176 513 : create_task_copyfn (gomp_task *task_stmt, omp_context *ctx)
12177 : {
12178 513 : struct function *child_cfun;
12179 513 : tree child_fn, t, c, src, dst, f, sf, arg, sarg, decl;
12180 513 : tree record_type, srecord_type, bind, list;
12181 513 : bool record_needs_remap = false, srecord_needs_remap = false;
12182 513 : splay_tree_node n;
12183 513 : struct omp_taskcopy_context tcctx;
12184 513 : location_t loc = gimple_location (task_stmt);
12185 513 : size_t looptempno = 0;
12186 :
12187 513 : child_fn = gimple_omp_task_copy_fn (task_stmt);
12188 513 : task_cpyfns.safe_push (task_stmt);
12189 513 : child_cfun = DECL_STRUCT_FUNCTION (child_fn);
12190 513 : gcc_assert (child_cfun->cfg == NULL);
12191 513 : DECL_SAVED_TREE (child_fn) = alloc_stmt_list ();
12192 :
12193 : /* Reset DECL_CONTEXT on function arguments. */
12194 1539 : for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
12195 1026 : DECL_CONTEXT (t) = child_fn;
12196 :
12197 : /* Populate the function. */
12198 513 : push_gimplify_context ();
12199 513 : push_cfun (child_cfun);
12200 :
12201 513 : bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
12202 513 : TREE_SIDE_EFFECTS (bind) = 1;
12203 513 : list = NULL;
12204 513 : DECL_SAVED_TREE (child_fn) = bind;
12205 513 : DECL_SOURCE_LOCATION (child_fn) = gimple_location (task_stmt);
12206 :
12207 : /* Remap src and dst argument types if needed. */
12208 513 : record_type = ctx->record_type;
12209 513 : srecord_type = ctx->srecord_type;
12210 3132 : for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
12211 2644 : if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
12212 : {
12213 : record_needs_remap = true;
12214 : break;
12215 : }
12216 2871 : for (f = TYPE_FIELDS (srecord_type); f ; f = DECL_CHAIN (f))
12217 2383 : if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
12218 : {
12219 : srecord_needs_remap = true;
12220 : break;
12221 : }
12222 :
12223 513 : if (record_needs_remap || srecord_needs_remap)
12224 : {
12225 25 : memset (&tcctx, '\0', sizeof (tcctx));
12226 25 : tcctx.cb.src_fn = ctx->cb.src_fn;
12227 25 : tcctx.cb.dst_fn = child_fn;
12228 25 : tcctx.cb.src_node = cgraph_node::get (tcctx.cb.src_fn);
12229 25 : gcc_checking_assert (tcctx.cb.src_node);
12230 25 : tcctx.cb.dst_node = tcctx.cb.src_node;
12231 25 : tcctx.cb.src_cfun = ctx->cb.src_cfun;
12232 25 : tcctx.cb.copy_decl = task_copyfn_copy_decl;
12233 25 : tcctx.cb.eh_lp_nr = 0;
12234 25 : tcctx.cb.transform_call_graph_edges = CB_CGE_MOVE;
12235 25 : tcctx.cb.decl_map = new hash_map<tree, tree>;
12236 25 : tcctx.ctx = ctx;
12237 :
12238 25 : if (record_needs_remap)
12239 25 : record_type = task_copyfn_remap_type (&tcctx, record_type);
12240 25 : if (srecord_needs_remap)
12241 25 : srecord_type = task_copyfn_remap_type (&tcctx, srecord_type);
12242 : }
12243 : else
12244 488 : tcctx.cb.decl_map = NULL;
12245 :
12246 513 : arg = DECL_ARGUMENTS (child_fn);
12247 513 : TREE_TYPE (arg) = build_pointer_type (record_type);
12248 513 : sarg = DECL_CHAIN (arg);
12249 513 : TREE_TYPE (sarg) = build_pointer_type (srecord_type);
12250 :
12251 : /* First pass: initialize temporaries used in record_type and srecord_type
12252 : sizes and field offsets. */
12253 513 : if (tcctx.cb.decl_map)
12254 631 : for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
12255 606 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
12256 : {
12257 497 : tree *p;
12258 :
12259 497 : decl = OMP_CLAUSE_DECL (c);
12260 497 : p = tcctx.cb.decl_map->get (decl);
12261 497 : if (p == NULL)
12262 339 : continue;
12263 158 : n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
12264 158 : sf = (tree) n->value;
12265 158 : sf = *tcctx.cb.decl_map->get (sf);
12266 158 : src = build_simple_mem_ref_loc (loc, sarg);
12267 158 : src = omp_build_component_ref (src, sf);
12268 158 : t = build2 (MODIFY_EXPR, TREE_TYPE (*p), *p, src);
12269 158 : append_to_statement_list (t, &list);
12270 : }
12271 :
12272 : /* Second pass: copy shared var pointers and copy construct non-VLA
12273 : firstprivate vars. */
12274 5895 : for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
12275 5382 : switch (OMP_CLAUSE_CODE (c))
12276 : {
12277 583 : splay_tree_key key;
12278 583 : case OMP_CLAUSE_SHARED:
12279 583 : decl = OMP_CLAUSE_DECL (c);
12280 583 : key = (splay_tree_key) decl;
12281 583 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
12282 273 : key = (splay_tree_key) &DECL_UID (decl);
12283 583 : n = splay_tree_lookup (ctx->field_map, key);
12284 583 : if (n == NULL)
12285 : break;
12286 149 : f = (tree) n->value;
12287 149 : if (tcctx.cb.decl_map)
12288 12 : f = *tcctx.cb.decl_map->get (f);
12289 149 : n = splay_tree_lookup (ctx->sfield_map, key);
12290 149 : sf = (tree) n->value;
12291 149 : if (tcctx.cb.decl_map)
12292 12 : sf = *tcctx.cb.decl_map->get (sf);
12293 149 : src = build_simple_mem_ref_loc (loc, sarg);
12294 149 : src = omp_build_component_ref (src, sf);
12295 149 : dst = build_simple_mem_ref_loc (loc, arg);
12296 149 : dst = omp_build_component_ref (dst, f);
12297 149 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12298 149 : append_to_statement_list (t, &list);
12299 149 : break;
12300 460 : case OMP_CLAUSE_REDUCTION:
12301 460 : case OMP_CLAUSE_IN_REDUCTION:
12302 460 : decl = OMP_CLAUSE_DECL (c);
12303 460 : if (TREE_CODE (decl) == MEM_REF)
12304 : {
12305 160 : decl = TREE_OPERAND (decl, 0);
12306 160 : if (TREE_CODE (decl) == POINTER_PLUS_EXPR)
12307 38 : decl = TREE_OPERAND (decl, 0);
12308 160 : if (TREE_CODE (decl) == INDIRECT_REF
12309 128 : || TREE_CODE (decl) == ADDR_EXPR)
12310 95 : decl = TREE_OPERAND (decl, 0);
12311 : }
12312 460 : key = (splay_tree_key) decl;
12313 460 : n = splay_tree_lookup (ctx->field_map, key);
12314 460 : if (n == NULL)
12315 : break;
12316 101 : f = (tree) n->value;
12317 101 : if (tcctx.cb.decl_map)
12318 20 : f = *tcctx.cb.decl_map->get (f);
12319 101 : n = splay_tree_lookup (ctx->sfield_map, key);
12320 101 : sf = (tree) n->value;
12321 101 : if (tcctx.cb.decl_map)
12322 20 : sf = *tcctx.cb.decl_map->get (sf);
12323 101 : src = build_simple_mem_ref_loc (loc, sarg);
12324 101 : src = omp_build_component_ref (src, sf);
12325 101 : if (decl != OMP_CLAUSE_DECL (c)
12326 95 : && TREE_CODE (TREE_TYPE (decl)) == REFERENCE_TYPE
12327 173 : && TREE_CODE (TREE_TYPE (TREE_TYPE (decl))) == POINTER_TYPE)
12328 32 : src = build_simple_mem_ref_loc (loc, src);
12329 101 : dst = build_simple_mem_ref_loc (loc, arg);
12330 101 : dst = omp_build_component_ref (dst, f);
12331 101 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12332 101 : append_to_statement_list (t, &list);
12333 101 : break;
12334 766 : case OMP_CLAUSE__LOOPTEMP_:
12335 : /* Fields for first two _looptemp_ clauses are initialized by
12336 : GOMP_taskloop*, the rest are handled like firstprivate. */
12337 766 : if (looptempno < 2)
12338 : {
12339 760 : looptempno++;
12340 760 : break;
12341 : }
12342 : /* FALLTHRU */
12343 1740 : case OMP_CLAUSE__REDUCTEMP_:
12344 1740 : case OMP_CLAUSE_FIRSTPRIVATE:
12345 1740 : decl = OMP_CLAUSE_DECL (c);
12346 1740 : if (is_variable_sized (decl))
12347 : break;
12348 1726 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
12349 1726 : if (n == NULL)
12350 : break;
12351 1726 : f = (tree) n->value;
12352 1726 : if (tcctx.cb.decl_map)
12353 490 : f = *tcctx.cb.decl_map->get (f);
12354 1726 : n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
12355 1726 : if (n != NULL)
12356 : {
12357 1476 : sf = (tree) n->value;
12358 1476 : if (tcctx.cb.decl_map)
12359 490 : sf = *tcctx.cb.decl_map->get (sf);
12360 1476 : src = build_simple_mem_ref_loc (loc, sarg);
12361 1476 : src = omp_build_component_ref (src, sf);
12362 1476 : if (use_pointer_for_field (decl, NULL)
12363 1476 : || omp_privatize_by_reference (decl))
12364 326 : src = build_simple_mem_ref_loc (loc, src);
12365 : }
12366 : else
12367 : src = decl;
12368 1726 : dst = build_simple_mem_ref_loc (loc, arg);
12369 1726 : dst = omp_build_component_ref (dst, f);
12370 1726 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FIRSTPRIVATE)
12371 192 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12372 : else
12373 : {
12374 1534 : if (ctx->allocate_map)
12375 93 : if (tree *allocatorp = ctx->allocate_map->get (decl))
12376 : {
12377 59 : tree allocator = *allocatorp;
12378 59 : HOST_WIDE_INT ialign = 0;
12379 59 : if (TREE_CODE (allocator) == TREE_LIST)
12380 : {
12381 4 : ialign = tree_to_uhwi (TREE_VALUE (allocator));
12382 4 : allocator = TREE_PURPOSE (allocator);
12383 : }
12384 59 : if (TREE_CODE (allocator) != INTEGER_CST)
12385 : {
12386 22 : n = splay_tree_lookup (ctx->sfield_map,
12387 : (splay_tree_key) allocator);
12388 22 : allocator = (tree) n->value;
12389 22 : if (tcctx.cb.decl_map)
12390 0 : allocator = *tcctx.cb.decl_map->get (allocator);
12391 22 : tree a = build_simple_mem_ref_loc (loc, sarg);
12392 22 : allocator = omp_build_component_ref (a, allocator);
12393 : }
12394 59 : allocator = fold_convert (pointer_sized_int_node, allocator);
12395 59 : tree a = builtin_decl_explicit (BUILT_IN_GOMP_ALLOC);
12396 59 : tree align = build_int_cst (size_type_node,
12397 59 : MAX (ialign,
12398 : DECL_ALIGN_UNIT (decl)));
12399 59 : tree sz = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (dst)));
12400 59 : tree ptr = build_call_expr_loc (loc, a, 3, align, sz,
12401 : allocator);
12402 59 : ptr = fold_convert (TREE_TYPE (dst), ptr);
12403 59 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, ptr);
12404 59 : append_to_statement_list (t, &list);
12405 59 : dst = build_simple_mem_ref_loc (loc, dst);
12406 : }
12407 1534 : t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
12408 : }
12409 1726 : append_to_statement_list (t, &list);
12410 1726 : break;
12411 495 : case OMP_CLAUSE_PRIVATE:
12412 495 : if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c))
12413 : break;
12414 12 : decl = OMP_CLAUSE_DECL (c);
12415 12 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
12416 12 : f = (tree) n->value;
12417 12 : if (tcctx.cb.decl_map)
12418 0 : f = *tcctx.cb.decl_map->get (f);
12419 12 : n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
12420 12 : if (n != NULL)
12421 : {
12422 12 : sf = (tree) n->value;
12423 12 : if (tcctx.cb.decl_map)
12424 0 : sf = *tcctx.cb.decl_map->get (sf);
12425 12 : src = build_simple_mem_ref_loc (loc, sarg);
12426 12 : src = omp_build_component_ref (src, sf);
12427 12 : if (use_pointer_for_field (decl, NULL))
12428 12 : src = build_simple_mem_ref_loc (loc, src);
12429 : }
12430 : else
12431 : src = decl;
12432 12 : dst = build_simple_mem_ref_loc (loc, arg);
12433 12 : dst = omp_build_component_ref (dst, f);
12434 12 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12435 12 : append_to_statement_list (t, &list);
12436 12 : break;
12437 : default:
12438 : break;
12439 : }
12440 :
12441 : /* Last pass: handle VLA firstprivates. */
12442 513 : if (tcctx.cb.decl_map)
12443 631 : for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
12444 606 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
12445 : {
12446 497 : tree ind, ptr, df;
12447 :
12448 497 : decl = OMP_CLAUSE_DECL (c);
12449 497 : if (!is_variable_sized (decl))
12450 483 : continue;
12451 14 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
12452 14 : if (n == NULL)
12453 0 : continue;
12454 14 : f = (tree) n->value;
12455 14 : f = *tcctx.cb.decl_map->get (f);
12456 14 : gcc_assert (DECL_HAS_VALUE_EXPR_P (decl));
12457 14 : ind = DECL_VALUE_EXPR (decl);
12458 14 : gcc_assert (TREE_CODE (ind) == INDIRECT_REF);
12459 14 : gcc_assert (DECL_P (TREE_OPERAND (ind, 0)));
12460 42 : n = splay_tree_lookup (ctx->sfield_map,
12461 14 : (splay_tree_key) TREE_OPERAND (ind, 0));
12462 14 : sf = (tree) n->value;
12463 14 : sf = *tcctx.cb.decl_map->get (sf);
12464 14 : src = build_simple_mem_ref_loc (loc, sarg);
12465 14 : src = omp_build_component_ref (src, sf);
12466 14 : src = build_simple_mem_ref_loc (loc, src);
12467 14 : dst = build_simple_mem_ref_loc (loc, arg);
12468 14 : dst = omp_build_component_ref (dst, f);
12469 14 : t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
12470 14 : append_to_statement_list (t, &list);
12471 42 : n = splay_tree_lookup (ctx->field_map,
12472 14 : (splay_tree_key) TREE_OPERAND (ind, 0));
12473 14 : df = (tree) n->value;
12474 14 : df = *tcctx.cb.decl_map->get (df);
12475 14 : ptr = build_simple_mem_ref_loc (loc, arg);
12476 14 : ptr = omp_build_component_ref (ptr, df);
12477 14 : t = build2 (MODIFY_EXPR, TREE_TYPE (ptr), ptr,
12478 : build_fold_addr_expr_loc (loc, dst));
12479 14 : append_to_statement_list (t, &list);
12480 : }
12481 :
12482 513 : t = build1 (RETURN_EXPR, void_type_node, NULL);
12483 513 : append_to_statement_list (t, &list);
12484 :
12485 513 : if (tcctx.cb.decl_map)
12486 25 : delete tcctx.cb.decl_map;
12487 513 : pop_gimplify_context (NULL);
12488 513 : BIND_EXPR_BODY (bind) = list;
12489 513 : pop_cfun ();
12490 513 : }
12491 :
12492 : static void
12493 1553 : lower_depend_clauses (tree *pclauses, gimple_seq *iseq, gimple_seq *oseq)
12494 : {
12495 1553 : tree c, clauses;
12496 1553 : gimple *g;
12497 1553 : size_t cnt[5] = { 0, 0, 0, 0, 0 }, idx = 2, i;
12498 :
12499 1553 : clauses = omp_find_clause (*pclauses, OMP_CLAUSE_DEPEND);
12500 1553 : gcc_assert (clauses);
12501 6316 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
12502 4888 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
12503 1713 : switch (OMP_CLAUSE_DEPEND_KIND (c))
12504 : {
12505 125 : case OMP_CLAUSE_DEPEND_LAST:
12506 : /* Lowering already done at gimplification. */
12507 125 : return;
12508 481 : case OMP_CLAUSE_DEPEND_IN:
12509 481 : cnt[2]++;
12510 481 : break;
12511 969 : case OMP_CLAUSE_DEPEND_OUT:
12512 969 : case OMP_CLAUSE_DEPEND_INOUT:
12513 969 : cnt[0]++;
12514 969 : break;
12515 30 : case OMP_CLAUSE_DEPEND_MUTEXINOUTSET:
12516 30 : cnt[1]++;
12517 30 : break;
12518 76 : case OMP_CLAUSE_DEPEND_DEPOBJ:
12519 76 : cnt[3]++;
12520 76 : break;
12521 32 : case OMP_CLAUSE_DEPEND_INOUTSET:
12522 32 : cnt[4]++;
12523 32 : break;
12524 0 : default:
12525 0 : gcc_unreachable ();
12526 : }
12527 1428 : if (cnt[1] || cnt[3] || cnt[4])
12528 135 : idx = 5;
12529 1428 : size_t total = cnt[0] + cnt[1] + cnt[2] + cnt[3] + cnt[4];
12530 1428 : size_t inoutidx = total + idx;
12531 1428 : tree type = build_array_type_nelts (ptr_type_node, total + idx + 2 * cnt[4]);
12532 1428 : tree array = create_tmp_var (type);
12533 1428 : TREE_ADDRESSABLE (array) = 1;
12534 1428 : tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE,
12535 : NULL_TREE);
12536 1428 : if (idx == 5)
12537 : {
12538 135 : g = gimple_build_assign (r, build_int_cst (ptr_type_node, 0));
12539 135 : gimple_seq_add_stmt (iseq, g);
12540 135 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE,
12541 : NULL_TREE);
12542 : }
12543 1428 : g = gimple_build_assign (r, build_int_cst (ptr_type_node, total));
12544 1428 : gimple_seq_add_stmt (iseq, g);
12545 7140 : for (i = 0; i < (idx == 5 ? 3 : 1); i++)
12546 : {
12547 1698 : r = build4 (ARRAY_REF, ptr_type_node, array,
12548 1698 : size_int (i + 1 + (idx == 5)), NULL_TREE, NULL_TREE);
12549 1698 : g = gimple_build_assign (r, build_int_cst (ptr_type_node, cnt[i]));
12550 1698 : gimple_seq_add_stmt (iseq, g);
12551 : }
12552 8568 : for (i = 0; i < 5; i++)
12553 : {
12554 7140 : if (cnt[i] == 0)
12555 5656 : continue;
12556 6565 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
12557 5081 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DEPEND)
12558 3354 : continue;
12559 : else
12560 : {
12561 1727 : switch (OMP_CLAUSE_DEPEND_KIND (c))
12562 : {
12563 547 : case OMP_CLAUSE_DEPEND_IN:
12564 547 : if (i != 2)
12565 139 : continue;
12566 : break;
12567 1025 : case OMP_CLAUSE_DEPEND_OUT:
12568 1025 : case OMP_CLAUSE_DEPEND_INOUT:
12569 1025 : if (i != 0)
12570 56 : continue;
12571 : break;
12572 44 : case OMP_CLAUSE_DEPEND_MUTEXINOUTSET:
12573 44 : if (i != 1)
12574 14 : continue;
12575 : break;
12576 79 : case OMP_CLAUSE_DEPEND_DEPOBJ:
12577 79 : if (i != 3)
12578 3 : continue;
12579 : break;
12580 32 : case OMP_CLAUSE_DEPEND_INOUTSET:
12581 32 : if (i != 4)
12582 0 : continue;
12583 : break;
12584 0 : default:
12585 0 : gcc_unreachable ();
12586 : }
12587 1588 : tree t = OMP_CLAUSE_DECL (c);
12588 1588 : if (i == 4)
12589 : {
12590 32 : t = build4 (ARRAY_REF, ptr_type_node, array,
12591 32 : size_int (inoutidx), NULL_TREE, NULL_TREE);
12592 32 : t = build_fold_addr_expr (t);
12593 32 : inoutidx += 2;
12594 : }
12595 1588 : t = fold_convert (ptr_type_node, t);
12596 1588 : gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
12597 1588 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
12598 : NULL_TREE, NULL_TREE);
12599 1588 : g = gimple_build_assign (r, t);
12600 1588 : gimple_seq_add_stmt (iseq, g);
12601 : }
12602 : }
12603 1428 : if (cnt[4])
12604 81 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
12605 49 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
12606 49 : && OMP_CLAUSE_DEPEND_KIND (c) == OMP_CLAUSE_DEPEND_INOUTSET)
12607 : {
12608 32 : tree t = OMP_CLAUSE_DECL (c);
12609 32 : t = fold_convert (ptr_type_node, t);
12610 32 : gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
12611 32 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
12612 : NULL_TREE, NULL_TREE);
12613 32 : g = gimple_build_assign (r, t);
12614 32 : gimple_seq_add_stmt (iseq, g);
12615 32 : t = build_int_cst (ptr_type_node, GOMP_DEPEND_INOUTSET);
12616 32 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
12617 : NULL_TREE, NULL_TREE);
12618 32 : g = gimple_build_assign (r, t);
12619 32 : gimple_seq_add_stmt (iseq, g);
12620 : }
12621 :
12622 1428 : c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND);
12623 1428 : OMP_CLAUSE_DEPEND_KIND (c) = OMP_CLAUSE_DEPEND_LAST;
12624 1428 : OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array);
12625 1428 : OMP_CLAUSE_CHAIN (c) = *pclauses;
12626 1428 : *pclauses = c;
12627 1428 : tree clobber = build_clobber (type);
12628 1428 : g = gimple_build_assign (array, clobber);
12629 1428 : gimple_seq_add_stmt (oseq, g);
12630 : }
12631 :
12632 : /* Lower the OpenMP parallel or task directive in the current statement
12633 : in GSI_P. CTX holds context information for the directive. */
12634 :
12635 : static void
12636 22809 : lower_omp_taskreg (gimple_stmt_iterator *gsi_p, omp_context *ctx)
12637 : {
12638 22809 : tree clauses;
12639 22809 : tree child_fn, t;
12640 22809 : gimple *stmt = gsi_stmt (*gsi_p);
12641 22809 : gbind *par_bind, *bind, *dep_bind = NULL;
12642 22809 : gimple_seq par_body;
12643 22809 : location_t loc = gimple_location (stmt);
12644 :
12645 22809 : clauses = gimple_omp_taskreg_clauses (stmt);
12646 22809 : if (gimple_code (stmt) == GIMPLE_OMP_TASK
12647 22809 : && gimple_omp_task_taskwait_p (stmt))
12648 : {
12649 84 : par_bind = NULL;
12650 84 : par_body = NULL;
12651 : }
12652 : else
12653 : {
12654 22725 : par_bind
12655 22725 : = as_a <gbind *> (gimple_seq_first_stmt (gimple_omp_body (stmt)));
12656 22725 : par_body = gimple_bind_body (par_bind);
12657 : }
12658 22809 : child_fn = ctx->cb.dst_fn;
12659 22809 : if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
12660 22809 : && !gimple_omp_parallel_combined_p (stmt))
12661 : {
12662 4349 : struct walk_stmt_info wi;
12663 4349 : int ws_num = 0;
12664 :
12665 4349 : memset (&wi, 0, sizeof (wi));
12666 4349 : wi.info = &ws_num;
12667 4349 : wi.val_only = true;
12668 4349 : walk_gimple_seq (par_body, check_combined_parallel, NULL, &wi);
12669 4349 : if (ws_num == 1)
12670 385 : gimple_omp_parallel_set_combined_p (stmt, true);
12671 : }
12672 22809 : gimple_seq dep_ilist = NULL;
12673 22809 : gimple_seq dep_olist = NULL;
12674 22809 : if (gimple_code (stmt) == GIMPLE_OMP_TASK
12675 22809 : && omp_find_clause (clauses, OMP_CLAUSE_DEPEND))
12676 : {
12677 1189 : push_gimplify_context ();
12678 1189 : dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12679 1189 : lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt),
12680 : &dep_ilist, &dep_olist);
12681 : }
12682 :
12683 22809 : if (gimple_code (stmt) == GIMPLE_OMP_TASK
12684 22809 : && gimple_omp_task_taskwait_p (stmt))
12685 : {
12686 84 : if (dep_bind)
12687 : {
12688 84 : gsi_replace (gsi_p, dep_bind, true);
12689 84 : gimple_bind_add_seq (dep_bind, dep_ilist);
12690 84 : gimple_bind_add_stmt (dep_bind, stmt);
12691 84 : gimple_bind_add_seq (dep_bind, dep_olist);
12692 84 : pop_gimplify_context (dep_bind);
12693 : }
12694 84 : return;
12695 : }
12696 :
12697 22725 : if (ctx->srecord_type)
12698 513 : create_task_copyfn (as_a <gomp_task *> (stmt), ctx);
12699 :
12700 22725 : gimple_seq tskred_ilist = NULL;
12701 22725 : gimple_seq tskred_olist = NULL;
12702 22725 : if ((is_task_ctx (ctx)
12703 3842 : && gimple_omp_task_taskloop_p (ctx->stmt)
12704 1336 : && omp_find_clause (gimple_omp_task_clauses (ctx->stmt),
12705 : OMP_CLAUSE_REDUCTION))
12706 26070 : || (is_parallel_ctx (ctx)
12707 16306 : && omp_find_clause (gimple_omp_parallel_clauses (stmt),
12708 : OMP_CLAUSE__REDUCTEMP_)))
12709 : {
12710 559 : if (dep_bind == NULL)
12711 : {
12712 559 : push_gimplify_context ();
12713 559 : dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12714 : }
12715 621 : lower_omp_task_reductions (ctx, is_task_ctx (ctx) ? OMP_TASKLOOP
12716 : : OMP_PARALLEL,
12717 559 : gimple_omp_taskreg_clauses (ctx->stmt),
12718 : &tskred_ilist, &tskred_olist);
12719 : }
12720 :
12721 22725 : push_gimplify_context ();
12722 :
12723 22725 : gimple_seq par_olist = NULL;
12724 22725 : gimple_seq par_ilist = NULL;
12725 22725 : gimple_seq par_rlist = NULL;
12726 22725 : lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx, NULL);
12727 22721 : lower_omp (&par_body, ctx);
12728 22721 : if (gimple_code (stmt) != GIMPLE_OMP_TASK)
12729 18879 : lower_reduction_clauses (clauses, &par_rlist, NULL, ctx);
12730 :
12731 : /* Declare all the variables created by mapping and the variables
12732 : declared in the scope of the parallel body. */
12733 22721 : record_vars_into (ctx->block_vars, child_fn);
12734 22721 : maybe_remove_omp_member_access_dummy_vars (par_bind);
12735 22721 : record_vars_into (gimple_bind_vars (par_bind), child_fn);
12736 :
12737 22721 : if (ctx->record_type)
12738 : {
12739 18181 : ctx->sender_decl
12740 35849 : = create_tmp_var (ctx->srecord_type ? ctx->srecord_type
12741 : : ctx->record_type, ".omp_data_o");
12742 18181 : DECL_NAMELESS (ctx->sender_decl) = 1;
12743 18181 : TREE_ADDRESSABLE (ctx->sender_decl) = 1;
12744 18181 : gimple_omp_taskreg_set_data_arg (stmt, ctx->sender_decl);
12745 : }
12746 :
12747 22721 : gimple_seq olist = NULL;
12748 22721 : gimple_seq ilist = NULL;
12749 22721 : lower_send_clauses (clauses, &ilist, &olist, ctx);
12750 22721 : lower_send_shared_vars (&ilist, &olist, ctx);
12751 :
12752 22721 : if (ctx->record_type)
12753 : {
12754 18181 : tree clobber = build_clobber (TREE_TYPE (ctx->sender_decl));
12755 18181 : gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
12756 : clobber));
12757 : }
12758 :
12759 : /* Once all the expansions are done, sequence all the different
12760 : fragments inside gimple_omp_body. */
12761 :
12762 22721 : gimple_seq new_body = NULL;
12763 :
12764 22721 : if (ctx->record_type)
12765 : {
12766 18181 : t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
12767 : /* fixup_child_record_type might have changed receiver_decl's type. */
12768 18181 : t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
12769 18181 : gimple_seq_add_stmt (&new_body,
12770 18181 : gimple_build_assign (ctx->receiver_decl, t));
12771 : }
12772 :
12773 22721 : gimple_seq_add_seq (&new_body, par_ilist);
12774 22721 : gimple_seq_add_seq (&new_body, par_body);
12775 22721 : gimple_seq_add_seq (&new_body, par_rlist);
12776 22721 : if (ctx->cancellable)
12777 146 : gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
12778 22721 : gimple_seq_add_seq (&new_body, par_olist);
12779 22721 : new_body = maybe_catch_exception (new_body);
12780 22721 : if (gimple_code (stmt) == GIMPLE_OMP_TASK)
12781 3842 : gimple_seq_add_stmt (&new_body,
12782 3842 : gimple_build_omp_continue (integer_zero_node,
12783 : integer_zero_node));
12784 22721 : gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
12785 22721 : gimple_omp_set_body (stmt, new_body);
12786 :
12787 22721 : if (dep_bind && gimple_bind_block (par_bind) == NULL_TREE)
12788 497 : bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12789 : else
12790 22224 : bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind));
12791 43778 : gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
12792 22721 : gimple_bind_add_seq (bind, ilist);
12793 22721 : gimple_bind_add_stmt (bind, stmt);
12794 22721 : gimple_bind_add_seq (bind, olist);
12795 :
12796 22721 : pop_gimplify_context (NULL);
12797 :
12798 22721 : if (dep_bind)
12799 : {
12800 1664 : gimple_bind_add_seq (dep_bind, dep_ilist);
12801 1664 : gimple_bind_add_seq (dep_bind, tskred_ilist);
12802 1664 : gimple_bind_add_stmt (dep_bind, bind);
12803 1664 : gimple_bind_add_seq (dep_bind, tskred_olist);
12804 1664 : gimple_bind_add_seq (dep_bind, dep_olist);
12805 1664 : pop_gimplify_context (dep_bind);
12806 : }
12807 : }
12808 :
12809 : /* Set EXPR as the hostaddr expression that should result from the clause C
12810 : in the target statement STMT. Returns the tree that should be
12811 : passed as the hostaddr (a pointer to the array containing the expanded
12812 : hostaddrs and sizes of the clause). */
12813 :
12814 : static tree
12815 37413 : lower_omp_map_iterator_expr (tree expr, tree c, gomp_target *stmt)
12816 : {
12817 37413 : if (!OMP_CLAUSE_HAS_ITERATORS (c))
12818 : return expr;
12819 :
12820 215 : tree iterator = OMP_CLAUSE_ITERATORS (c);
12821 215 : tree index = OMP_ITERATOR_INDEX (iterator);
12822 215 : tree elems = OMP_ITERATOR_ELEMS (iterator);
12823 215 : gimple_seq *loop_body_p = enter_omp_iterator_loop_context (c, stmt);
12824 :
12825 : /* IN LOOP BODY: */
12826 : /* elems[idx] = <expr>; */
12827 215 : tree lhs = build4 (ARRAY_REF, ptr_type_node, elems, index,
12828 : NULL_TREE, NULL_TREE);
12829 215 : tree mod_expr = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
12830 : void_type_node, lhs, expr);
12831 215 : gimplify_and_add (mod_expr, loop_body_p);
12832 215 : exit_omp_iterator_loop_context (c);
12833 :
12834 215 : return build_fold_addr_expr_with_type (elems, ptr_type_node);
12835 : }
12836 :
12837 : /* Set SIZE as the size expression that should result from the clause C
12838 : in the target statement STMT. Returns the tree that should be
12839 : passed as the clause size (a size_int with the value SIZE_MAX, indicating
12840 : that the clause uses an iterator). */
12841 :
12842 : static tree
12843 80263 : lower_omp_map_iterator_size (tree size, tree c, gomp_target *stmt)
12844 : {
12845 80263 : if (!OMP_CLAUSE_HAS_ITERATORS (c))
12846 : return size;
12847 :
12848 219 : tree iterator = OMP_CLAUSE_ITERATORS (c);
12849 219 : tree index = OMP_ITERATOR_INDEX (iterator);
12850 219 : tree elems = OMP_ITERATOR_ELEMS (iterator);
12851 219 : gimple_seq *loop_body_p = enter_omp_iterator_loop_context (c, stmt);
12852 :
12853 : /* IN LOOP BODY: */
12854 : /* elems[idx+1] = <size>; */
12855 219 : tree lhs = build4 (ARRAY_REF, ptr_type_node, elems,
12856 : size_binop (PLUS_EXPR, index, size_int (1)),
12857 : NULL_TREE, NULL_TREE);
12858 219 : tree mod_expr = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
12859 : void_type_node, lhs, size);
12860 219 : gimplify_and_add (mod_expr, loop_body_p);
12861 219 : exit_omp_iterator_loop_context (c);
12862 :
12863 219 : return size_int (SIZE_MAX);
12864 : }
12865 :
12866 : /* Lower the GIMPLE_OMP_TARGET in the current statement
12867 : in GSI_P. CTX holds context information for the directive. */
12868 :
12869 : static void
12870 37314 : lower_omp_target (gimple_stmt_iterator *gsi_p, omp_context *ctx)
12871 : {
12872 37314 : tree clauses;
12873 37314 : tree child_fn, t, c;
12874 37314 : gomp_target *stmt = as_a <gomp_target *> (gsi_stmt (*gsi_p));
12875 37314 : gbind *tgt_bind, *bind, *dep_bind = NULL;
12876 37314 : gimple_seq tgt_body, olist, ilist, fplist, new_body;
12877 37314 : location_t loc = gimple_location (stmt);
12878 37314 : bool offloaded, data_region;
12879 37314 : unsigned int map_cnt = 0;
12880 37314 : tree in_reduction_clauses = NULL_TREE;
12881 :
12882 37314 : tree deep_map_cnt = NULL_TREE;
12883 37314 : tree deep_map_data = NULL_TREE;
12884 37314 : tree deep_map_offset_data = NULL_TREE;
12885 37314 : tree deep_map_offset = NULL_TREE;
12886 :
12887 37314 : offloaded = is_gimple_omp_offloaded (stmt);
12888 37314 : switch (gimple_omp_target_kind (stmt))
12889 : {
12890 11916 : case GF_OMP_TARGET_KIND_REGION:
12891 11916 : tree *p, *q;
12892 11916 : q = &in_reduction_clauses;
12893 79969 : for (p = gimple_omp_target_clauses_ptr (stmt); *p; )
12894 68053 : if (OMP_CLAUSE_CODE (*p) == OMP_CLAUSE_IN_REDUCTION)
12895 : {
12896 422 : *q = *p;
12897 422 : q = &OMP_CLAUSE_CHAIN (*q);
12898 422 : *p = OMP_CLAUSE_CHAIN (*p);
12899 : }
12900 : else
12901 67631 : p = &OMP_CLAUSE_CHAIN (*p);
12902 11916 : *q = NULL_TREE;
12903 11916 : *p = in_reduction_clauses;
12904 : /* FALLTHRU */
12905 : case GF_OMP_TARGET_KIND_UPDATE:
12906 : case GF_OMP_TARGET_KIND_ENTER_DATA:
12907 : case GF_OMP_TARGET_KIND_EXIT_DATA:
12908 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
12909 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
12910 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
12911 : case GF_OMP_TARGET_KIND_OACC_UPDATE:
12912 : case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
12913 : case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
12914 : case GF_OMP_TARGET_KIND_OACC_DECLARE:
12915 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
12916 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
12917 : data_region = false;
12918 : break;
12919 : case GF_OMP_TARGET_KIND_DATA:
12920 : case GF_OMP_TARGET_KIND_OACC_DATA:
12921 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
12922 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
12923 37314 : data_region = true;
12924 : break;
12925 0 : default:
12926 0 : gcc_unreachable ();
12927 : }
12928 :
12929 : /* Ensure that requires map is written via output_offload_tables, even if only
12930 : 'target (enter/exit) data' is used in the translation unit. */
12931 37314 : if (ENABLE_OFFLOADING && (omp_requires_mask & OMP_REQUIRES_TARGET_USED))
12932 : g->have_offload = true;
12933 :
12934 37314 : clauses = gimple_omp_target_clauses (stmt);
12935 :
12936 37314 : gimple_seq dep_ilist = NULL;
12937 37314 : gimple_seq dep_olist = NULL;
12938 37314 : bool has_depend = omp_find_clause (clauses, OMP_CLAUSE_DEPEND) != NULL_TREE;
12939 37314 : if (has_depend || in_reduction_clauses)
12940 : {
12941 392 : push_gimplify_context ();
12942 392 : dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12943 392 : if (has_depend)
12944 329 : lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt),
12945 : &dep_ilist, &dep_olist);
12946 392 : if (in_reduction_clauses)
12947 314 : lower_rec_input_clauses (in_reduction_clauses, &dep_ilist, &dep_olist,
12948 : ctx, NULL);
12949 : }
12950 :
12951 37314 : tgt_bind = NULL;
12952 37314 : tgt_body = NULL;
12953 37314 : if (offloaded)
12954 : {
12955 21311 : tgt_bind = gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt));
12956 21311 : tgt_body = gimple_bind_body (tgt_bind);
12957 : }
12958 16003 : else if (data_region)
12959 4024 : tgt_body = gimple_omp_body (stmt);
12960 37314 : child_fn = ctx->cb.dst_fn;
12961 :
12962 37314 : push_gimplify_context ();
12963 37314 : fplist = NULL;
12964 :
12965 37314 : ilist = NULL;
12966 37314 : olist = NULL;
12967 :
12968 37314 : gimple_seq alloc_dlist = NULL;
12969 37314 : hash_map<tree, tree> alloc_map;
12970 74628 : hash_map<tree, gimple_seq> alloc_seq_map;
12971 :
12972 : /* The value expression for USES_ALLOCATORS needs to be setup before the
12973 : allocator is used via lower_private_allocate. */
12974 214424 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
12975 139796 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USES_ALLOCATORS)
12976 : {
12977 86 : tree allocator = OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c);
12978 86 : tree new_allocator = lookup_decl (allocator, ctx);
12979 86 : tree x = build_receiver_ref (allocator, false, ctx);
12980 86 : SET_DECL_VALUE_EXPR (new_allocator, x);
12981 86 : DECL_HAS_VALUE_EXPR_P (new_allocator) = 1;
12982 86 : map_cnt++;
12983 : }
12984 :
12985 177110 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
12986 139796 : switch (OMP_CLAUSE_CODE (c))
12987 : {
12988 : tree var, x, new_var, allocator, allocate_ptr, size;
12989 : gimple_seq alloc_seq;
12990 : bool by_ref;
12991 :
12992 : default:
12993 93255 : break;
12994 77986 : case OMP_CLAUSE_MAP:
12995 : #if CHECKING_P
12996 : /* First check what we're prepared to handle in the following. */
12997 77986 : switch (OMP_CLAUSE_MAP_KIND (c))
12998 : {
12999 : case GOMP_MAP_ALLOC:
13000 : case GOMP_MAP_TO:
13001 : case GOMP_MAP_FROM:
13002 : case GOMP_MAP_TOFROM:
13003 : case GOMP_MAP_POINTER:
13004 : case GOMP_MAP_TO_PSET:
13005 : case GOMP_MAP_DELETE:
13006 : case GOMP_MAP_RELEASE:
13007 : case GOMP_MAP_ALWAYS_TO:
13008 : case GOMP_MAP_ALWAYS_FROM:
13009 : case GOMP_MAP_ALWAYS_TOFROM:
13010 : case GOMP_MAP_FORCE_PRESENT:
13011 : case GOMP_MAP_ALWAYS_PRESENT_FROM:
13012 : case GOMP_MAP_ALWAYS_PRESENT_TO:
13013 : case GOMP_MAP_ALWAYS_PRESENT_TOFROM:
13014 :
13015 : case GOMP_MAP_FIRSTPRIVATE_POINTER:
13016 : case GOMP_MAP_FIRSTPRIVATE_REFERENCE:
13017 : case GOMP_MAP_STRUCT:
13018 : case GOMP_MAP_STRUCT_UNORD:
13019 : case GOMP_MAP_ALWAYS_POINTER:
13020 : case GOMP_MAP_ATTACH:
13021 : case GOMP_MAP_DETACH:
13022 : case GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION:
13023 : case GOMP_MAP_POINTER_TO_ZERO_LENGTH_ARRAY_SECTION:
13024 : break;
13025 2587 : case GOMP_MAP_IF_PRESENT:
13026 2587 : case GOMP_MAP_FORCE_ALLOC:
13027 2587 : case GOMP_MAP_FORCE_TO:
13028 2587 : case GOMP_MAP_FORCE_FROM:
13029 2587 : case GOMP_MAP_FORCE_TOFROM:
13030 2587 : case GOMP_MAP_FORCE_DEVICEPTR:
13031 2587 : case GOMP_MAP_DEVICE_RESIDENT:
13032 2587 : case GOMP_MAP_LINK:
13033 2587 : case GOMP_MAP_FORCE_DETACH:
13034 2587 : gcc_assert (is_gimple_omp_oacc (stmt));
13035 : break;
13036 0 : default:
13037 0 : gcc_unreachable ();
13038 : }
13039 : #endif
13040 : /* FALLTHRU */
13041 89575 : case OMP_CLAUSE_TO:
13042 89575 : case OMP_CLAUSE_FROM:
13043 89575 : oacc_firstprivate:
13044 89575 : var = OMP_CLAUSE_DECL (c);
13045 89575 : {
13046 89575 : tree extra = lang_hooks.decls.omp_deep_mapping_cnt (stmt, c, &ilist);
13047 89575 : if (extra != NULL_TREE && deep_map_cnt != NULL_TREE)
13048 59 : deep_map_cnt = fold_build2_loc (OMP_CLAUSE_LOCATION (c), PLUS_EXPR,
13049 : size_type_node, deep_map_cnt,
13050 : extra);
13051 89516 : else if (extra != NULL_TREE)
13052 : deep_map_cnt = extra;
13053 : }
13054 :
13055 89431 : if (deep_map_cnt
13056 89797 : && OMP_CLAUSE_HAS_ITERATORS (c))
13057 0 : sorry ("iterators used together with deep mapping are not "
13058 : "supported yet");
13059 :
13060 89575 : if (!DECL_P (var))
13061 : {
13062 36988 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
13063 36988 : || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
13064 35376 : && (OMP_CLAUSE_MAP_KIND (c)
13065 : != GOMP_MAP_FIRSTPRIVATE_POINTER)))
13066 36988 : map_cnt++;
13067 46541 : continue;
13068 : }
13069 :
13070 52587 : if (DECL_SIZE (var)
13071 52587 : && !poly_int_tree_p (DECL_SIZE (var)))
13072 : {
13073 719 : tree var2 = DECL_VALUE_EXPR (var);
13074 719 : gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
13075 719 : var2 = TREE_OPERAND (var2, 0);
13076 719 : gcc_assert (DECL_P (var2));
13077 : var = var2;
13078 : }
13079 :
13080 52587 : if (offloaded
13081 36688 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13082 85379 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
13083 29549 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE))
13084 : {
13085 3672 : if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
13086 : {
13087 415 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx))
13088 415 : && varpool_node::get_create (var)->offloadable)
13089 2 : continue;
13090 :
13091 413 : tree type = build_pointer_type (TREE_TYPE (var));
13092 413 : tree new_var = lookup_decl (var, ctx);
13093 413 : x = create_tmp_var_raw (type, get_name (new_var));
13094 413 : gimple_add_tmp_var (x);
13095 413 : x = build_simple_mem_ref (x);
13096 413 : SET_DECL_VALUE_EXPR (new_var, x);
13097 413 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13098 : }
13099 3670 : continue;
13100 3670 : }
13101 :
13102 48915 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13103 38938 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
13104 38727 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
13105 49180 : && is_omp_target (stmt))
13106 : {
13107 241 : gcc_assert (maybe_lookup_field (c, ctx));
13108 241 : map_cnt++;
13109 241 : continue;
13110 : }
13111 :
13112 48674 : if (!maybe_lookup_field (c, ctx))
13113 5640 : continue;
13114 :
13115 : /* Don't remap compute constructs' reduction variables, because the
13116 : intermediate result must be local to each gang. */
13117 70864 : if (offloaded && !(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13118 23934 : && is_gimple_omp_oacc (ctx->stmt)
13119 13572 : && OMP_CLAUSE_MAP_IN_REDUCTION (c)))
13120 : {
13121 27050 : x = build_receiver_ref (c, true, ctx);
13122 27050 : tree new_var = lookup_decl (var, ctx);
13123 :
13124 27050 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13125 23154 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
13126 3002 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
13127 30052 : && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
13128 425 : x = build_simple_mem_ref (x);
13129 27050 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
13130 : {
13131 3896 : gcc_assert (is_gimple_omp_oacc (ctx->stmt));
13132 3896 : if (omp_privatize_by_reference (new_var)
13133 3896 : && (TREE_CODE (TREE_TYPE (new_var)) != POINTER_TYPE
13134 44 : || DECL_BY_REFERENCE (var)))
13135 : {
13136 : /* Create a local object to hold the instance
13137 : value. */
13138 455 : tree type = TREE_TYPE (TREE_TYPE (new_var));
13139 455 : const char *id = IDENTIFIER_POINTER (DECL_NAME (new_var));
13140 455 : tree inst = create_tmp_var (type, id);
13141 455 : gimplify_assign (inst, fold_indirect_ref (x), &fplist);
13142 455 : x = build_fold_addr_expr (inst);
13143 : }
13144 3896 : gimplify_assign (new_var, x, &fplist);
13145 : }
13146 23154 : else if (DECL_P (new_var))
13147 : {
13148 23154 : SET_DECL_VALUE_EXPR (new_var, x);
13149 23154 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13150 : }
13151 : else
13152 0 : gcc_unreachable ();
13153 : }
13154 43034 : map_cnt++;
13155 43034 : break;
13156 :
13157 15467 : case OMP_CLAUSE_FIRSTPRIVATE:
13158 15467 : omp_firstprivate_recv:
13159 15467 : gcc_checking_assert (offloaded);
13160 15467 : if (is_gimple_omp_oacc (ctx->stmt))
13161 : {
13162 : /* No 'firstprivate' clauses on OpenACC 'kernels'. */
13163 3896 : gcc_checking_assert (!is_oacc_kernels (ctx));
13164 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
13165 3896 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
13166 :
13167 3896 : goto oacc_firstprivate;
13168 : }
13169 11571 : map_cnt++;
13170 11571 : var = OMP_CLAUSE_DECL (c);
13171 11571 : new_var = lookup_decl (var, ctx);
13172 11571 : allocator = NULL_TREE;
13173 11571 : allocate_ptr = NULL_TREE;
13174 11571 : size = TREE_TYPE (var);
13175 11571 : by_ref = omp_privatize_by_reference (var);
13176 11571 : if (by_ref)
13177 812 : size = TREE_TYPE (size);
13178 11571 : size = TYPE_SIZE_UNIT (size);
13179 11571 : if (is_variable_sized (var, by_ref))
13180 39 : size = lookup_decl (size, ctx);
13181 11571 : alloc_seq = NULL;
13182 11571 : if (lower_private_allocate (var, new_var, allocator, allocate_ptr,
13183 : &alloc_seq, ctx, by_ref, size))
13184 : {
13185 68 : alloc_map.put (new_var, allocate_ptr);
13186 68 : alloc_seq_map.put (new_var, alloc_seq);
13187 : }
13188 11571 : if (!by_ref && !is_gimple_reg_type (TREE_TYPE (var)))
13189 : {
13190 156 : if (is_variable_sized (var))
13191 : {
13192 10 : tree pvar = DECL_VALUE_EXPR (var);
13193 10 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
13194 10 : pvar = TREE_OPERAND (pvar, 0);
13195 10 : gcc_assert (DECL_P (pvar));
13196 10 : tree new_pvar = lookup_decl (pvar, ctx);
13197 10 : x = build_fold_indirect_ref (new_pvar);
13198 10 : TREE_THIS_NOTRAP (x) = 1;
13199 : }
13200 146 : else if (allocate_ptr)
13201 22 : x = build_fold_indirect_ref (allocate_ptr);
13202 : else
13203 124 : x = build_receiver_ref (var, true, ctx);
13204 156 : SET_DECL_VALUE_EXPR (new_var, x);
13205 156 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13206 : }
13207 : /* Fortran array descriptors: firstprivate of data + attach. */
13208 11571 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
13209 11552 : && lang_hooks.decls.omp_array_data (var, true)
13210 11602 : && lang_hooks.decls.omp_array_data_privatize (var))
13211 23 : map_cnt += 2;
13212 :
13213 11719 : do_dtor:
13214 11719 : if (allocator)
13215 : {
13216 125 : if (!is_gimple_val (allocator))
13217 : {
13218 0 : tree avar = create_tmp_var (TREE_TYPE (allocator));
13219 0 : gimplify_assign (avar, allocator, &alloc_dlist);
13220 0 : allocator = avar;
13221 : }
13222 125 : if (!is_gimple_val (allocate_ptr))
13223 : {
13224 0 : tree apvar = create_tmp_var (TREE_TYPE (allocate_ptr));
13225 0 : gimplify_assign (apvar, allocate_ptr, &alloc_dlist);
13226 0 : allocate_ptr = apvar;
13227 : }
13228 125 : tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
13229 125 : gimple *g = gimple_build_call (f, 2, allocate_ptr, allocator);
13230 125 : gimple_seq_add_stmt (&alloc_dlist, g);
13231 : }
13232 : break;
13233 :
13234 228 : case OMP_CLAUSE_PRIVATE:
13235 228 : gcc_checking_assert (offloaded);
13236 228 : if (is_gimple_omp_oacc (ctx->stmt))
13237 : {
13238 : /* No 'private' clauses on OpenACC 'kernels'. */
13239 80 : gcc_checking_assert (!is_oacc_kernels (ctx));
13240 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
13241 80 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
13242 :
13243 : break;
13244 : }
13245 148 : var = OMP_CLAUSE_DECL (c);
13246 148 : new_var = lookup_decl (var, ctx);
13247 148 : allocator = NULL_TREE;
13248 148 : allocate_ptr = NULL_TREE;
13249 148 : alloc_seq = NULL;
13250 148 : size = TREE_TYPE (var);
13251 148 : by_ref = omp_privatize_by_reference (var);
13252 148 : if (by_ref)
13253 22 : size = TREE_TYPE (size);
13254 148 : size = TYPE_SIZE_UNIT (size);
13255 148 : if (is_variable_sized (var, by_ref))
13256 21 : size = lookup_decl (size, ctx);
13257 148 : lower_private_allocate (var, new_var, allocator, allocate_ptr,
13258 : &alloc_seq, ctx, by_ref, size);
13259 148 : if (allocate_ptr)
13260 : {
13261 57 : alloc_map.put (new_var, allocate_ptr);
13262 57 : alloc_seq_map.put (new_var, alloc_seq);
13263 : }
13264 148 : if (!allocate_ptr && is_variable_sized (var))
13265 : {
13266 1 : tree new_var = lookup_decl (var, ctx);
13267 1 : tree pvar = DECL_VALUE_EXPR (var);
13268 1 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
13269 1 : pvar = TREE_OPERAND (pvar, 0);
13270 1 : gcc_assert (DECL_P (pvar));
13271 1 : tree new_pvar = lookup_decl (pvar, ctx);
13272 1 : x = build_fold_indirect_ref (new_pvar);
13273 1 : TREE_THIS_NOTRAP (x) = 1;
13274 1 : SET_DECL_VALUE_EXPR (new_var, x);
13275 1 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13276 : }
13277 148 : goto do_dtor;
13278 :
13279 2431 : case OMP_CLAUSE_USE_DEVICE_PTR:
13280 2431 : case OMP_CLAUSE_USE_DEVICE_ADDR:
13281 2431 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
13282 2431 : case OMP_CLAUSE_IS_DEVICE_PTR:
13283 2431 : var = OMP_CLAUSE_DECL (c);
13284 2431 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
13285 : {
13286 253 : while (TREE_CODE (var) == INDIRECT_REF
13287 253 : || TREE_CODE (var) == ARRAY_REF)
13288 17 : var = TREE_OPERAND (var, 0);
13289 236 : if (lang_hooks.decls.omp_array_data (var, true))
13290 19 : goto omp_firstprivate_recv;
13291 : }
13292 2412 : map_cnt++;
13293 2412 : if (is_variable_sized (var))
13294 : {
13295 12 : tree new_var = lookup_decl (var, ctx);
13296 12 : tree pvar = DECL_VALUE_EXPR (var);
13297 12 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
13298 12 : pvar = TREE_OPERAND (pvar, 0);
13299 12 : gcc_assert (DECL_P (pvar));
13300 12 : tree new_pvar = lookup_decl (pvar, ctx);
13301 12 : x = build_fold_indirect_ref (new_pvar);
13302 12 : TREE_THIS_NOTRAP (x) = 1;
13303 12 : SET_DECL_VALUE_EXPR (new_var, x);
13304 12 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13305 : }
13306 2400 : else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
13307 543 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
13308 2072 : && !omp_privatize_by_reference (var)
13309 703 : && !omp_is_allocatable_or_ptr (var)
13310 564 : && !lang_hooks.decls.omp_array_data (var, true))
13311 4435 : || TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
13312 : {
13313 389 : tree new_var = lookup_decl (var, ctx);
13314 389 : tree type = build_pointer_type (TREE_TYPE (var));
13315 389 : x = create_tmp_var_raw (type, get_name (new_var));
13316 389 : gimple_add_tmp_var (x);
13317 389 : x = build_simple_mem_ref (x);
13318 389 : SET_DECL_VALUE_EXPR (new_var, x);
13319 389 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13320 : }
13321 : else
13322 : {
13323 2011 : tree new_var = lookup_decl (var, ctx);
13324 2011 : x = create_tmp_var_raw (TREE_TYPE (new_var), get_name (new_var));
13325 2011 : gimple_add_tmp_var (x);
13326 2011 : SET_DECL_VALUE_EXPR (new_var, x);
13327 2011 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13328 : }
13329 : break;
13330 : }
13331 :
13332 37314 : if (offloaded)
13333 : {
13334 21311 : target_nesting_level++;
13335 21311 : lower_omp (&tgt_body, ctx);
13336 21311 : target_nesting_level--;
13337 : }
13338 16003 : else if (data_region)
13339 4024 : lower_omp (&tgt_body, ctx);
13340 :
13341 37314 : if (offloaded)
13342 : {
13343 : /* Declare all the variables created by mapping and the variables
13344 : declared in the scope of the target body. */
13345 21311 : record_vars_into (ctx->block_vars, child_fn);
13346 21311 : maybe_remove_omp_member_access_dummy_vars (tgt_bind);
13347 21311 : record_vars_into (gimple_bind_vars (tgt_bind), child_fn);
13348 : }
13349 :
13350 37314 : if (ctx->record_type)
13351 : {
13352 32403 : if (deep_map_cnt && TREE_CODE (deep_map_cnt) == INTEGER_CST)
13353 : /* map_cnt = map_cnt + tree_to_hwi (deep_map_cnt); */
13354 : /* deep_map_cnt = NULL_TREE; */
13355 0 : gcc_unreachable ();
13356 144 : else if (deep_map_cnt)
13357 : {
13358 144 : gcc_assert (flexible_array_type_p (ctx->record_type));
13359 144 : tree n = create_tmp_var_raw (size_type_node, "nn_map");
13360 144 : gimple_add_tmp_var (n);
13361 144 : gimplify_assign (n, deep_map_cnt, &ilist);
13362 144 : deep_map_cnt = n;
13363 : }
13364 144 : ctx->sender_decl
13365 32403 : = create_tmp_var (deep_map_cnt ? build_pointer_type (ctx->record_type)
13366 : : ctx->record_type, ".omp_data_arr");
13367 32403 : DECL_NAMELESS (ctx->sender_decl) = 1;
13368 32403 : TREE_ADDRESSABLE (ctx->sender_decl) = 1;
13369 64662 : t = make_tree_vec (deep_map_cnt ? 4 : 3);
13370 32403 : TREE_VEC_ELT (t, 0) = ctx->sender_decl;
13371 32403 : TREE_VEC_ELT (t, 1)
13372 64806 : = create_tmp_var (deep_map_cnt
13373 144 : ? build_pointer_type (size_type_node)
13374 32259 : : build_array_type_nelts (size_type_node, map_cnt),
13375 : ".omp_data_sizes");
13376 32403 : DECL_NAMELESS (TREE_VEC_ELT (t, 1)) = 1;
13377 32403 : TREE_ADDRESSABLE (TREE_VEC_ELT (t, 1)) = 1;
13378 32403 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 1;
13379 32403 : tree tkind_type = short_unsigned_type_node;
13380 32403 : int talign_shift = 8;
13381 32403 : TREE_VEC_ELT (t, 2)
13382 64806 : = create_tmp_var (deep_map_cnt
13383 144 : ? build_pointer_type (tkind_type)
13384 32259 : : build_array_type_nelts (tkind_type, map_cnt),
13385 : ".omp_data_kinds");
13386 32403 : DECL_NAMELESS (TREE_VEC_ELT (t, 2)) = 1;
13387 32403 : TREE_ADDRESSABLE (TREE_VEC_ELT (t, 2)) = 1;
13388 32403 : TREE_STATIC (TREE_VEC_ELT (t, 2)) = 1;
13389 32403 : gimple_omp_target_set_data_arg (stmt, t);
13390 :
13391 32403 : if (deep_map_cnt)
13392 : {
13393 144 : tree tmp, size;
13394 144 : size = create_tmp_var (size_type_node, NULL);
13395 144 : DECL_NAMELESS (size) = 1;
13396 144 : gimplify_assign (size,
13397 : fold_build2_loc (UNKNOWN_LOCATION, PLUS_EXPR,
13398 : size_type_node, deep_map_cnt,
13399 : build_int_cst (size_type_node,
13400 144 : map_cnt)), &ilist);
13401 144 : TREE_VEC_ELT (t, 3) = size;
13402 :
13403 144 : tree call = builtin_decl_explicit (BUILT_IN_MALLOC);
13404 144 : size = fold_build2_loc (UNKNOWN_LOCATION, MULT_EXPR,
13405 : size_type_node, deep_map_cnt,
13406 144 : TYPE_SIZE_UNIT (ptr_type_node));
13407 144 : size = fold_build2_loc (UNKNOWN_LOCATION, PLUS_EXPR,
13408 : size_type_node, size,
13409 144 : TYPE_SIZE_UNIT (ctx->record_type));
13410 144 : tmp = build_call_expr_loc (input_location, call, 1, size);
13411 144 : gimplify_assign (ctx->sender_decl, tmp, &ilist);
13412 :
13413 144 : size = fold_build2_loc (UNKNOWN_LOCATION, MULT_EXPR,
13414 144 : size_type_node, TREE_VEC_ELT (t, 3),
13415 144 : TYPE_SIZE_UNIT (size_type_node));
13416 144 : tmp = build_call_expr_loc (input_location, call, 1, size);
13417 144 : gimplify_assign (TREE_VEC_ELT (t, 1), tmp, &ilist);
13418 :
13419 144 : size = fold_build2_loc (UNKNOWN_LOCATION, MULT_EXPR,
13420 144 : size_type_node, TREE_VEC_ELT (t, 3),
13421 144 : TYPE_SIZE_UNIT (tkind_type));
13422 144 : tmp = build_call_expr_loc (input_location, call, 1, size);
13423 144 : gimplify_assign (TREE_VEC_ELT (t, 2), tmp, &ilist);
13424 144 : tree field = TYPE_FIELDS (TREE_TYPE (TREE_TYPE (ctx->sender_decl)));
13425 595 : for ( ; DECL_CHAIN (field) != NULL_TREE; field = DECL_CHAIN (field))
13426 : ;
13427 144 : gcc_assert (TREE_CODE (TREE_TYPE (field)));
13428 144 : tmp = build_fold_indirect_ref (ctx->sender_decl);
13429 144 : deep_map_data = omp_build_component_ref (tmp, field);
13430 144 : deep_map_offset_data = create_tmp_var_raw (size_type_node,
13431 : "map_offset_data");
13432 144 : deep_map_offset = create_tmp_var_raw (size_type_node, "map_offset");
13433 144 : gimple_add_tmp_var (deep_map_offset_data);
13434 144 : gimple_add_tmp_var (deep_map_offset);
13435 144 : gimplify_assign (deep_map_offset_data, build_int_cst (size_type_node,
13436 : 0), &ilist);
13437 144 : gimplify_assign (deep_map_offset, build_int_cst (size_type_node,
13438 144 : map_cnt), &ilist);
13439 : }
13440 :
13441 32403 : vec<constructor_elt, va_gc> *vsize;
13442 32403 : vec<constructor_elt, va_gc> *vkind;
13443 32403 : vec_alloc (vsize, map_cnt);
13444 32403 : vec_alloc (vkind, map_cnt);
13445 32403 : unsigned int map_idx = 0;
13446 :
13447 195168 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
13448 130362 : switch (OMP_CLAUSE_CODE (c))
13449 : {
13450 : tree ovar, nc, s, purpose, var, x, type;
13451 : unsigned int talign;
13452 :
13453 : default:
13454 126391 : break;
13455 :
13456 88074 : case OMP_CLAUSE_MAP:
13457 88074 : case OMP_CLAUSE_TO:
13458 88074 : case OMP_CLAUSE_FROM:
13459 88074 : oacc_firstprivate_map:
13460 88074 : nc = c;
13461 88074 : ovar = OMP_CLAUSE_DECL (c);
13462 88074 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13463 88074 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
13464 73074 : || (OMP_CLAUSE_MAP_KIND (c)
13465 : == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
13466 : break;
13467 84234 : if (deep_map_cnt)
13468 : {
13469 428 : unsigned HOST_WIDE_INT tkind2;
13470 428 : switch (OMP_CLAUSE_CODE (c))
13471 : {
13472 424 : case OMP_CLAUSE_MAP:
13473 424 : tkind2 = OMP_CLAUSE_MAP_KIND (c);
13474 424 : if (OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (c)
13475 424 : && (((tkind2 & GOMP_MAP_FLAG_SPECIAL_BITS)
13476 : & ~GOMP_MAP_IMPLICIT)
13477 : == 0))
13478 : {
13479 : /* If this is an implicit map, and the GOMP_MAP_IMPLICIT
13480 : bits are not interfered by other special bit
13481 : encodings, then turn the GOMP_IMPLICIT_BIT flag on
13482 : for the runtime to see. */
13483 120 : tkind2 |= GOMP_MAP_IMPLICIT;
13484 : }
13485 : break;
13486 : case OMP_CLAUSE_FIRSTPRIVATE: tkind2 = GOMP_MAP_TO; break;
13487 : case OMP_CLAUSE_TO: tkind2 = GOMP_MAP_TO; break;
13488 2 : case OMP_CLAUSE_FROM: tkind2 = GOMP_MAP_FROM; break;
13489 0 : default: gcc_unreachable ();
13490 : }
13491 428 : lang_hooks.decls.omp_deep_mapping (stmt, c, tkind2,
13492 : deep_map_data,
13493 428 : TREE_VEC_ELT (t, 1),
13494 428 : TREE_VEC_ELT (t, 2),
13495 : deep_map_offset_data,
13496 : deep_map_offset, &ilist);
13497 : }
13498 84234 : if (!DECL_P (ovar))
13499 : {
13500 36988 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13501 36988 : && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
13502 : {
13503 0 : nc = OMP_CLAUSE_CHAIN (c);
13504 0 : gcc_checking_assert (OMP_CLAUSE_DECL (nc)
13505 : == get_base_address (ovar));
13506 0 : ovar = OMP_CLAUSE_DECL (nc);
13507 : }
13508 : else
13509 : {
13510 36988 : tree x = build_sender_ref (c, ctx);
13511 36988 : tree v = ovar;
13512 36988 : if (in_reduction_clauses
13513 163 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13514 37151 : && OMP_CLAUSE_MAP_IN_REDUCTION (c))
13515 : {
13516 104 : v = unshare_expr (v);
13517 104 : tree *p = &v;
13518 104 : while (handled_component_p (*p)
13519 : || TREE_CODE (*p) == INDIRECT_REF
13520 : || TREE_CODE (*p) == ADDR_EXPR
13521 : || TREE_CODE (*p) == MEM_REF
13522 256 : || TREE_CODE (*p) == NON_LVALUE_EXPR)
13523 152 : p = &TREE_OPERAND (*p, 0);
13524 104 : tree d = *p;
13525 104 : if (is_variable_sized (d))
13526 : {
13527 16 : gcc_assert (DECL_HAS_VALUE_EXPR_P (d));
13528 16 : d = DECL_VALUE_EXPR (d);
13529 16 : gcc_assert (TREE_CODE (d) == INDIRECT_REF);
13530 16 : d = TREE_OPERAND (d, 0);
13531 16 : gcc_assert (DECL_P (d));
13532 : }
13533 104 : splay_tree_key key
13534 104 : = (splay_tree_key) &DECL_CONTEXT (d);
13535 104 : tree nd = (tree) splay_tree_lookup (ctx->field_map,
13536 104 : key)->value;
13537 104 : if (d == *p)
13538 88 : *p = nd;
13539 : else
13540 16 : *p = build_fold_indirect_ref (nd);
13541 : }
13542 36988 : v = build_fold_addr_expr_with_type (v, ptr_type_node);
13543 36988 : v = lower_omp_map_iterator_expr (v, c, stmt);
13544 36988 : gimplify_assign (x, v, &ilist);
13545 36988 : nc = NULL_TREE;
13546 : }
13547 : }
13548 : else
13549 : {
13550 47246 : if (DECL_SIZE (ovar)
13551 47246 : && !poly_int_tree_p (DECL_SIZE (ovar)))
13552 : {
13553 7 : tree ovar2 = DECL_VALUE_EXPR (ovar);
13554 7 : gcc_assert (TREE_CODE (ovar2) == INDIRECT_REF);
13555 7 : ovar2 = TREE_OPERAND (ovar2, 0);
13556 7 : gcc_assert (DECL_P (ovar2));
13557 : ovar = ovar2;
13558 : }
13559 47246 : if (!maybe_lookup_field (c, ctx)
13560 51217 : && !(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13561 3971 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
13562 3971 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)))
13563 3971 : continue;
13564 : }
13565 :
13566 80263 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar));
13567 80263 : if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign)
13568 6416 : talign = DECL_ALIGN_UNIT (ovar);
13569 :
13570 80263 : var = NULL_TREE;
13571 80263 : if (nc)
13572 : {
13573 43275 : if (in_reduction_clauses
13574 879 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13575 44154 : && OMP_CLAUSE_MAP_IN_REDUCTION (c))
13576 : {
13577 268 : tree d = ovar;
13578 268 : if (is_variable_sized (d))
13579 : {
13580 0 : gcc_assert (DECL_HAS_VALUE_EXPR_P (d));
13581 0 : d = DECL_VALUE_EXPR (d);
13582 0 : gcc_assert (TREE_CODE (d) == INDIRECT_REF);
13583 0 : d = TREE_OPERAND (d, 0);
13584 0 : gcc_assert (DECL_P (d));
13585 : }
13586 268 : splay_tree_key key
13587 268 : = (splay_tree_key) &DECL_CONTEXT (d);
13588 268 : tree nd = (tree) splay_tree_lookup (ctx->field_map,
13589 268 : key)->value;
13590 268 : if (d == ovar)
13591 : var = nd;
13592 : else
13593 0 : var = build_fold_indirect_ref (nd);
13594 : }
13595 : else
13596 43007 : var = lookup_decl_in_outer_ctx (ovar, ctx);
13597 : }
13598 43275 : if (nc
13599 43275 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13600 33298 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
13601 33087 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
13602 265 : && is_omp_target (stmt))
13603 : {
13604 241 : x = build_sender_ref (c, ctx);
13605 241 : gimplify_assign (x, build_fold_addr_expr (var), &ilist);
13606 : }
13607 80022 : else if (nc)
13608 : {
13609 43034 : x = build_sender_ref (nc, ctx);
13610 :
13611 43034 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13612 33057 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
13613 5172 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
13614 48206 : && TREE_CODE (TREE_TYPE (ovar)) == ARRAY_TYPE)
13615 : {
13616 425 : gcc_assert (offloaded);
13617 425 : tree avar = build_fold_addr_expr (var);
13618 425 : if (!OMP_CLAUSE_ITERATORS (c))
13619 : {
13620 425 : tree tmp = create_tmp_var (TREE_TYPE (TREE_TYPE (x)));
13621 425 : mark_addressable (tmp);
13622 425 : gimplify_assign (tmp, avar, &ilist);
13623 425 : avar = tmp;
13624 : }
13625 425 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (x)));
13626 425 : avar = build_fold_addr_expr (avar);
13627 425 : avar = lower_omp_map_iterator_expr (avar, c, stmt);
13628 425 : gimplify_assign (x, avar, &ilist);
13629 : }
13630 42609 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
13631 : {
13632 3896 : gcc_assert (is_gimple_omp_oacc (ctx->stmt));
13633 3896 : if (!omp_privatize_by_reference (var))
13634 : {
13635 3441 : if (is_gimple_reg (var)
13636 3441 : && OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
13637 2578 : suppress_warning (var);
13638 3441 : var = build_fold_addr_expr (var);
13639 : }
13640 : else
13641 455 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
13642 3896 : gimplify_assign (x, var, &ilist);
13643 : }
13644 38713 : else if (is_gimple_reg (var))
13645 : {
13646 5197 : gcc_assert (offloaded);
13647 5197 : tree avar = create_tmp_var (TREE_TYPE (var));
13648 5197 : mark_addressable (avar);
13649 5197 : enum gomp_map_kind map_kind = OMP_CLAUSE_MAP_KIND (c);
13650 5197 : if (GOMP_MAP_COPY_TO_P (map_kind)
13651 : || map_kind == GOMP_MAP_POINTER
13652 475 : || map_kind == GOMP_MAP_TO_PSET
13653 2 : || map_kind == GOMP_MAP_FORCE_DEVICEPTR)
13654 : {
13655 : /* If we need to initialize a temporary
13656 : with VAR because it is not addressable, and
13657 : the variable hasn't been initialized yet, then
13658 : we'll get a warning for the store to avar.
13659 : Don't warn in that case, the mapping might
13660 : be implicit. */
13661 5195 : suppress_warning (var, OPT_Wuninitialized);
13662 5195 : gimplify_assign (avar, var, &ilist);
13663 : }
13664 5197 : avar = build_fold_addr_expr (avar);
13665 5197 : gimplify_assign (x, avar, &ilist);
13666 5197 : if ((GOMP_MAP_COPY_FROM_P (map_kind)
13667 1404 : || map_kind == GOMP_MAP_FORCE_DEVICEPTR)
13668 8990 : && !TYPE_READONLY (TREE_TYPE (var)))
13669 : {
13670 3793 : x = unshare_expr (x);
13671 3793 : x = build_simple_mem_ref (x);
13672 3793 : gimplify_assign (var, x, &olist);
13673 : }
13674 : }
13675 : else
13676 : {
13677 : /* While MAP is handled explicitly by the FE,
13678 : for 'target update', only the identified is passed. */
13679 33516 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FROM
13680 28385 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TO)
13681 34466 : && (omp_is_allocatable_or_ptr (var)
13682 120 : && omp_check_optional_argument (var, false)))
13683 0 : var = build_fold_indirect_ref (var);
13684 33516 : else if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FROM
13685 28385 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_TO)
13686 34466 : || (!omp_is_allocatable_or_ptr (var)
13687 5961 : && !omp_check_optional_argument (var, false)))
13688 33396 : var = build_fold_addr_expr (var);
13689 33516 : gimplify_assign (x, var, &ilist);
13690 : }
13691 : }
13692 80263 : s = NULL_TREE;
13693 80263 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
13694 : {
13695 3896 : gcc_checking_assert (is_gimple_omp_oacc (ctx->stmt));
13696 3896 : s = TREE_TYPE (ovar);
13697 3896 : if (TREE_CODE (s) == REFERENCE_TYPE
13698 3896 : || omp_check_optional_argument (ovar, false))
13699 449 : s = TREE_TYPE (s);
13700 3896 : s = TYPE_SIZE_UNIT (s);
13701 : }
13702 : else
13703 76367 : s = OMP_CLAUSE_SIZE (c);
13704 80263 : if (s == NULL_TREE)
13705 155 : s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
13706 80263 : s = fold_convert (size_type_node, s);
13707 80263 : s = lower_omp_map_iterator_size (s, c, stmt);
13708 80263 : purpose = size_int (map_idx++);
13709 80263 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
13710 80263 : if (TREE_CODE (s) != INTEGER_CST)
13711 16158 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
13712 :
13713 80263 : unsigned HOST_WIDE_INT tkind, tkind_zero;
13714 80263 : switch (OMP_CLAUSE_CODE (c))
13715 : {
13716 68674 : case OMP_CLAUSE_MAP:
13717 68674 : tkind = OMP_CLAUSE_MAP_KIND (c);
13718 68674 : tkind_zero = tkind;
13719 68674 : if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c))
13720 5326 : switch (tkind)
13721 : {
13722 : case GOMP_MAP_ALLOC:
13723 : case GOMP_MAP_IF_PRESENT:
13724 : case GOMP_MAP_TO:
13725 : case GOMP_MAP_FROM:
13726 : case GOMP_MAP_TOFROM:
13727 : case GOMP_MAP_ALWAYS_TO:
13728 : case GOMP_MAP_ALWAYS_FROM:
13729 : case GOMP_MAP_ALWAYS_TOFROM:
13730 : case GOMP_MAP_ALWAYS_PRESENT_TO:
13731 : case GOMP_MAP_ALWAYS_PRESENT_FROM:
13732 : case GOMP_MAP_ALWAYS_PRESENT_TOFROM:
13733 : case GOMP_MAP_RELEASE:
13734 : case GOMP_MAP_FORCE_TO:
13735 : case GOMP_MAP_FORCE_FROM:
13736 : case GOMP_MAP_FORCE_TOFROM:
13737 : case GOMP_MAP_FORCE_PRESENT:
13738 68674 : tkind_zero = GOMP_MAP_ZERO_LEN_ARRAY_SECTION;
13739 : break;
13740 21 : case GOMP_MAP_DELETE:
13741 21 : tkind_zero = GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION;
13742 : default:
13743 : break;
13744 : }
13745 68674 : if (tkind_zero != tkind)
13746 : {
13747 5326 : if (integer_zerop (s))
13748 : tkind = tkind_zero;
13749 4594 : else if (integer_nonzerop (s))
13750 68674 : tkind_zero = tkind;
13751 : }
13752 68674 : if (tkind_zero == tkind
13753 67211 : && OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (c)
13754 68674 : && (((tkind & GOMP_MAP_FLAG_SPECIAL_BITS)
13755 5164 : & ~GOMP_MAP_IMPLICIT)
13756 : == 0))
13757 : {
13758 : /* If this is an implicit map, and the GOMP_MAP_IMPLICIT
13759 : bits are not interfered by other special bit encodings,
13760 : then turn the GOMP_IMPLICIT_BIT flag on for the runtime
13761 : to see. */
13762 5121 : tkind |= GOMP_MAP_IMPLICIT;
13763 5121 : tkind_zero = tkind;
13764 : }
13765 : break;
13766 3896 : case OMP_CLAUSE_FIRSTPRIVATE:
13767 3896 : gcc_checking_assert (is_gimple_omp_oacc (ctx->stmt));
13768 : tkind = GOMP_MAP_TO;
13769 : tkind_zero = tkind;
13770 : break;
13771 1659 : case OMP_CLAUSE_TO:
13772 1659 : tkind
13773 1659 : = (OMP_CLAUSE_MOTION_PRESENT (c)
13774 1659 : ? GOMP_MAP_ALWAYS_PRESENT_TO : GOMP_MAP_TO);
13775 : tkind_zero = tkind;
13776 : break;
13777 6034 : case OMP_CLAUSE_FROM:
13778 6034 : tkind
13779 6034 : = (OMP_CLAUSE_MOTION_PRESENT (c)
13780 6034 : ? GOMP_MAP_ALWAYS_PRESENT_FROM : GOMP_MAP_FROM);
13781 : tkind_zero = tkind;
13782 : break;
13783 0 : default:
13784 0 : gcc_unreachable ();
13785 : }
13786 68674 : gcc_checking_assert (tkind
13787 : < (HOST_WIDE_INT_C (1U) << talign_shift));
13788 80263 : gcc_checking_assert (tkind_zero
13789 : < (HOST_WIDE_INT_C (1U) << talign_shift));
13790 80263 : talign = ceil_log2 (talign);
13791 80263 : tkind |= talign << talign_shift;
13792 80263 : tkind_zero |= talign << talign_shift;
13793 80263 : gcc_checking_assert (tkind
13794 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
13795 80263 : gcc_checking_assert (tkind_zero
13796 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
13797 80263 : if (tkind == tkind_zero)
13798 78800 : x = build_int_cstu (tkind_type, tkind);
13799 : else
13800 : {
13801 1463 : TREE_STATIC (TREE_VEC_ELT (t, 2)) = 0;
13802 1463 : x = build3 (COND_EXPR, tkind_type,
13803 : fold_build2 (EQ_EXPR, boolean_type_node,
13804 : unshare_expr (s), size_zero_node),
13805 1463 : build_int_cstu (tkind_type, tkind_zero),
13806 1463 : build_int_cstu (tkind_type, tkind));
13807 : }
13808 80263 : CONSTRUCTOR_APPEND_ELT (vkind, purpose, x);
13809 80263 : if (nc && nc != c)
13810 0 : c = nc;
13811 : break;
13812 :
13813 15467 : case OMP_CLAUSE_FIRSTPRIVATE:
13814 15467 : omp_has_device_addr_descr:
13815 15467 : if (is_gimple_omp_oacc (ctx->stmt))
13816 3896 : goto oacc_firstprivate_map;
13817 11571 : ovar = OMP_CLAUSE_DECL (c);
13818 11571 : if (omp_privatize_by_reference (ovar))
13819 812 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
13820 : else
13821 10759 : talign = DECL_ALIGN_UNIT (ovar);
13822 11571 : var = lookup_decl_in_outer_ctx (ovar, ctx);
13823 11571 : x = build_sender_ref (ovar, ctx);
13824 11571 : tkind = GOMP_MAP_FIRSTPRIVATE;
13825 11571 : type = TREE_TYPE (ovar);
13826 11571 : if (omp_privatize_by_reference (ovar))
13827 812 : type = TREE_TYPE (type);
13828 11571 : if ((INTEGRAL_TYPE_P (type)
13829 11148 : && TYPE_PRECISION (type) <= POINTER_SIZE)
13830 11581 : || TREE_CODE (type) == POINTER_TYPE)
13831 : {
13832 11297 : tkind = GOMP_MAP_FIRSTPRIVATE_INT;
13833 11297 : tree t = var;
13834 11297 : if (omp_privatize_by_reference (var))
13835 757 : t = build_simple_mem_ref (var);
13836 10540 : else if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
13837 9331 : suppress_warning (var);
13838 11297 : if (TREE_CODE (type) != POINTER_TYPE)
13839 11138 : t = fold_convert (pointer_sized_int_node, t);
13840 11297 : t = fold_convert (TREE_TYPE (x), t);
13841 11297 : gimplify_assign (x, t, &ilist);
13842 : }
13843 274 : else if (omp_privatize_by_reference (var))
13844 55 : gimplify_assign (x, var, &ilist);
13845 219 : else if (is_gimple_reg (var))
13846 : {
13847 51 : tree avar = create_tmp_var (TREE_TYPE (var));
13848 51 : mark_addressable (avar);
13849 51 : if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
13850 44 : suppress_warning (var);
13851 51 : gimplify_assign (avar, var, &ilist);
13852 51 : avar = build_fold_addr_expr (avar);
13853 51 : gimplify_assign (x, avar, &ilist);
13854 : }
13855 : else
13856 : {
13857 168 : var = build_fold_addr_expr (var);
13858 168 : gimplify_assign (x, var, &ilist);
13859 : }
13860 274 : if (tkind == GOMP_MAP_FIRSTPRIVATE_INT)
13861 11297 : s = size_int (0);
13862 274 : else if (omp_privatize_by_reference (ovar))
13863 55 : s = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
13864 : else
13865 219 : s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
13866 11571 : s = fold_convert (size_type_node, s);
13867 11571 : purpose = size_int (map_idx++);
13868 11571 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
13869 11571 : if (TREE_CODE (s) != INTEGER_CST)
13870 39 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
13871 :
13872 11571 : gcc_checking_assert (tkind
13873 : < (HOST_WIDE_INT_C (1U) << talign_shift));
13874 11571 : talign = ceil_log2 (talign);
13875 11571 : tkind |= talign << talign_shift;
13876 11571 : gcc_checking_assert (tkind
13877 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
13878 11571 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
13879 : build_int_cstu (tkind_type, tkind));
13880 : /* Fortran array descriptors: firstprivate of data + attach. */
13881 11571 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
13882 11552 : && lang_hooks.decls.omp_array_data (ovar, true)
13883 11602 : && lang_hooks.decls.omp_array_data_privatize (ovar))
13884 : {
13885 23 : tree not_null_lb, null_lb, after_lb;
13886 23 : tree var1, var2, size1, size2;
13887 23 : tree present = omp_check_optional_argument (ovar, true);
13888 23 : if (present)
13889 : {
13890 1 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
13891 1 : not_null_lb = create_artificial_label (clause_loc);
13892 1 : null_lb = create_artificial_label (clause_loc);
13893 1 : after_lb = create_artificial_label (clause_loc);
13894 1 : gimple_seq seq = NULL;
13895 1 : present = force_gimple_operand (present, &seq, true,
13896 : NULL_TREE);
13897 1 : gimple_seq_add_seq (&ilist, seq);
13898 1 : gimple_seq_add_stmt (&ilist,
13899 1 : gimple_build_cond_from_tree (present,
13900 : not_null_lb, null_lb));
13901 1 : gimple_seq_add_stmt (&ilist,
13902 1 : gimple_build_label (not_null_lb));
13903 : }
13904 23 : var1 = lang_hooks.decls.omp_array_data (var, false);
13905 23 : size1 = lang_hooks.decls.omp_array_size (var, &ilist);
13906 23 : var2 = build_fold_addr_expr (x);
13907 23 : if (!POINTER_TYPE_P (TREE_TYPE (var)))
13908 0 : var = build_fold_addr_expr (var);
13909 23 : size2 = fold_build2 (POINTER_DIFF_EXPR, ssizetype,
13910 : build_fold_addr_expr (var1), var);
13911 23 : size2 = fold_convert (sizetype, size2);
13912 23 : if (present)
13913 : {
13914 1 : tree tmp = create_tmp_var (TREE_TYPE (var1));
13915 1 : gimplify_assign (tmp, var1, &ilist);
13916 1 : var1 = tmp;
13917 1 : tmp = create_tmp_var (TREE_TYPE (var2));
13918 1 : gimplify_assign (tmp, var2, &ilist);
13919 1 : var2 = tmp;
13920 1 : tmp = create_tmp_var (TREE_TYPE (size1));
13921 1 : gimplify_assign (tmp, size1, &ilist);
13922 1 : size1 = tmp;
13923 1 : tmp = create_tmp_var (TREE_TYPE (size2));
13924 1 : gimplify_assign (tmp, size2, &ilist);
13925 1 : size2 = tmp;
13926 1 : gimple_seq_add_stmt (&ilist, gimple_build_goto (after_lb));
13927 1 : gimple_seq_add_stmt (&ilist, gimple_build_label (null_lb));
13928 1 : gimplify_assign (var1, null_pointer_node, &ilist);
13929 1 : gimplify_assign (var2, null_pointer_node, &ilist);
13930 1 : gimplify_assign (size1, size_zero_node, &ilist);
13931 1 : gimplify_assign (size2, size_zero_node, &ilist);
13932 1 : gimple_seq_add_stmt (&ilist, gimple_build_label (after_lb));
13933 : }
13934 23 : x = build_sender_ref ((splay_tree_key) &DECL_NAME (ovar), ctx);
13935 23 : gimplify_assign (x, var1, &ilist);
13936 23 : tkind = GOMP_MAP_FIRSTPRIVATE;
13937 23 : talign = DECL_ALIGN_UNIT (ovar);
13938 23 : talign = ceil_log2 (talign);
13939 23 : tkind |= talign << talign_shift;
13940 23 : gcc_checking_assert (tkind
13941 : <= tree_to_uhwi (
13942 : TYPE_MAX_VALUE (tkind_type)));
13943 23 : purpose = size_int (map_idx++);
13944 23 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, size1);
13945 23 : if (TREE_CODE (size1) != INTEGER_CST)
13946 23 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
13947 23 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
13948 : build_int_cstu (tkind_type, tkind));
13949 23 : x = build_sender_ref ((splay_tree_key) &DECL_UID (ovar), ctx);
13950 23 : gimplify_assign (x, var2, &ilist);
13951 23 : tkind = GOMP_MAP_ATTACH;
13952 23 : purpose = size_int (map_idx++);
13953 23 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, size2);
13954 23 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
13955 : build_int_cstu (tkind_type, tkind));
13956 : }
13957 : break;
13958 :
13959 2431 : case OMP_CLAUSE_USE_DEVICE_PTR:
13960 2431 : case OMP_CLAUSE_USE_DEVICE_ADDR:
13961 2431 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
13962 2431 : case OMP_CLAUSE_IS_DEVICE_PTR:
13963 2431 : ovar = OMP_CLAUSE_DECL (c);
13964 2431 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
13965 : {
13966 236 : if (lang_hooks.decls.omp_array_data (ovar, true))
13967 19 : goto omp_has_device_addr_descr;
13968 234 : while (TREE_CODE (ovar) == INDIRECT_REF
13969 234 : || TREE_CODE (ovar) == ARRAY_REF)
13970 17 : ovar = TREE_OPERAND (ovar, 0);
13971 : }
13972 2412 : var = lookup_decl_in_outer_ctx (ovar, ctx);
13973 :
13974 2412 : if (lang_hooks.decls.omp_array_data (ovar, true))
13975 : {
13976 601 : tkind = ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
13977 601 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
13978 601 : ? GOMP_MAP_USE_DEVICE_PTR : GOMP_MAP_FIRSTPRIVATE_INT);
13979 601 : x = build_sender_ref ((splay_tree_key) &DECL_NAME (ovar), ctx);
13980 : }
13981 1811 : else if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
13982 1811 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
13983 : {
13984 1453 : tkind = GOMP_MAP_USE_DEVICE_PTR;
13985 1453 : x = build_sender_ref ((splay_tree_key) &DECL_UID (ovar), ctx);
13986 : }
13987 : else
13988 : {
13989 358 : tkind = GOMP_MAP_FIRSTPRIVATE_INT;
13990 358 : x = build_sender_ref (ovar, ctx);
13991 : }
13992 :
13993 2412 : if (is_gimple_omp_oacc (ctx->stmt))
13994 : {
13995 147 : gcc_assert (tkind == GOMP_MAP_USE_DEVICE_PTR);
13996 :
13997 147 : if (OMP_CLAUSE_USE_DEVICE_PTR_IF_PRESENT (c))
13998 57 : tkind = GOMP_MAP_USE_DEVICE_PTR_IF_PRESENT;
13999 : }
14000 :
14001 2412 : type = TREE_TYPE (ovar);
14002 2412 : if (lang_hooks.decls.omp_array_data (ovar, true))
14003 601 : var = lang_hooks.decls.omp_array_data (var, false);
14004 1811 : else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
14005 543 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
14006 1485 : && !omp_privatize_by_reference (ovar)
14007 512 : && !omp_is_allocatable_or_ptr (ovar))
14008 3249 : || TREE_CODE (type) == ARRAY_TYPE)
14009 401 : var = build_fold_addr_expr (var);
14010 : else
14011 : {
14012 1410 : if (omp_privatize_by_reference (ovar)
14013 370 : || omp_check_optional_argument (ovar, false)
14014 1780 : || omp_is_allocatable_or_ptr (ovar))
14015 : {
14016 1213 : type = TREE_TYPE (type);
14017 1213 : if (POINTER_TYPE_P (type)
14018 1213 : && TREE_CODE (type) != ARRAY_TYPE
14019 1572 : && ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_USE_DEVICE_ADDR
14020 36 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
14021 28 : && !omp_is_allocatable_or_ptr (ovar))
14022 337 : || (omp_privatize_by_reference (ovar)
14023 337 : && omp_is_allocatable_or_ptr (ovar))))
14024 357 : var = build_simple_mem_ref (var);
14025 1213 : var = fold_convert (TREE_TYPE (x), var);
14026 : }
14027 : }
14028 2412 : tree present;
14029 2412 : present = omp_check_optional_argument (ovar, true);
14030 2412 : if (present)
14031 : {
14032 878 : tree null_label = create_artificial_label (UNKNOWN_LOCATION);
14033 878 : tree notnull_label = create_artificial_label (UNKNOWN_LOCATION);
14034 878 : tree opt_arg_label = create_artificial_label (UNKNOWN_LOCATION);
14035 878 : tree new_x = unshare_expr (x);
14036 878 : gimplify_expr (&present, &ilist, NULL, is_gimple_val,
14037 : fb_rvalue);
14038 878 : gcond *cond = gimple_build_cond_from_tree (present,
14039 : notnull_label,
14040 : null_label);
14041 878 : gimple_seq_add_stmt (&ilist, cond);
14042 878 : gimple_seq_add_stmt (&ilist, gimple_build_label (null_label));
14043 878 : gimplify_assign (new_x, null_pointer_node, &ilist);
14044 878 : gimple_seq_add_stmt (&ilist, gimple_build_goto (opt_arg_label));
14045 878 : gimple_seq_add_stmt (&ilist,
14046 878 : gimple_build_label (notnull_label));
14047 878 : gimplify_assign (x, var, &ilist);
14048 878 : gimple_seq_add_stmt (&ilist,
14049 878 : gimple_build_label (opt_arg_label));
14050 : }
14051 : else
14052 1534 : gimplify_assign (x, var, &ilist);
14053 2412 : s = size_int (0);
14054 2412 : purpose = size_int (map_idx++);
14055 2412 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
14056 2412 : gcc_checking_assert (tkind
14057 : < (HOST_WIDE_INT_C (1U) << talign_shift));
14058 2412 : gcc_checking_assert (tkind
14059 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
14060 2412 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
14061 : build_int_cstu (tkind_type, tkind));
14062 2412 : break;
14063 :
14064 86 : case OMP_CLAUSE_USES_ALLOCATORS:
14065 86 : tree allocator = OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c);
14066 86 : tree memspace = OMP_CLAUSE_USES_ALLOCATORS_MEMSPACE (c);
14067 86 : tree traits = OMP_CLAUSE_USES_ALLOCATORS_TRAITS (c);
14068 :
14069 86 : tree ntraits, traits_var;
14070 86 : if (traits == NULL_TREE)
14071 : {
14072 29 : ntraits = integer_zero_node;
14073 29 : traits_var = null_pointer_node;
14074 : }
14075 : else
14076 : {
14077 57 : location_t loc = OMP_CLAUSE_LOCATION (c);
14078 57 : ntraits = array_type_nelts_top (TREE_TYPE (traits));
14079 57 : traits_var = build_fold_addr_expr_loc (loc, traits);
14080 : }
14081 :
14082 86 : if (memspace == NULL_TREE)
14083 61 : memspace = build_int_cst (pointer_sized_int_node, 0);
14084 : else
14085 25 : memspace = fold_convert (pointer_sized_int_node, memspace);
14086 :
14087 86 : tree arr_type
14088 86 : = build_array_type_nelts (pointer_sized_int_node, 3);
14089 86 : tree uses_allocators_descr
14090 86 : = create_tmp_var (arr_type, "uses_allocator_descr");
14091 86 : tree ua_descr[3];
14092 430 : for (int i = 0; i < 3; i++)
14093 258 : ua_descr[i] = build4 (ARRAY_REF, pointer_sized_int_node,
14094 : uses_allocators_descr,
14095 258 : build_int_cst (size_type_node, i),
14096 : NULL_TREE, NULL_TREE);
14097 86 : gimplify_assign (ua_descr[0],
14098 : fold_convert (pointer_sized_int_node, memspace),
14099 : &ilist);
14100 86 : gimplify_assign (ua_descr[1],
14101 : fold_convert (pointer_sized_int_node, ntraits),
14102 : &ilist);
14103 86 : gimplify_assign (ua_descr[2],
14104 : fold_convert (pointer_sized_int_node, traits_var),
14105 : &ilist);
14106 :
14107 86 : x = build_sender_ref (allocator, ctx);
14108 86 : tree ptr = build_fold_addr_expr (uses_allocators_descr);
14109 86 : gimplify_assign (x, fold_convert (TREE_TYPE (x), ptr), &ilist);
14110 :
14111 86 : s = size_int (0);
14112 86 : purpose = size_int (map_idx++);
14113 86 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
14114 86 : tkind = GOMP_MAP_USES_ALLOCATORS;
14115 86 : gcc_checking_assert (tkind
14116 : < (HOST_WIDE_INT_C (1U) << talign_shift));
14117 86 : talign = TYPE_ALIGN_UNIT (pointer_sized_int_node);
14118 86 : talign = ceil_log2 (talign);
14119 86 : tkind |= talign << talign_shift;
14120 86 : gcc_checking_assert (tkind
14121 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
14122 86 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
14123 : build_int_cstu (tkind_type, tkind));
14124 86 : break;
14125 : }
14126 :
14127 32403 : gcc_assert (map_idx == map_cnt);
14128 :
14129 32403 : if (!deep_map_cnt)
14130 : {
14131 32259 : DECL_INITIAL (TREE_VEC_ELT (t, 1))
14132 32259 : = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), vsize);
14133 32259 : DECL_INITIAL (TREE_VEC_ELT (t, 2))
14134 64518 : = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 2)), vkind);
14135 : }
14136 97209 : for (int i = 1; i <= 2; i++)
14137 64806 : if (deep_map_cnt || !TREE_STATIC (TREE_VEC_ELT (t, i)))
14138 : {
14139 9667 : tree tmp = TREE_VEC_ELT (t, i);
14140 9667 : if (deep_map_cnt)
14141 : {
14142 288 : const char *prefix = (i == 1 ? ".omp_data_sizes0"
14143 : : ".omp_data_kinds0");
14144 144 : tree type = (i == 1) ? size_type_node : tkind_type;
14145 288 : type = build_array_type_nelts (type, map_cnt);
14146 288 : tree var = create_tmp_var (type, prefix);
14147 288 : DECL_NAMELESS (var) = 1;
14148 288 : TREE_ADDRESSABLE (var) = 1;
14149 288 : TREE_STATIC (var) = TREE_STATIC (tmp);
14150 288 : DECL_INITIAL (var) = build_constructor (type, i == 1
14151 : ? vsize : vkind);
14152 288 : tmp = var;
14153 288 : TREE_STATIC (TREE_VEC_ELT (t, i)) = 0;
14154 : }
14155 :
14156 9667 : gimple_seq initlist = NULL;
14157 9667 : force_gimple_operand (build1 (DECL_EXPR, void_type_node, tmp),
14158 : &initlist, true, NULL_TREE);
14159 9667 : gimple_seq_add_seq (&ilist, initlist);
14160 :
14161 9667 : if (deep_map_cnt)
14162 : {
14163 288 : tree tmp2;
14164 288 : tree call = builtin_decl_explicit (BUILT_IN_MEMCPY);
14165 288 : tmp2 = TYPE_SIZE_UNIT (TREE_TYPE (tmp));
14166 576 : call = build_call_expr_loc (input_location, call, 3,
14167 288 : TREE_VEC_ELT (t, i),
14168 : build_fold_addr_expr (tmp), tmp2);
14169 288 : gimplify_and_add (call, &ilist);
14170 : }
14171 :
14172 9667 : if (!TREE_STATIC (tmp))
14173 : {
14174 9449 : tree clobber = build_clobber (TREE_TYPE (tmp));
14175 9449 : gimple_seq_add_stmt (&olist,
14176 9449 : gimple_build_assign (tmp, clobber));
14177 : }
14178 9667 : if (deep_map_cnt)
14179 : {
14180 288 : tmp = TREE_VEC_ELT (t, i);
14181 288 : tree call = builtin_decl_explicit (BUILT_IN_FREE);
14182 288 : call = build_call_expr_loc (input_location, call, 1, tmp);
14183 288 : gimplify_and_add (call, &olist);
14184 288 : tree clobber = build_clobber (TREE_TYPE (tmp));
14185 288 : gimple_seq_add_stmt (&olist,
14186 288 : gimple_build_assign (tmp, clobber));
14187 : }
14188 : }
14189 55139 : else if (omp_maybe_offloaded_ctx (ctx->outer))
14190 : {
14191 3296 : tree id = get_identifier ("omp declare target");
14192 3296 : tree decl = TREE_VEC_ELT (t, i);
14193 3296 : DECL_ATTRIBUTES (decl)
14194 3296 : = tree_cons (id, NULL_TREE, DECL_ATTRIBUTES (decl));
14195 3296 : varpool_node *node = varpool_node::get (decl);
14196 3296 : if (node)
14197 : {
14198 0 : node->offloadable = 1;
14199 0 : if (ENABLE_OFFLOADING)
14200 : {
14201 : g->have_offload = true;
14202 : vec_safe_push (offload_vars, t);
14203 : }
14204 : }
14205 : }
14206 :
14207 32403 : if (deep_map_cnt)
14208 : {
14209 144 : tree call = builtin_decl_explicit (BUILT_IN_FREE);
14210 144 : call = build_call_expr_loc (input_location, call, 1,
14211 144 : TREE_VEC_ELT (t, 0));
14212 144 : gimplify_and_add (call, &olist);
14213 :
14214 144 : gimplify_expr (&TREE_VEC_ELT (t, 1), &ilist, NULL, is_gimple_val,
14215 : fb_rvalue);
14216 : }
14217 :
14218 32403 : tree clobber = build_clobber (TREE_TYPE (ctx->sender_decl));
14219 32403 : gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
14220 : clobber));
14221 : }
14222 :
14223 : /* Once all the expansions are done, sequence all the different
14224 : fragments inside gimple_omp_body. */
14225 :
14226 37314 : new_body = NULL;
14227 :
14228 37314 : if (offloaded
14229 21311 : && ctx->record_type)
14230 : {
14231 16633 : t = ctx->sender_decl;
14232 16633 : if (!deep_map_cnt)
14233 16526 : t = build_fold_addr_expr_loc (loc, t);
14234 : /* fixup_child_record_type might have changed receiver_decl's type. */
14235 16633 : t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
14236 16633 : if (!AGGREGATE_TYPE_P (TREE_TYPE (ctx->sender_decl)))
14237 107 : gimplify_assign (ctx->receiver_decl, t, &new_body);
14238 : else
14239 16526 : gimple_seq_add_stmt (&new_body,
14240 16526 : gimple_build_assign (ctx->receiver_decl, t));
14241 : }
14242 37314 : gimple_seq_add_seq (&new_body, fplist);
14243 :
14244 37314 : if (offloaded || data_region)
14245 : {
14246 141697 : tree prev = NULL_TREE;
14247 : bool by_ref;
14248 141697 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
14249 116362 : switch (OMP_CLAUSE_CODE (c))
14250 : {
14251 : tree var, x, new_var, *allocate_ptr;
14252 : default:
14253 : break;
14254 15467 : case OMP_CLAUSE_FIRSTPRIVATE:
14255 15467 : omp_firstprivatize_data_region:
14256 15467 : if (is_gimple_omp_oacc (ctx->stmt))
14257 : break;
14258 11571 : var = OMP_CLAUSE_DECL (c);
14259 11571 : new_var = lookup_decl (var, ctx);
14260 11571 : allocate_ptr = alloc_map.get (new_var);
14261 11571 : by_ref = omp_privatize_by_reference (var);
14262 11571 : if (allocate_ptr)
14263 : {
14264 68 : if (is_variable_sized (var, by_ref))
14265 : /* Handle this in the next pass when the size is
14266 : available. */
14267 : break;
14268 :
14269 53 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14270 53 : gcc_assert (allocate_seq);
14271 53 : gimple_seq_add_seq (&new_body, *allocate_seq);
14272 :
14273 53 : if (by_ref)
14274 : {
14275 4 : x = fold_convert (TREE_TYPE (new_var), *allocate_ptr);
14276 4 : gimplify_assign (new_var, x, &new_body);
14277 4 : new_var = build_fold_indirect_ref (new_var);
14278 : }
14279 : else
14280 49 : new_var = build_simple_mem_ref (*allocate_ptr);
14281 : }
14282 11556 : if (by_ref || is_gimple_reg_type (TREE_TYPE (var)))
14283 : {
14284 11402 : tree type;
14285 11402 : type = TREE_TYPE (var);
14286 11402 : if (by_ref)
14287 799 : type = TREE_TYPE (type);
14288 11402 : if ((INTEGRAL_TYPE_P (type)
14289 11148 : && TYPE_PRECISION (type) <= POINTER_SIZE)
14290 11412 : || TREE_CODE (type) == POINTER_TYPE)
14291 : {
14292 11297 : x = build_receiver_ref (var, false, ctx);
14293 11297 : if (TREE_CODE (type) != POINTER_TYPE)
14294 11138 : x = fold_convert (pointer_sized_int_node, x);
14295 11297 : x = fold_convert (type, x);
14296 11297 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14297 : fb_rvalue);
14298 11297 : if (by_ref && !allocate_ptr)
14299 : {
14300 756 : tree v = create_tmp_var_raw (type, get_name (var));
14301 756 : gimple_add_tmp_var (v);
14302 756 : TREE_ADDRESSABLE (v) = 1;
14303 756 : gimple_seq_add_stmt (&new_body,
14304 756 : gimple_build_assign (v, x));
14305 756 : x = build_fold_addr_expr (v);
14306 : }
14307 11297 : gimplify_assign (new_var, x, &new_body);
14308 : }
14309 : else
14310 : {
14311 105 : x = build_receiver_ref (var, allocate_ptr || !by_ref, ctx);
14312 105 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14313 : fb_rvalue);
14314 105 : gimplify_assign (new_var, x, &new_body);
14315 : }
14316 : }
14317 154 : else if (is_variable_sized (var))
14318 : {
14319 8 : tree pvar = DECL_VALUE_EXPR (var);
14320 8 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14321 8 : pvar = TREE_OPERAND (pvar, 0);
14322 8 : gcc_assert (DECL_P (pvar));
14323 8 : tree new_var = lookup_decl (pvar, ctx);
14324 8 : x = build_receiver_ref (var, false, ctx);
14325 8 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14326 8 : gimple_seq_add_stmt (&new_body,
14327 8 : gimple_build_assign (new_var, x));
14328 : }
14329 146 : else if (allocate_ptr)
14330 : {
14331 22 : x = build_receiver_ref (var, true, ctx);
14332 22 : new_var = unshare_expr (new_var);
14333 22 : x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
14334 22 : gimplify_and_add (x, &new_body);
14335 : }
14336 : break;
14337 228 : case OMP_CLAUSE_PRIVATE:
14338 228 : if (is_gimple_omp_oacc (ctx->stmt))
14339 : break;
14340 148 : var = OMP_CLAUSE_DECL (c);
14341 148 : new_var = lookup_decl (var, ctx);
14342 148 : allocate_ptr = alloc_map.get (new_var);
14343 148 : by_ref = omp_privatize_by_reference (var);
14344 148 : if (allocate_ptr)
14345 : {
14346 57 : if (is_variable_sized (var, by_ref))
14347 : /* Handle this in the next pass when the size is
14348 : available. */
14349 : break;
14350 :
14351 45 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14352 45 : gcc_assert (allocate_seq);
14353 45 : gimple_seq_add_seq (&new_body, *allocate_seq);
14354 :
14355 45 : if (!by_ref)
14356 44 : new_var = build_simple_mem_ref (*allocate_ptr);
14357 : }
14358 135 : if (by_ref)
14359 : {
14360 12 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14361 12 : if (!allocate_ptr)
14362 : {
14363 11 : x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
14364 11 : if (TREE_CONSTANT (x))
14365 : {
14366 3 : x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
14367 : get_name (var));
14368 3 : gimple_add_tmp_var (x);
14369 3 : TREE_ADDRESSABLE (x) = 1;
14370 3 : x = build_fold_addr_expr_loc (clause_loc, x);
14371 : }
14372 : else
14373 : break;
14374 :
14375 3 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
14376 3 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14377 : fb_rvalue);
14378 : }
14379 : else
14380 1 : x = *allocate_ptr;
14381 :
14382 4 : gimple_seq_add_stmt (&new_body,
14383 4 : gimple_build_assign (new_var, x));
14384 : }
14385 : break;
14386 2431 : case OMP_CLAUSE_USE_DEVICE_PTR:
14387 2431 : case OMP_CLAUSE_USE_DEVICE_ADDR:
14388 2431 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
14389 2431 : case OMP_CLAUSE_IS_DEVICE_PTR:
14390 2431 : gimple_seq assign_body;
14391 2431 : bool is_array_data;
14392 2431 : bool do_optional_check;
14393 2431 : assign_body = NULL;
14394 2431 : do_optional_check = false;
14395 2431 : var = OMP_CLAUSE_DECL (c);
14396 2431 : is_array_data = lang_hooks.decls.omp_array_data (var, true) != NULL;
14397 2431 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR && is_array_data)
14398 19 : goto omp_firstprivatize_data_region;
14399 :
14400 2412 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
14401 2412 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
14402 4108 : x = build_sender_ref (is_array_data
14403 601 : ? (splay_tree_key) &DECL_NAME (var)
14404 1453 : : (splay_tree_key) &DECL_UID (var), ctx);
14405 : else
14406 : {
14407 358 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
14408 : {
14409 234 : while (TREE_CODE (var) == INDIRECT_REF
14410 234 : || TREE_CODE (var) == ARRAY_REF)
14411 17 : var = TREE_OPERAND (var, 0);
14412 : }
14413 358 : x = build_receiver_ref (var, false, ctx);
14414 : }
14415 :
14416 2412 : if (is_array_data)
14417 : {
14418 601 : bool is_ref = omp_privatize_by_reference (var);
14419 601 : do_optional_check = true;
14420 : /* First, we copy the descriptor data from the host; then
14421 : we update its data to point to the target address. */
14422 601 : new_var = lookup_decl (var, ctx);
14423 601 : new_var = DECL_VALUE_EXPR (new_var);
14424 601 : tree v = new_var;
14425 601 : tree v2 = var;
14426 601 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_PTR
14427 601 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR)
14428 601 : v2 = maybe_lookup_decl_in_outer_ctx (var, ctx);
14429 :
14430 601 : if (is_ref)
14431 : {
14432 396 : v2 = build_fold_indirect_ref (v2);
14433 396 : v = create_tmp_var_raw (TREE_TYPE (v2), get_name (var));
14434 396 : gimple_add_tmp_var (v);
14435 396 : TREE_ADDRESSABLE (v) = 1;
14436 396 : gimplify_assign (v, v2, &assign_body);
14437 396 : tree rhs = build_fold_addr_expr (v);
14438 396 : gimple_seq_add_stmt (&assign_body,
14439 396 : gimple_build_assign (new_var, rhs));
14440 : }
14441 : else
14442 205 : gimplify_assign (new_var, v2, &assign_body);
14443 :
14444 601 : v2 = lang_hooks.decls.omp_array_data (unshare_expr (v), false);
14445 601 : gcc_assert (v2);
14446 601 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14447 1202 : gimple_seq_add_stmt (&assign_body,
14448 601 : gimple_build_assign (v2, x));
14449 : }
14450 1811 : else if (is_variable_sized (var))
14451 : {
14452 12 : tree pvar = DECL_VALUE_EXPR (var);
14453 12 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14454 12 : pvar = TREE_OPERAND (pvar, 0);
14455 12 : gcc_assert (DECL_P (pvar));
14456 12 : new_var = lookup_decl (pvar, ctx);
14457 12 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14458 24 : gimple_seq_add_stmt (&assign_body,
14459 12 : gimple_build_assign (new_var, x));
14460 : }
14461 1799 : else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
14462 537 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
14463 1477 : && !omp_privatize_by_reference (var)
14464 504 : && !omp_is_allocatable_or_ptr (var))
14465 3233 : || TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
14466 : {
14467 389 : new_var = lookup_decl (var, ctx);
14468 389 : new_var = DECL_VALUE_EXPR (new_var);
14469 389 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
14470 389 : new_var = TREE_OPERAND (new_var, 0);
14471 389 : gcc_assert (DECL_P (new_var));
14472 389 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14473 778 : gimple_seq_add_stmt (&assign_body,
14474 389 : gimple_build_assign (new_var, x));
14475 : }
14476 : else
14477 : {
14478 1410 : tree type = TREE_TYPE (var);
14479 1410 : new_var = lookup_decl (var, ctx);
14480 1410 : if (omp_privatize_by_reference (var))
14481 : {
14482 1040 : type = TREE_TYPE (type);
14483 1040 : if (POINTER_TYPE_P (type)
14484 1040 : && TREE_CODE (type) != ARRAY_TYPE
14485 1399 : && ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_USE_DEVICE_ADDR
14486 36 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
14487 331 : || (omp_privatize_by_reference (var)
14488 331 : && omp_is_allocatable_or_ptr (var))))
14489 : {
14490 357 : tree v = create_tmp_var_raw (type, get_name (var));
14491 357 : gimple_add_tmp_var (v);
14492 357 : TREE_ADDRESSABLE (v) = 1;
14493 357 : x = fold_convert (type, x);
14494 357 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val,
14495 : fb_rvalue);
14496 357 : gimple_seq_add_stmt (&assign_body,
14497 357 : gimple_build_assign (v, x));
14498 357 : x = build_fold_addr_expr (v);
14499 357 : do_optional_check = true;
14500 : }
14501 : }
14502 1410 : new_var = DECL_VALUE_EXPR (new_var);
14503 1410 : x = fold_convert (TREE_TYPE (new_var), x);
14504 1410 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14505 1410 : gimple_seq_add_stmt (&assign_body,
14506 1410 : gimple_build_assign (new_var, x));
14507 : }
14508 2412 : tree present;
14509 1002 : present = ((do_optional_check
14510 958 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
14511 952 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR)
14512 2343 : ? omp_check_optional_argument (OMP_CLAUSE_DECL (c), true)
14513 : : NULL_TREE);
14514 2412 : if (present)
14515 : {
14516 480 : tree null_label = create_artificial_label (UNKNOWN_LOCATION);
14517 480 : tree notnull_label = create_artificial_label (UNKNOWN_LOCATION);
14518 480 : tree opt_arg_label = create_artificial_label (UNKNOWN_LOCATION);
14519 480 : glabel *null_glabel = gimple_build_label (null_label);
14520 480 : glabel *notnull_glabel = gimple_build_label (notnull_label);
14521 480 : ggoto *opt_arg_ggoto = gimple_build_goto (opt_arg_label);
14522 480 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14523 : fb_rvalue);
14524 480 : gimplify_expr (&present, &new_body, NULL, is_gimple_val,
14525 : fb_rvalue);
14526 480 : gcond *cond = gimple_build_cond_from_tree (present,
14527 : notnull_label,
14528 : null_label);
14529 480 : gimple_seq_add_stmt (&new_body, cond);
14530 480 : gimple_seq_add_stmt (&new_body, null_glabel);
14531 480 : gimplify_assign (new_var, null_pointer_node, &new_body);
14532 480 : gimple_seq_add_stmt (&new_body, opt_arg_ggoto);
14533 480 : gimple_seq_add_stmt (&new_body, notnull_glabel);
14534 480 : gimple_seq_add_seq (&new_body, assign_body);
14535 480 : gimple_seq_add_stmt (&new_body,
14536 480 : gimple_build_label (opt_arg_label));
14537 : }
14538 : else
14539 1932 : gimple_seq_add_seq (&new_body, assign_body);
14540 : break;
14541 : }
14542 : /* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass,
14543 : so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
14544 : are already handled. Similarly OMP_CLAUSE_PRIVATE for VLAs
14545 : or references to VLAs. */
14546 141697 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
14547 116362 : switch (OMP_CLAUSE_CODE (c))
14548 : {
14549 : tree var, new_var, *allocate_ptr;
14550 : default:
14551 100938 : break;
14552 63606 : case OMP_CLAUSE_MAP:
14553 63606 : if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
14554 63606 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
14555 : {
14556 3672 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14557 3672 : poly_int64 offset = 0;
14558 3672 : gcc_assert (prev);
14559 3672 : var = OMP_CLAUSE_DECL (c);
14560 3672 : if (DECL_P (var)
14561 3672 : && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
14562 1127 : && is_global_var (maybe_lookup_decl_in_outer_ctx (var,
14563 : ctx))
14564 3770 : && varpool_node::get_create (var)->offloadable)
14565 : break;
14566 3670 : if (TREE_CODE (var) == INDIRECT_REF
14567 3670 : && TREE_CODE (TREE_OPERAND (var, 0)) == COMPONENT_REF)
14568 0 : var = TREE_OPERAND (var, 0);
14569 3670 : if (TREE_CODE (var) == COMPONENT_REF)
14570 : {
14571 0 : var = get_addr_base_and_unit_offset (var, &offset);
14572 0 : gcc_assert (var != NULL_TREE && DECL_P (var));
14573 : }
14574 3670 : else if (DECL_SIZE (var)
14575 3670 : && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST)
14576 : {
14577 712 : tree var2 = DECL_VALUE_EXPR (var);
14578 712 : gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
14579 712 : var2 = TREE_OPERAND (var2, 0);
14580 712 : gcc_assert (DECL_P (var2));
14581 : var = var2;
14582 : }
14583 3670 : tree new_var = lookup_decl (var, ctx), x;
14584 3670 : tree type = TREE_TYPE (new_var);
14585 3670 : bool is_ref;
14586 3670 : if (TREE_CODE (OMP_CLAUSE_DECL (c)) == INDIRECT_REF
14587 3670 : && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c), 0))
14588 : == COMPONENT_REF))
14589 : {
14590 0 : type = TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c), 0));
14591 0 : is_ref = true;
14592 0 : new_var = build2 (MEM_REF, type,
14593 : build_fold_addr_expr (new_var),
14594 : build_int_cst (build_pointer_type (type),
14595 : offset));
14596 : }
14597 3670 : else if (TREE_CODE (OMP_CLAUSE_DECL (c)) == COMPONENT_REF)
14598 : {
14599 0 : type = TREE_TYPE (OMP_CLAUSE_DECL (c));
14600 0 : is_ref = TREE_CODE (type) == REFERENCE_TYPE;
14601 0 : new_var = build2 (MEM_REF, type,
14602 : build_fold_addr_expr (new_var),
14603 : build_int_cst (build_pointer_type (type),
14604 : offset));
14605 : }
14606 : else
14607 3670 : is_ref = omp_privatize_by_reference (var);
14608 3670 : if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
14609 : is_ref = false;
14610 6711 : bool ref_to_array = false;
14611 6711 : bool ref_to_ptr = false;
14612 3241 : if (is_ref)
14613 : {
14614 200 : type = TREE_TYPE (type);
14615 200 : if (TREE_CODE (type) == ARRAY_TYPE)
14616 : {
14617 194 : type = build_pointer_type (type);
14618 194 : ref_to_array = true;
14619 : }
14620 : }
14621 3470 : else if (TREE_CODE (type) == ARRAY_TYPE)
14622 : {
14623 413 : tree decl2 = DECL_VALUE_EXPR (new_var);
14624 413 : gcc_assert (TREE_CODE (decl2) == MEM_REF);
14625 413 : decl2 = TREE_OPERAND (decl2, 0);
14626 413 : gcc_assert (DECL_P (decl2));
14627 413 : new_var = decl2;
14628 413 : type = TREE_TYPE (new_var);
14629 : }
14630 3057 : else if (TREE_CODE (type) == REFERENCE_TYPE
14631 3057 : && TREE_CODE (TREE_TYPE (type)) == POINTER_TYPE)
14632 : {
14633 156 : type = TREE_TYPE (type);
14634 156 : ref_to_ptr = true;
14635 : }
14636 3670 : x = build_receiver_ref (prev, false, ctx);
14637 3670 : x = fold_convert_loc (clause_loc, type, x);
14638 3670 : if (!integer_zerop (OMP_CLAUSE_SIZE (c)))
14639 : {
14640 762 : tree bias = OMP_CLAUSE_SIZE (c);
14641 762 : if (DECL_P (bias))
14642 364 : bias = lookup_decl (bias, ctx);
14643 762 : bias = fold_convert_loc (clause_loc, sizetype, bias);
14644 762 : bias = fold_build1_loc (clause_loc, NEGATE_EXPR, sizetype,
14645 : bias);
14646 762 : x = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
14647 762 : TREE_TYPE (x), x, bias);
14648 : }
14649 3670 : if (ref_to_array)
14650 194 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
14651 3670 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14652 3670 : if ((is_ref && !ref_to_array)
14653 3664 : || ref_to_ptr)
14654 : {
14655 162 : tree t = create_tmp_var_raw (type, get_name (var));
14656 162 : gimple_add_tmp_var (t);
14657 162 : TREE_ADDRESSABLE (t) = 1;
14658 162 : gimple_seq_add_stmt (&new_body,
14659 162 : gimple_build_assign (t, x));
14660 162 : x = build_fold_addr_expr_loc (clause_loc, t);
14661 : }
14662 3670 : gimple_seq_add_stmt (&new_body,
14663 3670 : gimple_build_assign (new_var, x));
14664 3670 : prev = NULL_TREE;
14665 : }
14666 59934 : else if (OMP_CLAUSE_CHAIN (c)
14667 43281 : && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c))
14668 : == OMP_CLAUSE_MAP
14669 97780 : && (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c))
14670 : == GOMP_MAP_FIRSTPRIVATE_POINTER
14671 34603 : || (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c))
14672 : == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
14673 : prev = c;
14674 : break;
14675 228 : case OMP_CLAUSE_PRIVATE:
14676 228 : var = OMP_CLAUSE_DECL (c);
14677 228 : by_ref = omp_privatize_by_reference (var);
14678 228 : new_var = lookup_decl (var, ctx);
14679 228 : allocate_ptr = alloc_map.get (new_var);
14680 228 : if (is_variable_sized (var, by_ref)
14681 228 : || (!allocate_ptr && by_ref && !is_gimple_omp_oacc (ctx->stmt)))
14682 : {
14683 24 : if (allocate_ptr)
14684 : {
14685 12 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14686 12 : gcc_assert (allocate_seq);
14687 12 : gimple_seq_add_seq (&new_body, *allocate_seq);
14688 : }
14689 24 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14690 24 : tree pvar = var;
14691 24 : if (!by_ref)
14692 : {
14693 3 : pvar = DECL_VALUE_EXPR (var);
14694 3 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14695 3 : pvar = TREE_OPERAND (pvar, 0);
14696 : }
14697 24 : gcc_assert (DECL_P (pvar));
14698 24 : tree new_pvar = lookup_decl (pvar, ctx);
14699 24 : tree x;
14700 24 : if (!allocate_ptr)
14701 : {
14702 12 : tree atmp = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
14703 12 : tree ty = TREE_TYPE (new_var);
14704 12 : if (by_ref)
14705 11 : ty = TREE_TYPE (ty);
14706 12 : x = TYPE_SIZE_UNIT (ty);
14707 12 : if (TREE_CONSTANT (x))
14708 : break;
14709 9 : tree al = size_int (TYPE_ALIGN (ty));
14710 9 : x = build_call_expr_loc (clause_loc, atmp, 2, x, al);
14711 9 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_pvar), x);
14712 : }
14713 : else
14714 12 : x = *allocate_ptr;
14715 21 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14716 21 : gimple_seq_add_stmt (&new_body,
14717 21 : gimple_build_assign (new_pvar, x));
14718 : }
14719 : break;
14720 15448 : case OMP_CLAUSE_FIRSTPRIVATE:
14721 15448 : var = OMP_CLAUSE_DECL (c);
14722 15448 : by_ref = omp_privatize_by_reference (var);
14723 15448 : if (is_variable_sized (var, by_ref))
14724 : {
14725 39 : tree new_var = lookup_decl (var, ctx);
14726 39 : tree *allocate_ptr = alloc_map.get (new_var);
14727 39 : if (!allocate_ptr)
14728 : break;
14729 15 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14730 15 : gcc_assert (allocate_seq);
14731 15 : gimple_seq_add_seq (&new_body, *allocate_seq);
14732 :
14733 15 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14734 15 : tree pvar = var;
14735 15 : if (!by_ref)
14736 : {
14737 2 : pvar = DECL_VALUE_EXPR (var);
14738 2 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14739 2 : pvar = TREE_OPERAND (pvar, 0);
14740 : }
14741 15 : gcc_assert (DECL_P (pvar));
14742 15 : tree new_pvar = lookup_decl (pvar, ctx);
14743 15 : tree x = fold_convert_loc (clause_loc, TREE_TYPE (new_pvar),
14744 15 : *allocate_ptr);
14745 15 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14746 15 : gimple_seq_add_stmt (&new_body,
14747 15 : gimple_build_assign (new_pvar, x));
14748 :
14749 15 : x = build_receiver_ref (var, true, ctx);
14750 15 : new_var = unshare_expr (new_var);
14751 15 : if (by_ref)
14752 13 : new_var = build_fold_indirect_ref (new_var);
14753 15 : x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
14754 15 : gimplify_and_add (x, &new_body);
14755 : }
14756 : }
14757 :
14758 25335 : gimple_seq fork_seq = NULL;
14759 25335 : gimple_seq join_seq = NULL;
14760 :
14761 25335 : if (offloaded && is_gimple_omp_oacc (ctx->stmt))
14762 : {
14763 : /* If there are reductions on the offloaded region itself, treat
14764 : them as a dummy GANG loop. */
14765 9395 : tree level = build_int_cst (integer_type_node, GOMP_DIM_GANG);
14766 :
14767 9395 : gcall *private_marker = lower_oacc_private_marker (ctx);
14768 :
14769 9395 : if (private_marker)
14770 30 : gimple_call_set_arg (private_marker, 2, level);
14771 :
14772 9395 : lower_oacc_reductions (gimple_location (ctx->stmt), clauses, level,
14773 : false, NULL, private_marker, NULL, &fork_seq,
14774 : &join_seq, ctx);
14775 : }
14776 :
14777 25335 : gimple_seq_add_seq (&new_body, fork_seq);
14778 25335 : gimple_seq_add_seq (&new_body, tgt_body);
14779 25335 : gimple_seq_add_seq (&new_body, join_seq);
14780 25335 : gimple_seq_add_seq (&new_body, alloc_dlist);
14781 :
14782 25335 : if (offloaded)
14783 : {
14784 21311 : new_body = maybe_catch_exception (new_body);
14785 21311 : gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
14786 : }
14787 25335 : gimple_omp_set_body (stmt, new_body);
14788 : }
14789 :
14790 37314 : gsi_insert_seq_before (gsi_p, gimple_omp_target_iterator_loops (stmt),
14791 : GSI_SAME_STMT);
14792 37314 : gimple_omp_target_set_iterator_loops (stmt, NULL);
14793 58625 : bind = gimple_build_bind (NULL, NULL,
14794 21311 : tgt_bind ? gimple_bind_block (tgt_bind)
14795 : : NULL_TREE);
14796 74236 : gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
14797 37314 : gimple_bind_add_seq (bind, ilist);
14798 37314 : gimple_bind_add_stmt (bind, stmt);
14799 37314 : gimple_bind_add_seq (bind, olist);
14800 :
14801 37314 : pop_gimplify_context (NULL);
14802 :
14803 37314 : if (dep_bind)
14804 : {
14805 392 : gimple_bind_add_seq (dep_bind, dep_ilist);
14806 392 : gimple_bind_add_stmt (dep_bind, bind);
14807 392 : gimple_bind_add_seq (dep_bind, dep_olist);
14808 392 : pop_gimplify_context (dep_bind);
14809 : }
14810 37314 : }
14811 :
14812 : /* Generate code to implement the action-clauses (destroy, init, use) of an
14813 : OpenMP interop construct. */
14814 :
14815 : static void
14816 492 : lower_omp_interop_action_clauses (gimple_seq *seq, vec<tree> &objs,
14817 : vec<tree> *interop_types = NULL,
14818 : vec<tree> *prefer_types = NULL)
14819 : {
14820 492 : if (objs.length () == 0)
14821 229 : return;
14822 :
14823 263 : enum omp_clause_code action = OMP_CLAUSE_CODE (objs[0]);
14824 263 : if (action == OMP_CLAUSE_INIT)
14825 402 : gcc_checking_assert (objs.length () == interop_types->length ()
14826 : && objs.length () == prefer_types->length ());
14827 : else
14828 129 : gcc_assert (prefer_types == NULL && interop_types == NULL);
14829 :
14830 263 : tree ret_objs = NULL_TREE, ret_interop_types = NULL_TREE,
14831 263 : ret_prefer_types = NULL_TREE;
14832 :
14833 : /* Build an array of interop objects. */
14834 :
14835 263 : tree type_obj_pref = build_array_type_nelts (ptr_type_node, objs.length ());
14836 263 : ret_objs = create_tmp_var (type_obj_pref, "interopobjs");
14837 :
14838 263 : bool have_pref_type = false;
14839 263 : if (action == OMP_CLAUSE_INIT)
14840 : {
14841 592 : for (tree pref_type : prefer_types)
14842 260 : if (pref_type != NULL_TREE)
14843 : {
14844 : have_pref_type = true;
14845 : break;
14846 : }
14847 134 : tree type_tgtsync
14848 134 : = build_array_type_nelts (integer_type_node, objs.length ());
14849 134 : ret_interop_types = create_tmp_var (type_tgtsync, "tgt_tgtsync");
14850 134 : if (have_pref_type)
14851 70 : ret_prefer_types = create_tmp_var (type_obj_pref, "pref_type");
14852 : else
14853 : {
14854 64 : ret_prefer_types = null_pointer_node;
14855 64 : prefer_types->truncate (0);
14856 : }
14857 : }
14858 :
14859 856 : for (size_t i = 0; !objs.is_empty (); i++)
14860 : {
14861 593 : tree offset = build_int_cst (integer_type_node, i);
14862 593 : tree init = build4 (ARRAY_REF, ptr_type_node, ret_objs, offset, NULL_TREE,
14863 : NULL_TREE);
14864 593 : tree obj = OMP_CLAUSE_DECL (objs.pop ());
14865 593 : if (TREE_CODE (TREE_TYPE (obj)) == REFERENCE_TYPE)
14866 6 : obj = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (obj)), obj);
14867 593 : if (action != OMP_CLAUSE_USE
14868 593 : && TREE_CODE (TREE_TYPE (obj)) != POINTER_TYPE)
14869 : /* For modifying actions, we need a pointer. */
14870 483 : obj = build_fold_addr_expr (obj);
14871 110 : else if (action == OMP_CLAUSE_USE
14872 110 : && TREE_CODE (TREE_TYPE (obj)) == POINTER_TYPE)
14873 : /* For use action, we need the value. */
14874 2 : obj = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (obj)), obj);
14875 593 : init = build2 (MODIFY_EXPR, ptr_type_node, init,
14876 : fold_convert (ptr_type_node, obj));
14877 593 : gimplify_and_add (init, seq);
14878 :
14879 593 : if (action == OMP_CLAUSE_INIT)
14880 : {
14881 384 : init = build4 (ARRAY_REF, integer_type_node, ret_interop_types,
14882 : offset, NULL_TREE, NULL_TREE);
14883 384 : init = build2 (MODIFY_EXPR, integer_type_node, init,
14884 384 : interop_types->pop ());
14885 384 : gimplify_and_add (init, seq);
14886 :
14887 384 : if (have_pref_type)
14888 : {
14889 194 : tree prefer_type = prefer_types->pop ();
14890 194 : tree pref = (prefer_type == NULL_TREE
14891 194 : ? null_pointer_node
14892 194 : : build_fold_addr_expr (prefer_type));
14893 194 : init = build4 (ARRAY_REF, ptr_type_node, ret_prefer_types, offset,
14894 : NULL_TREE, NULL_TREE);
14895 194 : init = build2 (MODIFY_EXPR, ptr_type_node, init, pref);
14896 194 : gimplify_and_add (init, seq);
14897 : }
14898 : }
14899 : }
14900 263 : if (action == OMP_CLAUSE_INIT)
14901 : {
14902 134 : if (have_pref_type)
14903 70 : ret_prefer_types = build_fold_addr_expr (ret_prefer_types);
14904 134 : ret_interop_types = build_fold_addr_expr (ret_interop_types);
14905 : }
14906 263 : ret_objs = build_fold_addr_expr (ret_objs);
14907 :
14908 660 : gcc_assert (objs.is_empty ()
14909 : && (!interop_types || interop_types->is_empty ())
14910 : && (!prefer_types || prefer_types->is_empty ()));
14911 :
14912 263 : objs.safe_push (ret_objs);
14913 263 : if (action == OMP_CLAUSE_INIT)
14914 : {
14915 134 : interop_types->safe_push (ret_interop_types);
14916 134 : prefer_types->safe_push (ret_prefer_types);
14917 : }
14918 : }
14919 :
14920 : /* Lower code for an OpenMP interop directive. */
14921 :
14922 : static void
14923 164 : lower_omp_interop (gimple_stmt_iterator *gsi_p, omp_context *ctx)
14924 : {
14925 164 : push_gimplify_context ();
14926 :
14927 164 : tree block = make_node (BLOCK);
14928 164 : gbind *bind = gimple_build_bind (NULL, NULL, block);
14929 164 : gimple_seq bind_body = NULL;
14930 :
14931 : /* Emit call to GOMP_interop:
14932 : void
14933 : GOMP_interop (int device_num, int n_init, omp_interop_t **init,
14934 : const void *target_targetsync, const void *prefer_type,
14935 : int n_use, omp_interop_t *use, int n_destroy,
14936 : omp_interop_t **destroy, unsigned int flags,
14937 : void **depend) */
14938 :
14939 164 : tree flags = NULL_TREE;
14940 164 : tree depend = null_pointer_node;
14941 164 : tree device_num = NULL_TREE;
14942 :
14943 164 : auto_vec<tree> init_objs, use_objs, destroy_objs, prefer_type,
14944 164 : target_targetsync;
14945 164 : gimple_seq dep_ilist = NULL, dep_olist = NULL;
14946 164 : tree clauses = gimple_omp_interop_clauses (gsi_stmt (*gsi_p));
14947 868 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
14948 : {
14949 704 : switch (OMP_CLAUSE_CODE (c))
14950 : {
14951 384 : case OMP_CLAUSE_INIT:
14952 384 : {
14953 384 : init_objs.safe_push (c);
14954 384 : int target_targetsync_bits = 0;
14955 384 : if (OMP_CLAUSE_INIT_TARGET (c))
14956 229 : target_targetsync_bits |= GOMP_INTEROP_TARGET;
14957 384 : if (OMP_CLAUSE_INIT_TARGETSYNC (c))
14958 204 : target_targetsync_bits |= GOMP_INTEROP_TARGETSYNC;
14959 384 : tree t = build_int_cst (integer_type_node, target_targetsync_bits);
14960 384 : target_targetsync.safe_push (t);
14961 384 : prefer_type.safe_push (OMP_CLAUSE_INIT_PREFER_TYPE (c));
14962 : }
14963 384 : break;
14964 106 : case OMP_CLAUSE_USE:
14965 106 : use_objs.safe_push (c);
14966 106 : break;
14967 103 : case OMP_CLAUSE_DESTROY:
14968 103 : destroy_objs.safe_push (c);
14969 103 : break;
14970 50 : case OMP_CLAUSE_NOWAIT:
14971 50 : flags = build_int_cst (integer_type_node, GOMP_INTEROP_FLAG_NOWAIT);
14972 50 : break;
14973 35 : case OMP_CLAUSE_DEPEND:
14974 35 : {
14975 35 : tree *cp = gimple_omp_interop_clauses_ptr (gsi_stmt (*gsi_p));
14976 35 : lower_depend_clauses (cp, &dep_ilist, &dep_olist);
14977 35 : depend = OMP_CLAUSE_DECL (*cp);
14978 : }
14979 35 : break;
14980 26 : case OMP_CLAUSE_DEVICE:
14981 26 : device_num = OMP_CLAUSE_DEVICE_ID (c);
14982 26 : break;
14983 0 : default:
14984 0 : gcc_unreachable ();
14985 : }
14986 : }
14987 :
14988 164 : if (flags == NULL_TREE)
14989 114 : flags = build_int_cst (integer_type_node, 0);
14990 :
14991 164 : if (device_num == NULL_TREE)
14992 138 : device_num = build_int_cst (integer_type_node, GOMP_DEVICE_DEFAULT_OMP_61);
14993 :
14994 298 : tree n_init = build_int_cst (integer_type_node, init_objs.length ());
14995 234 : tree n_use = build_int_cst (integer_type_node, use_objs.length ());
14996 223 : tree n_destroy = build_int_cst (integer_type_node, destroy_objs.length ());
14997 :
14998 164 : lower_omp_interop_action_clauses (&bind_body, init_objs, &target_targetsync,
14999 : &prefer_type);
15000 164 : lower_omp_interop_action_clauses (&bind_body, use_objs);
15001 164 : lower_omp_interop_action_clauses (&bind_body, destroy_objs);
15002 :
15003 164 : gimple_seq_add_seq (&bind_body, dep_ilist);
15004 164 : tree fn = builtin_decl_explicit (BUILT_IN_GOMP_INTEROP);
15005 298 : tree init_arg = init_objs.length () ? init_objs.pop () : null_pointer_node;
15006 164 : tree target_targetsync_arg = target_targetsync.length ()
15007 134 : ? target_targetsync.pop ()
15008 164 : : null_pointer_node;
15009 164 : tree prefer_type_arg
15010 298 : = prefer_type.length () ? prefer_type.pop () : null_pointer_node;
15011 234 : tree use_arg = use_objs.length () ? use_objs.pop () : null_pointer_node;
15012 164 : tree destroy_arg
15013 223 : = destroy_objs.length () ? destroy_objs.pop () : null_pointer_node;
15014 164 : gcall *call
15015 164 : = gimple_build_call (fn, 11, device_num, n_init, init_arg,
15016 : target_targetsync_arg, prefer_type_arg, n_use, use_arg,
15017 : n_destroy, destroy_arg, flags, depend);
15018 164 : gimple_seq_add_stmt (&bind_body, call);
15019 164 : gimple_seq_add_seq (&bind_body, dep_olist);
15020 :
15021 164 : gsi_replace (gsi_p, bind, true);
15022 164 : gimple_bind_set_body (bind, bind_body);
15023 164 : pop_gimplify_context (bind);
15024 164 : gimple_bind_append_vars (bind, ctx->block_vars);
15025 164 : BLOCK_VARS (block) = ctx->block_vars;
15026 164 : }
15027 :
15028 : /* Expand code for an OpenMP teams directive. */
15029 :
15030 : static void
15031 5938 : lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx)
15032 : {
15033 5938 : gomp_teams *teams_stmt = as_a <gomp_teams *> (gsi_stmt (*gsi_p));
15034 5938 : push_gimplify_context ();
15035 :
15036 5938 : tree block = make_node (BLOCK);
15037 5938 : gbind *bind = gimple_build_bind (NULL, NULL, block);
15038 5938 : gsi_replace (gsi_p, bind, true);
15039 5938 : gimple_seq bind_body = NULL;
15040 5938 : gimple_seq dlist = NULL;
15041 5938 : gimple_seq olist = NULL;
15042 :
15043 5938 : tree num_teams = omp_find_clause (gimple_omp_teams_clauses (teams_stmt),
15044 5938 : OMP_CLAUSE_NUM_TEAMS);
15045 5938 : tree num_teams_lower = NULL_TREE;
15046 5938 : if (num_teams == NULL_TREE)
15047 5308 : num_teams = build_int_cst (unsigned_type_node, 0);
15048 : else
15049 : {
15050 630 : num_teams_lower = OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (num_teams);
15051 630 : if (num_teams_lower)
15052 : {
15053 150 : num_teams_lower = fold_convert (unsigned_type_node, num_teams_lower);
15054 150 : gimplify_expr (&num_teams_lower, &bind_body, NULL, is_gimple_val,
15055 : fb_rvalue);
15056 : }
15057 630 : num_teams = OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (num_teams);
15058 : // FIXME: Handle dim(x), cf. omp-expand.cc's get_target_arguments. */
15059 630 : if (TREE_CODE (num_teams) == TREE_LIST && TREE_CHAIN (num_teams))
15060 : {
15061 3 : num_teams_lower = TREE_VALUE (num_teams);
15062 3 : num_teams = TREE_VALUE (TREE_CHAIN (num_teams));
15063 : }
15064 627 : else if (TREE_CODE (num_teams) == TREE_LIST)
15065 2 : num_teams = TREE_VALUE (num_teams);
15066 630 : num_teams = fold_convert (unsigned_type_node, num_teams);
15067 630 : gimplify_expr (&num_teams, &bind_body, NULL, is_gimple_val, fb_rvalue);
15068 : }
15069 5938 : if (num_teams_lower == NULL_TREE)
15070 5785 : num_teams_lower = num_teams;
15071 5938 : tree thread_limit = omp_find_clause (gimple_omp_teams_clauses (teams_stmt),
15072 5938 : OMP_CLAUSE_THREAD_LIMIT);
15073 5938 : if (thread_limit == NULL_TREE)
15074 5457 : thread_limit = build_int_cst (unsigned_type_node, 0);
15075 : else
15076 : {
15077 481 : thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit);
15078 : // FIXME: Handle dim(x), cf. omp-expand.cc's get_target_arguments. */
15079 481 : if (TREE_CODE (thread_limit) == TREE_LIST)
15080 69 : thread_limit = TREE_VALUE (thread_limit);
15081 481 : thread_limit = fold_convert (unsigned_type_node, thread_limit);
15082 481 : gimplify_expr (&thread_limit, &bind_body, NULL, is_gimple_val,
15083 : fb_rvalue);
15084 : }
15085 5938 : location_t loc = gimple_location (teams_stmt);
15086 5938 : tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS4);
15087 5938 : tree rettype = TREE_TYPE (TREE_TYPE (decl));
15088 5938 : tree first = create_tmp_var (rettype);
15089 5938 : gimple_seq_add_stmt (&bind_body,
15090 5938 : gimple_build_assign (first, build_one_cst (rettype)));
15091 5938 : tree llabel = create_artificial_label (loc);
15092 5938 : gimple_seq_add_stmt (&bind_body, gimple_build_label (llabel));
15093 5938 : gimple *call
15094 5938 : = gimple_build_call (decl, 4, num_teams_lower, num_teams, thread_limit,
15095 : first);
15096 5938 : gimple_set_location (call, loc);
15097 5938 : tree temp = create_tmp_var (rettype);
15098 5938 : gimple_call_set_lhs (call, temp);
15099 5938 : gimple_seq_add_stmt (&bind_body, call);
15100 :
15101 5938 : tree tlabel = create_artificial_label (loc);
15102 5938 : tree flabel = create_artificial_label (loc);
15103 5938 : gimple *cond = gimple_build_cond (NE_EXPR, temp, build_zero_cst (rettype),
15104 : tlabel, flabel);
15105 5938 : gimple_seq_add_stmt (&bind_body, cond);
15106 5938 : gimple_seq_add_stmt (&bind_body, gimple_build_label (tlabel));
15107 5938 : gimple_seq_add_stmt (&bind_body,
15108 5938 : gimple_build_assign (first, build_zero_cst (rettype)));
15109 :
15110 5938 : lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt),
15111 : &bind_body, &dlist, ctx, NULL);
15112 5938 : lower_omp (gimple_omp_body_ptr (teams_stmt), ctx);
15113 5938 : lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt), &olist,
15114 : NULL, ctx);
15115 5938 : gimple_seq_add_stmt (&bind_body, teams_stmt);
15116 :
15117 5938 : gimple_seq_add_seq (&bind_body, gimple_omp_body (teams_stmt));
15118 5938 : gimple_omp_set_body (teams_stmt, NULL);
15119 5938 : gimple_seq_add_seq (&bind_body, olist);
15120 5938 : gimple_seq_add_seq (&bind_body, dlist);
15121 5938 : gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true));
15122 5938 : gimple_seq_add_stmt (&bind_body, gimple_build_goto (llabel));
15123 5938 : gimple_seq_add_stmt (&bind_body, gimple_build_label (flabel));
15124 5938 : gimple_bind_set_body (bind, bind_body);
15125 :
15126 5938 : pop_gimplify_context (bind);
15127 :
15128 5938 : gimple_bind_append_vars (bind, ctx->block_vars);
15129 5938 : BLOCK_VARS (block) = ctx->block_vars;
15130 5938 : if (BLOCK_VARS (block))
15131 689 : TREE_USED (block) = 1;
15132 5938 : }
15133 :
15134 : /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
15135 : regimplified. If DATA is non-NULL, lower_omp_1 is outside
15136 : of OMP context, but with make_addressable_vars set. */
15137 :
15138 : static tree
15139 3226433 : lower_omp_regimplify_p (tree *tp, int *walk_subtrees,
15140 : void *data)
15141 : {
15142 3226433 : tree t = *tp;
15143 :
15144 : /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
15145 3226433 : if ((VAR_P (t) || TREE_CODE (t) == PARM_DECL || TREE_CODE (t) == RESULT_DECL)
15146 1500736 : && data == NULL
15147 1440235 : && DECL_HAS_VALUE_EXPR_P (t))
15148 : return t;
15149 :
15150 3120264 : if (make_addressable_vars
15151 423226 : && DECL_P (t)
15152 3325655 : && bitmap_bit_p (make_addressable_vars, DECL_UID (t)))
15153 : return t;
15154 :
15155 : /* If a global variable has been privatized, TREE_CONSTANT on
15156 : ADDR_EXPR might be wrong. */
15157 3117675 : if (data == NULL && TREE_CODE (t) == ADDR_EXPR)
15158 110505 : recompute_tree_invariant_for_addr_expr (t);
15159 :
15160 3117675 : *walk_subtrees = !IS_TYPE_OR_DECL_P (t);
15161 3117675 : return NULL_TREE;
15162 : }
15163 :
15164 : /* Data to be communicated between lower_omp_regimplify_operands and
15165 : lower_omp_regimplify_operands_p. */
15166 :
15167 : struct lower_omp_regimplify_operands_data
15168 : {
15169 : omp_context *ctx;
15170 : vec<tree> *decls;
15171 : };
15172 :
15173 : /* Helper function for lower_omp_regimplify_operands. Find
15174 : omp_member_access_dummy_var vars and adjust temporarily their
15175 : DECL_VALUE_EXPRs if needed. */
15176 :
15177 : static tree
15178 428301 : lower_omp_regimplify_operands_p (tree *tp, int *walk_subtrees,
15179 : void *data)
15180 : {
15181 428301 : tree t = omp_member_access_dummy_var (*tp);
15182 428301 : if (t)
15183 : {
15184 9 : struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
15185 9 : lower_omp_regimplify_operands_data *ldata
15186 : = (lower_omp_regimplify_operands_data *) wi->info;
15187 9 : tree o = maybe_lookup_decl (t, ldata->ctx);
15188 9 : if (o == NULL_TREE)
15189 5 : o = maybe_lookup_decl_in_outer_ctx (t, ldata->ctx);
15190 9 : if (o != t)
15191 : {
15192 9 : ldata->decls->safe_push (DECL_VALUE_EXPR (*tp));
15193 9 : ldata->decls->safe_push (*tp);
15194 9 : tree v = unshare_and_remap (DECL_VALUE_EXPR (*tp), t, o);
15195 9 : SET_DECL_VALUE_EXPR (*tp, v);
15196 : }
15197 : }
15198 428301 : *walk_subtrees = !IS_TYPE_OR_DECL_P (*tp);
15199 428301 : return NULL_TREE;
15200 : }
15201 :
15202 : /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
15203 : of omp_member_access_dummy_var vars during regimplification. */
15204 :
15205 : static void
15206 108601 : lower_omp_regimplify_operands (omp_context *ctx, gimple *stmt,
15207 : gimple_stmt_iterator *gsi_p)
15208 : {
15209 108601 : auto_vec<tree, 10> decls;
15210 108601 : if (ctx)
15211 : {
15212 106079 : struct walk_stmt_info wi;
15213 106079 : memset (&wi, '\0', sizeof (wi));
15214 106079 : struct lower_omp_regimplify_operands_data data;
15215 106079 : data.ctx = ctx;
15216 106079 : data.decls = &decls;
15217 106079 : wi.info = &data;
15218 106079 : walk_gimple_op (stmt, lower_omp_regimplify_operands_p, &wi);
15219 : }
15220 108601 : gimple_regimplify_operands (stmt, gsi_p);
15221 325812 : while (!decls.is_empty ())
15222 : {
15223 9 : tree t = decls.pop ();
15224 9 : tree v = decls.pop ();
15225 9 : SET_DECL_VALUE_EXPR (t, v);
15226 : }
15227 108601 : }
15228 :
15229 : static void
15230 2524622 : lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx)
15231 : {
15232 2524622 : gimple *stmt = gsi_stmt (*gsi_p);
15233 2524622 : struct walk_stmt_info wi;
15234 2524622 : gcall *call_stmt;
15235 :
15236 2524622 : if (gimple_has_location (stmt))
15237 2071578 : input_location = gimple_location (stmt);
15238 :
15239 2524622 : if (make_addressable_vars)
15240 154287 : memset (&wi, '\0', sizeof (wi));
15241 :
15242 : /* If we have issued syntax errors, avoid doing any heavy lifting.
15243 : Just replace the OMP directives with a NOP to avoid
15244 : confusing RTL expansion. */
15245 2524622 : if (seen_error () && is_gimple_omp (stmt))
15246 : {
15247 11728 : gsi_replace (gsi_p, gimple_build_nop (), true);
15248 11728 : return;
15249 : }
15250 :
15251 2512894 : switch (gimple_code (stmt))
15252 : {
15253 157605 : case GIMPLE_COND:
15254 157605 : {
15255 157605 : gcond *cond_stmt = as_a <gcond *> (stmt);
15256 89792 : if ((ctx || make_addressable_vars)
15257 161218 : && (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
15258 : lower_omp_regimplify_p,
15259 : ctx ? NULL : &wi, NULL)
15260 137292 : || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
15261 : lower_omp_regimplify_p,
15262 : ctx ? NULL : &wi, NULL)))
15263 1247 : lower_omp_regimplify_operands (ctx, cond_stmt, gsi_p);
15264 : }
15265 : break;
15266 20 : case GIMPLE_CATCH:
15267 20 : lower_omp (gimple_catch_handler_ptr (as_a <gcatch *> (stmt)), ctx);
15268 20 : break;
15269 0 : case GIMPLE_EH_FILTER:
15270 0 : lower_omp (gimple_eh_filter_failure_ptr (stmt), ctx);
15271 0 : break;
15272 26211 : case GIMPLE_TRY:
15273 26211 : lower_omp (gimple_try_eval_ptr (stmt), ctx);
15274 26206 : lower_omp (gimple_try_cleanup_ptr (stmt), ctx);
15275 26206 : break;
15276 2 : case GIMPLE_ASSUME:
15277 2 : lower_omp (gimple_assume_body_ptr (stmt), ctx);
15278 2 : break;
15279 5 : case GIMPLE_TRANSACTION:
15280 5 : lower_omp (gimple_transaction_body_ptr (as_a <gtransaction *> (stmt)),
15281 : ctx);
15282 5 : break;
15283 169478 : case GIMPLE_BIND:
15284 169478 : if (ctx && is_gimple_omp_oacc (ctx->stmt))
15285 : {
15286 17785 : tree vars = gimple_bind_vars (as_a <gbind *> (stmt));
15287 17785 : oacc_privatization_scan_decl_chain (ctx, vars);
15288 : }
15289 169478 : lower_omp (gimple_bind_body_ptr (as_a <gbind *> (stmt)), ctx);
15290 169471 : maybe_remove_omp_member_access_dummy_vars (as_a <gbind *> (stmt));
15291 169471 : break;
15292 20232 : case GIMPLE_OMP_PARALLEL:
15293 20232 : case GIMPLE_OMP_TASK:
15294 20232 : ctx = maybe_lookup_ctx (stmt);
15295 20232 : gcc_assert (ctx);
15296 20232 : if (ctx->cancellable)
15297 146 : ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
15298 20232 : lower_omp_taskreg (gsi_p, ctx);
15299 20232 : break;
15300 47102 : case GIMPLE_OMP_FOR:
15301 47102 : ctx = maybe_lookup_ctx (stmt);
15302 47102 : gcc_assert (ctx);
15303 47102 : if (ctx->cancellable)
15304 84 : ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
15305 47102 : lower_omp_for (gsi_p, ctx);
15306 47102 : break;
15307 386 : case GIMPLE_OMP_SECTIONS:
15308 386 : ctx = maybe_lookup_ctx (stmt);
15309 386 : gcc_assert (ctx);
15310 386 : if (ctx->cancellable)
15311 19 : ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
15312 386 : lower_omp_sections (gsi_p, ctx);
15313 386 : break;
15314 161 : case GIMPLE_OMP_SCOPE:
15315 161 : ctx = maybe_lookup_ctx (stmt);
15316 161 : gcc_assert (ctx);
15317 161 : lower_omp_scope (gsi_p, ctx);
15318 161 : break;
15319 659 : case GIMPLE_OMP_DISPATCH:
15320 659 : ctx = maybe_lookup_ctx (stmt);
15321 659 : gcc_assert (ctx);
15322 659 : lower_omp_dispatch (gsi_p, ctx);
15323 659 : break;
15324 164 : case GIMPLE_OMP_INTEROP:
15325 164 : ctx = maybe_lookup_ctx (stmt);
15326 164 : gcc_assert (ctx);
15327 164 : lower_omp_interop (gsi_p, ctx);
15328 164 : break;
15329 1140 : case GIMPLE_OMP_SINGLE:
15330 1140 : ctx = maybe_lookup_ctx (stmt);
15331 1140 : gcc_assert (ctx);
15332 1140 : lower_omp_single (gsi_p, ctx);
15333 1140 : break;
15334 658 : case GIMPLE_OMP_STRUCTURED_BLOCK:
15335 : /* We have already done error checking at this point, so these nodes
15336 : can be completely removed and replaced with their body. */
15337 658 : ctx = maybe_lookup_ctx (stmt);
15338 658 : gcc_assert (ctx);
15339 658 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
15340 658 : gsi_replace_with_seq (gsi_p, gimple_omp_body (stmt), true);
15341 658 : break;
15342 1127 : case GIMPLE_OMP_MASTER:
15343 1127 : case GIMPLE_OMP_MASKED:
15344 1127 : ctx = maybe_lookup_ctx (stmt);
15345 1127 : gcc_assert (ctx);
15346 1127 : lower_omp_master (gsi_p, ctx);
15347 1127 : break;
15348 536 : case GIMPLE_OMP_TASKGROUP:
15349 536 : ctx = maybe_lookup_ctx (stmt);
15350 536 : gcc_assert (ctx);
15351 536 : lower_omp_taskgroup (gsi_p, ctx);
15352 536 : break;
15353 1128 : case GIMPLE_OMP_ORDERED:
15354 1128 : ctx = maybe_lookup_ctx (stmt);
15355 1128 : gcc_assert (ctx);
15356 1128 : lower_omp_ordered (gsi_p, ctx);
15357 1128 : break;
15358 1324 : case GIMPLE_OMP_SCAN:
15359 1324 : ctx = maybe_lookup_ctx (stmt);
15360 1324 : gcc_assert (ctx);
15361 1324 : lower_omp_scan (gsi_p, ctx);
15362 1324 : break;
15363 317 : case GIMPLE_OMP_CRITICAL:
15364 317 : ctx = maybe_lookup_ctx (stmt);
15365 317 : gcc_assert (ctx);
15366 317 : lower_omp_critical (gsi_p, ctx);
15367 317 : break;
15368 2566 : case GIMPLE_OMP_ATOMIC_LOAD:
15369 90 : if ((ctx || make_addressable_vars)
15370 2566 : && walk_tree (gimple_omp_atomic_load_rhs_ptr (
15371 : as_a <gomp_atomic_load *> (stmt)),
15372 : lower_omp_regimplify_p, ctx ? NULL : &wi, NULL))
15373 1442 : lower_omp_regimplify_operands (ctx, stmt, gsi_p);
15374 : break;
15375 37314 : case GIMPLE_OMP_TARGET:
15376 37314 : ctx = maybe_lookup_ctx (stmt);
15377 37314 : gcc_assert (ctx);
15378 37314 : lower_omp_target (gsi_p, ctx);
15379 37314 : break;
15380 8515 : case GIMPLE_OMP_TEAMS:
15381 8515 : ctx = maybe_lookup_ctx (stmt);
15382 8515 : gcc_assert (ctx);
15383 8515 : if (gimple_omp_teams_host (as_a <gomp_teams *> (stmt)))
15384 2577 : lower_omp_taskreg (gsi_p, ctx);
15385 : else
15386 5938 : lower_omp_teams (gsi_p, ctx);
15387 : break;
15388 136307 : case GIMPLE_CALL:
15389 136307 : tree fndecl;
15390 136307 : call_stmt = as_a <gcall *> (stmt);
15391 136307 : fndecl = gimple_call_fndecl (call_stmt);
15392 136307 : if (fndecl
15393 136307 : && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL))
15394 39123 : switch (DECL_FUNCTION_CODE (fndecl))
15395 : {
15396 742 : case BUILT_IN_GOMP_BARRIER:
15397 742 : if (ctx == NULL)
15398 : break;
15399 : /* FALLTHRU */
15400 1070 : case BUILT_IN_GOMP_CANCEL:
15401 1070 : case BUILT_IN_GOMP_CANCELLATION_POINT:
15402 1070 : omp_context *cctx;
15403 1070 : cctx = ctx;
15404 1070 : if (gimple_code (cctx->stmt) == GIMPLE_OMP_SECTION)
15405 43 : cctx = cctx->outer;
15406 1070 : gcc_assert (gimple_call_lhs (call_stmt) == NULL_TREE);
15407 1070 : if (!cctx->cancellable)
15408 : {
15409 688 : if (DECL_FUNCTION_CODE (fndecl)
15410 : == BUILT_IN_GOMP_CANCELLATION_POINT)
15411 : {
15412 0 : stmt = gimple_build_nop ();
15413 0 : gsi_replace (gsi_p, stmt, false);
15414 : }
15415 : break;
15416 : }
15417 382 : if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_GOMP_BARRIER)
15418 : {
15419 20 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL);
15420 20 : gimple_call_set_fndecl (call_stmt, fndecl);
15421 20 : gimple_call_set_fntype (call_stmt, TREE_TYPE (fndecl));
15422 : }
15423 382 : tree lhs;
15424 382 : lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl)));
15425 382 : gimple_call_set_lhs (call_stmt, lhs);
15426 382 : tree fallthru_label;
15427 382 : fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
15428 382 : gimple *g;
15429 382 : g = gimple_build_label (fallthru_label);
15430 382 : gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
15431 382 : g = gimple_build_cond (NE_EXPR, lhs,
15432 382 : fold_convert (TREE_TYPE (lhs),
15433 : boolean_false_node),
15434 : cctx->cancel_label, fallthru_label);
15435 382 : gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
15436 382 : break;
15437 : default:
15438 : break;
15439 : }
15440 136272 : goto regimplify;
15441 :
15442 : case GIMPLE_ASSIGN:
15443 1461389 : for (omp_context *up = ctx; up; up = up->outer)
15444 : {
15445 687431 : if (gimple_code (up->stmt) == GIMPLE_OMP_ORDERED
15446 : || gimple_code (up->stmt) == GIMPLE_OMP_CRITICAL
15447 : || gimple_code (up->stmt) == GIMPLE_OMP_TASKGROUP
15448 : || gimple_code (up->stmt) == GIMPLE_OMP_SCOPE
15449 : || gimple_code (up->stmt) == GIMPLE_OMP_SECTION
15450 : || gimple_code (up->stmt) == GIMPLE_OMP_SCAN
15451 : || (gimple_code (up->stmt) == GIMPLE_OMP_TARGET
15452 230338 : && (gimple_omp_target_kind (up->stmt)
15453 : == GF_OMP_TARGET_KIND_DATA)))
15454 90764 : continue;
15455 596667 : else if (!up->lastprivate_conditional_map)
15456 : break;
15457 4204 : tree lhs = get_base_address (gimple_assign_lhs (stmt));
15458 4204 : if (TREE_CODE (lhs) == MEM_REF
15459 26 : && DECL_P (TREE_OPERAND (lhs, 0))
15460 4227 : && TREE_CODE (TREE_TYPE (TREE_OPERAND (lhs,
15461 : 0))) == REFERENCE_TYPE)
15462 19 : lhs = TREE_OPERAND (lhs, 0);
15463 4204 : if (DECL_P (lhs))
15464 3161 : if (tree *v = up->lastprivate_conditional_map->get (lhs))
15465 : {
15466 382 : tree clauses;
15467 382 : if (up->combined_into_simd_safelen1)
15468 : {
15469 54 : up = up->outer;
15470 54 : if (gimple_code (up->stmt) == GIMPLE_OMP_SCAN)
15471 3 : up = up->outer;
15472 : }
15473 382 : if (gimple_code (up->stmt) == GIMPLE_OMP_FOR)
15474 298 : clauses = gimple_omp_for_clauses (up->stmt);
15475 : else
15476 84 : clauses = gimple_omp_sections_clauses (up->stmt);
15477 382 : tree c = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_);
15478 382 : if (!OMP_CLAUSE__CONDTEMP__ITER (c))
15479 348 : c = omp_find_clause (OMP_CLAUSE_CHAIN (c),
15480 : OMP_CLAUSE__CONDTEMP_);
15481 382 : gcc_assert (OMP_CLAUSE__CONDTEMP__ITER (c));
15482 382 : gimple *g = gimple_build_assign (*v, OMP_CLAUSE_DECL (c));
15483 382 : gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
15484 : }
15485 : }
15486 : /* FALLTHRU */
15487 :
15488 2036244 : default:
15489 2036244 : regimplify:
15490 1074761 : if ((ctx || make_addressable_vars)
15491 2112159 : && walk_gimple_op (stmt, lower_omp_regimplify_p,
15492 : ctx ? NULL : &wi))
15493 : {
15494 : /* Just remove clobbers, this should happen only if we have
15495 : "privatized" local addressable variables in SIMD regions,
15496 : the clobber isn't needed in that case and gimplifying address
15497 : of the ARRAY_REF into a pointer and creating MEM_REF based
15498 : clobber would create worse code than we get with the clobber
15499 : dropped. */
15500 106069 : if (gimple_clobber_p (stmt))
15501 : {
15502 157 : gsi_replace (gsi_p, gimple_build_nop (), true);
15503 157 : break;
15504 : }
15505 105912 : lower_omp_regimplify_operands (ctx, stmt, gsi_p);
15506 : }
15507 : break;
15508 : }
15509 : }
15510 :
15511 : static void
15512 414005 : lower_omp (gimple_seq *body, omp_context *ctx)
15513 : {
15514 414005 : location_t saved_location = input_location;
15515 414005 : gimple_stmt_iterator gsi;
15516 3304132 : for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
15517 2524622 : lower_omp_1 (&gsi, ctx);
15518 : /* During gimplification, we haven't folded statements inside offloading
15519 : or taskreg regions (gimplify.cc:maybe_fold_stmt); do that now. */
15520 413989 : if (target_nesting_level || taskreg_nesting_level)
15521 658434 : for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
15522 444775 : fold_stmt (&gsi);
15523 413989 : input_location = saved_location;
15524 413989 : }
15525 :
15526 : /* Main entry point. */
15527 :
15528 : static unsigned int
15529 3010141 : execute_lower_omp (void)
15530 : {
15531 3010141 : gimple_seq body;
15532 3010141 : int i;
15533 3010141 : omp_context *ctx;
15534 :
15535 : /* This pass always runs, to provide PROP_gimple_lomp.
15536 : But often, there is nothing to do. */
15537 3010141 : if (flag_openacc == 0 && flag_openmp == 0
15538 2952884 : && flag_openmp_simd == 0)
15539 : return 0;
15540 :
15541 59553 : all_contexts = splay_tree_new (splay_tree_compare_pointers, 0,
15542 : delete_omp_context);
15543 :
15544 59553 : body = gimple_body (current_function_decl);
15545 :
15546 59553 : scan_omp (&body, NULL);
15547 59553 : gcc_assert (taskreg_nesting_level == 0);
15548 86163 : FOR_EACH_VEC_ELT (taskreg_contexts, i, ctx)
15549 26610 : finish_taskreg_scan (ctx);
15550 59553 : taskreg_contexts.release ();
15551 :
15552 59553 : if (all_contexts->root)
15553 : {
15554 26058 : if (make_addressable_vars)
15555 624 : push_gimplify_context ();
15556 26058 : lower_omp (&body, NULL);
15557 26054 : if (make_addressable_vars)
15558 624 : pop_gimplify_context (NULL);
15559 : }
15560 :
15561 59549 : if (all_contexts)
15562 : {
15563 59549 : splay_tree_delete (all_contexts);
15564 59549 : all_contexts = NULL;
15565 : }
15566 59549 : BITMAP_FREE (make_addressable_vars);
15567 59549 : BITMAP_FREE (global_nonaddressable_vars);
15568 :
15569 : /* If current function is a method, remove artificial dummy VAR_DECL created
15570 : for non-static data member privatization, they aren't needed for
15571 : debuginfo nor anything else, have been already replaced everywhere in the
15572 : IL and cause problems with LTO. */
15573 59549 : if (DECL_ARGUMENTS (current_function_decl)
15574 34084 : && DECL_ARTIFICIAL (DECL_ARGUMENTS (current_function_decl))
15575 69950 : && (TREE_CODE (TREE_TYPE (DECL_ARGUMENTS (current_function_decl)))
15576 : == POINTER_TYPE))
15577 9838 : remove_member_access_dummy_vars (DECL_INITIAL (current_function_decl));
15578 :
15579 60540 : for (auto task_stmt : task_cpyfns)
15580 513 : finalize_task_copyfn (task_stmt);
15581 59549 : task_cpyfns.release ();
15582 59549 : return 0;
15583 : }
15584 :
15585 : namespace {
15586 :
15587 : const pass_data pass_data_lower_omp =
15588 : {
15589 : GIMPLE_PASS, /* type */
15590 : "omplower", /* name */
15591 : OPTGROUP_OMP, /* optinfo_flags */
15592 : TV_NONE, /* tv_id */
15593 : PROP_gimple_any, /* properties_required */
15594 : PROP_gimple_lomp | PROP_gimple_lomp_dev, /* properties_provided */
15595 : 0, /* properties_destroyed */
15596 : 0, /* todo_flags_start */
15597 : 0, /* todo_flags_finish */
15598 : };
15599 :
15600 : class pass_lower_omp : public gimple_opt_pass
15601 : {
15602 : public:
15603 297180 : pass_lower_omp (gcc::context *ctxt)
15604 594360 : : gimple_opt_pass (pass_data_lower_omp, ctxt)
15605 : {}
15606 :
15607 : /* opt_pass methods: */
15608 3010141 : unsigned int execute (function *) final override
15609 : {
15610 3010141 : return execute_lower_omp ();
15611 : }
15612 :
15613 : }; // class pass_lower_omp
15614 :
15615 : } // anon namespace
15616 :
15617 : gimple_opt_pass *
15618 297180 : make_pass_lower_omp (gcc::context *ctxt)
15619 : {
15620 297180 : return new pass_lower_omp (ctxt);
15621 : }
15622 :
15623 : /* The following is a utility to diagnose structured block violations.
15624 : It is not part of the "omplower" pass, as that's invoked too late. It
15625 : should be invoked by the respective front ends after gimplification. */
15626 :
15627 : static splay_tree all_labels;
15628 :
15629 : /* Check for mismatched contexts and generate an error if needed. Return
15630 : true if an error is detected. */
15631 :
15632 : static bool
15633 568384 : diagnose_sb_0 (gimple_stmt_iterator *gsi_p,
15634 : gimple *branch_ctx, gimple *label_ctx)
15635 : {
15636 568384 : gcc_checking_assert (!branch_ctx || is_gimple_omp (branch_ctx));
15637 568384 : gcc_checking_assert (!label_ctx || is_gimple_omp (label_ctx));
15638 :
15639 568384 : if (label_ctx == branch_ctx)
15640 : return false;
15641 :
15642 180 : const char* kind = NULL;
15643 :
15644 180 : if (flag_openacc)
15645 : {
15646 7 : if ((branch_ctx && is_gimple_omp_oacc (branch_ctx))
15647 16 : || (label_ctx && is_gimple_omp_oacc (label_ctx)))
15648 : {
15649 : gcc_checking_assert (kind == NULL);
15650 : kind = "OpenACC";
15651 : }
15652 : }
15653 : if (kind == NULL)
15654 : {
15655 164 : gcc_checking_assert (flag_openmp || flag_openmp_simd);
15656 : kind = "OpenMP";
15657 : }
15658 :
15659 : /* Previously we kept track of the label's entire context in diagnose_sb_[12]
15660 : so we could traverse it and issue a correct "exit" or "enter" error
15661 : message upon a structured block violation.
15662 :
15663 : We built the context by building a list with tree_cons'ing, but there is
15664 : no easy counterpart in gimple tuples. It seems like far too much work
15665 : for issuing exit/enter error messages. If someone really misses the
15666 : distinct error message... patches welcome. */
15667 :
15668 : #if 0
15669 : /* Try to avoid confusing the user by producing and error message
15670 : with correct "exit" or "enter" verbiage. We prefer "exit"
15671 : unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
15672 : if (branch_ctx == NULL)
15673 : exit_p = false;
15674 : else
15675 : {
15676 : while (label_ctx)
15677 : {
15678 : if (TREE_VALUE (label_ctx) == branch_ctx)
15679 : {
15680 : exit_p = false;
15681 : break;
15682 : }
15683 : label_ctx = TREE_CHAIN (label_ctx);
15684 : }
15685 : }
15686 :
15687 : if (exit_p)
15688 : error ("invalid exit from %s structured block", kind);
15689 : else
15690 : error ("invalid entry to %s structured block", kind);
15691 : #endif
15692 :
15693 : /* If it's obvious we have an invalid entry, be specific about the error. */
15694 180 : if (branch_ctx == NULL)
15695 53 : error ("invalid entry to %s structured block", kind);
15696 : else
15697 : {
15698 : /* Otherwise, be vague and lazy, but efficient. */
15699 127 : error ("invalid branch to/from %s structured block", kind);
15700 : }
15701 :
15702 180 : gsi_replace (gsi_p, gimple_build_nop (), false);
15703 180 : return true;
15704 : }
15705 :
15706 : /* Pass 1: Create a minimal tree of structured blocks, and record
15707 : where each label is found. */
15708 :
15709 : static tree
15710 3395322 : diagnose_sb_1 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
15711 : struct walk_stmt_info *wi)
15712 : {
15713 3395322 : gimple *context = (gimple *) wi->info;
15714 3395322 : gimple *inner_context;
15715 3395322 : gimple *stmt = gsi_stmt (*gsi_p);
15716 :
15717 3395322 : *handled_ops_p = true;
15718 :
15719 3395322 : switch (gimple_code (stmt))
15720 : {
15721 327782 : WALK_SUBSTMTS;
15722 :
15723 84560 : case GIMPLE_OMP_PARALLEL:
15724 84560 : case GIMPLE_OMP_TASK:
15725 84560 : case GIMPLE_OMP_SCOPE:
15726 84560 : case GIMPLE_OMP_SECTIONS:
15727 84560 : case GIMPLE_OMP_SINGLE:
15728 84560 : case GIMPLE_OMP_SECTION:
15729 84560 : case GIMPLE_OMP_STRUCTURED_BLOCK:
15730 84560 : case GIMPLE_OMP_MASTER:
15731 84560 : case GIMPLE_OMP_MASKED:
15732 84560 : case GIMPLE_OMP_ORDERED:
15733 84560 : case GIMPLE_OMP_SCAN:
15734 84560 : case GIMPLE_OMP_CRITICAL:
15735 84560 : case GIMPLE_OMP_TARGET:
15736 84560 : case GIMPLE_OMP_TEAMS:
15737 84560 : case GIMPLE_OMP_TASKGROUP:
15738 : /* The minimal context here is just the current OMP construct. */
15739 84560 : inner_context = stmt;
15740 84560 : wi->info = inner_context;
15741 84560 : walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
15742 84560 : wi->info = context;
15743 84560 : break;
15744 :
15745 53086 : case GIMPLE_OMP_FOR:
15746 53086 : inner_context = stmt;
15747 53086 : wi->info = inner_context;
15748 : /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
15749 : walk them. */
15750 53086 : walk_gimple_seq (gimple_omp_for_pre_body (stmt),
15751 : diagnose_sb_1, NULL, wi);
15752 53086 : walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
15753 53086 : wi->info = context;
15754 53086 : break;
15755 :
15756 563857 : case GIMPLE_LABEL:
15757 1127714 : splay_tree_insert (all_labels,
15758 563857 : (splay_tree_key) gimple_label_label (
15759 563857 : as_a <glabel *> (stmt)),
15760 : (splay_tree_value) context);
15761 563857 : break;
15762 :
15763 : default:
15764 : break;
15765 : }
15766 :
15767 3395322 : return NULL_TREE;
15768 : }
15769 :
15770 : /* Pass 2: Check each branch and see if its context differs from that of
15771 : the destination label's context. */
15772 :
15773 : static tree
15774 3395322 : diagnose_sb_2 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
15775 : struct walk_stmt_info *wi)
15776 : {
15777 3395322 : gimple *context = (gimple *) wi->info;
15778 3395322 : splay_tree_node n;
15779 3395322 : gimple *stmt = gsi_stmt (*gsi_p);
15780 :
15781 3395322 : *handled_ops_p = true;
15782 :
15783 3395322 : switch (gimple_code (stmt))
15784 : {
15785 327782 : WALK_SUBSTMTS;
15786 :
15787 84560 : case GIMPLE_OMP_PARALLEL:
15788 84560 : case GIMPLE_OMP_TASK:
15789 84560 : case GIMPLE_OMP_SCOPE:
15790 84560 : case GIMPLE_OMP_SECTIONS:
15791 84560 : case GIMPLE_OMP_SINGLE:
15792 84560 : case GIMPLE_OMP_SECTION:
15793 84560 : case GIMPLE_OMP_STRUCTURED_BLOCK:
15794 84560 : case GIMPLE_OMP_MASTER:
15795 84560 : case GIMPLE_OMP_MASKED:
15796 84560 : case GIMPLE_OMP_ORDERED:
15797 84560 : case GIMPLE_OMP_SCAN:
15798 84560 : case GIMPLE_OMP_CRITICAL:
15799 84560 : case GIMPLE_OMP_TARGET:
15800 84560 : case GIMPLE_OMP_TEAMS:
15801 84560 : case GIMPLE_OMP_TASKGROUP:
15802 84560 : wi->info = stmt;
15803 84560 : walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
15804 84560 : wi->info = context;
15805 84560 : break;
15806 :
15807 53086 : case GIMPLE_OMP_FOR:
15808 53086 : wi->info = stmt;
15809 : /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
15810 : walk them. */
15811 53086 : walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt),
15812 : diagnose_sb_2, NULL, wi);
15813 53086 : walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
15814 53086 : wi->info = context;
15815 53086 : break;
15816 :
15817 208786 : case GIMPLE_COND:
15818 208786 : {
15819 208786 : gcond *cond_stmt = as_a <gcond *> (stmt);
15820 208786 : tree lab = gimple_cond_true_label (cond_stmt);
15821 208786 : if (lab)
15822 : {
15823 208786 : n = splay_tree_lookup (all_labels,
15824 : (splay_tree_key) lab);
15825 417572 : diagnose_sb_0 (gsi_p, context,
15826 208786 : n ? (gimple *) n->value : NULL);
15827 : }
15828 208786 : lab = gimple_cond_false_label (cond_stmt);
15829 208786 : if (lab)
15830 : {
15831 208786 : n = splay_tree_lookup (all_labels,
15832 : (splay_tree_key) lab);
15833 417572 : diagnose_sb_0 (gsi_p, context,
15834 208786 : n ? (gimple *) n->value : NULL);
15835 : }
15836 : }
15837 : break;
15838 :
15839 113245 : case GIMPLE_GOTO:
15840 113245 : {
15841 113245 : tree lab = gimple_goto_dest (stmt);
15842 113245 : if (TREE_CODE (lab) != LABEL_DECL)
15843 : break;
15844 :
15845 113245 : n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
15846 113245 : diagnose_sb_0 (gsi_p, context, n ? (gimple *) n->value : NULL);
15847 : }
15848 113245 : break;
15849 :
15850 433 : case GIMPLE_SWITCH:
15851 433 : {
15852 433 : gswitch *switch_stmt = as_a <gswitch *> (stmt);
15853 433 : unsigned int i;
15854 2205 : for (i = 0; i < gimple_switch_num_labels (switch_stmt); ++i)
15855 : {
15856 1355 : tree lab = CASE_LABEL (gimple_switch_label (switch_stmt, i));
15857 1355 : n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
15858 1355 : if (n && diagnose_sb_0 (gsi_p, context, (gimple *) n->value))
15859 : break;
15860 : }
15861 : }
15862 : break;
15863 :
15864 756 : case GIMPLE_ASM:
15865 756 : {
15866 756 : gasm *asm_stmt = as_a <gasm *> (stmt);
15867 1527 : for (unsigned i = 0; i < gimple_asm_nlabels (asm_stmt); ++i)
15868 : {
15869 21 : tree lab = TREE_VALUE (gimple_asm_label_op (asm_stmt, i));
15870 21 : n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
15871 21 : if (n && diagnose_sb_0 (gsi_p, context, (gimple *) n->value))
15872 : break;
15873 : }
15874 : }
15875 : break;
15876 :
15877 36191 : case GIMPLE_RETURN:
15878 36191 : diagnose_sb_0 (gsi_p, context, NULL);
15879 36191 : break;
15880 :
15881 : default:
15882 : break;
15883 : }
15884 :
15885 3395322 : return NULL_TREE;
15886 : }
15887 :
15888 : static unsigned int
15889 59562 : diagnose_omp_structured_block_errors (void)
15890 : {
15891 59562 : struct walk_stmt_info wi;
15892 59562 : gimple_seq body = gimple_body (current_function_decl);
15893 :
15894 59562 : all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0);
15895 :
15896 59562 : memset (&wi, 0, sizeof (wi));
15897 59562 : walk_gimple_seq (body, diagnose_sb_1, NULL, &wi);
15898 :
15899 59562 : memset (&wi, 0, sizeof (wi));
15900 59562 : wi.want_locations = true;
15901 59562 : walk_gimple_seq_mod (&body, diagnose_sb_2, NULL, &wi);
15902 :
15903 59562 : gimple_set_body (current_function_decl, body);
15904 :
15905 59562 : splay_tree_delete (all_labels);
15906 59562 : all_labels = NULL;
15907 :
15908 59562 : return 0;
15909 : }
15910 :
15911 : namespace {
15912 :
15913 : const pass_data pass_data_diagnose_omp_blocks =
15914 : {
15915 : GIMPLE_PASS, /* type */
15916 : "*diagnose_omp_blocks", /* name */
15917 : OPTGROUP_OMP, /* optinfo_flags */
15918 : TV_NONE, /* tv_id */
15919 : PROP_gimple_any, /* properties_required */
15920 : 0, /* properties_provided */
15921 : 0, /* properties_destroyed */
15922 : 0, /* todo_flags_start */
15923 : 0, /* todo_flags_finish */
15924 : };
15925 :
15926 : class pass_diagnose_omp_blocks : public gimple_opt_pass
15927 : {
15928 : public:
15929 297180 : pass_diagnose_omp_blocks (gcc::context *ctxt)
15930 594360 : : gimple_opt_pass (pass_data_diagnose_omp_blocks, ctxt)
15931 : {}
15932 :
15933 : /* opt_pass methods: */
15934 3010155 : bool gate (function *) final override
15935 : {
15936 3010155 : return flag_openacc || flag_openmp || flag_openmp_simd;
15937 : }
15938 59562 : unsigned int execute (function *) final override
15939 : {
15940 59562 : return diagnose_omp_structured_block_errors ();
15941 : }
15942 :
15943 : }; // class pass_diagnose_omp_blocks
15944 :
15945 : } // anon namespace
15946 :
15947 : gimple_opt_pass *
15948 297180 : make_pass_diagnose_omp_blocks (gcc::context *ctxt)
15949 : {
15950 297180 : return new pass_diagnose_omp_blocks (ctxt);
15951 : }
15952 :
15953 :
15954 : #include "gt-omp-low.h"
|