Line data Source code
1 : /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 : calls to the runtime library (libgomp), data marshalling to implement data
3 : sharing and copying clauses, offloading to accelerators, and more.
4 :
5 : Contributed by Diego Novillo <dnovillo@redhat.com>
6 :
7 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
8 :
9 : This file is part of GCC.
10 :
11 : GCC is free software; you can redistribute it and/or modify it under
12 : the terms of the GNU General Public License as published by the Free
13 : Software Foundation; either version 3, or (at your option) any later
14 : version.
15 :
16 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 : for more details.
20 :
21 : You should have received a copy of the GNU General Public License
22 : along with GCC; see the file COPYING3. If not see
23 : <http://www.gnu.org/licenses/>. */
24 :
25 : #include "config.h"
26 : #include "system.h"
27 : #include "coretypes.h"
28 : #include "backend.h"
29 : #include "target.h"
30 : #include "tree.h"
31 : #include "gimple.h"
32 : #include "tree-pass.h"
33 : #include "ssa.h"
34 : #include "cgraph.h"
35 : #include "pretty-print.h"
36 : #include "diagnostic-core.h"
37 : #include "fold-const.h"
38 : #include "stor-layout.h"
39 : #include "internal-fn.h"
40 : #include "gimple-iterator.h"
41 : #include "gimple-fold.h"
42 : #include "gimplify.h"
43 : #include "gimplify-me.h"
44 : #include "gimple-walk.h"
45 : #include "tree-iterator.h"
46 : #include "tree-inline.h"
47 : #include "langhooks.h"
48 : #include "tree-dfa.h"
49 : #include "tree-ssa.h"
50 : #include "splay-tree.h"
51 : #include "omp-general.h"
52 : #include "omp-low.h"
53 : #include "gimple-low.h"
54 : #include "alloc-pool.h"
55 : #include "symbol-summary.h"
56 : #include "tree-nested.h"
57 : #include "context.h"
58 : #include "gomp-constants.h"
59 : #include "gimple-pretty-print.h"
60 : #include "stringpool.h"
61 : #include "attribs.h"
62 : #include "omp-offload.h"
63 :
64 : /* Lowering of OMP parallel and workshare constructs proceeds in two
65 : phases. The first phase scans the function looking for OMP statements
66 : and then for variables that must be replaced to satisfy data sharing
67 : clauses. The second phase expands code for the constructs, as well as
68 : re-gimplifying things when variables have been replaced with complex
69 : expressions.
70 :
71 : Final code generation is done by pass_expand_omp. The flowgraph is
72 : scanned for regions which are then moved to a new
73 : function, to be invoked by the thread library, or offloaded. */
74 :
75 : /* Context structure. Used to store information about each parallel
76 : directive in the code. */
77 :
78 : struct omp_context
79 : {
80 : /* This field must be at the beginning, as we do "inheritance": Some
81 : callback functions for tree-inline.cc (e.g., omp_copy_decl)
82 : receive a copy_body_data pointer that is up-casted to an
83 : omp_context pointer. */
84 : copy_body_data cb;
85 :
86 : /* The tree of contexts corresponding to the encountered constructs. */
87 : struct omp_context *outer;
88 : gimple *stmt;
89 :
90 : /* Map variables to fields in a structure that allows communication
91 : between sending and receiving threads. */
92 : splay_tree field_map;
93 : tree record_type;
94 : tree sender_decl;
95 : tree receiver_decl;
96 :
97 : /* These are used just by task contexts, if task firstprivate fn is
98 : needed. srecord_type is used to communicate from the thread
99 : that encountered the task construct to task firstprivate fn,
100 : record_type is allocated by GOMP_task, initialized by task firstprivate
101 : fn and passed to the task body fn. */
102 : splay_tree sfield_map;
103 : tree srecord_type;
104 :
105 : /* A chain of variables to add to the top-level block surrounding the
106 : construct. In the case of a parallel, this is in the child function. */
107 : tree block_vars;
108 :
109 : /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
110 : barriers should jump to during omplower pass. */
111 : tree cancel_label;
112 :
113 : /* The sibling GIMPLE_OMP_FOR simd with _simt_ clause or NULL
114 : otherwise. */
115 : gimple *simt_stmt;
116 :
117 : /* For task reductions registered in this context, a vector containing
118 : the length of the private copies block (if constant, otherwise NULL)
119 : and then offsets (if constant, otherwise NULL) for each entry. */
120 : vec<tree> task_reductions;
121 :
122 : /* A hash map from the reduction clauses to the registered array
123 : elts. */
124 : hash_map<tree, unsigned> *task_reduction_map;
125 :
126 : /* And a hash map from the lastprivate(conditional:) variables to their
127 : corresponding tracking loop iteration variables. */
128 : hash_map<tree, tree> *lastprivate_conditional_map;
129 :
130 : /* And a hash map from the allocate variables to their corresponding
131 : allocators. */
132 : hash_map<tree, tree> *allocate_map;
133 :
134 : /* A tree_list of the reduction clauses in this context. This is
135 : only used for checking the consistency of OpenACC reduction
136 : clauses in scan_omp_for and is not guaranteed to contain a valid
137 : value outside of this function. */
138 : tree local_reduction_clauses;
139 :
140 : /* A tree_list of the reduction clauses in outer contexts. This is
141 : only used for checking the consistency of OpenACC reduction
142 : clauses in scan_omp_for and is not guaranteed to contain a valid
143 : value outside of this function. */
144 : tree outer_reduction_clauses;
145 :
146 : /* Nesting depth of this context. Used to beautify error messages re
147 : invalid gotos. The outermost ctx is depth 1, with depth 0 being
148 : reserved for the main body of the function. */
149 : int depth;
150 :
151 : /* True if this parallel directive is nested within another. */
152 : bool is_nested;
153 :
154 : /* True if this construct can be cancelled. */
155 : bool cancellable;
156 :
157 : /* True if lower_omp_1 should look up lastprivate conditional in parent
158 : context. */
159 : bool combined_into_simd_safelen1;
160 :
161 : /* True if there is nested scan context with inclusive clause. */
162 : bool scan_inclusive;
163 :
164 : /* True if there is nested scan context with exclusive clause. */
165 : bool scan_exclusive;
166 :
167 : /* True in the second simd loop of for simd with inscan reductions. */
168 : bool for_simd_scan_phase;
169 :
170 : /* True if there is order(concurrent) clause on the construct. */
171 : bool order_concurrent;
172 :
173 : /* True if there is bind clause on the construct (i.e. a loop construct). */
174 : bool loop_p;
175 :
176 : /* Only used for omp target contexts. True if a teams construct is
177 : strictly nested in it. */
178 : bool teams_nested_p;
179 :
180 : /* Only used for omp target contexts. True if an OpenMP construct other
181 : than teams is strictly nested in it. */
182 : bool nonteams_nested_p;
183 :
184 : /* Candidates for adjusting OpenACC privatization level. */
185 : vec<tree> oacc_privatization_candidates;
186 : };
187 :
188 : static splay_tree all_contexts;
189 : static int taskreg_nesting_level;
190 : static int target_nesting_level;
191 : static bitmap make_addressable_vars;
192 : static bitmap global_nonaddressable_vars;
193 : static vec<omp_context *> taskreg_contexts;
194 : static vec<gomp_task *> task_cpyfns;
195 :
196 : static void scan_omp (gimple_seq *, omp_context *);
197 : static tree scan_omp_1_op (tree *, int *, void *);
198 : static bool omp_maybe_offloaded_ctx (omp_context *ctx);
199 :
200 : #define WALK_SUBSTMTS \
201 : case GIMPLE_BIND: \
202 : case GIMPLE_TRY: \
203 : case GIMPLE_CATCH: \
204 : case GIMPLE_EH_FILTER: \
205 : case GIMPLE_ASSUME: \
206 : case GIMPLE_TRANSACTION: \
207 : /* The sub-statements for these should be walked. */ \
208 : *handled_ops_p = false; \
209 : break;
210 :
211 : /* Return whether CTX represents an OpenACC 'parallel' or 'serial' construct.
212 : (This doesn't include OpenACC 'kernels' decomposed parts.) */
213 :
214 : static bool
215 33658 : is_oacc_parallel_or_serial (omp_context *ctx)
216 : {
217 33658 : enum gimple_code outer_type = gimple_code (ctx->stmt);
218 33658 : return ((outer_type == GIMPLE_OMP_TARGET)
219 33658 : && ((gimple_omp_target_kind (ctx->stmt)
220 : == GF_OMP_TARGET_KIND_OACC_PARALLEL)
221 15316 : || (gimple_omp_target_kind (ctx->stmt)
222 33658 : == GF_OMP_TARGET_KIND_OACC_SERIAL)));
223 : }
224 :
225 : /* Return whether CTX represents an OpenACC 'kernels' construct.
226 : (This doesn't include OpenACC 'kernels' decomposed parts.) */
227 :
228 : static bool
229 49900 : is_oacc_kernels (omp_context *ctx)
230 : {
231 49900 : enum gimple_code outer_type = gimple_code (ctx->stmt);
232 49900 : return ((outer_type == GIMPLE_OMP_TARGET)
233 49900 : && (gimple_omp_target_kind (ctx->stmt)
234 49900 : == GF_OMP_TARGET_KIND_OACC_KERNELS));
235 : }
236 :
237 : /* Return whether CTX represents an OpenACC 'kernels' decomposed part. */
238 :
239 : static bool
240 21645 : is_oacc_kernels_decomposed_part (omp_context *ctx)
241 : {
242 21645 : enum gimple_code outer_type = gimple_code (ctx->stmt);
243 21645 : return ((outer_type == GIMPLE_OMP_TARGET)
244 21645 : && ((gimple_omp_target_kind (ctx->stmt)
245 : == GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED)
246 14552 : || (gimple_omp_target_kind (ctx->stmt)
247 : == GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE)
248 14552 : || (gimple_omp_target_kind (ctx->stmt)
249 21645 : == GF_OMP_TARGET_KIND_OACC_DATA_KERNELS)));
250 : }
251 :
252 : /* Return true if STMT corresponds to an OpenMP target region. */
253 : static bool
254 42898 : is_omp_target (gimple *stmt)
255 : {
256 42898 : if (gimple_code (stmt) == GIMPLE_OMP_TARGET)
257 : {
258 6511 : int kind = gimple_omp_target_kind (stmt);
259 6511 : return (kind == GF_OMP_TARGET_KIND_REGION
260 6511 : || kind == GF_OMP_TARGET_KIND_DATA
261 6511 : || kind == GF_OMP_TARGET_KIND_ENTER_DATA
262 11147 : || kind == GF_OMP_TARGET_KIND_EXIT_DATA);
263 : }
264 : return false;
265 : }
266 :
267 : /* If DECL is the artificial dummy VAR_DECL created for non-static
268 : data member privatization, return the underlying "this" parameter,
269 : otherwise return NULL. */
270 :
271 : tree
272 536974 : omp_member_access_dummy_var (tree decl)
273 : {
274 536974 : if (!VAR_P (decl)
275 329509 : || !DECL_ARTIFICIAL (decl)
276 161805 : || !DECL_IGNORED_P (decl)
277 156377 : || !DECL_HAS_VALUE_EXPR_P (decl)
278 549759 : || !lang_hooks.decls.omp_disregard_value_expr (decl, false))
279 534714 : return NULL_TREE;
280 :
281 2260 : tree v = DECL_VALUE_EXPR (decl);
282 2260 : if (TREE_CODE (v) != COMPONENT_REF)
283 : return NULL_TREE;
284 :
285 16164 : while (1)
286 9197 : switch (TREE_CODE (v))
287 : {
288 6967 : case COMPONENT_REF:
289 6967 : case MEM_REF:
290 6967 : case INDIRECT_REF:
291 6967 : CASE_CONVERT:
292 6967 : case POINTER_PLUS_EXPR:
293 6967 : v = TREE_OPERAND (v, 0);
294 6967 : continue;
295 2211 : case PARM_DECL:
296 2211 : if (DECL_CONTEXT (v) == current_function_decl
297 2211 : && DECL_ARTIFICIAL (v)
298 4422 : && TREE_CODE (TREE_TYPE (v)) == POINTER_TYPE)
299 : return v;
300 : return NULL_TREE;
301 : default:
302 : return NULL_TREE;
303 : }
304 : }
305 :
306 : /* Helper for unshare_and_remap, called through walk_tree. */
307 :
308 : static tree
309 1579 : unshare_and_remap_1 (tree *tp, int *walk_subtrees, void *data)
310 : {
311 1579 : tree *pair = (tree *) data;
312 1579 : if (*tp == pair[0])
313 : {
314 233 : *tp = unshare_expr (pair[1]);
315 233 : *walk_subtrees = 0;
316 : }
317 1346 : else if (IS_TYPE_OR_DECL_P (*tp))
318 244 : *walk_subtrees = 0;
319 1579 : return NULL_TREE;
320 : }
321 :
322 : /* Return unshare_expr (X) with all occurrences of FROM
323 : replaced with TO. */
324 :
325 : static tree
326 150 : unshare_and_remap (tree x, tree from, tree to)
327 : {
328 150 : tree pair[2] = { from, to };
329 150 : x = unshare_expr (x);
330 150 : walk_tree (&x, unshare_and_remap_1, pair, NULL);
331 150 : return x;
332 : }
333 :
334 : /* Convenience function for calling scan_omp_1_op on tree operands. */
335 :
336 : static inline tree
337 338788 : scan_omp_op (tree *tp, omp_context *ctx)
338 : {
339 338788 : struct walk_stmt_info wi;
340 :
341 338788 : memset (&wi, 0, sizeof (wi));
342 338788 : wi.info = ctx;
343 338788 : wi.want_locations = true;
344 :
345 338788 : return walk_tree (tp, scan_omp_1_op, &wi, NULL);
346 : }
347 :
348 : static void lower_omp (gimple_seq *, omp_context *);
349 : static tree lookup_decl_in_outer_ctx (tree, omp_context *);
350 : static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *);
351 :
352 : /* Return true if CTX is for an omp parallel. */
353 :
354 : static inline bool
355 774811 : is_parallel_ctx (omp_context *ctx)
356 : {
357 61094 : return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL;
358 : }
359 :
360 :
361 : /* Return true if CTX is for an omp task. */
362 :
363 : static inline bool
364 236018 : is_task_ctx (omp_context *ctx)
365 : {
366 38770 : return gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
367 : }
368 :
369 :
370 : /* Return true if CTX is for an omp taskloop. */
371 :
372 : static inline bool
373 8095 : is_taskloop_ctx (omp_context *ctx)
374 : {
375 8095 : return gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
376 8095 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_TASKLOOP;
377 : }
378 :
379 :
380 : /* Return true if CTX is for a host omp teams. */
381 :
382 : static inline bool
383 520306 : is_host_teams_ctx (omp_context *ctx)
384 : {
385 520306 : return gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
386 520306 : && gimple_omp_teams_host (as_a <gomp_teams *> (ctx->stmt));
387 : }
388 :
389 : /* Return true if CTX is for an omp parallel or omp task or host omp teams
390 : (the last one is strictly not a task region in OpenMP speak, but we
391 : need to treat it similarly). */
392 :
393 : static inline bool
394 662359 : is_taskreg_ctx (omp_context *ctx)
395 : {
396 662359 : return is_parallel_ctx (ctx) || is_task_ctx (ctx) || is_host_teams_ctx (ctx);
397 : }
398 :
399 : /* Return true if EXPR is variable sized. If IS_REF is true, then
400 : EXPR is assumed to be a reference and the object it refers
401 : to is checked instead. */
402 :
403 : static inline bool
404 904585 : is_variable_sized (const_tree expr, bool is_ref = false)
405 : {
406 904585 : if (is_ref)
407 2166 : expr = TREE_TYPE (expr);
408 904585 : return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr)));
409 : }
410 :
411 : /* Lookup variables. The "maybe" form
412 : allows for the variable form to not have been entered, otherwise we
413 : assert that the variable must have been entered. */
414 :
415 : static inline tree
416 714764 : lookup_decl (tree var, omp_context *ctx)
417 : {
418 1429429 : tree *n = ctx->cb.decl_map->get (var);
419 714764 : return *n;
420 : }
421 :
422 : static inline tree
423 717285 : maybe_lookup_decl (const_tree var, omp_context *ctx)
424 : {
425 717285 : tree *n = ctx->cb.decl_map->get (const_cast<tree> (var));
426 717285 : return n ? *n : NULL_TREE;
427 : }
428 :
429 : static inline tree
430 127113 : lookup_field (tree var, omp_context *ctx)
431 : {
432 127113 : splay_tree_node n;
433 254226 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
434 127113 : return (tree) n->value;
435 : }
436 :
437 : static inline tree
438 169225 : lookup_sfield (splay_tree_key key, omp_context *ctx)
439 : {
440 169225 : splay_tree_node n;
441 169225 : n = splay_tree_lookup (ctx->sfield_map
442 : ? ctx->sfield_map : ctx->field_map, key);
443 169225 : return (tree) n->value;
444 : }
445 :
446 : static inline tree
447 967 : lookup_sfield (tree var, omp_context *ctx)
448 : {
449 967 : return lookup_sfield ((splay_tree_key) var, ctx);
450 : }
451 :
452 : static inline tree
453 221384 : maybe_lookup_field (splay_tree_key key, omp_context *ctx)
454 : {
455 221384 : splay_tree_node n;
456 221384 : n = splay_tree_lookup (ctx->field_map, key);
457 221384 : return n ? (tree) n->value : NULL_TREE;
458 : }
459 :
460 : static inline tree
461 221384 : maybe_lookup_field (tree var, omp_context *ctx)
462 : {
463 2555 : return maybe_lookup_field ((splay_tree_key) var, ctx);
464 : }
465 :
466 : /* Return true if DECL should be copied by pointer. SHARED_CTX is
467 : the parallel context if DECL is to be shared. */
468 :
469 : static bool
470 231402 : use_pointer_for_field (tree decl, omp_context *shared_ctx)
471 : {
472 451284 : if (AGGREGATE_TYPE_P (TREE_TYPE (decl))
473 214755 : || TYPE_ATOMIC (TREE_TYPE (decl))
474 446157 : || POLY_INT_CST_P (DECL_SIZE (decl)))
475 : return true;
476 :
477 : /* We can only use copy-in/copy-out semantics for shared variables
478 : when we know the value is not accessible from an outer scope. */
479 214725 : if (shared_ctx)
480 : {
481 28809 : gcc_assert (!is_gimple_omp_oacc (shared_ctx->stmt));
482 :
483 : /* ??? Trivially accessible from anywhere. But why would we even
484 : be passing an address in this case? Should we simply assert
485 : this to be false, or should we have a cleanup pass that removes
486 : these from the list of mappings? */
487 28809 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, shared_ctx)))
488 : return true;
489 :
490 : /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
491 : without analyzing the expression whether or not its location
492 : is accessible to anyone else. In the case of nested parallel
493 : regions it certainly may be. */
494 28809 : if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl))
495 : return true;
496 :
497 : /* Do not use copy-in/copy-out for variables that have their
498 : address taken. */
499 27895 : if (is_global_var (decl))
500 : {
501 : /* For file scope vars, track whether we've seen them as
502 : non-addressable initially and in that case, keep the same
503 : answer for the duration of the pass, even when they are made
504 : addressable later on e.g. through reduction expansion. Global
505 : variables which weren't addressable before the pass will not
506 : have their privatized copies address taken. See PR91216. */
507 1232 : if (!TREE_ADDRESSABLE (decl))
508 : {
509 479 : if (!global_nonaddressable_vars)
510 74 : global_nonaddressable_vars = BITMAP_ALLOC (NULL);
511 479 : bitmap_set_bit (global_nonaddressable_vars, DECL_UID (decl));
512 : }
513 753 : else if (!global_nonaddressable_vars
514 986 : || !bitmap_bit_p (global_nonaddressable_vars,
515 233 : DECL_UID (decl)))
516 731 : return true;
517 : }
518 26663 : else if (TREE_ADDRESSABLE (decl))
519 : return true;
520 :
521 : /* lower_send_shared_vars only uses copy-in, but not copy-out
522 : for these. */
523 18103 : if (TREE_READONLY (decl)
524 18103 : || ((TREE_CODE (decl) == RESULT_DECL
525 11847 : || TREE_CODE (decl) == PARM_DECL)
526 673 : && DECL_BY_REFERENCE (decl)))
527 : return false;
528 :
529 : /* Disallow copy-in/out in nested parallel if
530 : decl is shared in outer parallel, otherwise
531 : each thread could store the shared variable
532 : in its own copy-in location, making the
533 : variable no longer really shared. */
534 11615 : if (shared_ctx->is_nested)
535 : {
536 2470 : omp_context *up;
537 :
538 5975 : for (up = shared_ctx->outer; up; up = up->outer)
539 5702 : if ((is_taskreg_ctx (up)
540 3643 : || (gimple_code (up->stmt) == GIMPLE_OMP_TARGET
541 411 : && is_gimple_omp_offloaded (up->stmt)))
542 6113 : && maybe_lookup_decl (decl, up))
543 : break;
544 :
545 2470 : if (up)
546 : {
547 2197 : tree c;
548 :
549 2197 : if (gimple_code (up->stmt) == GIMPLE_OMP_TARGET)
550 : {
551 309 : for (c = gimple_omp_target_clauses (up->stmt);
552 2031 : c; c = OMP_CLAUSE_CHAIN (c))
553 1932 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
554 1932 : && OMP_CLAUSE_DECL (c) == decl)
555 : break;
556 : }
557 : else
558 1888 : for (c = gimple_omp_taskreg_clauses (up->stmt);
559 6909 : c; c = OMP_CLAUSE_CHAIN (c))
560 6454 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
561 6454 : && OMP_CLAUSE_DECL (c) == decl)
562 : break;
563 :
564 2197 : if (c)
565 1643 : goto maybe_mark_addressable_and_ret;
566 : }
567 : }
568 :
569 : /* For tasks avoid using copy-in/out. As tasks can be
570 : deferred or executed in different thread, when GOMP_task
571 : returns, the task hasn't necessarily terminated. */
572 9972 : if (is_task_ctx (shared_ctx))
573 : {
574 2195 : tree outer;
575 552 : maybe_mark_addressable_and_ret:
576 2195 : outer = maybe_lookup_decl_in_outer_ctx (decl, shared_ctx);
577 2195 : if (is_gimple_reg (outer) && !omp_member_access_dummy_var (outer))
578 : {
579 : /* Taking address of OUTER in lower_send_shared_vars
580 : might need regimplification of everything that uses the
581 : variable. */
582 984 : if (!make_addressable_vars)
583 491 : make_addressable_vars = BITMAP_ALLOC (NULL);
584 984 : bitmap_set_bit (make_addressable_vars, DECL_UID (outer));
585 984 : TREE_ADDRESSABLE (outer) = 1;
586 : }
587 2195 : return true;
588 : }
589 : }
590 :
591 : return false;
592 : }
593 :
594 : /* Construct a new automatic decl similar to VAR. */
595 :
596 : static tree
597 229288 : omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx)
598 : {
599 229288 : tree copy = copy_var_decl (var, name, type);
600 :
601 229288 : DECL_CONTEXT (copy) = current_function_decl;
602 :
603 229288 : if (ctx)
604 : {
605 229007 : DECL_CHAIN (copy) = ctx->block_vars;
606 229007 : ctx->block_vars = copy;
607 : }
608 : else
609 281 : record_vars (copy);
610 :
611 : /* If VAR is listed in make_addressable_vars, it wasn't
612 : originally addressable, but was only later made so.
613 : We don't need to take address of privatizations
614 : from that var. */
615 229288 : if (TREE_ADDRESSABLE (var)
616 229288 : && ((make_addressable_vars
617 3469 : && bitmap_bit_p (make_addressable_vars, DECL_UID (var)))
618 39449 : || (global_nonaddressable_vars
619 515 : && bitmap_bit_p (global_nonaddressable_vars, DECL_UID (var)))))
620 2079 : TREE_ADDRESSABLE (copy) = 0;
621 :
622 229288 : return copy;
623 : }
624 :
625 : static tree
626 229288 : omp_copy_decl_1 (tree var, omp_context *ctx)
627 : {
628 229288 : return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx);
629 : }
630 :
631 : /* Build tree nodes to access the field for VAR on the receiver side. */
632 :
633 : static tree
634 122655 : build_receiver_ref (tree var, bool by_ref, omp_context *ctx)
635 : {
636 122655 : tree x, field = lookup_field (var, ctx);
637 :
638 : /* If the receiver record type was remapped in the child function,
639 : remap the field into the new record type. */
640 122655 : x = maybe_lookup_field (field, ctx);
641 122655 : if (x != NULL)
642 11167 : field = x;
643 :
644 122655 : x = build_simple_mem_ref (ctx->receiver_decl);
645 122655 : TREE_THIS_NOTRAP (x) = 1;
646 122655 : x = omp_build_component_ref (x, field);
647 122655 : if (by_ref)
648 : {
649 36110 : x = build_simple_mem_ref (x);
650 36110 : TREE_THIS_NOTRAP (x) = 1;
651 : }
652 :
653 122655 : return x;
654 : }
655 :
656 : /* Build tree nodes to access VAR in the scope outer to CTX. In the case
657 : of a parallel, this is a component reference; for workshare constructs
658 : this is some variable. */
659 :
660 : static tree
661 114110 : build_outer_var_ref (tree var, omp_context *ctx,
662 : enum omp_clause_code code = OMP_CLAUSE_ERROR)
663 : {
664 114110 : tree x;
665 114110 : omp_context *outer = ctx->outer;
666 114499 : for (; outer; outer = outer->outer)
667 : {
668 82151 : if (gimple_code (outer->stmt) == GIMPLE_OMP_TASKGROUP)
669 373 : continue;
670 81794 : if (gimple_code (outer->stmt) == GIMPLE_OMP_SCOPE
671 81778 : && !maybe_lookup_decl (var, outer))
672 16 : continue;
673 : break;
674 : }
675 :
676 114110 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx)))
677 : x = var;
678 105969 : else if (is_variable_sized (var))
679 : {
680 47 : x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0);
681 47 : x = build_outer_var_ref (x, ctx, code);
682 47 : x = build_simple_mem_ref (x);
683 : }
684 105922 : else if (is_taskreg_ctx (ctx))
685 : {
686 63671 : bool by_ref = use_pointer_for_field (var, NULL);
687 63671 : x = build_receiver_ref (var, by_ref, ctx);
688 : }
689 42251 : else if ((gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
690 40882 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
691 13789 : || ctx->loop_p
692 13016 : || code == OMP_CLAUSE_ALLOCATE
693 55105 : || (code == OMP_CLAUSE_PRIVATE
694 40 : && (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
695 : || gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS
696 : || gimple_code (ctx->stmt) == GIMPLE_OMP_SINGLE)))
697 : {
698 : /* #pragma omp simd isn't a worksharing construct, and can reference
699 : even private vars in its linear etc. clauses.
700 : Similarly for OMP_CLAUSE_PRIVATE with outer ref, that can refer
701 : to private vars in all worksharing constructs. */
702 29417 : x = NULL_TREE;
703 29417 : if (outer && is_taskreg_ctx (outer))
704 912 : x = lookup_decl (var, outer);
705 28505 : else if (outer)
706 22315 : x = maybe_lookup_decl_in_outer_ctx (var, ctx);
707 23227 : if (x == NULL_TREE)
708 : x = var;
709 : }
710 12834 : else if (code == OMP_CLAUSE_LASTPRIVATE && is_taskloop_ctx (ctx))
711 : {
712 689 : gcc_assert (outer);
713 689 : splay_tree_node n
714 1378 : = splay_tree_lookup (outer->field_map,
715 689 : (splay_tree_key) &DECL_UID (var));
716 689 : if (n == NULL)
717 : {
718 610 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, outer)))
719 : x = var;
720 : else
721 384 : x = lookup_decl (var, outer);
722 : }
723 : else
724 : {
725 79 : tree field = (tree) n->value;
726 : /* If the receiver record type was remapped in the child function,
727 : remap the field into the new record type. */
728 79 : x = maybe_lookup_field (field, outer);
729 79 : if (x != NULL)
730 0 : field = x;
731 :
732 79 : x = build_simple_mem_ref (outer->receiver_decl);
733 79 : x = omp_build_component_ref (x, field);
734 79 : if (use_pointer_for_field (var, outer))
735 59 : x = build_simple_mem_ref (x);
736 : }
737 : }
738 12145 : else if (outer)
739 11953 : x = lookup_decl (var, outer);
740 192 : else if (omp_privatize_by_reference (var))
741 : /* This can happen with orphaned constructs. If var is reference, it is
742 : possible it is shared and as such valid. */
743 : x = var;
744 94 : else if (omp_member_access_dummy_var (var))
745 : x = var;
746 : else
747 0 : gcc_unreachable ();
748 :
749 114110 : if (x == var)
750 : {
751 16238 : tree t = omp_member_access_dummy_var (var);
752 16238 : if (t)
753 : {
754 220 : x = DECL_VALUE_EXPR (var);
755 220 : tree o = maybe_lookup_decl_in_outer_ctx (t, ctx);
756 220 : if (o != t)
757 54 : x = unshare_and_remap (x, t, o);
758 : else
759 166 : x = unshare_expr (x);
760 : }
761 : }
762 :
763 114110 : if (omp_privatize_by_reference (var))
764 3032 : x = build_simple_mem_ref (x);
765 :
766 114110 : return x;
767 : }
768 :
769 : /* Build tree nodes to access the field for VAR on the sender side. */
770 :
771 : static tree
772 168179 : build_sender_ref (splay_tree_key key, omp_context *ctx)
773 : {
774 168179 : tree field = lookup_sfield (key, ctx);
775 168179 : tree tmp = ctx->sender_decl;
776 168179 : if (POINTER_TYPE_P (TREE_TYPE (tmp)))
777 451 : tmp = build_fold_indirect_ref (tmp);
778 168179 : return omp_build_component_ref (tmp, field);
779 : }
780 :
781 : static tree
782 164025 : build_sender_ref (tree var, omp_context *ctx)
783 : {
784 0 : return build_sender_ref ((splay_tree_key) var, ctx);
785 : }
786 :
787 : /* Add a new field for VAR inside the structure CTX. If BY_REF is true,
788 : use a pointer to the VAR rather than VAR itself.
789 : MASK is a bit mask of other options. Bits are interpreted as:
790 : 1: Install VAR in ctx->field_map.
791 : 2: Install VAR in ctx->sfield_map.
792 : 4: VAR is an array, convert it to a pointer.
793 : 8: Use DECL_UID (VAR) instead of VAR as key.
794 : 16: Use DECL_NAME (VAR) instead of VAR as key.
795 : 32: Don't dereference omp_is_reference types.
796 : KEY_EXPR allows specifying something other than VAR as the lookup key.
797 : If specified, it also overrides the 8 and 16 MASK bits. */
798 :
799 : static void
800 131003 : install_var_field (tree var, bool by_ref, int mask, omp_context *ctx,
801 : tree key_expr = NULL_TREE)
802 : {
803 131003 : tree field, type, sfield = NULL_TREE;
804 131003 : splay_tree_key key = (splay_tree_key) var;
805 :
806 131003 : if (key_expr)
807 : /* Allow caller to explicitly set the expression used as the key. */
808 44762 : key = (splay_tree_key) key_expr;
809 : else
810 : {
811 86241 : if ((mask & 16) != 0)
812 : {
813 624 : key = (splay_tree_key) &DECL_NAME (var);
814 624 : gcc_checking_assert (key != (splay_tree_key) var);
815 : }
816 86241 : if ((mask & 8) != 0)
817 : {
818 1619 : key = (splay_tree_key) &DECL_UID (var);
819 1619 : gcc_checking_assert (key != (splay_tree_key) var);
820 : }
821 : }
822 131003 : gcc_assert ((mask & 1) == 0
823 : || !splay_tree_lookup (ctx->field_map, key));
824 131003 : gcc_assert ((mask & 2) == 0 || !ctx->sfield_map
825 : || !splay_tree_lookup (ctx->sfield_map, key));
826 131003 : gcc_assert ((mask & 3) == 3
827 : || !is_gimple_omp_oacc (ctx->stmt));
828 :
829 131003 : type = TREE_TYPE (var);
830 131003 : if ((mask & 16) != 0)
831 624 : type = lang_hooks.decls.omp_array_data (var, true);
832 :
833 : /* Prevent redeclaring the var in the split-off function with a restrict
834 : pointer type. Note that we only clear type itself, restrict qualifiers in
835 : the pointed-to type will be ignored by points-to analysis. */
836 131003 : if (POINTER_TYPE_P (type)
837 131003 : && TYPE_RESTRICT (type))
838 6843 : type = build_qualified_type (type, TYPE_QUALS (type) & ~TYPE_QUAL_RESTRICT);
839 :
840 131003 : if (mask & 4)
841 : {
842 425 : gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
843 425 : type = build_pointer_type (build_pointer_type (type));
844 : }
845 130578 : else if (by_ref)
846 63972 : type = build_pointer_type (type);
847 66606 : else if ((mask & (32 | 3)) == 1
848 66606 : && omp_privatize_by_reference (var))
849 162 : type = TREE_TYPE (type);
850 :
851 131003 : field = build_decl (DECL_SOURCE_LOCATION (var),
852 131003 : FIELD_DECL, DECL_NAME (var), type);
853 :
854 : /* Remember what variable this field was created for. This does have a
855 : side effect of making dwarf2out ignore this member, so for helpful
856 : debugging we clear it later in delete_omp_context. */
857 131003 : DECL_ABSTRACT_ORIGIN (field) = var;
858 131003 : if ((mask & 16) == 0 && type == TREE_TYPE (var))
859 : {
860 62282 : SET_DECL_ALIGN (field, DECL_ALIGN (var));
861 62282 : DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
862 62282 : TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
863 : }
864 : else
865 68721 : SET_DECL_ALIGN (field, TYPE_ALIGN (type));
866 :
867 131003 : if ((mask & 3) == 3)
868 : {
869 129999 : insert_field_into_struct (ctx->record_type, field);
870 129999 : if (ctx->srecord_type)
871 : {
872 644 : sfield = build_decl (DECL_SOURCE_LOCATION (var),
873 644 : FIELD_DECL, DECL_NAME (var), type);
874 644 : DECL_ABSTRACT_ORIGIN (sfield) = var;
875 644 : SET_DECL_ALIGN (sfield, DECL_ALIGN (field));
876 644 : DECL_USER_ALIGN (sfield) = DECL_USER_ALIGN (field);
877 644 : TREE_THIS_VOLATILE (sfield) = TREE_THIS_VOLATILE (field);
878 644 : insert_field_into_struct (ctx->srecord_type, sfield);
879 : }
880 : }
881 : else
882 : {
883 1004 : if (ctx->srecord_type == NULL_TREE)
884 : {
885 520 : tree t;
886 :
887 520 : ctx->srecord_type = lang_hooks.types.make_type (RECORD_TYPE);
888 520 : ctx->sfield_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
889 2003 : for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t))
890 : {
891 1483 : sfield = build_decl (DECL_SOURCE_LOCATION (t),
892 1483 : FIELD_DECL, DECL_NAME (t), TREE_TYPE (t));
893 1483 : DECL_ABSTRACT_ORIGIN (sfield) = DECL_ABSTRACT_ORIGIN (t);
894 1483 : insert_field_into_struct (ctx->srecord_type, sfield);
895 2966 : splay_tree_insert (ctx->sfield_map,
896 1483 : (splay_tree_key) DECL_ABSTRACT_ORIGIN (t),
897 : (splay_tree_value) sfield);
898 : }
899 : }
900 1004 : sfield = field;
901 1004 : insert_field_into_struct ((mask & 1) ? ctx->record_type
902 : : ctx->srecord_type, field);
903 : }
904 :
905 131003 : if (mask & 1)
906 130640 : splay_tree_insert (ctx->field_map, key, (splay_tree_value) field);
907 131003 : if ((mask & 2) && ctx->sfield_map)
908 1007 : splay_tree_insert (ctx->sfield_map, key, (splay_tree_value) sfield);
909 131003 : }
910 :
911 : static tree
912 228857 : install_var_local (tree var, omp_context *ctx)
913 : {
914 228857 : tree new_var = omp_copy_decl_1 (var, ctx);
915 228857 : insert_decl_map (&ctx->cb, var, new_var);
916 228857 : return new_var;
917 : }
918 :
919 : /* Adjust the replacement for DECL in CTX for the new context. This means
920 : copying the DECL_VALUE_EXPR, and fixing up the type. */
921 :
922 : static void
923 204045 : fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug)
924 : {
925 204045 : tree new_decl, size;
926 :
927 204045 : new_decl = lookup_decl (decl, ctx);
928 :
929 204045 : TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), &ctx->cb);
930 :
931 406957 : if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug)
932 204234 : && DECL_HAS_VALUE_EXPR_P (decl))
933 : {
934 1322 : tree ve = DECL_VALUE_EXPR (decl);
935 1322 : walk_tree (&ve, copy_tree_body_r, &ctx->cb, NULL);
936 1322 : SET_DECL_VALUE_EXPR (new_decl, ve);
937 1322 : DECL_HAS_VALUE_EXPR_P (new_decl) = 1;
938 : }
939 :
940 204045 : if (!TREE_CONSTANT (DECL_SIZE (new_decl)))
941 : {
942 1133 : size = remap_decl (DECL_SIZE (decl), &ctx->cb);
943 1133 : if (size == error_mark_node)
944 316 : size = TYPE_SIZE (TREE_TYPE (new_decl));
945 1133 : DECL_SIZE (new_decl) = size;
946 :
947 1133 : size = remap_decl (DECL_SIZE_UNIT (decl), &ctx->cb);
948 1133 : if (size == error_mark_node)
949 303 : size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl));
950 1133 : DECL_SIZE_UNIT (new_decl) = size;
951 : }
952 204045 : }
953 :
954 : /* The callback for remap_decl. Search all containing contexts for a
955 : mapping of the variable; this avoids having to duplicate the splay
956 : tree ahead of time. We know a mapping doesn't already exist in the
957 : given context. Create new mappings to implement default semantics. */
958 :
959 : static tree
960 462528 : omp_copy_decl (tree var, copy_body_data *cb)
961 : {
962 462528 : omp_context *ctx = (omp_context *) cb;
963 462528 : tree new_var;
964 :
965 462528 : if (TREE_CODE (var) == LABEL_DECL)
966 : {
967 187875 : if (FORCED_LABEL (var) || DECL_NONLOCAL (var))
968 : return var;
969 187869 : new_var = create_artificial_label (DECL_SOURCE_LOCATION (var));
970 187869 : DECL_CONTEXT (new_var) = current_function_decl;
971 187869 : insert_decl_map (&ctx->cb, var, new_var);
972 187869 : return new_var;
973 : }
974 :
975 356477 : while (!is_taskreg_ctx (ctx))
976 : {
977 337291 : ctx = ctx->outer;
978 337291 : if (ctx == NULL)
979 : return var;
980 279023 : new_var = maybe_lookup_decl (var, ctx);
981 279023 : if (new_var)
982 : return new_var;
983 : }
984 :
985 19186 : if (is_global_var (var) || decl_function_context (var) != ctx->cb.src_fn)
986 13365 : return var;
987 :
988 5821 : return error_mark_node;
989 : }
990 :
991 : /* Create a new context, with OUTER_CTX being the surrounding context. */
992 :
993 : static omp_context *
994 137599 : new_omp_context (gimple *stmt, omp_context *outer_ctx)
995 : {
996 137599 : omp_context *ctx = XCNEW (omp_context);
997 :
998 137599 : splay_tree_insert (all_contexts, (splay_tree_key) stmt,
999 : (splay_tree_value) ctx);
1000 137599 : ctx->stmt = stmt;
1001 :
1002 137599 : if (outer_ctx)
1003 : {
1004 74386 : ctx->outer = outer_ctx;
1005 74386 : ctx->cb = outer_ctx->cb;
1006 74386 : ctx->cb.block = NULL;
1007 74386 : ctx->depth = outer_ctx->depth + 1;
1008 : }
1009 : else
1010 : {
1011 63213 : ctx->cb.src_fn = current_function_decl;
1012 63213 : ctx->cb.dst_fn = current_function_decl;
1013 63213 : ctx->cb.src_node = cgraph_node::get (current_function_decl);
1014 63213 : gcc_checking_assert (ctx->cb.src_node);
1015 63213 : ctx->cb.dst_node = ctx->cb.src_node;
1016 63213 : ctx->cb.src_cfun = cfun;
1017 63213 : ctx->cb.copy_decl = omp_copy_decl;
1018 63213 : ctx->cb.eh_lp_nr = 0;
1019 63213 : ctx->cb.transform_call_graph_edges = CB_CGE_MOVE;
1020 63213 : ctx->cb.adjust_array_error_bounds = true;
1021 63213 : ctx->cb.dont_remap_vla_if_no_change = true;
1022 63213 : ctx->depth = 1;
1023 : }
1024 :
1025 137599 : ctx->cb.decl_map = new hash_map<tree, tree>;
1026 :
1027 137599 : return ctx;
1028 : }
1029 :
1030 : static gimple_seq maybe_catch_exception (gimple_seq);
1031 :
1032 : /* Finalize task copyfn. */
1033 :
1034 : static void
1035 480 : finalize_task_copyfn (gomp_task *task_stmt)
1036 : {
1037 480 : struct function *child_cfun;
1038 480 : tree child_fn;
1039 480 : gimple_seq seq = NULL, new_seq;
1040 480 : gbind *bind;
1041 :
1042 480 : child_fn = gimple_omp_task_copy_fn (task_stmt);
1043 480 : if (child_fn == NULL_TREE)
1044 0 : return;
1045 :
1046 480 : child_cfun = DECL_STRUCT_FUNCTION (child_fn);
1047 480 : DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
1048 :
1049 480 : push_cfun (child_cfun);
1050 480 : bind = gimplify_body (child_fn, false);
1051 480 : gimple_seq_add_stmt (&seq, bind);
1052 480 : new_seq = maybe_catch_exception (seq);
1053 480 : if (new_seq != seq)
1054 : {
1055 332 : bind = gimple_build_bind (NULL, new_seq, NULL);
1056 332 : seq = NULL;
1057 332 : gimple_seq_add_stmt (&seq, bind);
1058 : }
1059 480 : gimple_set_body (child_fn, seq);
1060 480 : pop_cfun ();
1061 :
1062 : /* Inform the callgraph about the new function. */
1063 480 : cgraph_node *node = cgraph_node::get_create (child_fn);
1064 480 : node->parallelized_function = 1;
1065 480 : cgraph_node::add_new_function (child_fn, false);
1066 : }
1067 :
1068 : /* Destroy a omp_context data structures. Called through the splay tree
1069 : value delete callback. */
1070 :
1071 : static void
1072 137593 : delete_omp_context (splay_tree_value value)
1073 : {
1074 137593 : omp_context *ctx = (omp_context *) value;
1075 :
1076 275186 : delete ctx->cb.decl_map;
1077 :
1078 137593 : if (ctx->field_map)
1079 69065 : splay_tree_delete (ctx->field_map);
1080 137593 : if (ctx->sfield_map)
1081 520 : splay_tree_delete (ctx->sfield_map);
1082 :
1083 : /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1084 : it produces corrupt debug information. */
1085 137593 : if (ctx->record_type)
1086 : {
1087 53353 : tree t;
1088 222194 : for (t = TYPE_FIELDS (ctx->record_type); t ; t = DECL_CHAIN (t))
1089 168841 : DECL_ABSTRACT_ORIGIN (t) = NULL;
1090 : }
1091 137593 : if (ctx->srecord_type)
1092 : {
1093 520 : tree t;
1094 3010 : for (t = TYPE_FIELDS (ctx->srecord_type); t ; t = DECL_CHAIN (t))
1095 2490 : DECL_ABSTRACT_ORIGIN (t) = NULL;
1096 : }
1097 :
1098 137593 : if (ctx->task_reduction_map)
1099 : {
1100 1165 : ctx->task_reductions.release ();
1101 2330 : delete ctx->task_reduction_map;
1102 : }
1103 :
1104 137860 : delete ctx->lastprivate_conditional_map;
1105 138891 : delete ctx->allocate_map;
1106 :
1107 137593 : XDELETE (ctx);
1108 137593 : }
1109 :
1110 : /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1111 : context. */
1112 :
1113 : static void
1114 36742 : fixup_child_record_type (omp_context *ctx)
1115 : {
1116 36742 : tree f, type = ctx->record_type;
1117 :
1118 36742 : if (!ctx->receiver_decl)
1119 : return;
1120 : /* ??? It isn't sufficient to just call remap_type here, because
1121 : variably_modified_type_p doesn't work the way we expect for
1122 : record types. Testing each field for whether it needs remapping
1123 : and creating a new record by hand works, however. */
1124 161362 : for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
1125 125889 : if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
1126 : break;
1127 36742 : if (f)
1128 : {
1129 1269 : tree name, new_fields = NULL;
1130 :
1131 1269 : type = lang_hooks.types.make_type (RECORD_TYPE);
1132 1269 : name = DECL_NAME (TYPE_NAME (ctx->record_type));
1133 1269 : name = build_decl (DECL_SOURCE_LOCATION (ctx->receiver_decl),
1134 : TYPE_DECL, name, type);
1135 1269 : TYPE_NAME (type) = name;
1136 :
1137 13448 : for (f = TYPE_FIELDS (ctx->record_type); f ; f = DECL_CHAIN (f))
1138 : {
1139 12179 : tree new_f = copy_node (f);
1140 12179 : DECL_CONTEXT (new_f) = type;
1141 12179 : TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &ctx->cb);
1142 12179 : DECL_CHAIN (new_f) = new_fields;
1143 12179 : walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &ctx->cb, NULL);
1144 12179 : walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r,
1145 : &ctx->cb, NULL);
1146 12179 : walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
1147 : &ctx->cb, NULL);
1148 12179 : new_fields = new_f;
1149 :
1150 : /* Arrange to be able to look up the receiver field
1151 : given the sender field. */
1152 12179 : splay_tree_insert (ctx->field_map, (splay_tree_key) f,
1153 : (splay_tree_value) new_f);
1154 : }
1155 1269 : TYPE_FIELDS (type) = nreverse (new_fields);
1156 1269 : layout_type (type);
1157 : }
1158 :
1159 : /* In a target region we never modify any of the pointers in *.omp_data_i,
1160 : so attempt to help the optimizers. */
1161 36742 : if (is_gimple_omp_offloaded (ctx->stmt))
1162 17601 : type = build_qualified_type (type, TYPE_QUAL_CONST);
1163 :
1164 36742 : TREE_TYPE (ctx->receiver_decl)
1165 110226 : = build_qualified_type (flexible_array_type_p (type)
1166 107 : ? build_pointer_type (type)
1167 36635 : : build_reference_type (type), TYPE_QUAL_RESTRICT);
1168 : }
1169 :
1170 : /* Instantiate decls as necessary in CTX to satisfy the data sharing
1171 : specified by CLAUSES. */
1172 :
1173 : static void
1174 129746 : scan_sharing_clauses (tree clauses, omp_context *ctx)
1175 : {
1176 129746 : tree c, decl;
1177 129746 : bool scan_array_reductions = false;
1178 129746 : bool flex_array_ptr = false;
1179 :
1180 549406 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1181 419660 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_ALLOCATE
1182 419660 : && (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) == NULL_TREE
1183 : /* omp_default_mem_alloc is 1 */
1184 1137 : || !integer_onep (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c))
1185 572 : || OMP_CLAUSE_ALLOCATE_ALIGN (c) != NULL_TREE))
1186 : {
1187 : /* The allocate clauses that appear on a target construct or on
1188 : constructs in a target region must specify an allocator expression
1189 : unless a requires directive with the dynamic_allocators clause
1190 : is present in the same compilation unit. */
1191 1629 : if (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) == NULL_TREE
1192 1020 : && ((omp_requires_mask & OMP_REQUIRES_DYNAMIC_ALLOCATORS) == 0)
1193 2637 : && omp_maybe_offloaded_ctx (ctx))
1194 10 : error_at (OMP_CLAUSE_LOCATION (c), "%<allocate%> clause must"
1195 : " specify an allocator here");
1196 1629 : if ((omp_requires_mask & OMP_REQUIRES_DYNAMIC_ALLOCATORS) == 0
1197 1617 : && OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) != NULL_TREE
1198 609 : && DECL_P (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c))
1199 1978 : && !DECL_ARTIFICIAL (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c)))
1200 : {
1201 275 : tree alloc2 = OMP_CLAUSE_ALLOCATE_ALLOCATOR (c);
1202 275 : if (TREE_CODE (alloc2) == MEM_REF
1203 275 : || TREE_CODE (alloc2) == INDIRECT_REF)
1204 0 : alloc2 = TREE_OPERAND (alloc2, 0);
1205 275 : omp_context *ctx2 = ctx;
1206 736 : for (; ctx2; ctx2 = ctx2->outer)
1207 476 : if (is_gimple_omp_offloaded (ctx2->stmt))
1208 : break;
1209 275 : if (ctx2 != NULL)
1210 : {
1211 15 : tree c2 = gimple_omp_target_clauses (ctx2->stmt);
1212 80 : for (; c2; c2 = OMP_CLAUSE_CHAIN (c2))
1213 78 : if (OMP_CLAUSE_CODE (c2) == OMP_CLAUSE_USES_ALLOCATORS
1214 91 : && operand_equal_p (
1215 13 : alloc2, OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c2)))
1216 : break;
1217 15 : if (c2 == NULL_TREE)
1218 4 : error_at (EXPR_LOC_OR_LOC (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c),
1219 : OMP_CLAUSE_LOCATION (c)),
1220 : "allocator %qE in %<allocate%> clause inside a "
1221 : "target region must be specified in an "
1222 : "%<uses_allocators%> clause on the %<target%> "
1223 : "directive", alloc2);
1224 : }
1225 : }
1226 1629 : if (ctx->allocate_map == NULL)
1227 1298 : ctx->allocate_map = new hash_map<tree, tree>;
1228 1629 : tree val = integer_zero_node;
1229 1629 : if (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c))
1230 609 : val = OMP_CLAUSE_ALLOCATE_ALLOCATOR (c);
1231 1629 : if (OMP_CLAUSE_ALLOCATE_ALIGN (c))
1232 295 : val = build_tree_list (val, OMP_CLAUSE_ALLOCATE_ALIGN (c));
1233 1629 : ctx->allocate_map->put (OMP_CLAUSE_DECL (c), val);
1234 : }
1235 :
1236 549406 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1237 : {
1238 419660 : bool by_ref;
1239 :
1240 419660 : switch (OMP_CLAUSE_CODE (c))
1241 : {
1242 73105 : case OMP_CLAUSE_PRIVATE:
1243 73105 : decl = OMP_CLAUSE_DECL (c);
1244 73105 : if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
1245 165 : goto do_private;
1246 72940 : else if (!is_variable_sized (decl))
1247 72577 : install_var_local (decl, ctx);
1248 : break;
1249 :
1250 46549 : case OMP_CLAUSE_SHARED:
1251 46549 : decl = OMP_CLAUSE_DECL (c);
1252 46549 : if (ctx->allocate_map && ctx->allocate_map->get (decl))
1253 37 : ctx->allocate_map->remove (decl);
1254 : /* Ignore shared directives in teams construct inside of
1255 : target construct. */
1256 46549 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
1257 46549 : && !is_host_teams_ctx (ctx))
1258 : {
1259 : /* Global variables don't need to be copied,
1260 : the receiver side will use them directly. */
1261 11856 : tree odecl = maybe_lookup_decl_in_outer_ctx (decl, ctx);
1262 11856 : if (is_global_var (odecl))
1263 : break;
1264 8127 : insert_decl_map (&ctx->cb, decl, odecl);
1265 8127 : break;
1266 : }
1267 34693 : gcc_assert (is_taskreg_ctx (ctx));
1268 34693 : gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl))
1269 : || !is_variable_sized (decl));
1270 : /* Global variables don't need to be copied,
1271 : the receiver side will use them directly. */
1272 34693 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1273 : break;
1274 28628 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
1275 : {
1276 86 : use_pointer_for_field (decl, ctx);
1277 86 : break;
1278 : }
1279 28542 : by_ref = use_pointer_for_field (decl, NULL);
1280 54728 : if ((! TREE_READONLY (decl) && !OMP_CLAUSE_SHARED_READONLY (c))
1281 19150 : || TREE_ADDRESSABLE (decl)
1282 18793 : || by_ref
1283 47335 : || omp_privatize_by_reference (decl))
1284 : {
1285 11751 : by_ref = use_pointer_for_field (decl, ctx);
1286 11751 : install_var_field (decl, by_ref, 3, ctx);
1287 11751 : install_var_local (decl, ctx);
1288 11751 : break;
1289 : }
1290 : /* We don't need to copy const scalar vars back. */
1291 16791 : OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE);
1292 16791 : goto do_private;
1293 :
1294 14529 : case OMP_CLAUSE_REDUCTION:
1295 : /* Collect 'reduction' clauses on OpenACC compute construct. */
1296 14529 : if (is_gimple_omp_oacc (ctx->stmt)
1297 14529 : && is_gimple_omp_offloaded (ctx->stmt))
1298 : {
1299 : /* No 'reduction' clauses on OpenACC 'kernels'. */
1300 790 : gcc_checking_assert (!is_oacc_kernels (ctx));
1301 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
1302 790 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
1303 :
1304 790 : ctx->local_reduction_clauses
1305 790 : = tree_cons (NULL, c, ctx->local_reduction_clauses);
1306 : }
1307 : /* FALLTHRU */
1308 :
1309 16603 : case OMP_CLAUSE_IN_REDUCTION:
1310 16603 : decl = OMP_CLAUSE_DECL (c);
1311 16603 : if (ctx->allocate_map
1312 16603 : && ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1313 365 : && (OMP_CLAUSE_REDUCTION_INSCAN (c)
1314 365 : || OMP_CLAUSE_REDUCTION_TASK (c)))
1315 373 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
1316 365 : || is_task_ctx (ctx)))
1317 : {
1318 : /* For now. */
1319 17 : if (ctx->allocate_map->get (decl))
1320 17 : ctx->allocate_map->remove (decl);
1321 : }
1322 16603 : if (TREE_CODE (decl) == MEM_REF)
1323 : {
1324 2366 : tree t = TREE_OPERAND (decl, 0);
1325 2366 : if (TREE_CODE (t) == POINTER_PLUS_EXPR)
1326 436 : t = TREE_OPERAND (t, 0);
1327 2366 : if (INDIRECT_REF_P (t)
1328 2272 : || TREE_CODE (t) == ADDR_EXPR)
1329 1378 : t = TREE_OPERAND (t, 0);
1330 2366 : if (is_omp_target (ctx->stmt))
1331 : {
1332 96 : if (is_variable_sized (t))
1333 : {
1334 32 : gcc_assert (DECL_HAS_VALUE_EXPR_P (t));
1335 32 : t = DECL_VALUE_EXPR (t);
1336 32 : gcc_assert (INDIRECT_REF_P (t));
1337 32 : t = TREE_OPERAND (t, 0);
1338 32 : gcc_assert (DECL_P (t));
1339 : }
1340 96 : tree at = t;
1341 96 : if (ctx->outer)
1342 96 : scan_omp_op (&at, ctx->outer);
1343 96 : tree nt = omp_copy_decl_1 (at, ctx->outer);
1344 192 : splay_tree_insert (ctx->field_map,
1345 96 : (splay_tree_key) &DECL_CONTEXT (t),
1346 : (splay_tree_value) nt);
1347 96 : if (at != t)
1348 96 : splay_tree_insert (ctx->field_map,
1349 48 : (splay_tree_key) &DECL_CONTEXT (at),
1350 : (splay_tree_value) nt);
1351 96 : break;
1352 : }
1353 2270 : install_var_local (t, ctx);
1354 2270 : if (is_taskreg_ctx (ctx)
1355 1822 : && (!is_global_var (maybe_lookup_decl_in_outer_ctx (t, ctx))
1356 387 : || (is_task_ctx (ctx)
1357 310 : && (TREE_CODE (TREE_TYPE (t)) == POINTER_TYPE
1358 234 : || (TREE_CODE (TREE_TYPE (t)) == REFERENCE_TYPE
1359 0 : && (TREE_CODE (TREE_TYPE (TREE_TYPE (t)))
1360 : == POINTER_TYPE)))))
1361 1511 : && !is_variable_sized (t)
1362 3722 : && (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
1363 617 : || (!OMP_CLAUSE_REDUCTION_TASK (c)
1364 578 : && !is_task_ctx (ctx))))
1365 : {
1366 1335 : by_ref = use_pointer_for_field (t, NULL);
1367 1335 : if (is_task_ctx (ctx)
1368 835 : && TREE_CODE (TREE_TYPE (t)) == REFERENCE_TYPE
1369 1495 : && TREE_CODE (TREE_TYPE (TREE_TYPE (t))) == POINTER_TYPE)
1370 : {
1371 32 : install_var_field (t, false, 1, ctx);
1372 32 : install_var_field (t, by_ref, 2, ctx);
1373 : }
1374 : else
1375 1303 : install_var_field (t, by_ref, 3, ctx);
1376 : }
1377 : break;
1378 : }
1379 14237 : if (is_omp_target (ctx->stmt))
1380 : {
1381 335 : tree at = decl;
1382 335 : if (ctx->outer)
1383 54 : scan_omp_op (&at, ctx->outer);
1384 335 : tree nt = omp_copy_decl_1 (at, ctx->outer);
1385 670 : splay_tree_insert (ctx->field_map,
1386 335 : (splay_tree_key) &DECL_CONTEXT (decl),
1387 : (splay_tree_value) nt);
1388 335 : if (at != decl)
1389 24 : splay_tree_insert (ctx->field_map,
1390 12 : (splay_tree_key) &DECL_CONTEXT (at),
1391 : (splay_tree_value) nt);
1392 335 : break;
1393 : }
1394 13902 : if (is_task_ctx (ctx)
1395 26734 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1396 12832 : && OMP_CLAUSE_REDUCTION_TASK (c)
1397 373 : && is_parallel_ctx (ctx)))
1398 : {
1399 : /* Global variables don't need to be copied,
1400 : the receiver side will use them directly. */
1401 1150 : if (!is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1402 : {
1403 462 : by_ref = use_pointer_for_field (decl, ctx);
1404 462 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION)
1405 304 : install_var_field (decl, by_ref, 3, ctx);
1406 : }
1407 1150 : install_var_local (decl, ctx);
1408 1150 : break;
1409 : }
1410 12752 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1411 12752 : && OMP_CLAUSE_REDUCTION_TASK (c))
1412 : {
1413 293 : install_var_local (decl, ctx);
1414 293 : break;
1415 : }
1416 12459 : goto do_private;
1417 :
1418 22664 : case OMP_CLAUSE_LASTPRIVATE:
1419 : /* Let the corresponding firstprivate clause create
1420 : the variable. */
1421 22664 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1422 : break;
1423 : /* FALLTHRU */
1424 :
1425 59206 : case OMP_CLAUSE_FIRSTPRIVATE:
1426 59206 : case OMP_CLAUSE_LINEAR:
1427 59206 : decl = OMP_CLAUSE_DECL (c);
1428 89009 : do_private:
1429 89009 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1430 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR
1431 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1432 45711 : && is_gimple_omp_offloaded (ctx->stmt))
1433 : {
1434 16255 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1435 16255 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR
1436 239 : && lang_hooks.decls.omp_array_data (decl, true)))
1437 : {
1438 : /* OpenACC firstprivate clauses are later processed with same
1439 : code path as map clauses in lower_omp_target, so follow
1440 : the same convention of using the whole clause expression
1441 : as splay-tree key. */
1442 15886 : tree k = (is_oacc_parallel_or_serial (ctx) ? c : NULL_TREE);
1443 15886 : by_ref = !omp_privatize_by_reference (decl);
1444 15886 : install_var_field (decl, by_ref, 3, ctx, k);
1445 : }
1446 369 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1447 : {
1448 220 : if (INDIRECT_REF_P (decl))
1449 0 : decl = TREE_OPERAND (decl, 0);
1450 220 : install_var_field (decl, true, 3, ctx);
1451 : }
1452 149 : else if (TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1453 8 : install_var_field (decl, true, 3, ctx);
1454 : else
1455 141 : install_var_field (decl, false, 3, ctx);
1456 : }
1457 89009 : if (is_variable_sized (decl))
1458 : {
1459 60 : if (is_task_ctx (ctx))
1460 : {
1461 14 : if (ctx->allocate_map
1462 14 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
1463 : {
1464 : /* For now. */
1465 0 : if (ctx->allocate_map->get (decl))
1466 0 : ctx->allocate_map->remove (decl);
1467 : }
1468 14 : install_var_field (decl, false, 1, ctx);
1469 : }
1470 : break;
1471 : }
1472 88949 : else if (is_taskreg_ctx (ctx))
1473 : {
1474 38284 : bool global
1475 38284 : = is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx));
1476 38284 : by_ref = use_pointer_for_field (decl, NULL);
1477 :
1478 38284 : if (is_task_ctx (ctx)
1479 38284 : && (global || by_ref || omp_privatize_by_reference (decl)))
1480 : {
1481 595 : if (ctx->allocate_map
1482 595 : && ctx->allocate_map->get (decl))
1483 35 : install_var_field (decl, by_ref, 32 | 1, ctx);
1484 : else
1485 560 : install_var_field (decl, false, 1, ctx);
1486 595 : if (!global)
1487 331 : install_var_field (decl, by_ref, 2, ctx);
1488 : }
1489 37689 : else if (!global)
1490 36420 : install_var_field (decl, by_ref, 3, ctx);
1491 : }
1492 88949 : install_var_local (decl, ctx);
1493 : /* For descr arrays on target: firstprivatize data + attach ptr. */
1494 88949 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1495 45276 : && is_gimple_omp_offloaded (ctx->stmt)
1496 15861 : && !is_gimple_omp_oacc (ctx->stmt)
1497 11613 : && lang_hooks.decls.omp_array_data (decl, true)
1498 88980 : && lang_hooks.decls.omp_array_data_privatize (decl))
1499 : {
1500 23 : install_var_field (decl, false, 16 | 3, ctx);
1501 23 : install_var_field (decl, true, 8 | 3, ctx);
1502 : }
1503 : break;
1504 :
1505 2118 : case OMP_CLAUSE_USE_DEVICE_PTR:
1506 2118 : case OMP_CLAUSE_USE_DEVICE_ADDR:
1507 2118 : decl = OMP_CLAUSE_DECL (c);
1508 :
1509 : /* Fortran array descriptors. */
1510 2118 : if (lang_hooks.decls.omp_array_data (decl, true))
1511 601 : install_var_field (decl, false, 19, ctx);
1512 1517 : else if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
1513 1312 : && !omp_privatize_by_reference (decl)
1514 513 : && !omp_is_allocatable_or_ptr (decl))
1515 2454 : || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1516 407 : install_var_field (decl, true, 11, ctx);
1517 : else
1518 1110 : install_var_field (decl, false, 11, ctx);
1519 2118 : if (DECL_SIZE (decl)
1520 2118 : && !poly_int_tree_p (DECL_SIZE (decl)))
1521 : {
1522 6 : tree decl2 = DECL_VALUE_EXPR (decl);
1523 6 : gcc_assert (INDIRECT_REF_P (decl2));
1524 6 : decl2 = TREE_OPERAND (decl2, 0);
1525 6 : gcc_assert (DECL_P (decl2));
1526 6 : install_var_local (decl2, ctx);
1527 : }
1528 2118 : install_var_local (decl, ctx);
1529 2118 : break;
1530 :
1531 239 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
1532 239 : decl = OMP_CLAUSE_DECL (c);
1533 239 : while (INDIRECT_REF_P (decl)
1534 256 : || TREE_CODE (decl) == ARRAY_REF)
1535 17 : decl = TREE_OPERAND (decl, 0);
1536 239 : goto do_private;
1537 :
1538 149 : case OMP_CLAUSE_IS_DEVICE_PTR:
1539 149 : decl = OMP_CLAUSE_DECL (c);
1540 149 : goto do_private;
1541 :
1542 20111 : case OMP_CLAUSE__LOOPTEMP_:
1543 20111 : case OMP_CLAUSE__REDUCTEMP_:
1544 20111 : gcc_assert (is_taskreg_ctx (ctx));
1545 20111 : decl = OMP_CLAUSE_DECL (c);
1546 20111 : install_var_field (decl, false, 3, ctx);
1547 20111 : install_var_local (decl, ctx);
1548 20111 : break;
1549 :
1550 896 : case OMP_CLAUSE_COPYPRIVATE:
1551 896 : case OMP_CLAUSE_COPYIN:
1552 896 : decl = OMP_CLAUSE_DECL (c);
1553 896 : by_ref = use_pointer_for_field (decl, NULL);
1554 896 : install_var_field (decl, by_ref, 3, ctx);
1555 896 : break;
1556 :
1557 55236 : case OMP_CLAUSE_FINAL:
1558 55236 : case OMP_CLAUSE_IF:
1559 55236 : case OMP_CLAUSE_SELF:
1560 55236 : case OMP_CLAUSE_NUM_THREADS:
1561 55236 : case OMP_CLAUSE_NUM_TEAMS:
1562 55236 : case OMP_CLAUSE_THREAD_LIMIT:
1563 55236 : case OMP_CLAUSE_DEVICE:
1564 55236 : case OMP_CLAUSE_SCHEDULE:
1565 55236 : case OMP_CLAUSE_DIST_SCHEDULE:
1566 55236 : case OMP_CLAUSE_DEPEND:
1567 55236 : case OMP_CLAUSE_PRIORITY:
1568 55236 : case OMP_CLAUSE_GRAINSIZE:
1569 55236 : case OMP_CLAUSE_NUM_TASKS:
1570 55236 : case OMP_CLAUSE_NUM_GANGS:
1571 55236 : case OMP_CLAUSE_NUM_WORKERS:
1572 55236 : case OMP_CLAUSE_VECTOR_LENGTH:
1573 55236 : case OMP_CLAUSE_DETACH:
1574 55236 : case OMP_CLAUSE_FILTER:
1575 55236 : if (ctx->outer)
1576 17573 : scan_omp_op (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer);
1577 : break;
1578 :
1579 88329 : case OMP_CLAUSE_TO:
1580 88329 : case OMP_CLAUSE_FROM:
1581 88329 : case OMP_CLAUSE_MAP:
1582 88329 : if (ctx->outer)
1583 15385 : scan_omp_op (&OMP_CLAUSE_SIZE (c), ctx->outer);
1584 88329 : decl = OMP_CLAUSE_DECL (c);
1585 : /* If requested, make 'decl' addressable. */
1586 88329 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1587 88329 : && OMP_CLAUSE_MAP_DECL_MAKE_ADDRESSABLE (c))
1588 : {
1589 433 : gcc_checking_assert (DECL_P (decl));
1590 :
1591 433 : bool decl_addressable = TREE_ADDRESSABLE (decl);
1592 433 : if (!decl_addressable)
1593 : {
1594 219 : if (!make_addressable_vars)
1595 131 : make_addressable_vars = BITMAP_ALLOC (NULL);
1596 219 : bitmap_set_bit (make_addressable_vars, DECL_UID (decl));
1597 219 : TREE_ADDRESSABLE (decl) = 1;
1598 : }
1599 :
1600 433 : if (dump_enabled_p ())
1601 : {
1602 421 : location_t loc = OMP_CLAUSE_LOCATION (c);
1603 421 : const dump_user_location_t d_u_loc
1604 421 : = dump_user_location_t::from_location_t (loc);
1605 : /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */
1606 : #if __GNUC__ >= 10
1607 421 : # pragma GCC diagnostic push
1608 421 : # pragma GCC diagnostic ignored "-Wformat"
1609 : #endif
1610 421 : if (!decl_addressable)
1611 207 : dump_printf_loc (MSG_NOTE, d_u_loc,
1612 : "variable %<%T%>"
1613 : " made addressable\n",
1614 : decl);
1615 : else
1616 214 : dump_printf_loc (MSG_NOTE, d_u_loc,
1617 : "variable %<%T%>"
1618 : " already made addressable\n",
1619 : decl);
1620 : #if __GNUC__ >= 10
1621 421 : # pragma GCC diagnostic pop
1622 : #endif
1623 : }
1624 :
1625 : /* Done. */
1626 433 : OMP_CLAUSE_MAP_DECL_MAKE_ADDRESSABLE (c) = 0;
1627 : }
1628 : /* Global variables with "omp declare target" attribute
1629 : don't need to be copied, the receiver side will use them
1630 : directly. However, global variables with "omp declare target link"
1631 : attribute need to be copied. Or when ALWAYS modifier is used. */
1632 88329 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1633 80544 : && DECL_P (decl)
1634 44359 : && ((OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER
1635 40542 : && (OMP_CLAUSE_MAP_KIND (c)
1636 : != GOMP_MAP_FIRSTPRIVATE_REFERENCE)
1637 40113 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ATTACH
1638 39879 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_DETACH)
1639 4545 : || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1640 41017 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_TO
1641 40882 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_FROM
1642 40853 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_TOFROM
1643 40365 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_TO
1644 40335 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_FROM
1645 40313 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_TOFROM
1646 40299 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_TO_PSET
1647 36308 : && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1648 7560 : && varpool_node::get_create (decl)->offloadable
1649 93595 : && !lookup_attribute ("omp declare target link",
1650 5266 : DECL_ATTRIBUTES (decl)))
1651 : break;
1652 83091 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1653 83091 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER)
1654 : {
1655 : /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
1656 : not offloaded; there is nothing to map for those. */
1657 10489 : if (!is_gimple_omp_offloaded (ctx->stmt)
1658 4978 : && !POINTER_TYPE_P (TREE_TYPE (decl))
1659 10753 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
1660 : break;
1661 : }
1662 82827 : if (!flex_array_ptr)
1663 82599 : flex_array_ptr = lang_hooks.decls.omp_deep_mapping_p (ctx->stmt, c);
1664 82827 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1665 75042 : && DECL_P (decl)
1666 38857 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
1667 38623 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
1668 83126 : && is_omp_target (ctx->stmt))
1669 : {
1670 : /* If this is an offloaded region, an attach operation should
1671 : only exist when the pointer variable is mapped in a prior
1672 : clause. An exception is if we have a reference (to pointer):
1673 : in that case we should have mapped "*decl" in a previous
1674 : mapping instead of "decl". Skip the assertion in that case.
1675 : If we had an error, we may not have attempted to sort clauses
1676 : properly, so avoid the test. */
1677 261 : if (TREE_CODE (TREE_TYPE (decl)) != REFERENCE_TYPE
1678 229 : && is_gimple_omp_offloaded (ctx->stmt)
1679 331 : && !seen_error ())
1680 62 : gcc_assert
1681 : (maybe_lookup_decl (decl, ctx)
1682 : || (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1683 : && lookup_attribute ("omp declare target",
1684 : DECL_ATTRIBUTES (decl))));
1685 :
1686 : /* By itself, attach/detach is generated as part of pointer
1687 : variable mapping and should not create new variables in the
1688 : offloaded region, however sender refs for it must be created
1689 : for its address to be passed to the runtime. */
1690 261 : tree field
1691 261 : = build_decl (OMP_CLAUSE_LOCATION (c),
1692 : FIELD_DECL, NULL_TREE, ptr_type_node);
1693 261 : SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
1694 261 : insert_field_into_struct (ctx->record_type, field);
1695 : /* To not clash with a map of the pointer variable itself,
1696 : attach/detach maps have their field looked up by the *clause*
1697 : tree expression, not the decl. */
1698 261 : gcc_assert (!splay_tree_lookup (ctx->field_map,
1699 : (splay_tree_key) c));
1700 261 : splay_tree_insert (ctx->field_map, (splay_tree_key) c,
1701 : (splay_tree_value) field);
1702 261 : break;
1703 : }
1704 82566 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1705 82566 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
1706 70966 : || (OMP_CLAUSE_MAP_KIND (c)
1707 : == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
1708 : {
1709 4244 : if (TREE_CODE (decl) == COMPONENT_REF
1710 4244 : || (INDIRECT_REF_P (decl)
1711 0 : && TREE_CODE (TREE_OPERAND (decl, 0)) == COMPONENT_REF
1712 0 : && (((TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0)))
1713 : == REFERENCE_TYPE)
1714 0 : || (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0)))
1715 : == POINTER_TYPE)))))
1716 : break;
1717 4244 : if (DECL_SIZE (decl)
1718 4244 : && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
1719 : {
1720 710 : tree decl2 = DECL_VALUE_EXPR (decl);
1721 710 : gcc_assert (INDIRECT_REF_P (decl2));
1722 710 : decl2 = TREE_OPERAND (decl2, 0);
1723 710 : gcc_assert (DECL_P (decl2));
1724 710 : install_var_local (decl2, ctx);
1725 : }
1726 4244 : install_var_local (decl, ctx);
1727 4244 : break;
1728 : }
1729 78322 : if (DECL_P (decl))
1730 : {
1731 40514 : if (DECL_SIZE (decl)
1732 40514 : && !poly_int_tree_p (DECL_SIZE (decl)))
1733 : {
1734 0 : tree decl2 = DECL_VALUE_EXPR (decl);
1735 0 : gcc_assert (INDIRECT_REF_P (decl2));
1736 0 : decl2 = TREE_OPERAND (decl2, 0);
1737 0 : gcc_assert (DECL_P (decl2));
1738 0 : install_var_field (decl2, true, 3, ctx, c);
1739 0 : install_var_local (decl2, ctx);
1740 0 : install_var_local (decl, ctx);
1741 : }
1742 : else
1743 : {
1744 40514 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1745 34352 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
1746 5204 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
1747 45718 : && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1748 425 : install_var_field (decl, true, 7, ctx, c);
1749 : else
1750 40089 : install_var_field (decl, true, 3, ctx, c);
1751 40514 : if (is_gimple_omp_offloaded (ctx->stmt)
1752 40514 : && !(is_gimple_omp_oacc (ctx->stmt)
1753 13857 : && OMP_CLAUSE_MAP_IN_REDUCTION (c)))
1754 23830 : install_var_local (decl, ctx);
1755 : }
1756 : }
1757 : else
1758 : {
1759 37808 : tree base = get_base_address (decl);
1760 37808 : tree nc = OMP_CLAUSE_CHAIN (c);
1761 37808 : if (DECL_P (base)
1762 9267 : && nc != NULL_TREE
1763 6677 : && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP
1764 5515 : && OMP_CLAUSE_DECL (nc) == base
1765 1184 : && OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_POINTER
1766 38476 : && integer_zerop (OMP_CLAUSE_SIZE (nc)))
1767 : {
1768 0 : OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) = 1;
1769 0 : OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc) = 1;
1770 : }
1771 : else
1772 : {
1773 37808 : if (ctx->outer)
1774 : {
1775 7152 : scan_omp_op (&OMP_CLAUSE_DECL (c), ctx->outer);
1776 7152 : decl = OMP_CLAUSE_DECL (c);
1777 : }
1778 37808 : gcc_assert (!splay_tree_lookup (ctx->field_map,
1779 : (splay_tree_key) decl));
1780 37808 : tree field
1781 37808 : = build_decl (OMP_CLAUSE_LOCATION (c),
1782 : FIELD_DECL, NULL_TREE, ptr_type_node);
1783 37808 : SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
1784 37808 : insert_field_into_struct (ctx->record_type, field);
1785 37808 : splay_tree_insert (ctx->field_map, (splay_tree_key) c,
1786 : (splay_tree_value) field);
1787 : }
1788 : }
1789 : break;
1790 :
1791 3973 : case OMP_CLAUSE_ORDER:
1792 3973 : ctx->order_concurrent = true;
1793 3973 : break;
1794 :
1795 1907 : case OMP_CLAUSE_BIND:
1796 1907 : ctx->loop_p = true;
1797 1907 : break;
1798 :
1799 : case OMP_CLAUSE_NOWAIT:
1800 : case OMP_CLAUSE_ORDERED:
1801 : case OMP_CLAUSE_COLLAPSE:
1802 : case OMP_CLAUSE_UNTIED:
1803 : case OMP_CLAUSE_MERGEABLE:
1804 : case OMP_CLAUSE_PROC_BIND:
1805 : case OMP_CLAUSE_SAFELEN:
1806 : case OMP_CLAUSE_SIMDLEN:
1807 : case OMP_CLAUSE_THREADS:
1808 : case OMP_CLAUSE_SIMD:
1809 : case OMP_CLAUSE_NOGROUP:
1810 : case OMP_CLAUSE_DEFAULTMAP:
1811 : case OMP_CLAUSE_ASYNC:
1812 : case OMP_CLAUSE_WAIT:
1813 : case OMP_CLAUSE_GANG:
1814 : case OMP_CLAUSE_WORKER:
1815 : case OMP_CLAUSE_VECTOR:
1816 : case OMP_CLAUSE_INDEPENDENT:
1817 : case OMP_CLAUSE_AUTO:
1818 : case OMP_CLAUSE_SEQ:
1819 : case OMP_CLAUSE_TILE:
1820 : case OMP_CLAUSE__SIMT_:
1821 : case OMP_CLAUSE_DEFAULT:
1822 : case OMP_CLAUSE_NONTEMPORAL:
1823 : case OMP_CLAUSE_IF_PRESENT:
1824 : case OMP_CLAUSE_FINALIZE:
1825 : case OMP_CLAUSE_TASK_REDUCTION:
1826 : case OMP_CLAUSE_ALLOCATE:
1827 : case OMP_CLAUSE_DEVICE_TYPE:
1828 : break;
1829 :
1830 90 : case OMP_CLAUSE_USES_ALLOCATORS:
1831 90 : decl = OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c);
1832 90 : gcc_assert (DECL_P (decl));
1833 90 : gcc_assert (is_gimple_omp_offloaded (ctx->stmt));
1834 90 : install_var_field (decl, false, 3, ctx);
1835 90 : install_var_local (decl, ctx);
1836 90 : break;
1837 :
1838 162 : case OMP_CLAUSE_ALIGNED:
1839 162 : decl = OMP_CLAUSE_DECL (c);
1840 162 : if (is_global_var (decl)
1841 162 : && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1842 86 : install_var_local (decl, ctx);
1843 : break;
1844 :
1845 349 : case OMP_CLAUSE__CONDTEMP_:
1846 349 : decl = OMP_CLAUSE_DECL (c);
1847 349 : if (is_parallel_ctx (ctx))
1848 : {
1849 112 : install_var_field (decl, false, 3, ctx);
1850 112 : install_var_local (decl, ctx);
1851 : }
1852 237 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
1853 237 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
1854 362 : && !OMP_CLAUSE__CONDTEMP__ITER (c))
1855 125 : install_var_local (decl, ctx);
1856 : break;
1857 :
1858 : case OMP_CLAUSE_INIT:
1859 : case OMP_CLAUSE_USE:
1860 : case OMP_CLAUSE_DESTROY:
1861 : break;
1862 :
1863 0 : case OMP_CLAUSE__CACHE_:
1864 0 : case OMP_CLAUSE_NOHOST:
1865 0 : default:
1866 0 : gcc_unreachable ();
1867 : }
1868 : }
1869 :
1870 549406 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1871 : {
1872 419660 : switch (OMP_CLAUSE_CODE (c))
1873 : {
1874 22664 : case OMP_CLAUSE_LASTPRIVATE:
1875 : /* Let the corresponding firstprivate clause create
1876 : the variable. */
1877 22664 : if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
1878 7203 : scan_array_reductions = true;
1879 22664 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1880 : break;
1881 : /* FALLTHRU */
1882 :
1883 149490 : case OMP_CLAUSE_FIRSTPRIVATE:
1884 149490 : case OMP_CLAUSE_PRIVATE:
1885 149490 : case OMP_CLAUSE_LINEAR:
1886 149490 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
1887 149490 : case OMP_CLAUSE_IS_DEVICE_PTR:
1888 149490 : decl = OMP_CLAUSE_DECL (c);
1889 149490 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1890 : {
1891 256 : while (INDIRECT_REF_P (decl)
1892 256 : || TREE_CODE (decl) == ARRAY_REF)
1893 17 : decl = TREE_OPERAND (decl, 0);
1894 : }
1895 :
1896 149490 : if (is_variable_sized (decl))
1897 : {
1898 423 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1899 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR
1900 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1901 53 : && is_gimple_omp_offloaded (ctx->stmt))
1902 : {
1903 12 : tree decl2 = DECL_VALUE_EXPR (decl);
1904 12 : gcc_assert (INDIRECT_REF_P (decl2));
1905 12 : decl2 = TREE_OPERAND (decl2, 0);
1906 12 : gcc_assert (DECL_P (decl2));
1907 12 : install_var_local (decl2, ctx);
1908 12 : fixup_remapped_decl (decl2, ctx, false);
1909 : }
1910 423 : install_var_local (decl, ctx);
1911 : }
1912 298980 : fixup_remapped_decl (decl, ctx,
1913 149490 : OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE
1914 149490 : && OMP_CLAUSE_PRIVATE_DEBUG (c));
1915 149490 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
1916 149490 : && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
1917 : scan_array_reductions = true;
1918 : break;
1919 :
1920 16603 : case OMP_CLAUSE_REDUCTION:
1921 16603 : case OMP_CLAUSE_IN_REDUCTION:
1922 16603 : decl = OMP_CLAUSE_DECL (c);
1923 16603 : if (TREE_CODE (decl) != MEM_REF && !is_omp_target (ctx->stmt))
1924 : {
1925 13902 : if (is_variable_sized (decl))
1926 0 : install_var_local (decl, ctx);
1927 13902 : fixup_remapped_decl (decl, ctx, false);
1928 : }
1929 16603 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1930 2834 : scan_array_reductions = true;
1931 : break;
1932 :
1933 530 : case OMP_CLAUSE_TASK_REDUCTION:
1934 530 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1935 2834 : scan_array_reductions = true;
1936 : break;
1937 :
1938 29758 : case OMP_CLAUSE_SHARED:
1939 : /* Ignore shared directives in teams construct inside of
1940 : target construct. */
1941 29758 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
1942 29758 : && !is_host_teams_ctx (ctx))
1943 : break;
1944 17902 : decl = OMP_CLAUSE_DECL (c);
1945 17902 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1946 : break;
1947 11837 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
1948 : {
1949 86 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl,
1950 : ctx->outer)))
1951 : break;
1952 79 : bool by_ref = use_pointer_for_field (decl, ctx);
1953 79 : install_var_field (decl, by_ref, 11, ctx);
1954 79 : break;
1955 : }
1956 11751 : fixup_remapped_decl (decl, ctx, false);
1957 11751 : break;
1958 :
1959 80544 : case OMP_CLAUSE_MAP:
1960 80544 : if (!is_gimple_omp_offloaded (ctx->stmt))
1961 : break;
1962 53805 : decl = OMP_CLAUSE_DECL (c);
1963 53805 : if (DECL_P (decl)
1964 33886 : && ((OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER
1965 30237 : && (OMP_CLAUSE_MAP_KIND (c)
1966 : != GOMP_MAP_FIRSTPRIVATE_REFERENCE))
1967 4078 : || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1968 31011 : && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1969 61188 : && varpool_node::get_create (decl)->offloadable)
1970 : break;
1971 48123 : if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
1972 45911 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
1973 2212 : && is_omp_target (ctx->stmt)
1974 50126 : && !is_gimple_omp_offloaded (ctx->stmt))
1975 : break;
1976 48123 : if (DECL_P (decl))
1977 : {
1978 28204 : if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
1979 25151 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER)
1980 6700 : && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
1981 29829 : && !COMPLETE_TYPE_P (TREE_TYPE (decl)))
1982 : {
1983 24 : tree new_decl = lookup_decl (decl, ctx);
1984 24 : TREE_TYPE (new_decl)
1985 48 : = remap_type (TREE_TYPE (decl), &ctx->cb);
1986 : }
1987 28180 : else if (DECL_SIZE (decl)
1988 28180 : && !poly_int_tree_p (DECL_SIZE (decl)))
1989 : {
1990 710 : tree decl2 = DECL_VALUE_EXPR (decl);
1991 710 : gcc_assert (INDIRECT_REF_P (decl2));
1992 710 : decl2 = TREE_OPERAND (decl2, 0);
1993 710 : gcc_assert (DECL_P (decl2));
1994 710 : fixup_remapped_decl (decl2, ctx, false);
1995 710 : fixup_remapped_decl (decl, ctx, true);
1996 : }
1997 : else
1998 27470 : fixup_remapped_decl (decl, ctx, false);
1999 : }
2000 : break;
2001 :
2002 : case OMP_CLAUSE_COPYPRIVATE:
2003 : case OMP_CLAUSE_COPYIN:
2004 : case OMP_CLAUSE_DEFAULT:
2005 : case OMP_CLAUSE_IF:
2006 : case OMP_CLAUSE_SELF:
2007 : case OMP_CLAUSE_NUM_THREADS:
2008 : case OMP_CLAUSE_NUM_TEAMS:
2009 : case OMP_CLAUSE_THREAD_LIMIT:
2010 : case OMP_CLAUSE_DEVICE:
2011 : case OMP_CLAUSE_SCHEDULE:
2012 : case OMP_CLAUSE_DIST_SCHEDULE:
2013 : case OMP_CLAUSE_NOWAIT:
2014 : case OMP_CLAUSE_ORDERED:
2015 : case OMP_CLAUSE_COLLAPSE:
2016 : case OMP_CLAUSE_UNTIED:
2017 : case OMP_CLAUSE_FINAL:
2018 : case OMP_CLAUSE_MERGEABLE:
2019 : case OMP_CLAUSE_PROC_BIND:
2020 : case OMP_CLAUSE_SAFELEN:
2021 : case OMP_CLAUSE_SIMDLEN:
2022 : case OMP_CLAUSE_ALIGNED:
2023 : case OMP_CLAUSE_DEPEND:
2024 : case OMP_CLAUSE_DETACH:
2025 : case OMP_CLAUSE_ALLOCATE:
2026 : case OMP_CLAUSE__LOOPTEMP_:
2027 : case OMP_CLAUSE__REDUCTEMP_:
2028 : case OMP_CLAUSE_TO:
2029 : case OMP_CLAUSE_FROM:
2030 : case OMP_CLAUSE_PRIORITY:
2031 : case OMP_CLAUSE_GRAINSIZE:
2032 : case OMP_CLAUSE_NUM_TASKS:
2033 : case OMP_CLAUSE_THREADS:
2034 : case OMP_CLAUSE_SIMD:
2035 : case OMP_CLAUSE_NOGROUP:
2036 : case OMP_CLAUSE_DEFAULTMAP:
2037 : case OMP_CLAUSE_ORDER:
2038 : case OMP_CLAUSE_BIND:
2039 : case OMP_CLAUSE_USE_DEVICE_PTR:
2040 : case OMP_CLAUSE_USE_DEVICE_ADDR:
2041 : case OMP_CLAUSE_NONTEMPORAL:
2042 : case OMP_CLAUSE_ASYNC:
2043 : case OMP_CLAUSE_WAIT:
2044 : case OMP_CLAUSE_NUM_GANGS:
2045 : case OMP_CLAUSE_NUM_WORKERS:
2046 : case OMP_CLAUSE_VECTOR_LENGTH:
2047 : case OMP_CLAUSE_GANG:
2048 : case OMP_CLAUSE_WORKER:
2049 : case OMP_CLAUSE_VECTOR:
2050 : case OMP_CLAUSE_INDEPENDENT:
2051 : case OMP_CLAUSE_AUTO:
2052 : case OMP_CLAUSE_SEQ:
2053 : case OMP_CLAUSE_TILE:
2054 : case OMP_CLAUSE__SIMT_:
2055 : case OMP_CLAUSE_IF_PRESENT:
2056 : case OMP_CLAUSE_FINALIZE:
2057 : case OMP_CLAUSE_FILTER:
2058 : case OMP_CLAUSE__CONDTEMP_:
2059 : case OMP_CLAUSE_INIT:
2060 : case OMP_CLAUSE_USE:
2061 : case OMP_CLAUSE_DESTROY:
2062 : case OMP_CLAUSE_DEVICE_TYPE:
2063 : case OMP_CLAUSE_USES_ALLOCATORS:
2064 : break;
2065 :
2066 0 : case OMP_CLAUSE__CACHE_:
2067 0 : case OMP_CLAUSE_NOHOST:
2068 0 : default:
2069 0 : gcc_unreachable ();
2070 : }
2071 : }
2072 :
2073 129746 : gcc_checking_assert (!scan_array_reductions
2074 : || !is_gimple_omp_oacc (ctx->stmt));
2075 : if (scan_array_reductions)
2076 : {
2077 34354 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2078 29182 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
2079 26806 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
2080 26182 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
2081 29886 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
2082 : {
2083 2178 : omp_context *rctx = ctx;
2084 2178 : if (is_omp_target (ctx->stmt))
2085 60 : rctx = ctx->outer;
2086 2178 : scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), rctx);
2087 2178 : scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), rctx);
2088 : }
2089 27004 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
2090 27004 : && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
2091 7203 : scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
2092 19801 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
2093 19801 : && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
2094 656 : scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
2095 : }
2096 129746 : if (flex_array_ptr)
2097 : {
2098 144 : tree field = build_range_type (size_type_node,
2099 : build_int_cstu (size_type_node, 0),
2100 : NULL_TREE);
2101 144 : field = build_array_type (ptr_type_node, field);
2102 144 : field = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE, field);
2103 144 : SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
2104 144 : DECL_CONTEXT (field) = ctx->record_type;
2105 144 : DECL_CHAIN (field) = TYPE_FIELDS (ctx->record_type);
2106 144 : TYPE_FIELDS (ctx->record_type) = field;
2107 : }
2108 129746 : }
2109 :
2110 : /* Create a new name for omp child function. Returns an identifier. */
2111 :
2112 : static tree
2113 51326 : create_omp_child_function_name (bool task_copy)
2114 : {
2115 102132 : return clone_function_name_numbered (current_function_decl,
2116 51326 : task_copy ? "_omp_cpyfn" : "_omp_fn");
2117 : }
2118 :
2119 : /* Return true if CTX may belong to offloaded code: either if current function
2120 : is offloaded, or any enclosing context corresponds to a target region. */
2121 :
2122 : static bool
2123 122793 : omp_maybe_offloaded_ctx (omp_context *ctx)
2124 : {
2125 122793 : if (cgraph_node::get (current_function_decl)->offloadable)
2126 : return true;
2127 203000 : for (; ctx; ctx = ctx->outer)
2128 119951 : if (is_gimple_omp_offloaded (ctx->stmt))
2129 : return true;
2130 : return false;
2131 : }
2132 :
2133 : /* Build a decl for the omp child function. It'll not contain a body
2134 : yet, just the bare decl. */
2135 :
2136 : static void
2137 51326 : create_omp_child_function (omp_context *ctx, bool task_copy)
2138 : {
2139 51326 : tree decl, type, name, t;
2140 :
2141 51326 : name = create_omp_child_function_name (task_copy);
2142 51326 : if (task_copy)
2143 520 : type = build_function_type_list (void_type_node, ptr_type_node,
2144 : ptr_type_node, NULL_TREE);
2145 : else
2146 50806 : type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE);
2147 :
2148 51326 : decl = build_decl (gimple_location (ctx->stmt), FUNCTION_DECL, name, type);
2149 :
2150 51326 : gcc_checking_assert (!is_gimple_omp_oacc (ctx->stmt)
2151 : || !task_copy);
2152 39940 : if (!task_copy)
2153 50806 : ctx->cb.dst_fn = decl;
2154 : else
2155 520 : gimple_omp_task_set_copy_fn (ctx->stmt, decl);
2156 :
2157 51326 : TREE_STATIC (decl) = 1;
2158 51326 : TREE_USED (decl) = 1;
2159 51326 : DECL_ARTIFICIAL (decl) = 1;
2160 51326 : DECL_IGNORED_P (decl) = 0;
2161 51326 : TREE_PUBLIC (decl) = 0;
2162 51326 : DECL_UNINLINABLE (decl) = 1;
2163 51326 : DECL_EXTERNAL (decl) = 0;
2164 51326 : DECL_CONTEXT (decl) = NULL_TREE;
2165 51326 : DECL_INITIAL (decl) = make_node (BLOCK);
2166 51326 : BLOCK_SUPERCONTEXT (DECL_INITIAL (decl)) = decl;
2167 51326 : DECL_ATTRIBUTES (decl) = DECL_ATTRIBUTES (current_function_decl);
2168 : /* Remove omp declare simd attribute from the new attributes. */
2169 51326 : if (tree a = lookup_attribute ("omp declare simd", DECL_ATTRIBUTES (decl)))
2170 : {
2171 9 : while (tree a2 = lookup_attribute ("omp declare simd", TREE_CHAIN (a)))
2172 : a = a2;
2173 9 : a = TREE_CHAIN (a);
2174 22 : for (tree *p = &DECL_ATTRIBUTES (decl); *p != a;)
2175 13 : if (is_attribute_p ("omp declare simd", get_attribute_name (*p)))
2176 9 : *p = TREE_CHAIN (*p);
2177 : else
2178 : {
2179 4 : tree chain = TREE_CHAIN (*p);
2180 4 : *p = copy_node (*p);
2181 4 : p = &TREE_CHAIN (*p);
2182 4 : *p = chain;
2183 : }
2184 : }
2185 102652 : DECL_FUNCTION_SPECIFIC_OPTIMIZATION (decl)
2186 51326 : = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (current_function_decl);
2187 102652 : DECL_FUNCTION_SPECIFIC_TARGET (decl)
2188 51326 : = DECL_FUNCTION_SPECIFIC_TARGET (current_function_decl);
2189 102652 : DECL_FUNCTION_VERSIONED (decl)
2190 51326 : = DECL_FUNCTION_VERSIONED (current_function_decl);
2191 :
2192 51326 : if (omp_maybe_offloaded_ctx (ctx))
2193 : {
2194 31015 : cgraph_node::get_create (decl)->offloadable = 1;
2195 31015 : if (ENABLE_OFFLOADING)
2196 : g->have_offload = true;
2197 : }
2198 :
2199 51326 : if (cgraph_node::get_create (decl)->offloadable)
2200 : {
2201 31015 : const char *target_attr = (is_gimple_omp_offloaded (ctx->stmt)
2202 31015 : ? "omp target entrypoint"
2203 6208 : : "omp declare target");
2204 31015 : if (lookup_attribute ("omp declare target",
2205 31015 : DECL_ATTRIBUTES (current_function_decl)))
2206 : {
2207 3032 : if (is_gimple_omp_offloaded (ctx->stmt))
2208 1620 : DECL_ATTRIBUTES (decl)
2209 1620 : = remove_attribute ("omp declare target",
2210 1620 : copy_list (DECL_ATTRIBUTES (decl)));
2211 : else
2212 : target_attr = NULL;
2213 : }
2214 1620 : if (target_attr
2215 29603 : && is_gimple_omp_offloaded (ctx->stmt)
2216 24807 : && lookup_attribute ("noclone", DECL_ATTRIBUTES (decl)) == NULL_TREE)
2217 19300 : DECL_ATTRIBUTES (decl) = tree_cons (get_identifier ("noclone"),
2218 19300 : NULL_TREE, DECL_ATTRIBUTES (decl));
2219 31015 : if (target_attr)
2220 29603 : DECL_ATTRIBUTES (decl)
2221 59206 : = tree_cons (get_identifier (target_attr),
2222 29603 : NULL_TREE, DECL_ATTRIBUTES (decl));
2223 : }
2224 :
2225 51326 : t = build_decl (DECL_SOURCE_LOCATION (decl),
2226 : RESULT_DECL, NULL_TREE, void_type_node);
2227 51326 : DECL_ARTIFICIAL (t) = 1;
2228 51326 : DECL_IGNORED_P (t) = 1;
2229 51326 : DECL_CONTEXT (t) = decl;
2230 51326 : DECL_RESULT (decl) = t;
2231 :
2232 51326 : tree data_name = get_identifier (".omp_data_i");
2233 51326 : t = build_decl (DECL_SOURCE_LOCATION (decl), PARM_DECL, data_name,
2234 : ptr_type_node);
2235 51326 : DECL_ARTIFICIAL (t) = 1;
2236 51326 : DECL_NAMELESS (t) = 1;
2237 51326 : DECL_ARG_TYPE (t) = ptr_type_node;
2238 51326 : DECL_CONTEXT (t) = current_function_decl;
2239 51326 : TREE_USED (t) = 1;
2240 51326 : TREE_READONLY (t) = 1;
2241 51326 : DECL_ARGUMENTS (decl) = t;
2242 51326 : if (!task_copy)
2243 50806 : ctx->receiver_decl = t;
2244 : else
2245 : {
2246 520 : t = build_decl (DECL_SOURCE_LOCATION (decl),
2247 : PARM_DECL, get_identifier (".omp_data_o"),
2248 : ptr_type_node);
2249 520 : DECL_ARTIFICIAL (t) = 1;
2250 520 : DECL_NAMELESS (t) = 1;
2251 520 : DECL_ARG_TYPE (t) = ptr_type_node;
2252 520 : DECL_CONTEXT (t) = current_function_decl;
2253 520 : TREE_USED (t) = 1;
2254 520 : TREE_ADDRESSABLE (t) = 1;
2255 520 : DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2256 520 : DECL_ARGUMENTS (decl) = t;
2257 : }
2258 :
2259 : /* Allocate memory for the function structure. The call to
2260 : allocate_struct_function clobbers CFUN, so we need to restore
2261 : it afterward. */
2262 51326 : push_struct_function (decl);
2263 51326 : cfun->function_end_locus = gimple_location (ctx->stmt);
2264 51326 : init_tree_ssa (cfun);
2265 51326 : pop_cfun ();
2266 51326 : }
2267 :
2268 : /* Callback for walk_gimple_seq. Check if combined parallel
2269 : contains gimple_omp_for_combined_into_p OMP_FOR. */
2270 :
2271 : tree
2272 35647 : omp_find_combined_for (gimple_stmt_iterator *gsi_p,
2273 : bool *handled_ops_p,
2274 : struct walk_stmt_info *wi)
2275 : {
2276 35647 : gimple *stmt = gsi_stmt (*gsi_p);
2277 :
2278 35647 : *handled_ops_p = true;
2279 35647 : switch (gimple_code (stmt))
2280 : {
2281 19972 : WALK_SUBSTMTS;
2282 :
2283 13502 : case GIMPLE_OMP_FOR:
2284 13502 : if (gimple_omp_for_combined_into_p (stmt)
2285 13502 : && gimple_omp_for_kind (stmt)
2286 7960 : == *(const enum gf_mask *) (wi->info))
2287 : {
2288 7960 : wi->info = stmt;
2289 7960 : return integer_zero_node;
2290 : }
2291 : break;
2292 : default:
2293 : break;
2294 : }
2295 : return NULL;
2296 : }
2297 :
2298 : /* Add _LOOPTEMP_/_REDUCTEMP_ clauses on OpenMP parallel or task. */
2299 :
2300 : static void
2301 14087 : add_taskreg_looptemp_clauses (enum gf_mask msk, gimple *stmt,
2302 : omp_context *outer_ctx)
2303 : {
2304 14087 : struct walk_stmt_info wi;
2305 :
2306 14087 : memset (&wi, 0, sizeof (wi));
2307 14087 : wi.val_only = true;
2308 14087 : wi.info = (void *) &msk;
2309 14087 : walk_gimple_seq (gimple_omp_body (stmt), omp_find_combined_for, NULL, &wi);
2310 14087 : if (wi.info != (void *) &msk)
2311 : {
2312 7960 : gomp_for *for_stmt = as_a <gomp_for *> ((gimple *) wi.info);
2313 7960 : struct omp_for_data fd;
2314 7960 : omp_extract_for_data (for_stmt, &fd, NULL);
2315 : /* We need two temporaries with fd.loop.v type (istart/iend)
2316 : and then (fd.collapse - 1) temporaries with the same
2317 : type for count2 ... countN-1 vars if not constant. */
2318 7960 : size_t count = 2, i;
2319 7960 : tree type = fd.iter_type;
2320 7960 : if (fd.collapse > 1
2321 2648 : && TREE_CODE (fd.loop.n2) != INTEGER_CST)
2322 : {
2323 1466 : count += fd.collapse - 1;
2324 : /* If there are lastprivate clauses on the inner
2325 : GIMPLE_OMP_FOR, add one more temporaries for the total number
2326 : of iterations (product of count1 ... countN-1). */
2327 1466 : if (omp_find_clause (gimple_omp_for_clauses (for_stmt),
2328 : OMP_CLAUSE_LASTPRIVATE)
2329 1466 : || (msk == GF_OMP_FOR_KIND_FOR
2330 1335 : && omp_find_clause (gimple_omp_parallel_clauses (stmt),
2331 : OMP_CLAUSE_LASTPRIVATE)))
2332 : {
2333 726 : tree temp = create_tmp_var (type);
2334 726 : tree c = build_omp_clause (UNKNOWN_LOCATION,
2335 : OMP_CLAUSE__LOOPTEMP_);
2336 726 : insert_decl_map (&outer_ctx->cb, temp, temp);
2337 726 : OMP_CLAUSE_DECL (c) = temp;
2338 726 : OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
2339 726 : gimple_omp_taskreg_set_clauses (stmt, c);
2340 : }
2341 1466 : if (fd.non_rect
2342 24 : && fd.last_nonrect == fd.first_nonrect + 1)
2343 12 : if (tree v = gimple_omp_for_index (for_stmt, fd.last_nonrect))
2344 12 : if (!TYPE_UNSIGNED (TREE_TYPE (v)))
2345 : {
2346 12 : v = gimple_omp_for_index (for_stmt, fd.first_nonrect);
2347 12 : tree type2 = TREE_TYPE (v);
2348 12 : count++;
2349 48 : for (i = 0; i < 3; i++)
2350 : {
2351 36 : tree temp = create_tmp_var (type2);
2352 36 : tree c = build_omp_clause (UNKNOWN_LOCATION,
2353 : OMP_CLAUSE__LOOPTEMP_);
2354 36 : insert_decl_map (&outer_ctx->cb, temp, temp);
2355 36 : OMP_CLAUSE_DECL (c) = temp;
2356 36 : OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
2357 36 : gimple_omp_taskreg_set_clauses (stmt, c);
2358 : }
2359 : }
2360 : }
2361 26725 : for (i = 0; i < count; i++)
2362 : {
2363 18765 : tree temp = create_tmp_var (type);
2364 18765 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
2365 18765 : insert_decl_map (&outer_ctx->cb, temp, temp);
2366 18765 : OMP_CLAUSE_DECL (c) = temp;
2367 18765 : OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
2368 18765 : gimple_omp_taskreg_set_clauses (stmt, c);
2369 : }
2370 : }
2371 14087 : if (msk == GF_OMP_FOR_KIND_TASKLOOP
2372 15654 : && omp_find_clause (gimple_omp_task_clauses (stmt),
2373 : OMP_CLAUSE_REDUCTION))
2374 : {
2375 517 : tree type = build_pointer_type (pointer_sized_int_node);
2376 517 : tree temp = create_tmp_var (type);
2377 517 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
2378 517 : insert_decl_map (&outer_ctx->cb, temp, temp);
2379 517 : OMP_CLAUSE_DECL (c) = temp;
2380 517 : OMP_CLAUSE_CHAIN (c) = gimple_omp_task_clauses (stmt);
2381 517 : gimple_omp_task_set_clauses (stmt, c);
2382 : }
2383 14087 : }
2384 :
2385 : /* Scan an OpenMP parallel directive. */
2386 :
2387 : static void
2388 18168 : scan_omp_parallel (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2389 : {
2390 18168 : omp_context *ctx;
2391 18168 : tree name;
2392 18168 : gomp_parallel *stmt = as_a <gomp_parallel *> (gsi_stmt (*gsi));
2393 :
2394 : /* Ignore parallel directives with empty bodies, unless there
2395 : are copyin clauses. */
2396 18168 : if (optimize > 0
2397 12465 : && empty_body_p (gimple_omp_body (stmt))
2398 18215 : && omp_find_clause (gimple_omp_parallel_clauses (stmt),
2399 : OMP_CLAUSE_COPYIN) == NULL)
2400 : {
2401 41 : gsi_replace (gsi, gimple_build_nop (), false);
2402 41 : return;
2403 : }
2404 :
2405 18127 : if (gimple_omp_parallel_combined_p (stmt))
2406 12520 : add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR, stmt, outer_ctx);
2407 18127 : for (tree c = omp_find_clause (gimple_omp_parallel_clauses (stmt),
2408 : OMP_CLAUSE_REDUCTION);
2409 20954 : c; c = omp_find_clause (OMP_CLAUSE_CHAIN (c), OMP_CLAUSE_REDUCTION))
2410 3914 : if (OMP_CLAUSE_REDUCTION_TASK (c))
2411 : {
2412 67 : tree type = build_pointer_type (pointer_sized_int_node);
2413 67 : tree temp = create_tmp_var (type);
2414 67 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
2415 67 : if (outer_ctx)
2416 18 : insert_decl_map (&outer_ctx->cb, temp, temp);
2417 67 : OMP_CLAUSE_DECL (c) = temp;
2418 67 : OMP_CLAUSE_CHAIN (c) = gimple_omp_parallel_clauses (stmt);
2419 67 : gimple_omp_parallel_set_clauses (stmt, c);
2420 67 : break;
2421 : }
2422 3847 : else if (OMP_CLAUSE_CHAIN (c) == NULL_TREE)
2423 : break;
2424 :
2425 18127 : ctx = new_omp_context (stmt, outer_ctx);
2426 18127 : taskreg_contexts.safe_push (ctx);
2427 18127 : if (taskreg_nesting_level > 1)
2428 6193 : ctx->is_nested = true;
2429 18127 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2430 18127 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2431 18127 : name = create_tmp_var_name (".omp_data_s");
2432 18127 : name = build_decl (gimple_location (stmt),
2433 : TYPE_DECL, name, ctx->record_type);
2434 18127 : DECL_ARTIFICIAL (name) = 1;
2435 18127 : DECL_NAMELESS (name) = 1;
2436 18127 : TYPE_NAME (ctx->record_type) = name;
2437 18127 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
2438 18127 : create_omp_child_function (ctx, false);
2439 18127 : gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn);
2440 :
2441 18127 : scan_sharing_clauses (gimple_omp_parallel_clauses (stmt), ctx);
2442 18127 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
2443 :
2444 18127 : if (TYPE_FIELDS (ctx->record_type) == NULL)
2445 3575 : ctx->record_type = ctx->receiver_decl = NULL;
2446 : }
2447 :
2448 : /* Scan an OpenMP task directive. */
2449 :
2450 : static void
2451 5344 : scan_omp_task (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2452 : {
2453 5344 : omp_context *ctx;
2454 5344 : tree name, t;
2455 5344 : gomp_task *stmt = as_a <gomp_task *> (gsi_stmt (*gsi));
2456 :
2457 : /* Ignore task directives with empty bodies, unless they have depend
2458 : clause. */
2459 5344 : if (optimize > 0
2460 2527 : && gimple_omp_body (stmt)
2461 2484 : && empty_body_p (gimple_omp_body (stmt))
2462 5398 : && !omp_find_clause (gimple_omp_task_clauses (stmt), OMP_CLAUSE_DEPEND))
2463 : {
2464 19 : gsi_replace (gsi, gimple_build_nop (), false);
2465 131 : return;
2466 : }
2467 :
2468 5325 : if (gimple_omp_task_taskloop_p (stmt))
2469 1567 : add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP, stmt, outer_ctx);
2470 :
2471 5325 : ctx = new_omp_context (stmt, outer_ctx);
2472 :
2473 5325 : if (gimple_omp_task_taskwait_p (stmt))
2474 : {
2475 93 : scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
2476 93 : return;
2477 : }
2478 :
2479 5232 : taskreg_contexts.safe_push (ctx);
2480 5232 : if (taskreg_nesting_level > 1)
2481 2296 : ctx->is_nested = true;
2482 5232 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2483 5232 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2484 5232 : name = create_tmp_var_name (".omp_data_s");
2485 5232 : name = build_decl (gimple_location (stmt),
2486 : TYPE_DECL, name, ctx->record_type);
2487 5232 : DECL_ARTIFICIAL (name) = 1;
2488 5232 : DECL_NAMELESS (name) = 1;
2489 5232 : TYPE_NAME (ctx->record_type) = name;
2490 5232 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
2491 5232 : create_omp_child_function (ctx, false);
2492 5232 : gimple_omp_task_set_child_fn (stmt, ctx->cb.dst_fn);
2493 :
2494 5232 : scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
2495 :
2496 5232 : if (ctx->srecord_type)
2497 : {
2498 520 : name = create_tmp_var_name (".omp_data_a");
2499 520 : name = build_decl (gimple_location (stmt),
2500 : TYPE_DECL, name, ctx->srecord_type);
2501 520 : DECL_ARTIFICIAL (name) = 1;
2502 520 : DECL_NAMELESS (name) = 1;
2503 520 : TYPE_NAME (ctx->srecord_type) = name;
2504 520 : TYPE_ARTIFICIAL (ctx->srecord_type) = 1;
2505 520 : create_omp_child_function (ctx, true);
2506 : }
2507 :
2508 5232 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
2509 :
2510 5232 : if (TYPE_FIELDS (ctx->record_type) == NULL)
2511 : {
2512 1837 : ctx->record_type = ctx->receiver_decl = NULL;
2513 1837 : t = build_int_cst (long_integer_type_node, 0);
2514 1837 : gimple_omp_task_set_arg_size (stmt, t);
2515 1837 : t = build_int_cst (long_integer_type_node, 1);
2516 1837 : gimple_omp_task_set_arg_align (stmt, t);
2517 : }
2518 : }
2519 :
2520 : /* Helper function for finish_taskreg_scan, called through walk_tree.
2521 : If maybe_lookup_decl_in_outer_context returns non-NULL for some
2522 : tree, replace it in the expression. */
2523 :
2524 : static tree
2525 276 : finish_taskreg_remap (tree *tp, int *walk_subtrees, void *data)
2526 : {
2527 276 : if (VAR_P (*tp))
2528 : {
2529 96 : omp_context *ctx = (omp_context *) data;
2530 96 : tree t = maybe_lookup_decl_in_outer_ctx (*tp, ctx);
2531 96 : if (t != *tp)
2532 : {
2533 9 : if (DECL_HAS_VALUE_EXPR_P (t))
2534 0 : t = unshare_expr (DECL_VALUE_EXPR (t));
2535 9 : *tp = t;
2536 : }
2537 96 : *walk_subtrees = 0;
2538 : }
2539 180 : else if (IS_TYPE_OR_DECL_P (*tp))
2540 0 : *walk_subtrees = 0;
2541 276 : return NULL_TREE;
2542 : }
2543 :
2544 : /* If any decls have been made addressable during scan_omp,
2545 : adjust their fields if needed, and layout record types
2546 : of parallel/task constructs. */
2547 :
2548 : static void
2549 25999 : finish_taskreg_scan (omp_context *ctx)
2550 : {
2551 25999 : if (ctx->record_type == NULL_TREE)
2552 : return;
2553 :
2554 : /* If any make_addressable_vars were needed, verify all
2555 : OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK,TEAMS}
2556 : statements if use_pointer_for_field hasn't changed
2557 : because of that. If it did, update field types now. */
2558 19141 : if (make_addressable_vars)
2559 : {
2560 2211 : tree c;
2561 :
2562 12357 : for (c = gimple_omp_taskreg_clauses (ctx->stmt);
2563 12357 : c; c = OMP_CLAUSE_CHAIN (c))
2564 10146 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
2565 10146 : && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
2566 : {
2567 2620 : tree decl = OMP_CLAUSE_DECL (c);
2568 :
2569 : /* Global variables don't need to be copied,
2570 : the receiver side will use them directly. */
2571 2620 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
2572 282 : continue;
2573 2338 : if (!bitmap_bit_p (make_addressable_vars, DECL_UID (decl))
2574 2338 : || !use_pointer_for_field (decl, ctx))
2575 1787 : continue;
2576 551 : tree field = lookup_field (decl, ctx);
2577 551 : if (TREE_CODE (TREE_TYPE (field)) == POINTER_TYPE
2578 551 : && TREE_TYPE (TREE_TYPE (field)) == TREE_TYPE (decl))
2579 497 : continue;
2580 54 : TREE_TYPE (field) = build_pointer_type (TREE_TYPE (decl));
2581 54 : TREE_THIS_VOLATILE (field) = 0;
2582 54 : DECL_USER_ALIGN (field) = 0;
2583 54 : SET_DECL_ALIGN (field, TYPE_ALIGN (TREE_TYPE (field)));
2584 54 : if (TYPE_ALIGN (ctx->record_type) < DECL_ALIGN (field))
2585 16 : SET_TYPE_ALIGN (ctx->record_type, DECL_ALIGN (field));
2586 54 : if (ctx->srecord_type)
2587 : {
2588 0 : tree sfield = lookup_sfield (decl, ctx);
2589 0 : TREE_TYPE (sfield) = TREE_TYPE (field);
2590 0 : TREE_THIS_VOLATILE (sfield) = 0;
2591 0 : DECL_USER_ALIGN (sfield) = 0;
2592 0 : SET_DECL_ALIGN (sfield, DECL_ALIGN (field));
2593 0 : if (TYPE_ALIGN (ctx->srecord_type) < DECL_ALIGN (sfield))
2594 0 : SET_TYPE_ALIGN (ctx->srecord_type, DECL_ALIGN (sfield));
2595 : }
2596 : }
2597 : }
2598 :
2599 19141 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL)
2600 : {
2601 14552 : tree clauses = gimple_omp_parallel_clauses (ctx->stmt);
2602 14552 : tree c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
2603 14552 : if (c)
2604 : {
2605 : /* Move the _reductemp_ clause first. GOMP_parallel_reductions
2606 : expects to find it at the start of data. */
2607 67 : tree f = lookup_field (OMP_CLAUSE_DECL (c), ctx);
2608 67 : tree *p = &TYPE_FIELDS (ctx->record_type);
2609 153 : while (*p)
2610 153 : if (*p == f)
2611 : {
2612 67 : *p = DECL_CHAIN (*p);
2613 67 : break;
2614 : }
2615 : else
2616 86 : p = &DECL_CHAIN (*p);
2617 67 : DECL_CHAIN (f) = TYPE_FIELDS (ctx->record_type);
2618 67 : TYPE_FIELDS (ctx->record_type) = f;
2619 : }
2620 14552 : layout_type (ctx->record_type);
2621 14552 : fixup_child_record_type (ctx);
2622 : }
2623 4589 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
2624 : {
2625 1194 : layout_type (ctx->record_type);
2626 1194 : fixup_child_record_type (ctx);
2627 : }
2628 : else
2629 : {
2630 3395 : location_t loc = gimple_location (ctx->stmt);
2631 3395 : tree *p, vla_fields = NULL_TREE, *q = &vla_fields;
2632 3395 : tree detach_clause
2633 3395 : = omp_find_clause (gimple_omp_task_clauses (ctx->stmt),
2634 : OMP_CLAUSE_DETACH);
2635 : /* Move VLA fields to the end. */
2636 3395 : p = &TYPE_FIELDS (ctx->record_type);
2637 15232 : while (*p)
2638 11837 : if (!TYPE_SIZE_UNIT (TREE_TYPE (*p))
2639 11837 : || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p))))
2640 : {
2641 96 : *q = *p;
2642 96 : *p = TREE_CHAIN (*p);
2643 96 : TREE_CHAIN (*q) = NULL_TREE;
2644 96 : q = &TREE_CHAIN (*q);
2645 : }
2646 : else
2647 11741 : p = &DECL_CHAIN (*p);
2648 3395 : *p = vla_fields;
2649 3395 : if (gimple_omp_task_taskloop_p (ctx->stmt))
2650 : {
2651 : /* Move fields corresponding to first and second _looptemp_
2652 : clause first. There are filled by GOMP_taskloop
2653 : and thus need to be in specific positions. */
2654 1567 : tree clauses = gimple_omp_task_clauses (ctx->stmt);
2655 1567 : tree c1 = omp_find_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
2656 1567 : tree c2 = omp_find_clause (OMP_CLAUSE_CHAIN (c1),
2657 : OMP_CLAUSE__LOOPTEMP_);
2658 1567 : tree c3 = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
2659 1567 : tree f1 = lookup_field (OMP_CLAUSE_DECL (c1), ctx);
2660 1567 : tree f2 = lookup_field (OMP_CLAUSE_DECL (c2), ctx);
2661 2084 : tree f3 = c3 ? lookup_field (OMP_CLAUSE_DECL (c3), ctx) : NULL_TREE;
2662 1567 : p = &TYPE_FIELDS (ctx->record_type);
2663 8873 : while (*p)
2664 7306 : if (*p == f1 || *p == f2 || *p == f3)
2665 3651 : *p = DECL_CHAIN (*p);
2666 : else
2667 3655 : p = &DECL_CHAIN (*p);
2668 1567 : DECL_CHAIN (f1) = f2;
2669 1567 : if (c3)
2670 : {
2671 517 : DECL_CHAIN (f2) = f3;
2672 517 : DECL_CHAIN (f3) = TYPE_FIELDS (ctx->record_type);
2673 : }
2674 : else
2675 1050 : DECL_CHAIN (f2) = TYPE_FIELDS (ctx->record_type);
2676 1567 : TYPE_FIELDS (ctx->record_type) = f1;
2677 1567 : if (ctx->srecord_type)
2678 : {
2679 389 : f1 = lookup_sfield (OMP_CLAUSE_DECL (c1), ctx);
2680 389 : f2 = lookup_sfield (OMP_CLAUSE_DECL (c2), ctx);
2681 389 : if (c3)
2682 186 : f3 = lookup_sfield (OMP_CLAUSE_DECL (c3), ctx);
2683 389 : p = &TYPE_FIELDS (ctx->srecord_type);
2684 2099 : while (*p)
2685 1710 : if (*p == f1 || *p == f2 || *p == f3)
2686 964 : *p = DECL_CHAIN (*p);
2687 : else
2688 746 : p = &DECL_CHAIN (*p);
2689 389 : DECL_CHAIN (f1) = f2;
2690 389 : DECL_CHAIN (f2) = TYPE_FIELDS (ctx->srecord_type);
2691 389 : if (c3)
2692 : {
2693 186 : DECL_CHAIN (f2) = f3;
2694 186 : DECL_CHAIN (f3) = TYPE_FIELDS (ctx->srecord_type);
2695 : }
2696 : else
2697 203 : DECL_CHAIN (f2) = TYPE_FIELDS (ctx->srecord_type);
2698 389 : TYPE_FIELDS (ctx->srecord_type) = f1;
2699 : }
2700 : }
2701 3395 : if (detach_clause)
2702 : {
2703 189 : tree c, field;
2704 :
2705 : /* Look for a firstprivate clause with the detach event handle. */
2706 540 : for (c = gimple_omp_taskreg_clauses (ctx->stmt);
2707 540 : c; c = OMP_CLAUSE_CHAIN (c))
2708 : {
2709 540 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FIRSTPRIVATE)
2710 347 : continue;
2711 193 : if (maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_DECL (c), ctx)
2712 193 : == OMP_CLAUSE_DECL (detach_clause))
2713 : break;
2714 : }
2715 :
2716 189 : gcc_assert (c);
2717 189 : field = lookup_field (OMP_CLAUSE_DECL (c), ctx);
2718 :
2719 : /* Move field corresponding to the detach clause first.
2720 : This is filled by GOMP_task and needs to be in a
2721 : specific position. */
2722 189 : p = &TYPE_FIELDS (ctx->record_type);
2723 518 : while (*p)
2724 329 : if (*p == field)
2725 189 : *p = DECL_CHAIN (*p);
2726 : else
2727 140 : p = &DECL_CHAIN (*p);
2728 189 : DECL_CHAIN (field) = TYPE_FIELDS (ctx->record_type);
2729 189 : TYPE_FIELDS (ctx->record_type) = field;
2730 189 : if (ctx->srecord_type)
2731 : {
2732 3 : field = lookup_sfield (OMP_CLAUSE_DECL (c), ctx);
2733 3 : p = &TYPE_FIELDS (ctx->srecord_type);
2734 9 : while (*p)
2735 6 : if (*p == field)
2736 3 : *p = DECL_CHAIN (*p);
2737 : else
2738 3 : p = &DECL_CHAIN (*p);
2739 3 : DECL_CHAIN (field) = TYPE_FIELDS (ctx->srecord_type);
2740 3 : TYPE_FIELDS (ctx->srecord_type) = field;
2741 : }
2742 : }
2743 3395 : layout_type (ctx->record_type);
2744 3395 : fixup_child_record_type (ctx);
2745 3395 : if (ctx->srecord_type)
2746 520 : layout_type (ctx->srecord_type);
2747 3395 : tree t = fold_convert_loc (loc, long_integer_type_node,
2748 3395 : TYPE_SIZE_UNIT (ctx->record_type));
2749 3395 : if (TREE_CODE (t) != INTEGER_CST)
2750 : {
2751 21 : t = unshare_expr (t);
2752 21 : walk_tree (&t, finish_taskreg_remap, ctx, NULL);
2753 : }
2754 3395 : gimple_omp_task_set_arg_size (ctx->stmt, t);
2755 3395 : t = build_int_cst (long_integer_type_node,
2756 3395 : TYPE_ALIGN_UNIT (ctx->record_type));
2757 3395 : gimple_omp_task_set_arg_align (ctx->stmt, t);
2758 : }
2759 : }
2760 :
2761 : /* Find the enclosing offload context. */
2762 :
2763 : static omp_context *
2764 9665 : enclosing_target_ctx (omp_context *ctx)
2765 : {
2766 43359 : for (; ctx; ctx = ctx->outer)
2767 41745 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET)
2768 : break;
2769 :
2770 22111 : return ctx;
2771 : }
2772 :
2773 : /* Return whether CTX's parent compute construct is an OpenACC 'kernels'
2774 : construct.
2775 : (This doesn't include OpenACC 'kernels' decomposed parts.) */
2776 :
2777 : static bool
2778 11325 : ctx_in_oacc_kernels_region (omp_context *ctx)
2779 : {
2780 39137 : for (;ctx != NULL; ctx = ctx->outer)
2781 : {
2782 29472 : gimple *stmt = ctx->stmt;
2783 29472 : if (gimple_code (stmt) == GIMPLE_OMP_TARGET
2784 29472 : && gimple_omp_target_kind (stmt) == GF_OMP_TARGET_KIND_OACC_KERNELS)
2785 : return true;
2786 : }
2787 :
2788 : return false;
2789 : }
2790 :
2791 : /* Check the parallelism clauses inside a OpenACC 'kernels' region.
2792 : (This doesn't include OpenACC 'kernels' decomposed parts.)
2793 : Until kernels handling moves to use the same loop indirection
2794 : scheme as parallel, we need to do this checking early. */
2795 :
2796 : static unsigned
2797 7151 : check_oacc_kernel_gwv (gomp_for *stmt, omp_context *ctx)
2798 : {
2799 7151 : bool checking = true;
2800 7151 : unsigned outer_mask = 0;
2801 7151 : unsigned this_mask = 0;
2802 7151 : bool has_seq = false, has_auto = false;
2803 :
2804 7151 : if (ctx->outer)
2805 4819 : outer_mask = check_oacc_kernel_gwv (NULL, ctx->outer);
2806 7151 : if (!stmt)
2807 : {
2808 4819 : checking = false;
2809 4819 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR)
2810 : return outer_mask;
2811 1730 : stmt = as_a <gomp_for *> (ctx->stmt);
2812 : }
2813 :
2814 11330 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
2815 : {
2816 7268 : switch (OMP_CLAUSE_CODE (c))
2817 : {
2818 645 : case OMP_CLAUSE_GANG:
2819 645 : this_mask |= GOMP_DIM_MASK (GOMP_DIM_GANG);
2820 645 : break;
2821 439 : case OMP_CLAUSE_WORKER:
2822 439 : this_mask |= GOMP_DIM_MASK (GOMP_DIM_WORKER);
2823 439 : break;
2824 342 : case OMP_CLAUSE_VECTOR:
2825 342 : this_mask |= GOMP_DIM_MASK (GOMP_DIM_VECTOR);
2826 342 : break;
2827 63 : case OMP_CLAUSE_SEQ:
2828 63 : has_seq = true;
2829 63 : break;
2830 94 : case OMP_CLAUSE_AUTO:
2831 94 : has_auto = true;
2832 94 : break;
2833 : default:
2834 : break;
2835 : }
2836 : }
2837 :
2838 4062 : if (checking)
2839 : {
2840 2332 : if (has_seq && (this_mask || has_auto))
2841 36 : error_at (gimple_location (stmt), "%<seq%> overrides other"
2842 : " OpenACC loop specifiers");
2843 2296 : else if (has_auto && this_mask)
2844 30 : error_at (gimple_location (stmt), "%<auto%> conflicts with other"
2845 : " OpenACC loop specifiers");
2846 :
2847 2332 : if (this_mask & outer_mask)
2848 15 : error_at (gimple_location (stmt), "inner loop uses same"
2849 : " OpenACC parallelism as containing loop");
2850 : }
2851 :
2852 4062 : return outer_mask | this_mask;
2853 : }
2854 :
2855 : /* Scan a GIMPLE_OMP_FOR. */
2856 :
2857 : static omp_context *
2858 52612 : scan_omp_for (gomp_for *stmt, omp_context *outer_ctx)
2859 : {
2860 52612 : omp_context *ctx;
2861 52612 : size_t i;
2862 52612 : tree clauses = gimple_omp_for_clauses (stmt);
2863 :
2864 52612 : ctx = new_omp_context (stmt, outer_ctx);
2865 :
2866 52612 : if (is_gimple_omp_oacc (stmt))
2867 : {
2868 12446 : omp_context *tgt = enclosing_target_ctx (outer_ctx);
2869 :
2870 12446 : if (!(tgt && is_oacc_kernels (tgt)))
2871 32251 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2872 : {
2873 22137 : tree c_op0;
2874 22137 : switch (OMP_CLAUSE_CODE (c))
2875 : {
2876 1952 : case OMP_CLAUSE_GANG:
2877 1952 : c_op0 = OMP_CLAUSE_GANG_EXPR (c);
2878 1952 : break;
2879 :
2880 1426 : case OMP_CLAUSE_WORKER:
2881 1426 : c_op0 = OMP_CLAUSE_WORKER_EXPR (c);
2882 1426 : break;
2883 :
2884 1627 : case OMP_CLAUSE_VECTOR:
2885 1627 : c_op0 = OMP_CLAUSE_VECTOR_EXPR (c);
2886 1627 : break;
2887 :
2888 17132 : default:
2889 17132 : continue;
2890 : }
2891 :
2892 5005 : if (c_op0)
2893 : {
2894 : /* By construction, this is impossible for OpenACC 'kernels'
2895 : decomposed parts. */
2896 210 : gcc_assert (!(tgt && is_oacc_kernels_decomposed_part (tgt)));
2897 :
2898 210 : error_at (OMP_CLAUSE_LOCATION (c),
2899 : "argument not permitted on %qs clause",
2900 210 : omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
2901 210 : if (tgt)
2902 180 : inform (gimple_location (tgt->stmt),
2903 : "enclosing parent compute construct");
2904 30 : else if (oacc_get_fn_attrib (current_function_decl))
2905 30 : inform (DECL_SOURCE_LOCATION (current_function_decl),
2906 : "enclosing routine");
2907 : else
2908 0 : gcc_unreachable ();
2909 : }
2910 : }
2911 :
2912 12446 : if (tgt && is_oacc_kernels (tgt))
2913 2332 : check_oacc_kernel_gwv (stmt, ctx);
2914 :
2915 : /* Collect all variables named in reductions on this loop. Ensure
2916 : that, if this loop has a reduction on some variable v, and there is
2917 : a reduction on v somewhere in an outer context, then there is a
2918 : reduction on v on all intervening loops as well. */
2919 12446 : tree local_reduction_clauses = NULL;
2920 39374 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
2921 : {
2922 26928 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
2923 4712 : local_reduction_clauses
2924 4712 : = tree_cons (NULL, c, local_reduction_clauses);
2925 : }
2926 12446 : if (ctx->outer_reduction_clauses == NULL && ctx->outer != NULL)
2927 11920 : ctx->outer_reduction_clauses
2928 11920 : = chainon (unshare_expr (ctx->outer->local_reduction_clauses),
2929 : ctx->outer->outer_reduction_clauses);
2930 12446 : tree outer_reduction_clauses = ctx->outer_reduction_clauses;
2931 12446 : tree local_iter = local_reduction_clauses;
2932 17158 : for (; local_iter; local_iter = TREE_CHAIN (local_iter))
2933 : {
2934 4712 : tree local_clause = TREE_VALUE (local_iter);
2935 4712 : tree local_var = OMP_CLAUSE_DECL (local_clause);
2936 4712 : tree_code local_op = OMP_CLAUSE_REDUCTION_CODE (local_clause);
2937 4712 : bool have_outer_reduction = false;
2938 4712 : tree ctx_iter = outer_reduction_clauses;
2939 5826 : for (; ctx_iter; ctx_iter = TREE_CHAIN (ctx_iter))
2940 : {
2941 3325 : tree outer_clause = TREE_VALUE (ctx_iter);
2942 3325 : tree outer_var = OMP_CLAUSE_DECL (outer_clause);
2943 3325 : tree_code outer_op = OMP_CLAUSE_REDUCTION_CODE (outer_clause);
2944 3325 : if (outer_var == local_var && outer_op != local_op)
2945 : {
2946 535 : if (warning_at (OMP_CLAUSE_LOCATION (local_clause),
2947 535 : OPT_Wopenmp, "conflicting reduction "
2948 : "operations for %qE",
2949 : local_var))
2950 535 : inform (OMP_CLAUSE_LOCATION (outer_clause),
2951 : "location of the previous reduction for %qE",
2952 : outer_var);
2953 : }
2954 3325 : if (outer_var == local_var)
2955 : {
2956 : have_outer_reduction = true;
2957 : break;
2958 : }
2959 : }
2960 4712 : if (have_outer_reduction)
2961 : {
2962 : /* There is a reduction on outer_var both on this loop and on
2963 : some enclosing loop. Walk up the context tree until such a
2964 : loop with a reduction on outer_var is found, and complain
2965 : about all intervening loops that do not have such a
2966 : reduction. */
2967 2211 : struct omp_context *curr_loop = ctx->outer;
2968 2211 : bool found = false;
2969 2826 : while (curr_loop != NULL)
2970 : {
2971 2826 : tree curr_iter = curr_loop->local_reduction_clauses;
2972 3460 : for (; curr_iter; curr_iter = TREE_CHAIN (curr_iter))
2973 : {
2974 2845 : tree curr_clause = TREE_VALUE (curr_iter);
2975 2845 : tree curr_var = OMP_CLAUSE_DECL (curr_clause);
2976 2845 : if (curr_var == local_var)
2977 : {
2978 : found = true;
2979 : break;
2980 : }
2981 : }
2982 2826 : if (!found)
2983 615 : warning_at (gimple_location (curr_loop->stmt), OPT_Wopenmp,
2984 : "nested loop in reduction needs "
2985 : "reduction clause for %qE",
2986 : local_var);
2987 : else
2988 : break;
2989 615 : curr_loop = curr_loop->outer;
2990 : }
2991 : }
2992 : }
2993 12446 : ctx->local_reduction_clauses = local_reduction_clauses;
2994 12446 : ctx->outer_reduction_clauses
2995 12446 : = chainon (unshare_expr (ctx->local_reduction_clauses),
2996 : ctx->outer_reduction_clauses);
2997 :
2998 12446 : if (tgt && is_oacc_kernels (tgt))
2999 : {
3000 : /* Strip out reductions, as they are not handled yet. */
3001 : tree *prev_ptr = &clauses;
3002 :
3003 7123 : while (tree probe = *prev_ptr)
3004 : {
3005 4791 : tree *next_ptr = &OMP_CLAUSE_CHAIN (probe);
3006 :
3007 4791 : if (OMP_CLAUSE_CODE (probe) == OMP_CLAUSE_REDUCTION)
3008 718 : *prev_ptr = *next_ptr;
3009 : else
3010 : prev_ptr = next_ptr;
3011 : }
3012 :
3013 2332 : gimple_omp_for_set_clauses (stmt, clauses);
3014 : }
3015 : }
3016 :
3017 52612 : scan_sharing_clauses (clauses, ctx);
3018 :
3019 52612 : scan_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
3020 179856 : for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
3021 : {
3022 74632 : scan_omp_op (gimple_omp_for_index_ptr (stmt, i), ctx);
3023 74632 : scan_omp_op (gimple_omp_for_initial_ptr (stmt, i), ctx);
3024 74632 : scan_omp_op (gimple_omp_for_final_ptr (stmt, i), ctx);
3025 74632 : scan_omp_op (gimple_omp_for_incr_ptr (stmt, i), ctx);
3026 : }
3027 52612 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3028 52612 : return ctx;
3029 : }
3030 :
3031 : /* Duplicate #pragma omp simd, one for SIMT, another one for SIMD. */
3032 :
3033 : static void
3034 0 : scan_omp_simd (gimple_stmt_iterator *gsi, gomp_for *stmt,
3035 : omp_context *outer_ctx)
3036 : {
3037 0 : gbind *bind = gimple_build_bind (NULL, NULL, NULL);
3038 0 : gsi_replace (gsi, bind, false);
3039 0 : gimple_seq seq = NULL;
3040 0 : gimple *g = gimple_build_call_internal (IFN_GOMP_USE_SIMT, 0);
3041 0 : tree cond = create_tmp_var_raw (integer_type_node);
3042 0 : DECL_CONTEXT (cond) = current_function_decl;
3043 0 : DECL_SEEN_IN_BIND_EXPR_P (cond) = 1;
3044 0 : gimple_bind_set_vars (bind, cond);
3045 0 : gimple_call_set_lhs (g, cond);
3046 0 : gimple_seq_add_stmt (&seq, g);
3047 0 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
3048 0 : tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
3049 0 : tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
3050 0 : g = gimple_build_cond (NE_EXPR, cond, integer_zero_node, lab1, lab2);
3051 0 : gimple_seq_add_stmt (&seq, g);
3052 0 : g = gimple_build_label (lab1);
3053 0 : gimple_seq_add_stmt (&seq, g);
3054 0 : gimple_seq new_seq = copy_gimple_seq_and_replace_locals (stmt);
3055 0 : gomp_for *new_stmt = as_a <gomp_for *> (new_seq);
3056 0 : tree clause = build_omp_clause (gimple_location (stmt), OMP_CLAUSE__SIMT_);
3057 0 : OMP_CLAUSE_CHAIN (clause) = gimple_omp_for_clauses (new_stmt);
3058 0 : gimple_omp_for_set_clauses (new_stmt, clause);
3059 0 : gimple_seq_add_stmt (&seq, new_stmt);
3060 0 : g = gimple_build_goto (lab3);
3061 0 : gimple_seq_add_stmt (&seq, g);
3062 0 : g = gimple_build_label (lab2);
3063 0 : gimple_seq_add_stmt (&seq, g);
3064 0 : gimple_seq_add_stmt (&seq, stmt);
3065 0 : g = gimple_build_label (lab3);
3066 0 : gimple_seq_add_stmt (&seq, g);
3067 0 : gimple_bind_set_body (bind, seq);
3068 0 : update_stmt (bind);
3069 0 : scan_omp_for (new_stmt, outer_ctx);
3070 0 : scan_omp_for (stmt, outer_ctx)->simt_stmt = new_stmt;
3071 0 : }
3072 :
3073 : static tree omp_find_scan (gimple_stmt_iterator *, bool *,
3074 : struct walk_stmt_info *);
3075 : static omp_context *maybe_lookup_ctx (gimple *);
3076 :
3077 : /* Duplicate #pragma omp simd, one for the scan input phase loop and one
3078 : for scan phase loop. */
3079 :
3080 : static void
3081 83 : scan_omp_simd_scan (gimple_stmt_iterator *gsi, gomp_for *stmt,
3082 : omp_context *outer_ctx)
3083 : {
3084 : /* The only change between inclusive and exclusive scan will be
3085 : within the first simd loop, so just use inclusive in the
3086 : worksharing loop. */
3087 83 : outer_ctx->scan_inclusive = true;
3088 83 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_INCLUSIVE);
3089 83 : OMP_CLAUSE_DECL (c) = integer_zero_node;
3090 :
3091 83 : gomp_scan *input_stmt = gimple_build_omp_scan (NULL, NULL_TREE);
3092 83 : gomp_scan *scan_stmt = gimple_build_omp_scan (NULL, c);
3093 83 : gsi_replace (gsi, input_stmt, false);
3094 83 : gimple_seq input_body = NULL;
3095 83 : gimple_seq_add_stmt (&input_body, stmt);
3096 83 : gsi_insert_after (gsi, scan_stmt, GSI_NEW_STMT);
3097 :
3098 83 : gimple_stmt_iterator input1_gsi = gsi_none ();
3099 83 : struct walk_stmt_info wi;
3100 83 : memset (&wi, 0, sizeof (wi));
3101 83 : wi.val_only = true;
3102 83 : wi.info = (void *) &input1_gsi;
3103 83 : walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), omp_find_scan, NULL, &wi);
3104 83 : gcc_assert (!gsi_end_p (input1_gsi));
3105 :
3106 83 : gimple *input_stmt1 = gsi_stmt (input1_gsi);
3107 83 : gsi_next (&input1_gsi);
3108 83 : gimple *scan_stmt1 = gsi_stmt (input1_gsi);
3109 83 : gcc_assert (scan_stmt1 && gimple_code (scan_stmt1) == GIMPLE_OMP_SCAN);
3110 83 : c = gimple_omp_scan_clauses (as_a <gomp_scan *> (scan_stmt1));
3111 166 : if (c && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_EXCLUSIVE)
3112 : std::swap (input_stmt1, scan_stmt1);
3113 :
3114 83 : gimple_seq input_body1 = gimple_omp_body (input_stmt1);
3115 83 : gimple_omp_set_body (input_stmt1, NULL);
3116 :
3117 83 : gimple_seq scan_body = copy_gimple_seq_and_replace_locals (stmt);
3118 83 : gomp_for *new_stmt = as_a <gomp_for *> (scan_body);
3119 :
3120 83 : gimple_omp_set_body (input_stmt1, input_body1);
3121 83 : gimple_omp_set_body (scan_stmt1, NULL);
3122 :
3123 83 : gimple_stmt_iterator input2_gsi = gsi_none ();
3124 83 : memset (&wi, 0, sizeof (wi));
3125 83 : wi.val_only = true;
3126 83 : wi.info = (void *) &input2_gsi;
3127 83 : walk_gimple_seq_mod (gimple_omp_body_ptr (new_stmt), omp_find_scan,
3128 : NULL, &wi);
3129 83 : gcc_assert (!gsi_end_p (input2_gsi));
3130 :
3131 83 : gimple *input_stmt2 = gsi_stmt (input2_gsi);
3132 83 : gsi_next (&input2_gsi);
3133 83 : gimple *scan_stmt2 = gsi_stmt (input2_gsi);
3134 83 : gcc_assert (scan_stmt2 && gimple_code (scan_stmt2) == GIMPLE_OMP_SCAN);
3135 166 : if (c && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_EXCLUSIVE)
3136 : std::swap (input_stmt2, scan_stmt2);
3137 :
3138 83 : gimple_omp_set_body (input_stmt2, NULL);
3139 :
3140 83 : gimple_omp_set_body (input_stmt, input_body);
3141 83 : gimple_omp_set_body (scan_stmt, scan_body);
3142 :
3143 83 : omp_context *ctx = new_omp_context (input_stmt, outer_ctx);
3144 83 : scan_omp (gimple_omp_body_ptr (input_stmt), ctx);
3145 :
3146 83 : ctx = new_omp_context (scan_stmt, outer_ctx);
3147 83 : scan_omp (gimple_omp_body_ptr (scan_stmt), ctx);
3148 :
3149 83 : maybe_lookup_ctx (new_stmt)->for_simd_scan_phase = true;
3150 83 : }
3151 :
3152 : /* Scan an OpenMP sections directive. */
3153 :
3154 : static void
3155 589 : scan_omp_sections (gomp_sections *stmt, omp_context *outer_ctx)
3156 : {
3157 589 : omp_context *ctx;
3158 :
3159 589 : ctx = new_omp_context (stmt, outer_ctx);
3160 589 : scan_sharing_clauses (gimple_omp_sections_clauses (stmt), ctx);
3161 589 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3162 589 : }
3163 :
3164 : /* Scan an OpenMP single directive. */
3165 :
3166 : static void
3167 1236 : scan_omp_single (gomp_single *stmt, omp_context *outer_ctx)
3168 : {
3169 1236 : omp_context *ctx;
3170 1236 : tree name;
3171 :
3172 1236 : ctx = new_omp_context (stmt, outer_ctx);
3173 1236 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
3174 1236 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
3175 1236 : name = create_tmp_var_name (".omp_copy_s");
3176 1236 : name = build_decl (gimple_location (stmt),
3177 : TYPE_DECL, name, ctx->record_type);
3178 1236 : TYPE_NAME (ctx->record_type) = name;
3179 :
3180 1236 : scan_sharing_clauses (gimple_omp_single_clauses (stmt), ctx);
3181 1236 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3182 :
3183 1236 : if (TYPE_FIELDS (ctx->record_type) == NULL)
3184 1089 : ctx->record_type = NULL;
3185 : else
3186 147 : layout_type (ctx->record_type);
3187 1236 : }
3188 :
3189 : /* Scan a GIMPLE_OMP_TARGET. */
3190 :
3191 : static void
3192 41836 : scan_omp_target (gomp_target *stmt, omp_context *outer_ctx)
3193 : {
3194 41836 : omp_context *ctx;
3195 41836 : tree name;
3196 41836 : bool offloaded = is_gimple_omp_offloaded (stmt);
3197 41836 : tree clauses = gimple_omp_target_clauses (stmt);
3198 :
3199 41836 : ctx = new_omp_context (stmt, outer_ctx);
3200 41836 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
3201 41836 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
3202 41836 : name = create_tmp_var_name (".omp_data_t");
3203 41836 : name = build_decl (gimple_location (stmt),
3204 : TYPE_DECL, name, ctx->record_type);
3205 41836 : DECL_ARTIFICIAL (name) = 1;
3206 41836 : DECL_NAMELESS (name) = 1;
3207 41836 : TYPE_NAME (ctx->record_type) = name;
3208 41836 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
3209 :
3210 41836 : if (offloaded)
3211 : {
3212 24807 : create_omp_child_function (ctx, false);
3213 24807 : gimple_omp_target_set_child_fn (stmt, ctx->cb.dst_fn);
3214 : }
3215 :
3216 41836 : scan_sharing_clauses (clauses, ctx);
3217 41836 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3218 :
3219 41836 : if (TYPE_FIELDS (ctx->record_type) == NULL)
3220 7765 : ctx->record_type = ctx->receiver_decl = NULL;
3221 : else
3222 : {
3223 34071 : TYPE_FIELDS (ctx->record_type)
3224 34071 : = nreverse (TYPE_FIELDS (ctx->record_type));
3225 34071 : if (flag_checking)
3226 : {
3227 34071 : unsigned int align = DECL_ALIGN (TYPE_FIELDS (ctx->record_type));
3228 34071 : for (tree field = TYPE_FIELDS (ctx->record_type);
3229 131307 : field;
3230 97236 : field = DECL_CHAIN (field))
3231 97236 : gcc_assert (DECL_ALIGN (field) == align);
3232 : }
3233 34071 : layout_type (ctx->record_type);
3234 34071 : if (offloaded)
3235 17601 : fixup_child_record_type (ctx);
3236 : }
3237 :
3238 41836 : if (ctx->teams_nested_p && ctx->nonteams_nested_p)
3239 : {
3240 4 : error_at (gimple_location (stmt),
3241 : "%<target%> construct with nested %<teams%> construct "
3242 : "contains directives outside of the %<teams%> construct");
3243 4 : gimple_omp_set_body (stmt, gimple_build_bind (NULL, NULL, NULL));
3244 : }
3245 41836 : }
3246 :
3247 : /* Scan an OpenMP teams directive. */
3248 :
3249 : static void
3250 8761 : scan_omp_teams (gomp_teams *stmt, omp_context *outer_ctx)
3251 : {
3252 8761 : omp_context *ctx = new_omp_context (stmt, outer_ctx);
3253 :
3254 8761 : if (!gimple_omp_teams_host (stmt))
3255 : {
3256 6121 : scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
3257 6121 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3258 6121 : return;
3259 : }
3260 2640 : taskreg_contexts.safe_push (ctx);
3261 2640 : gcc_assert (taskreg_nesting_level == 1);
3262 2640 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
3263 2640 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
3264 2640 : tree name = create_tmp_var_name (".omp_data_s");
3265 2640 : name = build_decl (gimple_location (stmt),
3266 : TYPE_DECL, name, ctx->record_type);
3267 2640 : DECL_ARTIFICIAL (name) = 1;
3268 2640 : DECL_NAMELESS (name) = 1;
3269 2640 : TYPE_NAME (ctx->record_type) = name;
3270 2640 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
3271 2640 : create_omp_child_function (ctx, false);
3272 2640 : gimple_omp_teams_set_child_fn (stmt, ctx->cb.dst_fn);
3273 :
3274 2640 : scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
3275 2640 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3276 :
3277 2640 : if (TYPE_FIELDS (ctx->record_type) == NULL)
3278 1446 : ctx->record_type = ctx->receiver_decl = NULL;
3279 : }
3280 :
3281 : /* Check nesting restrictions. */
3282 : static bool
3283 156699 : check_omp_nesting_restrictions (gimple *stmt, omp_context *ctx)
3284 : {
3285 156699 : tree c;
3286 :
3287 : /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
3288 : inside an OpenACC CTX. */
3289 156699 : if (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
3290 156699 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE)
3291 : /* ..., except for the atomic codes that OpenACC shares with OpenMP. */
3292 : ;
3293 281799 : else if (!(is_gimple_omp (stmt)
3294 138840 : && is_gimple_omp_oacc (stmt)))
3295 : {
3296 112695 : if (oacc_get_fn_attrib (cfun->decl) != NULL)
3297 : {
3298 48 : error_at (gimple_location (stmt),
3299 : "non-OpenACC construct inside of OpenACC routine");
3300 48 : return false;
3301 : }
3302 : else
3303 235869 : for (omp_context *octx = ctx; octx != NULL; octx = octx->outer)
3304 246620 : if (is_gimple_omp (octx->stmt)
3305 123398 : && is_gimple_omp_oacc (octx->stmt))
3306 : {
3307 176 : error_at (gimple_location (stmt),
3308 : "non-OpenACC construct inside of OpenACC region");
3309 176 : return false;
3310 : }
3311 : }
3312 :
3313 156475 : if (ctx != NULL)
3314 : {
3315 83905 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
3316 83905 : && gimple_omp_target_kind (ctx->stmt) == GF_OMP_TARGET_KIND_REGION)
3317 : {
3318 8325 : c = omp_find_clause (gimple_omp_target_clauses (ctx->stmt),
3319 : OMP_CLAUSE_DEVICE);
3320 8325 : if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
3321 : {
3322 9 : error_at (gimple_location (stmt),
3323 : "OpenMP constructs are not allowed in target region "
3324 : "with %<ancestor%>");
3325 9 : return false;
3326 : }
3327 :
3328 8316 : if (gimple_code (stmt) == GIMPLE_OMP_TEAMS && !ctx->teams_nested_p)
3329 6117 : ctx->teams_nested_p = true;
3330 : else
3331 2199 : ctx->nonteams_nested_p = true;
3332 : }
3333 83896 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_SCAN
3334 172 : && ctx->outer
3335 84068 : && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR)
3336 : ctx = ctx->outer;
3337 83896 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3338 28194 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
3339 85705 : && !ctx->loop_p)
3340 : {
3341 1589 : c = NULL_TREE;
3342 1589 : if (ctx->order_concurrent
3343 1589 : && (gimple_code (stmt) == GIMPLE_OMP_ORDERED
3344 280 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
3345 190 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE))
3346 : {
3347 210 : error_at (gimple_location (stmt),
3348 : "OpenMP constructs other than %<parallel%>, %<loop%>"
3349 : " or %<simd%> may not be nested inside a region with"
3350 : " the %<order(concurrent)%> clause");
3351 210 : return false;
3352 : }
3353 1379 : if (gimple_code (stmt) == GIMPLE_OMP_ORDERED)
3354 : {
3355 133 : c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
3356 133 : if (omp_find_clause (c, OMP_CLAUSE_SIMD))
3357 : {
3358 133 : if (omp_find_clause (c, OMP_CLAUSE_THREADS)
3359 133 : && (ctx->outer == NULL
3360 29 : || !gimple_omp_for_combined_into_p (ctx->stmt)
3361 25 : || gimple_code (ctx->outer->stmt) != GIMPLE_OMP_FOR
3362 25 : || (gimple_omp_for_kind (ctx->outer->stmt)
3363 : != GF_OMP_FOR_KIND_FOR)
3364 25 : || !gimple_omp_for_combined_p (ctx->outer->stmt)))
3365 : {
3366 8 : error_at (gimple_location (stmt),
3367 : "%<ordered simd threads%> must be closely "
3368 : "nested inside of %<%s simd%> region",
3369 8 : lang_GNU_Fortran () ? "do" : "for");
3370 8 : return false;
3371 : }
3372 : return true;
3373 : }
3374 : }
3375 : else if (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
3376 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE
3377 : || gimple_code (stmt) == GIMPLE_OMP_SCAN
3378 : || gimple_code (stmt) == GIMPLE_OMP_STRUCTURED_BLOCK)
3379 : return true;
3380 : else if (gimple_code (stmt) == GIMPLE_OMP_FOR
3381 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
3382 : return true;
3383 60 : error_at (gimple_location (stmt),
3384 : "OpenMP constructs other than "
3385 : "%<ordered simd%>, %<simd%>, %<loop%> or %<atomic%> may "
3386 : "not be nested inside %<simd%> region");
3387 60 : return false;
3388 : }
3389 82307 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
3390 : {
3391 5848 : if ((gimple_code (stmt) != GIMPLE_OMP_FOR
3392 5547 : || (gimple_omp_for_kind (stmt) != GF_OMP_FOR_KIND_DISTRIBUTE
3393 25 : && omp_find_clause (gimple_omp_for_clauses (stmt),
3394 : OMP_CLAUSE_BIND) == NULL_TREE))
3395 5864 : && gimple_code (stmt) != GIMPLE_OMP_PARALLEL)
3396 : {
3397 132 : error_at (gimple_location (stmt),
3398 : "only %<distribute%>, %<parallel%> or %<loop%> "
3399 : "regions are allowed to be strictly nested inside "
3400 : "%<teams%> region");
3401 132 : return false;
3402 : }
3403 : }
3404 76459 : else if (ctx->order_concurrent
3405 1944 : && gimple_code (stmt) != GIMPLE_OMP_PARALLEL
3406 1465 : && (gimple_code (stmt) != GIMPLE_OMP_FOR
3407 1191 : || gimple_omp_for_kind (stmt) != GF_OMP_FOR_KIND_SIMD)
3408 289 : && gimple_code (stmt) != GIMPLE_OMP_SCAN
3409 76748 : && gimple_code (stmt) != GIMPLE_OMP_STRUCTURED_BLOCK)
3410 : {
3411 285 : if (ctx->loop_p)
3412 135 : error_at (gimple_location (stmt),
3413 : "OpenMP constructs other than %<parallel%>, %<loop%> or "
3414 : "%<simd%> may not be nested inside a %<loop%> region");
3415 : else
3416 150 : error_at (gimple_location (stmt),
3417 : "OpenMP constructs other than %<parallel%>, %<loop%> or "
3418 : "%<simd%> may not be nested inside a region with "
3419 : "the %<order(concurrent)%> clause");
3420 285 : return false;
3421 : }
3422 : }
3423 154460 : switch (gimple_code (stmt))
3424 : {
3425 52731 : case GIMPLE_OMP_FOR:
3426 52731 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_SIMD)
3427 : return true;
3428 41973 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
3429 : {
3430 8238 : if (ctx != NULL && gimple_code (ctx->stmt) != GIMPLE_OMP_TEAMS)
3431 : {
3432 0 : error_at (gimple_location (stmt),
3433 : "%<distribute%> region must be strictly nested "
3434 : "inside %<teams%> construct");
3435 0 : return false;
3436 : }
3437 : return true;
3438 : }
3439 : /* We split taskloop into task and nested taskloop in it. */
3440 33735 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_TASKLOOP)
3441 : return true;
3442 : /* For now, hope this will change and loop bind(parallel) will not
3443 : be allowed in lots of contexts. */
3444 30601 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR
3445 48691 : && omp_find_clause (gimple_omp_for_clauses (stmt), OMP_CLAUSE_BIND))
3446 : return true;
3447 29972 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
3448 : {
3449 12511 : bool ok = false;
3450 :
3451 12511 : if (ctx)
3452 11964 : switch (gimple_code (ctx->stmt))
3453 : {
3454 4049 : case GIMPLE_OMP_FOR:
3455 4049 : ok = (gimple_omp_for_kind (ctx->stmt)
3456 : == GF_OMP_FOR_KIND_OACC_LOOP);
3457 4049 : break;
3458 :
3459 7887 : case GIMPLE_OMP_TARGET:
3460 7887 : switch (gimple_omp_target_kind (ctx->stmt))
3461 : {
3462 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
3463 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
3464 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
3465 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
3466 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
3467 : ok = true;
3468 : break;
3469 :
3470 : default:
3471 : break;
3472 : }
3473 :
3474 : default:
3475 : break;
3476 : }
3477 547 : else if (oacc_get_fn_attrib (current_function_decl))
3478 : ok = true;
3479 4049 : if (!ok)
3480 : {
3481 65 : error_at (gimple_location (stmt),
3482 : "OpenACC loop directive must be associated with"
3483 : " an OpenACC compute region");
3484 65 : return false;
3485 : }
3486 : }
3487 : /* FALLTHRU */
3488 33966 : case GIMPLE_CALL:
3489 33966 : if (is_gimple_call (stmt)
3490 33966 : && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3491 : == BUILT_IN_GOMP_CANCEL
3492 3033 : || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3493 : == BUILT_IN_GOMP_CANCELLATION_POINT))
3494 : {
3495 1829 : const char *bad = NULL;
3496 1829 : const char *kind = NULL;
3497 1829 : const char *construct
3498 1829 : = (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3499 : == BUILT_IN_GOMP_CANCEL)
3500 1829 : ? "cancel"
3501 803 : : "cancellation point";
3502 1829 : if (ctx == NULL)
3503 : {
3504 41 : error_at (gimple_location (stmt), "orphaned %qs construct",
3505 : construct);
3506 41 : return false;
3507 : }
3508 1788 : switch (tree_fits_shwi_p (gimple_call_arg (stmt, 0))
3509 1788 : ? tree_to_shwi (gimple_call_arg (stmt, 0))
3510 : : 0)
3511 : {
3512 498 : case 1:
3513 498 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_PARALLEL)
3514 : bad = "parallel";
3515 178 : else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3516 : == BUILT_IN_GOMP_CANCEL
3517 178 : && !integer_zerop (gimple_call_arg (stmt, 1)))
3518 134 : ctx->cancellable = true;
3519 134 : kind = "parallel";
3520 134 : break;
3521 379 : case 2:
3522 379 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
3523 379 : || gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR)
3524 : bad = "for";
3525 69 : else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3526 : == BUILT_IN_GOMP_CANCEL
3527 69 : && !integer_zerop (gimple_call_arg (stmt, 1)))
3528 : {
3529 58 : ctx->cancellable = true;
3530 58 : if (omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3531 : OMP_CLAUSE_NOWAIT))
3532 5 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3533 : "%<cancel for%> inside "
3534 : "%<nowait%> for construct");
3535 58 : if (omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3536 : OMP_CLAUSE_ORDERED))
3537 5 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3538 : "%<cancel for%> inside "
3539 : "%<ordered%> for construct");
3540 : }
3541 5 : kind = "for";
3542 5 : break;
3543 383 : case 4:
3544 383 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_SECTIONS
3545 383 : && gimple_code (ctx->stmt) != GIMPLE_OMP_SECTION)
3546 : bad = "sections";
3547 93 : else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3548 : == BUILT_IN_GOMP_CANCEL
3549 93 : && !integer_zerop (gimple_call_arg (stmt, 1)))
3550 : {
3551 73 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
3552 : {
3553 0 : ctx->cancellable = true;
3554 0 : if (omp_find_clause (gimple_omp_sections_clauses
3555 0 : (ctx->stmt),
3556 : OMP_CLAUSE_NOWAIT))
3557 0 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3558 : "%<cancel sections%> inside "
3559 : "%<nowait%> sections construct");
3560 : }
3561 : else
3562 : {
3563 73 : gcc_assert (ctx->outer
3564 : && gimple_code (ctx->outer->stmt)
3565 : == GIMPLE_OMP_SECTIONS);
3566 73 : ctx->outer->cancellable = true;
3567 73 : if (omp_find_clause (gimple_omp_sections_clauses
3568 73 : (ctx->outer->stmt),
3569 : OMP_CLAUSE_NOWAIT))
3570 10 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3571 : "%<cancel sections%> inside "
3572 : "%<nowait%> sections construct");
3573 : }
3574 : }
3575 10 : kind = "sections";
3576 10 : break;
3577 528 : case 8:
3578 528 : if (!is_task_ctx (ctx)
3579 528 : && (!is_taskloop_ctx (ctx)
3580 54 : || ctx->outer == NULL
3581 54 : || !is_task_ctx (ctx->outer)))
3582 : bad = "task";
3583 : else
3584 : {
3585 248 : for (omp_context *octx = ctx->outer;
3586 439 : octx; octx = octx->outer)
3587 : {
3588 425 : switch (gimple_code (octx->stmt))
3589 : {
3590 : case GIMPLE_OMP_TASKGROUP:
3591 : break;
3592 40 : case GIMPLE_OMP_TARGET:
3593 40 : if (gimple_omp_target_kind (octx->stmt)
3594 : != GF_OMP_TARGET_KIND_REGION)
3595 20 : continue;
3596 : /* FALLTHRU */
3597 99 : case GIMPLE_OMP_PARALLEL:
3598 99 : case GIMPLE_OMP_TEAMS:
3599 99 : error_at (gimple_location (stmt),
3600 : "%<%s taskgroup%> construct not closely "
3601 : "nested inside of %<taskgroup%> region",
3602 : construct);
3603 99 : return false;
3604 104 : case GIMPLE_OMP_TASK:
3605 104 : if (gimple_omp_task_taskloop_p (octx->stmt)
3606 94 : && octx->outer
3607 198 : && is_taskloop_ctx (octx->outer))
3608 : {
3609 94 : tree clauses
3610 94 : = gimple_omp_for_clauses (octx->outer->stmt);
3611 94 : if (!omp_find_clause (clauses, OMP_CLAUSE_NOGROUP))
3612 : break;
3613 : }
3614 60 : continue;
3615 111 : default:
3616 111 : continue;
3617 171 : }
3618 : break;
3619 : }
3620 149 : ctx->cancellable = true;
3621 : }
3622 149 : kind = "taskgroup";
3623 149 : break;
3624 0 : default:
3625 0 : error_at (gimple_location (stmt), "invalid arguments");
3626 0 : return false;
3627 : }
3628 298 : if (bad)
3629 : {
3630 1200 : error_at (gimple_location (stmt),
3631 : "%<%s %s%> construct not closely nested inside of %qs",
3632 : construct, kind, bad);
3633 1200 : return false;
3634 : }
3635 : }
3636 : /* FALLTHRU */
3637 : case GIMPLE_OMP_SECTIONS:
3638 : case GIMPLE_OMP_SINGLE:
3639 55442 : for (; ctx != NULL; ctx = ctx->outer)
3640 37331 : switch (gimple_code (ctx->stmt))
3641 : {
3642 6825 : case GIMPLE_OMP_FOR:
3643 6825 : if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
3644 6825 : && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
3645 : break;
3646 : /* FALLTHRU */
3647 730 : case GIMPLE_OMP_SECTIONS:
3648 730 : case GIMPLE_OMP_SINGLE:
3649 730 : case GIMPLE_OMP_ORDERED:
3650 730 : case GIMPLE_OMP_MASTER:
3651 730 : case GIMPLE_OMP_MASKED:
3652 730 : case GIMPLE_OMP_TASK:
3653 730 : case GIMPLE_OMP_CRITICAL:
3654 730 : if (is_gimple_call (stmt))
3655 : {
3656 659 : if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3657 : != BUILT_IN_GOMP_BARRIER)
3658 : return true;
3659 23 : error_at (gimple_location (stmt),
3660 : "barrier region may not be closely nested inside "
3661 : "of work-sharing, %<loop%>, %<critical%>, "
3662 : "%<ordered%>, %<master%>, %<masked%>, explicit "
3663 : "%<task%> or %<taskloop%> region");
3664 23 : return false;
3665 : }
3666 71 : error_at (gimple_location (stmt),
3667 : "work-sharing region may not be closely nested inside "
3668 : "of work-sharing, %<loop%>, %<critical%>, %<ordered%>, "
3669 : "%<master%>, %<masked%>, explicit %<task%> or "
3670 : "%<taskloop%> region");
3671 71 : return false;
3672 : case GIMPLE_OMP_PARALLEL:
3673 : case GIMPLE_OMP_TEAMS:
3674 : return true;
3675 13437 : case GIMPLE_OMP_TARGET:
3676 13437 : if (gimple_omp_target_kind (ctx->stmt)
3677 : == GF_OMP_TARGET_KIND_REGION)
3678 : return true;
3679 : break;
3680 : default:
3681 : break;
3682 : }
3683 : break;
3684 : case GIMPLE_OMP_MASTER:
3685 : case GIMPLE_OMP_MASKED:
3686 1523 : for (; ctx != NULL; ctx = ctx->outer)
3687 1014 : switch (gimple_code (ctx->stmt))
3688 : {
3689 14 : case GIMPLE_OMP_FOR:
3690 14 : if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
3691 14 : && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
3692 : break;
3693 : /* FALLTHRU */
3694 30 : case GIMPLE_OMP_SECTIONS:
3695 30 : case GIMPLE_OMP_SINGLE:
3696 30 : case GIMPLE_OMP_TASK:
3697 60 : error_at (gimple_location (stmt),
3698 : "%qs region may not be closely nested inside "
3699 : "of work-sharing, %<loop%>, explicit %<task%> or "
3700 : "%<taskloop%> region",
3701 30 : gimple_code (stmt) == GIMPLE_OMP_MASTER
3702 : ? "master" : "masked");
3703 30 : return false;
3704 : case GIMPLE_OMP_PARALLEL:
3705 : case GIMPLE_OMP_TEAMS:
3706 : return true;
3707 10 : case GIMPLE_OMP_TARGET:
3708 10 : if (gimple_omp_target_kind (ctx->stmt)
3709 : == GF_OMP_TARGET_KIND_REGION)
3710 : return true;
3711 : break;
3712 : default:
3713 : break;
3714 : }
3715 : break;
3716 : case GIMPLE_OMP_SCOPE:
3717 286 : for (; ctx != NULL; ctx = ctx->outer)
3718 183 : switch (gimple_code (ctx->stmt))
3719 : {
3720 7 : case GIMPLE_OMP_FOR:
3721 7 : if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
3722 7 : && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
3723 : break;
3724 : /* FALLTHRU */
3725 25 : case GIMPLE_OMP_SECTIONS:
3726 25 : case GIMPLE_OMP_SINGLE:
3727 25 : case GIMPLE_OMP_TASK:
3728 25 : case GIMPLE_OMP_CRITICAL:
3729 25 : case GIMPLE_OMP_ORDERED:
3730 25 : case GIMPLE_OMP_MASTER:
3731 25 : case GIMPLE_OMP_MASKED:
3732 25 : error_at (gimple_location (stmt),
3733 : "%<scope%> region may not be closely nested inside "
3734 : "of work-sharing, %<loop%>, explicit %<task%>, "
3735 : "%<taskloop%>, %<critical%>, %<ordered%>, %<master%>, "
3736 : "or %<masked%> region");
3737 25 : return false;
3738 : case GIMPLE_OMP_PARALLEL:
3739 : case GIMPLE_OMP_TEAMS:
3740 : return true;
3741 5 : case GIMPLE_OMP_TARGET:
3742 5 : if (gimple_omp_target_kind (ctx->stmt)
3743 : == GF_OMP_TARGET_KIND_REGION)
3744 : return true;
3745 : break;
3746 : default:
3747 : break;
3748 : }
3749 : break;
3750 5348 : case GIMPLE_OMP_TASK:
3751 22761 : for (c = gimple_omp_task_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
3752 17417 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS)
3753 : {
3754 4 : enum omp_clause_doacross_kind kind = OMP_CLAUSE_DOACROSS_KIND (c);
3755 4 : error_at (OMP_CLAUSE_LOCATION (c),
3756 : "%<%s(%s)%> is only allowed in %<omp ordered%>",
3757 4 : OMP_CLAUSE_DOACROSS_DEPEND (c) ? "depend" : "doacross",
3758 : kind == OMP_CLAUSE_DOACROSS_SOURCE ? "source" : "sink");
3759 4 : return false;
3760 : }
3761 : break;
3762 1631 : case GIMPLE_OMP_ORDERED:
3763 2873 : for (c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
3764 2873 : c; c = OMP_CLAUSE_CHAIN (c))
3765 : {
3766 1258 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DOACROSS)
3767 : {
3768 169 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
3769 : {
3770 4 : error_at (OMP_CLAUSE_LOCATION (c),
3771 : "invalid depend kind in omp %<ordered%> %<depend%>");
3772 4 : return false;
3773 : }
3774 165 : gcc_assert (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_THREADS
3775 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SIMD);
3776 165 : continue;
3777 : }
3778 :
3779 1089 : tree oclause;
3780 : /* Look for containing ordered(N) loop. */
3781 1089 : if (ctx == NULL
3782 1077 : || gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
3783 1089 : || (oclause
3784 1077 : = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3785 : OMP_CLAUSE_ORDERED)) == NULL_TREE)
3786 : {
3787 12 : error_at (OMP_CLAUSE_LOCATION (c),
3788 : "%<ordered%> construct with %<depend%> clause "
3789 : "must be closely nested inside an %<ordered%> loop");
3790 12 : return false;
3791 : }
3792 : }
3793 1615 : c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
3794 1615 : if (omp_find_clause (c, OMP_CLAUSE_SIMD))
3795 : {
3796 : /* ordered simd must be closely nested inside of simd region,
3797 : and simd region must not encounter constructs other than
3798 : ordered simd, therefore ordered simd may be either orphaned,
3799 : or ctx->stmt must be simd. The latter case is handled already
3800 : earlier. */
3801 34 : if (ctx != NULL)
3802 : {
3803 10 : error_at (gimple_location (stmt),
3804 : "%<ordered%> %<simd%> must be closely nested inside "
3805 : "%<simd%> region");
3806 10 : return false;
3807 : }
3808 : }
3809 1605 : for (; ctx != NULL; ctx = ctx->outer)
3810 1474 : switch (gimple_code (ctx->stmt))
3811 : {
3812 38 : case GIMPLE_OMP_CRITICAL:
3813 38 : case GIMPLE_OMP_TASK:
3814 38 : case GIMPLE_OMP_ORDERED:
3815 38 : ordered_in_taskloop:
3816 38 : error_at (gimple_location (stmt),
3817 : "%<ordered%> region may not be closely nested inside "
3818 : "of %<critical%>, %<ordered%>, explicit %<task%> or "
3819 : "%<taskloop%> region");
3820 38 : return false;
3821 1416 : case GIMPLE_OMP_FOR:
3822 1416 : if (gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_TASKLOOP)
3823 10 : goto ordered_in_taskloop;
3824 1406 : tree o;
3825 1406 : o = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3826 : OMP_CLAUSE_ORDERED);
3827 1406 : if (o == NULL)
3828 : {
3829 22 : error_at (gimple_location (stmt),
3830 : "%<ordered%> region must be closely nested inside "
3831 : "a loop region with an %<ordered%> clause");
3832 22 : return false;
3833 : }
3834 1384 : if (!gimple_omp_ordered_standalone_p (stmt))
3835 : {
3836 375 : if (OMP_CLAUSE_ORDERED_DOACROSS (o))
3837 : {
3838 10 : error_at (gimple_location (stmt),
3839 : "%<ordered%> construct without %<doacross%> or "
3840 : "%<depend%> clauses must not have the same "
3841 : "binding region as %<ordered%> construct with "
3842 : "those clauses");
3843 10 : return false;
3844 : }
3845 365 : else if (OMP_CLAUSE_ORDERED_EXPR (o))
3846 : {
3847 23 : tree co
3848 23 : = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3849 : OMP_CLAUSE_COLLAPSE);
3850 23 : HOST_WIDE_INT
3851 23 : o_n = tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (o));
3852 23 : HOST_WIDE_INT c_n = 1;
3853 23 : if (co)
3854 5 : c_n = tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (co));
3855 23 : if (o_n != c_n)
3856 : {
3857 10 : error_at (gimple_location (stmt),
3858 : "%<ordered%> construct without %<doacross%> "
3859 : "or %<depend%> clauses binds to loop where "
3860 : "%<collapse%> argument %wd is different from "
3861 : "%<ordered%> argument %wd", c_n, o_n);
3862 10 : return false;
3863 : }
3864 : }
3865 : }
3866 : return true;
3867 10 : case GIMPLE_OMP_TARGET:
3868 10 : if (gimple_omp_target_kind (ctx->stmt)
3869 : != GF_OMP_TARGET_KIND_REGION)
3870 : break;
3871 : /* FALLTHRU */
3872 30 : case GIMPLE_OMP_PARALLEL:
3873 30 : case GIMPLE_OMP_TEAMS:
3874 30 : error_at (gimple_location (stmt),
3875 : "%<ordered%> region must be closely nested inside "
3876 : "a loop region with an %<ordered%> clause");
3877 30 : return false;
3878 : default:
3879 : break;
3880 : }
3881 : break;
3882 462 : case GIMPLE_OMP_CRITICAL:
3883 462 : {
3884 462 : tree this_stmt_name
3885 462 : = gimple_omp_critical_name (as_a <gomp_critical *> (stmt));
3886 888 : for (; ctx != NULL; ctx = ctx->outer)
3887 432 : if (gomp_critical *other_crit
3888 451 : = dyn_cast <gomp_critical *> (ctx->stmt))
3889 25 : if (this_stmt_name == gimple_omp_critical_name (other_crit))
3890 : {
3891 6 : error_at (gimple_location (stmt),
3892 : "%<critical%> region may not be nested inside "
3893 : "a %<critical%> region with the same name");
3894 6 : return false;
3895 : }
3896 : }
3897 : break;
3898 8805 : case GIMPLE_OMP_TEAMS:
3899 8805 : if (ctx == NULL)
3900 : break;
3901 6165 : else if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET
3902 6165 : || (gimple_omp_target_kind (ctx->stmt)
3903 : != GF_OMP_TARGET_KIND_REGION))
3904 : {
3905 : /* Teams construct can appear either strictly nested inside of
3906 : target construct with no intervening stmts, or can be encountered
3907 : only by initial task (so must not appear inside any OpenMP
3908 : construct. */
3909 44 : error_at (gimple_location (stmt),
3910 : "%<teams%> construct must be closely nested inside of "
3911 : "%<target%> construct or not nested in any OpenMP "
3912 : "construct");
3913 44 : return false;
3914 : }
3915 : break;
3916 42219 : case GIMPLE_OMP_TARGET:
3917 189290 : for (c = gimple_omp_target_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
3918 147075 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS)
3919 : {
3920 4 : enum omp_clause_doacross_kind kind = OMP_CLAUSE_DOACROSS_KIND (c);
3921 4 : error_at (OMP_CLAUSE_LOCATION (c),
3922 : "%<depend(%s)%> is only allowed in %<omp ordered%>",
3923 : kind == OMP_CLAUSE_DOACROSS_SOURCE ? "source" : "sink");
3924 4 : return false;
3925 : }
3926 42215 : if (is_gimple_omp_offloaded (stmt)
3927 42215 : && oacc_get_fn_attrib (cfun->decl) != NULL)
3928 : {
3929 4 : error_at (gimple_location (stmt),
3930 : "OpenACC region inside of OpenACC routine, nested "
3931 : "parallelism not supported yet");
3932 4 : return false;
3933 : }
3934 49625 : for (; ctx != NULL; ctx = ctx->outer)
3935 : {
3936 7820 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET)
3937 : {
3938 768 : if (is_gimple_omp (stmt)
3939 768 : && is_gimple_omp_oacc (stmt)
3940 242 : && is_gimple_omp (ctx->stmt))
3941 : {
3942 242 : error_at (gimple_location (stmt),
3943 : "OpenACC construct inside of non-OpenACC region");
3944 242 : return false;
3945 : }
3946 526 : continue;
3947 : }
3948 :
3949 7052 : const char *stmt_name, *ctx_stmt_name;
3950 7052 : switch (gimple_omp_target_kind (stmt))
3951 : {
3952 : case GF_OMP_TARGET_KIND_REGION: stmt_name = "target"; break;
3953 399 : case GF_OMP_TARGET_KIND_DATA: stmt_name = "target data"; break;
3954 1688 : case GF_OMP_TARGET_KIND_UPDATE: stmt_name = "target update"; break;
3955 12 : case GF_OMP_TARGET_KIND_ENTER_DATA:
3956 12 : stmt_name = "target enter data"; break;
3957 18 : case GF_OMP_TARGET_KIND_EXIT_DATA:
3958 18 : stmt_name = "target exit data"; break;
3959 1442 : case GF_OMP_TARGET_KIND_OACC_PARALLEL: stmt_name = "parallel"; break;
3960 1621 : case GF_OMP_TARGET_KIND_OACC_KERNELS: stmt_name = "kernels"; break;
3961 183 : case GF_OMP_TARGET_KIND_OACC_SERIAL: stmt_name = "serial"; break;
3962 386 : case GF_OMP_TARGET_KIND_OACC_DATA: stmt_name = "data"; break;
3963 256 : case GF_OMP_TARGET_KIND_OACC_UPDATE: stmt_name = "update"; break;
3964 94 : case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
3965 94 : stmt_name = "enter data"; break;
3966 86 : case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
3967 86 : stmt_name = "exit data"; break;
3968 64 : case GF_OMP_TARGET_KIND_OACC_DECLARE: stmt_name = "declare"; break;
3969 128 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA: stmt_name = "host_data";
3970 128 : break;
3971 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
3972 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
3973 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
3974 : /* OpenACC 'kernels' decomposed parts. */
3975 1621 : stmt_name = "kernels"; break;
3976 0 : default: gcc_unreachable ();
3977 : }
3978 7052 : switch (gimple_omp_target_kind (ctx->stmt))
3979 : {
3980 : case GF_OMP_TARGET_KIND_REGION: ctx_stmt_name = "target"; break;
3981 2712 : case GF_OMP_TARGET_KIND_DATA: ctx_stmt_name = "target data"; break;
3982 35 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
3983 35 : ctx_stmt_name = "parallel"; break;
3984 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
3985 1037 : ctx_stmt_name = "kernels"; break;
3986 35 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
3987 35 : ctx_stmt_name = "serial"; break;
3988 3117 : case GF_OMP_TARGET_KIND_OACC_DATA: ctx_stmt_name = "data"; break;
3989 8 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
3990 8 : ctx_stmt_name = "host_data"; break;
3991 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
3992 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
3993 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
3994 : /* OpenACC 'kernels' decomposed parts. */
3995 1037 : ctx_stmt_name = "kernels"; break;
3996 0 : default: gcc_unreachable ();
3997 : }
3998 :
3999 : /* OpenACC/OpenMP mismatch? */
4000 14104 : if (is_gimple_omp_oacc (stmt)
4001 7052 : != is_gimple_omp_oacc (ctx->stmt))
4002 : {
4003 56 : error_at (gimple_location (stmt),
4004 : "%s %qs construct inside of %s %qs region",
4005 : (is_gimple_omp_oacc (stmt)
4006 : ? "OpenACC" : "OpenMP"), stmt_name,
4007 : (is_gimple_omp_oacc (ctx->stmt)
4008 : ? "OpenACC" : "OpenMP"), ctx_stmt_name);
4009 28 : return false;
4010 : }
4011 7024 : if (is_gimple_omp_offloaded (ctx->stmt))
4012 : {
4013 : /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
4014 185 : if (is_gimple_omp_oacc (ctx->stmt))
4015 : {
4016 105 : error_at (gimple_location (stmt),
4017 : "%qs construct inside of %qs region",
4018 : stmt_name, ctx_stmt_name);
4019 105 : return false;
4020 : }
4021 : else
4022 : {
4023 80 : if ((gimple_omp_target_kind (ctx->stmt)
4024 : == GF_OMP_TARGET_KIND_REGION)
4025 80 : && (gimple_omp_target_kind (stmt)
4026 : == GF_OMP_TARGET_KIND_REGION))
4027 : {
4028 58 : c = omp_find_clause (gimple_omp_target_clauses (stmt),
4029 : OMP_CLAUSE_DEVICE);
4030 58 : if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
4031 : break;
4032 : }
4033 49 : warning_at (gimple_location (stmt), OPT_Wopenmp,
4034 : "%qs construct inside of %qs region",
4035 : stmt_name, ctx_stmt_name);
4036 : }
4037 : }
4038 : }
4039 : break;
4040 : default:
4041 : break;
4042 : }
4043 : return true;
4044 : }
4045 :
4046 :
4047 : /* Helper function scan_omp.
4048 :
4049 : Callback for walk_tree or operators in walk_gimple_stmt used to
4050 : scan for OMP directives in TP. */
4051 :
4052 : static tree
4053 9800862 : scan_omp_1_op (tree *tp, int *walk_subtrees, void *data)
4054 : {
4055 9800862 : struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
4056 9800862 : omp_context *ctx = (omp_context *) wi->info;
4057 9800862 : tree t = *tp;
4058 9800862 : tree tmp;
4059 :
4060 9800862 : switch (TREE_CODE (t))
4061 : {
4062 4148086 : case VAR_DECL:
4063 4148086 : case PARM_DECL:
4064 4148086 : case LABEL_DECL:
4065 4148086 : case RESULT_DECL:
4066 4148086 : if (ctx)
4067 : {
4068 2124189 : tmp = NULL_TREE;
4069 2124189 : if (TREE_CODE (t) == VAR_DECL
4070 3806649 : && (tmp = lookup_attribute ("omp allocate var",
4071 1682460 : DECL_ATTRIBUTES (t))) != NULL_TREE)
4072 61 : t = TREE_VALUE (TREE_VALUE (tmp));
4073 2124189 : tree repl = remap_decl (t, &ctx->cb);
4074 2124189 : gcc_checking_assert (TREE_CODE (repl) != ERROR_MARK);
4075 2124189 : if (tmp != NULL_TREE && t != repl)
4076 31 : *tp = build_fold_addr_expr (repl);
4077 2124158 : else if (tmp == NULL_TREE)
4078 2124128 : *tp = repl;
4079 : }
4080 : break;
4081 :
4082 192793 : case INDIRECT_REF:
4083 192793 : case MEM_REF:
4084 192793 : if (ctx
4085 77636 : && TREE_CODE (TREE_OPERAND (t, 0)) == VAR_DECL
4086 251760 : && ((tmp = lookup_attribute ("omp allocate var",
4087 58967 : DECL_ATTRIBUTES (TREE_OPERAND (t, 0))))
4088 : != NULL_TREE))
4089 : {
4090 127 : tmp = TREE_VALUE (TREE_VALUE (tmp));
4091 127 : tree repl = remap_decl (tmp, &ctx->cb);
4092 127 : gcc_checking_assert (TREE_CODE (repl) != ERROR_MARK);
4093 127 : if (tmp != repl)
4094 118 : *tp = repl;
4095 : break;
4096 : }
4097 5652649 : gcc_fallthrough ();
4098 :
4099 5652649 : default:
4100 5652649 : if (ctx && TYPE_P (t))
4101 5 : *tp = remap_type (t, &ctx->cb);
4102 5652644 : else if (!DECL_P (t))
4103 : {
4104 4689245 : *walk_subtrees = 1;
4105 4689245 : if (ctx)
4106 : {
4107 1424797 : tree tem = remap_type (TREE_TYPE (t), &ctx->cb);
4108 1424797 : if (tem != TREE_TYPE (t))
4109 : {
4110 9814 : if (TREE_CODE (t) == INTEGER_CST)
4111 4136 : *tp = wide_int_to_tree (tem, wi::to_wide (t));
4112 : else
4113 5678 : TREE_TYPE (t) = tem;
4114 : }
4115 : }
4116 : }
4117 : break;
4118 : }
4119 :
4120 9800862 : return NULL_TREE;
4121 : }
4122 :
4123 : /* Return true if FNDECL is a setjmp or a longjmp. */
4124 :
4125 : static bool
4126 3042 : setjmp_or_longjmp_p (const_tree fndecl)
4127 : {
4128 3042 : if (fndecl_built_in_p (fndecl, BUILT_IN_SETJMP, BUILT_IN_LONGJMP))
4129 : return true;
4130 :
4131 3042 : tree declname = DECL_NAME (fndecl);
4132 3042 : if (!declname
4133 3042 : || (DECL_CONTEXT (fndecl) != NULL_TREE
4134 2347 : && TREE_CODE (DECL_CONTEXT (fndecl)) != TRANSLATION_UNIT_DECL)
4135 5833 : || !TREE_PUBLIC (fndecl))
4136 : return false;
4137 :
4138 2629 : const char *name = IDENTIFIER_POINTER (declname);
4139 2629 : return !strcmp (name, "setjmp") || !strcmp (name, "longjmp");
4140 : }
4141 :
4142 : /* Helper function for scan_omp.
4143 :
4144 : Callback for walk_gimple_stmt used to scan for OMP directives in
4145 : the current statement in GSI. */
4146 :
4147 : static tree
4148 3404667 : scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
4149 : struct walk_stmt_info *wi)
4150 : {
4151 3404667 : gimple *stmt = gsi_stmt (*gsi);
4152 3404667 : omp_context *ctx = (omp_context *) wi->info;
4153 :
4154 3404667 : if (gimple_has_location (stmt))
4155 2778954 : input_location = gimple_location (stmt);
4156 :
4157 : /* Check the nesting restrictions. */
4158 3404667 : bool remove = false;
4159 3404667 : if (is_gimple_omp (stmt))
4160 152580 : remove = !check_omp_nesting_restrictions (stmt, ctx);
4161 3252087 : else if (is_gimple_call (stmt))
4162 : {
4163 230986 : tree fndecl = gimple_call_fndecl (stmt);
4164 230986 : if (fndecl)
4165 : {
4166 215158 : if (ctx
4167 67514 : && gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
4168 14457 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
4169 3042 : && setjmp_or_longjmp_p (fndecl)
4170 215166 : && !ctx->loop_p)
4171 : {
4172 4 : remove = true;
4173 4 : error_at (gimple_location (stmt),
4174 : "setjmp/longjmp inside %<simd%> construct");
4175 : }
4176 215154 : else if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
4177 63290 : switch (DECL_FUNCTION_CODE (fndecl))
4178 : {
4179 4119 : case BUILT_IN_GOMP_BARRIER:
4180 4119 : case BUILT_IN_GOMP_CANCEL:
4181 4119 : case BUILT_IN_GOMP_CANCELLATION_POINT:
4182 4119 : case BUILT_IN_GOMP_TASKYIELD:
4183 4119 : case BUILT_IN_GOMP_TASKWAIT:
4184 4119 : case BUILT_IN_GOMP_TASKGROUP_START:
4185 4119 : case BUILT_IN_GOMP_TASKGROUP_END:
4186 4119 : remove = !check_omp_nesting_restrictions (stmt, ctx);
4187 4119 : break;
4188 : default:
4189 : break;
4190 : }
4191 151864 : else if (ctx)
4192 : {
4193 45659 : omp_context *octx = ctx;
4194 45659 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_SCAN && ctx->outer)
4195 45659 : octx = ctx->outer;
4196 45659 : if (octx->order_concurrent && omp_runtime_api_call (fndecl))
4197 : {
4198 240 : remove = true;
4199 240 : error_at (gimple_location (stmt),
4200 : "OpenMP runtime API call %qD in a region with "
4201 : "%<order(concurrent)%> clause", fndecl);
4202 : }
4203 45659 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
4204 3337 : && omp_runtime_api_call (fndecl)
4205 339 : && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
4206 : != strlen ("omp_get_num_teams"))
4207 185 : || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
4208 : "omp_get_num_teams") != 0)
4209 169 : && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
4210 : != strlen ("omp_get_num_teams_dim"))
4211 6 : || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
4212 : "omp_get_num_teams_dim") != 0)
4213 163 : && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
4214 : != strlen ("omp_get_team_num"))
4215 124 : || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
4216 : "omp_get_team_num") != 0)
4217 45698 : && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
4218 : != strlen ("omp_get_team_num_dim"))
4219 12 : || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
4220 : "omp_get_team_num_dim") != 0))
4221 : {
4222 27 : remove = true;
4223 27 : error_at (gimple_location (stmt),
4224 : "OpenMP runtime API call %qD strictly nested in a "
4225 : "%<teams%> region", fndecl);
4226 : }
4227 45659 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
4228 13340 : && (gimple_omp_target_kind (ctx->stmt)
4229 : == GF_OMP_TARGET_KIND_REGION)
4230 50613 : && omp_runtime_api_call (fndecl))
4231 : {
4232 379 : tree tgt_clauses = gimple_omp_target_clauses (ctx->stmt);
4233 379 : tree c = omp_find_clause (tgt_clauses, OMP_CLAUSE_DEVICE);
4234 379 : if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
4235 5 : error_at (gimple_location (stmt),
4236 : "OpenMP runtime API call %qD in a region with "
4237 : "%<device(ancestor)%> clause", fndecl);
4238 : }
4239 : }
4240 : }
4241 : }
4242 202362 : if (remove)
4243 : {
4244 3326 : stmt = gimple_build_nop ();
4245 3326 : gsi_replace (gsi, stmt, false);
4246 : }
4247 :
4248 3404667 : *handled_ops_p = true;
4249 :
4250 3404667 : switch (gimple_code (stmt))
4251 : {
4252 18168 : case GIMPLE_OMP_PARALLEL:
4253 18168 : taskreg_nesting_level++;
4254 18168 : scan_omp_parallel (gsi, ctx);
4255 18168 : taskreg_nesting_level--;
4256 18168 : break;
4257 :
4258 5344 : case GIMPLE_OMP_TASK:
4259 5344 : taskreg_nesting_level++;
4260 5344 : scan_omp_task (gsi, ctx);
4261 5344 : taskreg_nesting_level--;
4262 5344 : break;
4263 :
4264 52695 : case GIMPLE_OMP_FOR:
4265 52695 : if ((gimple_omp_for_kind (as_a <gomp_for *> (stmt))
4266 : == GF_OMP_FOR_KIND_SIMD)
4267 10810 : && gimple_omp_for_combined_into_p (stmt)
4268 60387 : && gimple_code (ctx->stmt) != GIMPLE_OMP_SCAN)
4269 : {
4270 7526 : tree clauses = gimple_omp_for_clauses (as_a <gomp_for *> (stmt));
4271 7526 : tree c = omp_find_clause (clauses, OMP_CLAUSE_REDUCTION);
4272 7526 : if (c && OMP_CLAUSE_REDUCTION_INSCAN (c) && !seen_error ())
4273 : {
4274 83 : scan_omp_simd_scan (gsi, as_a <gomp_for *> (stmt), ctx);
4275 83 : break;
4276 : }
4277 : }
4278 52612 : if ((gimple_omp_for_kind (as_a <gomp_for *> (stmt))
4279 : == GF_OMP_FOR_KIND_SIMD)
4280 10727 : && omp_maybe_offloaded_ctx (ctx)
4281 3641 : && omp_max_simt_vf ()
4282 52612 : && gimple_omp_for_collapse (stmt) == 1)
4283 0 : scan_omp_simd (gsi, as_a <gomp_for *> (stmt), ctx);
4284 : else
4285 52612 : scan_omp_for (as_a <gomp_for *> (stmt), ctx);
4286 : break;
4287 :
4288 196 : case GIMPLE_OMP_SCOPE:
4289 196 : ctx = new_omp_context (stmt, ctx);
4290 196 : scan_sharing_clauses (gimple_omp_scope_clauses (stmt), ctx);
4291 196 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4292 196 : break;
4293 :
4294 852 : case GIMPLE_OMP_DISPATCH:
4295 852 : ctx = new_omp_context (stmt, ctx);
4296 852 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4297 852 : break;
4298 :
4299 612 : case GIMPLE_OMP_INTEROP:
4300 612 : ctx = new_omp_context (stmt, ctx);
4301 612 : break;
4302 :
4303 589 : case GIMPLE_OMP_SECTIONS:
4304 589 : scan_omp_sections (as_a <gomp_sections *> (stmt), ctx);
4305 589 : break;
4306 :
4307 1236 : case GIMPLE_OMP_SINGLE:
4308 1236 : scan_omp_single (as_a <gomp_single *> (stmt), ctx);
4309 1236 : break;
4310 :
4311 1292 : case GIMPLE_OMP_SCAN:
4312 1292 : if (tree clauses = gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt)))
4313 : {
4314 610 : if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_INCLUSIVE)
4315 308 : ctx->scan_inclusive = true;
4316 302 : else if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_EXCLUSIVE)
4317 302 : ctx->scan_exclusive = true;
4318 : }
4319 : /* FALLTHRU */
4320 6223 : case GIMPLE_OMP_SECTION:
4321 6223 : case GIMPLE_OMP_STRUCTURED_BLOCK:
4322 6223 : case GIMPLE_OMP_MASTER:
4323 6223 : case GIMPLE_OMP_ORDERED:
4324 6223 : case GIMPLE_OMP_CRITICAL:
4325 6223 : ctx = new_omp_context (stmt, ctx);
4326 6223 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4327 6223 : break;
4328 :
4329 457 : case GIMPLE_OMP_MASKED:
4330 457 : ctx = new_omp_context (stmt, ctx);
4331 457 : scan_sharing_clauses (gimple_omp_masked_clauses (stmt), ctx);
4332 457 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4333 457 : break;
4334 :
4335 607 : case GIMPLE_OMP_TASKGROUP:
4336 607 : ctx = new_omp_context (stmt, ctx);
4337 607 : scan_sharing_clauses (gimple_omp_taskgroup_clauses (stmt), ctx);
4338 607 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4339 607 : break;
4340 :
4341 41836 : case GIMPLE_OMP_TARGET:
4342 41836 : if (is_gimple_omp_offloaded (stmt))
4343 : {
4344 24807 : taskreg_nesting_level++;
4345 24807 : scan_omp_target (as_a <gomp_target *> (stmt), ctx);
4346 24807 : taskreg_nesting_level--;
4347 : }
4348 : else
4349 17029 : scan_omp_target (as_a <gomp_target *> (stmt), ctx);
4350 : break;
4351 :
4352 8761 : case GIMPLE_OMP_TEAMS:
4353 8761 : if (gimple_omp_teams_host (as_a <gomp_teams *> (stmt)))
4354 : {
4355 2640 : taskreg_nesting_level++;
4356 2640 : scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
4357 2640 : taskreg_nesting_level--;
4358 : }
4359 : else
4360 6121 : scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
4361 : break;
4362 :
4363 281271 : case GIMPLE_BIND:
4364 281271 : {
4365 281271 : tree var;
4366 :
4367 281271 : *handled_ops_p = false;
4368 281271 : if (ctx)
4369 152322 : for (var = gimple_bind_vars (as_a <gbind *> (stmt));
4370 578133 : var ;
4371 425811 : var = DECL_CHAIN (var))
4372 425811 : insert_decl_map (&ctx->cb, var, var);
4373 : }
4374 : break;
4375 2985820 : default:
4376 2985820 : *handled_ops_p = false;
4377 2985820 : break;
4378 : }
4379 :
4380 3404667 : return NULL_TREE;
4381 : }
4382 :
4383 :
4384 : /* Scan all the statements starting at the current statement. CTX
4385 : contains context information about the OMP directives and
4386 : clauses found during the scan. */
4387 :
4388 : static void
4389 258541 : scan_omp (gimple_seq *body_p, omp_context *ctx)
4390 : {
4391 258541 : location_t saved_location;
4392 258541 : struct walk_stmt_info wi;
4393 :
4394 258541 : memset (&wi, 0, sizeof (wi));
4395 258541 : wi.info = ctx;
4396 258541 : wi.want_locations = true;
4397 :
4398 258541 : saved_location = input_location;
4399 258541 : walk_gimple_seq_mod (body_p, scan_omp_1_stmt, scan_omp_1_op, &wi);
4400 258541 : input_location = saved_location;
4401 258541 : }
4402 :
4403 : /* Re-gimplification and code generation routines. */
4404 :
4405 : /* Remove omp_member_access_dummy_var variables from gimple_bind_vars
4406 : of BIND if in a method. */
4407 :
4408 : static void
4409 258383 : maybe_remove_omp_member_access_dummy_vars (gbind *bind)
4410 : {
4411 258383 : if (DECL_ARGUMENTS (current_function_decl)
4412 97552 : && DECL_ARTIFICIAL (DECL_ARGUMENTS (current_function_decl))
4413 269133 : && (TREE_CODE (TREE_TYPE (DECL_ARGUMENTS (current_function_decl)))
4414 : == POINTER_TYPE))
4415 : {
4416 3520 : tree vars = gimple_bind_vars (bind);
4417 19340 : for (tree *pvar = &vars; *pvar; )
4418 15820 : if (omp_member_access_dummy_var (*pvar))
4419 719 : *pvar = DECL_CHAIN (*pvar);
4420 : else
4421 15101 : pvar = &DECL_CHAIN (*pvar);
4422 3520 : gimple_bind_set_vars (bind, vars);
4423 : }
4424 258383 : }
4425 :
4426 : /* Remove omp_member_access_dummy_var variables from BLOCK_VARS of
4427 : block and its subblocks. */
4428 :
4429 : static void
4430 12852 : remove_member_access_dummy_vars (tree block)
4431 : {
4432 22413 : for (tree *pvar = &BLOCK_VARS (block); *pvar; )
4433 9561 : if (omp_member_access_dummy_var (*pvar))
4434 108 : *pvar = DECL_CHAIN (*pvar);
4435 : else
4436 9453 : pvar = &DECL_CHAIN (*pvar);
4437 :
4438 16737 : for (block = BLOCK_SUBBLOCKS (block); block; block = BLOCK_CHAIN (block))
4439 3885 : remove_member_access_dummy_vars (block);
4440 12852 : }
4441 :
4442 : /* If a context was created for STMT when it was scanned, return it. */
4443 :
4444 : static omp_context *
4445 121281 : maybe_lookup_ctx (gimple *stmt)
4446 : {
4447 121281 : splay_tree_node n;
4448 121281 : n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt);
4449 121281 : return n ? (omp_context *) n->value : NULL;
4450 : }
4451 :
4452 :
4453 : /* Find the mapping for DECL in CTX or the immediately enclosing
4454 : context that has a mapping for DECL.
4455 :
4456 : If CTX is a nested parallel directive, we may have to use the decl
4457 : mappings created in CTX's parent context. Suppose that we have the
4458 : following parallel nesting (variable UIDs showed for clarity):
4459 :
4460 : iD.1562 = 0;
4461 : #omp parallel shared(iD.1562) -> outer parallel
4462 : iD.1562 = iD.1562 + 1;
4463 :
4464 : #omp parallel shared (iD.1562) -> inner parallel
4465 : iD.1562 = iD.1562 - 1;
4466 :
4467 : Each parallel structure will create a distinct .omp_data_s structure
4468 : for copying iD.1562 in/out of the directive:
4469 :
4470 : outer parallel .omp_data_s.1.i -> iD.1562
4471 : inner parallel .omp_data_s.2.i -> iD.1562
4472 :
4473 : A shared variable mapping will produce a copy-out operation before
4474 : the parallel directive and a copy-in operation after it. So, in
4475 : this case we would have:
4476 :
4477 : iD.1562 = 0;
4478 : .omp_data_o.1.i = iD.1562;
4479 : #omp parallel shared(iD.1562) -> outer parallel
4480 : .omp_data_i.1 = &.omp_data_o.1
4481 : .omp_data_i.1->i = .omp_data_i.1->i + 1;
4482 :
4483 : .omp_data_o.2.i = iD.1562; -> **
4484 : #omp parallel shared(iD.1562) -> inner parallel
4485 : .omp_data_i.2 = &.omp_data_o.2
4486 : .omp_data_i.2->i = .omp_data_i.2->i - 1;
4487 :
4488 :
4489 : ** This is a problem. The symbol iD.1562 cannot be referenced
4490 : inside the body of the outer parallel region. But since we are
4491 : emitting this copy operation while expanding the inner parallel
4492 : directive, we need to access the CTX structure of the outer
4493 : parallel directive to get the correct mapping:
4494 :
4495 : .omp_data_o.2.i = .omp_data_i.1->i
4496 :
4497 : Since there may be other workshare or parallel directives enclosing
4498 : the parallel directive, it may be necessary to walk up the context
4499 : parent chain. This is not a problem in general because nested
4500 : parallelism happens only rarely. */
4501 :
4502 : static tree
4503 126033 : lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
4504 : {
4505 126033 : tree t;
4506 126033 : omp_context *up;
4507 :
4508 182685 : for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
4509 56652 : t = maybe_lookup_decl (decl, up);
4510 :
4511 126033 : gcc_assert (!ctx->is_nested || t || is_global_var (decl));
4512 :
4513 97784 : return t ? t : decl;
4514 : }
4515 :
4516 :
4517 : /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
4518 : in outer contexts. */
4519 :
4520 : static tree
4521 355624 : maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
4522 : {
4523 355624 : tree t = NULL;
4524 355624 : omp_context *up;
4525 :
4526 626198 : for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
4527 270574 : t = maybe_lookup_decl (decl, up);
4528 :
4529 355624 : return t ? t : decl;
4530 : }
4531 :
4532 :
4533 : /* Construct the initialization value for reduction operation OP. */
4534 :
4535 : tree
4536 13101 : omp_reduction_init_op (location_t loc, enum tree_code op, tree type)
4537 : {
4538 13101 : switch (op)
4539 : {
4540 10766 : case PLUS_EXPR:
4541 10766 : case MINUS_EXPR:
4542 10766 : case BIT_IOR_EXPR:
4543 10766 : case BIT_XOR_EXPR:
4544 10766 : case TRUTH_OR_EXPR:
4545 10766 : case TRUTH_ORIF_EXPR:
4546 10766 : case TRUTH_XOR_EXPR:
4547 10766 : case NE_EXPR:
4548 10766 : return build_zero_cst (type);
4549 :
4550 1431 : case MULT_EXPR:
4551 1431 : case TRUTH_AND_EXPR:
4552 1431 : case TRUTH_ANDIF_EXPR:
4553 1431 : case EQ_EXPR:
4554 1431 : return fold_convert_loc (loc, type, integer_one_node);
4555 :
4556 208 : case BIT_AND_EXPR:
4557 208 : return fold_convert_loc (loc, type, integer_minus_one_node);
4558 :
4559 388 : case MAX_EXPR:
4560 388 : if (SCALAR_FLOAT_TYPE_P (type))
4561 : {
4562 158 : REAL_VALUE_TYPE min;
4563 158 : if (HONOR_INFINITIES (type))
4564 158 : real_arithmetic (&min, NEGATE_EXPR, &dconstinf, NULL);
4565 : else
4566 0 : real_maxval (&min, 1, TYPE_MODE (type));
4567 158 : return build_real (type, min);
4568 : }
4569 230 : else if (POINTER_TYPE_P (type))
4570 : {
4571 2 : wide_int min
4572 2 : = wi::min_value (TYPE_PRECISION (type), TYPE_SIGN (type));
4573 2 : return wide_int_to_tree (type, min);
4574 2 : }
4575 : else
4576 : {
4577 228 : gcc_assert (INTEGRAL_TYPE_P (type));
4578 228 : return TYPE_MIN_VALUE (type);
4579 : }
4580 :
4581 308 : case MIN_EXPR:
4582 308 : if (SCALAR_FLOAT_TYPE_P (type))
4583 : {
4584 108 : REAL_VALUE_TYPE max;
4585 108 : if (HONOR_INFINITIES (type))
4586 108 : max = dconstinf;
4587 : else
4588 0 : real_maxval (&max, 0, TYPE_MODE (type));
4589 108 : return build_real (type, max);
4590 : }
4591 200 : else if (POINTER_TYPE_P (type))
4592 : {
4593 2 : wide_int max
4594 2 : = wi::max_value (TYPE_PRECISION (type), TYPE_SIGN (type));
4595 2 : return wide_int_to_tree (type, max);
4596 2 : }
4597 : else
4598 : {
4599 198 : gcc_assert (INTEGRAL_TYPE_P (type));
4600 198 : return TYPE_MAX_VALUE (type);
4601 : }
4602 :
4603 0 : default:
4604 0 : gcc_unreachable ();
4605 : }
4606 : }
4607 :
4608 : /* Construct the initialization value for reduction CLAUSE. */
4609 :
4610 : tree
4611 10480 : omp_reduction_init (tree clause, tree type)
4612 : {
4613 10480 : return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause),
4614 10480 : OMP_CLAUSE_REDUCTION_CODE (clause), type);
4615 : }
4616 :
4617 : /* Return alignment to be assumed for var in CLAUSE, which should be
4618 : OMP_CLAUSE_ALIGNED. */
4619 :
4620 : static tree
4621 140 : omp_clause_aligned_alignment (tree clause)
4622 : {
4623 140 : if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause))
4624 123 : return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause);
4625 :
4626 : /* Otherwise return implementation defined alignment. */
4627 17 : unsigned int al = 1;
4628 17 : opt_scalar_mode mode_iter;
4629 17 : auto_vector_modes modes;
4630 17 : targetm.vectorize.autovectorize_vector_modes (&modes, true);
4631 17 : static enum mode_class classes[]
4632 : = { MODE_INT, MODE_VECTOR_INT, MODE_FLOAT, MODE_VECTOR_FLOAT };
4633 51 : for (int i = 0; i < 4; i += 2)
4634 : /* The for loop above dictates that we only walk through scalar classes. */
4635 255 : FOR_EACH_MODE_IN_CLASS (mode_iter, classes[i])
4636 : {
4637 221 : scalar_mode mode = mode_iter.require ();
4638 221 : machine_mode vmode = targetm.vectorize.preferred_simd_mode (mode);
4639 221 : if (GET_MODE_CLASS (vmode) != classes[i + 1])
4640 323 : continue;
4641 : machine_mode alt_vmode;
4642 476 : for (unsigned int j = 0; j < modes.length (); ++j)
4643 527 : if (related_vector_mode (modes[j], mode).exists (&alt_vmode)
4644 578 : && known_ge (GET_MODE_SIZE (alt_vmode), GET_MODE_SIZE (vmode)))
4645 119 : vmode = alt_vmode;
4646 :
4647 119 : tree type = lang_hooks.types.type_for_mode (mode, 1);
4648 119 : if (type == NULL_TREE || TYPE_MODE (type) != mode)
4649 17 : continue;
4650 102 : type = build_vector_type_for_mode (type, vmode);
4651 102 : if (TYPE_MODE (type) != vmode)
4652 0 : continue;
4653 102 : if (TYPE_ALIGN_UNIT (type) > al)
4654 17 : al = TYPE_ALIGN_UNIT (type);
4655 : }
4656 17 : return build_int_cst (integer_type_node, al);
4657 17 : }
4658 :
4659 :
4660 : /* This structure is part of the interface between lower_rec_simd_input_clauses
4661 : and lower_rec_input_clauses. */
4662 :
4663 : class omplow_simd_context {
4664 : public:
4665 77138 : omplow_simd_context () { memset (this, 0, sizeof (*this)); }
4666 : tree idx;
4667 : tree lane;
4668 : tree lastlane;
4669 : vec<tree, va_heap> simt_eargs;
4670 : gimple_seq simt_dlist;
4671 : poly_uint64 max_vf;
4672 : bool is_simt;
4673 : };
4674 :
4675 : /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
4676 : privatization. */
4677 :
4678 : static bool
4679 10228 : lower_rec_simd_input_clauses (tree new_var, omp_context *ctx,
4680 : omplow_simd_context *sctx, tree &ivar,
4681 : tree &lvar, tree *rvar = NULL,
4682 : tree *rvar2 = NULL)
4683 : {
4684 10228 : if (known_eq (sctx->max_vf, 0U))
4685 : {
4686 4790 : sctx->max_vf = (sctx->is_simt ? omp_max_simt_vf ()
4687 4790 : : omp_max_vf (omp_maybe_offloaded_ctx (ctx)));
4688 4790 : if (maybe_gt (sctx->max_vf, 1U))
4689 : {
4690 3452 : tree c = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
4691 : OMP_CLAUSE_SAFELEN);
4692 3452 : if (c)
4693 : {
4694 89 : poly_uint64 safe_len;
4695 89 : if (!poly_int_tree_p (OMP_CLAUSE_SAFELEN_EXPR (c), &safe_len)
4696 89 : || maybe_lt (safe_len, 1U))
4697 6 : sctx->max_vf = 1;
4698 : else
4699 83 : sctx->max_vf = lower_bound (sctx->max_vf, safe_len);
4700 : }
4701 : }
4702 4790 : if (sctx->is_simt && !known_eq (sctx->max_vf, 1U))
4703 : {
4704 0 : for (tree c = gimple_omp_for_clauses (ctx->stmt); c;
4705 0 : c = OMP_CLAUSE_CHAIN (c))
4706 : {
4707 0 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
4708 0 : continue;
4709 :
4710 0 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
4711 : {
4712 : /* UDR reductions are not supported yet for SIMT, disable
4713 : SIMT. */
4714 0 : sctx->max_vf = 1;
4715 0 : break;
4716 : }
4717 :
4718 0 : if (truth_value_p (OMP_CLAUSE_REDUCTION_CODE (c))
4719 0 : && !INTEGRAL_TYPE_P (TREE_TYPE (new_var)))
4720 : {
4721 : /* Doing boolean operations on non-integral types is
4722 : for conformance only, it's not worth supporting this
4723 : for SIMT. */
4724 0 : sctx->max_vf = 1;
4725 0 : break;
4726 : }
4727 : }
4728 : }
4729 4790 : if (maybe_gt (sctx->max_vf, 1U))
4730 : {
4731 3433 : sctx->idx = create_tmp_var (unsigned_type_node);
4732 3433 : sctx->lane = create_tmp_var (unsigned_type_node);
4733 : }
4734 : }
4735 10228 : if (known_eq (sctx->max_vf, 1U))
4736 : return false;
4737 :
4738 7174 : if (sctx->is_simt)
4739 : {
4740 0 : if (is_gimple_reg (new_var))
4741 : {
4742 0 : ivar = lvar = new_var;
4743 0 : return true;
4744 : }
4745 0 : tree type = TREE_TYPE (new_var), ptype = build_pointer_type (type);
4746 0 : ivar = lvar = create_tmp_var (type);
4747 0 : TREE_ADDRESSABLE (ivar) = 1;
4748 0 : DECL_ATTRIBUTES (ivar) = tree_cons (get_identifier ("omp simt private"),
4749 0 : NULL, DECL_ATTRIBUTES (ivar));
4750 0 : sctx->simt_eargs.safe_push (build1 (ADDR_EXPR, ptype, ivar));
4751 0 : tree clobber = build_clobber (type);
4752 0 : gimple *g = gimple_build_assign (ivar, clobber);
4753 0 : gimple_seq_add_stmt (&sctx->simt_dlist, g);
4754 : }
4755 : else
4756 : {
4757 7174 : tree atype = build_array_type_nelts (TREE_TYPE (new_var), sctx->max_vf);
4758 7174 : tree avar = create_tmp_var_raw (atype);
4759 7174 : if (TREE_ADDRESSABLE (new_var))
4760 912 : TREE_ADDRESSABLE (avar) = 1;
4761 7174 : DECL_ATTRIBUTES (avar)
4762 7174 : = tree_cons (get_identifier ("omp simd array"), NULL,
4763 7174 : DECL_ATTRIBUTES (avar));
4764 7174 : gimple_add_tmp_var (avar);
4765 7174 : tree iavar = avar;
4766 7174 : if (rvar && !ctx->for_simd_scan_phase)
4767 : {
4768 : /* For inscan reductions, create another array temporary,
4769 : which will hold the reduced value. */
4770 232 : iavar = create_tmp_var_raw (atype);
4771 232 : if (TREE_ADDRESSABLE (new_var))
4772 33 : TREE_ADDRESSABLE (iavar) = 1;
4773 232 : DECL_ATTRIBUTES (iavar)
4774 232 : = tree_cons (get_identifier ("omp simd array"), NULL,
4775 : tree_cons (get_identifier ("omp simd inscan"), NULL,
4776 232 : DECL_ATTRIBUTES (iavar)));
4777 232 : gimple_add_tmp_var (iavar);
4778 232 : ctx->cb.decl_map->put (avar, iavar);
4779 232 : if (sctx->lastlane == NULL_TREE)
4780 184 : sctx->lastlane = create_tmp_var (unsigned_type_node);
4781 232 : *rvar = build4 (ARRAY_REF, TREE_TYPE (new_var), iavar,
4782 : sctx->lastlane, NULL_TREE, NULL_TREE);
4783 232 : TREE_THIS_NOTRAP (*rvar) = 1;
4784 :
4785 232 : if (ctx->scan_exclusive)
4786 : {
4787 : /* And for exclusive scan yet another one, which will
4788 : hold the value during the scan phase. */
4789 114 : tree savar = create_tmp_var_raw (atype);
4790 114 : if (TREE_ADDRESSABLE (new_var))
4791 17 : TREE_ADDRESSABLE (savar) = 1;
4792 114 : DECL_ATTRIBUTES (savar)
4793 114 : = tree_cons (get_identifier ("omp simd array"), NULL,
4794 : tree_cons (get_identifier ("omp simd inscan "
4795 : "exclusive"), NULL,
4796 114 : DECL_ATTRIBUTES (savar)));
4797 114 : gimple_add_tmp_var (savar);
4798 114 : ctx->cb.decl_map->put (iavar, savar);
4799 114 : *rvar2 = build4 (ARRAY_REF, TREE_TYPE (new_var), savar,
4800 : sctx->idx, NULL_TREE, NULL_TREE);
4801 114 : TREE_THIS_NOTRAP (*rvar2) = 1;
4802 : }
4803 : }
4804 7174 : ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), iavar, sctx->idx,
4805 : NULL_TREE, NULL_TREE);
4806 7174 : lvar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, sctx->lane,
4807 : NULL_TREE, NULL_TREE);
4808 7174 : TREE_THIS_NOTRAP (ivar) = 1;
4809 7174 : TREE_THIS_NOTRAP (lvar) = 1;
4810 : }
4811 7174 : if (DECL_P (new_var))
4812 : {
4813 7020 : SET_DECL_VALUE_EXPR (new_var, lvar);
4814 7020 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
4815 : }
4816 : return true;
4817 : }
4818 :
4819 : /* Helper function of lower_rec_input_clauses. For a reference
4820 : in simd reduction, add an underlying variable it will reference. */
4821 :
4822 : static void
4823 64 : handle_simd_reference (location_t loc, tree new_vard, gimple_seq *ilist)
4824 : {
4825 64 : tree z = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard)));
4826 64 : if (TREE_CONSTANT (z))
4827 : {
4828 64 : z = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard)),
4829 : get_name (new_vard));
4830 64 : gimple_add_tmp_var (z);
4831 64 : TREE_ADDRESSABLE (z) = 1;
4832 64 : z = build_fold_addr_expr_loc (loc, z);
4833 64 : gimplify_assign (new_vard, z, ilist);
4834 : }
4835 64 : }
4836 :
4837 : /* Helper function for lower_rec_input_clauses. Emit into ilist sequence
4838 : code to emit (type) (tskred_temp[idx]). */
4839 :
4840 : static tree
4841 1120 : task_reduction_read (gimple_seq *ilist, tree tskred_temp, tree type,
4842 : unsigned idx)
4843 : {
4844 1120 : unsigned HOST_WIDE_INT sz
4845 1120 : = tree_to_uhwi (TYPE_SIZE_UNIT (pointer_sized_int_node));
4846 1120 : tree r = build2 (MEM_REF, pointer_sized_int_node,
4847 1120 : tskred_temp, build_int_cst (TREE_TYPE (tskred_temp),
4848 1120 : idx * sz));
4849 1120 : tree v = create_tmp_var (pointer_sized_int_node);
4850 1120 : gimple *g = gimple_build_assign (v, r);
4851 1120 : gimple_seq_add_stmt (ilist, g);
4852 1120 : if (!useless_type_conversion_p (type, pointer_sized_int_node))
4853 : {
4854 872 : v = create_tmp_var (type);
4855 872 : g = gimple_build_assign (v, NOP_EXPR, gimple_assign_lhs (g));
4856 872 : gimple_seq_add_stmt (ilist, g);
4857 : }
4858 1120 : return v;
4859 : }
4860 :
4861 : /* Lower early initialization of privatized variable NEW_VAR
4862 : if it needs an allocator (has allocate clause). */
4863 :
4864 : static bool
4865 144318 : lower_private_allocate (tree var, tree new_var, tree &allocator,
4866 : tree &allocate_ptr, gimple_seq *ilist,
4867 : omp_context *ctx, bool is_ref, tree size)
4868 : {
4869 144318 : if (allocator)
4870 : return false;
4871 144266 : gcc_assert (allocate_ptr == NULL_TREE);
4872 144266 : if (ctx->allocate_map
4873 2798 : && (DECL_P (new_var) || (TYPE_P (new_var) && size)))
4874 2795 : if (tree *allocatorp = ctx->allocate_map->get (var))
4875 953 : allocator = *allocatorp;
4876 144266 : if (allocator == NULL_TREE)
4877 : return false;
4878 953 : if (!is_ref && omp_privatize_by_reference (var))
4879 : {
4880 0 : allocator = NULL_TREE;
4881 0 : return false;
4882 : }
4883 :
4884 953 : unsigned HOST_WIDE_INT ialign = 0;
4885 953 : if (TREE_CODE (allocator) == TREE_LIST)
4886 : {
4887 248 : ialign = tree_to_uhwi (TREE_VALUE (allocator));
4888 248 : allocator = TREE_PURPOSE (allocator);
4889 : }
4890 953 : if (TREE_CODE (allocator) != INTEGER_CST)
4891 295 : allocator = build_outer_var_ref (allocator, ctx, OMP_CLAUSE_ALLOCATE);
4892 953 : allocator = fold_convert (pointer_sized_int_node, allocator);
4893 953 : if (TREE_CODE (allocator) != INTEGER_CST)
4894 : {
4895 295 : tree var = create_tmp_var (TREE_TYPE (allocator));
4896 295 : gimplify_assign (var, allocator, ilist);
4897 295 : allocator = var;
4898 : }
4899 :
4900 953 : tree ptr_type, align, sz = size;
4901 953 : if (TYPE_P (new_var))
4902 : {
4903 225 : ptr_type = build_pointer_type (new_var);
4904 225 : ialign = MAX (ialign, TYPE_ALIGN_UNIT (new_var));
4905 : }
4906 728 : else if (is_ref)
4907 : {
4908 72 : ptr_type = build_pointer_type (TREE_TYPE (TREE_TYPE (new_var)));
4909 72 : ialign = MAX (ialign, TYPE_ALIGN_UNIT (TREE_TYPE (ptr_type)));
4910 : }
4911 : else
4912 : {
4913 656 : ptr_type = build_pointer_type (TREE_TYPE (new_var));
4914 656 : ialign = MAX (ialign, DECL_ALIGN_UNIT (new_var));
4915 656 : if (sz == NULL_TREE)
4916 558 : sz = fold_convert (size_type_node, DECL_SIZE_UNIT (new_var));
4917 : }
4918 953 : align = build_int_cst (size_type_node, ialign);
4919 953 : if (TREE_CODE (sz) != INTEGER_CST)
4920 : {
4921 42 : tree szvar = create_tmp_var (size_type_node);
4922 42 : gimplify_assign (szvar, sz, ilist);
4923 42 : sz = szvar;
4924 : }
4925 953 : allocate_ptr = create_tmp_var (ptr_type);
4926 953 : tree a = builtin_decl_explicit (BUILT_IN_GOMP_ALLOC);
4927 953 : gimple *g = gimple_build_call (a, 3, align, sz, allocator);
4928 953 : gimple_call_set_lhs (g, allocate_ptr);
4929 953 : gimple_seq_add_stmt (ilist, g);
4930 953 : if (!is_ref)
4931 : {
4932 656 : tree x = build_simple_mem_ref (allocate_ptr);
4933 656 : TREE_THIS_NOTRAP (x) = 1;
4934 656 : SET_DECL_VALUE_EXPR (new_var, x);
4935 656 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
4936 : }
4937 : return true;
4938 : }
4939 :
4940 : /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
4941 : from the receiver (aka child) side and initializers for REFERENCE_TYPE
4942 : private variables. Initialization statements go in ILIST, while calls
4943 : to destructors go in DLIST. */
4944 :
4945 : static void
4946 77138 : lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist,
4947 : omp_context *ctx, struct omp_for_data *fd)
4948 : {
4949 77138 : tree c, copyin_seq, x, ptr;
4950 77138 : bool copyin_by_ref = false;
4951 77138 : bool lastprivate_firstprivate = false;
4952 77138 : bool reduction_omp_orig_ref = false;
4953 77138 : int pass;
4954 77138 : bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
4955 77138 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD);
4956 77138 : omplow_simd_context sctx = omplow_simd_context ();
4957 77138 : tree simt_lane = NULL_TREE, simtrec = NULL_TREE;
4958 77138 : tree ivar = NULL_TREE, lvar = NULL_TREE, uid = NULL_TREE;
4959 77138 : gimple_seq llist[4] = { };
4960 77138 : tree nonconst_simd_if = NULL_TREE;
4961 :
4962 77138 : copyin_seq = NULL;
4963 77138 : sctx.is_simt = is_simd && omp_find_clause (clauses, OMP_CLAUSE__SIMT_);
4964 :
4965 : /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
4966 : with data sharing clauses referencing variable sized vars. That
4967 : is unnecessarily hard to support and very unlikely to result in
4968 : vectorized code anyway. */
4969 9355 : if (is_simd)
4970 61463 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
4971 52108 : switch (OMP_CLAUSE_CODE (c))
4972 : {
4973 7579 : case OMP_CLAUSE_LINEAR:
4974 7579 : if (OMP_CLAUSE_LINEAR_ARRAY (c))
4975 216 : sctx.max_vf = 1;
4976 : /* FALLTHRU */
4977 25155 : case OMP_CLAUSE_PRIVATE:
4978 25155 : case OMP_CLAUSE_FIRSTPRIVATE:
4979 25155 : case OMP_CLAUSE_LASTPRIVATE:
4980 25155 : if (is_variable_sized (OMP_CLAUSE_DECL (c)))
4981 0 : sctx.max_vf = 1;
4982 25155 : else if (omp_privatize_by_reference (OMP_CLAUSE_DECL (c)))
4983 : {
4984 279 : tree rtype = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
4985 279 : if (!TREE_CONSTANT (TYPE_SIZE_UNIT (rtype)))
4986 96 : sctx.max_vf = 1;
4987 : }
4988 : break;
4989 2578 : case OMP_CLAUSE_REDUCTION:
4990 2578 : case OMP_CLAUSE_IN_REDUCTION:
4991 2578 : if (TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF
4992 2578 : || is_variable_sized (OMP_CLAUSE_DECL (c)))
4993 180 : sctx.max_vf = 1;
4994 2398 : else if (omp_privatize_by_reference (OMP_CLAUSE_DECL (c)))
4995 : {
4996 144 : tree rtype = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
4997 144 : if (!TREE_CONSTANT (TYPE_SIZE_UNIT (rtype)))
4998 0 : sctx.max_vf = 1;
4999 : }
5000 : break;
5001 759 : case OMP_CLAUSE_IF:
5002 759 : if (integer_zerop (OMP_CLAUSE_IF_EXPR (c)))
5003 124 : sctx.max_vf = 1;
5004 635 : else if (TREE_CODE (OMP_CLAUSE_IF_EXPR (c)) != INTEGER_CST)
5005 625 : nonconst_simd_if = OMP_CLAUSE_IF_EXPR (c);
5006 : break;
5007 626 : case OMP_CLAUSE_SIMDLEN:
5008 626 : if (integer_onep (OMP_CLAUSE_SIMDLEN_EXPR (c)))
5009 110 : sctx.max_vf = 1;
5010 : break;
5011 184 : case OMP_CLAUSE__CONDTEMP_:
5012 : /* FIXME: lastprivate(conditional:) not handled for SIMT yet. */
5013 184 : if (sctx.is_simt)
5014 0 : sctx.max_vf = 1;
5015 : break;
5016 22806 : default:
5017 22806 : continue;
5018 22806 : }
5019 :
5020 : /* Add a placeholder for simduid. */
5021 77138 : if (sctx.is_simt && maybe_ne (sctx.max_vf, 1U))
5022 0 : sctx.simt_eargs.safe_push (NULL_TREE);
5023 :
5024 : unsigned task_reduction_cnt = 0;
5025 : unsigned task_reduction_cntorig = 0;
5026 : unsigned task_reduction_cnt_full = 0;
5027 : unsigned task_reduction_cntorig_full = 0;
5028 : unsigned task_reduction_other_cnt = 0;
5029 234469 : tree tskred_atype = NULL_TREE, tskred_avar = NULL_TREE;
5030 : tree tskred_base = NULL_TREE, tskred_temp = NULL_TREE;
5031 : /* Do all the fixed sized types in the first pass, and the variable sized
5032 : types in the second pass. This makes sure that the scalar arguments to
5033 : the variable sized types are processed before we use them in the
5034 : variable sized operations. For task reductions we use 4 passes, in the
5035 : first two we ignore them, in the third one gather arguments for
5036 : GOMP_task_reduction_remap call and in the last pass actually handle
5037 : the task reductions. */
5038 391800 : for (pass = 0; pass < ((task_reduction_cnt || task_reduction_other_cnt)
5039 234469 : ? 4 : 2); ++pass)
5040 : {
5041 157335 : if (pass == 2 && task_reduction_cnt)
5042 : {
5043 871 : tskred_atype
5044 871 : = build_array_type_nelts (ptr_type_node, task_reduction_cnt
5045 871 : + task_reduction_cntorig);
5046 871 : tskred_avar = create_tmp_var_raw (tskred_atype);
5047 871 : gimple_add_tmp_var (tskred_avar);
5048 871 : TREE_ADDRESSABLE (tskred_avar) = 1;
5049 871 : task_reduction_cnt_full = task_reduction_cnt;
5050 871 : task_reduction_cntorig_full = task_reduction_cntorig;
5051 : }
5052 156464 : else if (pass == 3 && task_reduction_cnt)
5053 : {
5054 871 : x = builtin_decl_explicit (BUILT_IN_GOMP_TASK_REDUCTION_REMAP);
5055 871 : gimple *g
5056 871 : = gimple_build_call (x, 3, size_int (task_reduction_cnt),
5057 871 : size_int (task_reduction_cntorig),
5058 : build_fold_addr_expr (tskred_avar));
5059 871 : gimple_seq_add_stmt (ilist, g);
5060 : }
5061 157335 : if (pass == 3 && task_reduction_other_cnt)
5062 : {
5063 : /* For reduction clauses, build
5064 : tskred_base = (void *) tskred_temp[2]
5065 : + omp_get_thread_num () * tskred_temp[1]
5066 : or if tskred_temp[1] is known to be constant, that constant
5067 : directly. This is the start of the private reduction copy block
5068 : for the current thread. */
5069 838 : tree v = create_tmp_var (integer_type_node);
5070 838 : x = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
5071 838 : gimple *g = gimple_build_call (x, 0);
5072 838 : gimple_call_set_lhs (g, v);
5073 838 : gimple_seq_add_stmt (ilist, g);
5074 838 : c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
5075 838 : tskred_temp = OMP_CLAUSE_DECL (c);
5076 838 : if (is_taskreg_ctx (ctx))
5077 557 : tskred_temp = lookup_decl (tskred_temp, ctx);
5078 838 : tree v2 = create_tmp_var (sizetype);
5079 838 : g = gimple_build_assign (v2, NOP_EXPR, v);
5080 838 : gimple_seq_add_stmt (ilist, g);
5081 838 : if (ctx->task_reductions[0])
5082 821 : v = fold_convert (sizetype, ctx->task_reductions[0]);
5083 : else
5084 17 : v = task_reduction_read (ilist, tskred_temp, sizetype, 1);
5085 838 : tree v3 = create_tmp_var (sizetype);
5086 838 : g = gimple_build_assign (v3, MULT_EXPR, v2, v);
5087 838 : gimple_seq_add_stmt (ilist, g);
5088 838 : v = task_reduction_read (ilist, tskred_temp, ptr_type_node, 2);
5089 838 : tskred_base = create_tmp_var (ptr_type_node);
5090 838 : g = gimple_build_assign (tskred_base, POINTER_PLUS_EXPR, v, v3);
5091 838 : gimple_seq_add_stmt (ilist, g);
5092 : }
5093 157335 : task_reduction_cnt = 0;
5094 157335 : task_reduction_cntorig = 0;
5095 157335 : task_reduction_other_cnt = 0;
5096 760130 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
5097 : {
5098 602799 : enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c);
5099 602799 : tree var, new_var;
5100 602799 : bool by_ref;
5101 602799 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
5102 602799 : bool task_reduction_p = false;
5103 602799 : bool task_reduction_needs_orig_p = false;
5104 602799 : tree cond = NULL_TREE;
5105 602799 : tree allocator, allocate_ptr;
5106 :
5107 602799 : switch (c_kind)
5108 : {
5109 135788 : case OMP_CLAUSE_PRIVATE:
5110 135788 : if (OMP_CLAUSE_PRIVATE_DEBUG (c))
5111 434804 : continue;
5112 : break;
5113 59602 : case OMP_CLAUSE_SHARED:
5114 : /* Ignore shared directives in teams construct inside
5115 : of target construct. */
5116 59602 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
5117 59602 : && !is_host_teams_ctx (ctx))
5118 23496 : continue;
5119 36106 : if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL)
5120 : {
5121 12080 : gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c)
5122 : || is_global_var (OMP_CLAUSE_DECL (c)));
5123 12080 : continue;
5124 : }
5125 : case OMP_CLAUSE_FIRSTPRIVATE:
5126 : case OMP_CLAUSE_COPYIN:
5127 : break;
5128 15564 : case OMP_CLAUSE_LINEAR:
5129 15564 : if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c)
5130 15564 : && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
5131 : lastprivate_firstprivate = true;
5132 : break;
5133 37216 : case OMP_CLAUSE_REDUCTION:
5134 37216 : case OMP_CLAUSE_IN_REDUCTION:
5135 37216 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
5136 28968 : || is_task_ctx (ctx)
5137 63844 : || OMP_CLAUSE_REDUCTION_TASK (c))
5138 : {
5139 12780 : task_reduction_p = true;
5140 12780 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
5141 : {
5142 4532 : task_reduction_other_cnt++;
5143 4532 : if (pass == 2)
5144 1133 : continue;
5145 : }
5146 : else
5147 8248 : task_reduction_cnt++;
5148 11647 : if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5149 : {
5150 696 : var = OMP_CLAUSE_DECL (c);
5151 : /* If var is a global variable that isn't privatized
5152 : in outer contexts, we don't need to look up the
5153 : original address, it is always the address of the
5154 : global variable itself. */
5155 696 : if (!DECL_P (var)
5156 272 : || omp_privatize_by_reference (var)
5157 923 : || !is_global_var
5158 227 : (maybe_lookup_decl_in_outer_ctx (var, ctx)))
5159 : {
5160 594 : task_reduction_needs_orig_p = true;
5161 594 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5162 492 : task_reduction_cntorig++;
5163 : }
5164 : }
5165 : }
5166 24436 : else if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5167 356681 : reduction_omp_orig_ref = true;
5168 : break;
5169 3352 : case OMP_CLAUSE__REDUCTEMP_:
5170 3352 : if (!is_taskreg_ctx (ctx))
5171 1124 : continue;
5172 : /* FALLTHRU */
5173 111456 : case OMP_CLAUSE__LOOPTEMP_:
5174 : /* Handle _looptemp_/_reductemp_ clauses only on
5175 : parallel/task. */
5176 111456 : if (fd)
5177 69210 : continue;
5178 : break;
5179 43558 : case OMP_CLAUSE_LASTPRIVATE:
5180 43558 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
5181 : {
5182 2066 : lastprivate_firstprivate = true;
5183 2066 : if (pass != 0 || is_taskloop_ctx (ctx))
5184 1347 : continue;
5185 : }
5186 : /* Even without corresponding firstprivate, if
5187 : decl is Fortran allocatable, it needs outer var
5188 : reference. */
5189 41492 : else if (pass == 0
5190 62202 : && lang_hooks.decls.omp_private_outer_ref
5191 20710 : (OMP_CLAUSE_DECL (c)))
5192 : lastprivate_firstprivate = true;
5193 : break;
5194 280 : case OMP_CLAUSE_ALIGNED:
5195 280 : if (pass != 1)
5196 140 : continue;
5197 140 : var = OMP_CLAUSE_DECL (c);
5198 140 : if (TREE_CODE (TREE_TYPE (var)) == POINTER_TYPE
5199 140 : && !is_global_var (var))
5200 : {
5201 76 : new_var = maybe_lookup_decl (var, ctx);
5202 76 : if (new_var == NULL_TREE)
5203 6 : new_var = maybe_lookup_decl_in_outer_ctx (var, ctx);
5204 76 : x = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
5205 76 : tree alarg = omp_clause_aligned_alignment (c);
5206 76 : alarg = fold_convert_loc (clause_loc, size_type_node, alarg);
5207 76 : x = build_call_expr_loc (clause_loc, x, 2, new_var, alarg);
5208 76 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
5209 76 : x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
5210 76 : gimplify_and_add (x, ilist);
5211 : }
5212 64 : else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
5213 64 : && is_global_var (var))
5214 : {
5215 64 : tree ptype = build_pointer_type (TREE_TYPE (var)), t, t2;
5216 64 : new_var = lookup_decl (var, ctx);
5217 64 : t = maybe_lookup_decl_in_outer_ctx (var, ctx);
5218 64 : t = build_fold_addr_expr_loc (clause_loc, t);
5219 64 : t2 = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
5220 64 : tree alarg = omp_clause_aligned_alignment (c);
5221 64 : alarg = fold_convert_loc (clause_loc, size_type_node, alarg);
5222 64 : t = build_call_expr_loc (clause_loc, t2, 2, t, alarg);
5223 64 : t = fold_convert_loc (clause_loc, ptype, t);
5224 64 : x = create_tmp_var (ptype);
5225 64 : t = build2 (MODIFY_EXPR, ptype, x, t);
5226 64 : gimplify_and_add (t, ilist);
5227 64 : t = build_simple_mem_ref_loc (clause_loc, x);
5228 64 : SET_DECL_VALUE_EXPR (new_var, t);
5229 64 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5230 : }
5231 140 : continue;
5232 1478 : case OMP_CLAUSE__CONDTEMP_:
5233 1478 : if (is_parallel_ctx (ctx)
5234 1478 : || (is_simd && !OMP_CLAUSE__CONDTEMP__ITER (c)))
5235 : break;
5236 1070 : continue;
5237 135456 : default:
5238 135456 : continue;
5239 136666 : }
5240 :
5241 356681 : if (task_reduction_p != (pass >= 2))
5242 15616 : continue;
5243 :
5244 341065 : allocator = NULL_TREE;
5245 341065 : allocate_ptr = NULL_TREE;
5246 341065 : new_var = var = OMP_CLAUSE_DECL (c);
5247 341065 : if ((c_kind == OMP_CLAUSE_REDUCTION
5248 341065 : || c_kind == OMP_CLAUSE_IN_REDUCTION)
5249 29639 : && TREE_CODE (var) == MEM_REF)
5250 : {
5251 4356 : var = TREE_OPERAND (var, 0);
5252 4356 : if (TREE_CODE (var) == POINTER_PLUS_EXPR)
5253 773 : var = TREE_OPERAND (var, 0);
5254 4356 : if (TREE_CODE (var) == INDIRECT_REF
5255 4168 : || TREE_CODE (var) == ADDR_EXPR)
5256 2506 : var = TREE_OPERAND (var, 0);
5257 4356 : if (is_variable_sized (var))
5258 : {
5259 179 : gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
5260 179 : var = DECL_VALUE_EXPR (var);
5261 179 : gcc_assert (TREE_CODE (var) == INDIRECT_REF);
5262 179 : var = TREE_OPERAND (var, 0);
5263 179 : gcc_assert (DECL_P (var));
5264 : }
5265 4356 : new_var = var;
5266 : }
5267 29639 : if (c_kind == OMP_CLAUSE_IN_REDUCTION && is_omp_target (ctx->stmt))
5268 : {
5269 844 : splay_tree_key key = (splay_tree_key) &DECL_CONTEXT (var);
5270 844 : new_var = (tree) splay_tree_lookup (ctx->field_map, key)->value;
5271 : }
5272 340221 : else if (c_kind != OMP_CLAUSE_COPYIN)
5273 339193 : new_var = lookup_decl (var, ctx);
5274 :
5275 341065 : if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN)
5276 : {
5277 24378 : if (pass != 0)
5278 12188 : continue;
5279 : }
5280 : /* C/C++ array section reductions. */
5281 316687 : else if ((c_kind == OMP_CLAUSE_REDUCTION
5282 : || c_kind == OMP_CLAUSE_IN_REDUCTION)
5283 316687 : && var != OMP_CLAUSE_DECL (c))
5284 : {
5285 4356 : if (pass == 0)
5286 873 : continue;
5287 :
5288 3483 : tree bias = TREE_OPERAND (OMP_CLAUSE_DECL (c), 1);
5289 3483 : tree orig_var = TREE_OPERAND (OMP_CLAUSE_DECL (c), 0);
5290 :
5291 3483 : if (TREE_CODE (orig_var) == POINTER_PLUS_EXPR)
5292 : {
5293 723 : tree b = TREE_OPERAND (orig_var, 1);
5294 723 : if (is_omp_target (ctx->stmt))
5295 : b = NULL_TREE;
5296 : else
5297 723 : b = maybe_lookup_decl (b, ctx);
5298 723 : if (b == NULL)
5299 : {
5300 17 : b = TREE_OPERAND (orig_var, 1);
5301 17 : b = maybe_lookup_decl_in_outer_ctx (b, ctx);
5302 : }
5303 723 : if (integer_zerop (bias))
5304 : bias = b;
5305 : else
5306 : {
5307 0 : bias = fold_convert_loc (clause_loc,
5308 0 : TREE_TYPE (b), bias);
5309 0 : bias = fold_build2_loc (clause_loc, PLUS_EXPR,
5310 0 : TREE_TYPE (b), b, bias);
5311 : }
5312 723 : orig_var = TREE_OPERAND (orig_var, 0);
5313 : }
5314 3483 : if (pass == 2)
5315 : {
5316 1149 : tree out = maybe_lookup_decl_in_outer_ctx (var, ctx);
5317 1149 : if (is_global_var (out)
5318 241 : && TREE_CODE (TREE_TYPE (out)) != POINTER_TYPE
5319 1336 : && (TREE_CODE (TREE_TYPE (out)) != REFERENCE_TYPE
5320 0 : || (TREE_CODE (TREE_TYPE (TREE_TYPE (out)))
5321 : != POINTER_TYPE)))
5322 : x = var;
5323 962 : else if (is_omp_target (ctx->stmt))
5324 : x = out;
5325 : else
5326 : {
5327 866 : bool by_ref = use_pointer_for_field (var, NULL);
5328 866 : x = build_receiver_ref (var, by_ref, ctx);
5329 866 : if (TREE_CODE (TREE_TYPE (var)) == REFERENCE_TYPE
5330 866 : && (TREE_CODE (TREE_TYPE (TREE_TYPE (var)))
5331 : == POINTER_TYPE))
5332 32 : x = build_fold_addr_expr (x);
5333 : }
5334 1149 : if (TREE_CODE (orig_var) == INDIRECT_REF)
5335 40 : x = build_simple_mem_ref (x);
5336 1109 : else if (TREE_CODE (orig_var) == ADDR_EXPR)
5337 : {
5338 646 : if (var == TREE_OPERAND (orig_var, 0))
5339 583 : x = build_fold_addr_expr (x);
5340 : }
5341 1149 : bias = fold_convert (sizetype, bias);
5342 1149 : x = fold_convert (ptr_type_node, x);
5343 1149 : x = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
5344 1149 : TREE_TYPE (x), x, bias);
5345 1149 : unsigned cnt = task_reduction_cnt - 1;
5346 1149 : if (!task_reduction_needs_orig_p)
5347 1061 : cnt += (task_reduction_cntorig_full
5348 1061 : - task_reduction_cntorig);
5349 : else
5350 88 : cnt = task_reduction_cntorig - 1;
5351 1149 : tree r = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5352 1149 : size_int (cnt), NULL_TREE, NULL_TREE);
5353 1149 : gimplify_assign (r, x, ilist);
5354 1149 : continue;
5355 1149 : }
5356 :
5357 2334 : if (TREE_CODE (orig_var) == INDIRECT_REF
5358 2240 : || TREE_CODE (orig_var) == ADDR_EXPR)
5359 1362 : orig_var = TREE_OPERAND (orig_var, 0);
5360 2334 : tree d = OMP_CLAUSE_DECL (c);
5361 2334 : tree type = TREE_TYPE (d);
5362 2334 : gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
5363 2334 : tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
5364 2334 : tree sz = v;
5365 2334 : const char *name = get_name (orig_var);
5366 2334 : if (pass != 3 && !TREE_CONSTANT (v))
5367 : {
5368 93 : tree t;
5369 93 : if (is_omp_target (ctx->stmt))
5370 : t = NULL_TREE;
5371 : else
5372 93 : t = maybe_lookup_decl (v, ctx);
5373 93 : if (t)
5374 88 : v = t;
5375 : else
5376 5 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
5377 93 : gimplify_expr (&v, ilist, NULL, is_gimple_val, fb_rvalue);
5378 186 : t = fold_build2_loc (clause_loc, PLUS_EXPR,
5379 93 : TREE_TYPE (v), v,
5380 93 : build_int_cst (TREE_TYPE (v), 1));
5381 93 : sz = fold_build2_loc (clause_loc, MULT_EXPR,
5382 93 : TREE_TYPE (v), t,
5383 93 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5384 : }
5385 2334 : if (pass == 3)
5386 : {
5387 1461 : tree xv = create_tmp_var (ptr_type_node);
5388 1461 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5389 : {
5390 1149 : unsigned cnt = task_reduction_cnt - 1;
5391 1149 : if (!task_reduction_needs_orig_p)
5392 1061 : cnt += (task_reduction_cntorig_full
5393 1061 : - task_reduction_cntorig);
5394 : else
5395 88 : cnt = task_reduction_cntorig - 1;
5396 1149 : x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5397 1149 : size_int (cnt), NULL_TREE, NULL_TREE);
5398 :
5399 1149 : gimple *g = gimple_build_assign (xv, x);
5400 1149 : gimple_seq_add_stmt (ilist, g);
5401 : }
5402 : else
5403 : {
5404 312 : unsigned int idx = *ctx->task_reduction_map->get (c);
5405 312 : tree off;
5406 312 : if (ctx->task_reductions[1 + idx])
5407 81 : off = fold_convert (sizetype,
5408 : ctx->task_reductions[1 + idx]);
5409 : else
5410 231 : off = task_reduction_read (ilist, tskred_temp, sizetype,
5411 231 : 7 + 3 * idx + 1);
5412 312 : gimple *g = gimple_build_assign (xv, POINTER_PLUS_EXPR,
5413 : tskred_base, off);
5414 312 : gimple_seq_add_stmt (ilist, g);
5415 : }
5416 1461 : x = fold_convert (build_pointer_type (boolean_type_node),
5417 : xv);
5418 1461 : if (TREE_CONSTANT (v))
5419 1065 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (x), x,
5420 : TYPE_SIZE_UNIT (type));
5421 : else
5422 : {
5423 396 : tree t;
5424 396 : if (is_omp_target (ctx->stmt))
5425 : t = NULL_TREE;
5426 : else
5427 336 : t = maybe_lookup_decl (v, ctx);
5428 336 : if (t)
5429 316 : v = t;
5430 : else
5431 80 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
5432 396 : gimplify_expr (&v, ilist, NULL, is_gimple_val,
5433 : fb_rvalue);
5434 792 : t = fold_build2_loc (clause_loc, PLUS_EXPR,
5435 396 : TREE_TYPE (v), v,
5436 396 : build_int_cst (TREE_TYPE (v), 1));
5437 396 : t = fold_build2_loc (clause_loc, MULT_EXPR,
5438 396 : TREE_TYPE (v), t,
5439 396 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5440 396 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (x), x, t);
5441 : }
5442 1461 : cond = create_tmp_var (TREE_TYPE (x));
5443 1461 : gimplify_assign (cond, x, ilist);
5444 1461 : x = xv;
5445 : }
5446 873 : else if (lower_private_allocate (var, type, allocator,
5447 : allocate_ptr, ilist, ctx,
5448 : true,
5449 873 : TREE_CONSTANT (v)
5450 780 : ? TYPE_SIZE_UNIT (type)
5451 : : sz))
5452 225 : x = allocate_ptr;
5453 648 : else if (TREE_CONSTANT (v))
5454 : {
5455 560 : x = create_tmp_var_raw (type, name);
5456 560 : gimple_add_tmp_var (x);
5457 560 : TREE_ADDRESSABLE (x) = 1;
5458 560 : x = build_fold_addr_expr_loc (clause_loc, x);
5459 : }
5460 : else
5461 : {
5462 88 : tree atmp
5463 88 : = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
5464 88 : tree al = size_int (TYPE_ALIGN (TREE_TYPE (type)));
5465 88 : x = build_call_expr_loc (clause_loc, atmp, 2, sz, al);
5466 : }
5467 :
5468 2334 : tree ptype = build_pointer_type (TREE_TYPE (type));
5469 2334 : x = fold_convert_loc (clause_loc, ptype, x);
5470 2334 : tree y = create_tmp_var (ptype, name);
5471 2334 : gimplify_assign (y, x, ilist);
5472 2334 : x = y;
5473 2334 : tree yb = y;
5474 :
5475 2334 : if (!integer_zerop (bias))
5476 : {
5477 1512 : bias = fold_convert_loc (clause_loc, pointer_sized_int_node,
5478 : bias);
5479 1512 : yb = fold_convert_loc (clause_loc, pointer_sized_int_node,
5480 : x);
5481 1512 : yb = fold_build2_loc (clause_loc, MINUS_EXPR,
5482 : pointer_sized_int_node, yb, bias);
5483 1512 : x = fold_convert_loc (clause_loc, TREE_TYPE (x), yb);
5484 1512 : yb = create_tmp_var (ptype, name);
5485 1512 : gimplify_assign (yb, x, ilist);
5486 1512 : x = yb;
5487 : }
5488 :
5489 2334 : d = TREE_OPERAND (d, 0);
5490 2334 : if (TREE_CODE (d) == POINTER_PLUS_EXPR)
5491 420 : d = TREE_OPERAND (d, 0);
5492 2334 : if (TREE_CODE (d) == ADDR_EXPR)
5493 : {
5494 1268 : if (orig_var != var)
5495 : {
5496 91 : gcc_assert (is_variable_sized (orig_var));
5497 91 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var),
5498 : x);
5499 91 : gimplify_assign (new_var, x, ilist);
5500 91 : tree new_orig_var = lookup_decl (orig_var, ctx);
5501 91 : tree t = build_fold_indirect_ref (new_var);
5502 91 : DECL_IGNORED_P (new_var) = 0;
5503 91 : TREE_THIS_NOTRAP (t) = 1;
5504 91 : SET_DECL_VALUE_EXPR (new_orig_var, t);
5505 91 : DECL_HAS_VALUE_EXPR_P (new_orig_var) = 1;
5506 : }
5507 : else
5508 : {
5509 1177 : x = build2 (MEM_REF, TREE_TYPE (new_var), x,
5510 : build_int_cst (ptype, 0));
5511 1177 : SET_DECL_VALUE_EXPR (new_var, x);
5512 1177 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5513 : }
5514 : }
5515 : else
5516 : {
5517 1066 : gcc_assert (orig_var == var);
5518 1066 : if (TREE_CODE (d) == INDIRECT_REF)
5519 : {
5520 94 : x = create_tmp_var (ptype, name);
5521 94 : TREE_ADDRESSABLE (x) = 1;
5522 94 : gimplify_assign (x, yb, ilist);
5523 94 : x = build_fold_addr_expr_loc (clause_loc, x);
5524 : }
5525 1066 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
5526 1066 : gimplify_assign (new_var, x, ilist);
5527 : }
5528 : /* GOMP_taskgroup_reduction_register memsets the whole
5529 : array to zero. If the initializer is zero, we don't
5530 : need to initialize it again, just mark it as ever
5531 : used unconditionally, i.e. cond = true. */
5532 2334 : if (cond
5533 1461 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) == NULL_TREE
5534 3267 : && initializer_zerop (omp_reduction_init (c,
5535 933 : TREE_TYPE (type))))
5536 : {
5537 624 : gimple *g = gimple_build_assign (build_simple_mem_ref (cond),
5538 : boolean_true_node);
5539 624 : gimple_seq_add_stmt (ilist, g);
5540 624 : continue;
5541 624 : }
5542 1710 : tree end = create_artificial_label (UNKNOWN_LOCATION);
5543 1710 : if (cond)
5544 : {
5545 837 : gimple *g;
5546 837 : if (!is_parallel_ctx (ctx))
5547 : {
5548 786 : tree condv = create_tmp_var (boolean_type_node);
5549 786 : g = gimple_build_assign (condv,
5550 : build_simple_mem_ref (cond));
5551 786 : gimple_seq_add_stmt (ilist, g);
5552 786 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
5553 786 : g = gimple_build_cond (NE_EXPR, condv,
5554 : boolean_false_node, end, lab1);
5555 786 : gimple_seq_add_stmt (ilist, g);
5556 786 : gimple_seq_add_stmt (ilist, gimple_build_label (lab1));
5557 : }
5558 837 : g = gimple_build_assign (build_simple_mem_ref (cond),
5559 : boolean_true_node);
5560 837 : gimple_seq_add_stmt (ilist, g);
5561 : }
5562 :
5563 1710 : tree y1 = create_tmp_var (ptype);
5564 1710 : gimplify_assign (y1, y, ilist);
5565 1710 : tree i2 = NULL_TREE, y2 = NULL_TREE;
5566 1710 : tree body2 = NULL_TREE, end2 = NULL_TREE;
5567 1710 : tree y3 = NULL_TREE, y4 = NULL_TREE;
5568 1710 : if (task_reduction_needs_orig_p)
5569 : {
5570 112 : y3 = create_tmp_var (ptype);
5571 112 : tree ref;
5572 112 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5573 88 : ref = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5574 88 : size_int (task_reduction_cnt_full
5575 : + task_reduction_cntorig - 1),
5576 : NULL_TREE, NULL_TREE);
5577 : else
5578 : {
5579 24 : unsigned int idx = *ctx->task_reduction_map->get (c);
5580 24 : ref = task_reduction_read (ilist, tskred_temp, ptype,
5581 24 : 7 + 3 * idx);
5582 : }
5583 112 : gimplify_assign (y3, ref, ilist);
5584 : }
5585 1598 : else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) || is_simd)
5586 : {
5587 668 : if (pass != 3)
5588 : {
5589 252 : y2 = create_tmp_var (ptype);
5590 252 : gimplify_assign (y2, y, ilist);
5591 : }
5592 668 : if (is_simd || OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5593 : {
5594 188 : tree ref = build_outer_var_ref (var, ctx);
5595 : /* For ref build_outer_var_ref already performs this. */
5596 188 : if (TREE_CODE (d) == INDIRECT_REF)
5597 0 : gcc_assert (omp_privatize_by_reference (var));
5598 188 : else if (TREE_CODE (d) == ADDR_EXPR)
5599 96 : ref = build_fold_addr_expr (ref);
5600 92 : else if (omp_privatize_by_reference (var))
5601 8 : ref = build_fold_addr_expr (ref);
5602 188 : ref = fold_convert_loc (clause_loc, ptype, ref);
5603 188 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)
5604 188 : && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5605 : {
5606 8 : y3 = create_tmp_var (ptype);
5607 8 : gimplify_assign (y3, unshare_expr (ref), ilist);
5608 : }
5609 188 : if (is_simd)
5610 : {
5611 180 : y4 = create_tmp_var (ptype);
5612 180 : gimplify_assign (y4, ref, dlist);
5613 : }
5614 : }
5615 : }
5616 1710 : tree i = create_tmp_var (TREE_TYPE (v));
5617 1710 : gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), ilist);
5618 1710 : tree body = create_artificial_label (UNKNOWN_LOCATION);
5619 1710 : gimple_seq_add_stmt (ilist, gimple_build_label (body));
5620 1710 : if (y2)
5621 : {
5622 252 : i2 = create_tmp_var (TREE_TYPE (v));
5623 252 : gimplify_assign (i2, build_int_cst (TREE_TYPE (v), 0), dlist);
5624 252 : body2 = create_artificial_label (UNKNOWN_LOCATION);
5625 252 : end2 = create_artificial_label (UNKNOWN_LOCATION);
5626 252 : gimple_seq_add_stmt (dlist, gimple_build_label (body2));
5627 : }
5628 1710 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
5629 : {
5630 600 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
5631 600 : tree decl_placeholder
5632 600 : = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
5633 600 : SET_DECL_VALUE_EXPR (decl_placeholder,
5634 : build_simple_mem_ref (y1));
5635 600 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
5636 600 : SET_DECL_VALUE_EXPR (placeholder,
5637 : y3 ? build_simple_mem_ref (y3)
5638 : : error_mark_node);
5639 600 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
5640 600 : x = lang_hooks.decls.omp_clause_default_ctor
5641 720 : (c, build_simple_mem_ref (y1),
5642 120 : y3 ? build_simple_mem_ref (y3) : NULL_TREE);
5643 600 : if (x)
5644 152 : gimplify_and_add (x, ilist);
5645 600 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
5646 : {
5647 576 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
5648 576 : lower_omp (&tseq, ctx);
5649 576 : gimple_seq_add_seq (ilist, tseq);
5650 : }
5651 600 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
5652 600 : if (is_simd)
5653 : {
5654 0 : SET_DECL_VALUE_EXPR (decl_placeholder,
5655 : build_simple_mem_ref (y2));
5656 0 : SET_DECL_VALUE_EXPR (placeholder,
5657 : build_simple_mem_ref (y4));
5658 0 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
5659 0 : lower_omp (&tseq, ctx);
5660 0 : gimple_seq_add_seq (dlist, tseq);
5661 0 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
5662 : }
5663 600 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
5664 600 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 0;
5665 600 : if (y2)
5666 : {
5667 72 : x = lang_hooks.decls.omp_clause_dtor
5668 72 : (c, build_simple_mem_ref (y2));
5669 72 : if (x)
5670 40 : gimplify_and_add (x, dlist);
5671 : }
5672 : }
5673 : else
5674 : {
5675 1110 : x = omp_reduction_init (c, TREE_TYPE (type));
5676 1110 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
5677 :
5678 : /* reduction(-:var) sums up the partial results, so it
5679 : acts identically to reduction(+:var). */
5680 1110 : if (code == MINUS_EXPR)
5681 0 : code = PLUS_EXPR;
5682 :
5683 1110 : gimplify_assign (build_simple_mem_ref (y1), x, ilist);
5684 1110 : if (is_simd)
5685 : {
5686 180 : x = build2 (code, TREE_TYPE (type),
5687 : build_simple_mem_ref (y4),
5688 : build_simple_mem_ref (y2));
5689 180 : gimplify_assign (build_simple_mem_ref (y4), x, dlist);
5690 : }
5691 : }
5692 1710 : gimple *g
5693 1710 : = gimple_build_assign (y1, POINTER_PLUS_EXPR, y1,
5694 1710 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5695 1710 : gimple_seq_add_stmt (ilist, g);
5696 1710 : if (y3)
5697 : {
5698 120 : g = gimple_build_assign (y3, POINTER_PLUS_EXPR, y3,
5699 120 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5700 120 : gimple_seq_add_stmt (ilist, g);
5701 : }
5702 1710 : g = gimple_build_assign (i, PLUS_EXPR, i,
5703 1710 : build_int_cst (TREE_TYPE (i), 1));
5704 1710 : gimple_seq_add_stmt (ilist, g);
5705 1710 : g = gimple_build_cond (LE_EXPR, i, v, body, end);
5706 1710 : gimple_seq_add_stmt (ilist, g);
5707 1710 : gimple_seq_add_stmt (ilist, gimple_build_label (end));
5708 1710 : if (y2)
5709 : {
5710 252 : g = gimple_build_assign (y2, POINTER_PLUS_EXPR, y2,
5711 252 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5712 252 : gimple_seq_add_stmt (dlist, g);
5713 252 : if (y4)
5714 : {
5715 180 : g = gimple_build_assign
5716 180 : (y4, POINTER_PLUS_EXPR, y4,
5717 180 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5718 180 : gimple_seq_add_stmt (dlist, g);
5719 : }
5720 252 : g = gimple_build_assign (i2, PLUS_EXPR, i2,
5721 252 : build_int_cst (TREE_TYPE (i2), 1));
5722 252 : gimple_seq_add_stmt (dlist, g);
5723 252 : g = gimple_build_cond (LE_EXPR, i2, v, body2, end2);
5724 252 : gimple_seq_add_stmt (dlist, g);
5725 252 : gimple_seq_add_stmt (dlist, gimple_build_label (end2));
5726 : }
5727 1710 : if (allocator)
5728 : {
5729 225 : tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
5730 225 : g = gimple_build_call (f, 2, allocate_ptr, allocator);
5731 225 : gimple_seq_add_stmt (dlist, g);
5732 : }
5733 1710 : continue;
5734 1710 : }
5735 312331 : else if (pass == 2)
5736 : {
5737 913 : tree out = maybe_lookup_decl_in_outer_ctx (var, ctx);
5738 913 : if (is_global_var (out))
5739 : x = var;
5740 361 : else if (is_omp_target (ctx->stmt))
5741 : x = out;
5742 : else
5743 : {
5744 301 : bool by_ref = use_pointer_for_field (var, ctx);
5745 301 : x = build_receiver_ref (var, by_ref, ctx);
5746 : }
5747 913 : if (!omp_privatize_by_reference (var))
5748 823 : x = build_fold_addr_expr (x);
5749 913 : x = fold_convert (ptr_type_node, x);
5750 913 : unsigned cnt = task_reduction_cnt - 1;
5751 913 : if (!task_reduction_needs_orig_p)
5752 878 : cnt += task_reduction_cntorig_full - task_reduction_cntorig;
5753 : else
5754 35 : cnt = task_reduction_cntorig - 1;
5755 913 : tree r = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5756 913 : size_int (cnt), NULL_TREE, NULL_TREE);
5757 913 : gimplify_assign (r, x, ilist);
5758 913 : continue;
5759 913 : }
5760 311418 : else if (pass == 3)
5761 : {
5762 1734 : tree type = TREE_TYPE (new_var);
5763 1734 : if (!omp_privatize_by_reference (var))
5764 1628 : type = build_pointer_type (type);
5765 1734 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5766 : {
5767 913 : unsigned cnt = task_reduction_cnt - 1;
5768 913 : if (!task_reduction_needs_orig_p)
5769 878 : cnt += (task_reduction_cntorig_full
5770 878 : - task_reduction_cntorig);
5771 : else
5772 35 : cnt = task_reduction_cntorig - 1;
5773 913 : x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5774 913 : size_int (cnt), NULL_TREE, NULL_TREE);
5775 : }
5776 : else
5777 : {
5778 821 : unsigned int idx = *ctx->task_reduction_map->get (c);
5779 821 : tree off;
5780 821 : if (ctx->task_reductions[1 + idx])
5781 821 : off = fold_convert (sizetype,
5782 : ctx->task_reductions[1 + idx]);
5783 : else
5784 0 : off = task_reduction_read (ilist, tskred_temp, sizetype,
5785 0 : 7 + 3 * idx + 1);
5786 821 : x = fold_build2 (POINTER_PLUS_EXPR, ptr_type_node,
5787 : tskred_base, off);
5788 : }
5789 1734 : x = fold_convert (type, x);
5790 1734 : tree t;
5791 1734 : if (omp_privatize_by_reference (var))
5792 : {
5793 106 : gimplify_assign (new_var, x, ilist);
5794 106 : t = new_var;
5795 106 : new_var = build_simple_mem_ref (new_var);
5796 : }
5797 : else
5798 : {
5799 1628 : t = create_tmp_var (type);
5800 1628 : gimplify_assign (t, x, ilist);
5801 1628 : SET_DECL_VALUE_EXPR (new_var, build_simple_mem_ref (t));
5802 1628 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5803 : }
5804 1734 : t = fold_convert (build_pointer_type (boolean_type_node), t);
5805 1734 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t,
5806 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5807 1734 : cond = create_tmp_var (TREE_TYPE (t));
5808 1734 : gimplify_assign (cond, t, ilist);
5809 : }
5810 309684 : else if (is_variable_sized (var))
5811 : {
5812 : /* For variable sized types, we need to allocate the
5813 : actual storage here. Call alloca and store the
5814 : result in the pointer decl that we created elsewhere. */
5815 262 : if (pass == 0)
5816 134 : continue;
5817 :
5818 128 : if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx))
5819 : {
5820 114 : tree tmp;
5821 :
5822 114 : ptr = DECL_VALUE_EXPR (new_var);
5823 114 : gcc_assert (TREE_CODE (ptr) == INDIRECT_REF);
5824 114 : ptr = TREE_OPERAND (ptr, 0);
5825 114 : gcc_assert (DECL_P (ptr));
5826 114 : x = TYPE_SIZE_UNIT (TREE_TYPE (new_var));
5827 :
5828 114 : if (lower_private_allocate (var, new_var, allocator,
5829 : allocate_ptr, ilist, ctx,
5830 : false, x))
5831 8 : tmp = allocate_ptr;
5832 : else
5833 : {
5834 : /* void *tmp = __builtin_alloca */
5835 106 : tree atmp
5836 106 : = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
5837 106 : gcall *stmt
5838 106 : = gimple_build_call (atmp, 2, x,
5839 106 : size_int (DECL_ALIGN (var)));
5840 106 : cfun->calls_alloca = 1;
5841 106 : tmp = create_tmp_var_raw (ptr_type_node);
5842 106 : gimple_add_tmp_var (tmp);
5843 106 : gimple_call_set_lhs (stmt, tmp);
5844 :
5845 106 : gimple_seq_add_stmt (ilist, stmt);
5846 : }
5847 :
5848 114 : x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp);
5849 114 : gimplify_assign (ptr, x, ilist);
5850 : }
5851 : }
5852 309422 : else if (omp_privatize_by_reference (var)
5853 309422 : && (c_kind != OMP_CLAUSE_FIRSTPRIVATE
5854 1944 : || !OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c)))
5855 : {
5856 : /* For references that are being privatized for Fortran,
5857 : allocate new backing storage for the new pointer
5858 : variable. This allows us to avoid changing all the
5859 : code that expects a pointer to something that expects
5860 : a direct variable. */
5861 4787 : if (pass == 0)
5862 2496 : continue;
5863 :
5864 2291 : x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
5865 2291 : if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx))
5866 : {
5867 134 : x = build_receiver_ref (var, false, ctx);
5868 134 : if (ctx->allocate_map)
5869 4 : if (tree *allocatep = ctx->allocate_map->get (var))
5870 : {
5871 4 : allocator = *allocatep;
5872 4 : if (TREE_CODE (allocator) == TREE_LIST)
5873 0 : allocator = TREE_PURPOSE (allocator);
5874 4 : if (TREE_CODE (allocator) != INTEGER_CST)
5875 2 : allocator = build_outer_var_ref (allocator, ctx);
5876 4 : allocator = fold_convert (pointer_sized_int_node,
5877 : allocator);
5878 4 : allocate_ptr = unshare_expr (x);
5879 : }
5880 134 : if (allocator == NULL_TREE)
5881 130 : x = build_fold_addr_expr_loc (clause_loc, x);
5882 : }
5883 2157 : else if (lower_private_allocate (var, new_var, allocator,
5884 : allocate_ptr,
5885 : ilist, ctx, true, x))
5886 44 : x = allocate_ptr;
5887 2113 : else if (TREE_CONSTANT (x))
5888 : {
5889 : /* For reduction in SIMD loop, defer adding the
5890 : initialization of the reference, because if we decide
5891 : to use SIMD array for it, the initialization could cause
5892 : expansion ICE. Ditto for other privatization clauses. */
5893 1404 : if (is_simd)
5894 : x = NULL_TREE;
5895 : else
5896 : {
5897 1077 : x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
5898 : get_name (var));
5899 1077 : gimple_add_tmp_var (x);
5900 1077 : TREE_ADDRESSABLE (x) = 1;
5901 1077 : x = build_fold_addr_expr_loc (clause_loc, x);
5902 : }
5903 : }
5904 : else
5905 : {
5906 709 : tree atmp
5907 709 : = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
5908 709 : tree rtype = TREE_TYPE (TREE_TYPE (new_var));
5909 709 : tree al = size_int (TYPE_ALIGN (rtype));
5910 709 : x = build_call_expr_loc (clause_loc, atmp, 2, x, al);
5911 : }
5912 :
5913 1964 : if (x)
5914 : {
5915 1964 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
5916 1964 : gimplify_assign (new_var, x, ilist);
5917 : }
5918 :
5919 2291 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
5920 : }
5921 304635 : else if ((c_kind == OMP_CLAUSE_REDUCTION
5922 : || c_kind == OMP_CLAUSE_IN_REDUCTION)
5923 304635 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
5924 : {
5925 1968 : if (pass == 0)
5926 984 : continue;
5927 : }
5928 302667 : else if (pass != 0)
5929 151077 : continue;
5930 :
5931 168917 : switch (OMP_CLAUSE_CODE (c))
5932 : {
5933 11676 : case OMP_CLAUSE_SHARED:
5934 : /* Ignore shared directives in teams construct inside
5935 : target construct. */
5936 11676 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
5937 11676 : && !is_host_teams_ctx (ctx))
5938 0 : continue;
5939 : /* Shared global vars are just accessed directly. */
5940 11676 : if (is_global_var (new_var))
5941 : break;
5942 : /* For taskloop firstprivate/lastprivate, represented
5943 : as firstprivate and shared clause on the task, new_var
5944 : is the firstprivate var. */
5945 11522 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
5946 : break;
5947 : /* Set up the DECL_VALUE_EXPR for shared variables now. This
5948 : needs to be delayed until after fixup_child_record_type so
5949 : that we get the correct type during the dereference. */
5950 11210 : by_ref = use_pointer_for_field (var, ctx);
5951 11210 : x = build_receiver_ref (var, by_ref, ctx);
5952 11210 : SET_DECL_VALUE_EXPR (new_var, x);
5953 11210 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5954 :
5955 : /* ??? If VAR is not passed by reference, and the variable
5956 : hasn't been initialized yet, then we'll get a warning for
5957 : the store into the omp_data_s structure. Ideally, we'd be
5958 : able to notice this and not store anything at all, but
5959 : we're generating code too early. Suppress the warning. */
5960 11210 : if (!by_ref)
5961 5116 : suppress_warning (var, OPT_Wuninitialized);
5962 : break;
5963 :
5964 204 : case OMP_CLAUSE__CONDTEMP_:
5965 204 : if (is_parallel_ctx (ctx))
5966 : {
5967 97 : x = build_receiver_ref (var, false, ctx);
5968 97 : SET_DECL_VALUE_EXPR (new_var, x);
5969 97 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5970 : }
5971 107 : else if (is_simd && !OMP_CLAUSE__CONDTEMP__ITER (c))
5972 : {
5973 107 : x = build_zero_cst (TREE_TYPE (var));
5974 107 : goto do_private;
5975 : }
5976 : break;
5977 :
5978 21218 : case OMP_CLAUSE_LASTPRIVATE:
5979 21218 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
5980 : break;
5981 : /* FALLTHRU */
5982 :
5983 87200 : case OMP_CLAUSE_PRIVATE:
5984 87200 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_PRIVATE)
5985 20710 : x = build_outer_var_ref (var, ctx);
5986 66490 : else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
5987 : {
5988 163 : if (is_task_ctx (ctx))
5989 12 : x = build_receiver_ref (var, false, ctx);
5990 : else
5991 151 : x = build_outer_var_ref (var, ctx, OMP_CLAUSE_PRIVATE);
5992 : }
5993 : else
5994 : x = NULL;
5995 93844 : do_private:
5996 93844 : tree nx;
5997 93844 : bool copy_ctor;
5998 93844 : copy_ctor = false;
5999 93844 : lower_private_allocate (var, new_var, allocator, allocate_ptr,
6000 : ilist, ctx, false, NULL_TREE);
6001 93844 : nx = unshare_expr (new_var);
6002 93844 : if (is_simd
6003 24199 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6004 101230 : && OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c))
6005 22 : copy_ctor = true;
6006 93844 : if (copy_ctor)
6007 22 : nx = lang_hooks.decls.omp_clause_copy_ctor (c, nx, x);
6008 : else
6009 93822 : nx = lang_hooks.decls.omp_clause_default_ctor (c, nx, x);
6010 93844 : if (is_simd)
6011 : {
6012 24199 : tree y = lang_hooks.decls.omp_clause_dtor (c, new_var);
6013 23517 : if ((TREE_ADDRESSABLE (new_var) || nx || y
6014 23501 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6015 7010 : && (gimple_omp_for_collapse (ctx->stmt) != 1
6016 1375 : || (gimple_omp_for_index (ctx->stmt, 0)
6017 : != new_var)))
6018 16793 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE__CONDTEMP_
6019 16686 : || omp_privatize_by_reference (var))
6020 31064 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
6021 : ivar, lvar))
6022 : {
6023 5981 : if (omp_privatize_by_reference (var))
6024 : {
6025 52 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6026 52 : tree new_vard = TREE_OPERAND (new_var, 0);
6027 52 : gcc_assert (DECL_P (new_vard));
6028 52 : SET_DECL_VALUE_EXPR (new_vard,
6029 : build_fold_addr_expr (lvar));
6030 52 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6031 : }
6032 :
6033 5981 : if (nx)
6034 : {
6035 30 : tree iv = unshare_expr (ivar);
6036 30 : if (copy_ctor)
6037 22 : x = lang_hooks.decls.omp_clause_copy_ctor (c, iv,
6038 : x);
6039 : else
6040 8 : x = lang_hooks.decls.omp_clause_default_ctor (c,
6041 : iv,
6042 : x);
6043 : }
6044 5951 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__CONDTEMP_)
6045 : {
6046 37 : x = build2 (MODIFY_EXPR, TREE_TYPE (ivar),
6047 : unshare_expr (ivar), x);
6048 37 : nx = x;
6049 : }
6050 5981 : if (nx && x)
6051 67 : gimplify_and_add (x, &llist[0]);
6052 5981 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6053 5981 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
6054 : {
6055 37 : tree v = new_var;
6056 37 : if (!DECL_P (v))
6057 : {
6058 4 : gcc_assert (TREE_CODE (v) == MEM_REF);
6059 4 : v = TREE_OPERAND (v, 0);
6060 4 : gcc_assert (DECL_P (v));
6061 : }
6062 37 : v = *ctx->lastprivate_conditional_map->get (v);
6063 37 : tree t = create_tmp_var (TREE_TYPE (v));
6064 37 : tree z = build_zero_cst (TREE_TYPE (v));
6065 37 : tree orig_v
6066 37 : = build_outer_var_ref (var, ctx,
6067 : OMP_CLAUSE_LASTPRIVATE);
6068 37 : gimple_seq_add_stmt (dlist,
6069 37 : gimple_build_assign (t, z));
6070 37 : gcc_assert (DECL_HAS_VALUE_EXPR_P (v));
6071 37 : tree civar = DECL_VALUE_EXPR (v);
6072 37 : gcc_assert (TREE_CODE (civar) == ARRAY_REF);
6073 37 : civar = unshare_expr (civar);
6074 37 : TREE_OPERAND (civar, 1) = sctx.idx;
6075 37 : x = build2 (MODIFY_EXPR, TREE_TYPE (t), t,
6076 : unshare_expr (civar));
6077 74 : x = build2 (COMPOUND_EXPR, TREE_TYPE (orig_v), x,
6078 37 : build2 (MODIFY_EXPR, TREE_TYPE (orig_v),
6079 : orig_v, unshare_expr (ivar)));
6080 37 : tree cond = build2 (LT_EXPR, boolean_type_node, t,
6081 : civar);
6082 37 : x = build3 (COND_EXPR, void_type_node, cond, x,
6083 : void_node);
6084 37 : gimple_seq tseq = NULL;
6085 37 : gimplify_and_add (x, &tseq);
6086 37 : if (ctx->outer)
6087 27 : lower_omp (&tseq, ctx->outer);
6088 37 : gimple_seq_add_seq (&llist[1], tseq);
6089 : }
6090 5981 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6091 5981 : && ctx->for_simd_scan_phase)
6092 : {
6093 9 : x = unshare_expr (ivar);
6094 9 : tree orig_v
6095 9 : = build_outer_var_ref (var, ctx,
6096 : OMP_CLAUSE_LASTPRIVATE);
6097 9 : x = lang_hooks.decls.omp_clause_assign_op (c, x,
6098 : orig_v);
6099 9 : gimplify_and_add (x, &llist[0]);
6100 : }
6101 5981 : if (y)
6102 : {
6103 30 : y = lang_hooks.decls.omp_clause_dtor (c, ivar);
6104 30 : if (y)
6105 30 : gimplify_and_add (y, &llist[1]);
6106 : }
6107 : break;
6108 : }
6109 18218 : if (omp_privatize_by_reference (var))
6110 : {
6111 28 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6112 28 : tree new_vard = TREE_OPERAND (new_var, 0);
6113 28 : gcc_assert (DECL_P (new_vard));
6114 28 : tree type = TREE_TYPE (TREE_TYPE (new_vard));
6115 28 : x = TYPE_SIZE_UNIT (type);
6116 28 : if (TREE_CONSTANT (x))
6117 : {
6118 28 : x = create_tmp_var_raw (type, get_name (var));
6119 28 : gimple_add_tmp_var (x);
6120 28 : TREE_ADDRESSABLE (x) = 1;
6121 28 : x = build_fold_addr_expr_loc (clause_loc, x);
6122 28 : x = fold_convert_loc (clause_loc,
6123 28 : TREE_TYPE (new_vard), x);
6124 28 : gimplify_assign (new_vard, x, ilist);
6125 : }
6126 : }
6127 : }
6128 87863 : if (nx)
6129 488 : gimplify_and_add (nx, ilist);
6130 87863 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6131 15087 : && is_simd
6132 89626 : && ctx->for_simd_scan_phase)
6133 : {
6134 5 : tree orig_v = build_outer_var_ref (var, ctx,
6135 : OMP_CLAUSE_LASTPRIVATE);
6136 5 : x = lang_hooks.decls.omp_clause_assign_op (c, new_var,
6137 : orig_v);
6138 5 : gimplify_and_add (x, ilist);
6139 : }
6140 : /* FALLTHRU */
6141 :
6142 118607 : do_dtor:
6143 118607 : x = lang_hooks.decls.omp_clause_dtor (c, new_var);
6144 118607 : if (x)
6145 1134 : gimplify_and_add (x, dlist);
6146 118607 : if (allocator)
6147 : {
6148 636 : if (!is_gimple_val (allocator))
6149 : {
6150 10 : tree avar = create_tmp_var (TREE_TYPE (allocator));
6151 10 : gimplify_assign (avar, allocator, dlist);
6152 10 : allocator = avar;
6153 : }
6154 636 : if (!is_gimple_val (allocate_ptr))
6155 : {
6156 26 : tree apvar = create_tmp_var (TREE_TYPE (allocate_ptr));
6157 26 : gimplify_assign (apvar, allocate_ptr, dlist);
6158 26 : allocate_ptr = apvar;
6159 : }
6160 636 : tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
6161 636 : gimple *g
6162 636 : = gimple_build_call (f, 2, allocate_ptr, allocator);
6163 636 : gimple_seq_add_stmt (dlist, g);
6164 : }
6165 : break;
6166 :
6167 7782 : case OMP_CLAUSE_LINEAR:
6168 7782 : if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c))
6169 1245 : goto do_firstprivate;
6170 6537 : if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
6171 : x = NULL;
6172 : else
6173 3598 : x = build_outer_var_ref (var, ctx);
6174 6537 : goto do_private;
6175 :
6176 28629 : case OMP_CLAUSE_FIRSTPRIVATE:
6177 28629 : if (is_task_ctx (ctx))
6178 : {
6179 4589 : if ((omp_privatize_by_reference (var)
6180 134 : && !OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c))
6181 4589 : || is_variable_sized (var))
6182 148 : goto do_dtor;
6183 4441 : else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var,
6184 : ctx))
6185 4441 : || use_pointer_for_field (var, NULL))
6186 : {
6187 409 : x = build_receiver_ref (var, false, ctx);
6188 409 : if (ctx->allocate_map)
6189 22 : if (tree *allocatep = ctx->allocate_map->get (var))
6190 : {
6191 22 : allocator = *allocatep;
6192 22 : if (TREE_CODE (allocator) == TREE_LIST)
6193 4 : allocator = TREE_PURPOSE (allocator);
6194 22 : if (TREE_CODE (allocator) != INTEGER_CST)
6195 8 : allocator = build_outer_var_ref (allocator, ctx);
6196 22 : allocator = fold_convert (pointer_sized_int_node,
6197 : allocator);
6198 22 : allocate_ptr = unshare_expr (x);
6199 22 : x = build_simple_mem_ref (x);
6200 22 : TREE_THIS_NOTRAP (x) = 1;
6201 : }
6202 409 : SET_DECL_VALUE_EXPR (new_var, x);
6203 409 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
6204 409 : goto do_dtor;
6205 : }
6206 : }
6207 28072 : if (OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c)
6208 28072 : && omp_privatize_by_reference (var))
6209 : {
6210 0 : x = build_outer_var_ref (var, ctx);
6211 0 : gcc_assert (TREE_CODE (x) == MEM_REF
6212 : && integer_zerop (TREE_OPERAND (x, 1)));
6213 0 : x = TREE_OPERAND (x, 0);
6214 0 : x = lang_hooks.decls.omp_clause_copy_ctor
6215 0 : (c, unshare_expr (new_var), x);
6216 0 : gimplify_and_add (x, ilist);
6217 0 : goto do_dtor;
6218 : }
6219 29317 : do_firstprivate:
6220 29317 : lower_private_allocate (var, new_var, allocator, allocate_ptr,
6221 : ilist, ctx, false, NULL_TREE);
6222 29317 : x = build_outer_var_ref (var, ctx);
6223 29317 : if (is_simd)
6224 : {
6225 1063 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
6226 1063 : && gimple_omp_for_combined_into_p (ctx->stmt))
6227 : {
6228 668 : tree t = OMP_CLAUSE_LINEAR_STEP (c);
6229 668 : if (DECL_P (t))
6230 114 : t = build_outer_var_ref (t, ctx);
6231 668 : tree stept = TREE_TYPE (t);
6232 668 : tree ct = omp_find_clause (clauses,
6233 : OMP_CLAUSE__LOOPTEMP_);
6234 668 : gcc_assert (ct);
6235 668 : tree l = OMP_CLAUSE_DECL (ct);
6236 668 : tree n1 = fd->loop.n1;
6237 668 : tree step = fd->loop.step;
6238 668 : tree itype = TREE_TYPE (l);
6239 668 : if (POINTER_TYPE_P (itype))
6240 0 : itype = signed_type_for (itype);
6241 668 : l = fold_build2 (MINUS_EXPR, itype, l, n1);
6242 668 : if (TYPE_UNSIGNED (itype)
6243 668 : && fd->loop.cond_code == GT_EXPR)
6244 0 : l = fold_build2 (TRUNC_DIV_EXPR, itype,
6245 : fold_build1 (NEGATE_EXPR, itype, l),
6246 : fold_build1 (NEGATE_EXPR,
6247 : itype, step));
6248 : else
6249 668 : l = fold_build2 (TRUNC_DIV_EXPR, itype, l, step);
6250 668 : t = fold_build2 (MULT_EXPR, stept,
6251 : fold_convert (stept, l), t);
6252 :
6253 668 : if (OMP_CLAUSE_LINEAR_ARRAY (c))
6254 : {
6255 108 : if (omp_privatize_by_reference (var))
6256 : {
6257 72 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6258 72 : tree new_vard = TREE_OPERAND (new_var, 0);
6259 72 : gcc_assert (DECL_P (new_vard));
6260 72 : tree type = TREE_TYPE (TREE_TYPE (new_vard));
6261 72 : nx = TYPE_SIZE_UNIT (type);
6262 72 : if (TREE_CONSTANT (nx))
6263 : {
6264 24 : nx = create_tmp_var_raw (type,
6265 : get_name (var));
6266 24 : gimple_add_tmp_var (nx);
6267 24 : TREE_ADDRESSABLE (nx) = 1;
6268 24 : nx = build_fold_addr_expr_loc (clause_loc,
6269 : nx);
6270 24 : nx = fold_convert_loc (clause_loc,
6271 24 : TREE_TYPE (new_vard),
6272 : nx);
6273 24 : gimplify_assign (new_vard, nx, ilist);
6274 : }
6275 : }
6276 :
6277 108 : x = lang_hooks.decls.omp_clause_linear_ctor
6278 108 : (c, new_var, x, t);
6279 108 : gimplify_and_add (x, ilist);
6280 108 : goto do_dtor;
6281 : }
6282 :
6283 560 : if (POINTER_TYPE_P (TREE_TYPE (x)))
6284 11 : x = fold_build_pointer_plus (x, t);
6285 : else
6286 549 : x = fold_build2 (PLUS_EXPR, TREE_TYPE (x), x,
6287 : fold_convert (TREE_TYPE (x), t));
6288 : }
6289 :
6290 955 : if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LINEAR
6291 934 : || TREE_ADDRESSABLE (new_var)
6292 799 : || omp_privatize_by_reference (var))
6293 1217 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
6294 : ivar, lvar))
6295 : {
6296 152 : if (omp_privatize_by_reference (var))
6297 : {
6298 22 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6299 22 : tree new_vard = TREE_OPERAND (new_var, 0);
6300 22 : gcc_assert (DECL_P (new_vard));
6301 22 : SET_DECL_VALUE_EXPR (new_vard,
6302 : build_fold_addr_expr (lvar));
6303 22 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6304 : }
6305 152 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR)
6306 : {
6307 131 : tree iv = create_tmp_var (TREE_TYPE (new_var));
6308 131 : x = lang_hooks.decls.omp_clause_copy_ctor (c, iv, x);
6309 131 : gimplify_and_add (x, ilist);
6310 131 : gimple_stmt_iterator gsi
6311 131 : = gsi_start (*gimple_omp_body_ptr (ctx->stmt));
6312 131 : gassign *g
6313 131 : = gimple_build_assign (unshare_expr (lvar), iv);
6314 131 : gsi_insert_before_without_update (&gsi, g,
6315 : GSI_SAME_STMT);
6316 131 : tree t = OMP_CLAUSE_LINEAR_STEP (c);
6317 131 : enum tree_code code = PLUS_EXPR;
6318 131 : if (POINTER_TYPE_P (TREE_TYPE (new_var)))
6319 : code = POINTER_PLUS_EXPR;
6320 131 : g = gimple_build_assign (iv, code, iv, t);
6321 131 : gsi_insert_before_without_update (&gsi, g,
6322 : GSI_SAME_STMT);
6323 131 : break;
6324 : }
6325 21 : x = lang_hooks.decls.omp_clause_copy_ctor
6326 21 : (c, unshare_expr (ivar), x);
6327 21 : gimplify_and_add (x, &llist[0]);
6328 21 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6329 21 : if (x)
6330 21 : gimplify_and_add (x, &llist[1]);
6331 : break;
6332 : }
6333 803 : if (omp_privatize_by_reference (var))
6334 : {
6335 105 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6336 105 : tree new_vard = TREE_OPERAND (new_var, 0);
6337 105 : gcc_assert (DECL_P (new_vard));
6338 105 : tree type = TREE_TYPE (TREE_TYPE (new_vard));
6339 105 : nx = TYPE_SIZE_UNIT (type);
6340 105 : if (TREE_CONSTANT (nx))
6341 : {
6342 57 : nx = create_tmp_var_raw (type, get_name (var));
6343 57 : gimple_add_tmp_var (nx);
6344 57 : TREE_ADDRESSABLE (nx) = 1;
6345 57 : nx = build_fold_addr_expr_loc (clause_loc, nx);
6346 57 : nx = fold_convert_loc (clause_loc,
6347 57 : TREE_TYPE (new_vard), nx);
6348 57 : gimplify_assign (new_vard, nx, ilist);
6349 : }
6350 : }
6351 : }
6352 29057 : x = lang_hooks.decls.omp_clause_copy_ctor
6353 29057 : (c, unshare_expr (new_var), x);
6354 29053 : gimplify_and_add (x, ilist);
6355 29053 : goto do_dtor;
6356 :
6357 19352 : case OMP_CLAUSE__LOOPTEMP_:
6358 19352 : case OMP_CLAUSE__REDUCTEMP_:
6359 19352 : gcc_assert (is_taskreg_ctx (ctx));
6360 19352 : x = build_outer_var_ref (var, ctx);
6361 19352 : x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
6362 19352 : gimplify_and_add (x, ilist);
6363 19352 : break;
6364 :
6365 514 : case OMP_CLAUSE_COPYIN:
6366 514 : by_ref = use_pointer_for_field (var, NULL);
6367 514 : x = build_receiver_ref (var, by_ref, ctx);
6368 514 : x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x);
6369 514 : append_to_statement_list (x, ©in_seq);
6370 514 : copyin_by_ref |= by_ref;
6371 514 : break;
6372 :
6373 13052 : case OMP_CLAUSE_REDUCTION:
6374 13052 : case OMP_CLAUSE_IN_REDUCTION:
6375 : /* OpenACC reductions are initialized using the
6376 : GOACC_REDUCTION internal function. */
6377 13052 : if (is_gimple_omp_oacc (ctx->stmt))
6378 : break;
6379 9070 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
6380 : {
6381 1440 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
6382 1440 : gimple *tseq;
6383 1440 : tree ptype = TREE_TYPE (placeholder);
6384 1440 : if (cond)
6385 : {
6386 295 : x = error_mark_node;
6387 295 : if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)
6388 295 : && !task_reduction_needs_orig_p)
6389 28 : x = var;
6390 267 : else if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
6391 : {
6392 45 : tree pptype = build_pointer_type (ptype);
6393 45 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
6394 35 : x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
6395 35 : size_int (task_reduction_cnt_full
6396 : + task_reduction_cntorig - 1),
6397 : NULL_TREE, NULL_TREE);
6398 : else
6399 : {
6400 10 : unsigned int idx
6401 10 : = *ctx->task_reduction_map->get (c);
6402 10 : x = task_reduction_read (ilist, tskred_temp,
6403 10 : pptype, 7 + 3 * idx);
6404 : }
6405 45 : x = fold_convert (pptype, x);
6406 45 : x = build_simple_mem_ref (x);
6407 : }
6408 : }
6409 : else
6410 : {
6411 1145 : lower_private_allocate (var, new_var, allocator,
6412 : allocate_ptr, ilist, ctx, false,
6413 : NULL_TREE);
6414 1145 : x = build_outer_var_ref (var, ctx);
6415 :
6416 1145 : if (omp_privatize_by_reference (var)
6417 1145 : && !useless_type_conversion_p (ptype, TREE_TYPE (x)))
6418 37 : x = build_fold_addr_expr_loc (clause_loc, x);
6419 : }
6420 1440 : SET_DECL_VALUE_EXPR (placeholder, x);
6421 1440 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
6422 1440 : tree new_vard = new_var;
6423 1440 : if (omp_privatize_by_reference (var))
6424 : {
6425 191 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6426 191 : new_vard = TREE_OPERAND (new_var, 0);
6427 191 : gcc_assert (DECL_P (new_vard));
6428 : }
6429 1440 : tree rvar = NULL_TREE, *rvarp = NULL, rvar2 = NULL_TREE;
6430 1440 : if (is_simd
6431 311 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6432 1751 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6433 : rvarp = &rvar;
6434 1440 : if (is_simd
6435 1440 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
6436 : ivar, lvar, rvarp,
6437 : &rvar2))
6438 : {
6439 174 : if (new_vard == new_var)
6440 : {
6441 128 : gcc_assert (DECL_VALUE_EXPR (new_var) == lvar);
6442 128 : SET_DECL_VALUE_EXPR (new_var, ivar);
6443 : }
6444 : else
6445 : {
6446 46 : SET_DECL_VALUE_EXPR (new_vard,
6447 : build_fold_addr_expr (ivar));
6448 46 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6449 : }
6450 174 : x = lang_hooks.decls.omp_clause_default_ctor
6451 174 : (c, unshare_expr (ivar),
6452 : build_outer_var_ref (var, ctx));
6453 174 : if (rvarp && ctx->for_simd_scan_phase)
6454 : {
6455 16 : if (x)
6456 8 : gimplify_and_add (x, &llist[0]);
6457 16 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6458 16 : if (x)
6459 8 : gimplify_and_add (x, &llist[1]);
6460 469 : break;
6461 : }
6462 78 : else if (rvarp)
6463 : {
6464 78 : if (x)
6465 : {
6466 38 : gimplify_and_add (x, &llist[0]);
6467 :
6468 38 : tree ivar2 = unshare_expr (lvar);
6469 38 : TREE_OPERAND (ivar2, 1) = sctx.idx;
6470 38 : x = lang_hooks.decls.omp_clause_default_ctor
6471 38 : (c, ivar2, build_outer_var_ref (var, ctx));
6472 38 : gimplify_and_add (x, &llist[0]);
6473 :
6474 38 : if (rvar2)
6475 : {
6476 16 : x = lang_hooks.decls.omp_clause_default_ctor
6477 16 : (c, unshare_expr (rvar2),
6478 : build_outer_var_ref (var, ctx));
6479 16 : gimplify_and_add (x, &llist[0]);
6480 : }
6481 :
6482 : /* For types that need construction, add another
6483 : private var which will be default constructed
6484 : and optionally initialized with
6485 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT, as in the
6486 : loop we want to assign this value instead of
6487 : constructing and destructing it in each
6488 : iteration. */
6489 38 : tree nv = create_tmp_var_raw (TREE_TYPE (ivar));
6490 38 : gimple_add_tmp_var (nv);
6491 60 : ctx->cb.decl_map->put (TREE_OPERAND (rvar2
6492 : ? rvar2
6493 : : ivar, 0),
6494 : nv);
6495 38 : x = lang_hooks.decls.omp_clause_default_ctor
6496 38 : (c, nv, build_outer_var_ref (var, ctx));
6497 38 : gimplify_and_add (x, ilist);
6498 :
6499 38 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6500 : {
6501 19 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6502 19 : x = DECL_VALUE_EXPR (new_vard);
6503 19 : tree vexpr = nv;
6504 19 : if (new_vard != new_var)
6505 11 : vexpr = build_fold_addr_expr (nv);
6506 19 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
6507 19 : lower_omp (&tseq, ctx);
6508 19 : SET_DECL_VALUE_EXPR (new_vard, x);
6509 19 : gimple_seq_add_seq (ilist, tseq);
6510 19 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6511 : }
6512 :
6513 38 : x = lang_hooks.decls.omp_clause_dtor (c, nv);
6514 38 : if (x)
6515 38 : gimplify_and_add (x, dlist);
6516 : }
6517 :
6518 78 : tree ref = build_outer_var_ref (var, ctx);
6519 78 : x = unshare_expr (ivar);
6520 78 : x = lang_hooks.decls.omp_clause_assign_op (c, x,
6521 : ref);
6522 78 : gimplify_and_add (x, &llist[0]);
6523 :
6524 78 : ref = build_outer_var_ref (var, ctx);
6525 78 : x = lang_hooks.decls.omp_clause_assign_op (c, ref,
6526 : rvar);
6527 78 : gimplify_and_add (x, &llist[3]);
6528 :
6529 78 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6530 78 : if (new_vard == new_var)
6531 48 : SET_DECL_VALUE_EXPR (new_var, lvar);
6532 : else
6533 30 : SET_DECL_VALUE_EXPR (new_vard,
6534 : build_fold_addr_expr (lvar));
6535 :
6536 78 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6537 78 : if (x)
6538 38 : gimplify_and_add (x, &llist[1]);
6539 :
6540 78 : tree ivar2 = unshare_expr (lvar);
6541 78 : TREE_OPERAND (ivar2, 1) = sctx.idx;
6542 78 : x = lang_hooks.decls.omp_clause_dtor (c, ivar2);
6543 78 : if (x)
6544 38 : gimplify_and_add (x, &llist[1]);
6545 :
6546 78 : if (rvar2)
6547 : {
6548 36 : x = lang_hooks.decls.omp_clause_dtor (c, rvar2);
6549 36 : if (x)
6550 16 : gimplify_and_add (x, &llist[1]);
6551 : }
6552 : break;
6553 : }
6554 80 : if (x)
6555 34 : gimplify_and_add (x, &llist[0]);
6556 80 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6557 : {
6558 76 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6559 76 : lower_omp (&tseq, ctx);
6560 76 : gimple_seq_add_seq (&llist[0], tseq);
6561 : }
6562 80 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6563 80 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
6564 80 : lower_omp (&tseq, ctx);
6565 80 : gimple_seq_add_seq (&llist[1], tseq);
6566 80 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
6567 80 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6568 80 : if (new_vard == new_var)
6569 70 : SET_DECL_VALUE_EXPR (new_var, lvar);
6570 : else
6571 10 : SET_DECL_VALUE_EXPR (new_vard,
6572 : build_fold_addr_expr (lvar));
6573 80 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6574 80 : if (x)
6575 38 : gimplify_and_add (x, &llist[1]);
6576 : break;
6577 : }
6578 : /* If this is a reference to constant size reduction var
6579 : with placeholder, we haven't emitted the initializer
6580 : for it because it is undesirable if SIMD arrays are used.
6581 : But if they aren't used, we need to emit the deferred
6582 : initialization now. */
6583 1266 : else if (omp_privatize_by_reference (var) && is_simd)
6584 46 : handle_simd_reference (clause_loc, new_vard, ilist);
6585 :
6586 1266 : tree lab2 = NULL_TREE;
6587 1266 : if (cond)
6588 : {
6589 295 : gimple *g;
6590 295 : if (!is_parallel_ctx (ctx))
6591 : {
6592 278 : tree condv = create_tmp_var (boolean_type_node);
6593 278 : tree m = build_simple_mem_ref (cond);
6594 278 : g = gimple_build_assign (condv, m);
6595 278 : gimple_seq_add_stmt (ilist, g);
6596 278 : tree lab1
6597 278 : = create_artificial_label (UNKNOWN_LOCATION);
6598 278 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
6599 278 : g = gimple_build_cond (NE_EXPR, condv,
6600 : boolean_false_node,
6601 : lab2, lab1);
6602 278 : gimple_seq_add_stmt (ilist, g);
6603 278 : gimple_seq_add_stmt (ilist,
6604 278 : gimple_build_label (lab1));
6605 : }
6606 295 : g = gimple_build_assign (build_simple_mem_ref (cond),
6607 : boolean_true_node);
6608 295 : gimple_seq_add_stmt (ilist, g);
6609 : }
6610 295 : x = lang_hooks.decls.omp_clause_default_ctor
6611 1266 : (c, unshare_expr (new_var),
6612 : cond ? NULL_TREE
6613 971 : : build_outer_var_ref (var, ctx));
6614 1266 : if (x)
6615 286 : gimplify_and_add (x, ilist);
6616 :
6617 1266 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6618 1266 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6619 : {
6620 158 : if (ctx->for_simd_scan_phase)
6621 971 : goto do_dtor;
6622 142 : if (x || (!is_simd
6623 32 : && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)))
6624 : {
6625 70 : tree nv = create_tmp_var_raw (TREE_TYPE (new_var));
6626 70 : gimple_add_tmp_var (nv);
6627 70 : ctx->cb.decl_map->put (new_vard, nv);
6628 70 : x = lang_hooks.decls.omp_clause_default_ctor
6629 70 : (c, nv, build_outer_var_ref (var, ctx));
6630 70 : if (x)
6631 70 : gimplify_and_add (x, ilist);
6632 70 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6633 : {
6634 35 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6635 35 : tree vexpr = nv;
6636 35 : if (new_vard != new_var)
6637 19 : vexpr = build_fold_addr_expr (nv);
6638 35 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
6639 35 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6640 35 : lower_omp (&tseq, ctx);
6641 35 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
6642 35 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
6643 35 : gimple_seq_add_seq (ilist, tseq);
6644 : }
6645 70 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6646 70 : if (is_simd && ctx->scan_exclusive)
6647 : {
6648 16 : tree nv2
6649 16 : = create_tmp_var_raw (TREE_TYPE (new_var));
6650 16 : gimple_add_tmp_var (nv2);
6651 16 : ctx->cb.decl_map->put (nv, nv2);
6652 16 : x = lang_hooks.decls.omp_clause_default_ctor
6653 16 : (c, nv2, build_outer_var_ref (var, ctx));
6654 16 : gimplify_and_add (x, ilist);
6655 16 : x = lang_hooks.decls.omp_clause_dtor (c, nv2);
6656 16 : if (x)
6657 16 : gimplify_and_add (x, dlist);
6658 : }
6659 70 : x = lang_hooks.decls.omp_clause_dtor (c, nv);
6660 70 : if (x)
6661 70 : gimplify_and_add (x, dlist);
6662 : }
6663 72 : else if (is_simd
6664 40 : && ctx->scan_exclusive
6665 92 : && TREE_ADDRESSABLE (TREE_TYPE (new_var)))
6666 : {
6667 0 : tree nv2 = create_tmp_var_raw (TREE_TYPE (new_var));
6668 0 : gimple_add_tmp_var (nv2);
6669 0 : ctx->cb.decl_map->put (new_vard, nv2);
6670 0 : x = lang_hooks.decls.omp_clause_dtor (c, nv2);
6671 0 : if (x)
6672 0 : gimplify_and_add (x, dlist);
6673 : }
6674 142 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6675 142 : goto do_dtor;
6676 : }
6677 :
6678 1108 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6679 : {
6680 1060 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6681 1060 : if (c_kind == OMP_CLAUSE_IN_REDUCTION
6682 1060 : && is_omp_target (ctx->stmt))
6683 : {
6684 12 : tree d = maybe_lookup_decl_in_outer_ctx (var, ctx);
6685 12 : tree oldv = NULL_TREE;
6686 12 : gcc_assert (d);
6687 12 : if (DECL_HAS_VALUE_EXPR_P (d))
6688 6 : oldv = DECL_VALUE_EXPR (d);
6689 12 : SET_DECL_VALUE_EXPR (d, new_vard);
6690 12 : DECL_HAS_VALUE_EXPR_P (d) = 1;
6691 12 : lower_omp (&tseq, ctx);
6692 12 : if (oldv)
6693 6 : SET_DECL_VALUE_EXPR (d, oldv);
6694 : else
6695 : {
6696 6 : SET_DECL_VALUE_EXPR (d, NULL_TREE);
6697 6 : DECL_HAS_VALUE_EXPR_P (d) = 0;
6698 : }
6699 : }
6700 : else
6701 1048 : lower_omp (&tseq, ctx);
6702 1060 : gimple_seq_add_seq (ilist, tseq);
6703 : }
6704 1108 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6705 1108 : if (is_simd)
6706 : {
6707 43 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
6708 43 : lower_omp (&tseq, ctx);
6709 43 : gimple_seq_add_seq (dlist, tseq);
6710 43 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
6711 : }
6712 1108 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6713 1108 : if (cond)
6714 : {
6715 295 : if (lab2)
6716 278 : gimple_seq_add_stmt (ilist, gimple_build_label (lab2));
6717 : break;
6718 : }
6719 813 : goto do_dtor;
6720 : }
6721 : else
6722 : {
6723 7630 : x = omp_reduction_init (c, TREE_TYPE (new_var));
6724 7630 : gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE);
6725 7630 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
6726 :
6727 7630 : if (cond)
6728 : {
6729 1439 : gimple *g;
6730 1439 : tree lab2 = NULL_TREE;
6731 : /* GOMP_taskgroup_reduction_register memsets the whole
6732 : array to zero. If the initializer is zero, we don't
6733 : need to initialize it again, just mark it as ever
6734 : used unconditionally, i.e. cond = true. */
6735 1439 : if (initializer_zerop (x))
6736 : {
6737 1258 : g = gimple_build_assign (build_simple_mem_ref (cond),
6738 : boolean_true_node);
6739 1258 : gimple_seq_add_stmt (ilist, g);
6740 3227 : break;
6741 : }
6742 :
6743 : /* Otherwise, emit
6744 : if (!cond) { cond = true; new_var = x; } */
6745 181 : if (!is_parallel_ctx (ctx))
6746 : {
6747 177 : tree condv = create_tmp_var (boolean_type_node);
6748 177 : tree m = build_simple_mem_ref (cond);
6749 177 : g = gimple_build_assign (condv, m);
6750 177 : gimple_seq_add_stmt (ilist, g);
6751 177 : tree lab1
6752 177 : = create_artificial_label (UNKNOWN_LOCATION);
6753 177 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
6754 177 : g = gimple_build_cond (NE_EXPR, condv,
6755 : boolean_false_node,
6756 : lab2, lab1);
6757 177 : gimple_seq_add_stmt (ilist, g);
6758 177 : gimple_seq_add_stmt (ilist,
6759 177 : gimple_build_label (lab1));
6760 : }
6761 181 : g = gimple_build_assign (build_simple_mem_ref (cond),
6762 : boolean_true_node);
6763 181 : gimple_seq_add_stmt (ilist, g);
6764 181 : gimplify_assign (new_var, x, ilist);
6765 181 : if (lab2)
6766 177 : gimple_seq_add_stmt (ilist, gimple_build_label (lab2));
6767 : break;
6768 : }
6769 :
6770 : /* reduction(-:var) sums up the partial results, so it
6771 : acts identically to reduction(+:var). */
6772 6191 : if (code == MINUS_EXPR)
6773 33 : code = PLUS_EXPR;
6774 :
6775 6191 : bool is_truth_op
6776 6191 : = (code == TRUTH_ANDIF_EXPR || code == TRUTH_ORIF_EXPR);
6777 6191 : tree new_vard = new_var;
6778 6191 : if (is_simd && omp_privatize_by_reference (var))
6779 : {
6780 52 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6781 52 : new_vard = TREE_OPERAND (new_var, 0);
6782 52 : gcc_assert (DECL_P (new_vard));
6783 : }
6784 6191 : tree rvar = NULL_TREE, *rvarp = NULL, rvar2 = NULL_TREE;
6785 6191 : if (is_simd
6786 2087 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6787 8278 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6788 : rvarp = &rvar;
6789 6191 : if (is_simd
6790 6191 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
6791 : ivar, lvar, rvarp,
6792 : &rvar2))
6793 : {
6794 867 : if (new_vard != new_var)
6795 : {
6796 34 : SET_DECL_VALUE_EXPR (new_vard,
6797 : build_fold_addr_expr (lvar));
6798 34 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6799 : }
6800 :
6801 867 : tree ref = build_outer_var_ref (var, ctx);
6802 :
6803 867 : if (rvarp)
6804 : {
6805 193 : if (ctx->for_simd_scan_phase)
6806 : break;
6807 154 : gimplify_assign (ivar, ref, &llist[0]);
6808 154 : ref = build_outer_var_ref (var, ctx);
6809 154 : gimplify_assign (ref, rvar, &llist[3]);
6810 154 : break;
6811 : }
6812 :
6813 674 : gimplify_assign (unshare_expr (ivar), x, &llist[0]);
6814 :
6815 674 : if (sctx.is_simt)
6816 : {
6817 0 : if (!simt_lane)
6818 0 : simt_lane = create_tmp_var (unsigned_type_node);
6819 0 : x = build_call_expr_internal_loc
6820 0 : (UNKNOWN_LOCATION, IFN_GOMP_SIMT_XCHG_BFLY,
6821 0 : TREE_TYPE (ivar), 2, ivar, simt_lane);
6822 : /* Make sure x is evaluated unconditionally. */
6823 0 : tree bfly_var = create_tmp_var (TREE_TYPE (ivar));
6824 0 : gimplify_assign (bfly_var, x, &llist[2]);
6825 0 : x = build2 (code, TREE_TYPE (ivar), ivar, bfly_var);
6826 0 : gimplify_assign (ivar, x, &llist[2]);
6827 : }
6828 674 : tree ivar2 = ivar;
6829 674 : tree ref2 = ref;
6830 674 : if (is_truth_op)
6831 : {
6832 99 : tree zero = build_zero_cst (TREE_TYPE (ivar));
6833 99 : ivar2 = fold_build2_loc (clause_loc, NE_EXPR,
6834 : boolean_type_node, ivar,
6835 : zero);
6836 99 : ref2 = fold_build2_loc (clause_loc, NE_EXPR,
6837 : boolean_type_node, ref,
6838 : zero);
6839 : }
6840 674 : x = build2 (code, TREE_TYPE (ref), ref2, ivar2);
6841 674 : if (is_truth_op)
6842 99 : x = fold_convert (TREE_TYPE (ref), x);
6843 674 : ref = build_outer_var_ref (var, ctx);
6844 674 : gimplify_assign (ref, x, &llist[1]);
6845 :
6846 : }
6847 : else
6848 : {
6849 5324 : lower_private_allocate (var, new_var, allocator,
6850 : allocate_ptr, ilist, ctx,
6851 : false, NULL_TREE);
6852 5324 : if (omp_privatize_by_reference (var) && is_simd)
6853 18 : handle_simd_reference (clause_loc, new_vard, ilist);
6854 5324 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6855 5324 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6856 : break;
6857 4987 : gimplify_assign (new_var, x, ilist);
6858 4987 : if (is_simd)
6859 : {
6860 1028 : tree ref = build_outer_var_ref (var, ctx);
6861 1028 : tree new_var2 = new_var;
6862 1028 : tree ref2 = ref;
6863 1028 : if (is_truth_op)
6864 : {
6865 24 : tree zero = build_zero_cst (TREE_TYPE (new_var));
6866 24 : new_var2
6867 24 : = fold_build2_loc (clause_loc, NE_EXPR,
6868 : boolean_type_node, new_var,
6869 : zero);
6870 24 : ref2 = fold_build2_loc (clause_loc, NE_EXPR,
6871 : boolean_type_node, ref,
6872 : zero);
6873 : }
6874 1028 : x = build2 (code, TREE_TYPE (ref2), ref2, new_var2);
6875 1028 : if (is_truth_op)
6876 24 : x = fold_convert (TREE_TYPE (new_var), x);
6877 1028 : ref = build_outer_var_ref (var, ctx);
6878 1028 : gimplify_assign (ref, x, dlist);
6879 : }
6880 4987 : if (allocator)
6881 55 : goto do_dtor;
6882 : }
6883 : }
6884 5606 : break;
6885 :
6886 0 : default:
6887 0 : gcc_unreachable ();
6888 : }
6889 : }
6890 : }
6891 77134 : if (tskred_avar)
6892 : {
6893 871 : tree clobber = build_clobber (TREE_TYPE (tskred_avar));
6894 871 : gimple_seq_add_stmt (ilist, gimple_build_assign (tskred_avar, clobber));
6895 : }
6896 :
6897 77134 : if (known_eq (sctx.max_vf, 1U))
6898 : {
6899 1779 : sctx.is_simt = false;
6900 1779 : if (ctx->lastprivate_conditional_map)
6901 : {
6902 52 : if (gimple_omp_for_combined_into_p (ctx->stmt))
6903 : {
6904 : /* Signal to lower_omp_1 that it should use parent context. */
6905 43 : ctx->combined_into_simd_safelen1 = true;
6906 370 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
6907 327 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6908 327 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
6909 : {
6910 57 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
6911 57 : omp_context *outer = ctx->outer;
6912 57 : if (gimple_code (outer->stmt) == GIMPLE_OMP_SCAN)
6913 6 : outer = outer->outer;
6914 57 : tree *v = ctx->lastprivate_conditional_map->get (o);
6915 57 : tree po = lookup_decl (OMP_CLAUSE_DECL (c), outer);
6916 57 : tree *pv = outer->lastprivate_conditional_map->get (po);
6917 57 : *v = *pv;
6918 : }
6919 : }
6920 : else
6921 : {
6922 : /* When not vectorized, treat lastprivate(conditional:) like
6923 : normal lastprivate, as there will be just one simd lane
6924 : writing the privatized variable. */
6925 9 : delete ctx->lastprivate_conditional_map;
6926 9 : ctx->lastprivate_conditional_map = NULL;
6927 : }
6928 : }
6929 : }
6930 :
6931 77134 : if (nonconst_simd_if)
6932 : {
6933 625 : if (sctx.lane == NULL_TREE)
6934 : {
6935 570 : sctx.idx = create_tmp_var (unsigned_type_node);
6936 570 : sctx.lane = create_tmp_var (unsigned_type_node);
6937 : }
6938 : /* FIXME: For now. */
6939 625 : sctx.is_simt = false;
6940 : }
6941 :
6942 77134 : if (sctx.lane || sctx.is_simt)
6943 : {
6944 4003 : uid = create_tmp_var (ptr_type_node, "simduid");
6945 : /* Don't want uninit warnings on simduid, it is always uninitialized,
6946 : but we use it not for the value, but for the DECL_UID only. */
6947 4003 : suppress_warning (uid, OPT_Wuninitialized);
6948 4003 : c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SIMDUID_);
6949 4003 : OMP_CLAUSE__SIMDUID__DECL (c) = uid;
6950 4003 : OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
6951 4003 : gimple_omp_for_set_clauses (ctx->stmt, c);
6952 : }
6953 : /* Emit calls denoting privatized variables and initializing a pointer to
6954 : structure that holds private variables as fields after ompdevlow pass. */
6955 77134 : if (sctx.is_simt)
6956 : {
6957 0 : sctx.simt_eargs[0] = uid;
6958 0 : gimple *g
6959 0 : = gimple_build_call_internal_vec (IFN_GOMP_SIMT_ENTER, sctx.simt_eargs);
6960 0 : gimple_call_set_lhs (g, uid);
6961 0 : gimple_seq_add_stmt (ilist, g);
6962 0 : sctx.simt_eargs.release ();
6963 :
6964 0 : simtrec = create_tmp_var (ptr_type_node, ".omp_simt");
6965 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_ENTER_ALLOC, 1, uid);
6966 0 : gimple_call_set_lhs (g, simtrec);
6967 0 : gimple_seq_add_stmt (ilist, g);
6968 : }
6969 77134 : if (sctx.lane)
6970 : {
6971 7381 : gimple *g = gimple_build_call_internal (IFN_GOMP_SIMD_LANE,
6972 : 2 + (nonconst_simd_if != NULL),
6973 : uid, integer_zero_node,
6974 : nonconst_simd_if);
6975 4003 : gimple_call_set_lhs (g, sctx.lane);
6976 4003 : gimple_stmt_iterator gsi = gsi_start (*gimple_omp_body_ptr (ctx->stmt));
6977 4003 : gsi_insert_before_without_update (&gsi, g, GSI_SAME_STMT);
6978 4003 : g = gimple_build_assign (sctx.lane, INTEGER_CST,
6979 : build_int_cst (unsigned_type_node, 0));
6980 4003 : gimple_seq_add_stmt (ilist, g);
6981 4003 : if (sctx.lastlane)
6982 : {
6983 184 : g = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
6984 : 2, uid, sctx.lane);
6985 184 : gimple_call_set_lhs (g, sctx.lastlane);
6986 184 : gimple_seq_add_stmt (dlist, g);
6987 184 : gimple_seq_add_seq (dlist, llist[3]);
6988 : }
6989 : /* Emit reductions across SIMT lanes in log_2(simt_vf) steps. */
6990 4003 : if (llist[2])
6991 : {
6992 0 : tree simt_vf = create_tmp_var (unsigned_type_node);
6993 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_VF, 0);
6994 0 : gimple_call_set_lhs (g, simt_vf);
6995 0 : gimple_seq_add_stmt (dlist, g);
6996 :
6997 0 : tree t = build_int_cst (unsigned_type_node, 1);
6998 0 : g = gimple_build_assign (simt_lane, INTEGER_CST, t);
6999 0 : gimple_seq_add_stmt (dlist, g);
7000 :
7001 0 : t = build_int_cst (unsigned_type_node, 0);
7002 0 : g = gimple_build_assign (sctx.idx, INTEGER_CST, t);
7003 0 : gimple_seq_add_stmt (dlist, g);
7004 :
7005 0 : tree body = create_artificial_label (UNKNOWN_LOCATION);
7006 0 : tree header = create_artificial_label (UNKNOWN_LOCATION);
7007 0 : tree end = create_artificial_label (UNKNOWN_LOCATION);
7008 0 : gimple_seq_add_stmt (dlist, gimple_build_goto (header));
7009 0 : gimple_seq_add_stmt (dlist, gimple_build_label (body));
7010 :
7011 0 : gimple_seq_add_seq (dlist, llist[2]);
7012 :
7013 0 : g = gimple_build_assign (simt_lane, LSHIFT_EXPR, simt_lane, integer_one_node);
7014 0 : gimple_seq_add_stmt (dlist, g);
7015 :
7016 0 : gimple_seq_add_stmt (dlist, gimple_build_label (header));
7017 0 : g = gimple_build_cond (LT_EXPR, simt_lane, simt_vf, body, end);
7018 0 : gimple_seq_add_stmt (dlist, g);
7019 :
7020 0 : gimple_seq_add_stmt (dlist, gimple_build_label (end));
7021 : }
7022 12009 : for (int i = 0; i < 2; i++)
7023 8006 : if (llist[i])
7024 : {
7025 1716 : tree vf = create_tmp_var (unsigned_type_node);
7026 1716 : g = gimple_build_call_internal (IFN_GOMP_SIMD_VF, 1, uid);
7027 1716 : gimple_call_set_lhs (g, vf);
7028 1716 : gimple_seq *seq = i == 0 ? ilist : dlist;
7029 1716 : gimple_seq_add_stmt (seq, g);
7030 1716 : tree t = build_int_cst (unsigned_type_node, 0);
7031 1716 : g = gimple_build_assign (sctx.idx, INTEGER_CST, t);
7032 1716 : gimple_seq_add_stmt (seq, g);
7033 1716 : tree body = create_artificial_label (UNKNOWN_LOCATION);
7034 1716 : tree header = create_artificial_label (UNKNOWN_LOCATION);
7035 1716 : tree end = create_artificial_label (UNKNOWN_LOCATION);
7036 1716 : gimple_seq_add_stmt (seq, gimple_build_goto (header));
7037 1716 : gimple_seq_add_stmt (seq, gimple_build_label (body));
7038 1716 : gimple_seq_add_seq (seq, llist[i]);
7039 1716 : t = build_int_cst (unsigned_type_node, 1);
7040 1716 : g = gimple_build_assign (sctx.idx, PLUS_EXPR, sctx.idx, t);
7041 1716 : gimple_seq_add_stmt (seq, g);
7042 1716 : gimple_seq_add_stmt (seq, gimple_build_label (header));
7043 1716 : g = gimple_build_cond (LT_EXPR, sctx.idx, vf, body, end);
7044 1716 : gimple_seq_add_stmt (seq, g);
7045 1716 : gimple_seq_add_stmt (seq, gimple_build_label (end));
7046 : }
7047 : }
7048 77134 : if (sctx.is_simt)
7049 : {
7050 0 : gimple_seq_add_seq (dlist, sctx.simt_dlist);
7051 0 : gimple *g
7052 0 : = gimple_build_call_internal (IFN_GOMP_SIMT_EXIT, 1, simtrec);
7053 0 : gimple_seq_add_stmt (dlist, g);
7054 : }
7055 :
7056 : /* The copyin sequence is not to be executed by the main thread, since
7057 : that would result in self-copies. Perhaps not visible to scalars,
7058 : but it certainly is to C++ operator=. */
7059 77134 : if (copyin_seq)
7060 : {
7061 440 : x = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM),
7062 : 0);
7063 440 : x = build2 (NE_EXPR, boolean_type_node, x,
7064 440 : build_int_cst (TREE_TYPE (x), 0));
7065 440 : x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL);
7066 440 : gimplify_and_add (x, ilist);
7067 : }
7068 :
7069 : /* If any copyin variable is passed by reference, we must ensure the
7070 : master thread doesn't modify it before it is copied over in all
7071 : threads. Similarly for variables in both firstprivate and
7072 : lastprivate clauses we need to ensure the lastprivate copying
7073 : happens after firstprivate copying in all threads. And similarly
7074 : for UDRs if initializer expression refers to omp_orig. */
7075 77134 : if (copyin_by_ref || lastprivate_firstprivate
7076 75218 : || (reduction_omp_orig_ref
7077 101 : && !ctx->scan_inclusive
7078 101 : && !ctx->scan_exclusive))
7079 : {
7080 : /* Don't add any barrier for #pragma omp simd or
7081 : #pragma omp distribute. */
7082 2017 : if (!is_task_ctx (ctx)
7083 2017 : && (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
7084 1462 : || gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_FOR))
7085 : {
7086 880 : int barrier_kind;
7087 880 : switch (gimple_code (ctx->stmt))
7088 : {
7089 : case GIMPLE_OMP_SECTIONS:
7090 : case GIMPLE_OMP_SCOPE:
7091 : case GIMPLE_OMP_FOR:
7092 : barrier_kind = GOMP_BARRIER_IMPLICIT_WORKSHARE;
7093 : break;
7094 540 : case GIMPLE_OMP_PARALLEL:
7095 540 : case GIMPLE_OMP_TEAMS:
7096 540 : barrier_kind = GOMP_BARRIER_IMPLICIT_PARALLEL;
7097 540 : break;
7098 0 : default:
7099 0 : gcc_unreachable ();
7100 : }
7101 880 : gimple_seq_add_stmt (ilist,
7102 : omp_build_barrier (NULL_TREE, barrier_kind));
7103 : }
7104 : }
7105 :
7106 : /* If max_vf is non-zero, then we can use only a vectorization factor
7107 : up to the max_vf we chose. So stick it into the safelen clause. */
7108 77134 : if (maybe_ne (sctx.max_vf, 0U))
7109 : {
7110 5212 : tree c = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
7111 : OMP_CLAUSE_SAFELEN);
7112 5212 : poly_uint64 safe_len;
7113 5212 : if (c == NULL_TREE
7114 5212 : || (poly_int_tree_p (OMP_CLAUSE_SAFELEN_EXPR (c), &safe_len)
7115 613 : && maybe_gt (safe_len, sctx.max_vf)))
7116 : {
7117 5122 : c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_SAFELEN);
7118 5122 : OMP_CLAUSE_SAFELEN_EXPR (c) = build_int_cst (integer_type_node,
7119 5122 : sctx.max_vf);
7120 5122 : OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
7121 5122 : gimple_omp_for_set_clauses (ctx->stmt, c);
7122 : }
7123 : }
7124 77134 : }
7125 :
7126 : /* Create temporary variables for lastprivate(conditional:) implementation
7127 : in context CTX with CLAUSES. */
7128 :
7129 : static void
7130 47245 : lower_lastprivate_conditional_clauses (tree *clauses, omp_context *ctx)
7131 : {
7132 47245 : tree iter_type = NULL_TREE;
7133 47245 : tree cond_ptr = NULL_TREE;
7134 47245 : tree iter_var = NULL_TREE;
7135 47245 : bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
7136 47245 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD);
7137 47245 : tree next = *clauses;
7138 226762 : for (tree c = *clauses; c; c = OMP_CLAUSE_CHAIN (c))
7139 179517 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7140 179517 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
7141 : {
7142 380 : if (is_simd)
7143 : {
7144 107 : tree cc = omp_find_clause (next, OMP_CLAUSE__CONDTEMP_);
7145 107 : gcc_assert (cc);
7146 107 : if (iter_type == NULL_TREE)
7147 : {
7148 77 : iter_type = TREE_TYPE (OMP_CLAUSE_DECL (cc));
7149 77 : iter_var = create_tmp_var_raw (iter_type);
7150 77 : DECL_CONTEXT (iter_var) = current_function_decl;
7151 77 : DECL_SEEN_IN_BIND_EXPR_P (iter_var) = 1;
7152 77 : DECL_CHAIN (iter_var) = ctx->block_vars;
7153 77 : ctx->block_vars = iter_var;
7154 77 : tree c3
7155 77 : = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__CONDTEMP_);
7156 77 : OMP_CLAUSE__CONDTEMP__ITER (c3) = 1;
7157 77 : OMP_CLAUSE_DECL (c3) = iter_var;
7158 77 : OMP_CLAUSE_CHAIN (c3) = *clauses;
7159 77 : *clauses = c3;
7160 77 : ctx->lastprivate_conditional_map = new hash_map<tree, tree>;
7161 : }
7162 107 : next = OMP_CLAUSE_CHAIN (cc);
7163 107 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
7164 107 : tree v = lookup_decl (OMP_CLAUSE_DECL (cc), ctx);
7165 107 : ctx->lastprivate_conditional_map->put (o, v);
7166 107 : continue;
7167 107 : }
7168 273 : if (iter_type == NULL)
7169 : {
7170 199 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR)
7171 : {
7172 187 : struct omp_for_data fd;
7173 187 : omp_extract_for_data (as_a <gomp_for *> (ctx->stmt), &fd,
7174 : NULL);
7175 187 : iter_type = unsigned_type_for (fd.iter_type);
7176 : }
7177 12 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
7178 12 : iter_type = unsigned_type_node;
7179 199 : tree c2 = omp_find_clause (*clauses, OMP_CLAUSE__CONDTEMP_);
7180 199 : if (c2)
7181 : {
7182 97 : cond_ptr
7183 97 : = lookup_decl_in_outer_ctx (OMP_CLAUSE_DECL (c2), ctx);
7184 97 : OMP_CLAUSE_DECL (c2) = cond_ptr;
7185 : }
7186 : else
7187 : {
7188 102 : cond_ptr = create_tmp_var_raw (build_pointer_type (iter_type));
7189 102 : DECL_CONTEXT (cond_ptr) = current_function_decl;
7190 102 : DECL_SEEN_IN_BIND_EXPR_P (cond_ptr) = 1;
7191 102 : DECL_CHAIN (cond_ptr) = ctx->block_vars;
7192 102 : ctx->block_vars = cond_ptr;
7193 102 : c2 = build_omp_clause (UNKNOWN_LOCATION,
7194 : OMP_CLAUSE__CONDTEMP_);
7195 102 : OMP_CLAUSE_DECL (c2) = cond_ptr;
7196 102 : OMP_CLAUSE_CHAIN (c2) = *clauses;
7197 102 : *clauses = c2;
7198 : }
7199 199 : iter_var = create_tmp_var_raw (iter_type);
7200 199 : DECL_CONTEXT (iter_var) = current_function_decl;
7201 199 : DECL_SEEN_IN_BIND_EXPR_P (iter_var) = 1;
7202 199 : DECL_CHAIN (iter_var) = ctx->block_vars;
7203 199 : ctx->block_vars = iter_var;
7204 199 : tree c3
7205 199 : = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__CONDTEMP_);
7206 199 : OMP_CLAUSE__CONDTEMP__ITER (c3) = 1;
7207 199 : OMP_CLAUSE_DECL (c3) = iter_var;
7208 199 : OMP_CLAUSE_CHAIN (c3) = OMP_CLAUSE_CHAIN (c2);
7209 199 : OMP_CLAUSE_CHAIN (c2) = c3;
7210 199 : ctx->lastprivate_conditional_map = new hash_map<tree, tree>;
7211 : }
7212 273 : tree v = create_tmp_var_raw (iter_type);
7213 273 : DECL_CONTEXT (v) = current_function_decl;
7214 273 : DECL_SEEN_IN_BIND_EXPR_P (v) = 1;
7215 273 : DECL_CHAIN (v) = ctx->block_vars;
7216 273 : ctx->block_vars = v;
7217 273 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
7218 273 : ctx->lastprivate_conditional_map->put (o, v);
7219 : }
7220 47245 : }
7221 :
7222 :
7223 : /* Generate code to implement the LASTPRIVATE clauses. This is used for
7224 : both parallel and workshare constructs. PREDICATE may be NULL if it's
7225 : always true. BODY_P is the sequence to insert early initialization
7226 : if needed, STMT_LIST is where the non-conditional lastprivate handling
7227 : goes into and CSTMT_LIST is a sequence that needs to be run in a critical
7228 : section. */
7229 :
7230 : static void
7231 47245 : lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *body_p,
7232 : gimple_seq *stmt_list, gimple_seq *cstmt_list,
7233 : omp_context *ctx)
7234 : {
7235 47245 : tree x, c, label = NULL, orig_clauses = clauses;
7236 47245 : bool par_clauses = false;
7237 47245 : tree simduid = NULL, lastlane = NULL, simtcond = NULL, simtlast = NULL;
7238 47245 : unsigned HOST_WIDE_INT conditional_off = 0;
7239 47245 : gimple_seq post_stmt_list = NULL;
7240 :
7241 : /* Early exit if there are no lastprivate or linear clauses. */
7242 196287 : for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
7243 161493 : if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LASTPRIVATE
7244 161493 : || (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LINEAR
7245 6757 : && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses)))
7246 : break;
7247 47245 : if (clauses == NULL)
7248 : {
7249 : /* If this was a workshare clause, see if it had been combined
7250 : with its parallel. In that case, look for the clauses on the
7251 : parallel statement itself. */
7252 34794 : if (is_parallel_ctx (ctx))
7253 30794 : return;
7254 :
7255 34794 : ctx = ctx->outer;
7256 34794 : if (ctx == NULL || !is_parallel_ctx (ctx))
7257 : return;
7258 :
7259 12209 : clauses = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
7260 : OMP_CLAUSE_LASTPRIVATE);
7261 12209 : if (clauses == NULL)
7262 : return;
7263 : par_clauses = true;
7264 : }
7265 :
7266 16451 : bool maybe_simt = false;
7267 16451 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
7268 16451 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
7269 : {
7270 7424 : maybe_simt = omp_find_clause (orig_clauses, OMP_CLAUSE__SIMT_);
7271 7424 : simduid = omp_find_clause (orig_clauses, OMP_CLAUSE__SIMDUID_);
7272 7424 : if (simduid)
7273 3551 : simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
7274 : }
7275 :
7276 16451 : if (predicate)
7277 : {
7278 16328 : gcond *stmt;
7279 16328 : tree label_true, arm1, arm2;
7280 16328 : enum tree_code pred_code = TREE_CODE (predicate);
7281 :
7282 16328 : label = create_artificial_label (UNKNOWN_LOCATION);
7283 16328 : label_true = create_artificial_label (UNKNOWN_LOCATION);
7284 16328 : if (TREE_CODE_CLASS (pred_code) == tcc_comparison)
7285 : {
7286 16328 : arm1 = TREE_OPERAND (predicate, 0);
7287 16328 : arm2 = TREE_OPERAND (predicate, 1);
7288 16328 : gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
7289 16328 : gimplify_expr (&arm2, stmt_list, NULL, is_gimple_val, fb_rvalue);
7290 : }
7291 : else
7292 : {
7293 0 : arm1 = predicate;
7294 0 : gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
7295 0 : arm2 = boolean_false_node;
7296 0 : pred_code = NE_EXPR;
7297 : }
7298 16328 : if (maybe_simt)
7299 : {
7300 0 : c = build2 (pred_code, boolean_type_node, arm1, arm2);
7301 0 : c = fold_convert (integer_type_node, c);
7302 0 : simtcond = create_tmp_var (integer_type_node);
7303 0 : gimplify_assign (simtcond, c, stmt_list);
7304 0 : gcall *g = gimple_build_call_internal (IFN_GOMP_SIMT_VOTE_ANY,
7305 : 1, simtcond);
7306 0 : c = create_tmp_var (integer_type_node);
7307 0 : gimple_call_set_lhs (g, c);
7308 0 : gimple_seq_add_stmt (stmt_list, g);
7309 0 : stmt = gimple_build_cond (NE_EXPR, c, integer_zero_node,
7310 : label_true, label);
7311 : }
7312 : else
7313 16328 : stmt = gimple_build_cond (pred_code, arm1, arm2, label_true, label);
7314 16328 : gimple_seq_add_stmt (stmt_list, stmt);
7315 16328 : gimple_seq_add_stmt (stmt_list, gimple_build_label (label_true));
7316 : }
7317 :
7318 16451 : tree cond_ptr = NULL_TREE;
7319 61204 : for (c = clauses; c ;)
7320 : {
7321 56011 : tree var, new_var;
7322 56011 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
7323 56011 : gimple_seq *this_stmt_list = stmt_list;
7324 56011 : tree lab2 = NULL_TREE;
7325 :
7326 56011 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7327 21741 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c)
7328 380 : && ctx->lastprivate_conditional_map
7329 56378 : && !ctx->combined_into_simd_safelen1)
7330 : {
7331 310 : gcc_assert (body_p);
7332 310 : if (simduid)
7333 37 : goto next;
7334 273 : if (cond_ptr == NULL_TREE)
7335 : {
7336 199 : cond_ptr = omp_find_clause (orig_clauses, OMP_CLAUSE__CONDTEMP_);
7337 199 : cond_ptr = OMP_CLAUSE_DECL (cond_ptr);
7338 : }
7339 273 : tree type = TREE_TYPE (TREE_TYPE (cond_ptr));
7340 273 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
7341 273 : tree v = *ctx->lastprivate_conditional_map->get (o);
7342 273 : gimplify_assign (v, build_zero_cst (type), body_p);
7343 273 : this_stmt_list = cstmt_list;
7344 273 : tree mem;
7345 273 : if (POINTER_TYPE_P (TREE_TYPE (cond_ptr)))
7346 : {
7347 136 : mem = build2 (MEM_REF, type, cond_ptr,
7348 136 : build_int_cst (TREE_TYPE (cond_ptr),
7349 136 : conditional_off));
7350 136 : conditional_off += tree_to_uhwi (TYPE_SIZE_UNIT (type));
7351 : }
7352 : else
7353 137 : mem = build4 (ARRAY_REF, type, cond_ptr,
7354 137 : size_int (conditional_off++), NULL_TREE, NULL_TREE);
7355 273 : tree mem2 = copy_node (mem);
7356 273 : gimple_seq seq = NULL;
7357 273 : mem = force_gimple_operand (mem, &seq, true, NULL_TREE);
7358 273 : gimple_seq_add_seq (this_stmt_list, seq);
7359 273 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
7360 273 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
7361 273 : gimple *g = gimple_build_cond (GT_EXPR, v, mem, lab1, lab2);
7362 273 : gimple_seq_add_stmt (this_stmt_list, g);
7363 273 : gimple_seq_add_stmt (this_stmt_list, gimple_build_label (lab1));
7364 273 : gimplify_assign (mem2, v, this_stmt_list);
7365 : }
7366 55701 : else if (predicate
7367 55455 : && ctx->combined_into_simd_safelen1
7368 116 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7369 57 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c)
7370 55758 : && ctx->lastprivate_conditional_map)
7371 : this_stmt_list = &post_stmt_list;
7372 :
7373 55974 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7374 55974 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
7375 4843 : && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c)))
7376 : {
7377 26547 : var = OMP_CLAUSE_DECL (c);
7378 26547 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7379 21704 : && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)
7380 27578 : && is_taskloop_ctx (ctx))
7381 : {
7382 312 : gcc_checking_assert (ctx->outer && is_task_ctx (ctx->outer));
7383 312 : new_var = lookup_decl (var, ctx->outer);
7384 : }
7385 : else
7386 : {
7387 26235 : new_var = lookup_decl (var, ctx);
7388 : /* Avoid uninitialized warnings for lastprivate and
7389 : for linear iterators. */
7390 26235 : if (predicate
7391 52277 : && (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7392 4843 : || OMP_CLAUSE_LINEAR_NO_COPYIN (c)))
7393 24797 : suppress_warning (new_var, OPT_Wuninitialized);
7394 : }
7395 :
7396 26547 : if (!maybe_simt && simduid && DECL_HAS_VALUE_EXPR_P (new_var))
7397 : {
7398 5864 : tree val = DECL_VALUE_EXPR (new_var);
7399 5864 : if (TREE_CODE (val) == ARRAY_REF
7400 5816 : && VAR_P (TREE_OPERAND (val, 0))
7401 11680 : && lookup_attribute ("omp simd array",
7402 5816 : DECL_ATTRIBUTES (TREE_OPERAND (val,
7403 : 0))))
7404 : {
7405 5816 : if (lastlane == NULL)
7406 : {
7407 2759 : lastlane = create_tmp_var (unsigned_type_node);
7408 2759 : gcall *g
7409 2759 : = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
7410 : 2, simduid,
7411 2759 : TREE_OPERAND (val, 1));
7412 2759 : gimple_call_set_lhs (g, lastlane);
7413 2759 : gimple_seq_add_stmt (this_stmt_list, g);
7414 : }
7415 5816 : new_var = build4 (ARRAY_REF, TREE_TYPE (val),
7416 5816 : TREE_OPERAND (val, 0), lastlane,
7417 : NULL_TREE, NULL_TREE);
7418 5816 : TREE_THIS_NOTRAP (new_var) = 1;
7419 : }
7420 : }
7421 20683 : else if (maybe_simt)
7422 : {
7423 0 : tree val = (DECL_HAS_VALUE_EXPR_P (new_var)
7424 0 : ? DECL_VALUE_EXPR (new_var)
7425 : : new_var);
7426 0 : if (simtlast == NULL)
7427 : {
7428 0 : simtlast = create_tmp_var (unsigned_type_node);
7429 0 : gcall *g = gimple_build_call_internal
7430 0 : (IFN_GOMP_SIMT_LAST_LANE, 1, simtcond);
7431 0 : gimple_call_set_lhs (g, simtlast);
7432 0 : gimple_seq_add_stmt (this_stmt_list, g);
7433 : }
7434 0 : x = build_call_expr_internal_loc
7435 0 : (UNKNOWN_LOCATION, IFN_GOMP_SIMT_XCHG_IDX,
7436 0 : TREE_TYPE (val), 2, val, simtlast);
7437 0 : new_var = unshare_expr (new_var);
7438 0 : gimplify_assign (new_var, x, this_stmt_list);
7439 0 : new_var = unshare_expr (new_var);
7440 : }
7441 :
7442 26547 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7443 26547 : && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
7444 : {
7445 7117 : lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
7446 7117 : gimple_seq_add_seq (this_stmt_list,
7447 7117 : OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c));
7448 7117 : OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL;
7449 : }
7450 19430 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
7451 19430 : && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
7452 : {
7453 426 : lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
7454 426 : gimple_seq_add_seq (this_stmt_list,
7455 426 : OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c));
7456 426 : OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c) = NULL;
7457 : }
7458 :
7459 26547 : x = NULL_TREE;
7460 26547 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7461 21704 : && OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c)
7462 26607 : && is_taskloop_ctx (ctx))
7463 : {
7464 76 : tree ovar = maybe_lookup_decl_in_outer_ctx (var,
7465 38 : ctx->outer->outer);
7466 38 : if (is_global_var (ovar))
7467 7 : x = ovar;
7468 : }
7469 7 : if (!x)
7470 26540 : x = build_outer_var_ref (var, ctx, OMP_CLAUSE_LASTPRIVATE);
7471 26547 : if (omp_privatize_by_reference (var))
7472 657 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
7473 26547 : x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var);
7474 26547 : gimplify_and_add (x, this_stmt_list);
7475 :
7476 26547 : if (lab2)
7477 273 : gimple_seq_add_stmt (this_stmt_list, gimple_build_label (lab2));
7478 : }
7479 :
7480 56011 : next:
7481 56011 : c = OMP_CLAUSE_CHAIN (c);
7482 56011 : if (c == NULL && !par_clauses)
7483 : {
7484 : /* If this was a workshare clause, see if it had been combined
7485 : with its parallel. In that case, continue looking for the
7486 : clauses also on the parallel statement itself. */
7487 12451 : if (is_parallel_ctx (ctx))
7488 : break;
7489 :
7490 12451 : ctx = ctx->outer;
7491 12451 : if (ctx == NULL || !is_parallel_ctx (ctx))
7492 : break;
7493 :
7494 1193 : c = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
7495 : OMP_CLAUSE_LASTPRIVATE);
7496 1193 : par_clauses = true;
7497 : }
7498 : }
7499 :
7500 16451 : if (label)
7501 16328 : gimple_seq_add_stmt (stmt_list, gimple_build_label (label));
7502 16451 : gimple_seq_add_seq (stmt_list, post_stmt_list);
7503 : }
7504 :
7505 : /* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL
7506 : (which might be a placeholder). INNER is true if this is an inner
7507 : axis of a multi-axis loop. FORK and JOIN are (optional) fork and
7508 : join markers. Generate the before-loop forking sequence in
7509 : FORK_SEQ and the after-loop joining sequence to JOIN_SEQ. The
7510 : general form of these sequences is
7511 :
7512 : GOACC_REDUCTION_SETUP
7513 : GOACC_FORK
7514 : GOACC_REDUCTION_INIT
7515 : ...
7516 : GOACC_REDUCTION_FINI
7517 : GOACC_JOIN
7518 : GOACC_REDUCTION_TEARDOWN. */
7519 :
7520 : static void
7521 25946 : lower_oacc_reductions (location_t loc, tree clauses, tree level, bool inner,
7522 : gcall *fork, gcall *private_marker, gcall *join,
7523 : gimple_seq *fork_seq, gimple_seq *join_seq,
7524 : omp_context *ctx)
7525 : {
7526 25946 : gimple_seq before_fork = NULL;
7527 25946 : gimple_seq after_fork = NULL;
7528 25946 : gimple_seq before_join = NULL;
7529 25946 : gimple_seq after_join = NULL;
7530 25946 : tree init_code = NULL_TREE, fini_code = NULL_TREE,
7531 25946 : setup_code = NULL_TREE, teardown_code = NULL_TREE;
7532 25946 : unsigned offset = 0;
7533 :
7534 96516 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
7535 70570 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
7536 : {
7537 : /* No 'reduction' clauses on OpenACC 'kernels'. */
7538 7751 : gcc_checking_assert (!is_oacc_kernels (ctx));
7539 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
7540 7751 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
7541 :
7542 7751 : tree orig = OMP_CLAUSE_DECL (c);
7543 7751 : tree orig_clause;
7544 7751 : tree var = maybe_lookup_decl (orig, ctx);
7545 7751 : tree ref_to_res = NULL_TREE;
7546 7751 : tree incoming, outgoing, v1, v2, v3;
7547 7751 : bool is_private = false;
7548 :
7549 7751 : enum tree_code rcode = OMP_CLAUSE_REDUCTION_CODE (c);
7550 7751 : if (rcode == MINUS_EXPR)
7551 : rcode = PLUS_EXPR;
7552 6609 : else if (rcode == TRUTH_ANDIF_EXPR)
7553 : rcode = BIT_AND_EXPR;
7554 6431 : else if (rcode == TRUTH_ORIF_EXPR)
7555 433 : rcode = BIT_IOR_EXPR;
7556 7751 : tree op = build_int_cst (unsigned_type_node, rcode);
7557 :
7558 7751 : if (!var)
7559 4 : var = orig;
7560 :
7561 7751 : incoming = outgoing = var;
7562 :
7563 7751 : if (!inner)
7564 : {
7565 : /* See if an outer construct also reduces this variable. */
7566 : omp_context *outer = ctx;
7567 :
7568 7385 : while (omp_context *probe = outer->outer)
7569 : {
7570 4708 : enum gimple_code type = gimple_code (probe->stmt);
7571 4708 : tree cls;
7572 :
7573 4708 : switch (type)
7574 : {
7575 2220 : case GIMPLE_OMP_FOR:
7576 2220 : cls = gimple_omp_for_clauses (probe->stmt);
7577 2220 : break;
7578 :
7579 2488 : case GIMPLE_OMP_TARGET:
7580 : /* No 'reduction' clauses inside OpenACC 'kernels'
7581 : regions. */
7582 2488 : gcc_checking_assert (!is_oacc_kernels (probe));
7583 :
7584 2488 : if (!is_gimple_omp_offloaded (probe->stmt))
7585 26 : goto do_lookup;
7586 :
7587 2462 : cls = gimple_omp_target_clauses (probe->stmt);
7588 2462 : break;
7589 :
7590 0 : default:
7591 0 : goto do_lookup;
7592 : }
7593 :
7594 14249 : outer = probe;
7595 14249 : for (; cls; cls = OMP_CLAUSE_CHAIN (cls))
7596 11632 : if (OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_REDUCTION
7597 11632 : && orig == OMP_CLAUSE_DECL (cls))
7598 : {
7599 1772 : incoming = outgoing = lookup_decl (orig, probe);
7600 1772 : goto has_outer_reduction;
7601 : }
7602 9860 : else if ((OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_FIRSTPRIVATE
7603 8971 : || OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_PRIVATE)
7604 11164 : && orig == OMP_CLAUSE_DECL (cls))
7605 : {
7606 293 : is_private = true;
7607 293 : goto do_lookup;
7608 : }
7609 : }
7610 :
7611 2677 : do_lookup:
7612 : /* This is the outermost construct with this reduction,
7613 : see if there's a mapping for it. */
7614 2996 : orig_clause = NULL_TREE;
7615 2996 : if (gimple_code (outer->stmt) == GIMPLE_OMP_TARGET)
7616 7557 : for (tree cls = gimple_omp_target_clauses (outer->stmt);
7617 7557 : cls; cls = OMP_CLAUSE_CHAIN (cls))
7618 7250 : if (OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_MAP
7619 4823 : && orig == OMP_CLAUSE_DECL (cls)
7620 9805 : && maybe_lookup_field (cls, outer))
7621 : {
7622 : orig_clause = cls;
7623 : break;
7624 : }
7625 2996 : if (orig_clause != NULL_TREE && !is_private)
7626 : {
7627 2553 : ref_to_res = build_receiver_ref (orig_clause, false, outer);
7628 2553 : if (omp_privatize_by_reference (orig))
7629 79 : ref_to_res = build_simple_mem_ref (ref_to_res);
7630 :
7631 2553 : tree type = TREE_TYPE (var);
7632 2553 : if (POINTER_TYPE_P (type))
7633 79 : type = TREE_TYPE (type);
7634 :
7635 2553 : outgoing = var;
7636 2553 : incoming = omp_reduction_init_op (loc, rcode, type);
7637 : }
7638 : else
7639 : {
7640 : /* Try to look at enclosing contexts for reduction var,
7641 : use original if no mapping found. */
7642 443 : tree t = NULL_TREE;
7643 443 : omp_context *c = ctx->outer;
7644 895 : while (c && !t)
7645 : {
7646 452 : t = maybe_lookup_decl (orig, c);
7647 452 : c = c->outer;
7648 : }
7649 443 : incoming = outgoing = (t ? t : orig);
7650 : }
7651 :
7652 2553 : has_outer_reduction:;
7653 : }
7654 :
7655 4325 : if (!ref_to_res)
7656 5198 : ref_to_res = integer_zero_node;
7657 :
7658 7751 : if (omp_privatize_by_reference (orig))
7659 : {
7660 159 : tree type = TREE_TYPE (var);
7661 159 : const char *id = IDENTIFIER_POINTER (DECL_NAME (var));
7662 :
7663 159 : if (!inner)
7664 : {
7665 110 : tree x = create_tmp_var (TREE_TYPE (type), id);
7666 110 : gimplify_assign (var, build_fold_addr_expr (x), fork_seq);
7667 : }
7668 :
7669 159 : v1 = create_tmp_var (type, id);
7670 159 : v2 = create_tmp_var (type, id);
7671 159 : v3 = create_tmp_var (type, id);
7672 :
7673 159 : gimplify_assign (v1, var, fork_seq);
7674 159 : gimplify_assign (v2, var, fork_seq);
7675 159 : gimplify_assign (v3, var, fork_seq);
7676 :
7677 159 : var = build_simple_mem_ref (var);
7678 159 : v1 = build_simple_mem_ref (v1);
7679 159 : v2 = build_simple_mem_ref (v2);
7680 159 : v3 = build_simple_mem_ref (v3);
7681 159 : outgoing = build_simple_mem_ref (outgoing);
7682 :
7683 159 : if (!TREE_CONSTANT (incoming))
7684 80 : incoming = build_simple_mem_ref (incoming);
7685 : }
7686 : else
7687 : /* Note that 'var' might be a mem ref. */
7688 : v1 = v2 = v3 = var;
7689 :
7690 : /* Determine position in reduction buffer, which may be used
7691 : by target. The parser has ensured that this is not a
7692 : variable-sized type. */
7693 7751 : fixed_size_mode mode
7694 7751 : = as_a <fixed_size_mode> (TYPE_MODE (TREE_TYPE (var)));
7695 7751 : unsigned align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT;
7696 7751 : offset = (offset + align - 1) & ~(align - 1);
7697 7751 : tree off = build_int_cst (sizetype, offset);
7698 7751 : offset += GET_MODE_SIZE (mode);
7699 :
7700 7751 : if (!init_code)
7701 : {
7702 6969 : init_code = build_int_cst (integer_type_node,
7703 : IFN_GOACC_REDUCTION_INIT);
7704 6969 : fini_code = build_int_cst (integer_type_node,
7705 : IFN_GOACC_REDUCTION_FINI);
7706 6969 : setup_code = build_int_cst (integer_type_node,
7707 : IFN_GOACC_REDUCTION_SETUP);
7708 6969 : teardown_code = build_int_cst (integer_type_node,
7709 : IFN_GOACC_REDUCTION_TEARDOWN);
7710 : }
7711 :
7712 7751 : tree setup_call
7713 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7714 7751 : TREE_TYPE (var), 6, setup_code,
7715 : unshare_expr (ref_to_res),
7716 : unshare_expr (incoming),
7717 : level, op, off);
7718 7751 : tree init_call
7719 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7720 7751 : TREE_TYPE (var), 6, init_code,
7721 : unshare_expr (ref_to_res),
7722 : unshare_expr (v1), level, op, off);
7723 7751 : tree fini_call
7724 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7725 7751 : TREE_TYPE (var), 6, fini_code,
7726 : unshare_expr (ref_to_res),
7727 : unshare_expr (v2), level, op, off);
7728 7751 : tree teardown_call
7729 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7730 7751 : TREE_TYPE (var), 6, teardown_code,
7731 : ref_to_res, unshare_expr (v3),
7732 : level, op, off);
7733 :
7734 7751 : gimplify_assign (unshare_expr (v1), setup_call, &before_fork);
7735 7751 : gimplify_assign (unshare_expr (v2), init_call, &after_fork);
7736 7751 : gimplify_assign (unshare_expr (v3), fini_call, &before_join);
7737 7751 : gimplify_assign (unshare_expr (outgoing), teardown_call, &after_join);
7738 : }
7739 :
7740 : /* Now stitch things together. */
7741 25946 : gimple_seq_add_seq (fork_seq, before_fork);
7742 25946 : if (private_marker)
7743 259 : gimple_seq_add_stmt (fork_seq, private_marker);
7744 25946 : if (fork)
7745 16551 : gimple_seq_add_stmt (fork_seq, fork);
7746 25946 : gimple_seq_add_seq (fork_seq, after_fork);
7747 :
7748 25946 : gimple_seq_add_seq (join_seq, before_join);
7749 25946 : if (join)
7750 16551 : gimple_seq_add_stmt (join_seq, join);
7751 25946 : gimple_seq_add_seq (join_seq, after_join);
7752 25946 : }
7753 :
7754 : /* Generate code to implement the REDUCTION clauses, append it
7755 : to STMT_SEQP. CLIST if non-NULL is a pointer to a sequence
7756 : that should be emitted also inside of the critical section,
7757 : in that case clear *CLIST afterwards, otherwise leave it as is
7758 : and let the caller emit it itself. */
7759 :
7760 : static void
7761 71945 : lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp,
7762 : gimple_seq *clist, omp_context *ctx)
7763 : {
7764 71945 : gimple_seq sub_seq = NULL;
7765 71945 : gimple *stmt;
7766 71945 : tree x, c;
7767 71945 : int count = 0;
7768 :
7769 : /* OpenACC loop reductions are handled elsewhere. */
7770 71945 : if (is_gimple_omp_oacc (ctx->stmt))
7771 70892 : return;
7772 :
7773 : /* SIMD reductions are handled in lower_rec_input_clauses. */
7774 60620 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
7775 60620 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
7776 : return;
7777 :
7778 : /* inscan reductions are handled elsewhere. */
7779 51265 : if (ctx->scan_inclusive || ctx->scan_exclusive)
7780 : return;
7781 :
7782 : /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
7783 : update in that case, otherwise use a lock. */
7784 241714 : for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c))
7785 191482 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
7786 191482 : && !OMP_CLAUSE_REDUCTION_TASK (c))
7787 : {
7788 4618 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)
7789 4618 : || TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF)
7790 : {
7791 : /* Never use OMP_ATOMIC for array reductions or UDRs. */
7792 : count = -1;
7793 : break;
7794 : }
7795 3762 : count++;
7796 : }
7797 :
7798 51088 : if (count == 0)
7799 : return;
7800 :
7801 14929 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
7802 : {
7803 13876 : tree var, ref, new_var, orig_var;
7804 13876 : enum tree_code code;
7805 13876 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
7806 :
7807 22334 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
7808 13876 : || OMP_CLAUSE_REDUCTION_TASK (c))
7809 8458 : continue;
7810 :
7811 5418 : enum omp_clause_code ccode = OMP_CLAUSE_REDUCTION;
7812 5418 : orig_var = var = OMP_CLAUSE_DECL (c);
7813 5418 : if (TREE_CODE (var) == MEM_REF)
7814 : {
7815 689 : var = TREE_OPERAND (var, 0);
7816 689 : if (TREE_CODE (var) == POINTER_PLUS_EXPR)
7817 50 : var = TREE_OPERAND (var, 0);
7818 689 : if (TREE_CODE (var) == ADDR_EXPR)
7819 304 : var = TREE_OPERAND (var, 0);
7820 : else
7821 : {
7822 : /* If this is a pointer or referenced based array
7823 : section, the var could be private in the outer
7824 : context e.g. on orphaned loop construct. Pretend this
7825 : is private variable's outer reference. */
7826 385 : ccode = OMP_CLAUSE_PRIVATE;
7827 385 : if (INDIRECT_REF_P (var))
7828 54 : var = TREE_OPERAND (var, 0);
7829 : }
7830 689 : orig_var = var;
7831 689 : if (is_variable_sized (var))
7832 : {
7833 25 : gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
7834 25 : var = DECL_VALUE_EXPR (var);
7835 25 : gcc_assert (INDIRECT_REF_P (var));
7836 25 : var = TREE_OPERAND (var, 0);
7837 25 : gcc_assert (DECL_P (var));
7838 : }
7839 : }
7840 5418 : new_var = lookup_decl (var, ctx);
7841 5418 : if (var == OMP_CLAUSE_DECL (c)
7842 5418 : && omp_privatize_by_reference (var))
7843 140 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
7844 5418 : ref = build_outer_var_ref (var, ctx, ccode);
7845 5418 : code = OMP_CLAUSE_REDUCTION_CODE (c);
7846 :
7847 : /* reduction(-:var) sums up the partial results, so it acts
7848 : identically to reduction(+:var). */
7849 5418 : if (code == MINUS_EXPR)
7850 57 : code = PLUS_EXPR;
7851 :
7852 5418 : bool is_truth_op = (code == TRUTH_ANDIF_EXPR || code == TRUTH_ORIF_EXPR);
7853 5418 : if (count == 1)
7854 : {
7855 3313 : tree addr = build_fold_addr_expr_loc (clause_loc, ref);
7856 :
7857 3313 : addr = save_expr (addr);
7858 3313 : ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr);
7859 3313 : tree new_var2 = new_var;
7860 3313 : tree ref2 = ref;
7861 3313 : if (is_truth_op)
7862 : {
7863 784 : tree zero = build_zero_cst (TREE_TYPE (new_var));
7864 784 : new_var2 = fold_build2_loc (clause_loc, NE_EXPR,
7865 : boolean_type_node, new_var, zero);
7866 784 : ref2 = fold_build2_loc (clause_loc, NE_EXPR, boolean_type_node,
7867 : ref, zero);
7868 : }
7869 3313 : x = fold_build2_loc (clause_loc, code, TREE_TYPE (new_var2), ref2,
7870 : new_var2);
7871 3313 : if (is_truth_op)
7872 784 : x = fold_convert (TREE_TYPE (new_var), x);
7873 3313 : x = build2 (OMP_ATOMIC, void_type_node, addr, x);
7874 3313 : OMP_ATOMIC_MEMORY_ORDER (x) = OMP_MEMORY_ORDER_RELAXED;
7875 3313 : gimplify_and_add (x, stmt_seqp);
7876 3313 : return;
7877 : }
7878 2105 : else if (TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF)
7879 : {
7880 689 : tree d = OMP_CLAUSE_DECL (c);
7881 689 : tree type = TREE_TYPE (d);
7882 689 : tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
7883 689 : tree i = create_tmp_var (TREE_TYPE (v));
7884 689 : tree ptype = build_pointer_type (TREE_TYPE (type));
7885 689 : tree bias = TREE_OPERAND (d, 1);
7886 689 : d = TREE_OPERAND (d, 0);
7887 689 : if (TREE_CODE (d) == POINTER_PLUS_EXPR)
7888 : {
7889 50 : tree b = TREE_OPERAND (d, 1);
7890 50 : b = maybe_lookup_decl (b, ctx);
7891 50 : if (b == NULL)
7892 : {
7893 2 : b = TREE_OPERAND (d, 1);
7894 2 : b = maybe_lookup_decl_in_outer_ctx (b, ctx);
7895 : }
7896 50 : if (integer_zerop (bias))
7897 : bias = b;
7898 : else
7899 : {
7900 0 : bias = fold_convert_loc (clause_loc, TREE_TYPE (b), bias);
7901 0 : bias = fold_build2_loc (clause_loc, PLUS_EXPR,
7902 0 : TREE_TYPE (b), b, bias);
7903 : }
7904 50 : d = TREE_OPERAND (d, 0);
7905 : }
7906 : /* For ref build_outer_var_ref already performs this, so
7907 : only new_var needs a dereference. */
7908 689 : if (INDIRECT_REF_P (d))
7909 : {
7910 54 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
7911 54 : gcc_assert (omp_privatize_by_reference (var)
7912 : && var == orig_var);
7913 : }
7914 635 : else if (TREE_CODE (d) == ADDR_EXPR)
7915 : {
7916 304 : if (orig_var == var)
7917 : {
7918 279 : new_var = build_fold_addr_expr (new_var);
7919 279 : ref = build_fold_addr_expr (ref);
7920 : }
7921 : }
7922 : else
7923 : {
7924 331 : gcc_assert (orig_var == var);
7925 331 : if (omp_privatize_by_reference (var))
7926 79 : ref = build_fold_addr_expr (ref);
7927 : }
7928 689 : if (DECL_P (v))
7929 : {
7930 93 : tree t = maybe_lookup_decl (v, ctx);
7931 93 : if (t)
7932 88 : v = t;
7933 : else
7934 5 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
7935 93 : gimplify_expr (&v, stmt_seqp, NULL, is_gimple_val, fb_rvalue);
7936 : }
7937 689 : if (!integer_zerop (bias))
7938 : {
7939 402 : bias = fold_convert_loc (clause_loc, sizetype, bias);
7940 804 : new_var = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
7941 402 : TREE_TYPE (new_var), new_var,
7942 : unshare_expr (bias));
7943 402 : ref = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
7944 402 : TREE_TYPE (ref), ref, bias);
7945 : }
7946 689 : new_var = fold_convert_loc (clause_loc, ptype, new_var);
7947 689 : ref = fold_convert_loc (clause_loc, ptype, ref);
7948 689 : tree m = create_tmp_var (ptype);
7949 689 : gimplify_assign (m, new_var, stmt_seqp);
7950 689 : new_var = m;
7951 689 : m = create_tmp_var (ptype);
7952 689 : gimplify_assign (m, ref, stmt_seqp);
7953 689 : ref = m;
7954 689 : gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), stmt_seqp);
7955 689 : tree body = create_artificial_label (UNKNOWN_LOCATION);
7956 689 : tree end = create_artificial_label (UNKNOWN_LOCATION);
7957 689 : gimple_seq_add_stmt (&sub_seq, gimple_build_label (body));
7958 689 : tree priv = build_simple_mem_ref_loc (clause_loc, new_var);
7959 689 : tree out = build_simple_mem_ref_loc (clause_loc, ref);
7960 689 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
7961 : {
7962 72 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
7963 72 : tree decl_placeholder
7964 72 : = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
7965 72 : SET_DECL_VALUE_EXPR (placeholder, out);
7966 72 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
7967 72 : SET_DECL_VALUE_EXPR (decl_placeholder, priv);
7968 72 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
7969 72 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
7970 72 : gimple_seq_add_seq (&sub_seq,
7971 72 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
7972 72 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
7973 72 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
7974 72 : OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c) = NULL;
7975 : }
7976 : else
7977 : {
7978 617 : tree out2 = out;
7979 617 : tree priv2 = priv;
7980 617 : if (is_truth_op)
7981 : {
7982 0 : tree zero = build_zero_cst (TREE_TYPE (out));
7983 0 : out2 = fold_build2_loc (clause_loc, NE_EXPR,
7984 : boolean_type_node, out, zero);
7985 0 : priv2 = fold_build2_loc (clause_loc, NE_EXPR,
7986 : boolean_type_node, priv, zero);
7987 : }
7988 617 : x = build2 (code, TREE_TYPE (out2), out2, priv2);
7989 617 : if (is_truth_op)
7990 0 : x = fold_convert (TREE_TYPE (out), x);
7991 617 : out = unshare_expr (out);
7992 617 : gimplify_assign (out, x, &sub_seq);
7993 : }
7994 689 : gimple *g = gimple_build_assign (new_var, POINTER_PLUS_EXPR, new_var,
7995 689 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
7996 689 : gimple_seq_add_stmt (&sub_seq, g);
7997 689 : g = gimple_build_assign (ref, POINTER_PLUS_EXPR, ref,
7998 689 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
7999 689 : gimple_seq_add_stmt (&sub_seq, g);
8000 689 : g = gimple_build_assign (i, PLUS_EXPR, i,
8001 689 : build_int_cst (TREE_TYPE (i), 1));
8002 689 : gimple_seq_add_stmt (&sub_seq, g);
8003 689 : g = gimple_build_cond (LE_EXPR, i, v, body, end);
8004 689 : gimple_seq_add_stmt (&sub_seq, g);
8005 689 : gimple_seq_add_stmt (&sub_seq, gimple_build_label (end));
8006 : }
8007 1416 : else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
8008 : {
8009 770 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
8010 :
8011 770 : if (omp_privatize_by_reference (var)
8012 815 : && !useless_type_conversion_p (TREE_TYPE (placeholder),
8013 45 : TREE_TYPE (ref)))
8014 37 : ref = build_fold_addr_expr_loc (clause_loc, ref);
8015 770 : SET_DECL_VALUE_EXPR (placeholder, ref);
8016 770 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
8017 770 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
8018 770 : gimple_seq_add_seq (&sub_seq, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
8019 770 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
8020 770 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
8021 : }
8022 : else
8023 : {
8024 646 : tree new_var2 = new_var;
8025 646 : tree ref2 = ref;
8026 646 : if (is_truth_op)
8027 : {
8028 78 : tree zero = build_zero_cst (TREE_TYPE (new_var));
8029 78 : new_var2 = fold_build2_loc (clause_loc, NE_EXPR,
8030 : boolean_type_node, new_var, zero);
8031 78 : ref2 = fold_build2_loc (clause_loc, NE_EXPR, boolean_type_node,
8032 : ref, zero);
8033 : }
8034 646 : x = build2 (code, TREE_TYPE (ref), ref2, new_var2);
8035 646 : if (is_truth_op)
8036 78 : x = fold_convert (TREE_TYPE (new_var), x);
8037 646 : ref = build_outer_var_ref (var, ctx);
8038 646 : gimplify_assign (ref, x, &sub_seq);
8039 : }
8040 : }
8041 :
8042 1053 : stmt
8043 1053 : = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_START),
8044 : 0);
8045 1053 : gimple_seq_add_stmt (stmt_seqp, stmt);
8046 :
8047 1053 : gimple_seq_add_seq (stmt_seqp, sub_seq);
8048 :
8049 1053 : if (clist)
8050 : {
8051 167 : gimple_seq_add_seq (stmt_seqp, *clist);
8052 167 : *clist = NULL;
8053 : }
8054 :
8055 1053 : stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_END),
8056 : 0);
8057 1053 : gimple_seq_add_stmt (stmt_seqp, stmt);
8058 : }
8059 :
8060 :
8061 : /* Generate code to implement the COPYPRIVATE clauses. */
8062 :
8063 : static void
8064 115 : lower_copyprivate_clauses (tree clauses, gimple_seq *slist, gimple_seq *rlist,
8065 : omp_context *ctx)
8066 : {
8067 115 : tree c;
8068 :
8069 463 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
8070 : {
8071 348 : tree var, new_var, ref, x;
8072 348 : bool by_ref;
8073 348 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
8074 :
8075 348 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE)
8076 6 : continue;
8077 :
8078 342 : var = OMP_CLAUSE_DECL (c);
8079 342 : by_ref = use_pointer_for_field (var, NULL);
8080 :
8081 342 : ref = build_sender_ref (var, ctx);
8082 342 : x = new_var = lookup_decl_in_outer_ctx (var, ctx);
8083 342 : if (by_ref)
8084 : {
8085 74 : x = build_fold_addr_expr_loc (clause_loc, new_var);
8086 74 : x = fold_convert_loc (clause_loc, TREE_TYPE (ref), x);
8087 : }
8088 342 : gimplify_assign (ref, x, slist);
8089 :
8090 342 : ref = build_receiver_ref (var, false, ctx);
8091 342 : if (by_ref)
8092 : {
8093 74 : ref = fold_convert_loc (clause_loc,
8094 74 : build_pointer_type (TREE_TYPE (new_var)),
8095 : ref);
8096 74 : ref = build_fold_indirect_ref_loc (clause_loc, ref);
8097 : }
8098 342 : if (omp_privatize_by_reference (var))
8099 : {
8100 171 : ref = fold_convert_loc (clause_loc, TREE_TYPE (new_var), ref);
8101 171 : ref = build_simple_mem_ref_loc (clause_loc, ref);
8102 171 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
8103 : }
8104 342 : x = lang_hooks.decls.omp_clause_assign_op (c, new_var, ref);
8105 342 : gimplify_and_add (x, rlist);
8106 : }
8107 115 : }
8108 :
8109 :
8110 : /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
8111 : and REDUCTION from the sender (aka parent) side. */
8112 :
8113 : static void
8114 22383 : lower_send_clauses (tree clauses, gimple_seq *ilist, gimple_seq *olist,
8115 : omp_context *ctx)
8116 : {
8117 22383 : tree c, t;
8118 22383 : int ignored_looptemp = 0;
8119 22383 : bool is_taskloop = false;
8120 :
8121 : /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
8122 : by GOMP_taskloop. */
8123 22383 : if (is_task_ctx (ctx) && gimple_omp_task_taskloop_p (ctx->stmt))
8124 : {
8125 : ignored_looptemp = 2;
8126 : is_taskloop = true;
8127 : }
8128 :
8129 116545 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
8130 : {
8131 94162 : tree val, ref, x, var;
8132 94162 : bool by_ref, do_in = false, do_out = false;
8133 94162 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
8134 :
8135 94162 : switch (OMP_CLAUSE_CODE (c))
8136 : {
8137 4606 : case OMP_CLAUSE_PRIVATE:
8138 4606 : if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
8139 : break;
8140 4451 : continue;
8141 : case OMP_CLAUSE_FIRSTPRIVATE:
8142 : case OMP_CLAUSE_COPYIN:
8143 : case OMP_CLAUSE_LASTPRIVATE:
8144 : case OMP_CLAUSE_IN_REDUCTION:
8145 : case OMP_CLAUSE__REDUCTEMP_:
8146 : break;
8147 4800 : case OMP_CLAUSE_REDUCTION:
8148 4800 : if (is_task_ctx (ctx) || OMP_CLAUSE_REDUCTION_TASK (c))
8149 745 : continue;
8150 : break;
8151 17340 : case OMP_CLAUSE_SHARED:
8152 17340 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
8153 : break;
8154 17028 : continue;
8155 18795 : case OMP_CLAUSE__LOOPTEMP_:
8156 18795 : if (ignored_looptemp)
8157 : {
8158 2660 : ignored_looptemp--;
8159 2660 : continue;
8160 : }
8161 : break;
8162 11827 : default:
8163 11827 : continue;
8164 : }
8165 :
8166 57451 : val = OMP_CLAUSE_DECL (c);
8167 57451 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
8168 53396 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION)
8169 59091 : && TREE_CODE (val) == MEM_REF)
8170 : {
8171 1578 : val = TREE_OPERAND (val, 0);
8172 1578 : if (TREE_CODE (val) == POINTER_PLUS_EXPR)
8173 351 : val = TREE_OPERAND (val, 0);
8174 1578 : if (INDIRECT_REF_P (val)
8175 1494 : || TREE_CODE (val) == ADDR_EXPR)
8176 906 : val = TREE_OPERAND (val, 0);
8177 1578 : if (is_variable_sized (val))
8178 56 : continue;
8179 : }
8180 :
8181 : /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
8182 : outer taskloop region. */
8183 57395 : omp_context *ctx_for_o = ctx;
8184 57395 : if (is_taskloop
8185 3911 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
8186 57707 : && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
8187 312 : ctx_for_o = ctx->outer;
8188 :
8189 57395 : var = lookup_decl_in_outer_ctx (val, ctx_for_o);
8190 :
8191 57395 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN
8192 56881 : && is_global_var (var)
8193 59700 : && (val == OMP_CLAUSE_DECL (c)
8194 273 : || !is_task_ctx (ctx)
8195 241 : || (TREE_CODE (TREE_TYPE (val)) != POINTER_TYPE
8196 187 : && (TREE_CODE (TREE_TYPE (val)) != REFERENCE_TYPE
8197 0 : || (TREE_CODE (TREE_TYPE (TREE_TYPE (val)))
8198 : != POINTER_TYPE)))))
8199 2251 : continue;
8200 :
8201 55144 : t = omp_member_access_dummy_var (var);
8202 55144 : if (t)
8203 : {
8204 249 : var = DECL_VALUE_EXPR (var);
8205 249 : tree o = maybe_lookup_decl_in_outer_ctx (t, ctx_for_o);
8206 249 : if (o != t)
8207 69 : var = unshare_and_remap (var, t, o);
8208 : else
8209 180 : var = unshare_expr (var);
8210 : }
8211 :
8212 55144 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED)
8213 : {
8214 : /* Handle taskloop firstprivate/lastprivate, where the
8215 : lastprivate on GIMPLE_OMP_TASK is represented as
8216 : OMP_CLAUSE_SHARED_FIRSTPRIVATE. */
8217 79 : tree f = lookup_sfield ((splay_tree_key) &DECL_UID (val), ctx);
8218 79 : x = omp_build_component_ref (ctx->sender_decl, f);
8219 79 : if (use_pointer_for_field (val, ctx))
8220 59 : var = build_fold_addr_expr (var);
8221 79 : gimplify_assign (x, var, ilist);
8222 79 : DECL_ABSTRACT_ORIGIN (f) = NULL;
8223 79 : continue;
8224 79 : }
8225 :
8226 55119 : if (((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
8227 51755 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IN_REDUCTION)
8228 4446 : || val == OMP_CLAUSE_DECL (c))
8229 108827 : && is_variable_sized (val))
8230 54 : continue;
8231 55011 : by_ref = use_pointer_for_field (val, NULL);
8232 :
8233 55011 : switch (OMP_CLAUSE_CODE (c))
8234 : {
8235 26747 : case OMP_CLAUSE_FIRSTPRIVATE:
8236 26747 : if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c)
8237 7238 : && !by_ref
8238 33933 : && is_task_ctx (ctx))
8239 2325 : suppress_warning (var);
8240 : do_in = true;
8241 : break;
8242 :
8243 : case OMP_CLAUSE_PRIVATE:
8244 : case OMP_CLAUSE_COPYIN:
8245 : case OMP_CLAUSE__LOOPTEMP_:
8246 : case OMP_CLAUSE__REDUCTEMP_:
8247 : do_in = true;
8248 : break;
8249 :
8250 6458 : case OMP_CLAUSE_LASTPRIVATE:
8251 6458 : if (by_ref || omp_privatize_by_reference (val))
8252 : {
8253 384 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
8254 239 : continue;
8255 : do_in = true;
8256 : }
8257 : else
8258 : {
8259 6074 : do_out = true;
8260 6074 : if (lang_hooks.decls.omp_private_outer_ref (val))
8261 : do_in = true;
8262 : }
8263 : break;
8264 :
8265 4446 : case OMP_CLAUSE_REDUCTION:
8266 4446 : case OMP_CLAUSE_IN_REDUCTION:
8267 4446 : do_in = true;
8268 4446 : if (val == OMP_CLAUSE_DECL (c))
8269 : {
8270 3143 : if (is_task_ctx (ctx))
8271 301 : by_ref = use_pointer_for_field (val, ctx);
8272 : else
8273 2842 : do_out = !(by_ref || omp_privatize_by_reference (val));
8274 : }
8275 : else
8276 1303 : by_ref = TREE_CODE (TREE_TYPE (val)) == ARRAY_TYPE;
8277 : break;
8278 :
8279 0 : default:
8280 0 : gcc_unreachable ();
8281 : }
8282 :
8283 1604 : if (do_in)
8284 : {
8285 48722 : ref = build_sender_ref (val, ctx);
8286 48722 : x = by_ref ? build_fold_addr_expr_loc (clause_loc, var) : var;
8287 48722 : gimplify_assign (ref, x, ilist);
8288 48722 : if (is_task_ctx (ctx))
8289 6190 : DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref, 1)) = NULL;
8290 : }
8291 :
8292 54772 : if (do_out)
8293 : {
8294 8373 : ref = build_sender_ref (val, ctx);
8295 8373 : gimplify_assign (var, ref, olist);
8296 : }
8297 : }
8298 22383 : }
8299 :
8300 : /* Generate code to implement SHARED from the sender (aka parent)
8301 : side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
8302 : list things that got automatically shared. */
8303 :
8304 : static void
8305 22383 : lower_send_shared_vars (gimple_seq *ilist, gimple_seq *olist, omp_context *ctx)
8306 : {
8307 22383 : tree var, ovar, nvar, t, f, x, record_type;
8308 :
8309 22383 : if (ctx->record_type == NULL)
8310 4456 : return;
8311 :
8312 17927 : record_type = ctx->srecord_type ? ctx->srecord_type : ctx->record_type;
8313 86490 : for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
8314 : {
8315 68563 : ovar = DECL_ABSTRACT_ORIGIN (f);
8316 68563 : if (!ovar || TREE_CODE (ovar) == FIELD_DECL)
8317 6269 : continue;
8318 :
8319 62294 : nvar = maybe_lookup_decl (ovar, ctx);
8320 113180 : if (!nvar
8321 61881 : || !DECL_HAS_VALUE_EXPR_P (nvar)
8322 73888 : || (ctx->allocate_map
8323 231 : && ctx->allocate_map->get (ovar)))
8324 50886 : continue;
8325 :
8326 : /* If CTX is a nested parallel directive. Find the immediately
8327 : enclosing parallel or workshare construct that contains a
8328 : mapping for OVAR. */
8329 11408 : var = lookup_decl_in_outer_ctx (ovar, ctx);
8330 :
8331 11408 : t = omp_member_access_dummy_var (var);
8332 11408 : if (t)
8333 : {
8334 49 : var = DECL_VALUE_EXPR (var);
8335 49 : tree o = maybe_lookup_decl_in_outer_ctx (t, ctx);
8336 49 : if (o != t)
8337 18 : var = unshare_and_remap (var, t, o);
8338 : else
8339 31 : var = unshare_expr (var);
8340 : }
8341 :
8342 11408 : if (use_pointer_for_field (ovar, ctx))
8343 : {
8344 6294 : x = build_sender_ref (ovar, ctx);
8345 6294 : if (TREE_CODE (TREE_TYPE (f)) == ARRAY_TYPE
8346 6294 : && TREE_TYPE (f) == TREE_TYPE (ovar))
8347 : {
8348 97 : gcc_assert (is_parallel_ctx (ctx)
8349 : && DECL_ARTIFICIAL (ovar));
8350 : /* _condtemp_ clause. */
8351 97 : var = build_constructor (TREE_TYPE (x), NULL);
8352 : }
8353 : else
8354 6197 : var = build_fold_addr_expr (var);
8355 6294 : gimplify_assign (x, var, ilist);
8356 : }
8357 : else
8358 : {
8359 5114 : x = build_sender_ref (ovar, ctx);
8360 5114 : gimplify_assign (x, var, ilist);
8361 :
8362 5114 : if (!TREE_READONLY (var)
8363 : /* We don't need to receive a new reference to a result
8364 : or parm decl. In fact we may not store to it as we will
8365 : invalidate any pending RSO and generate wrong gimple
8366 : during inlining. */
8367 5114 : && !((TREE_CODE (var) == RESULT_DECL
8368 3296 : || TREE_CODE (var) == PARM_DECL)
8369 182 : && DECL_BY_REFERENCE (var)))
8370 : {
8371 3225 : x = build_sender_ref (ovar, ctx);
8372 3225 : gimplify_assign (var, x, olist);
8373 : }
8374 : }
8375 : }
8376 : }
8377 :
8378 : /* Emit an OpenACC head marker call, encapulating the partitioning and
8379 : other information that must be processed by the target compiler.
8380 : Return the maximum number of dimensions the associated loop might
8381 : be partitioned over. */
8382 :
8383 : static unsigned
8384 9665 : lower_oacc_head_mark (location_t loc, tree ddvar, tree clauses,
8385 : gimple_seq *seq, omp_context *ctx)
8386 : {
8387 9665 : unsigned levels = 0;
8388 9665 : unsigned tag = 0;
8389 9665 : tree gang_static = NULL_TREE;
8390 9665 : auto_vec<tree, 5> args;
8391 :
8392 19330 : args.quick_push (build_int_cst
8393 9665 : (integer_type_node, IFN_UNIQUE_OACC_HEAD_MARK));
8394 9665 : args.quick_push (ddvar);
8395 30854 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
8396 : {
8397 21189 : switch (OMP_CLAUSE_CODE (c))
8398 : {
8399 1856 : case OMP_CLAUSE_GANG:
8400 1856 : tag |= OLF_DIM_GANG;
8401 1856 : gang_static = OMP_CLAUSE_GANG_STATIC_EXPR (c);
8402 : /* static:* is represented by -1, and we can ignore it, as
8403 : scheduling is always static. */
8404 1856 : if (gang_static && integer_minus_onep (gang_static))
8405 70 : gang_static = NULL_TREE;
8406 1856 : levels++;
8407 1856 : break;
8408 :
8409 1330 : case OMP_CLAUSE_WORKER:
8410 1330 : tag |= OLF_DIM_WORKER;
8411 1330 : levels++;
8412 1330 : break;
8413 :
8414 1523 : case OMP_CLAUSE_VECTOR:
8415 1523 : tag |= OLF_DIM_VECTOR;
8416 1523 : levels++;
8417 1523 : break;
8418 :
8419 314 : case OMP_CLAUSE_SEQ:
8420 314 : tag |= OLF_SEQ;
8421 314 : break;
8422 :
8423 409 : case OMP_CLAUSE_AUTO:
8424 409 : tag |= OLF_AUTO;
8425 409 : break;
8426 :
8427 260 : case OMP_CLAUSE_INDEPENDENT:
8428 260 : tag |= OLF_INDEPENDENT;
8429 260 : break;
8430 :
8431 136 : case OMP_CLAUSE_TILE:
8432 136 : tag |= OLF_TILE;
8433 136 : break;
8434 :
8435 3986 : case OMP_CLAUSE_REDUCTION:
8436 3986 : tag |= OLF_REDUCTION;
8437 3986 : break;
8438 :
8439 11375 : default:
8440 11375 : continue;
8441 : }
8442 : }
8443 :
8444 9665 : if (gang_static)
8445 : {
8446 146 : if (DECL_P (gang_static))
8447 16 : gang_static = build_outer_var_ref (gang_static, ctx);
8448 146 : tag |= OLF_GANG_STATIC;
8449 : }
8450 :
8451 9665 : omp_context *tgt = enclosing_target_ctx (ctx);
8452 9665 : if (!tgt || is_oacc_parallel_or_serial (tgt))
8453 : ;
8454 62 : else if (is_oacc_kernels (tgt))
8455 : /* Not using this loops handling inside OpenACC 'kernels' regions. */
8456 0 : gcc_unreachable ();
8457 62 : else if (is_oacc_kernels_decomposed_part (tgt))
8458 : ;
8459 : else
8460 0 : gcc_unreachable ();
8461 :
8462 : /* In a parallel region, loops are implicitly INDEPENDENT. */
8463 9665 : if (!tgt || is_oacc_parallel_or_serial (tgt))
8464 9603 : tag |= OLF_INDEPENDENT;
8465 :
8466 : /* Loops inside OpenACC 'kernels' decomposed parts' regions are expected to
8467 : have an explicit 'seq' or 'independent' clause, and no 'auto' clause. */
8468 9665 : if (tgt && is_oacc_kernels_decomposed_part (tgt))
8469 : {
8470 62 : gcc_assert (tag & (OLF_SEQ | OLF_INDEPENDENT));
8471 62 : gcc_assert (!(tag & OLF_AUTO));
8472 : }
8473 :
8474 9665 : if (tag & OLF_TILE)
8475 : /* Tiling could use all 3 levels. */
8476 : levels = 3;
8477 : else
8478 : {
8479 : /* A loop lacking SEQ, GANG, WORKER and/or VECTOR could be AUTO.
8480 : Ensure at least one level, or 2 for possible auto
8481 : partitioning */
8482 9529 : bool maybe_auto = !(tag & (((GOMP_DIM_MASK (GOMP_DIM_MAX) - 1)
8483 : << OLF_DIM_BASE) | OLF_SEQ));
8484 :
8485 9529 : if (levels < 1u + maybe_auto)
8486 : levels = 1u + maybe_auto;
8487 : }
8488 :
8489 9665 : args.quick_push (build_int_cst (integer_type_node, levels));
8490 9665 : args.quick_push (build_int_cst (integer_type_node, tag));
8491 9665 : if (gang_static)
8492 146 : args.quick_push (gang_static);
8493 :
8494 9665 : gcall *call = gimple_build_call_internal_vec (IFN_UNIQUE, args);
8495 9665 : gimple_set_location (call, loc);
8496 9665 : gimple_set_lhs (call, ddvar);
8497 9665 : gimple_seq_add_stmt (seq, call);
8498 :
8499 9665 : return levels;
8500 9665 : }
8501 :
8502 : /* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the
8503 : partitioning level of the enclosed region. */
8504 :
8505 : static void
8506 42767 : lower_oacc_loop_marker (location_t loc, tree ddvar, bool head,
8507 : tree tofollow, gimple_seq *seq)
8508 : {
8509 42767 : int marker_kind = (head ? IFN_UNIQUE_OACC_HEAD_MARK
8510 : : IFN_UNIQUE_OACC_TAIL_MARK);
8511 42767 : tree marker = build_int_cst (integer_type_node, marker_kind);
8512 42767 : int nargs = 2 + (tofollow != NULL_TREE);
8513 42767 : gcall *call = gimple_build_call_internal (IFN_UNIQUE, nargs,
8514 : marker, ddvar, tofollow);
8515 42767 : gimple_set_location (call, loc);
8516 42767 : gimple_set_lhs (call, ddvar);
8517 42767 : gimple_seq_add_stmt (seq, call);
8518 42767 : }
8519 :
8520 : /* Generate the before and after OpenACC loop sequences. CLAUSES are
8521 : the loop clauses, from which we extract reductions. Initialize
8522 : HEAD and TAIL. */
8523 :
8524 : static void
8525 9665 : lower_oacc_head_tail (location_t loc, tree clauses, gcall *private_marker,
8526 : gimple_seq *head, gimple_seq *tail, omp_context *ctx)
8527 : {
8528 9665 : bool inner = false;
8529 9665 : tree ddvar = create_tmp_var (integer_type_node, ".data_dep");
8530 9665 : gimple_seq_add_stmt (head, gimple_build_assign (ddvar, integer_zero_node));
8531 :
8532 9665 : unsigned count = lower_oacc_head_mark (loc, ddvar, clauses, head, ctx);
8533 :
8534 9665 : if (private_marker)
8535 : {
8536 229 : gimple_set_location (private_marker, loc);
8537 229 : gimple_call_set_lhs (private_marker, ddvar);
8538 229 : gimple_call_set_arg (private_marker, 1, ddvar);
8539 : }
8540 :
8541 9665 : tree fork_kind = build_int_cst (unsigned_type_node, IFN_UNIQUE_OACC_FORK);
8542 9665 : tree join_kind = build_int_cst (unsigned_type_node, IFN_UNIQUE_OACC_JOIN);
8543 :
8544 9665 : gcc_assert (count);
8545 26216 : for (unsigned done = 1; count; count--, done++)
8546 : {
8547 16551 : gimple_seq fork_seq = NULL;
8548 16551 : gimple_seq join_seq = NULL;
8549 :
8550 16551 : tree place = build_int_cst (integer_type_node, -1);
8551 16551 : gcall *fork = gimple_build_call_internal (IFN_UNIQUE, 3,
8552 : fork_kind, ddvar, place);
8553 16551 : gimple_set_location (fork, loc);
8554 16551 : gimple_set_lhs (fork, ddvar);
8555 :
8556 16551 : gcall *join = gimple_build_call_internal (IFN_UNIQUE, 3,
8557 : join_kind, ddvar, place);
8558 16551 : gimple_set_location (join, loc);
8559 16551 : gimple_set_lhs (join, ddvar);
8560 :
8561 : /* Mark the beginning of this level sequence. */
8562 16551 : if (inner)
8563 6886 : lower_oacc_loop_marker (loc, ddvar, true,
8564 6886 : build_int_cst (integer_type_node, count),
8565 : &fork_seq);
8566 16551 : lower_oacc_loop_marker (loc, ddvar, false,
8567 16551 : build_int_cst (integer_type_node, done),
8568 : &join_seq);
8569 :
8570 23437 : lower_oacc_reductions (loc, clauses, place, inner,
8571 : fork, (count == 1) ? private_marker : NULL,
8572 : join, &fork_seq, &join_seq, ctx);
8573 :
8574 : /* Append this level to head. */
8575 16551 : gimple_seq_add_seq (head, fork_seq);
8576 : /* Prepend it to tail. */
8577 16551 : gimple_seq_add_seq (&join_seq, *tail);
8578 16551 : *tail = join_seq;
8579 :
8580 16551 : inner = true;
8581 : }
8582 :
8583 : /* Mark the end of the sequence. */
8584 9665 : lower_oacc_loop_marker (loc, ddvar, true, NULL_TREE, head);
8585 9665 : lower_oacc_loop_marker (loc, ddvar, false, NULL_TREE, tail);
8586 9665 : }
8587 :
8588 : /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
8589 : catch handler and return it. This prevents programs from violating the
8590 : structured block semantics with throws. */
8591 :
8592 : static gimple_seq
8593 95066 : maybe_catch_exception (gimple_seq body)
8594 : {
8595 95066 : gimple *g;
8596 95066 : tree decl;
8597 :
8598 95066 : if (!flag_exceptions)
8599 : return body;
8600 :
8601 42682 : if (lang_hooks.eh_protect_cleanup_actions != NULL)
8602 42654 : decl = lang_hooks.eh_protect_cleanup_actions ();
8603 : else
8604 28 : decl = builtin_decl_explicit (BUILT_IN_TRAP);
8605 :
8606 42682 : g = gimple_build_eh_must_not_throw (decl);
8607 42682 : g = gimple_build_try (body, gimple_seq_alloc_with_stmt (g),
8608 : GIMPLE_TRY_CATCH);
8609 :
8610 42682 : return gimple_seq_alloc_with_stmt (g);
8611 : }
8612 :
8613 :
8614 : /* Routines to lower OMP directives into OMP-GIMPLE. */
8615 :
8616 : /* If ctx is a worksharing context inside of a cancellable parallel
8617 : region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8618 : and conditional branch to parallel's cancel_label to handle
8619 : cancellation in the implicit barrier. */
8620 :
8621 : static void
8622 48517 : maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple *omp_return,
8623 : gimple_seq *body)
8624 : {
8625 48517 : gcc_assert (gimple_code (omp_return) == GIMPLE_OMP_RETURN);
8626 48517 : if (gimple_omp_return_nowait_p (omp_return))
8627 : return;
8628 19520 : for (omp_context *outer = ctx->outer; outer; outer = outer->outer)
8629 15943 : if (gimple_code (outer->stmt) == GIMPLE_OMP_PARALLEL
8630 15943 : && outer->cancellable)
8631 : {
8632 40 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_CANCEL);
8633 40 : tree c_bool_type = TREE_TYPE (TREE_TYPE (fndecl));
8634 40 : tree lhs = create_tmp_var (c_bool_type);
8635 40 : gimple_omp_return_set_lhs (omp_return, lhs);
8636 40 : tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
8637 40 : gimple *g = gimple_build_cond (NE_EXPR, lhs,
8638 : fold_convert (c_bool_type,
8639 : boolean_false_node),
8640 : outer->cancel_label, fallthru_label);
8641 40 : gimple_seq_add_stmt (body, g);
8642 40 : gimple_seq_add_stmt (body, gimple_build_label (fallthru_label));
8643 : }
8644 15903 : else if (gimple_code (outer->stmt) != GIMPLE_OMP_TASKGROUP
8645 15903 : && gimple_code (outer->stmt) != GIMPLE_OMP_SCOPE)
8646 : return;
8647 : }
8648 :
8649 : /* Find the first task_reduction or reduction clause or return NULL
8650 : if there are none. */
8651 :
8652 : static inline tree
8653 48764 : omp_task_reductions_find_first (tree clauses, enum tree_code code,
8654 : enum omp_clause_code ccode)
8655 : {
8656 64110 : while (1)
8657 : {
8658 56437 : clauses = omp_find_clause (clauses, ccode);
8659 56437 : if (clauses == NULL_TREE)
8660 : return NULL_TREE;
8661 9119 : if (ccode != OMP_CLAUSE_REDUCTION
8662 9119 : || code == OMP_TASKLOOP
8663 9119 : || OMP_CLAUSE_REDUCTION_TASK (clauses))
8664 : return clauses;
8665 7673 : clauses = OMP_CLAUSE_CHAIN (clauses);
8666 : }
8667 : }
8668 :
8669 : static void lower_omp_task_reductions (omp_context *, enum tree_code, tree,
8670 : gimple_seq *, gimple_seq *);
8671 :
8672 : /* Lower the OpenMP sections directive in the current statement in GSI_P.
8673 : CTX is the enclosing OMP context for the current statement. */
8674 :
8675 : static void
8676 386 : lower_omp_sections (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8677 : {
8678 386 : tree block, control;
8679 386 : gimple_stmt_iterator tgsi;
8680 386 : gomp_sections *stmt;
8681 386 : gimple *t;
8682 386 : gbind *new_stmt, *bind;
8683 386 : gimple_seq ilist, dlist, olist, tred_dlist = NULL, clist = NULL, new_body;
8684 :
8685 386 : stmt = as_a <gomp_sections *> (gsi_stmt (*gsi_p));
8686 :
8687 386 : push_gimplify_context ();
8688 :
8689 386 : dlist = NULL;
8690 386 : ilist = NULL;
8691 :
8692 386 : tree rclauses
8693 386 : = omp_task_reductions_find_first (gimple_omp_sections_clauses (stmt),
8694 : OMP_SECTIONS, OMP_CLAUSE_REDUCTION);
8695 386 : tree rtmp = NULL_TREE;
8696 386 : if (rclauses)
8697 : {
8698 8 : tree type = build_pointer_type (pointer_sized_int_node);
8699 8 : tree temp = create_tmp_var (type);
8700 8 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
8701 8 : OMP_CLAUSE_DECL (c) = temp;
8702 8 : OMP_CLAUSE_CHAIN (c) = gimple_omp_sections_clauses (stmt);
8703 8 : gimple_omp_sections_set_clauses (stmt, c);
8704 8 : lower_omp_task_reductions (ctx, OMP_SECTIONS,
8705 : gimple_omp_sections_clauses (stmt),
8706 : &ilist, &tred_dlist);
8707 8 : rclauses = c;
8708 8 : rtmp = make_ssa_name (type);
8709 8 : gimple_seq_add_stmt (&ilist, gimple_build_assign (rtmp, temp));
8710 : }
8711 :
8712 386 : tree *clauses_ptr = gimple_omp_sections_clauses_ptr (stmt);
8713 386 : lower_lastprivate_conditional_clauses (clauses_ptr, ctx);
8714 :
8715 386 : lower_rec_input_clauses (gimple_omp_sections_clauses (stmt),
8716 : &ilist, &dlist, ctx, NULL);
8717 :
8718 386 : control = create_tmp_var (unsigned_type_node, ".section");
8719 386 : gimple_omp_sections_set_control (stmt, control);
8720 :
8721 386 : new_body = gimple_omp_body (stmt);
8722 386 : gimple_omp_set_body (stmt, NULL);
8723 386 : tgsi = gsi_start (new_body);
8724 1249 : for (; !gsi_end_p (tgsi); gsi_next (&tgsi))
8725 : {
8726 863 : omp_context *sctx;
8727 863 : gimple *sec_start;
8728 :
8729 863 : sec_start = gsi_stmt (tgsi);
8730 863 : sctx = maybe_lookup_ctx (sec_start);
8731 863 : gcc_assert (sctx);
8732 :
8733 863 : lower_omp (gimple_omp_body_ptr (sec_start), sctx);
8734 863 : gsi_insert_seq_after (&tgsi, gimple_omp_body (sec_start),
8735 : GSI_CONTINUE_LINKING);
8736 863 : gimple_omp_set_body (sec_start, NULL);
8737 :
8738 863 : if (gsi_one_before_end_p (tgsi))
8739 : {
8740 386 : gimple_seq l = NULL;
8741 386 : lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt), NULL,
8742 : &ilist, &l, &clist, ctx);
8743 386 : gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING);
8744 386 : gimple_omp_section_set_last (sec_start);
8745 : }
8746 :
8747 863 : gsi_insert_after (&tgsi, gimple_build_omp_return (false),
8748 : GSI_CONTINUE_LINKING);
8749 : }
8750 :
8751 386 : block = make_node (BLOCK);
8752 386 : bind = gimple_build_bind (NULL, new_body, block);
8753 :
8754 386 : olist = NULL;
8755 386 : lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist,
8756 : &clist, ctx);
8757 386 : if (clist)
8758 : {
8759 10 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_START);
8760 10 : gcall *g = gimple_build_call (fndecl, 0);
8761 10 : gimple_seq_add_stmt (&olist, g);
8762 10 : gimple_seq_add_seq (&olist, clist);
8763 10 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_END);
8764 10 : g = gimple_build_call (fndecl, 0);
8765 10 : gimple_seq_add_stmt (&olist, g);
8766 : }
8767 :
8768 386 : block = make_node (BLOCK);
8769 386 : new_stmt = gimple_build_bind (NULL, NULL, block);
8770 386 : gsi_replace (gsi_p, new_stmt, true);
8771 :
8772 386 : pop_gimplify_context (new_stmt);
8773 386 : gimple_bind_append_vars (new_stmt, ctx->block_vars);
8774 386 : BLOCK_VARS (block) = gimple_bind_vars (bind);
8775 386 : if (BLOCK_VARS (block))
8776 0 : TREE_USED (block) = 1;
8777 :
8778 386 : new_body = NULL;
8779 386 : gimple_seq_add_seq (&new_body, ilist);
8780 386 : gimple_seq_add_stmt (&new_body, stmt);
8781 386 : gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ());
8782 386 : gimple_seq_add_stmt (&new_body, bind);
8783 :
8784 386 : t = gimple_build_omp_continue (control, control);
8785 386 : gimple_seq_add_stmt (&new_body, t);
8786 :
8787 386 : gimple_seq_add_seq (&new_body, olist);
8788 386 : if (ctx->cancellable)
8789 19 : gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
8790 386 : gimple_seq_add_seq (&new_body, dlist);
8791 :
8792 386 : new_body = maybe_catch_exception (new_body);
8793 :
8794 386 : bool nowait = omp_find_clause (gimple_omp_sections_clauses (stmt),
8795 386 : OMP_CLAUSE_NOWAIT) != NULL_TREE;
8796 386 : t = gimple_build_omp_return (nowait);
8797 386 : gimple_seq_add_stmt (&new_body, t);
8798 386 : gimple_seq_add_seq (&new_body, tred_dlist);
8799 386 : maybe_add_implicit_barrier_cancel (ctx, t, &new_body);
8800 :
8801 386 : if (rclauses)
8802 8 : OMP_CLAUSE_DECL (rclauses) = rtmp;
8803 :
8804 386 : gimple_bind_set_body (new_stmt, new_body);
8805 386 : }
8806 :
8807 :
8808 : /* A subroutine of lower_omp_single. Expand the simple form of
8809 : a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8810 :
8811 : if (GOMP_single_start ())
8812 : BODY;
8813 : [ GOMP_barrier (); ] -> unless 'nowait' is present.
8814 :
8815 : FIXME. It may be better to delay expanding the logic of this until
8816 : pass_expand_omp. The expanded logic may make the job more difficult
8817 : to a synchronization analysis pass. */
8818 :
8819 : static void
8820 1012 : lower_omp_single_simple (gomp_single *single_stmt, gimple_seq *pre_p)
8821 : {
8822 1012 : location_t loc = gimple_location (single_stmt);
8823 1012 : tree tlabel = create_artificial_label (loc);
8824 1012 : tree flabel = create_artificial_label (loc);
8825 1012 : gimple *call, *cond;
8826 1012 : tree lhs, decl;
8827 :
8828 2020 : decl = builtin_decl_explicit (flag_openmp_ompt
8829 : ? BUILT_IN_GOMP_SINGLE_START_WITH_END
8830 : : BUILT_IN_GOMP_SINGLE_START);
8831 1012 : lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)));
8832 1012 : call = gimple_build_call (decl, 0);
8833 1012 : gimple_call_set_lhs (call, lhs);
8834 1012 : gimple_seq_add_stmt (pre_p, call);
8835 :
8836 1012 : cond = gimple_build_cond (EQ_EXPR, lhs,
8837 1012 : fold_convert_loc (loc, TREE_TYPE (lhs),
8838 : boolean_true_node),
8839 : tlabel, flabel);
8840 1012 : gimple_seq_add_stmt (pre_p, cond);
8841 1012 : gimple_seq_add_stmt (pre_p, gimple_build_label (tlabel));
8842 1012 : gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
8843 1012 : gimple_seq_add_stmt (pre_p, gimple_build_label (flabel));
8844 1012 : }
8845 :
8846 :
8847 : /* A subroutine of lower_omp_single. Expand the simple form of
8848 : a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8849 :
8850 : #pragma omp single copyprivate (a, b, c)
8851 :
8852 : Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
8853 :
8854 : {
8855 : if ((copyout_p = GOMP_single_copy_start ()) == NULL)
8856 : {
8857 : BODY;
8858 : copyout.a = a;
8859 : copyout.b = b;
8860 : copyout.c = c;
8861 : GOMP_single_copy_end (©out);
8862 : }
8863 : else
8864 : {
8865 : a = copyout_p->a;
8866 : b = copyout_p->b;
8867 : c = copyout_p->c;
8868 : }
8869 : GOMP_barrier ();
8870 : }
8871 :
8872 : FIXME. It may be better to delay expanding the logic of this until
8873 : pass_expand_omp. The expanded logic may make the job more difficult
8874 : to a synchronization analysis pass. */
8875 :
8876 : static void
8877 115 : lower_omp_single_copy (gomp_single *single_stmt, gimple_seq *pre_p,
8878 : omp_context *ctx)
8879 : {
8880 115 : tree ptr_type, t, l0, l1, l2, bfn_decl;
8881 115 : gimple_seq copyin_seq;
8882 115 : location_t loc = gimple_location (single_stmt);
8883 :
8884 115 : ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o");
8885 :
8886 115 : ptr_type = build_pointer_type (ctx->record_type);
8887 115 : ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i");
8888 :
8889 115 : l0 = create_artificial_label (loc);
8890 115 : l1 = create_artificial_label (loc);
8891 115 : l2 = create_artificial_label (loc);
8892 :
8893 115 : bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START);
8894 115 : t = build_call_expr_loc (loc, bfn_decl, 0);
8895 115 : t = fold_convert_loc (loc, ptr_type, t);
8896 115 : gimplify_assign (ctx->receiver_decl, t, pre_p);
8897 :
8898 115 : t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl,
8899 : build_int_cst (ptr_type, 0));
8900 115 : t = build3 (COND_EXPR, void_type_node, t,
8901 : build_and_jump (&l0), build_and_jump (&l1));
8902 115 : gimplify_and_add (t, pre_p);
8903 :
8904 115 : gimple_seq_add_stmt (pre_p, gimple_build_label (l0));
8905 :
8906 115 : gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
8907 :
8908 115 : copyin_seq = NULL;
8909 115 : lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt), pre_p,
8910 : ©in_seq, ctx);
8911 :
8912 115 : t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
8913 115 : bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END);
8914 115 : t = build_call_expr_loc (loc, bfn_decl, 1, t);
8915 115 : gimplify_and_add (t, pre_p);
8916 :
8917 115 : t = build_and_jump (&l2);
8918 115 : gimplify_and_add (t, pre_p);
8919 :
8920 115 : gimple_seq_add_stmt (pre_p, gimple_build_label (l1));
8921 :
8922 115 : gimple_seq_add_seq (pre_p, copyin_seq);
8923 :
8924 115 : gimple_seq_add_stmt (pre_p, gimple_build_label (l2));
8925 115 : }
8926 :
8927 :
8928 : /* Expand code for an OpenMP single directive. */
8929 :
8930 : static void
8931 1127 : lower_omp_single (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8932 : {
8933 1127 : tree block;
8934 1127 : gomp_single *single_stmt = as_a <gomp_single *> (gsi_stmt (*gsi_p));
8935 1127 : gbind *bind;
8936 1127 : gimple_seq bind_body, bind_body_tail = NULL, dlist;
8937 :
8938 1127 : push_gimplify_context ();
8939 :
8940 1127 : block = make_node (BLOCK);
8941 1127 : bind = gimple_build_bind (NULL, NULL, block);
8942 1127 : gsi_replace (gsi_p, bind, true);
8943 1127 : bind_body = NULL;
8944 1127 : dlist = NULL;
8945 1127 : lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt),
8946 : &bind_body, &dlist, ctx, NULL);
8947 1127 : lower_omp (gimple_omp_body_ptr (single_stmt), ctx);
8948 :
8949 1127 : gimple_seq_add_stmt (&bind_body, single_stmt);
8950 :
8951 1127 : if (ctx->record_type)
8952 115 : lower_omp_single_copy (single_stmt, &bind_body, ctx);
8953 : else
8954 1012 : lower_omp_single_simple (single_stmt, &bind_body);
8955 :
8956 1127 : gimple_omp_set_body (single_stmt, NULL);
8957 :
8958 1127 : gimple_seq_add_seq (&bind_body, dlist);
8959 :
8960 1127 : bind_body = maybe_catch_exception (bind_body);
8961 :
8962 1127 : bool nowait = omp_find_clause (gimple_omp_single_clauses (single_stmt),
8963 1127 : OMP_CLAUSE_NOWAIT) != NULL_TREE;
8964 1127 : gimple *g = gimple_build_omp_return (nowait);
8965 1127 : gimple_seq_add_stmt (&bind_body_tail, g);
8966 1127 : maybe_add_implicit_barrier_cancel (ctx, g, &bind_body_tail);
8967 :
8968 1127 : if (flag_openmp_ompt && !ctx->record_type)
8969 : {
8970 : /* Insert call to GOMP_single_end. */
8971 4 : tree decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_END);
8972 4 : gimple *stmt = gimple_build_call (decl, 0);
8973 4 : gimple_seq_add_stmt (&bind_body_tail, stmt);
8974 : }
8975 :
8976 1127 : if (ctx->record_type)
8977 : {
8978 115 : gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
8979 115 : tree clobber = build_clobber (ctx->record_type);
8980 115 : gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
8981 : clobber), GSI_SAME_STMT);
8982 : }
8983 1127 : gimple_seq_add_seq (&bind_body, bind_body_tail);
8984 1127 : gimple_bind_set_body (bind, bind_body);
8985 :
8986 1127 : pop_gimplify_context (bind);
8987 :
8988 1127 : gimple_bind_append_vars (bind, ctx->block_vars);
8989 1127 : BLOCK_VARS (block) = ctx->block_vars;
8990 1127 : if (BLOCK_VARS (block))
8991 26 : TREE_USED (block) = 1;
8992 1127 : }
8993 :
8994 :
8995 : /* Lower code for an OMP scope directive. */
8996 :
8997 : static void
8998 145 : lower_omp_scope (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8999 : {
9000 145 : tree block;
9001 145 : gimple *scope_stmt = gsi_stmt (*gsi_p);
9002 145 : gbind *bind;
9003 145 : gimple_seq bind_body, bind_body_tail = NULL, dlist;
9004 145 : gimple_seq tred_dlist = NULL;
9005 :
9006 145 : push_gimplify_context ();
9007 :
9008 145 : block = make_node (BLOCK);
9009 145 : bind = gimple_build_bind (NULL, NULL, block);
9010 145 : gsi_replace (gsi_p, bind, true);
9011 145 : bind_body = NULL;
9012 145 : dlist = NULL;
9013 :
9014 145 : tree rclauses
9015 145 : = omp_task_reductions_find_first (gimple_omp_scope_clauses (scope_stmt),
9016 : OMP_SCOPE, OMP_CLAUSE_REDUCTION);
9017 145 : if (rclauses)
9018 : {
9019 46 : tree type = build_pointer_type (pointer_sized_int_node);
9020 46 : tree temp = create_tmp_var (type);
9021 46 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
9022 46 : OMP_CLAUSE_DECL (c) = temp;
9023 46 : OMP_CLAUSE_CHAIN (c) = gimple_omp_scope_clauses (scope_stmt);
9024 46 : gimple_omp_scope_set_clauses (scope_stmt, c);
9025 46 : lower_omp_task_reductions (ctx, OMP_SCOPE,
9026 : gimple_omp_scope_clauses (scope_stmt),
9027 : &bind_body, &tred_dlist);
9028 46 : rclauses = c;
9029 46 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_SCOPE_START);
9030 46 : gimple *stmt = gimple_build_call (fndecl, 1, temp);
9031 46 : gimple_seq_add_stmt (&bind_body, stmt);
9032 : }
9033 :
9034 145 : lower_rec_input_clauses (gimple_omp_scope_clauses (scope_stmt),
9035 : &bind_body, &dlist, ctx, NULL);
9036 145 : lower_omp (gimple_omp_body_ptr (scope_stmt), ctx);
9037 :
9038 145 : gimple_seq_add_stmt (&bind_body, scope_stmt);
9039 :
9040 145 : gimple_seq_add_seq (&bind_body, gimple_omp_body (scope_stmt));
9041 :
9042 145 : gimple_omp_set_body (scope_stmt, NULL);
9043 :
9044 145 : gimple_seq clist = NULL;
9045 145 : lower_reduction_clauses (gimple_omp_scope_clauses (scope_stmt),
9046 : &bind_body, &clist, ctx);
9047 145 : if (clist)
9048 : {
9049 0 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_START);
9050 0 : gcall *g = gimple_build_call (fndecl, 0);
9051 0 : gimple_seq_add_stmt (&bind_body, g);
9052 0 : gimple_seq_add_seq (&bind_body, clist);
9053 0 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_END);
9054 0 : g = gimple_build_call (fndecl, 0);
9055 0 : gimple_seq_add_stmt (&bind_body, g);
9056 : }
9057 :
9058 145 : gimple_seq_add_seq (&bind_body, dlist);
9059 :
9060 145 : bind_body = maybe_catch_exception (bind_body);
9061 :
9062 145 : bool nowait = omp_find_clause (gimple_omp_scope_clauses (scope_stmt),
9063 145 : OMP_CLAUSE_NOWAIT) != NULL_TREE;
9064 145 : gimple *g = gimple_build_omp_return (nowait);
9065 145 : gimple_seq_add_stmt (&bind_body_tail, g);
9066 145 : gimple_seq_add_seq (&bind_body_tail, tred_dlist);
9067 145 : maybe_add_implicit_barrier_cancel (ctx, g, &bind_body_tail);
9068 145 : if (ctx->record_type)
9069 : {
9070 0 : gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
9071 0 : tree clobber = build_clobber (ctx->record_type);
9072 0 : gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
9073 : clobber), GSI_SAME_STMT);
9074 : }
9075 145 : gimple_seq_add_seq (&bind_body, bind_body_tail);
9076 :
9077 145 : gimple_bind_set_body (bind, bind_body);
9078 :
9079 145 : pop_gimplify_context (bind);
9080 :
9081 145 : gimple_bind_append_vars (bind, ctx->block_vars);
9082 145 : BLOCK_VARS (block) = ctx->block_vars;
9083 145 : if (BLOCK_VARS (block))
9084 113 : TREE_USED (block) = 1;
9085 145 : }
9086 :
9087 : /* Lower code for an OMP dispatch directive. */
9088 :
9089 : static void
9090 659 : lower_omp_dispatch (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9091 : {
9092 659 : tree block;
9093 659 : gimple *stmt = gsi_stmt (*gsi_p);
9094 659 : gbind *bind;
9095 :
9096 659 : push_gimplify_context ();
9097 :
9098 659 : block = make_node (BLOCK);
9099 659 : bind = gimple_build_bind (NULL, NULL, block);
9100 659 : gsi_replace (gsi_p, bind, true);
9101 :
9102 659 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
9103 659 : gimple_bind_set_body (bind, maybe_catch_exception (gimple_omp_body (stmt)));
9104 :
9105 659 : pop_gimplify_context (bind);
9106 :
9107 659 : gimple_bind_append_vars (bind, ctx->block_vars);
9108 659 : BLOCK_VARS (block) = ctx->block_vars;
9109 659 : }
9110 :
9111 : /* Expand code for an OpenMP master or masked directive. */
9112 :
9113 : static void
9114 1118 : lower_omp_master (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9115 : {
9116 1118 : tree block, lab = NULL, x, bfn_decl;
9117 1118 : gimple *stmt = gsi_stmt (*gsi_p);
9118 1118 : gbind *bind;
9119 1118 : location_t loc = gimple_location (stmt);
9120 1118 : gimple_seq tseq;
9121 1118 : tree filter = integer_zero_node;
9122 :
9123 1118 : push_gimplify_context ();
9124 :
9125 1118 : if (gimple_code (stmt) == GIMPLE_OMP_MASKED)
9126 : {
9127 387 : filter = omp_find_clause (gimple_omp_masked_clauses (stmt),
9128 : OMP_CLAUSE_FILTER);
9129 387 : if (filter)
9130 252 : filter = fold_convert (integer_type_node,
9131 : OMP_CLAUSE_FILTER_EXPR (filter));
9132 : else
9133 135 : filter = integer_zero_node;
9134 : }
9135 1118 : block = make_node (BLOCK);
9136 1118 : bind = gimple_build_bind (NULL, NULL, block);
9137 1118 : gsi_replace (gsi_p, bind, true);
9138 1118 : gimple_bind_add_stmt (bind, stmt);
9139 :
9140 1118 : bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_HAS_MASKED_THREAD_NUM);
9141 1118 : x = build_call_expr_loc (loc, bfn_decl, 1, filter);
9142 1118 : x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab));
9143 1118 : tseq = NULL;
9144 1118 : gimplify_and_add (x, &tseq);
9145 1118 : gimple_bind_add_seq (bind, tseq);
9146 :
9147 1118 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
9148 1118 : gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
9149 1118 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
9150 1118 : gimple_omp_set_body (stmt, NULL);
9151 :
9152 1118 : if (flag_openmp_ompt)
9153 : {
9154 : /* Insert call to GOMP_masked_end at the end of the body. */
9155 4 : tree decl = builtin_decl_explicit (BUILT_IN_GOMP_MASKED_END);
9156 4 : gcall *g = gimple_build_call (decl, 0);
9157 4 : gimple_bind_add_stmt (bind, g);
9158 : }
9159 :
9160 1118 : gimple_bind_add_stmt (bind, gimple_build_label (lab));
9161 :
9162 1118 : gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
9163 :
9164 1118 : pop_gimplify_context (bind);
9165 :
9166 1118 : gimple_bind_append_vars (bind, ctx->block_vars);
9167 1118 : BLOCK_VARS (block) = ctx->block_vars;
9168 1118 : }
9169 :
9170 : /* Helper function for lower_omp_task_reductions. For a specific PASS
9171 : find out the current clause it should be processed, or return false
9172 : if all have been processed already. */
9173 :
9174 : static inline bool
9175 7952 : omp_task_reduction_iterate (int pass, enum tree_code code,
9176 : enum omp_clause_code ccode, tree *c, tree *decl,
9177 : tree *type, tree *next)
9178 : {
9179 11300 : for (; *c; *c = omp_find_clause (OMP_CLAUSE_CHAIN (*c), ccode))
9180 : {
9181 6696 : if (ccode == OMP_CLAUSE_REDUCTION
9182 6640 : && code != OMP_TASKLOOP
9183 6640 : && !OMP_CLAUSE_REDUCTION_TASK (*c))
9184 56 : continue;
9185 6584 : *decl = OMP_CLAUSE_DECL (*c);
9186 6584 : *type = TREE_TYPE (*decl);
9187 6584 : if (TREE_CODE (*decl) == MEM_REF)
9188 : {
9189 1940 : if (pass != 1)
9190 970 : continue;
9191 : }
9192 : else
9193 : {
9194 4644 : if (omp_privatize_by_reference (*decl))
9195 152 : *type = TREE_TYPE (*type);
9196 4644 : if (pass != (!TREE_CONSTANT (TYPE_SIZE_UNIT (*type))))
9197 2322 : continue;
9198 : }
9199 3292 : *next = omp_find_clause (OMP_CLAUSE_CHAIN (*c), ccode);
9200 3292 : return true;
9201 : }
9202 4660 : *decl = NULL_TREE;
9203 4660 : *type = NULL_TREE;
9204 4660 : *next = NULL_TREE;
9205 4660 : return false;
9206 : }
9207 :
9208 : /* Lower task_reduction and reduction clauses (the latter unless CODE is
9209 : OMP_TASKGROUP only with task modifier). Register mapping of those in
9210 : START sequence and reducing them and unregister them in the END sequence. */
9211 :
9212 : static void
9213 1374 : lower_omp_task_reductions (omp_context *ctx, enum tree_code code, tree clauses,
9214 : gimple_seq *start, gimple_seq *end)
9215 : {
9216 838 : enum omp_clause_code ccode
9217 : = (code == OMP_TASKGROUP
9218 1374 : ? OMP_CLAUSE_TASK_REDUCTION : OMP_CLAUSE_REDUCTION);
9219 1374 : tree cancellable = NULL_TREE;
9220 1374 : clauses = omp_task_reductions_find_first (clauses, code, ccode);
9221 1374 : if (clauses == NULL_TREE)
9222 209 : return;
9223 1165 : if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE)
9224 : {
9225 316 : for (omp_context *outer = ctx->outer; outer; outer = outer->outer)
9226 238 : if (gimple_code (outer->stmt) == GIMPLE_OMP_PARALLEL
9227 238 : && outer->cancellable)
9228 : {
9229 14 : cancellable = error_mark_node;
9230 14 : break;
9231 : }
9232 224 : else if (gimple_code (outer->stmt) != GIMPLE_OMP_TASKGROUP
9233 224 : && gimple_code (outer->stmt) != GIMPLE_OMP_SCOPE)
9234 : break;
9235 : }
9236 1165 : tree record_type = lang_hooks.types.make_type (RECORD_TYPE);
9237 1165 : tree *last = &TYPE_FIELDS (record_type);
9238 1165 : unsigned cnt = 0;
9239 1165 : if (cancellable)
9240 : {
9241 14 : tree field = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE,
9242 : ptr_type_node);
9243 14 : tree ifield = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE,
9244 : integer_type_node);
9245 14 : *last = field;
9246 14 : DECL_CHAIN (field) = ifield;
9247 14 : last = &DECL_CHAIN (ifield);
9248 14 : DECL_CONTEXT (field) = record_type;
9249 14 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (field))
9250 0 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (field));
9251 14 : DECL_CONTEXT (ifield) = record_type;
9252 14 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (ifield))
9253 0 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (ifield));
9254 : }
9255 3495 : for (int pass = 0; pass < 2; pass++)
9256 : {
9257 2330 : tree decl, type, next;
9258 2330 : for (tree c = clauses;
9259 3976 : omp_task_reduction_iterate (pass, code, ccode,
9260 1646 : &c, &decl, &type, &next); c = next)
9261 : {
9262 1646 : ++cnt;
9263 1646 : tree new_type = type;
9264 1646 : if (ctx->outer)
9265 1165 : new_type = remap_type (type, &ctx->outer->cb);
9266 1646 : tree field
9267 1646 : = build_decl (OMP_CLAUSE_LOCATION (c), FIELD_DECL,
9268 1646 : DECL_P (decl) ? DECL_NAME (decl) : NULL_TREE,
9269 : new_type);
9270 1646 : if (DECL_P (decl) && type == TREE_TYPE (decl))
9271 : {
9272 1123 : SET_DECL_ALIGN (field, DECL_ALIGN (decl));
9273 1123 : DECL_USER_ALIGN (field) = DECL_USER_ALIGN (decl);
9274 1123 : TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (decl);
9275 : }
9276 : else
9277 523 : SET_DECL_ALIGN (field, TYPE_ALIGN (type));
9278 1646 : DECL_CONTEXT (field) = record_type;
9279 1646 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (field))
9280 1187 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (field));
9281 1646 : *last = field;
9282 1646 : last = &DECL_CHAIN (field);
9283 1646 : tree bfield
9284 1646 : = build_decl (OMP_CLAUSE_LOCATION (c), FIELD_DECL, NULL_TREE,
9285 : boolean_type_node);
9286 1646 : DECL_CONTEXT (bfield) = record_type;
9287 1646 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (bfield))
9288 0 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (bfield));
9289 1646 : *last = bfield;
9290 1646 : last = &DECL_CHAIN (bfield);
9291 : }
9292 : }
9293 1165 : *last = NULL_TREE;
9294 1165 : layout_type (record_type);
9295 :
9296 : /* Build up an array which registers with the runtime all the reductions
9297 : and deregisters them at the end. Format documented in libgomp/task.c. */
9298 1165 : tree atype = build_array_type_nelts (pointer_sized_int_node, 7 + cnt * 3);
9299 1165 : tree avar = create_tmp_var_raw (atype);
9300 1165 : gimple_add_tmp_var (avar);
9301 1165 : TREE_ADDRESSABLE (avar) = 1;
9302 1165 : tree r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_zero_node,
9303 : NULL_TREE, NULL_TREE);
9304 1165 : tree t = build_int_cst (pointer_sized_int_node, cnt);
9305 1165 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9306 1165 : gimple_seq seq = NULL;
9307 1165 : tree sz = fold_convert (pointer_sized_int_node,
9308 : TYPE_SIZE_UNIT (record_type));
9309 1165 : int cachesz = 64;
9310 1165 : sz = fold_build2 (PLUS_EXPR, pointer_sized_int_node, sz,
9311 : build_int_cst (pointer_sized_int_node, cachesz - 1));
9312 1165 : sz = fold_build2 (BIT_AND_EXPR, pointer_sized_int_node, sz,
9313 : build_int_cst (pointer_sized_int_node, ~(cachesz - 1)));
9314 1165 : ctx->task_reductions.create (1 + cnt);
9315 1165 : ctx->task_reduction_map = new hash_map<tree, unsigned>;
9316 2330 : ctx->task_reductions.quick_push (TREE_CODE (sz) == INTEGER_CST
9317 1205 : ? sz : NULL_TREE);
9318 1165 : sz = force_gimple_operand (sz, &seq, true, NULL_TREE);
9319 1165 : gimple_seq_add_seq (start, seq);
9320 1165 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_one_node,
9321 : NULL_TREE, NULL_TREE);
9322 1165 : gimple_seq_add_stmt (start, gimple_build_assign (r, sz));
9323 1165 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (2),
9324 : NULL_TREE, NULL_TREE);
9325 1165 : t = build_int_cst (pointer_sized_int_node,
9326 1165 : MAX (TYPE_ALIGN_UNIT (record_type), (unsigned) cachesz));
9327 1165 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9328 1165 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (3),
9329 : NULL_TREE, NULL_TREE);
9330 1165 : t = build_int_cst (pointer_sized_int_node, -1);
9331 1165 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9332 1165 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (4),
9333 : NULL_TREE, NULL_TREE);
9334 1165 : t = build_int_cst (pointer_sized_int_node, 0);
9335 1165 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9336 :
9337 : /* In end, build a loop that iterates from 0 to < omp_get_num_threads ()
9338 : and for each task reduction checks a bool right after the private variable
9339 : within that thread's chunk; if the bool is clear, it hasn't been
9340 : initialized and thus isn't going to be reduced nor destructed, otherwise
9341 : reduce and destruct it. */
9342 1165 : tree idx = create_tmp_var (size_type_node);
9343 1165 : gimple_seq_add_stmt (end, gimple_build_assign (idx, size_zero_node));
9344 1165 : tree num_thr_sz = create_tmp_var (size_type_node);
9345 1165 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
9346 1165 : tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
9347 1165 : tree lab3 = NULL_TREE, lab7 = NULL_TREE;
9348 1165 : gimple *g;
9349 1165 : if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE)
9350 : {
9351 : /* For worksharing constructs or scope, only perform it in the master
9352 : thread, with the exception of cancelled implicit barriers - then only
9353 : handle the current thread. */
9354 281 : tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
9355 281 : t = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
9356 281 : tree thr_num = create_tmp_var (integer_type_node);
9357 281 : g = gimple_build_call (t, 0);
9358 281 : gimple_call_set_lhs (g, thr_num);
9359 281 : gimple_seq_add_stmt (end, g);
9360 281 : if (cancellable)
9361 : {
9362 14 : tree c;
9363 14 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9364 14 : tree lab6 = create_artificial_label (UNKNOWN_LOCATION);
9365 14 : lab3 = create_artificial_label (UNKNOWN_LOCATION);
9366 14 : if (code == OMP_FOR)
9367 4 : c = gimple_omp_for_clauses (ctx->stmt);
9368 10 : else if (code == OMP_SECTIONS)
9369 0 : c = gimple_omp_sections_clauses (ctx->stmt);
9370 : else /* if (code == OMP_SCOPE) */
9371 10 : c = gimple_omp_scope_clauses (ctx->stmt);
9372 14 : c = OMP_CLAUSE_DECL (omp_find_clause (c, OMP_CLAUSE__REDUCTEMP_));
9373 14 : cancellable = c;
9374 14 : g = gimple_build_cond (NE_EXPR, c, build_zero_cst (TREE_TYPE (c)),
9375 : lab5, lab6);
9376 14 : gimple_seq_add_stmt (end, g);
9377 14 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9378 14 : g = gimple_build_assign (idx, NOP_EXPR, thr_num);
9379 14 : gimple_seq_add_stmt (end, g);
9380 14 : g = gimple_build_assign (num_thr_sz, PLUS_EXPR, idx,
9381 14 : build_one_cst (TREE_TYPE (idx)));
9382 14 : gimple_seq_add_stmt (end, g);
9383 14 : gimple_seq_add_stmt (end, gimple_build_goto (lab3));
9384 14 : gimple_seq_add_stmt (end, gimple_build_label (lab6));
9385 : }
9386 281 : g = gimple_build_cond (NE_EXPR, thr_num, integer_zero_node, lab2, lab4);
9387 281 : gimple_seq_add_stmt (end, g);
9388 281 : gimple_seq_add_stmt (end, gimple_build_label (lab4));
9389 : }
9390 1165 : if (code != OMP_PARALLEL)
9391 : {
9392 1105 : t = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
9393 1105 : tree num_thr = create_tmp_var (integer_type_node);
9394 1105 : g = gimple_build_call (t, 0);
9395 1105 : gimple_call_set_lhs (g, num_thr);
9396 1105 : gimple_seq_add_stmt (end, g);
9397 1105 : g = gimple_build_assign (num_thr_sz, NOP_EXPR, num_thr);
9398 1105 : gimple_seq_add_stmt (end, g);
9399 1105 : if (cancellable)
9400 14 : gimple_seq_add_stmt (end, gimple_build_label (lab3));
9401 : }
9402 : else
9403 : {
9404 60 : tree c = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
9405 : OMP_CLAUSE__REDUCTEMP_);
9406 60 : t = fold_convert (pointer_sized_int_node, OMP_CLAUSE_DECL (c));
9407 60 : t = fold_convert (size_type_node, t);
9408 60 : gimplify_assign (num_thr_sz, t, end);
9409 : }
9410 1165 : t = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (2),
9411 : NULL_TREE, NULL_TREE);
9412 1165 : tree data = create_tmp_var (pointer_sized_int_node);
9413 1165 : gimple_seq_add_stmt (end, gimple_build_assign (data, t));
9414 1165 : if (code == OMP_TASKLOOP)
9415 : {
9416 497 : lab7 = create_artificial_label (UNKNOWN_LOCATION);
9417 497 : g = gimple_build_cond (NE_EXPR, data,
9418 : build_zero_cst (pointer_sized_int_node),
9419 : lab1, lab7);
9420 497 : gimple_seq_add_stmt (end, g);
9421 : }
9422 1165 : gimple_seq_add_stmt (end, gimple_build_label (lab1));
9423 1165 : tree ptr;
9424 1165 : if (TREE_CODE (TYPE_SIZE_UNIT (record_type)) == INTEGER_CST)
9425 1125 : ptr = create_tmp_var (build_pointer_type (record_type));
9426 : else
9427 40 : ptr = create_tmp_var (ptr_type_node);
9428 1165 : gimple_seq_add_stmt (end, gimple_build_assign (ptr, NOP_EXPR, data));
9429 :
9430 1165 : tree field = TYPE_FIELDS (record_type);
9431 1165 : cnt = 0;
9432 1165 : if (cancellable)
9433 14 : field = DECL_CHAIN (DECL_CHAIN (field));
9434 3495 : for (int pass = 0; pass < 2; pass++)
9435 : {
9436 2330 : tree decl, type, next;
9437 2330 : for (tree c = clauses;
9438 3976 : omp_task_reduction_iterate (pass, code, ccode,
9439 1646 : &c, &decl, &type, &next); c = next)
9440 : {
9441 1646 : tree var = decl, ref;
9442 1646 : if (TREE_CODE (decl) == MEM_REF)
9443 : {
9444 485 : var = TREE_OPERAND (var, 0);
9445 485 : if (TREE_CODE (var) == POINTER_PLUS_EXPR)
9446 108 : var = TREE_OPERAND (var, 0);
9447 485 : tree v = var;
9448 485 : if (TREE_CODE (var) == ADDR_EXPR)
9449 310 : var = TREE_OPERAND (var, 0);
9450 175 : else if (INDIRECT_REF_P (var))
9451 4 : var = TREE_OPERAND (var, 0);
9452 485 : tree orig_var = var;
9453 485 : if (is_variable_sized (var))
9454 : {
9455 31 : gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
9456 31 : var = DECL_VALUE_EXPR (var);
9457 31 : gcc_assert (INDIRECT_REF_P (var));
9458 31 : var = TREE_OPERAND (var, 0);
9459 31 : gcc_assert (DECL_P (var));
9460 : }
9461 485 : t = ref = maybe_lookup_decl_in_outer_ctx (var, ctx);
9462 485 : if (orig_var != var)
9463 31 : gcc_assert (TREE_CODE (v) == ADDR_EXPR);
9464 454 : else if (TREE_CODE (v) == ADDR_EXPR)
9465 279 : t = build_fold_addr_expr (t);
9466 175 : else if (INDIRECT_REF_P (v))
9467 4 : t = build_fold_indirect_ref (t);
9468 485 : if (TREE_CODE (TREE_OPERAND (decl, 0)) == POINTER_PLUS_EXPR)
9469 : {
9470 108 : tree b = TREE_OPERAND (TREE_OPERAND (decl, 0), 1);
9471 108 : b = maybe_lookup_decl_in_outer_ctx (b, ctx);
9472 108 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t, b);
9473 : }
9474 485 : if (!integer_zerop (TREE_OPERAND (decl, 1)))
9475 219 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t,
9476 : fold_convert (size_type_node,
9477 : TREE_OPERAND (decl, 1)));
9478 : }
9479 : else
9480 : {
9481 1161 : t = ref = maybe_lookup_decl_in_outer_ctx (var, ctx);
9482 1161 : if (!omp_privatize_by_reference (decl))
9483 1123 : t = build_fold_addr_expr (t);
9484 : }
9485 1646 : t = fold_convert (pointer_sized_int_node, t);
9486 1646 : seq = NULL;
9487 1646 : t = force_gimple_operand (t, &seq, true, NULL_TREE);
9488 1646 : gimple_seq_add_seq (start, seq);
9489 1646 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar,
9490 1646 : size_int (7 + cnt * 3), NULL_TREE, NULL_TREE);
9491 1646 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9492 1646 : t = unshare_expr (byte_position (field));
9493 1646 : t = fold_convert (pointer_sized_int_node, t);
9494 1646 : ctx->task_reduction_map->put (c, cnt);
9495 3292 : ctx->task_reductions.quick_push (TREE_CODE (t) == INTEGER_CST
9496 2018 : ? t : NULL_TREE);
9497 1646 : seq = NULL;
9498 1646 : t = force_gimple_operand (t, &seq, true, NULL_TREE);
9499 1646 : gimple_seq_add_seq (start, seq);
9500 1646 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar,
9501 1646 : size_int (7 + cnt * 3 + 1), NULL_TREE, NULL_TREE);
9502 1646 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9503 :
9504 1646 : tree bfield = DECL_CHAIN (field);
9505 1646 : tree cond;
9506 1646 : if (code == OMP_PARALLEL
9507 1646 : || code == OMP_FOR
9508 : || code == OMP_SECTIONS
9509 : || code == OMP_SCOPE)
9510 : /* In parallel, worksharing or scope all threads unconditionally
9511 : initialize all their task reduction private variables. */
9512 548 : cond = boolean_true_node;
9513 1098 : else if (TREE_TYPE (ptr) == ptr_type_node)
9514 : {
9515 261 : cond = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr,
9516 : unshare_expr (byte_position (bfield)));
9517 261 : seq = NULL;
9518 261 : cond = force_gimple_operand (cond, &seq, true, NULL_TREE);
9519 261 : gimple_seq_add_seq (end, seq);
9520 261 : tree pbool = build_pointer_type (TREE_TYPE (bfield));
9521 261 : cond = build2 (MEM_REF, TREE_TYPE (bfield), cond,
9522 : build_int_cst (pbool, 0));
9523 : }
9524 : else
9525 837 : cond = build3 (COMPONENT_REF, TREE_TYPE (bfield),
9526 : build_simple_mem_ref (ptr), bfield, NULL_TREE);
9527 1646 : tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
9528 1646 : tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
9529 1646 : tree condv = create_tmp_var (boolean_type_node);
9530 1646 : gimple_seq_add_stmt (end, gimple_build_assign (condv, cond));
9531 1646 : g = gimple_build_cond (NE_EXPR, condv, boolean_false_node,
9532 : lab3, lab4);
9533 1646 : gimple_seq_add_stmt (end, g);
9534 1646 : gimple_seq_add_stmt (end, gimple_build_label (lab3));
9535 1646 : if (cancellable && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) == NULL_TREE)
9536 : {
9537 : /* If this reduction doesn't need destruction and parallel
9538 : has been cancelled, there is nothing to do for this
9539 : reduction, so jump around the merge operation. */
9540 18 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9541 18 : g = gimple_build_cond (NE_EXPR, cancellable,
9542 18 : build_zero_cst (TREE_TYPE (cancellable)),
9543 : lab4, lab5);
9544 18 : gimple_seq_add_stmt (end, g);
9545 18 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9546 : }
9547 :
9548 1646 : tree new_var;
9549 1646 : if (TREE_TYPE (ptr) == ptr_type_node)
9550 : {
9551 428 : new_var = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr,
9552 : unshare_expr (byte_position (field)));
9553 428 : seq = NULL;
9554 428 : new_var = force_gimple_operand (new_var, &seq, true, NULL_TREE);
9555 428 : gimple_seq_add_seq (end, seq);
9556 428 : tree pbool = build_pointer_type (TREE_TYPE (field));
9557 428 : new_var = build2 (MEM_REF, TREE_TYPE (field), new_var,
9558 : build_int_cst (pbool, 0));
9559 : }
9560 : else
9561 1218 : new_var = build3 (COMPONENT_REF, TREE_TYPE (field),
9562 : build_simple_mem_ref (ptr), field, NULL_TREE);
9563 :
9564 1646 : enum tree_code rcode = OMP_CLAUSE_REDUCTION_CODE (c);
9565 1646 : if (TREE_CODE (decl) != MEM_REF
9566 1646 : && omp_privatize_by_reference (decl))
9567 38 : ref = build_simple_mem_ref (ref);
9568 : /* reduction(-:var) sums up the partial results, so it acts
9569 : identically to reduction(+:var). */
9570 1646 : if (rcode == MINUS_EXPR)
9571 0 : rcode = PLUS_EXPR;
9572 1646 : if (TREE_CODE (decl) == MEM_REF)
9573 : {
9574 485 : tree type = TREE_TYPE (new_var);
9575 485 : tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
9576 485 : tree i = create_tmp_var (TREE_TYPE (v));
9577 485 : tree ptype = build_pointer_type (TREE_TYPE (type));
9578 485 : if (DECL_P (v))
9579 : {
9580 142 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
9581 142 : tree vv = create_tmp_var (TREE_TYPE (v));
9582 142 : gimplify_assign (vv, v, start);
9583 142 : v = vv;
9584 : }
9585 485 : ref = build4 (ARRAY_REF, pointer_sized_int_node, avar,
9586 485 : size_int (7 + cnt * 3), NULL_TREE, NULL_TREE);
9587 485 : new_var = build_fold_addr_expr (new_var);
9588 485 : new_var = fold_convert (ptype, new_var);
9589 485 : ref = fold_convert (ptype, ref);
9590 485 : tree m = create_tmp_var (ptype);
9591 485 : gimplify_assign (m, new_var, end);
9592 485 : new_var = m;
9593 485 : m = create_tmp_var (ptype);
9594 485 : gimplify_assign (m, ref, end);
9595 485 : ref = m;
9596 485 : gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), end);
9597 485 : tree body = create_artificial_label (UNKNOWN_LOCATION);
9598 485 : tree endl = create_artificial_label (UNKNOWN_LOCATION);
9599 485 : gimple_seq_add_stmt (end, gimple_build_label (body));
9600 485 : tree priv = build_simple_mem_ref (new_var);
9601 485 : tree out = build_simple_mem_ref (ref);
9602 485 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
9603 : {
9604 161 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
9605 161 : tree decl_placeholder
9606 161 : = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
9607 161 : tree lab6 = NULL_TREE;
9608 161 : if (cancellable)
9609 : {
9610 : /* If this reduction needs destruction and parallel
9611 : has been cancelled, jump around the merge operation
9612 : to the destruction. */
9613 4 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9614 4 : lab6 = create_artificial_label (UNKNOWN_LOCATION);
9615 4 : tree zero = build_zero_cst (TREE_TYPE (cancellable));
9616 4 : g = gimple_build_cond (NE_EXPR, cancellable, zero,
9617 : lab6, lab5);
9618 4 : gimple_seq_add_stmt (end, g);
9619 4 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9620 : }
9621 161 : SET_DECL_VALUE_EXPR (placeholder, out);
9622 161 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
9623 161 : SET_DECL_VALUE_EXPR (decl_placeholder, priv);
9624 161 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
9625 161 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
9626 322 : gimple_seq_add_seq (end,
9627 161 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
9628 161 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
9629 161 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
9630 : {
9631 56 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
9632 56 : OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c) = NULL;
9633 : }
9634 161 : if (cancellable)
9635 4 : gimple_seq_add_stmt (end, gimple_build_label (lab6));
9636 161 : tree x = lang_hooks.decls.omp_clause_dtor (c, priv);
9637 161 : if (x)
9638 : {
9639 137 : gimple_seq tseq = NULL;
9640 137 : gimplify_stmt (&x, &tseq);
9641 137 : gimple_seq_add_seq (end, tseq);
9642 : }
9643 : }
9644 : else
9645 : {
9646 324 : tree x = build2 (rcode, TREE_TYPE (out), out, priv);
9647 324 : out = unshare_expr (out);
9648 324 : gimplify_assign (out, x, end);
9649 : }
9650 485 : gimple *g
9651 485 : = gimple_build_assign (new_var, POINTER_PLUS_EXPR, new_var,
9652 485 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
9653 485 : gimple_seq_add_stmt (end, g);
9654 485 : g = gimple_build_assign (ref, POINTER_PLUS_EXPR, ref,
9655 485 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
9656 485 : gimple_seq_add_stmt (end, g);
9657 485 : g = gimple_build_assign (i, PLUS_EXPR, i,
9658 485 : build_int_cst (TREE_TYPE (i), 1));
9659 485 : gimple_seq_add_stmt (end, g);
9660 485 : g = gimple_build_cond (LE_EXPR, i, v, body, endl);
9661 485 : gimple_seq_add_stmt (end, g);
9662 485 : gimple_seq_add_stmt (end, gimple_build_label (endl));
9663 : }
9664 1161 : else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
9665 : {
9666 127 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
9667 127 : tree oldv = NULL_TREE;
9668 127 : tree lab6 = NULL_TREE;
9669 127 : if (cancellable)
9670 : {
9671 : /* If this reduction needs destruction and parallel
9672 : has been cancelled, jump around the merge operation
9673 : to the destruction. */
9674 4 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9675 4 : lab6 = create_artificial_label (UNKNOWN_LOCATION);
9676 4 : tree zero = build_zero_cst (TREE_TYPE (cancellable));
9677 4 : g = gimple_build_cond (NE_EXPR, cancellable, zero,
9678 : lab6, lab5);
9679 4 : gimple_seq_add_stmt (end, g);
9680 4 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9681 : }
9682 127 : if (omp_privatize_by_reference (decl)
9683 137 : && !useless_type_conversion_p (TREE_TYPE (placeholder),
9684 10 : TREE_TYPE (ref)))
9685 0 : ref = build_fold_addr_expr_loc (OMP_CLAUSE_LOCATION (c), ref);
9686 127 : ref = build_fold_addr_expr_loc (OMP_CLAUSE_LOCATION (c), ref);
9687 127 : tree refv = create_tmp_var (TREE_TYPE (ref));
9688 127 : gimplify_assign (refv, ref, end);
9689 127 : ref = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), refv);
9690 127 : SET_DECL_VALUE_EXPR (placeholder, ref);
9691 127 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
9692 127 : tree d = maybe_lookup_decl (decl, ctx);
9693 127 : gcc_assert (d);
9694 127 : if (DECL_HAS_VALUE_EXPR_P (d))
9695 7 : oldv = DECL_VALUE_EXPR (d);
9696 127 : if (omp_privatize_by_reference (var))
9697 : {
9698 10 : tree v = fold_convert (TREE_TYPE (d),
9699 : build_fold_addr_expr (new_var));
9700 10 : SET_DECL_VALUE_EXPR (d, v);
9701 : }
9702 : else
9703 117 : SET_DECL_VALUE_EXPR (d, new_var);
9704 127 : DECL_HAS_VALUE_EXPR_P (d) = 1;
9705 127 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
9706 127 : if (oldv)
9707 7 : SET_DECL_VALUE_EXPR (d, oldv);
9708 : else
9709 : {
9710 120 : SET_DECL_VALUE_EXPR (d, NULL_TREE);
9711 120 : DECL_HAS_VALUE_EXPR_P (d) = 0;
9712 : }
9713 127 : gimple_seq_add_seq (end, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
9714 127 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
9715 127 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
9716 24 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
9717 127 : if (cancellable)
9718 4 : gimple_seq_add_stmt (end, gimple_build_label (lab6));
9719 127 : tree x = lang_hooks.decls.omp_clause_dtor (c, new_var);
9720 127 : if (x)
9721 : {
9722 29 : gimple_seq tseq = NULL;
9723 29 : gimplify_stmt (&x, &tseq);
9724 29 : gimple_seq_add_seq (end, tseq);
9725 : }
9726 : }
9727 : else
9728 : {
9729 1034 : tree x = build2 (rcode, TREE_TYPE (ref), ref, new_var);
9730 1034 : ref = unshare_expr (ref);
9731 1034 : gimplify_assign (ref, x, end);
9732 : }
9733 1646 : gimple_seq_add_stmt (end, gimple_build_label (lab4));
9734 1646 : ++cnt;
9735 1646 : field = DECL_CHAIN (bfield);
9736 : }
9737 : }
9738 :
9739 1165 : if (code == OMP_TASKGROUP)
9740 : {
9741 327 : t = builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_REDUCTION_REGISTER);
9742 327 : g = gimple_build_call (t, 1, build_fold_addr_expr (avar));
9743 327 : gimple_seq_add_stmt (start, g);
9744 : }
9745 : else
9746 : {
9747 838 : tree c;
9748 838 : if (code == OMP_FOR)
9749 227 : c = gimple_omp_for_clauses (ctx->stmt);
9750 611 : else if (code == OMP_SECTIONS)
9751 8 : c = gimple_omp_sections_clauses (ctx->stmt);
9752 603 : else if (code == OMP_SCOPE)
9753 46 : c = gimple_omp_scope_clauses (ctx->stmt);
9754 : else
9755 557 : c = gimple_omp_taskreg_clauses (ctx->stmt);
9756 838 : c = omp_find_clause (c, OMP_CLAUSE__REDUCTEMP_);
9757 838 : t = fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (c)),
9758 : build_fold_addr_expr (avar));
9759 838 : gimplify_assign (OMP_CLAUSE_DECL (c), t, start);
9760 : }
9761 :
9762 1165 : gimple_seq_add_stmt (end, gimple_build_assign (data, PLUS_EXPR, data, sz));
9763 1165 : gimple_seq_add_stmt (end, gimple_build_assign (idx, PLUS_EXPR, idx,
9764 : size_one_node));
9765 1165 : g = gimple_build_cond (NE_EXPR, idx, num_thr_sz, lab1, lab2);
9766 1165 : gimple_seq_add_stmt (end, g);
9767 1165 : gimple_seq_add_stmt (end, gimple_build_label (lab2));
9768 1165 : if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE)
9769 : {
9770 281 : enum built_in_function bfn
9771 : = BUILT_IN_GOMP_WORKSHARE_TASK_REDUCTION_UNREGISTER;
9772 281 : t = builtin_decl_explicit (bfn);
9773 281 : tree c_bool_type = TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (t)));
9774 281 : tree arg;
9775 281 : if (cancellable)
9776 : {
9777 14 : arg = create_tmp_var (c_bool_type);
9778 14 : gimple_seq_add_stmt (end, gimple_build_assign (arg, NOP_EXPR,
9779 : cancellable));
9780 : }
9781 : else
9782 267 : arg = build_int_cst (c_bool_type, 0);
9783 281 : g = gimple_build_call (t, 1, arg);
9784 281 : }
9785 : else
9786 : {
9787 884 : t = builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_REDUCTION_UNREGISTER);
9788 884 : g = gimple_build_call (t, 1, build_fold_addr_expr (avar));
9789 : }
9790 1165 : gimple_seq_add_stmt (end, g);
9791 1165 : if (lab7)
9792 497 : gimple_seq_add_stmt (end, gimple_build_label (lab7));
9793 1165 : t = build_constructor (atype, NULL);
9794 1165 : TREE_THIS_VOLATILE (t) = 1;
9795 1165 : gimple_seq_add_stmt (end, gimple_build_assign (avar, t));
9796 : }
9797 :
9798 : /* Expand code for an OpenMP taskgroup directive. */
9799 :
9800 : static void
9801 536 : lower_omp_taskgroup (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9802 : {
9803 536 : gimple *stmt = gsi_stmt (*gsi_p);
9804 536 : gcall *x;
9805 536 : gbind *bind;
9806 536 : gimple_seq dseq = NULL;
9807 536 : tree block = make_node (BLOCK);
9808 :
9809 536 : bind = gimple_build_bind (NULL, NULL, block);
9810 536 : gsi_replace (gsi_p, bind, true);
9811 536 : gimple_bind_add_stmt (bind, stmt);
9812 :
9813 536 : push_gimplify_context ();
9814 :
9815 536 : x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START),
9816 : 0);
9817 536 : gimple_bind_add_stmt (bind, x);
9818 :
9819 536 : lower_omp_task_reductions (ctx, OMP_TASKGROUP,
9820 : gimple_omp_taskgroup_clauses (stmt),
9821 : gimple_bind_body_ptr (bind), &dseq);
9822 :
9823 536 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
9824 536 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
9825 536 : gimple_omp_set_body (stmt, NULL);
9826 :
9827 536 : gimple_bind_add_seq (bind, dseq);
9828 :
9829 536 : pop_gimplify_context (bind);
9830 :
9831 536 : gimple_bind_append_vars (bind, ctx->block_vars);
9832 536 : BLOCK_VARS (block) = ctx->block_vars;
9833 536 : }
9834 :
9835 :
9836 : /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */
9837 :
9838 : static void
9839 0 : lower_omp_ordered_clauses (gimple_stmt_iterator *gsi_p, gomp_ordered *ord_stmt,
9840 : omp_context *ctx)
9841 : {
9842 0 : struct omp_for_data fd;
9843 0 : if (!ctx->outer || gimple_code (ctx->outer->stmt) != GIMPLE_OMP_FOR)
9844 0 : return;
9845 :
9846 0 : unsigned int len = gimple_omp_for_collapse (ctx->outer->stmt);
9847 0 : struct omp_for_data_loop *loops = XALLOCAVEC (struct omp_for_data_loop, len);
9848 0 : omp_extract_for_data (as_a <gomp_for *> (ctx->outer->stmt), &fd, loops);
9849 0 : if (!fd.ordered)
9850 : return;
9851 :
9852 0 : tree *list_p = gimple_omp_ordered_clauses_ptr (ord_stmt);
9853 0 : tree c = gimple_omp_ordered_clauses (ord_stmt);
9854 0 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS
9855 0 : && OMP_CLAUSE_DOACROSS_KIND (c) == OMP_CLAUSE_DOACROSS_SINK)
9856 : {
9857 : /* Merge depend clauses from multiple adjacent
9858 : #pragma omp ordered depend(sink:...) constructs
9859 : into one #pragma omp ordered depend(sink:...), so that
9860 : we can optimize them together. */
9861 0 : gimple_stmt_iterator gsi = *gsi_p;
9862 0 : gsi_next (&gsi);
9863 0 : while (!gsi_end_p (gsi))
9864 : {
9865 0 : gimple *stmt = gsi_stmt (gsi);
9866 0 : if (is_gimple_debug (stmt)
9867 0 : || gimple_code (stmt) == GIMPLE_NOP)
9868 : {
9869 0 : gsi_next (&gsi);
9870 0 : continue;
9871 : }
9872 0 : if (gimple_code (stmt) != GIMPLE_OMP_ORDERED)
9873 : break;
9874 0 : gomp_ordered *ord_stmt2 = as_a <gomp_ordered *> (stmt);
9875 0 : c = gimple_omp_ordered_clauses (ord_stmt2);
9876 0 : if (c == NULL_TREE
9877 0 : || OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DOACROSS
9878 0 : || OMP_CLAUSE_DOACROSS_KIND (c) != OMP_CLAUSE_DOACROSS_SINK)
9879 : break;
9880 0 : while (*list_p)
9881 0 : list_p = &OMP_CLAUSE_CHAIN (*list_p);
9882 0 : *list_p = c;
9883 0 : gsi_remove (&gsi, true);
9884 : }
9885 : }
9886 :
9887 : /* Canonicalize sink dependence clauses into one folded clause if
9888 : possible.
9889 :
9890 : The basic algorithm is to create a sink vector whose first
9891 : element is the GCD of all the first elements, and whose remaining
9892 : elements are the minimum of the subsequent columns.
9893 :
9894 : We ignore dependence vectors whose first element is zero because
9895 : such dependencies are known to be executed by the same thread.
9896 :
9897 : We take into account the direction of the loop, so a minimum
9898 : becomes a maximum if the loop is iterating forwards. We also
9899 : ignore sink clauses where the loop direction is unknown, or where
9900 : the offsets are clearly invalid because they are not a multiple
9901 : of the loop increment.
9902 :
9903 : For example:
9904 :
9905 : #pragma omp for ordered(2)
9906 : for (i=0; i < N; ++i)
9907 : for (j=0; j < M; ++j)
9908 : {
9909 : #pragma omp ordered \
9910 : depend(sink:i-8,j-2) \
9911 : depend(sink:i,j-1) \ // Completely ignored because i+0.
9912 : depend(sink:i-4,j-3) \
9913 : depend(sink:i-6,j-4)
9914 : #pragma omp ordered depend(source)
9915 : }
9916 :
9917 : Folded clause is:
9918 :
9919 : depend(sink:-gcd(8,4,6),-min(2,3,4))
9920 : -or-
9921 : depend(sink:-2,-2)
9922 : */
9923 :
9924 : /* FIXME: Computing GCD's where the first element is zero is
9925 : non-trivial in the presence of collapsed loops. Do this later. */
9926 0 : if (fd.collapse > 1)
9927 : return;
9928 :
9929 0 : wide_int *folded_deps = XALLOCAVEC (wide_int, 2 * len - 1);
9930 :
9931 : /* wide_int is not a POD so it must be default-constructed. */
9932 0 : for (unsigned i = 0; i != 2 * len - 1; ++i)
9933 0 : new (static_cast<void*>(folded_deps + i)) wide_int ();
9934 :
9935 : tree folded_dep = NULL_TREE;
9936 : /* TRUE if the first dimension's offset is negative. */
9937 : bool neg_offset_p = false;
9938 :
9939 : list_p = gimple_omp_ordered_clauses_ptr (ord_stmt);
9940 : unsigned int i;
9941 0 : while ((c = *list_p) != NULL)
9942 : {
9943 0 : bool remove = false;
9944 :
9945 0 : gcc_assert (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS);
9946 0 : if (OMP_CLAUSE_DOACROSS_KIND (c) != OMP_CLAUSE_DOACROSS_SINK)
9947 0 : goto next_ordered_clause;
9948 :
9949 0 : tree vec;
9950 0 : for (vec = OMP_CLAUSE_DECL (c), i = 0;
9951 0 : vec && TREE_CODE (vec) == TREE_LIST;
9952 0 : vec = TREE_CHAIN (vec), ++i)
9953 : {
9954 0 : gcc_assert (i < len);
9955 :
9956 : /* omp_extract_for_data has canonicalized the condition. */
9957 0 : gcc_assert (fd.loops[i].cond_code == LT_EXPR
9958 : || fd.loops[i].cond_code == GT_EXPR);
9959 0 : bool forward = fd.loops[i].cond_code == LT_EXPR;
9960 0 : bool maybe_lexically_later = true;
9961 :
9962 : /* While the committee makes up its mind, bail if we have any
9963 : non-constant steps. */
9964 0 : if (TREE_CODE (fd.loops[i].step) != INTEGER_CST)
9965 0 : goto lower_omp_ordered_ret;
9966 :
9967 0 : tree itype = TREE_TYPE (TREE_VALUE (vec));
9968 0 : if (POINTER_TYPE_P (itype))
9969 0 : itype = sizetype;
9970 0 : wide_int offset = wide_int::from (wi::to_wide (TREE_PURPOSE (vec)),
9971 0 : TYPE_PRECISION (itype),
9972 0 : TYPE_SIGN (itype));
9973 :
9974 : /* Ignore invalid offsets that are not multiples of the step. */
9975 0 : if (!wi::multiple_of_p (wi::abs (offset),
9976 0 : wi::abs (wi::to_wide (fd.loops[i].step)),
9977 : UNSIGNED))
9978 : {
9979 0 : warning_at (OMP_CLAUSE_LOCATION (c), OPT_Wopenmp,
9980 : "ignoring %<sink%> clause with offset that is not "
9981 : "a multiple of the loop step");
9982 0 : remove = true;
9983 0 : goto next_ordered_clause;
9984 : }
9985 :
9986 : /* Calculate the first dimension. The first dimension of
9987 : the folded dependency vector is the GCD of the first
9988 : elements, while ignoring any first elements whose offset
9989 : is 0. */
9990 0 : if (i == 0)
9991 : {
9992 : /* Ignore dependence vectors whose first dimension is 0. */
9993 0 : if (offset == 0)
9994 : {
9995 0 : remove = true;
9996 0 : goto next_ordered_clause;
9997 : }
9998 : else
9999 : {
10000 0 : if (!TYPE_UNSIGNED (itype) && (forward ^ wi::neg_p (offset)))
10001 : {
10002 0 : error_at (OMP_CLAUSE_LOCATION (c),
10003 : "first offset must be in opposite direction "
10004 : "of loop iterations");
10005 0 : goto lower_omp_ordered_ret;
10006 : }
10007 0 : if (forward)
10008 0 : offset = -offset;
10009 0 : neg_offset_p = forward;
10010 : /* Initialize the first time around. */
10011 0 : if (folded_dep == NULL_TREE)
10012 : {
10013 0 : folded_dep = c;
10014 0 : folded_deps[0] = offset;
10015 : }
10016 : else
10017 0 : folded_deps[0] = wi::gcd (folded_deps[0],
10018 0 : offset, UNSIGNED);
10019 : }
10020 : }
10021 : /* Calculate minimum for the remaining dimensions. */
10022 : else
10023 : {
10024 0 : folded_deps[len + i - 1] = offset;
10025 0 : if (folded_dep == c)
10026 0 : folded_deps[i] = offset;
10027 0 : else if (maybe_lexically_later
10028 0 : && !wi::eq_p (folded_deps[i], offset))
10029 : {
10030 0 : if (forward ^ wi::gts_p (folded_deps[i], offset))
10031 : {
10032 : unsigned int j;
10033 0 : folded_dep = c;
10034 0 : for (j = 1; j <= i; j++)
10035 0 : folded_deps[j] = folded_deps[len + j - 1];
10036 : }
10037 : else
10038 0 : maybe_lexically_later = false;
10039 : }
10040 : }
10041 0 : }
10042 0 : gcc_assert (i == len);
10043 :
10044 : remove = true;
10045 :
10046 0 : next_ordered_clause:
10047 0 : if (remove)
10048 0 : *list_p = OMP_CLAUSE_CHAIN (c);
10049 : else
10050 0 : list_p = &OMP_CLAUSE_CHAIN (c);
10051 : }
10052 :
10053 0 : if (folded_dep)
10054 : {
10055 0 : if (neg_offset_p)
10056 0 : folded_deps[0] = -folded_deps[0];
10057 :
10058 0 : tree itype = TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep)));
10059 0 : if (POINTER_TYPE_P (itype))
10060 0 : itype = sizetype;
10061 :
10062 0 : TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep))
10063 0 : = wide_int_to_tree (itype, folded_deps[0]);
10064 0 : OMP_CLAUSE_CHAIN (folded_dep) = gimple_omp_ordered_clauses (ord_stmt);
10065 0 : *gimple_omp_ordered_clauses_ptr (ord_stmt) = folded_dep;
10066 : }
10067 :
10068 0 : lower_omp_ordered_ret:
10069 :
10070 : /* Ordered without clauses is #pragma omp threads, while we want
10071 : a nop instead if we remove all clauses. */
10072 0 : if (gimple_omp_ordered_clauses (ord_stmt) == NULL_TREE)
10073 0 : gsi_replace (gsi_p, gimple_build_nop (), true);
10074 : }
10075 :
10076 :
10077 : /* Expand code for an OpenMP ordered directive. */
10078 :
10079 : static void
10080 1124 : lower_omp_ordered (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10081 : {
10082 1124 : tree block;
10083 1124 : gimple *stmt = gsi_stmt (*gsi_p), *g;
10084 1124 : gomp_ordered *ord_stmt = as_a <gomp_ordered *> (stmt);
10085 1124 : gcall *x;
10086 1124 : gbind *bind;
10087 1124 : bool simd = omp_find_clause (gimple_omp_ordered_clauses (ord_stmt),
10088 1124 : OMP_CLAUSE_SIMD);
10089 : /* FIXME: this should check presence of OMP_CLAUSE__SIMT_ on the enclosing
10090 : loop. */
10091 1124 : bool maybe_simt
10092 1124 : = simd && omp_maybe_offloaded_ctx (ctx) && omp_max_simt_vf () > 1;
10093 1124 : bool threads = omp_find_clause (gimple_omp_ordered_clauses (ord_stmt),
10094 1124 : OMP_CLAUSE_THREADS);
10095 :
10096 1124 : if (gimple_omp_ordered_standalone_p (ord_stmt))
10097 : {
10098 : /* FIXME: This is needs to be moved to the expansion to verify various
10099 : conditions only testable on cfg with dominators computed, and also
10100 : all the depend clauses to be merged still might need to be available
10101 : for the runtime checks. */
10102 : if (0)
10103 : lower_omp_ordered_clauses (gsi_p, ord_stmt, ctx);
10104 1124 : return;
10105 : }
10106 :
10107 411 : push_gimplify_context ();
10108 :
10109 411 : block = make_node (BLOCK);
10110 411 : bind = gimple_build_bind (NULL, NULL, block);
10111 411 : gsi_replace (gsi_p, bind, true);
10112 411 : gimple_bind_add_stmt (bind, stmt);
10113 :
10114 411 : if (simd)
10115 : {
10116 79 : x = gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START, 1,
10117 79 : build_int_cst (NULL_TREE, threads));
10118 79 : cfun->has_simduid_loops = true;
10119 : }
10120 : else
10121 332 : x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START),
10122 : 0);
10123 411 : gimple_bind_add_stmt (bind, x);
10124 :
10125 411 : tree counter = NULL_TREE, test = NULL_TREE, body = NULL_TREE;
10126 411 : if (maybe_simt)
10127 : {
10128 0 : counter = create_tmp_var (integer_type_node);
10129 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_LANE, 0);
10130 0 : gimple_call_set_lhs (g, counter);
10131 0 : gimple_bind_add_stmt (bind, g);
10132 :
10133 0 : body = create_artificial_label (UNKNOWN_LOCATION);
10134 0 : test = create_artificial_label (UNKNOWN_LOCATION);
10135 0 : gimple_bind_add_stmt (bind, gimple_build_label (body));
10136 :
10137 0 : tree simt_pred = create_tmp_var (integer_type_node);
10138 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_ORDERED_PRED, 1, counter);
10139 0 : gimple_call_set_lhs (g, simt_pred);
10140 0 : gimple_bind_add_stmt (bind, g);
10141 :
10142 0 : tree t = create_artificial_label (UNKNOWN_LOCATION);
10143 0 : g = gimple_build_cond (EQ_EXPR, simt_pred, integer_zero_node, t, test);
10144 0 : gimple_bind_add_stmt (bind, g);
10145 :
10146 0 : gimple_bind_add_stmt (bind, gimple_build_label (t));
10147 : }
10148 411 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
10149 411 : gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
10150 411 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10151 411 : gimple_omp_set_body (stmt, NULL);
10152 :
10153 411 : if (maybe_simt)
10154 : {
10155 0 : gimple_bind_add_stmt (bind, gimple_build_label (test));
10156 0 : g = gimple_build_assign (counter, MINUS_EXPR, counter, integer_one_node);
10157 0 : gimple_bind_add_stmt (bind, g);
10158 :
10159 0 : tree c = build2 (GE_EXPR, boolean_type_node, counter, integer_zero_node);
10160 0 : tree nonneg = create_tmp_var (integer_type_node);
10161 0 : gimple_seq tseq = NULL;
10162 0 : gimplify_assign (nonneg, fold_convert (integer_type_node, c), &tseq);
10163 0 : gimple_bind_add_seq (bind, tseq);
10164 :
10165 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_VOTE_ANY, 1, nonneg);
10166 0 : gimple_call_set_lhs (g, nonneg);
10167 0 : gimple_bind_add_stmt (bind, g);
10168 :
10169 0 : tree end = create_artificial_label (UNKNOWN_LOCATION);
10170 0 : g = gimple_build_cond (NE_EXPR, nonneg, integer_zero_node, body, end);
10171 0 : gimple_bind_add_stmt (bind, g);
10172 :
10173 0 : gimple_bind_add_stmt (bind, gimple_build_label (end));
10174 : }
10175 411 : if (simd)
10176 79 : x = gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END, 1,
10177 79 : build_int_cst (NULL_TREE, threads));
10178 : else
10179 332 : x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END),
10180 : 0);
10181 411 : gimple_bind_add_stmt (bind, x);
10182 :
10183 411 : gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10184 :
10185 411 : pop_gimplify_context (bind);
10186 :
10187 411 : gimple_bind_append_vars (bind, ctx->block_vars);
10188 411 : BLOCK_VARS (block) = gimple_bind_vars (bind);
10189 : }
10190 :
10191 :
10192 : /* Expand code for an OpenMP scan directive and the structured block
10193 : before the scan directive. */
10194 :
10195 : static void
10196 1544 : lower_omp_scan (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10197 : {
10198 1544 : gimple *stmt = gsi_stmt (*gsi_p);
10199 1544 : bool has_clauses
10200 1544 : = gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt)) != NULL;
10201 1544 : tree lane = NULL_TREE;
10202 1544 : gimple_seq before = NULL;
10203 1544 : omp_context *octx = ctx->outer;
10204 1544 : gcc_assert (octx);
10205 1544 : if (octx->scan_exclusive && !has_clauses)
10206 : {
10207 536 : gimple_stmt_iterator gsi2 = *gsi_p;
10208 536 : gsi_next (&gsi2);
10209 536 : gimple *stmt2 = gsi_stmt (gsi2);
10210 : /* For exclusive scan, swap GIMPLE_OMP_SCAN without clauses
10211 : with following GIMPLE_OMP_SCAN with clauses, so that input_phase,
10212 : the one with exclusive clause(s), comes first. */
10213 536 : if (stmt2
10214 274 : && gimple_code (stmt2) == GIMPLE_OMP_SCAN
10215 804 : && gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt2)) != NULL)
10216 : {
10217 268 : gsi_remove (gsi_p, false);
10218 268 : gsi_insert_after (gsi_p, stmt, GSI_SAME_STMT);
10219 268 : ctx = maybe_lookup_ctx (stmt2);
10220 268 : gcc_assert (ctx);
10221 268 : lower_omp_scan (gsi_p, ctx);
10222 268 : return;
10223 : }
10224 : }
10225 :
10226 1276 : bool input_phase = has_clauses ^ octx->scan_inclusive;
10227 1276 : bool is_simd = (gimple_code (octx->stmt) == GIMPLE_OMP_FOR
10228 1276 : && gimple_omp_for_kind (octx->stmt) == GF_OMP_FOR_KIND_SIMD);
10229 1276 : bool is_for = (gimple_code (octx->stmt) == GIMPLE_OMP_FOR
10230 1276 : && gimple_omp_for_kind (octx->stmt) == GF_OMP_FOR_KIND_FOR
10231 1630 : && !gimple_omp_for_combined_p (octx->stmt));
10232 1276 : bool is_for_simd = is_simd && gimple_omp_for_combined_into_p (octx->stmt);
10233 332 : if (is_for_simd && octx->for_simd_scan_phase)
10234 : is_simd = false;
10235 1110 : if (is_simd)
10236 756 : if (tree c = omp_find_clause (gimple_omp_for_clauses (octx->stmt),
10237 : OMP_CLAUSE__SIMDUID_))
10238 : {
10239 368 : tree uid = OMP_CLAUSE__SIMDUID__DECL (c);
10240 368 : lane = create_tmp_var (unsigned_type_node);
10241 368 : tree t = build_int_cst (integer_type_node,
10242 552 : input_phase ? 1
10243 184 : : octx->scan_inclusive ? 2 : 3);
10244 368 : gimple *g
10245 368 : = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 2, uid, t);
10246 368 : gimple_call_set_lhs (g, lane);
10247 368 : gimple_seq_add_stmt (&before, g);
10248 : }
10249 :
10250 1276 : if (is_simd || is_for)
10251 : {
10252 5004 : for (tree c = gimple_omp_for_clauses (octx->stmt);
10253 5004 : c; c = OMP_CLAUSE_CHAIN (c))
10254 4060 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
10255 4060 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
10256 : {
10257 1168 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
10258 1168 : tree var = OMP_CLAUSE_DECL (c);
10259 1168 : tree new_var = lookup_decl (var, octx);
10260 1168 : tree val = new_var;
10261 1168 : tree var2 = NULL_TREE;
10262 1168 : tree var3 = NULL_TREE;
10263 1168 : tree var4 = NULL_TREE;
10264 1168 : tree lane0 = NULL_TREE;
10265 1168 : tree new_vard = new_var;
10266 1168 : if (omp_privatize_by_reference (var))
10267 : {
10268 184 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
10269 184 : val = new_var;
10270 : }
10271 1168 : if (DECL_HAS_VALUE_EXPR_P (new_vard))
10272 : {
10273 464 : val = DECL_VALUE_EXPR (new_vard);
10274 464 : if (new_vard != new_var)
10275 : {
10276 76 : gcc_assert (TREE_CODE (val) == ADDR_EXPR);
10277 76 : val = TREE_OPERAND (val, 0);
10278 : }
10279 464 : if (TREE_CODE (val) == ARRAY_REF
10280 464 : && VAR_P (TREE_OPERAND (val, 0)))
10281 : {
10282 464 : tree v = TREE_OPERAND (val, 0);
10283 464 : if (lookup_attribute ("omp simd array",
10284 464 : DECL_ATTRIBUTES (v)))
10285 : {
10286 464 : val = unshare_expr (val);
10287 464 : lane0 = TREE_OPERAND (val, 1);
10288 464 : TREE_OPERAND (val, 1) = lane;
10289 464 : var2 = lookup_decl (v, octx);
10290 464 : if (octx->scan_exclusive)
10291 228 : var4 = lookup_decl (var2, octx);
10292 464 : if (input_phase
10293 464 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
10294 120 : var3 = maybe_lookup_decl (var4 ? var4 : var2, octx);
10295 464 : if (!input_phase)
10296 : {
10297 232 : var2 = build4 (ARRAY_REF, TREE_TYPE (val),
10298 : var2, lane, NULL_TREE, NULL_TREE);
10299 232 : TREE_THIS_NOTRAP (var2) = 1;
10300 232 : if (octx->scan_exclusive)
10301 : {
10302 114 : var4 = build4 (ARRAY_REF, TREE_TYPE (val),
10303 : var4, lane, NULL_TREE,
10304 : NULL_TREE);
10305 114 : TREE_THIS_NOTRAP (var4) = 1;
10306 : }
10307 : }
10308 : else
10309 : var2 = val;
10310 : }
10311 : }
10312 464 : gcc_assert (var2);
10313 : }
10314 : else
10315 : {
10316 704 : var2 = build_outer_var_ref (var, octx);
10317 704 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
10318 : {
10319 220 : var3 = maybe_lookup_decl (new_vard, octx);
10320 220 : if (var3 == new_vard || var3 == NULL_TREE)
10321 : var3 = NULL_TREE;
10322 108 : else if (is_simd && octx->scan_exclusive && !input_phase)
10323 : {
10324 16 : var4 = maybe_lookup_decl (var3, octx);
10325 16 : if (var4 == var3 || var4 == NULL_TREE)
10326 : {
10327 0 : if (TREE_ADDRESSABLE (TREE_TYPE (new_var)))
10328 : {
10329 : var4 = var3;
10330 : var3 = NULL_TREE;
10331 : }
10332 : else
10333 16 : var4 = NULL_TREE;
10334 : }
10335 : }
10336 : }
10337 672 : if (is_simd
10338 484 : && octx->scan_exclusive
10339 244 : && !input_phase
10340 244 : && var4 == NULL_TREE)
10341 106 : var4 = create_tmp_var (TREE_TYPE (val));
10342 : }
10343 1168 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
10344 : {
10345 376 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
10346 376 : if (input_phase)
10347 : {
10348 188 : if (var3)
10349 : {
10350 : /* If we've added a separate identity element
10351 : variable, copy it over into val. */
10352 92 : tree x = lang_hooks.decls.omp_clause_assign_op (c, val,
10353 : var3);
10354 92 : gimplify_and_add (x, &before);
10355 : }
10356 96 : else if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
10357 : {
10358 : /* Otherwise, assign to it the identity element. */
10359 96 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
10360 96 : if (is_for)
10361 16 : tseq = copy_gimple_seq_and_replace_locals (tseq);
10362 96 : tree ref = build_outer_var_ref (var, octx);
10363 96 : tree x = (DECL_HAS_VALUE_EXPR_P (new_vard)
10364 96 : ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
10365 40 : if (x)
10366 : {
10367 40 : if (new_vard != new_var)
10368 8 : val = build_fold_addr_expr_loc (clause_loc, val);
10369 40 : SET_DECL_VALUE_EXPR (new_vard, val);
10370 : }
10371 96 : SET_DECL_VALUE_EXPR (placeholder, ref);
10372 96 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
10373 96 : lower_omp (&tseq, octx);
10374 96 : if (x)
10375 40 : SET_DECL_VALUE_EXPR (new_vard, x);
10376 96 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
10377 96 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
10378 96 : gimple_seq_add_seq (&before, tseq);
10379 96 : if (is_simd)
10380 80 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
10381 : }
10382 : }
10383 188 : else if (is_simd)
10384 : {
10385 156 : tree x;
10386 156 : if (octx->scan_exclusive)
10387 : {
10388 72 : tree v4 = unshare_expr (var4);
10389 72 : tree v2 = unshare_expr (var2);
10390 72 : x = lang_hooks.decls.omp_clause_assign_op (c, v4, v2);
10391 72 : gimplify_and_add (x, &before);
10392 : }
10393 156 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
10394 156 : x = (DECL_HAS_VALUE_EXPR_P (new_vard)
10395 156 : ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
10396 156 : tree vexpr = val;
10397 156 : if (x && new_vard != new_var)
10398 30 : vexpr = build_fold_addr_expr_loc (clause_loc, val);
10399 156 : if (x)
10400 78 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
10401 156 : SET_DECL_VALUE_EXPR (placeholder, var2);
10402 156 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
10403 156 : lower_omp (&tseq, octx);
10404 156 : gimple_seq_add_seq (&before, tseq);
10405 156 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
10406 156 : if (x)
10407 78 : SET_DECL_VALUE_EXPR (new_vard, x);
10408 156 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
10409 156 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
10410 156 : if (octx->scan_inclusive)
10411 : {
10412 84 : x = lang_hooks.decls.omp_clause_assign_op (c, val,
10413 : var2);
10414 84 : gimplify_and_add (x, &before);
10415 : }
10416 72 : else if (lane0 == NULL_TREE)
10417 : {
10418 36 : x = lang_hooks.decls.omp_clause_assign_op (c, val,
10419 : var4);
10420 36 : gimplify_and_add (x, &before);
10421 : }
10422 : }
10423 : }
10424 : else
10425 : {
10426 792 : if (input_phase)
10427 : {
10428 : /* input phase. Set val to initializer before
10429 : the body. */
10430 396 : tree x = omp_reduction_init (c, TREE_TYPE (new_var));
10431 396 : gimplify_assign (val, x, &before);
10432 : }
10433 396 : else if (is_simd)
10434 : {
10435 : /* scan phase. */
10436 318 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
10437 318 : if (code == MINUS_EXPR)
10438 0 : code = PLUS_EXPR;
10439 :
10440 318 : tree x = build2 (code, TREE_TYPE (var2),
10441 : unshare_expr (var2), unshare_expr (val));
10442 318 : if (octx->scan_inclusive)
10443 : {
10444 154 : gimplify_assign (unshare_expr (var2), x, &before);
10445 154 : gimplify_assign (val, var2, &before);
10446 : }
10447 : else
10448 : {
10449 164 : gimplify_assign (unshare_expr (var4),
10450 : unshare_expr (var2), &before);
10451 164 : gimplify_assign (var2, x, &before);
10452 164 : if (lane0 == NULL_TREE)
10453 86 : gimplify_assign (val, var4, &before);
10454 : }
10455 : }
10456 : }
10457 1168 : if (octx->scan_exclusive && !input_phase && lane0)
10458 : {
10459 114 : tree vexpr = unshare_expr (var4);
10460 114 : TREE_OPERAND (vexpr, 1) = lane0;
10461 114 : if (new_vard != new_var)
10462 16 : vexpr = build_fold_addr_expr_loc (clause_loc, vexpr);
10463 114 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
10464 : }
10465 : }
10466 : }
10467 1276 : if (is_simd && !is_for_simd)
10468 : {
10469 590 : gsi_insert_seq_after (gsi_p, gimple_omp_body (stmt), GSI_SAME_STMT);
10470 590 : gsi_insert_seq_after (gsi_p, before, GSI_SAME_STMT);
10471 590 : gsi_replace (gsi_p, gimple_build_nop (), true);
10472 590 : return;
10473 : }
10474 686 : lower_omp (gimple_omp_body_ptr (stmt), octx);
10475 686 : if (before)
10476 : {
10477 260 : gimple_stmt_iterator gsi = gsi_start (*gimple_omp_body_ptr (stmt));
10478 260 : gsi_insert_seq_before (&gsi, before, GSI_SAME_STMT);
10479 : }
10480 : }
10481 :
10482 :
10483 : /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
10484 : substitution of a couple of function calls. But in the NAMED case,
10485 : requires that languages coordinate a symbol name. It is therefore
10486 : best put here in common code. */
10487 :
10488 : static GTY(()) hash_map<tree, tree> *critical_name_mutexes;
10489 :
10490 : static void
10491 311 : lower_omp_critical (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10492 : {
10493 311 : tree block;
10494 311 : tree name, lock, unlock;
10495 311 : gomp_critical *stmt = as_a <gomp_critical *> (gsi_stmt (*gsi_p));
10496 311 : gbind *bind;
10497 311 : location_t loc = gimple_location (stmt);
10498 311 : gimple_seq tbody;
10499 :
10500 311 : name = gimple_omp_critical_name (stmt);
10501 311 : if (name)
10502 : {
10503 88 : tree decl;
10504 :
10505 88 : if (!critical_name_mutexes)
10506 48 : critical_name_mutexes = hash_map<tree, tree>::create_ggc (10);
10507 :
10508 88 : tree *n = critical_name_mutexes->get (name);
10509 88 : if (n == NULL)
10510 : {
10511 70 : char *new_str;
10512 :
10513 70 : decl = create_tmp_var_raw (ptr_type_node);
10514 :
10515 70 : new_str = ACONCAT ((".gomp_critical_user_",
10516 : IDENTIFIER_POINTER (name), NULL));
10517 70 : DECL_NAME (decl) = get_identifier (new_str);
10518 70 : TREE_PUBLIC (decl) = 1;
10519 70 : TREE_STATIC (decl) = 1;
10520 70 : DECL_COMMON (decl) = 1;
10521 70 : DECL_ARTIFICIAL (decl) = 1;
10522 70 : DECL_IGNORED_P (decl) = 1;
10523 :
10524 70 : varpool_node::finalize_decl (decl);
10525 :
10526 70 : critical_name_mutexes->put (name, decl);
10527 : }
10528 : else
10529 18 : decl = *n;
10530 :
10531 : /* If '#pragma omp critical' is inside offloaded region or
10532 : inside function marked as offloadable, the symbol must be
10533 : marked as offloadable too. */
10534 88 : omp_context *octx;
10535 88 : if (cgraph_node::get (current_function_decl)->offloadable)
10536 1 : varpool_node::get_create (decl)->offloadable = 1;
10537 : else
10538 173 : for (octx = ctx->outer; octx; octx = octx->outer)
10539 87 : if (is_gimple_omp_offloaded (octx->stmt))
10540 : {
10541 1 : varpool_node::get_create (decl)->offloadable = 1;
10542 1 : break;
10543 : }
10544 :
10545 88 : lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START);
10546 88 : lock = build_call_expr_loc (loc, lock, 1,
10547 : build_fold_addr_expr_loc (loc, decl));
10548 :
10549 88 : unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END);
10550 88 : unlock = build_call_expr_loc (loc, unlock, 1,
10551 : build_fold_addr_expr_loc (loc, decl));
10552 : }
10553 : else
10554 : {
10555 223 : lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START);
10556 223 : lock = build_call_expr_loc (loc, lock, 0);
10557 :
10558 223 : unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END);
10559 223 : unlock = build_call_expr_loc (loc, unlock, 0);
10560 : }
10561 :
10562 311 : push_gimplify_context ();
10563 :
10564 311 : block = make_node (BLOCK);
10565 311 : bind = gimple_build_bind (NULL, NULL, block);
10566 311 : gsi_replace (gsi_p, bind, true);
10567 311 : gimple_bind_add_stmt (bind, stmt);
10568 :
10569 311 : tbody = gimple_bind_body (bind);
10570 311 : gimplify_and_add (lock, &tbody);
10571 311 : gimple_bind_set_body (bind, tbody);
10572 :
10573 311 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
10574 311 : gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
10575 311 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10576 311 : gimple_omp_set_body (stmt, NULL);
10577 :
10578 311 : tbody = gimple_bind_body (bind);
10579 311 : gimplify_and_add (unlock, &tbody);
10580 311 : gimple_bind_set_body (bind, tbody);
10581 :
10582 311 : gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10583 :
10584 311 : pop_gimplify_context (bind);
10585 311 : gimple_bind_append_vars (bind, ctx->block_vars);
10586 311 : BLOCK_VARS (block) = gimple_bind_vars (bind);
10587 311 : }
10588 :
10589 : /* A subroutine of lower_omp_for. Generate code to emit the predicate
10590 : for a lastprivate clause. Given a loop control predicate of (V
10591 : cond N2), we gate the clause on (!(V cond N2)). The lowered form
10592 : is appended to *DLIST, iterator initialization is appended to
10593 : *BODY_P. *CLIST is for lastprivate(conditional:) code that needs
10594 : to be emitted in a critical section. */
10595 :
10596 : static void
10597 46859 : lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p,
10598 : gimple_seq *dlist, gimple_seq *clist,
10599 : struct omp_context *ctx)
10600 : {
10601 46859 : tree clauses, cond, vinit;
10602 46859 : enum tree_code cond_code;
10603 46859 : gimple_seq stmts;
10604 :
10605 46859 : cond_code = fd->loop.cond_code;
10606 46859 : cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR;
10607 :
10608 : /* When possible, use a strict equality expression. This can let VRP
10609 : type optimizations deduce the value and remove a copy. */
10610 46859 : if (tree_fits_shwi_p (fd->loop.step))
10611 : {
10612 44620 : HOST_WIDE_INT step = tree_to_shwi (fd->loop.step);
10613 44620 : if (step == 1 || step == -1)
10614 46859 : cond_code = EQ_EXPR;
10615 : }
10616 :
10617 46859 : tree n2 = fd->loop.n2;
10618 46859 : if (fd->collapse > 1
10619 11233 : && TREE_CODE (n2) != INTEGER_CST
10620 53073 : && gimple_omp_for_combined_into_p (fd->for_stmt))
10621 : {
10622 2692 : struct omp_context *taskreg_ctx = NULL;
10623 2692 : if (gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR)
10624 : {
10625 1226 : gomp_for *gfor = as_a <gomp_for *> (ctx->outer->stmt);
10626 1226 : if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_FOR
10627 1226 : || gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_DISTRIBUTE)
10628 : {
10629 1194 : if (gimple_omp_for_combined_into_p (gfor))
10630 : {
10631 672 : gcc_assert (ctx->outer->outer
10632 : && is_parallel_ctx (ctx->outer->outer));
10633 : taskreg_ctx = ctx->outer->outer;
10634 : }
10635 : else
10636 : {
10637 522 : struct omp_for_data outer_fd;
10638 522 : omp_extract_for_data (gfor, &outer_fd, NULL);
10639 522 : n2 = fold_convert (TREE_TYPE (n2), outer_fd.loop.n2);
10640 : }
10641 : }
10642 32 : else if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_TASKLOOP)
10643 32 : taskreg_ctx = ctx->outer->outer;
10644 : }
10645 1466 : else if (is_taskreg_ctx (ctx->outer))
10646 : taskreg_ctx = ctx->outer;
10647 2692 : if (taskreg_ctx)
10648 : {
10649 2170 : int i;
10650 2170 : tree taskreg_clauses
10651 2170 : = gimple_omp_taskreg_clauses (taskreg_ctx->stmt);
10652 2170 : tree innerc = omp_find_clause (taskreg_clauses,
10653 : OMP_CLAUSE__LOOPTEMP_);
10654 2170 : gcc_assert (innerc);
10655 2170 : int count = fd->collapse;
10656 2170 : if (fd->non_rect
10657 24 : && fd->last_nonrect == fd->first_nonrect + 1)
10658 12 : if (tree v = gimple_omp_for_index (fd->for_stmt, fd->last_nonrect))
10659 12 : if (!TYPE_UNSIGNED (TREE_TYPE (v)))
10660 12 : count += 4;
10661 8582 : for (i = 0; i < count; i++)
10662 : {
10663 6412 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
10664 : OMP_CLAUSE__LOOPTEMP_);
10665 6412 : gcc_assert (innerc);
10666 : }
10667 2170 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
10668 : OMP_CLAUSE__LOOPTEMP_);
10669 2170 : if (innerc)
10670 1419 : n2 = fold_convert (TREE_TYPE (n2),
10671 : lookup_decl (OMP_CLAUSE_DECL (innerc),
10672 : taskreg_ctx));
10673 : }
10674 : }
10675 46859 : cond = build2 (cond_code, boolean_type_node, fd->loop.v, n2);
10676 :
10677 46859 : clauses = gimple_omp_for_clauses (fd->for_stmt);
10678 46859 : stmts = NULL;
10679 46859 : lower_lastprivate_clauses (clauses, cond, body_p, &stmts, clist, ctx);
10680 46859 : if (!gimple_seq_empty_p (stmts))
10681 : {
10682 16328 : gimple_seq_add_seq (&stmts, *dlist);
10683 16328 : *dlist = stmts;
10684 :
10685 : /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
10686 16328 : vinit = fd->loop.n1;
10687 16328 : if (cond_code == EQ_EXPR
10688 14347 : && tree_fits_shwi_p (fd->loop.n2)
10689 25861 : && ! integer_zerop (fd->loop.n2))
10690 8317 : vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0);
10691 : else
10692 8011 : vinit = unshare_expr (vinit);
10693 :
10694 : /* Initialize the iterator variable, so that threads that don't execute
10695 : any iterations don't execute the lastprivate clauses by accident. */
10696 16328 : gimplify_assign (fd->loop.v, vinit, body_p);
10697 : }
10698 46859 : }
10699 :
10700 : /* OpenACC privatization.
10701 :
10702 : Or, in other words, *sharing* at the respective OpenACC level of
10703 : parallelism.
10704 :
10705 : From a correctness perspective, a non-addressable variable can't be accessed
10706 : outside the current thread, so it can go in a (faster than shared memory)
10707 : register -- though that register may need to be broadcast in some
10708 : circumstances. A variable can only meaningfully be "shared" across workers
10709 : or vector lanes if its address is taken, e.g. by a call to an atomic
10710 : builtin.
10711 :
10712 : From an optimisation perspective, the answer might be fuzzier: maybe
10713 : sometimes, using shared memory directly would be faster than
10714 : broadcasting. */
10715 :
10716 : static void
10717 18997 : oacc_privatization_begin_diagnose_var (const dump_flags_t l_dump_flags,
10718 : const location_t loc, const tree c,
10719 : const tree decl)
10720 : {
10721 18997 : const dump_user_location_t d_u_loc
10722 18997 : = dump_user_location_t::from_location_t (loc);
10723 : /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */
10724 : #if __GNUC__ >= 10
10725 18997 : # pragma GCC diagnostic push
10726 18997 : # pragma GCC diagnostic ignored "-Wformat"
10727 : #endif
10728 18997 : dump_printf_loc (l_dump_flags, d_u_loc,
10729 : "variable %<%T%> ", decl);
10730 : #if __GNUC__ >= 10
10731 18997 : # pragma GCC diagnostic pop
10732 : #endif
10733 18997 : if (c)
10734 2797 : dump_printf (l_dump_flags,
10735 : "in %qs clause ",
10736 2797 : omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
10737 : else
10738 16200 : dump_printf (l_dump_flags,
10739 : "declared in block ");
10740 18997 : }
10741 :
10742 : static bool
10743 41114 : oacc_privatization_candidate_p (const location_t loc, const tree c,
10744 : const tree decl)
10745 : {
10746 41114 : dump_flags_t l_dump_flags = get_openacc_privatization_dump_flags ();
10747 :
10748 : /* There is some differentiation depending on block vs. clause. */
10749 41114 : bool block = !c;
10750 :
10751 41114 : bool res = true;
10752 :
10753 41114 : if (res && !VAR_P (decl))
10754 : {
10755 : /* A PARM_DECL (appearing in a 'private' clause) is expected to have been
10756 : privatized into a new VAR_DECL. */
10757 712 : gcc_checking_assert (TREE_CODE (decl) != PARM_DECL);
10758 :
10759 712 : res = false;
10760 :
10761 712 : if (dump_enabled_p ())
10762 : {
10763 683 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10764 683 : dump_printf (l_dump_flags,
10765 : "potentially has improper OpenACC privatization level: %qs\n",
10766 683 : get_tree_code_name (TREE_CODE (decl)));
10767 : }
10768 : }
10769 :
10770 41085 : if (res && block && TREE_STATIC (decl))
10771 : {
10772 53 : res = false;
10773 :
10774 53 : if (dump_enabled_p ())
10775 : {
10776 28 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10777 28 : dump_printf (l_dump_flags,
10778 : "isn%'t candidate for adjusting OpenACC privatization level: %s\n",
10779 : "static");
10780 : }
10781 : }
10782 :
10783 41089 : if (res && block && DECL_EXTERNAL (decl))
10784 : {
10785 12 : res = false;
10786 :
10787 12 : if (dump_enabled_p ())
10788 : {
10789 12 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10790 12 : dump_printf (l_dump_flags,
10791 : "isn%'t candidate for adjusting OpenACC privatization level: %s\n",
10792 : "external");
10793 : }
10794 : }
10795 :
10796 41114 : if (res && !TREE_ADDRESSABLE (decl))
10797 : {
10798 39245 : res = false;
10799 :
10800 39245 : if (dump_enabled_p ())
10801 : {
10802 17456 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10803 17456 : dump_printf (l_dump_flags,
10804 : "isn%'t candidate for adjusting OpenACC privatization level: %s\n",
10805 : "not addressable");
10806 : }
10807 : }
10808 :
10809 : /* If an artificial variable has been added to a bind, e.g.
10810 : a compiler-generated temporary structure used by the Fortran front-end, do
10811 : not consider it as a privatization candidate. Note that variables on
10812 : the stack are private per-thread by default: making them "gang-private"
10813 : for OpenACC actually means to share a single instance of a variable
10814 : amongst all workers and threads spawned within each gang.
10815 : At present, no compiler-generated artificial variables require such
10816 : sharing semantics, so this is safe. */
10817 :
10818 18548 : if (res && block && DECL_ARTIFICIAL (decl))
10819 : {
10820 548 : res = false;
10821 :
10822 548 : if (dump_enabled_p ())
10823 : {
10824 360 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10825 360 : dump_printf (l_dump_flags,
10826 : "isn%'t candidate for adjusting OpenACC privatization "
10827 : "level: %s\n", "artificial");
10828 : }
10829 : }
10830 :
10831 40926 : if (res)
10832 : {
10833 544 : if (dump_enabled_p ())
10834 : {
10835 458 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10836 458 : dump_printf (l_dump_flags,
10837 : "is candidate for adjusting OpenACC privatization level\n");
10838 : }
10839 : }
10840 :
10841 41114 : if (dump_file && (dump_flags & TDF_DETAILS))
10842 : {
10843 0 : print_generic_decl (dump_file, decl, dump_flags);
10844 0 : fprintf (dump_file, "\n");
10845 : }
10846 :
10847 41114 : return res;
10848 : }
10849 :
10850 : /* Scan CLAUSES for candidates for adjusting OpenACC privatization level in
10851 : CTX. */
10852 :
10853 : static void
10854 11325 : oacc_privatization_scan_clause_chain (omp_context *ctx, tree clauses)
10855 : {
10856 35223 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
10857 23898 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE)
10858 : {
10859 12986 : tree decl = OMP_CLAUSE_DECL (c);
10860 :
10861 12986 : tree new_decl = lookup_decl (decl, ctx);
10862 :
10863 12986 : if (!oacc_privatization_candidate_p (OMP_CLAUSE_LOCATION (c), c,
10864 : new_decl))
10865 12654 : continue;
10866 :
10867 332 : gcc_checking_assert
10868 : (!ctx->oacc_privatization_candidates.contains (new_decl));
10869 332 : ctx->oacc_privatization_candidates.safe_push (new_decl);
10870 : }
10871 11325 : }
10872 :
10873 : /* Scan DECLS for candidates for adjusting OpenACC privatization level in
10874 : CTX. */
10875 :
10876 : static void
10877 23090 : oacc_privatization_scan_decl_chain (omp_context *ctx, tree decls)
10878 : {
10879 51218 : for (tree decl = decls; decl; decl = DECL_CHAIN (decl))
10880 : {
10881 28128 : tree new_decl = lookup_decl (decl, ctx);
10882 28128 : gcc_checking_assert (new_decl == decl);
10883 :
10884 28128 : if (!oacc_privatization_candidate_p (gimple_location (ctx->stmt), NULL,
10885 : new_decl))
10886 27916 : continue;
10887 :
10888 212 : gcc_checking_assert
10889 : (!ctx->oacc_privatization_candidates.contains (new_decl));
10890 212 : ctx->oacc_privatization_candidates.safe_push (new_decl);
10891 : }
10892 23090 : }
10893 :
10894 : /* Callback for walk_gimple_seq. Find #pragma omp scan statement. */
10895 :
10896 : static tree
10897 2233 : omp_find_scan (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
10898 : struct walk_stmt_info *wi)
10899 : {
10900 2233 : gimple *stmt = gsi_stmt (*gsi_p);
10901 :
10902 2233 : *handled_ops_p = true;
10903 2233 : switch (gimple_code (stmt))
10904 : {
10905 418 : WALK_SUBSTMTS;
10906 :
10907 166 : case GIMPLE_OMP_FOR:
10908 166 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_SIMD
10909 166 : && gimple_omp_for_combined_into_p (stmt))
10910 166 : *handled_ops_p = false;
10911 : break;
10912 :
10913 686 : case GIMPLE_OMP_SCAN:
10914 686 : *(gimple_stmt_iterator *) (wi->info) = *gsi_p;
10915 686 : return integer_zero_node;
10916 : default:
10917 : break;
10918 : }
10919 : return NULL;
10920 : }
10921 :
10922 : /* Helper function for lower_omp_for, add transformations for a worksharing
10923 : loop with scan directives inside of it.
10924 : For worksharing loop not combined with simd, transform:
10925 : #pragma omp for reduction(inscan,+:r) private(i)
10926 : for (i = 0; i < n; i = i + 1)
10927 : {
10928 : {
10929 : update (r);
10930 : }
10931 : #pragma omp scan inclusive(r)
10932 : {
10933 : use (r);
10934 : }
10935 : }
10936 :
10937 : into two worksharing loops + code to merge results:
10938 :
10939 : num_threads = omp_get_num_threads ();
10940 : thread_num = omp_get_thread_num ();
10941 : if (thread_num == 0) goto <D.2099>; else goto <D.2100>;
10942 : <D.2099>:
10943 : var2 = r;
10944 : goto <D.2101>;
10945 : <D.2100>:
10946 : // For UDRs this is UDR init, or if ctors are needed, copy from
10947 : // var3 that has been constructed to contain the neutral element.
10948 : var2 = 0;
10949 : <D.2101>:
10950 : ivar = 0;
10951 : // The _scantemp_ clauses will arrange for rpriva to be initialized to
10952 : // a shared array with num_threads elements and rprivb to a local array
10953 : // number of elements equal to the number of (contiguous) iterations the
10954 : // current thread will perform. controlb and controlp variables are
10955 : // temporaries to handle deallocation of rprivb at the end of second
10956 : // GOMP_FOR.
10957 : #pragma omp for _scantemp_(rpriva) _scantemp_(rprivb) _scantemp_(controlb) \
10958 : _scantemp_(controlp) reduction(inscan,+:r) private(i) nowait
10959 : for (i = 0; i < n; i = i + 1)
10960 : {
10961 : {
10962 : // For UDRs this is UDR init or copy from var3.
10963 : r = 0;
10964 : // This is the input phase from user code.
10965 : update (r);
10966 : }
10967 : {
10968 : // For UDRs this is UDR merge.
10969 : var2 = var2 + r;
10970 : // Rather than handing it over to the user, save to local thread's
10971 : // array.
10972 : rprivb[ivar] = var2;
10973 : // For exclusive scan, the above two statements are swapped.
10974 : ivar = ivar + 1;
10975 : }
10976 : }
10977 : // And remember the final value from this thread's into the shared
10978 : // rpriva array.
10979 : rpriva[(sizetype) thread_num] = var2;
10980 : // If more than one thread, compute using Work-Efficient prefix sum
10981 : // the inclusive parallel scan of the rpriva array.
10982 : if (num_threads > 1) goto <D.2102>; else goto <D.2103>;
10983 : <D.2102>:
10984 : GOMP_barrier ();
10985 : down = 0;
10986 : k = 1;
10987 : num_threadsu = (unsigned int) num_threads;
10988 : thread_numup1 = (unsigned int) thread_num + 1;
10989 : <D.2108>:
10990 : twok = k << 1;
10991 : if (twok > num_threadsu) goto <D.2110>; else goto <D.2111>;
10992 : <D.2110>:
10993 : down = 4294967295;
10994 : k = k >> 1;
10995 : if (k == num_threadsu) goto <D.2112>; else goto <D.2111>;
10996 : <D.2112>:
10997 : k = k >> 1;
10998 : <D.2111>:
10999 : twok = k << 1;
11000 : cplx = .MUL_OVERFLOW (thread_nump1, twok);
11001 : mul = REALPART_EXPR <cplx>;
11002 : ovf = IMAGPART_EXPR <cplx>;
11003 : if (ovf == 0) goto <D.2116>; else goto <D.2117>;
11004 : <D.2116>:
11005 : andv = k & down;
11006 : andvm1 = andv + 4294967295;
11007 : l = mul + andvm1;
11008 : if (l < num_threadsu) goto <D.2120>; else goto <D.2117>;
11009 : <D.2120>:
11010 : // For UDRs this is UDR merge, performed using var2 variable as temporary,
11011 : // i.e. var2 = rpriva[l - k]; UDR merge (var2, rpriva[l]); rpriva[l] = var2;
11012 : rpriva[l] = rpriva[l - k] + rpriva[l];
11013 : <D.2117>:
11014 : if (down == 0) goto <D.2121>; else goto <D.2122>;
11015 : <D.2121>:
11016 : k = k << 1;
11017 : goto <D.2123>;
11018 : <D.2122>:
11019 : k = k >> 1;
11020 : <D.2123>:
11021 : GOMP_barrier ();
11022 : if (k != 0) goto <D.2108>; else goto <D.2103>;
11023 : <D.2103>:
11024 : if (thread_num == 0) goto <D.2124>; else goto <D.2125>;
11025 : <D.2124>:
11026 : // For UDRs this is UDR init or copy from var3.
11027 : var2 = 0;
11028 : goto <D.2126>;
11029 : <D.2125>:
11030 : var2 = rpriva[thread_num - 1];
11031 : <D.2126>:
11032 : ivar = 0;
11033 : #pragma omp for _scantemp_(controlb) _scantemp_(controlp) \
11034 : reduction(inscan,+:r) private(i)
11035 : for (i = 0; i < n; i = i + 1)
11036 : {
11037 : {
11038 : // For UDRs, this is r = var2; UDR merge (r, rprivb[ivar]);
11039 : r = var2 + rprivb[ivar];
11040 : }
11041 : {
11042 : // This is the scan phase from user code.
11043 : use (r);
11044 : // Plus a bump of the iterator.
11045 : ivar = ivar + 1;
11046 : }
11047 : } */
11048 :
11049 : static void
11050 177 : lower_omp_for_scan (gimple_seq *body_p, gimple_seq *dlist, gomp_for *stmt,
11051 : struct omp_for_data *fd, omp_context *ctx)
11052 : {
11053 177 : bool is_for_simd = gimple_omp_for_combined_p (stmt);
11054 177 : gcc_assert (ctx->scan_inclusive || ctx->scan_exclusive);
11055 :
11056 177 : gimple_seq body = gimple_omp_body (stmt);
11057 177 : gimple_stmt_iterator input1_gsi = gsi_none ();
11058 177 : struct walk_stmt_info wi;
11059 177 : memset (&wi, 0, sizeof (wi));
11060 177 : wi.val_only = true;
11061 177 : wi.info = (void *) &input1_gsi;
11062 177 : walk_gimple_seq_mod (&body, omp_find_scan, NULL, &wi);
11063 177 : gcc_assert (!gsi_end_p (input1_gsi));
11064 :
11065 177 : gimple *input_stmt1 = gsi_stmt (input1_gsi);
11066 177 : gimple_stmt_iterator gsi = input1_gsi;
11067 177 : gsi_next (&gsi);
11068 177 : gimple_stmt_iterator scan1_gsi = gsi;
11069 177 : gimple *scan_stmt1 = gsi_stmt (gsi);
11070 177 : gcc_assert (scan_stmt1 && gimple_code (scan_stmt1) == GIMPLE_OMP_SCAN);
11071 :
11072 177 : gimple_seq input_body = gimple_omp_body (input_stmt1);
11073 177 : gimple_seq scan_body = gimple_omp_body (scan_stmt1);
11074 177 : gimple_omp_set_body (input_stmt1, NULL);
11075 177 : gimple_omp_set_body (scan_stmt1, NULL);
11076 177 : gimple_omp_set_body (stmt, NULL);
11077 :
11078 177 : gomp_for *new_stmt = as_a <gomp_for *> (gimple_copy (stmt));
11079 177 : gimple_seq new_body = copy_gimple_seq_and_replace_locals (body);
11080 177 : gimple_omp_set_body (stmt, body);
11081 177 : gimple_omp_set_body (input_stmt1, input_body);
11082 :
11083 177 : gimple_stmt_iterator input2_gsi = gsi_none ();
11084 177 : memset (&wi, 0, sizeof (wi));
11085 177 : wi.val_only = true;
11086 177 : wi.info = (void *) &input2_gsi;
11087 177 : walk_gimple_seq_mod (&new_body, omp_find_scan, NULL, &wi);
11088 177 : gcc_assert (!gsi_end_p (input2_gsi));
11089 :
11090 177 : gimple *input_stmt2 = gsi_stmt (input2_gsi);
11091 177 : gsi = input2_gsi;
11092 177 : gsi_next (&gsi);
11093 177 : gimple_stmt_iterator scan2_gsi = gsi;
11094 177 : gimple *scan_stmt2 = gsi_stmt (gsi);
11095 177 : gcc_assert (scan_stmt2 && gimple_code (scan_stmt2) == GIMPLE_OMP_SCAN);
11096 177 : gimple_omp_set_body (scan_stmt2, scan_body);
11097 :
11098 177 : gimple_stmt_iterator input3_gsi = gsi_none ();
11099 177 : gimple_stmt_iterator scan3_gsi = gsi_none ();
11100 177 : gimple_stmt_iterator input4_gsi = gsi_none ();
11101 177 : gimple_stmt_iterator scan4_gsi = gsi_none ();
11102 177 : gimple *input_stmt3 = NULL, *scan_stmt3 = NULL;
11103 177 : gimple *input_stmt4 = NULL, *scan_stmt4 = NULL;
11104 177 : omp_context *input_simd_ctx = NULL, *scan_simd_ctx = NULL;
11105 177 : if (is_for_simd)
11106 : {
11107 83 : memset (&wi, 0, sizeof (wi));
11108 83 : wi.val_only = true;
11109 83 : wi.info = (void *) &input3_gsi;
11110 83 : walk_gimple_seq_mod (&input_body, omp_find_scan, NULL, &wi);
11111 83 : gcc_assert (!gsi_end_p (input3_gsi));
11112 :
11113 83 : input_stmt3 = gsi_stmt (input3_gsi);
11114 83 : gsi = input3_gsi;
11115 83 : gsi_next (&gsi);
11116 83 : scan3_gsi = gsi;
11117 83 : scan_stmt3 = gsi_stmt (gsi);
11118 83 : gcc_assert (scan_stmt3 && gimple_code (scan_stmt3) == GIMPLE_OMP_SCAN);
11119 :
11120 83 : memset (&wi, 0, sizeof (wi));
11121 83 : wi.val_only = true;
11122 83 : wi.info = (void *) &input4_gsi;
11123 83 : walk_gimple_seq_mod (&scan_body, omp_find_scan, NULL, &wi);
11124 83 : gcc_assert (!gsi_end_p (input4_gsi));
11125 :
11126 83 : input_stmt4 = gsi_stmt (input4_gsi);
11127 83 : gsi = input4_gsi;
11128 83 : gsi_next (&gsi);
11129 83 : scan4_gsi = gsi;
11130 83 : scan_stmt4 = gsi_stmt (gsi);
11131 83 : gcc_assert (scan_stmt4 && gimple_code (scan_stmt4) == GIMPLE_OMP_SCAN);
11132 :
11133 83 : input_simd_ctx = maybe_lookup_ctx (input_stmt3)->outer;
11134 83 : scan_simd_ctx = maybe_lookup_ctx (input_stmt4)->outer;
11135 : }
11136 :
11137 177 : tree num_threads = create_tmp_var (integer_type_node);
11138 177 : tree thread_num = create_tmp_var (integer_type_node);
11139 177 : tree nthreads_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
11140 177 : tree threadnum_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
11141 177 : gimple *g = gimple_build_call (nthreads_decl, 0);
11142 177 : gimple_call_set_lhs (g, num_threads);
11143 177 : gimple_seq_add_stmt (body_p, g);
11144 177 : g = gimple_build_call (threadnum_decl, 0);
11145 177 : gimple_call_set_lhs (g, thread_num);
11146 177 : gimple_seq_add_stmt (body_p, g);
11147 :
11148 177 : tree ivar = create_tmp_var (sizetype);
11149 177 : tree new_clauses1 = NULL_TREE, new_clauses2 = NULL_TREE;
11150 177 : tree *cp1 = &new_clauses1, *cp2 = &new_clauses2;
11151 177 : tree k = create_tmp_var (unsigned_type_node);
11152 177 : tree l = create_tmp_var (unsigned_type_node);
11153 :
11154 177 : gimple_seq clist = NULL, mdlist = NULL;
11155 177 : gimple_seq thr01_list = NULL, thrn1_list = NULL;
11156 177 : gimple_seq thr02_list = NULL, thrn2_list = NULL;
11157 177 : gimple_seq scan1_list = NULL, input2_list = NULL;
11158 177 : gimple_seq last_list = NULL, reduc_list = NULL;
11159 686 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
11160 509 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
11161 509 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
11162 : {
11163 209 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
11164 209 : tree var = OMP_CLAUSE_DECL (c);
11165 209 : tree new_var = lookup_decl (var, ctx);
11166 209 : tree var3 = NULL_TREE;
11167 209 : tree new_vard = new_var;
11168 209 : if (omp_privatize_by_reference (var))
11169 32 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
11170 209 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
11171 : {
11172 64 : var3 = maybe_lookup_decl (new_vard, ctx);
11173 64 : if (var3 == new_vard)
11174 177 : var3 = NULL_TREE;
11175 : }
11176 :
11177 209 : tree ptype = build_pointer_type (TREE_TYPE (new_var));
11178 209 : tree rpriva = create_tmp_var (ptype);
11179 209 : tree nc = build_omp_clause (clause_loc, OMP_CLAUSE__SCANTEMP_);
11180 209 : OMP_CLAUSE_DECL (nc) = rpriva;
11181 209 : *cp1 = nc;
11182 209 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11183 :
11184 209 : tree rprivb = create_tmp_var (ptype);
11185 209 : nc = build_omp_clause (clause_loc, OMP_CLAUSE__SCANTEMP_);
11186 209 : OMP_CLAUSE_DECL (nc) = rprivb;
11187 209 : OMP_CLAUSE__SCANTEMP__ALLOC (nc) = 1;
11188 209 : *cp1 = nc;
11189 209 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11190 :
11191 209 : tree var2 = create_tmp_var_raw (TREE_TYPE (new_var));
11192 209 : if (new_vard != new_var)
11193 32 : TREE_ADDRESSABLE (var2) = 1;
11194 209 : gimple_add_tmp_var (var2);
11195 :
11196 209 : tree x = fold_convert_loc (clause_loc, sizetype, thread_num);
11197 209 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11198 209 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11199 209 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11200 209 : tree rpriva_ref = build_simple_mem_ref_loc (clause_loc, x);
11201 :
11202 209 : x = fold_build2_loc (clause_loc, PLUS_EXPR, integer_type_node,
11203 : thread_num, integer_minus_one_node);
11204 209 : x = fold_convert_loc (clause_loc, sizetype, x);
11205 209 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11206 209 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11207 209 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11208 209 : tree rprivam1_ref = build_simple_mem_ref_loc (clause_loc, x);
11209 :
11210 209 : x = fold_convert_loc (clause_loc, sizetype, l);
11211 209 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11212 209 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11213 209 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11214 209 : tree rprival_ref = build_simple_mem_ref_loc (clause_loc, x);
11215 :
11216 209 : x = fold_build2_loc (clause_loc, MINUS_EXPR, unsigned_type_node, l, k);
11217 209 : x = fold_convert_loc (clause_loc, sizetype, x);
11218 209 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11219 209 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11220 209 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11221 209 : tree rprivalmk_ref = build_simple_mem_ref_loc (clause_loc, x);
11222 :
11223 209 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, ivar,
11224 209 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11225 209 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rprivb), rprivb, x);
11226 209 : tree rprivb_ref = build_simple_mem_ref_loc (clause_loc, x);
11227 :
11228 209 : tree var4 = is_for_simd ? new_var : var2;
11229 99 : tree var5 = NULL_TREE, var6 = NULL_TREE;
11230 99 : if (is_for_simd)
11231 : {
11232 99 : var5 = lookup_decl (var, input_simd_ctx);
11233 99 : var6 = lookup_decl (var, scan_simd_ctx);
11234 99 : if (new_vard != new_var)
11235 : {
11236 16 : var5 = build_simple_mem_ref_loc (clause_loc, var5);
11237 16 : var6 = build_simple_mem_ref_loc (clause_loc, var6);
11238 : }
11239 : }
11240 209 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
11241 : {
11242 64 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
11243 64 : tree val = var2;
11244 :
11245 64 : x = lang_hooks.decls.omp_clause_default_ctor
11246 64 : (c, var2, build_outer_var_ref (var, ctx));
11247 64 : if (x)
11248 32 : gimplify_and_add (x, &clist);
11249 :
11250 64 : x = build_outer_var_ref (var, ctx);
11251 64 : x = lang_hooks.decls.omp_clause_assign_op (c, unshare_expr (var4),
11252 : x);
11253 64 : gimplify_and_add (x, &thr01_list);
11254 :
11255 64 : tree y = (DECL_HAS_VALUE_EXPR_P (new_vard)
11256 64 : ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
11257 64 : if (var3)
11258 : {
11259 32 : x = unshare_expr (var4);
11260 32 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var3);
11261 32 : gimplify_and_add (x, &thrn1_list);
11262 32 : x = unshare_expr (var4);
11263 32 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var3);
11264 32 : gimplify_and_add (x, &thr02_list);
11265 : }
11266 32 : else if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
11267 : {
11268 : /* Otherwise, assign to it the identity element. */
11269 32 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
11270 32 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11271 32 : if (!is_for_simd)
11272 : {
11273 16 : if (new_vard != new_var)
11274 4 : val = build_fold_addr_expr_loc (clause_loc, val);
11275 16 : SET_DECL_VALUE_EXPR (new_vard, val);
11276 16 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
11277 : }
11278 32 : SET_DECL_VALUE_EXPR (placeholder, error_mark_node);
11279 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11280 32 : lower_omp (&tseq, ctx);
11281 32 : gimple_seq_add_seq (&thrn1_list, tseq);
11282 32 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
11283 32 : lower_omp (&tseq, ctx);
11284 32 : gimple_seq_add_seq (&thr02_list, tseq);
11285 32 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
11286 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
11287 32 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
11288 32 : if (y)
11289 0 : SET_DECL_VALUE_EXPR (new_vard, y);
11290 : else
11291 : {
11292 32 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
11293 32 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
11294 : }
11295 : }
11296 :
11297 64 : x = unshare_expr (var4);
11298 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x, rprivam1_ref);
11299 64 : gimplify_and_add (x, &thrn2_list);
11300 :
11301 64 : if (is_for_simd)
11302 : {
11303 32 : x = unshare_expr (rprivb_ref);
11304 32 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var5);
11305 32 : gimplify_and_add (x, &scan1_list);
11306 : }
11307 : else
11308 : {
11309 32 : if (ctx->scan_exclusive)
11310 : {
11311 16 : x = unshare_expr (rprivb_ref);
11312 16 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var2);
11313 16 : gimplify_and_add (x, &scan1_list);
11314 : }
11315 :
11316 32 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
11317 32 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11318 32 : SET_DECL_VALUE_EXPR (placeholder, var2);
11319 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11320 32 : lower_omp (&tseq, ctx);
11321 32 : gimple_seq_add_seq (&scan1_list, tseq);
11322 :
11323 32 : if (ctx->scan_inclusive)
11324 : {
11325 16 : x = unshare_expr (rprivb_ref);
11326 16 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var2);
11327 16 : gimplify_and_add (x, &scan1_list);
11328 : }
11329 : }
11330 :
11331 64 : x = unshare_expr (rpriva_ref);
11332 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x,
11333 : unshare_expr (var4));
11334 64 : gimplify_and_add (x, &mdlist);
11335 :
11336 96 : x = unshare_expr (is_for_simd ? var6 : new_var);
11337 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var4);
11338 64 : gimplify_and_add (x, &input2_list);
11339 :
11340 64 : val = rprivb_ref;
11341 64 : if (new_vard != new_var)
11342 24 : val = build_fold_addr_expr_loc (clause_loc, val);
11343 :
11344 64 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
11345 64 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11346 64 : SET_DECL_VALUE_EXPR (new_vard, val);
11347 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
11348 64 : if (is_for_simd)
11349 : {
11350 32 : SET_DECL_VALUE_EXPR (placeholder, var6);
11351 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11352 : }
11353 : else
11354 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
11355 64 : lower_omp (&tseq, ctx);
11356 64 : if (y)
11357 0 : SET_DECL_VALUE_EXPR (new_vard, y);
11358 : else
11359 : {
11360 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
11361 64 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
11362 : }
11363 64 : if (!is_for_simd)
11364 : {
11365 32 : SET_DECL_VALUE_EXPR (placeholder, new_var);
11366 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11367 32 : lower_omp (&tseq, ctx);
11368 : }
11369 64 : gimple_seq_add_seq (&input2_list, tseq);
11370 :
11371 64 : x = build_outer_var_ref (var, ctx);
11372 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x, rpriva_ref);
11373 64 : gimplify_and_add (x, &last_list);
11374 :
11375 64 : x = lang_hooks.decls.omp_clause_assign_op (c, var2, rprivalmk_ref);
11376 64 : gimplify_and_add (x, &reduc_list);
11377 64 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
11378 64 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11379 64 : val = rprival_ref;
11380 64 : if (new_vard != new_var)
11381 24 : val = build_fold_addr_expr_loc (clause_loc, val);
11382 64 : SET_DECL_VALUE_EXPR (new_vard, val);
11383 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
11384 64 : SET_DECL_VALUE_EXPR (placeholder, var2);
11385 64 : lower_omp (&tseq, ctx);
11386 64 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
11387 64 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
11388 64 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
11389 64 : if (y)
11390 0 : SET_DECL_VALUE_EXPR (new_vard, y);
11391 : else
11392 : {
11393 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
11394 64 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
11395 : }
11396 64 : gimple_seq_add_seq (&reduc_list, tseq);
11397 64 : x = lang_hooks.decls.omp_clause_assign_op (c, rprival_ref, var2);
11398 64 : gimplify_and_add (x, &reduc_list);
11399 :
11400 64 : x = lang_hooks.decls.omp_clause_dtor (c, var2);
11401 64 : if (x)
11402 32 : gimplify_and_add (x, dlist);
11403 : }
11404 : else
11405 : {
11406 145 : x = build_outer_var_ref (var, ctx);
11407 145 : gimplify_assign (unshare_expr (var4), x, &thr01_list);
11408 :
11409 145 : x = omp_reduction_init (c, TREE_TYPE (new_var));
11410 145 : gimplify_assign (unshare_expr (var4), unshare_expr (x),
11411 : &thrn1_list);
11412 145 : gimplify_assign (unshare_expr (var4), x, &thr02_list);
11413 :
11414 145 : gimplify_assign (unshare_expr (var4), rprivam1_ref, &thrn2_list);
11415 :
11416 145 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
11417 145 : if (code == MINUS_EXPR)
11418 0 : code = PLUS_EXPR;
11419 :
11420 145 : if (is_for_simd)
11421 67 : gimplify_assign (unshare_expr (rprivb_ref), var5, &scan1_list);
11422 : else
11423 : {
11424 78 : if (ctx->scan_exclusive)
11425 28 : gimplify_assign (unshare_expr (rprivb_ref), var2,
11426 : &scan1_list);
11427 78 : x = build2 (code, TREE_TYPE (new_var), var2, new_var);
11428 78 : gimplify_assign (var2, x, &scan1_list);
11429 78 : if (ctx->scan_inclusive)
11430 50 : gimplify_assign (unshare_expr (rprivb_ref), var2,
11431 : &scan1_list);
11432 : }
11433 :
11434 145 : gimplify_assign (unshare_expr (rpriva_ref), unshare_expr (var4),
11435 : &mdlist);
11436 :
11437 145 : x = build2 (code, TREE_TYPE (new_var), var4, rprivb_ref);
11438 223 : gimplify_assign (is_for_simd ? var6 : new_var, x, &input2_list);
11439 :
11440 145 : gimplify_assign (build_outer_var_ref (var, ctx), rpriva_ref,
11441 : &last_list);
11442 :
11443 145 : x = build2 (code, TREE_TYPE (new_var), rprivalmk_ref,
11444 : unshare_expr (rprival_ref));
11445 145 : gimplify_assign (rprival_ref, x, &reduc_list);
11446 : }
11447 : }
11448 :
11449 177 : g = gimple_build_assign (ivar, PLUS_EXPR, ivar, size_one_node);
11450 177 : gimple_seq_add_stmt (&scan1_list, g);
11451 177 : g = gimple_build_assign (ivar, PLUS_EXPR, ivar, size_one_node);
11452 271 : gimple_seq_add_stmt (gimple_omp_body_ptr (is_for_simd
11453 : ? scan_stmt4 : scan_stmt2), g);
11454 :
11455 177 : tree controlb = create_tmp_var (boolean_type_node);
11456 177 : tree controlp = create_tmp_var (ptr_type_node);
11457 177 : tree nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11458 177 : OMP_CLAUSE_DECL (nc) = controlb;
11459 177 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11460 177 : *cp1 = nc;
11461 177 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11462 177 : nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11463 177 : OMP_CLAUSE_DECL (nc) = controlp;
11464 177 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11465 177 : *cp1 = nc;
11466 177 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11467 177 : nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11468 177 : OMP_CLAUSE_DECL (nc) = controlb;
11469 177 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11470 177 : *cp2 = nc;
11471 177 : cp2 = &OMP_CLAUSE_CHAIN (nc);
11472 177 : nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11473 177 : OMP_CLAUSE_DECL (nc) = controlp;
11474 177 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11475 177 : *cp2 = nc;
11476 177 : cp2 = &OMP_CLAUSE_CHAIN (nc);
11477 :
11478 177 : *cp1 = gimple_omp_for_clauses (stmt);
11479 177 : gimple_omp_for_set_clauses (stmt, new_clauses1);
11480 177 : *cp2 = gimple_omp_for_clauses (new_stmt);
11481 177 : gimple_omp_for_set_clauses (new_stmt, new_clauses2);
11482 :
11483 177 : if (is_for_simd)
11484 : {
11485 83 : gimple_seq_add_seq (gimple_omp_body_ptr (scan_stmt3), scan1_list);
11486 83 : gimple_seq_add_seq (gimple_omp_body_ptr (input_stmt4), input2_list);
11487 :
11488 83 : gsi_insert_seq_after (&input3_gsi, gimple_omp_body (input_stmt3),
11489 : GSI_SAME_STMT);
11490 83 : gsi_remove (&input3_gsi, true);
11491 83 : gsi_insert_seq_after (&scan3_gsi, gimple_omp_body (scan_stmt3),
11492 : GSI_SAME_STMT);
11493 83 : gsi_remove (&scan3_gsi, true);
11494 83 : gsi_insert_seq_after (&input4_gsi, gimple_omp_body (input_stmt4),
11495 : GSI_SAME_STMT);
11496 83 : gsi_remove (&input4_gsi, true);
11497 83 : gsi_insert_seq_after (&scan4_gsi, gimple_omp_body (scan_stmt4),
11498 : GSI_SAME_STMT);
11499 83 : gsi_remove (&scan4_gsi, true);
11500 : }
11501 : else
11502 : {
11503 94 : gimple_omp_set_body (scan_stmt1, scan1_list);
11504 94 : gimple_omp_set_body (input_stmt2, input2_list);
11505 : }
11506 :
11507 177 : gsi_insert_seq_after (&input1_gsi, gimple_omp_body (input_stmt1),
11508 : GSI_SAME_STMT);
11509 177 : gsi_remove (&input1_gsi, true);
11510 177 : gsi_insert_seq_after (&scan1_gsi, gimple_omp_body (scan_stmt1),
11511 : GSI_SAME_STMT);
11512 177 : gsi_remove (&scan1_gsi, true);
11513 177 : gsi_insert_seq_after (&input2_gsi, gimple_omp_body (input_stmt2),
11514 : GSI_SAME_STMT);
11515 177 : gsi_remove (&input2_gsi, true);
11516 177 : gsi_insert_seq_after (&scan2_gsi, gimple_omp_body (scan_stmt2),
11517 : GSI_SAME_STMT);
11518 177 : gsi_remove (&scan2_gsi, true);
11519 :
11520 177 : gimple_seq_add_seq (body_p, clist);
11521 :
11522 177 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
11523 177 : tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
11524 177 : tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
11525 177 : g = gimple_build_cond (EQ_EXPR, thread_num, integer_zero_node, lab1, lab2);
11526 177 : gimple_seq_add_stmt (body_p, g);
11527 177 : g = gimple_build_label (lab1);
11528 177 : gimple_seq_add_stmt (body_p, g);
11529 177 : gimple_seq_add_seq (body_p, thr01_list);
11530 177 : g = gimple_build_goto (lab3);
11531 177 : gimple_seq_add_stmt (body_p, g);
11532 177 : g = gimple_build_label (lab2);
11533 177 : gimple_seq_add_stmt (body_p, g);
11534 177 : gimple_seq_add_seq (body_p, thrn1_list);
11535 177 : g = gimple_build_label (lab3);
11536 177 : gimple_seq_add_stmt (body_p, g);
11537 :
11538 177 : g = gimple_build_assign (ivar, size_zero_node);
11539 177 : gimple_seq_add_stmt (body_p, g);
11540 :
11541 177 : gimple_seq_add_stmt (body_p, stmt);
11542 177 : gimple_seq_add_seq (body_p, body);
11543 177 : gimple_seq_add_stmt (body_p, gimple_build_omp_continue (fd->loop.v,
11544 : fd->loop.v));
11545 :
11546 177 : g = gimple_build_omp_return (true);
11547 177 : gimple_seq_add_stmt (body_p, g);
11548 177 : gimple_seq_add_seq (body_p, mdlist);
11549 :
11550 177 : lab1 = create_artificial_label (UNKNOWN_LOCATION);
11551 177 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
11552 177 : g = gimple_build_cond (GT_EXPR, num_threads, integer_one_node, lab1, lab2);
11553 177 : gimple_seq_add_stmt (body_p, g);
11554 177 : g = gimple_build_label (lab1);
11555 177 : gimple_seq_add_stmt (body_p, g);
11556 :
11557 177 : g = omp_build_barrier (NULL, GOMP_BARRIER_IMPLICIT_WORKSHARE);
11558 177 : gimple_seq_add_stmt (body_p, g);
11559 :
11560 177 : tree down = create_tmp_var (unsigned_type_node);
11561 177 : g = gimple_build_assign (down, build_zero_cst (unsigned_type_node));
11562 177 : gimple_seq_add_stmt (body_p, g);
11563 :
11564 177 : g = gimple_build_assign (k, build_one_cst (unsigned_type_node));
11565 177 : gimple_seq_add_stmt (body_p, g);
11566 :
11567 177 : tree num_threadsu = create_tmp_var (unsigned_type_node);
11568 177 : g = gimple_build_assign (num_threadsu, NOP_EXPR, num_threads);
11569 177 : gimple_seq_add_stmt (body_p, g);
11570 :
11571 177 : tree thread_numu = create_tmp_var (unsigned_type_node);
11572 177 : g = gimple_build_assign (thread_numu, NOP_EXPR, thread_num);
11573 177 : gimple_seq_add_stmt (body_p, g);
11574 :
11575 177 : tree thread_nump1 = create_tmp_var (unsigned_type_node);
11576 177 : g = gimple_build_assign (thread_nump1, PLUS_EXPR, thread_numu,
11577 : build_int_cst (unsigned_type_node, 1));
11578 177 : gimple_seq_add_stmt (body_p, g);
11579 :
11580 177 : lab3 = create_artificial_label (UNKNOWN_LOCATION);
11581 177 : g = gimple_build_label (lab3);
11582 177 : gimple_seq_add_stmt (body_p, g);
11583 :
11584 177 : tree twok = create_tmp_var (unsigned_type_node);
11585 177 : g = gimple_build_assign (twok, LSHIFT_EXPR, k, integer_one_node);
11586 177 : gimple_seq_add_stmt (body_p, g);
11587 :
11588 177 : tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
11589 177 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
11590 177 : tree lab6 = create_artificial_label (UNKNOWN_LOCATION);
11591 177 : g = gimple_build_cond (GT_EXPR, twok, num_threadsu, lab4, lab5);
11592 177 : gimple_seq_add_stmt (body_p, g);
11593 177 : g = gimple_build_label (lab4);
11594 177 : gimple_seq_add_stmt (body_p, g);
11595 177 : g = gimple_build_assign (down, build_all_ones_cst (unsigned_type_node));
11596 177 : gimple_seq_add_stmt (body_p, g);
11597 177 : g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
11598 177 : gimple_seq_add_stmt (body_p, g);
11599 :
11600 177 : g = gimple_build_cond (EQ_EXPR, k, num_threadsu, lab6, lab5);
11601 177 : gimple_seq_add_stmt (body_p, g);
11602 177 : g = gimple_build_label (lab6);
11603 177 : gimple_seq_add_stmt (body_p, g);
11604 :
11605 177 : g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
11606 177 : gimple_seq_add_stmt (body_p, g);
11607 :
11608 177 : g = gimple_build_label (lab5);
11609 177 : gimple_seq_add_stmt (body_p, g);
11610 :
11611 177 : g = gimple_build_assign (twok, LSHIFT_EXPR, k, integer_one_node);
11612 177 : gimple_seq_add_stmt (body_p, g);
11613 :
11614 177 : tree cplx = create_tmp_var (build_complex_type (unsigned_type_node, false));
11615 177 : g = gimple_build_call_internal (IFN_MUL_OVERFLOW, 2, thread_nump1, twok);
11616 177 : gimple_call_set_lhs (g, cplx);
11617 177 : gimple_seq_add_stmt (body_p, g);
11618 177 : tree mul = create_tmp_var (unsigned_type_node);
11619 177 : g = gimple_build_assign (mul, REALPART_EXPR,
11620 : build1 (REALPART_EXPR, unsigned_type_node, cplx));
11621 177 : gimple_seq_add_stmt (body_p, g);
11622 177 : tree ovf = create_tmp_var (unsigned_type_node);
11623 177 : g = gimple_build_assign (ovf, IMAGPART_EXPR,
11624 : build1 (IMAGPART_EXPR, unsigned_type_node, cplx));
11625 177 : gimple_seq_add_stmt (body_p, g);
11626 :
11627 177 : tree lab7 = create_artificial_label (UNKNOWN_LOCATION);
11628 177 : tree lab8 = create_artificial_label (UNKNOWN_LOCATION);
11629 177 : g = gimple_build_cond (EQ_EXPR, ovf, build_zero_cst (unsigned_type_node),
11630 : lab7, lab8);
11631 177 : gimple_seq_add_stmt (body_p, g);
11632 177 : g = gimple_build_label (lab7);
11633 177 : gimple_seq_add_stmt (body_p, g);
11634 :
11635 177 : tree andv = create_tmp_var (unsigned_type_node);
11636 177 : g = gimple_build_assign (andv, BIT_AND_EXPR, k, down);
11637 177 : gimple_seq_add_stmt (body_p, g);
11638 177 : tree andvm1 = create_tmp_var (unsigned_type_node);
11639 177 : g = gimple_build_assign (andvm1, PLUS_EXPR, andv,
11640 : build_minus_one_cst (unsigned_type_node));
11641 177 : gimple_seq_add_stmt (body_p, g);
11642 :
11643 177 : g = gimple_build_assign (l, PLUS_EXPR, mul, andvm1);
11644 177 : gimple_seq_add_stmt (body_p, g);
11645 :
11646 177 : tree lab9 = create_artificial_label (UNKNOWN_LOCATION);
11647 177 : g = gimple_build_cond (LT_EXPR, l, num_threadsu, lab9, lab8);
11648 177 : gimple_seq_add_stmt (body_p, g);
11649 177 : g = gimple_build_label (lab9);
11650 177 : gimple_seq_add_stmt (body_p, g);
11651 177 : gimple_seq_add_seq (body_p, reduc_list);
11652 177 : g = gimple_build_label (lab8);
11653 177 : gimple_seq_add_stmt (body_p, g);
11654 :
11655 177 : tree lab10 = create_artificial_label (UNKNOWN_LOCATION);
11656 177 : tree lab11 = create_artificial_label (UNKNOWN_LOCATION);
11657 177 : tree lab12 = create_artificial_label (UNKNOWN_LOCATION);
11658 177 : g = gimple_build_cond (EQ_EXPR, down, build_zero_cst (unsigned_type_node),
11659 : lab10, lab11);
11660 177 : gimple_seq_add_stmt (body_p, g);
11661 177 : g = gimple_build_label (lab10);
11662 177 : gimple_seq_add_stmt (body_p, g);
11663 177 : g = gimple_build_assign (k, LSHIFT_EXPR, k, integer_one_node);
11664 177 : gimple_seq_add_stmt (body_p, g);
11665 177 : g = gimple_build_goto (lab12);
11666 177 : gimple_seq_add_stmt (body_p, g);
11667 177 : g = gimple_build_label (lab11);
11668 177 : gimple_seq_add_stmt (body_p, g);
11669 177 : g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
11670 177 : gimple_seq_add_stmt (body_p, g);
11671 177 : g = gimple_build_label (lab12);
11672 177 : gimple_seq_add_stmt (body_p, g);
11673 :
11674 177 : g = omp_build_barrier (NULL, GOMP_BARRIER_IMPLICIT_WORKSHARE);
11675 177 : gimple_seq_add_stmt (body_p, g);
11676 :
11677 177 : g = gimple_build_cond (NE_EXPR, k, build_zero_cst (unsigned_type_node),
11678 : lab3, lab2);
11679 177 : gimple_seq_add_stmt (body_p, g);
11680 :
11681 177 : g = gimple_build_label (lab2);
11682 177 : gimple_seq_add_stmt (body_p, g);
11683 :
11684 177 : lab1 = create_artificial_label (UNKNOWN_LOCATION);
11685 177 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
11686 177 : lab3 = create_artificial_label (UNKNOWN_LOCATION);
11687 177 : g = gimple_build_cond (EQ_EXPR, thread_num, integer_zero_node, lab1, lab2);
11688 177 : gimple_seq_add_stmt (body_p, g);
11689 177 : g = gimple_build_label (lab1);
11690 177 : gimple_seq_add_stmt (body_p, g);
11691 177 : gimple_seq_add_seq (body_p, thr02_list);
11692 177 : g = gimple_build_goto (lab3);
11693 177 : gimple_seq_add_stmt (body_p, g);
11694 177 : g = gimple_build_label (lab2);
11695 177 : gimple_seq_add_stmt (body_p, g);
11696 177 : gimple_seq_add_seq (body_p, thrn2_list);
11697 177 : g = gimple_build_label (lab3);
11698 177 : gimple_seq_add_stmt (body_p, g);
11699 :
11700 177 : g = gimple_build_assign (ivar, size_zero_node);
11701 177 : gimple_seq_add_stmt (body_p, g);
11702 177 : gimple_seq_add_stmt (body_p, new_stmt);
11703 177 : gimple_seq_add_seq (body_p, new_body);
11704 :
11705 177 : gimple_seq new_dlist = NULL;
11706 177 : lab1 = create_artificial_label (UNKNOWN_LOCATION);
11707 177 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
11708 177 : tree num_threadsm1 = create_tmp_var (integer_type_node);
11709 177 : g = gimple_build_assign (num_threadsm1, PLUS_EXPR, num_threads,
11710 : integer_minus_one_node);
11711 177 : gimple_seq_add_stmt (&new_dlist, g);
11712 177 : g = gimple_build_cond (EQ_EXPR, thread_num, num_threadsm1, lab1, lab2);
11713 177 : gimple_seq_add_stmt (&new_dlist, g);
11714 177 : g = gimple_build_label (lab1);
11715 177 : gimple_seq_add_stmt (&new_dlist, g);
11716 177 : gimple_seq_add_seq (&new_dlist, last_list);
11717 177 : g = gimple_build_label (lab2);
11718 177 : gimple_seq_add_stmt (&new_dlist, g);
11719 177 : gimple_seq_add_seq (&new_dlist, *dlist);
11720 177 : *dlist = new_dlist;
11721 177 : }
11722 :
11723 : /* Build an internal UNIQUE function with type IFN_UNIQUE_OACC_PRIVATE listing
11724 : the addresses of variables to be made private at the surrounding
11725 : parallelism level. Such functions appear in the gimple code stream in two
11726 : forms, e.g. for a partitioned loop:
11727 :
11728 : .data_dep.6 = .UNIQUE (OACC_HEAD_MARK, .data_dep.6, 1, 68);
11729 : .data_dep.6 = .UNIQUE (OACC_PRIVATE, .data_dep.6, -1, &w);
11730 : .data_dep.6 = .UNIQUE (OACC_FORK, .data_dep.6, -1);
11731 : .data_dep.6 = .UNIQUE (OACC_HEAD_MARK, .data_dep.6);
11732 :
11733 : or alternatively, OACC_PRIVATE can appear at the top level of a parallel,
11734 : not as part of a HEAD_MARK sequence:
11735 :
11736 : .UNIQUE (OACC_PRIVATE, 0, 0, &w);
11737 :
11738 : For such stand-alone appearances, the 3rd argument is always 0, denoting
11739 : gang partitioning. */
11740 :
11741 : static gcall *
11742 19728 : lower_oacc_private_marker (omp_context *ctx)
11743 : {
11744 19728 : if (ctx->oacc_privatization_candidates.length () == 0)
11745 : return NULL;
11746 :
11747 308 : auto_vec<tree, 5> args;
11748 :
11749 308 : args.quick_push (build_int_cst (integer_type_node, IFN_UNIQUE_OACC_PRIVATE));
11750 308 : args.quick_push (integer_zero_node);
11751 308 : args.quick_push (integer_minus_one_node);
11752 :
11753 308 : int i;
11754 308 : tree decl;
11755 696 : FOR_EACH_VEC_ELT (ctx->oacc_privatization_candidates, i, decl)
11756 : {
11757 388 : gcc_checking_assert (TREE_ADDRESSABLE (decl));
11758 388 : tree addr = build_fold_addr_expr (decl);
11759 388 : args.safe_push (addr);
11760 : }
11761 :
11762 308 : return gimple_build_call_internal_vec (IFN_UNIQUE, args);
11763 308 : }
11764 :
11765 : /* Lower code for an OMP loop directive. */
11766 :
11767 : static void
11768 46859 : lower_omp_for (gimple_stmt_iterator *gsi_p, omp_context *ctx)
11769 : {
11770 46859 : tree *rhs_p, block;
11771 46859 : struct omp_for_data fd, *fdp = NULL;
11772 46859 : gomp_for *stmt = as_a <gomp_for *> (gsi_stmt (*gsi_p));
11773 46859 : gbind *new_stmt;
11774 46859 : gimple_seq omp_for_body, body, dlist, tred_ilist = NULL, tred_dlist = NULL;
11775 46859 : gimple_seq cnt_list = NULL, clist = NULL;
11776 46859 : gimple_seq oacc_head = NULL, oacc_tail = NULL;
11777 46859 : size_t i;
11778 :
11779 46859 : push_gimplify_context ();
11780 :
11781 46859 : if (is_gimple_omp_oacc (ctx->stmt))
11782 11325 : oacc_privatization_scan_clause_chain (ctx, gimple_omp_for_clauses (stmt));
11783 :
11784 46859 : lower_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
11785 :
11786 46859 : block = make_node (BLOCK);
11787 46859 : new_stmt = gimple_build_bind (NULL, NULL, block);
11788 : /* Replace at gsi right away, so that 'stmt' is no member
11789 : of a sequence anymore as we're going to add to a different
11790 : one below. */
11791 46859 : gsi_replace (gsi_p, new_stmt, true);
11792 :
11793 : /* Move declaration of temporaries in the loop body before we make
11794 : it go away. */
11795 46859 : omp_for_body = gimple_omp_body (stmt);
11796 46859 : if (!gimple_seq_empty_p (omp_for_body)
11797 46859 : && gimple_code (gimple_seq_first_stmt (omp_for_body)) == GIMPLE_BIND)
11798 : {
11799 19551 : gbind *inner_bind
11800 19551 : = as_a <gbind *> (gimple_seq_first_stmt (omp_for_body));
11801 19551 : tree vars = gimple_bind_vars (inner_bind);
11802 19551 : if (is_gimple_omp_oacc (ctx->stmt))
11803 5305 : oacc_privatization_scan_decl_chain (ctx, vars);
11804 19551 : gimple_bind_append_vars (new_stmt, vars);
11805 : /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
11806 : keep them on the inner_bind and it's block. */
11807 19551 : gimple_bind_set_vars (inner_bind, NULL_TREE);
11808 19551 : if (gimple_bind_block (inner_bind))
11809 16995 : BLOCK_VARS (gimple_bind_block (inner_bind)) = NULL_TREE;
11810 : }
11811 :
11812 46859 : if (gimple_omp_for_combined_into_p (stmt))
11813 : {
11814 14598 : omp_extract_for_data (stmt, &fd, NULL);
11815 14598 : fdp = &fd;
11816 :
11817 : /* We need two temporaries with fd.loop.v type (istart/iend)
11818 : and then (fd.collapse - 1) temporaries with the same
11819 : type for count2 ... countN-1 vars if not constant. */
11820 14598 : size_t count = 2;
11821 14598 : tree type = fd.iter_type;
11822 14598 : if (fd.collapse > 1
11823 4867 : && TREE_CODE (fd.loop.n2) != INTEGER_CST)
11824 2692 : count += fd.collapse - 1;
11825 14598 : size_t count2 = 0;
11826 14598 : tree type2 = NULL_TREE;
11827 14598 : bool taskreg_for
11828 14598 : = (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR
11829 14598 : || gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_TASKLOOP);
11830 14598 : tree outerc = NULL, *pc = gimple_omp_for_clauses_ptr (stmt);
11831 14598 : tree simtc = NULL;
11832 14598 : tree clauses = *pc;
11833 14598 : if (fd.collapse > 1
11834 4867 : && fd.non_rect
11835 137 : && fd.last_nonrect == fd.first_nonrect + 1
11836 89 : && TREE_CODE (fd.loop.n2) != INTEGER_CST)
11837 64 : if (tree v = gimple_omp_for_index (stmt, fd.last_nonrect))
11838 64 : if (!TYPE_UNSIGNED (TREE_TYPE (v)))
11839 : {
11840 60 : v = gimple_omp_for_index (stmt, fd.first_nonrect);
11841 60 : type2 = TREE_TYPE (v);
11842 60 : count++;
11843 60 : count2 = 3;
11844 : }
11845 14598 : if (taskreg_for)
11846 7594 : outerc
11847 7594 : = omp_find_clause (gimple_omp_taskreg_clauses (ctx->outer->stmt),
11848 : OMP_CLAUSE__LOOPTEMP_);
11849 14598 : if (ctx->simt_stmt)
11850 0 : simtc = omp_find_clause (gimple_omp_for_clauses (ctx->simt_stmt),
11851 : OMP_CLAUSE__LOOPTEMP_);
11852 49203 : for (i = 0; i < count + count2; i++)
11853 : {
11854 34605 : tree temp;
11855 34605 : if (taskreg_for)
11856 : {
11857 18069 : gcc_assert (outerc);
11858 18069 : temp = lookup_decl (OMP_CLAUSE_DECL (outerc), ctx->outer);
11859 18069 : outerc = omp_find_clause (OMP_CLAUSE_CHAIN (outerc),
11860 : OMP_CLAUSE__LOOPTEMP_);
11861 : }
11862 : else
11863 : {
11864 : /* If there are 2 adjacent SIMD stmts, one with _simt_
11865 : clause, another without, make sure they have the same
11866 : decls in _looptemp_ clauses, because the outer stmt
11867 : they are combined into will look up just one inner_stmt. */
11868 16536 : if (ctx->simt_stmt)
11869 0 : temp = OMP_CLAUSE_DECL (simtc);
11870 : else
11871 32928 : temp = create_tmp_var (i >= count ? type2 : type);
11872 16536 : insert_decl_map (&ctx->outer->cb, temp, temp);
11873 : }
11874 34605 : *pc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
11875 34605 : OMP_CLAUSE_DECL (*pc) = temp;
11876 34605 : pc = &OMP_CLAUSE_CHAIN (*pc);
11877 34605 : if (ctx->simt_stmt)
11878 0 : simtc = omp_find_clause (OMP_CLAUSE_CHAIN (simtc),
11879 : OMP_CLAUSE__LOOPTEMP_);
11880 : }
11881 14598 : *pc = clauses;
11882 : }
11883 :
11884 : /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
11885 46859 : dlist = NULL;
11886 46859 : body = NULL;
11887 46859 : tree rclauses
11888 46859 : = omp_task_reductions_find_first (gimple_omp_for_clauses (stmt), OMP_FOR,
11889 : OMP_CLAUSE_REDUCTION);
11890 46859 : tree rtmp = NULL_TREE;
11891 46859 : if (rclauses)
11892 : {
11893 227 : tree type = build_pointer_type (pointer_sized_int_node);
11894 227 : tree temp = create_tmp_var (type);
11895 227 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
11896 227 : OMP_CLAUSE_DECL (c) = temp;
11897 227 : OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (stmt);
11898 227 : gimple_omp_for_set_clauses (stmt, c);
11899 227 : lower_omp_task_reductions (ctx, OMP_FOR,
11900 : gimple_omp_for_clauses (stmt),
11901 : &tred_ilist, &tred_dlist);
11902 227 : rclauses = c;
11903 227 : rtmp = make_ssa_name (type);
11904 227 : gimple_seq_add_stmt (&body, gimple_build_assign (rtmp, temp));
11905 : }
11906 :
11907 46859 : lower_lastprivate_conditional_clauses (gimple_omp_for_clauses_ptr (stmt),
11908 : ctx);
11909 :
11910 46859 : lower_rec_input_clauses (gimple_omp_for_clauses (stmt), &body, &dlist, ctx,
11911 : fdp);
11912 93491 : gimple_seq_add_seq (rclauses ? &tred_ilist : &body,
11913 : gimple_omp_for_pre_body (stmt));
11914 :
11915 46859 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
11916 :
11917 46859 : gcall *private_marker = NULL;
11918 46859 : if (is_gimple_omp_oacc (ctx->stmt)
11919 46859 : && !gimple_seq_empty_p (omp_for_body))
11920 10333 : private_marker = lower_oacc_private_marker (ctx);
11921 :
11922 : /* Lower the header expressions. At this point, we can assume that
11923 : the header is of the form:
11924 :
11925 : #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
11926 :
11927 : We just need to make sure that VAL1, VAL2 and VAL3 are lowered
11928 : using the .omp_data_s mapping, if needed. */
11929 114984 : for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
11930 : {
11931 68125 : rhs_p = gimple_omp_for_initial_ptr (stmt, i);
11932 68125 : if (TREE_CODE (*rhs_p) == TREE_VEC)
11933 : {
11934 639 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 1)))
11935 149 : TREE_VEC_ELT (*rhs_p, 1)
11936 298 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 1), &cnt_list);
11937 639 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 2)))
11938 161 : TREE_VEC_ELT (*rhs_p, 2)
11939 322 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 2), &cnt_list);
11940 : }
11941 67486 : else if (!is_gimple_min_invariant (*rhs_p))
11942 11465 : *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
11943 56021 : else if (TREE_CODE (*rhs_p) == ADDR_EXPR)
11944 4713 : recompute_tree_invariant_for_addr_expr (*rhs_p);
11945 :
11946 68125 : rhs_p = gimple_omp_for_final_ptr (stmt, i);
11947 68125 : if (TREE_CODE (*rhs_p) == TREE_VEC)
11948 : {
11949 519 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 1)))
11950 142 : TREE_VEC_ELT (*rhs_p, 1)
11951 284 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 1), &cnt_list);
11952 519 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 2)))
11953 157 : TREE_VEC_ELT (*rhs_p, 2)
11954 314 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 2), &cnt_list);
11955 : }
11956 67606 : else if (!is_gimple_min_invariant (*rhs_p))
11957 16851 : *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
11958 50755 : else if (TREE_CODE (*rhs_p) == ADDR_EXPR)
11959 4630 : recompute_tree_invariant_for_addr_expr (*rhs_p);
11960 :
11961 68125 : rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1);
11962 68125 : if (!is_gimple_min_invariant (*rhs_p))
11963 6318 : *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
11964 : }
11965 46859 : if (rclauses)
11966 227 : gimple_seq_add_seq (&tred_ilist, cnt_list);
11967 : else
11968 46632 : gimple_seq_add_seq (&body, cnt_list);
11969 :
11970 : /* Once lowered, extract the bounds and clauses. */
11971 46859 : omp_extract_for_data (stmt, &fd, NULL);
11972 :
11973 46859 : if (is_gimple_omp_oacc (ctx->stmt)
11974 46859 : && !ctx_in_oacc_kernels_region (ctx))
11975 9665 : lower_oacc_head_tail (gimple_location (stmt),
11976 : gimple_omp_for_clauses (stmt), private_marker,
11977 : &oacc_head, &oacc_tail, ctx);
11978 :
11979 : /* Add OpenACC partitioning and reduction markers just before the loop. */
11980 46859 : if (oacc_head)
11981 9665 : gimple_seq_add_seq (&body, oacc_head);
11982 :
11983 46859 : lower_omp_for_lastprivate (&fd, &body, &dlist, &clist, ctx);
11984 :
11985 46859 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
11986 85430 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
11987 69910 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
11988 69910 : && !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
11989 : {
11990 203 : OMP_CLAUSE_DECL (c) = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
11991 203 : if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c)))
11992 42 : OMP_CLAUSE_LINEAR_STEP (c)
11993 84 : = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c),
11994 : ctx);
11995 : }
11996 :
11997 46489 : if ((ctx->scan_inclusive || ctx->scan_exclusive)
11998 47127 : && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
11999 177 : lower_omp_for_scan (&body, &dlist, stmt, &fd, ctx);
12000 : else
12001 : {
12002 46682 : gimple_seq_add_stmt (&body, stmt);
12003 46682 : gimple_seq_add_seq (&body, gimple_omp_body (stmt));
12004 : }
12005 :
12006 46859 : gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v,
12007 : fd.loop.v));
12008 :
12009 : /* After the loop, add exit clauses. */
12010 46859 : lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, &clist, ctx);
12011 :
12012 46859 : if (clist)
12013 : {
12014 173 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_START);
12015 173 : gcall *g = gimple_build_call (fndecl, 0);
12016 173 : gimple_seq_add_stmt (&body, g);
12017 173 : gimple_seq_add_seq (&body, clist);
12018 173 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_END);
12019 173 : g = gimple_build_call (fndecl, 0);
12020 173 : gimple_seq_add_stmt (&body, g);
12021 : }
12022 :
12023 46859 : if (ctx->cancellable)
12024 36 : gimple_seq_add_stmt (&body, gimple_build_label (ctx->cancel_label));
12025 :
12026 46859 : gimple_seq_add_seq (&body, dlist);
12027 :
12028 46859 : if (rclauses)
12029 : {
12030 227 : gimple_seq_add_seq (&tred_ilist, body);
12031 227 : body = tred_ilist;
12032 : }
12033 :
12034 46859 : body = maybe_catch_exception (body);
12035 :
12036 : /* Region exit marker goes at the end of the loop body. */
12037 46859 : gimple *g = gimple_build_omp_return (fd.have_nowait);
12038 46859 : gimple_seq_add_stmt (&body, g);
12039 :
12040 46859 : gimple_seq_add_seq (&body, tred_dlist);
12041 :
12042 46859 : maybe_add_implicit_barrier_cancel (ctx, g, &body);
12043 :
12044 46859 : if (rclauses)
12045 227 : OMP_CLAUSE_DECL (rclauses) = rtmp;
12046 :
12047 : /* Add OpenACC joining and reduction markers just after the loop. */
12048 46859 : if (oacc_tail)
12049 9665 : gimple_seq_add_seq (&body, oacc_tail);
12050 :
12051 46859 : pop_gimplify_context (new_stmt);
12052 :
12053 46859 : gimple_bind_append_vars (new_stmt, ctx->block_vars);
12054 46859 : maybe_remove_omp_member_access_dummy_vars (new_stmt);
12055 46859 : BLOCK_VARS (block) = gimple_bind_vars (new_stmt);
12056 46859 : if (BLOCK_VARS (block))
12057 46216 : TREE_USED (block) = 1;
12058 :
12059 46859 : gimple_bind_set_body (new_stmt, body);
12060 46859 : gimple_omp_set_body (stmt, NULL);
12061 46859 : gimple_omp_for_set_pre_body (stmt, NULL);
12062 46859 : }
12063 :
12064 : /* Callback for walk_stmts. Check if the current statement only contains
12065 : GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
12066 :
12067 : static tree
12068 213311 : check_combined_parallel (gimple_stmt_iterator *gsi_p,
12069 : bool *handled_ops_p,
12070 : struct walk_stmt_info *wi)
12071 : {
12072 213311 : int *info = (int *) wi->info;
12073 213311 : gimple *stmt = gsi_stmt (*gsi_p);
12074 :
12075 213311 : *handled_ops_p = true;
12076 213311 : switch (gimple_code (stmt))
12077 : {
12078 11592 : WALK_SUBSTMTS;
12079 :
12080 : case GIMPLE_DEBUG:
12081 : break;
12082 1826 : case GIMPLE_OMP_FOR:
12083 1826 : case GIMPLE_OMP_SECTIONS:
12084 1826 : *info = *info == 0 ? 1 : -1;
12085 1826 : break;
12086 199854 : default:
12087 199854 : *info = -1;
12088 199854 : break;
12089 : }
12090 213311 : return NULL;
12091 : }
12092 :
12093 : struct omp_taskcopy_context
12094 : {
12095 : /* This field must be at the beginning, as we do "inheritance": Some
12096 : callback functions for tree-inline.cc (e.g., omp_copy_decl)
12097 : receive a copy_body_data pointer that is up-casted to an
12098 : omp_context pointer. */
12099 : copy_body_data cb;
12100 : omp_context *ctx;
12101 : };
12102 :
12103 : static tree
12104 366 : task_copyfn_copy_decl (tree var, copy_body_data *cb)
12105 : {
12106 366 : struct omp_taskcopy_context *tcctx = (struct omp_taskcopy_context *) cb;
12107 :
12108 366 : if (splay_tree_lookup (tcctx->ctx->sfield_map, (splay_tree_key) var))
12109 158 : return create_tmp_var (TREE_TYPE (var));
12110 :
12111 : return var;
12112 : }
12113 :
12114 : static tree
12115 50 : task_copyfn_remap_type (struct omp_taskcopy_context *tcctx, tree orig_type)
12116 : {
12117 50 : tree name, new_fields = NULL, type, f;
12118 :
12119 50 : type = lang_hooks.types.make_type (RECORD_TYPE);
12120 50 : name = DECL_NAME (TYPE_NAME (orig_type));
12121 50 : name = build_decl (gimple_location (tcctx->ctx->stmt),
12122 : TYPE_DECL, name, type);
12123 50 : TYPE_NAME (type) = name;
12124 :
12125 1140 : for (f = TYPE_FIELDS (orig_type); f ; f = TREE_CHAIN (f))
12126 : {
12127 1090 : tree new_f = copy_node (f);
12128 1090 : DECL_CONTEXT (new_f) = type;
12129 1090 : TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &tcctx->cb);
12130 1090 : TREE_CHAIN (new_f) = new_fields;
12131 1090 : walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &tcctx->cb, NULL);
12132 1090 : walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, &tcctx->cb, NULL);
12133 1090 : walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
12134 : &tcctx->cb, NULL);
12135 1090 : new_fields = new_f;
12136 1090 : tcctx->cb.decl_map->put (f, new_f);
12137 : }
12138 50 : TYPE_FIELDS (type) = nreverse (new_fields);
12139 50 : layout_type (type);
12140 50 : return type;
12141 : }
12142 :
12143 : /* Create task copyfn. */
12144 :
12145 : static void
12146 480 : create_task_copyfn (gomp_task *task_stmt, omp_context *ctx)
12147 : {
12148 480 : struct function *child_cfun;
12149 480 : tree child_fn, t, c, src, dst, f, sf, arg, sarg, decl;
12150 480 : tree record_type, srecord_type, bind, list;
12151 480 : bool record_needs_remap = false, srecord_needs_remap = false;
12152 480 : splay_tree_node n;
12153 480 : struct omp_taskcopy_context tcctx;
12154 480 : location_t loc = gimple_location (task_stmt);
12155 480 : size_t looptempno = 0;
12156 :
12157 480 : child_fn = gimple_omp_task_copy_fn (task_stmt);
12158 480 : task_cpyfns.safe_push (task_stmt);
12159 480 : child_cfun = DECL_STRUCT_FUNCTION (child_fn);
12160 480 : gcc_assert (child_cfun->cfg == NULL);
12161 480 : DECL_SAVED_TREE (child_fn) = alloc_stmt_list ();
12162 :
12163 : /* Reset DECL_CONTEXT on function arguments. */
12164 1440 : for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
12165 960 : DECL_CONTEXT (t) = child_fn;
12166 :
12167 : /* Populate the function. */
12168 480 : push_gimplify_context ();
12169 480 : push_cfun (child_cfun);
12170 :
12171 480 : bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
12172 480 : TREE_SIDE_EFFECTS (bind) = 1;
12173 480 : list = NULL;
12174 480 : DECL_SAVED_TREE (child_fn) = bind;
12175 480 : DECL_SOURCE_LOCATION (child_fn) = gimple_location (task_stmt);
12176 :
12177 : /* Remap src and dst argument types if needed. */
12178 480 : record_type = ctx->record_type;
12179 480 : srecord_type = ctx->srecord_type;
12180 3018 : for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
12181 2563 : if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
12182 : {
12183 : record_needs_remap = true;
12184 : break;
12185 : }
12186 2757 : for (f = TYPE_FIELDS (srecord_type); f ; f = DECL_CHAIN (f))
12187 2302 : if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
12188 : {
12189 : srecord_needs_remap = true;
12190 : break;
12191 : }
12192 :
12193 480 : if (record_needs_remap || srecord_needs_remap)
12194 : {
12195 25 : memset (&tcctx, '\0', sizeof (tcctx));
12196 25 : tcctx.cb.src_fn = ctx->cb.src_fn;
12197 25 : tcctx.cb.dst_fn = child_fn;
12198 25 : tcctx.cb.src_node = cgraph_node::get (tcctx.cb.src_fn);
12199 25 : gcc_checking_assert (tcctx.cb.src_node);
12200 25 : tcctx.cb.dst_node = tcctx.cb.src_node;
12201 25 : tcctx.cb.src_cfun = ctx->cb.src_cfun;
12202 25 : tcctx.cb.copy_decl = task_copyfn_copy_decl;
12203 25 : tcctx.cb.eh_lp_nr = 0;
12204 25 : tcctx.cb.transform_call_graph_edges = CB_CGE_MOVE;
12205 25 : tcctx.cb.decl_map = new hash_map<tree, tree>;
12206 25 : tcctx.ctx = ctx;
12207 :
12208 25 : if (record_needs_remap)
12209 25 : record_type = task_copyfn_remap_type (&tcctx, record_type);
12210 25 : if (srecord_needs_remap)
12211 25 : srecord_type = task_copyfn_remap_type (&tcctx, srecord_type);
12212 : }
12213 : else
12214 455 : tcctx.cb.decl_map = NULL;
12215 :
12216 480 : arg = DECL_ARGUMENTS (child_fn);
12217 480 : TREE_TYPE (arg) = build_pointer_type (record_type);
12218 480 : sarg = DECL_CHAIN (arg);
12219 480 : TREE_TYPE (sarg) = build_pointer_type (srecord_type);
12220 :
12221 : /* First pass: initialize temporaries used in record_type and srecord_type
12222 : sizes and field offsets. */
12223 480 : if (tcctx.cb.decl_map)
12224 631 : for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
12225 606 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
12226 : {
12227 497 : tree *p;
12228 :
12229 497 : decl = OMP_CLAUSE_DECL (c);
12230 497 : p = tcctx.cb.decl_map->get (decl);
12231 497 : if (p == NULL)
12232 339 : continue;
12233 158 : n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
12234 158 : sf = (tree) n->value;
12235 158 : sf = *tcctx.cb.decl_map->get (sf);
12236 158 : src = build_simple_mem_ref_loc (loc, sarg);
12237 158 : src = omp_build_component_ref (src, sf);
12238 158 : t = build2 (MODIFY_EXPR, TREE_TYPE (*p), *p, src);
12239 158 : append_to_statement_list (t, &list);
12240 : }
12241 :
12242 : /* Second pass: copy shared var pointers and copy construct non-VLA
12243 : firstprivate vars. */
12244 5694 : for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
12245 5214 : switch (OMP_CLAUSE_CODE (c))
12246 : {
12247 559 : splay_tree_key key;
12248 559 : case OMP_CLAUSE_SHARED:
12249 559 : decl = OMP_CLAUSE_DECL (c);
12250 559 : key = (splay_tree_key) decl;
12251 559 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
12252 273 : key = (splay_tree_key) &DECL_UID (decl);
12253 559 : n = splay_tree_lookup (ctx->field_map, key);
12254 559 : if (n == NULL)
12255 : break;
12256 125 : f = (tree) n->value;
12257 125 : if (tcctx.cb.decl_map)
12258 12 : f = *tcctx.cb.decl_map->get (f);
12259 125 : n = splay_tree_lookup (ctx->sfield_map, key);
12260 125 : sf = (tree) n->value;
12261 125 : if (tcctx.cb.decl_map)
12262 12 : sf = *tcctx.cb.decl_map->get (sf);
12263 125 : src = build_simple_mem_ref_loc (loc, sarg);
12264 125 : src = omp_build_component_ref (src, sf);
12265 125 : dst = build_simple_mem_ref_loc (loc, arg);
12266 125 : dst = omp_build_component_ref (dst, f);
12267 125 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12268 125 : append_to_statement_list (t, &list);
12269 125 : break;
12270 460 : case OMP_CLAUSE_REDUCTION:
12271 460 : case OMP_CLAUSE_IN_REDUCTION:
12272 460 : decl = OMP_CLAUSE_DECL (c);
12273 460 : if (TREE_CODE (decl) == MEM_REF)
12274 : {
12275 160 : decl = TREE_OPERAND (decl, 0);
12276 160 : if (TREE_CODE (decl) == POINTER_PLUS_EXPR)
12277 38 : decl = TREE_OPERAND (decl, 0);
12278 160 : if (TREE_CODE (decl) == INDIRECT_REF
12279 128 : || TREE_CODE (decl) == ADDR_EXPR)
12280 95 : decl = TREE_OPERAND (decl, 0);
12281 : }
12282 460 : key = (splay_tree_key) decl;
12283 460 : n = splay_tree_lookup (ctx->field_map, key);
12284 460 : if (n == NULL)
12285 : break;
12286 101 : f = (tree) n->value;
12287 101 : if (tcctx.cb.decl_map)
12288 20 : f = *tcctx.cb.decl_map->get (f);
12289 101 : n = splay_tree_lookup (ctx->sfield_map, key);
12290 101 : sf = (tree) n->value;
12291 101 : if (tcctx.cb.decl_map)
12292 20 : sf = *tcctx.cb.decl_map->get (sf);
12293 101 : src = build_simple_mem_ref_loc (loc, sarg);
12294 101 : src = omp_build_component_ref (src, sf);
12295 101 : if (decl != OMP_CLAUSE_DECL (c)
12296 95 : && TREE_CODE (TREE_TYPE (decl)) == REFERENCE_TYPE
12297 173 : && TREE_CODE (TREE_TYPE (TREE_TYPE (decl))) == POINTER_TYPE)
12298 32 : src = build_simple_mem_ref_loc (loc, src);
12299 101 : dst = build_simple_mem_ref_loc (loc, arg);
12300 101 : dst = omp_build_component_ref (dst, f);
12301 101 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12302 101 : append_to_statement_list (t, &list);
12303 101 : break;
12304 754 : case OMP_CLAUSE__LOOPTEMP_:
12305 : /* Fields for first two _looptemp_ clauses are initialized by
12306 : GOMP_taskloop*, the rest are handled like firstprivate. */
12307 754 : if (looptempno < 2)
12308 : {
12309 748 : looptempno++;
12310 748 : break;
12311 : }
12312 : /* FALLTHRU */
12313 1695 : case OMP_CLAUSE__REDUCTEMP_:
12314 1695 : case OMP_CLAUSE_FIRSTPRIVATE:
12315 1695 : decl = OMP_CLAUSE_DECL (c);
12316 1695 : if (is_variable_sized (decl))
12317 : break;
12318 1681 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
12319 1681 : if (n == NULL)
12320 : break;
12321 1681 : f = (tree) n->value;
12322 1681 : if (tcctx.cb.decl_map)
12323 490 : f = *tcctx.cb.decl_map->get (f);
12324 1681 : n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
12325 1681 : if (n != NULL)
12326 : {
12327 1431 : sf = (tree) n->value;
12328 1431 : if (tcctx.cb.decl_map)
12329 490 : sf = *tcctx.cb.decl_map->get (sf);
12330 1431 : src = build_simple_mem_ref_loc (loc, sarg);
12331 1431 : src = omp_build_component_ref (src, sf);
12332 1431 : if (use_pointer_for_field (decl, NULL)
12333 1431 : || omp_privatize_by_reference (decl))
12334 293 : src = build_simple_mem_ref_loc (loc, src);
12335 : }
12336 : else
12337 : src = decl;
12338 1681 : dst = build_simple_mem_ref_loc (loc, arg);
12339 1681 : dst = omp_build_component_ref (dst, f);
12340 1681 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FIRSTPRIVATE)
12341 192 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12342 : else
12343 : {
12344 1489 : if (ctx->allocate_map)
12345 48 : if (tree *allocatorp = ctx->allocate_map->get (decl))
12346 : {
12347 26 : tree allocator = *allocatorp;
12348 26 : HOST_WIDE_INT ialign = 0;
12349 26 : if (TREE_CODE (allocator) == TREE_LIST)
12350 : {
12351 4 : ialign = tree_to_uhwi (TREE_VALUE (allocator));
12352 4 : allocator = TREE_PURPOSE (allocator);
12353 : }
12354 26 : if (TREE_CODE (allocator) != INTEGER_CST)
12355 : {
12356 10 : n = splay_tree_lookup (ctx->sfield_map,
12357 : (splay_tree_key) allocator);
12358 10 : allocator = (tree) n->value;
12359 10 : if (tcctx.cb.decl_map)
12360 0 : allocator = *tcctx.cb.decl_map->get (allocator);
12361 10 : tree a = build_simple_mem_ref_loc (loc, sarg);
12362 10 : allocator = omp_build_component_ref (a, allocator);
12363 : }
12364 26 : allocator = fold_convert (pointer_sized_int_node, allocator);
12365 26 : tree a = builtin_decl_explicit (BUILT_IN_GOMP_ALLOC);
12366 26 : tree align = build_int_cst (size_type_node,
12367 26 : MAX (ialign,
12368 : DECL_ALIGN_UNIT (decl)));
12369 26 : tree sz = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (dst)));
12370 26 : tree ptr = build_call_expr_loc (loc, a, 3, align, sz,
12371 : allocator);
12372 26 : ptr = fold_convert (TREE_TYPE (dst), ptr);
12373 26 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, ptr);
12374 26 : append_to_statement_list (t, &list);
12375 26 : dst = build_simple_mem_ref_loc (loc, dst);
12376 : }
12377 1489 : t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
12378 : }
12379 1681 : append_to_statement_list (t, &list);
12380 1681 : break;
12381 465 : case OMP_CLAUSE_PRIVATE:
12382 465 : if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c))
12383 : break;
12384 12 : decl = OMP_CLAUSE_DECL (c);
12385 12 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
12386 12 : f = (tree) n->value;
12387 12 : if (tcctx.cb.decl_map)
12388 0 : f = *tcctx.cb.decl_map->get (f);
12389 12 : n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
12390 12 : if (n != NULL)
12391 : {
12392 12 : sf = (tree) n->value;
12393 12 : if (tcctx.cb.decl_map)
12394 0 : sf = *tcctx.cb.decl_map->get (sf);
12395 12 : src = build_simple_mem_ref_loc (loc, sarg);
12396 12 : src = omp_build_component_ref (src, sf);
12397 12 : if (use_pointer_for_field (decl, NULL))
12398 12 : src = build_simple_mem_ref_loc (loc, src);
12399 : }
12400 : else
12401 : src = decl;
12402 12 : dst = build_simple_mem_ref_loc (loc, arg);
12403 12 : dst = omp_build_component_ref (dst, f);
12404 12 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12405 12 : append_to_statement_list (t, &list);
12406 12 : break;
12407 : default:
12408 : break;
12409 : }
12410 :
12411 : /* Last pass: handle VLA firstprivates. */
12412 480 : if (tcctx.cb.decl_map)
12413 631 : for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
12414 606 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
12415 : {
12416 497 : tree ind, ptr, df;
12417 :
12418 497 : decl = OMP_CLAUSE_DECL (c);
12419 497 : if (!is_variable_sized (decl))
12420 483 : continue;
12421 14 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
12422 14 : if (n == NULL)
12423 0 : continue;
12424 14 : f = (tree) n->value;
12425 14 : f = *tcctx.cb.decl_map->get (f);
12426 14 : gcc_assert (DECL_HAS_VALUE_EXPR_P (decl));
12427 14 : ind = DECL_VALUE_EXPR (decl);
12428 14 : gcc_assert (TREE_CODE (ind) == INDIRECT_REF);
12429 14 : gcc_assert (DECL_P (TREE_OPERAND (ind, 0)));
12430 42 : n = splay_tree_lookup (ctx->sfield_map,
12431 14 : (splay_tree_key) TREE_OPERAND (ind, 0));
12432 14 : sf = (tree) n->value;
12433 14 : sf = *tcctx.cb.decl_map->get (sf);
12434 14 : src = build_simple_mem_ref_loc (loc, sarg);
12435 14 : src = omp_build_component_ref (src, sf);
12436 14 : src = build_simple_mem_ref_loc (loc, src);
12437 14 : dst = build_simple_mem_ref_loc (loc, arg);
12438 14 : dst = omp_build_component_ref (dst, f);
12439 14 : t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
12440 14 : append_to_statement_list (t, &list);
12441 42 : n = splay_tree_lookup (ctx->field_map,
12442 14 : (splay_tree_key) TREE_OPERAND (ind, 0));
12443 14 : df = (tree) n->value;
12444 14 : df = *tcctx.cb.decl_map->get (df);
12445 14 : ptr = build_simple_mem_ref_loc (loc, arg);
12446 14 : ptr = omp_build_component_ref (ptr, df);
12447 14 : t = build2 (MODIFY_EXPR, TREE_TYPE (ptr), ptr,
12448 : build_fold_addr_expr_loc (loc, dst));
12449 14 : append_to_statement_list (t, &list);
12450 : }
12451 :
12452 480 : t = build1 (RETURN_EXPR, void_type_node, NULL);
12453 480 : append_to_statement_list (t, &list);
12454 :
12455 480 : if (tcctx.cb.decl_map)
12456 25 : delete tcctx.cb.decl_map;
12457 480 : pop_gimplify_context (NULL);
12458 480 : BIND_EXPR_BODY (bind) = list;
12459 480 : pop_cfun ();
12460 480 : }
12461 :
12462 : static void
12463 1553 : lower_depend_clauses (tree *pclauses, gimple_seq *iseq, gimple_seq *oseq)
12464 : {
12465 1553 : tree c, clauses;
12466 1553 : gimple *g;
12467 1553 : size_t cnt[5] = { 0, 0, 0, 0, 0 }, idx = 2, i;
12468 :
12469 1553 : clauses = omp_find_clause (*pclauses, OMP_CLAUSE_DEPEND);
12470 1553 : gcc_assert (clauses);
12471 6314 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
12472 4886 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
12473 1713 : switch (OMP_CLAUSE_DEPEND_KIND (c))
12474 : {
12475 125 : case OMP_CLAUSE_DEPEND_LAST:
12476 : /* Lowering already done at gimplification. */
12477 125 : return;
12478 481 : case OMP_CLAUSE_DEPEND_IN:
12479 481 : cnt[2]++;
12480 481 : break;
12481 969 : case OMP_CLAUSE_DEPEND_OUT:
12482 969 : case OMP_CLAUSE_DEPEND_INOUT:
12483 969 : cnt[0]++;
12484 969 : break;
12485 30 : case OMP_CLAUSE_DEPEND_MUTEXINOUTSET:
12486 30 : cnt[1]++;
12487 30 : break;
12488 76 : case OMP_CLAUSE_DEPEND_DEPOBJ:
12489 76 : cnt[3]++;
12490 76 : break;
12491 32 : case OMP_CLAUSE_DEPEND_INOUTSET:
12492 32 : cnt[4]++;
12493 32 : break;
12494 0 : default:
12495 0 : gcc_unreachable ();
12496 : }
12497 1428 : if (cnt[1] || cnt[3] || cnt[4])
12498 135 : idx = 5;
12499 1428 : size_t total = cnt[0] + cnt[1] + cnt[2] + cnt[3] + cnt[4];
12500 1428 : size_t inoutidx = total + idx;
12501 1428 : tree type = build_array_type_nelts (ptr_type_node, total + idx + 2 * cnt[4]);
12502 1428 : tree array = create_tmp_var (type);
12503 1428 : TREE_ADDRESSABLE (array) = 1;
12504 1428 : tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE,
12505 : NULL_TREE);
12506 1428 : if (idx == 5)
12507 : {
12508 135 : g = gimple_build_assign (r, build_int_cst (ptr_type_node, 0));
12509 135 : gimple_seq_add_stmt (iseq, g);
12510 135 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE,
12511 : NULL_TREE);
12512 : }
12513 1428 : g = gimple_build_assign (r, build_int_cst (ptr_type_node, total));
12514 1428 : gimple_seq_add_stmt (iseq, g);
12515 7140 : for (i = 0; i < (idx == 5 ? 3 : 1); i++)
12516 : {
12517 1698 : r = build4 (ARRAY_REF, ptr_type_node, array,
12518 1698 : size_int (i + 1 + (idx == 5)), NULL_TREE, NULL_TREE);
12519 1698 : g = gimple_build_assign (r, build_int_cst (ptr_type_node, cnt[i]));
12520 1698 : gimple_seq_add_stmt (iseq, g);
12521 : }
12522 8568 : for (i = 0; i < 5; i++)
12523 : {
12524 7140 : if (cnt[i] == 0)
12525 5656 : continue;
12526 6563 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
12527 5079 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DEPEND)
12528 3352 : continue;
12529 : else
12530 : {
12531 1727 : switch (OMP_CLAUSE_DEPEND_KIND (c))
12532 : {
12533 547 : case OMP_CLAUSE_DEPEND_IN:
12534 547 : if (i != 2)
12535 139 : continue;
12536 : break;
12537 1025 : case OMP_CLAUSE_DEPEND_OUT:
12538 1025 : case OMP_CLAUSE_DEPEND_INOUT:
12539 1025 : if (i != 0)
12540 56 : continue;
12541 : break;
12542 44 : case OMP_CLAUSE_DEPEND_MUTEXINOUTSET:
12543 44 : if (i != 1)
12544 14 : continue;
12545 : break;
12546 79 : case OMP_CLAUSE_DEPEND_DEPOBJ:
12547 79 : if (i != 3)
12548 3 : continue;
12549 : break;
12550 32 : case OMP_CLAUSE_DEPEND_INOUTSET:
12551 32 : if (i != 4)
12552 0 : continue;
12553 : break;
12554 0 : default:
12555 0 : gcc_unreachable ();
12556 : }
12557 1588 : tree t = OMP_CLAUSE_DECL (c);
12558 1588 : if (i == 4)
12559 : {
12560 32 : t = build4 (ARRAY_REF, ptr_type_node, array,
12561 32 : size_int (inoutidx), NULL_TREE, NULL_TREE);
12562 32 : t = build_fold_addr_expr (t);
12563 32 : inoutidx += 2;
12564 : }
12565 1588 : t = fold_convert (ptr_type_node, t);
12566 1588 : gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
12567 1588 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
12568 : NULL_TREE, NULL_TREE);
12569 1588 : g = gimple_build_assign (r, t);
12570 1588 : gimple_seq_add_stmt (iseq, g);
12571 : }
12572 : }
12573 1428 : if (cnt[4])
12574 81 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
12575 49 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
12576 49 : && OMP_CLAUSE_DEPEND_KIND (c) == OMP_CLAUSE_DEPEND_INOUTSET)
12577 : {
12578 32 : tree t = OMP_CLAUSE_DECL (c);
12579 32 : t = fold_convert (ptr_type_node, t);
12580 32 : gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
12581 32 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
12582 : NULL_TREE, NULL_TREE);
12583 32 : g = gimple_build_assign (r, t);
12584 32 : gimple_seq_add_stmt (iseq, g);
12585 32 : t = build_int_cst (ptr_type_node, GOMP_DEPEND_INOUTSET);
12586 32 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
12587 : NULL_TREE, NULL_TREE);
12588 32 : g = gimple_build_assign (r, t);
12589 32 : gimple_seq_add_stmt (iseq, g);
12590 : }
12591 :
12592 1428 : c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND);
12593 1428 : OMP_CLAUSE_DEPEND_KIND (c) = OMP_CLAUSE_DEPEND_LAST;
12594 1428 : OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array);
12595 1428 : OMP_CLAUSE_CHAIN (c) = *pclauses;
12596 1428 : *pclauses = c;
12597 1428 : tree clobber = build_clobber (type);
12598 1428 : g = gimple_build_assign (array, clobber);
12599 1428 : gimple_seq_add_stmt (oseq, g);
12600 : }
12601 :
12602 : /* Lower the OpenMP parallel or task directive in the current statement
12603 : in GSI_P. CTX holds context information for the directive. */
12604 :
12605 : static void
12606 22471 : lower_omp_taskreg (gimple_stmt_iterator *gsi_p, omp_context *ctx)
12607 : {
12608 22471 : tree clauses;
12609 22471 : tree child_fn, t;
12610 22471 : gimple *stmt = gsi_stmt (*gsi_p);
12611 22471 : gbind *par_bind, *bind, *dep_bind = NULL;
12612 22471 : gimple_seq par_body;
12613 22471 : location_t loc = gimple_location (stmt);
12614 :
12615 22471 : clauses = gimple_omp_taskreg_clauses (stmt);
12616 22471 : if (gimple_code (stmt) == GIMPLE_OMP_TASK
12617 22471 : && gimple_omp_task_taskwait_p (stmt))
12618 : {
12619 84 : par_bind = NULL;
12620 84 : par_body = NULL;
12621 : }
12622 : else
12623 : {
12624 22387 : par_bind
12625 22387 : = as_a <gbind *> (gimple_seq_first_stmt (gimple_omp_body (stmt)));
12626 22387 : par_body = gimple_bind_body (par_bind);
12627 : }
12628 22471 : child_fn = ctx->cb.dst_fn;
12629 22471 : if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
12630 22471 : && !gimple_omp_parallel_combined_p (stmt))
12631 : {
12632 4218 : struct walk_stmt_info wi;
12633 4218 : int ws_num = 0;
12634 :
12635 4218 : memset (&wi, 0, sizeof (wi));
12636 4218 : wi.info = &ws_num;
12637 4218 : wi.val_only = true;
12638 4218 : walk_gimple_seq (par_body, check_combined_parallel, NULL, &wi);
12639 4218 : if (ws_num == 1)
12640 386 : gimple_omp_parallel_set_combined_p (stmt, true);
12641 : }
12642 22471 : gimple_seq dep_ilist = NULL;
12643 22471 : gimple_seq dep_olist = NULL;
12644 22471 : if (gimple_code (stmt) == GIMPLE_OMP_TASK
12645 22471 : && omp_find_clause (clauses, OMP_CLAUSE_DEPEND))
12646 : {
12647 1189 : push_gimplify_context ();
12648 1189 : dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12649 1189 : lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt),
12650 : &dep_ilist, &dep_olist);
12651 : }
12652 :
12653 22471 : if (gimple_code (stmt) == GIMPLE_OMP_TASK
12654 22471 : && gimple_omp_task_taskwait_p (stmt))
12655 : {
12656 84 : if (dep_bind)
12657 : {
12658 84 : gsi_replace (gsi_p, dep_bind, true);
12659 84 : gimple_bind_add_seq (dep_bind, dep_ilist);
12660 84 : gimple_bind_add_stmt (dep_bind, stmt);
12661 84 : gimple_bind_add_seq (dep_bind, dep_olist);
12662 84 : pop_gimplify_context (dep_bind);
12663 : }
12664 84 : return;
12665 : }
12666 :
12667 22387 : if (ctx->srecord_type)
12668 480 : create_task_copyfn (as_a <gomp_task *> (stmt), ctx);
12669 :
12670 22387 : gimple_seq tskred_ilist = NULL;
12671 22387 : gimple_seq tskred_olist = NULL;
12672 22387 : if ((is_task_ctx (ctx)
12673 3748 : && gimple_omp_task_taskloop_p (ctx->stmt)
12674 1330 : && omp_find_clause (gimple_omp_task_clauses (ctx->stmt),
12675 : OMP_CLAUSE_REDUCTION))
12676 25638 : || (is_parallel_ctx (ctx)
12677 16121 : && omp_find_clause (gimple_omp_parallel_clauses (stmt),
12678 : OMP_CLAUSE__REDUCTEMP_)))
12679 : {
12680 557 : if (dep_bind == NULL)
12681 : {
12682 557 : push_gimplify_context ();
12683 557 : dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12684 : }
12685 617 : lower_omp_task_reductions (ctx, is_task_ctx (ctx) ? OMP_TASKLOOP
12686 : : OMP_PARALLEL,
12687 557 : gimple_omp_taskreg_clauses (ctx->stmt),
12688 : &tskred_ilist, &tskred_olist);
12689 : }
12690 :
12691 22387 : push_gimplify_context ();
12692 :
12693 22387 : gimple_seq par_olist = NULL;
12694 22387 : gimple_seq par_ilist = NULL;
12695 22387 : gimple_seq par_rlist = NULL;
12696 22387 : lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx, NULL);
12697 22383 : lower_omp (&par_body, ctx);
12698 22383 : if (gimple_code (stmt) != GIMPLE_OMP_TASK)
12699 18635 : lower_reduction_clauses (clauses, &par_rlist, NULL, ctx);
12700 :
12701 : /* Declare all the variables created by mapping and the variables
12702 : declared in the scope of the parallel body. */
12703 22383 : record_vars_into (ctx->block_vars, child_fn);
12704 22383 : maybe_remove_omp_member_access_dummy_vars (par_bind);
12705 22383 : record_vars_into (gimple_bind_vars (par_bind), child_fn);
12706 :
12707 22383 : if (ctx->record_type)
12708 : {
12709 17927 : ctx->sender_decl
12710 35374 : = create_tmp_var (ctx->srecord_type ? ctx->srecord_type
12711 : : ctx->record_type, ".omp_data_o");
12712 17927 : DECL_NAMELESS (ctx->sender_decl) = 1;
12713 17927 : TREE_ADDRESSABLE (ctx->sender_decl) = 1;
12714 17927 : gimple_omp_taskreg_set_data_arg (stmt, ctx->sender_decl);
12715 : }
12716 :
12717 22383 : gimple_seq olist = NULL;
12718 22383 : gimple_seq ilist = NULL;
12719 22383 : lower_send_clauses (clauses, &ilist, &olist, ctx);
12720 22383 : lower_send_shared_vars (&ilist, &olist, ctx);
12721 :
12722 22383 : if (ctx->record_type)
12723 : {
12724 17927 : tree clobber = build_clobber (TREE_TYPE (ctx->sender_decl));
12725 17927 : gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
12726 : clobber));
12727 : }
12728 :
12729 : /* Once all the expansions are done, sequence all the different
12730 : fragments inside gimple_omp_body. */
12731 :
12732 22383 : gimple_seq new_body = NULL;
12733 :
12734 22383 : if (ctx->record_type)
12735 : {
12736 17927 : t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
12737 : /* fixup_child_record_type might have changed receiver_decl's type. */
12738 17927 : t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
12739 17927 : gimple_seq_add_stmt (&new_body,
12740 17927 : gimple_build_assign (ctx->receiver_decl, t));
12741 : }
12742 :
12743 22383 : gimple_seq_add_seq (&new_body, par_ilist);
12744 22383 : gimple_seq_add_seq (&new_body, par_body);
12745 22383 : gimple_seq_add_seq (&new_body, par_rlist);
12746 22383 : if (ctx->cancellable)
12747 129 : gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
12748 22383 : gimple_seq_add_seq (&new_body, par_olist);
12749 22383 : new_body = maybe_catch_exception (new_body);
12750 22383 : if (gimple_code (stmt) == GIMPLE_OMP_TASK)
12751 3748 : gimple_seq_add_stmt (&new_body,
12752 3748 : gimple_build_omp_continue (integer_zero_node,
12753 : integer_zero_node));
12754 22383 : gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
12755 22383 : gimple_omp_set_body (stmt, new_body);
12756 :
12757 22383 : if (dep_bind && gimple_bind_block (par_bind) == NULL_TREE)
12758 497 : bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12759 : else
12760 21886 : bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind));
12761 43104 : gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
12762 22383 : gimple_bind_add_seq (bind, ilist);
12763 22383 : gimple_bind_add_stmt (bind, stmt);
12764 22383 : gimple_bind_add_seq (bind, olist);
12765 :
12766 22383 : pop_gimplify_context (NULL);
12767 :
12768 22383 : if (dep_bind)
12769 : {
12770 1662 : gimple_bind_add_seq (dep_bind, dep_ilist);
12771 1662 : gimple_bind_add_seq (dep_bind, tskred_ilist);
12772 1662 : gimple_bind_add_stmt (dep_bind, bind);
12773 1662 : gimple_bind_add_seq (dep_bind, tskred_olist);
12774 1662 : gimple_bind_add_seq (dep_bind, dep_olist);
12775 1662 : pop_gimplify_context (dep_bind);
12776 : }
12777 : }
12778 :
12779 : /* Set EXPR as the hostaddr expression that should result from the clause C
12780 : in the target statement STMT. Returns the tree that should be
12781 : passed as the hostaddr (a pointer to the array containing the expanded
12782 : hostaddrs and sizes of the clause). */
12783 :
12784 : static tree
12785 37307 : lower_omp_map_iterator_expr (tree expr, tree c, gomp_target *stmt)
12786 : {
12787 37307 : if (!OMP_CLAUSE_HAS_ITERATORS (c))
12788 : return expr;
12789 :
12790 215 : tree iterator = OMP_CLAUSE_ITERATORS (c);
12791 215 : tree index = OMP_ITERATOR_INDEX (iterator);
12792 215 : tree elems = OMP_ITERATOR_ELEMS (iterator);
12793 215 : gimple_seq *loop_body_p = enter_omp_iterator_loop_context (c, stmt);
12794 :
12795 : /* IN LOOP BODY: */
12796 : /* elems[idx] = <expr>; */
12797 215 : tree lhs = build4 (ARRAY_REF, ptr_type_node, elems, index,
12798 : NULL_TREE, NULL_TREE);
12799 215 : tree mod_expr = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
12800 : void_type_node, lhs, expr);
12801 215 : gimplify_and_add (mod_expr, loop_body_p);
12802 215 : exit_omp_iterator_loop_context (c);
12803 :
12804 215 : return build_fold_addr_expr_with_type (elems, ptr_type_node);
12805 : }
12806 :
12807 : /* Set SIZE as the size expression that should result from the clause C
12808 : in the target statement STMT. Returns the tree that should be
12809 : passed as the clause size (a size_int with the value SIZE_MAX, indicating
12810 : that the clause uses an iterator). */
12811 :
12812 : static tree
12813 80132 : lower_omp_map_iterator_size (tree size, tree c, gomp_target *stmt)
12814 : {
12815 80132 : if (!OMP_CLAUSE_HAS_ITERATORS (c))
12816 : return size;
12817 :
12818 219 : tree iterator = OMP_CLAUSE_ITERATORS (c);
12819 219 : tree index = OMP_ITERATOR_INDEX (iterator);
12820 219 : tree elems = OMP_ITERATOR_ELEMS (iterator);
12821 219 : gimple_seq *loop_body_p = enter_omp_iterator_loop_context (c, stmt);
12822 :
12823 : /* IN LOOP BODY: */
12824 : /* elems[idx+1] = <size>; */
12825 219 : tree lhs = build4 (ARRAY_REF, ptr_type_node, elems,
12826 : size_binop (PLUS_EXPR, index, size_int (1)),
12827 : NULL_TREE, NULL_TREE);
12828 219 : tree mod_expr = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
12829 : void_type_node, lhs, size);
12830 219 : gimplify_and_add (mod_expr, loop_body_p);
12831 219 : exit_omp_iterator_loop_context (c);
12832 :
12833 219 : return size_int (SIZE_MAX);
12834 : }
12835 :
12836 : /* Lower the GIMPLE_OMP_TARGET in the current statement
12837 : in GSI_P. CTX holds context information for the directive. */
12838 :
12839 : static void
12840 37147 : lower_omp_target (gimple_stmt_iterator *gsi_p, omp_context *ctx)
12841 : {
12842 37147 : tree clauses;
12843 37147 : tree child_fn, t, c;
12844 37147 : gomp_target *stmt = as_a <gomp_target *> (gsi_stmt (*gsi_p));
12845 37147 : gbind *tgt_bind, *bind, *dep_bind = NULL;
12846 37147 : gimple_seq tgt_body, olist, ilist, fplist, new_body;
12847 37147 : location_t loc = gimple_location (stmt);
12848 37147 : bool offloaded, data_region;
12849 37147 : unsigned int map_cnt = 0;
12850 37147 : tree in_reduction_clauses = NULL_TREE;
12851 :
12852 37147 : tree deep_map_cnt = NULL_TREE;
12853 37147 : tree deep_map_data = NULL_TREE;
12854 37147 : tree deep_map_offset_data = NULL_TREE;
12855 37147 : tree deep_map_offset = NULL_TREE;
12856 :
12857 37147 : offloaded = is_gimple_omp_offloaded (stmt);
12858 37147 : switch (gimple_omp_target_kind (stmt))
12859 : {
12860 11792 : case GF_OMP_TARGET_KIND_REGION:
12861 11792 : tree *p, *q;
12862 11792 : q = &in_reduction_clauses;
12863 79316 : for (p = gimple_omp_target_clauses_ptr (stmt); *p; )
12864 67524 : if (OMP_CLAUSE_CODE (*p) == OMP_CLAUSE_IN_REDUCTION)
12865 : {
12866 422 : *q = *p;
12867 422 : q = &OMP_CLAUSE_CHAIN (*q);
12868 422 : *p = OMP_CLAUSE_CHAIN (*p);
12869 : }
12870 : else
12871 67102 : p = &OMP_CLAUSE_CHAIN (*p);
12872 11792 : *q = NULL_TREE;
12873 11792 : *p = in_reduction_clauses;
12874 : /* FALLTHRU */
12875 : case GF_OMP_TARGET_KIND_UPDATE:
12876 : case GF_OMP_TARGET_KIND_ENTER_DATA:
12877 : case GF_OMP_TARGET_KIND_EXIT_DATA:
12878 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
12879 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
12880 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
12881 : case GF_OMP_TARGET_KIND_OACC_UPDATE:
12882 : case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
12883 : case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
12884 : case GF_OMP_TARGET_KIND_OACC_DECLARE:
12885 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
12886 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
12887 : data_region = false;
12888 : break;
12889 : case GF_OMP_TARGET_KIND_DATA:
12890 : case GF_OMP_TARGET_KIND_OACC_DATA:
12891 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
12892 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
12893 : data_region = true;
12894 : break;
12895 0 : default:
12896 0 : gcc_unreachable ();
12897 : }
12898 :
12899 : /* Ensure that requires map is written via output_offload_tables, even if only
12900 : 'target (enter/exit) data' is used in the translation unit. */
12901 37147 : if (ENABLE_OFFLOADING && (omp_requires_mask & OMP_REQUIRES_TARGET_USED))
12902 : g->have_offload = true;
12903 :
12904 37147 : clauses = gimple_omp_target_clauses (stmt);
12905 :
12906 37147 : gimple_seq dep_ilist = NULL;
12907 37147 : gimple_seq dep_olist = NULL;
12908 37147 : bool has_depend = omp_find_clause (clauses, OMP_CLAUSE_DEPEND) != NULL_TREE;
12909 37147 : if (has_depend || in_reduction_clauses)
12910 : {
12911 392 : push_gimplify_context ();
12912 392 : dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12913 392 : if (has_depend)
12914 329 : lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt),
12915 : &dep_ilist, &dep_olist);
12916 392 : if (in_reduction_clauses)
12917 314 : lower_rec_input_clauses (in_reduction_clauses, &dep_ilist, &dep_olist,
12918 : ctx, NULL);
12919 : }
12920 :
12921 37147 : tgt_bind = NULL;
12922 37147 : tgt_body = NULL;
12923 37147 : if (offloaded)
12924 : {
12925 21187 : tgt_bind = gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt));
12926 21187 : tgt_body = gimple_bind_body (tgt_bind);
12927 : }
12928 15960 : else if (data_region)
12929 4021 : tgt_body = gimple_omp_body (stmt);
12930 37147 : child_fn = ctx->cb.dst_fn;
12931 :
12932 37147 : push_gimplify_context ();
12933 37147 : fplist = NULL;
12934 :
12935 37147 : ilist = NULL;
12936 37147 : olist = NULL;
12937 :
12938 37147 : gimple_seq alloc_dlist = NULL;
12939 37147 : hash_map<tree, tree> alloc_map;
12940 74294 : hash_map<tree, gimple_seq> alloc_seq_map;
12941 :
12942 176349 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
12943 139202 : switch (OMP_CLAUSE_CODE (c))
12944 : {
12945 : tree var, x, new_var, allocator, allocate_ptr, size;
12946 : gimple_seq alloc_seq;
12947 : bool by_ref;
12948 :
12949 : default:
12950 92788 : break;
12951 77857 : case OMP_CLAUSE_MAP:
12952 : #if CHECKING_P
12953 : /* First check what we're prepared to handle in the following. */
12954 77857 : switch (OMP_CLAUSE_MAP_KIND (c))
12955 : {
12956 : case GOMP_MAP_ALLOC:
12957 : case GOMP_MAP_TO:
12958 : case GOMP_MAP_FROM:
12959 : case GOMP_MAP_TOFROM:
12960 : case GOMP_MAP_POINTER:
12961 : case GOMP_MAP_TO_PSET:
12962 : case GOMP_MAP_DELETE:
12963 : case GOMP_MAP_RELEASE:
12964 : case GOMP_MAP_ALWAYS_TO:
12965 : case GOMP_MAP_ALWAYS_FROM:
12966 : case GOMP_MAP_ALWAYS_TOFROM:
12967 : case GOMP_MAP_FORCE_PRESENT:
12968 : case GOMP_MAP_ALWAYS_PRESENT_FROM:
12969 : case GOMP_MAP_ALWAYS_PRESENT_TO:
12970 : case GOMP_MAP_ALWAYS_PRESENT_TOFROM:
12971 :
12972 : case GOMP_MAP_FIRSTPRIVATE_POINTER:
12973 : case GOMP_MAP_FIRSTPRIVATE_REFERENCE:
12974 : case GOMP_MAP_STRUCT:
12975 : case GOMP_MAP_STRUCT_UNORD:
12976 : case GOMP_MAP_ALWAYS_POINTER:
12977 : case GOMP_MAP_ATTACH:
12978 : case GOMP_MAP_DETACH:
12979 : case GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION:
12980 : case GOMP_MAP_POINTER_TO_ZERO_LENGTH_ARRAY_SECTION:
12981 : break;
12982 2587 : case GOMP_MAP_IF_PRESENT:
12983 2587 : case GOMP_MAP_FORCE_ALLOC:
12984 2587 : case GOMP_MAP_FORCE_TO:
12985 2587 : case GOMP_MAP_FORCE_FROM:
12986 2587 : case GOMP_MAP_FORCE_TOFROM:
12987 2587 : case GOMP_MAP_FORCE_DEVICEPTR:
12988 2587 : case GOMP_MAP_DEVICE_RESIDENT:
12989 2587 : case GOMP_MAP_LINK:
12990 2587 : case GOMP_MAP_FORCE_DETACH:
12991 2587 : gcc_assert (is_gimple_omp_oacc (stmt));
12992 : break;
12993 0 : default:
12994 0 : gcc_unreachable ();
12995 : }
12996 : #endif
12997 : /* FALLTHRU */
12998 89427 : case OMP_CLAUSE_TO:
12999 89427 : case OMP_CLAUSE_FROM:
13000 89427 : oacc_firstprivate:
13001 89427 : var = OMP_CLAUSE_DECL (c);
13002 89427 : {
13003 89427 : tree extra = lang_hooks.decls.omp_deep_mapping_cnt (stmt, c, &ilist);
13004 89427 : if (extra != NULL_TREE && deep_map_cnt != NULL_TREE)
13005 59 : deep_map_cnt = fold_build2_loc (OMP_CLAUSE_LOCATION (c), PLUS_EXPR,
13006 : size_type_node, deep_map_cnt,
13007 : extra);
13008 89368 : else if (extra != NULL_TREE)
13009 : deep_map_cnt = extra;
13010 : }
13011 :
13012 89283 : if (deep_map_cnt
13013 89649 : && OMP_CLAUSE_HAS_ITERATORS (c))
13014 0 : sorry ("iterators used together with deep mapping are not "
13015 : "supported yet");
13016 :
13017 89427 : if (!DECL_P (var))
13018 : {
13019 36882 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
13020 36882 : || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
13021 35276 : && (OMP_CLAUSE_MAP_KIND (c)
13022 : != GOMP_MAP_FIRSTPRIVATE_POINTER)))
13023 36882 : map_cnt++;
13024 46414 : continue;
13025 : }
13026 :
13027 52545 : if (DECL_SIZE (var)
13028 52545 : && !poly_int_tree_p (DECL_SIZE (var)))
13029 : {
13030 717 : tree var2 = DECL_VALUE_EXPR (var);
13031 717 : gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
13032 717 : var2 = TREE_OPERAND (var2, 0);
13033 717 : gcc_assert (DECL_P (var2));
13034 : var = var2;
13035 : }
13036 :
13037 52545 : if (offloaded
13038 36679 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13039 85328 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
13040 29548 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE))
13041 : {
13042 3664 : if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
13043 : {
13044 415 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx))
13045 415 : && varpool_node::get_create (var)->offloadable)
13046 2 : continue;
13047 :
13048 413 : tree type = build_pointer_type (TREE_TYPE (var));
13049 413 : tree new_var = lookup_decl (var, ctx);
13050 413 : x = create_tmp_var_raw (type, get_name (new_var));
13051 413 : gimple_add_tmp_var (x);
13052 413 : x = build_simple_mem_ref (x);
13053 413 : SET_DECL_VALUE_EXPR (new_var, x);
13054 413 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13055 : }
13056 3662 : continue;
13057 3662 : }
13058 :
13059 48881 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13060 38917 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
13061 38710 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
13062 49142 : && is_omp_target (stmt))
13063 : {
13064 237 : gcc_assert (maybe_lookup_field (c, ctx));
13065 237 : map_cnt++;
13066 237 : continue;
13067 : }
13068 :
13069 48644 : if (!maybe_lookup_field (c, ctx))
13070 5631 : continue;
13071 :
13072 : /* Don't remap compute constructs' reduction variables, because the
13073 : intermediate result must be local to each gang. */
13074 70854 : if (offloaded && !(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13075 23945 : && is_gimple_omp_oacc (ctx->stmt)
13076 13572 : && OMP_CLAUSE_MAP_IN_REDUCTION (c)))
13077 : {
13078 27061 : x = build_receiver_ref (c, true, ctx);
13079 27061 : tree new_var = lookup_decl (var, ctx);
13080 :
13081 27061 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13082 23165 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
13083 3014 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
13084 30075 : && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
13085 425 : x = build_simple_mem_ref (x);
13086 27061 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
13087 : {
13088 3896 : gcc_assert (is_gimple_omp_oacc (ctx->stmt));
13089 3896 : if (omp_privatize_by_reference (new_var)
13090 3896 : && (TREE_CODE (TREE_TYPE (new_var)) != POINTER_TYPE
13091 44 : || DECL_BY_REFERENCE (var)))
13092 : {
13093 : /* Create a local object to hold the instance
13094 : value. */
13095 455 : tree type = TREE_TYPE (TREE_TYPE (new_var));
13096 455 : const char *id = IDENTIFIER_POINTER (DECL_NAME (new_var));
13097 455 : tree inst = create_tmp_var (type, id);
13098 455 : gimplify_assign (inst, fold_indirect_ref (x), &fplist);
13099 455 : x = build_fold_addr_expr (inst);
13100 : }
13101 3896 : gimplify_assign (new_var, x, &fplist);
13102 : }
13103 23165 : else if (DECL_P (new_var))
13104 : {
13105 23165 : SET_DECL_VALUE_EXPR (new_var, x);
13106 23165 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13107 : }
13108 : else
13109 0 : gcc_unreachable ();
13110 : }
13111 43013 : map_cnt++;
13112 43013 : break;
13113 :
13114 15294 : case OMP_CLAUSE_FIRSTPRIVATE:
13115 15294 : omp_firstprivate_recv:
13116 15294 : gcc_checking_assert (offloaded);
13117 15294 : if (is_gimple_omp_oacc (ctx->stmt))
13118 : {
13119 : /* No 'firstprivate' clauses on OpenACC 'kernels'. */
13120 3896 : gcc_checking_assert (!is_oacc_kernels (ctx));
13121 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
13122 3896 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
13123 :
13124 3896 : goto oacc_firstprivate;
13125 : }
13126 11398 : map_cnt++;
13127 11398 : var = OMP_CLAUSE_DECL (c);
13128 11398 : new_var = lookup_decl (var, ctx);
13129 11398 : allocator = NULL_TREE;
13130 11398 : allocate_ptr = NULL_TREE;
13131 11398 : size = TREE_TYPE (var);
13132 11398 : by_ref = omp_privatize_by_reference (var);
13133 11398 : if (by_ref)
13134 822 : size = TREE_TYPE (size);
13135 11398 : size = TYPE_SIZE_UNIT (size);
13136 11398 : if (is_variable_sized (var, by_ref))
13137 35 : size = lookup_decl (size, ctx);
13138 11398 : alloc_seq = NULL;
13139 11398 : if (lower_private_allocate (var, new_var, allocator, allocate_ptr,
13140 : &alloc_seq, ctx, by_ref, size))
13141 : {
13142 63 : alloc_map.put (new_var, allocate_ptr);
13143 63 : alloc_seq_map.put (new_var, alloc_seq);
13144 : }
13145 11398 : if (!by_ref && !is_gimple_reg_type (TREE_TYPE (var)))
13146 : {
13147 143 : if (is_variable_sized (var))
13148 : {
13149 6 : tree pvar = DECL_VALUE_EXPR (var);
13150 6 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
13151 6 : pvar = TREE_OPERAND (pvar, 0);
13152 6 : gcc_assert (DECL_P (pvar));
13153 6 : tree new_pvar = lookup_decl (pvar, ctx);
13154 6 : x = build_fold_indirect_ref (new_pvar);
13155 6 : TREE_THIS_NOTRAP (x) = 1;
13156 : }
13157 137 : else if (allocate_ptr)
13158 19 : x = build_fold_indirect_ref (allocate_ptr);
13159 : else
13160 118 : x = build_receiver_ref (var, true, ctx);
13161 143 : SET_DECL_VALUE_EXPR (new_var, x);
13162 143 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13163 : }
13164 : /* Fortran array descriptors: firstprivate of data + attach. */
13165 11398 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
13166 11379 : && lang_hooks.decls.omp_array_data (var, true)
13167 11429 : && lang_hooks.decls.omp_array_data_privatize (var))
13168 23 : map_cnt += 2;
13169 :
13170 11544 : do_dtor:
13171 11544 : if (allocator)
13172 : {
13173 118 : if (!is_gimple_val (allocator))
13174 : {
13175 0 : tree avar = create_tmp_var (TREE_TYPE (allocator));
13176 0 : gimplify_assign (avar, allocator, &alloc_dlist);
13177 0 : allocator = avar;
13178 : }
13179 118 : if (!is_gimple_val (allocate_ptr))
13180 : {
13181 0 : tree apvar = create_tmp_var (TREE_TYPE (allocate_ptr));
13182 0 : gimplify_assign (apvar, allocate_ptr, &alloc_dlist);
13183 0 : allocate_ptr = apvar;
13184 : }
13185 118 : tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
13186 118 : gimple *g = gimple_build_call (f, 2, allocate_ptr, allocator);
13187 118 : gimple_seq_add_stmt (&alloc_dlist, g);
13188 : }
13189 : break;
13190 :
13191 226 : case OMP_CLAUSE_PRIVATE:
13192 226 : gcc_checking_assert (offloaded);
13193 226 : if (is_gimple_omp_oacc (ctx->stmt))
13194 : {
13195 : /* No 'private' clauses on OpenACC 'kernels'. */
13196 80 : gcc_checking_assert (!is_oacc_kernels (ctx));
13197 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
13198 80 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
13199 :
13200 : break;
13201 : }
13202 146 : var = OMP_CLAUSE_DECL (c);
13203 146 : new_var = lookup_decl (var, ctx);
13204 146 : allocator = NULL_TREE;
13205 146 : allocate_ptr = NULL_TREE;
13206 146 : alloc_seq = NULL;
13207 146 : size = TREE_TYPE (var);
13208 146 : by_ref = omp_privatize_by_reference (var);
13209 146 : if (by_ref)
13210 22 : size = TREE_TYPE (size);
13211 146 : size = TYPE_SIZE_UNIT (size);
13212 146 : if (is_variable_sized (var, by_ref))
13213 21 : size = lookup_decl (size, ctx);
13214 146 : lower_private_allocate (var, new_var, allocator, allocate_ptr,
13215 : &alloc_seq, ctx, by_ref, size);
13216 146 : if (allocate_ptr)
13217 : {
13218 55 : alloc_map.put (new_var, allocate_ptr);
13219 55 : alloc_seq_map.put (new_var, alloc_seq);
13220 : }
13221 146 : if (!allocate_ptr && is_variable_sized (var))
13222 : {
13223 1 : tree new_var = lookup_decl (var, ctx);
13224 1 : tree pvar = DECL_VALUE_EXPR (var);
13225 1 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
13226 1 : pvar = TREE_OPERAND (pvar, 0);
13227 1 : gcc_assert (DECL_P (pvar));
13228 1 : tree new_pvar = lookup_decl (pvar, ctx);
13229 1 : x = build_fold_indirect_ref (new_pvar);
13230 1 : TREE_THIS_NOTRAP (x) = 1;
13231 1 : SET_DECL_VALUE_EXPR (new_var, x);
13232 1 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13233 : }
13234 146 : goto do_dtor;
13235 :
13236 2430 : case OMP_CLAUSE_USE_DEVICE_PTR:
13237 2430 : case OMP_CLAUSE_USE_DEVICE_ADDR:
13238 2430 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
13239 2430 : case OMP_CLAUSE_IS_DEVICE_PTR:
13240 2430 : var = OMP_CLAUSE_DECL (c);
13241 2430 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
13242 : {
13243 252 : while (TREE_CODE (var) == INDIRECT_REF
13244 252 : || TREE_CODE (var) == ARRAY_REF)
13245 17 : var = TREE_OPERAND (var, 0);
13246 235 : if (lang_hooks.decls.omp_array_data (var, true))
13247 19 : goto omp_firstprivate_recv;
13248 : }
13249 2411 : map_cnt++;
13250 2411 : if (is_variable_sized (var))
13251 : {
13252 12 : tree new_var = lookup_decl (var, ctx);
13253 12 : tree pvar = DECL_VALUE_EXPR (var);
13254 12 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
13255 12 : pvar = TREE_OPERAND (pvar, 0);
13256 12 : gcc_assert (DECL_P (pvar));
13257 12 : tree new_pvar = lookup_decl (pvar, ctx);
13258 12 : x = build_fold_indirect_ref (new_pvar);
13259 12 : TREE_THIS_NOTRAP (x) = 1;
13260 12 : SET_DECL_VALUE_EXPR (new_var, x);
13261 12 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13262 : }
13263 2399 : else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
13264 542 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
13265 2071 : && !omp_privatize_by_reference (var)
13266 703 : && !omp_is_allocatable_or_ptr (var)
13267 564 : && !lang_hooks.decls.omp_array_data (var, true))
13268 4433 : || TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
13269 : {
13270 389 : tree new_var = lookup_decl (var, ctx);
13271 389 : tree type = build_pointer_type (TREE_TYPE (var));
13272 389 : x = create_tmp_var_raw (type, get_name (new_var));
13273 389 : gimple_add_tmp_var (x);
13274 389 : x = build_simple_mem_ref (x);
13275 389 : SET_DECL_VALUE_EXPR (new_var, x);
13276 389 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13277 : }
13278 : else
13279 : {
13280 2010 : tree new_var = lookup_decl (var, ctx);
13281 2010 : x = create_tmp_var_raw (TREE_TYPE (new_var), get_name (new_var));
13282 2010 : gimple_add_tmp_var (x);
13283 2010 : SET_DECL_VALUE_EXPR (new_var, x);
13284 2010 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13285 : }
13286 : break;
13287 10 : case OMP_CLAUSE_DEVICE_TYPE:
13288 : /* FIXME: Ensure that 'nohost' also has not implied before that
13289 : 'g->have_offload = true' or an implicit declare target. */
13290 10 : if (OMP_CLAUSE_DEVICE_TYPE_KIND (c) != OMP_CLAUSE_DEVICE_TYPE_ANY)
13291 5 : sorry_at (OMP_CLAUSE_LOCATION (c),
13292 : "only the %<device_type(any)%> is supported");
13293 : break;
13294 68 : case OMP_CLAUSE_USES_ALLOCATORS:
13295 68 : allocator = OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c);
13296 68 : tree new_allocator = lookup_decl (allocator, ctx);
13297 68 : x = build_receiver_ref (allocator, false, ctx);
13298 68 : SET_DECL_VALUE_EXPR (new_allocator, x);
13299 68 : DECL_HAS_VALUE_EXPR_P (new_allocator) = 1;
13300 68 : map_cnt++;
13301 68 : break;
13302 : }
13303 :
13304 37147 : if (offloaded)
13305 : {
13306 21187 : target_nesting_level++;
13307 21187 : lower_omp (&tgt_body, ctx);
13308 21187 : target_nesting_level--;
13309 : }
13310 15960 : else if (data_region)
13311 4021 : lower_omp (&tgt_body, ctx);
13312 :
13313 37147 : if (offloaded)
13314 : {
13315 : /* Declare all the variables created by mapping and the variables
13316 : declared in the scope of the target body. */
13317 21187 : record_vars_into (ctx->block_vars, child_fn);
13318 21187 : maybe_remove_omp_member_access_dummy_vars (tgt_bind);
13319 21187 : record_vars_into (gimple_bind_vars (tgt_bind), child_fn);
13320 : }
13321 :
13322 37147 : if (ctx->record_type)
13323 : {
13324 32260 : if (deep_map_cnt && TREE_CODE (deep_map_cnt) == INTEGER_CST)
13325 : /* map_cnt = map_cnt + tree_to_hwi (deep_map_cnt); */
13326 : /* deep_map_cnt = NULL_TREE; */
13327 0 : gcc_unreachable ();
13328 144 : else if (deep_map_cnt)
13329 : {
13330 144 : gcc_assert (flexible_array_type_p (ctx->record_type));
13331 144 : tree n = create_tmp_var_raw (size_type_node, "nn_map");
13332 144 : gimple_add_tmp_var (n);
13333 144 : gimplify_assign (n, deep_map_cnt, &ilist);
13334 144 : deep_map_cnt = n;
13335 : }
13336 144 : ctx->sender_decl
13337 32260 : = create_tmp_var (deep_map_cnt ? build_pointer_type (ctx->record_type)
13338 : : ctx->record_type, ".omp_data_arr");
13339 32260 : DECL_NAMELESS (ctx->sender_decl) = 1;
13340 32260 : TREE_ADDRESSABLE (ctx->sender_decl) = 1;
13341 64376 : t = make_tree_vec (deep_map_cnt ? 4 : 3);
13342 32260 : TREE_VEC_ELT (t, 0) = ctx->sender_decl;
13343 32260 : TREE_VEC_ELT (t, 1)
13344 64520 : = create_tmp_var (deep_map_cnt
13345 144 : ? build_pointer_type (size_type_node)
13346 32116 : : build_array_type_nelts (size_type_node, map_cnt),
13347 : ".omp_data_sizes");
13348 32260 : DECL_NAMELESS (TREE_VEC_ELT (t, 1)) = 1;
13349 32260 : TREE_ADDRESSABLE (TREE_VEC_ELT (t, 1)) = 1;
13350 32260 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 1;
13351 32260 : tree tkind_type = short_unsigned_type_node;
13352 32260 : int talign_shift = 8;
13353 32260 : TREE_VEC_ELT (t, 2)
13354 64520 : = create_tmp_var (deep_map_cnt
13355 144 : ? build_pointer_type (tkind_type)
13356 32116 : : build_array_type_nelts (tkind_type, map_cnt),
13357 : ".omp_data_kinds");
13358 32260 : DECL_NAMELESS (TREE_VEC_ELT (t, 2)) = 1;
13359 32260 : TREE_ADDRESSABLE (TREE_VEC_ELT (t, 2)) = 1;
13360 32260 : TREE_STATIC (TREE_VEC_ELT (t, 2)) = 1;
13361 32260 : gimple_omp_target_set_data_arg (stmt, t);
13362 :
13363 32260 : if (deep_map_cnt)
13364 : {
13365 144 : tree tmp, size;
13366 144 : size = create_tmp_var (size_type_node, NULL);
13367 144 : DECL_NAMELESS (size) = 1;
13368 144 : gimplify_assign (size,
13369 : fold_build2_loc (UNKNOWN_LOCATION, PLUS_EXPR,
13370 : size_type_node, deep_map_cnt,
13371 : build_int_cst (size_type_node,
13372 144 : map_cnt)), &ilist);
13373 144 : TREE_VEC_ELT (t, 3) = size;
13374 :
13375 144 : tree call = builtin_decl_explicit (BUILT_IN_MALLOC);
13376 144 : size = fold_build2_loc (UNKNOWN_LOCATION, MULT_EXPR,
13377 : size_type_node, deep_map_cnt,
13378 144 : TYPE_SIZE_UNIT (ptr_type_node));
13379 144 : size = fold_build2_loc (UNKNOWN_LOCATION, PLUS_EXPR,
13380 : size_type_node, size,
13381 144 : TYPE_SIZE_UNIT (ctx->record_type));
13382 144 : tmp = build_call_expr_loc (input_location, call, 1, size);
13383 144 : gimplify_assign (ctx->sender_decl, tmp, &ilist);
13384 :
13385 144 : size = fold_build2_loc (UNKNOWN_LOCATION, MULT_EXPR,
13386 144 : size_type_node, TREE_VEC_ELT (t, 3),
13387 144 : TYPE_SIZE_UNIT (size_type_node));
13388 144 : tmp = build_call_expr_loc (input_location, call, 1, size);
13389 144 : gimplify_assign (TREE_VEC_ELT (t, 1), tmp, &ilist);
13390 :
13391 144 : size = fold_build2_loc (UNKNOWN_LOCATION, MULT_EXPR,
13392 144 : size_type_node, TREE_VEC_ELT (t, 3),
13393 144 : TYPE_SIZE_UNIT (tkind_type));
13394 144 : tmp = build_call_expr_loc (input_location, call, 1, size);
13395 144 : gimplify_assign (TREE_VEC_ELT (t, 2), tmp, &ilist);
13396 144 : tree field = TYPE_FIELDS (TREE_TYPE (TREE_TYPE (ctx->sender_decl)));
13397 595 : for ( ; DECL_CHAIN (field) != NULL_TREE; field = DECL_CHAIN (field))
13398 : ;
13399 144 : gcc_assert (TREE_CODE (TREE_TYPE (field)));
13400 144 : tmp = build_fold_indirect_ref (ctx->sender_decl);
13401 144 : deep_map_data = omp_build_component_ref (tmp, field);
13402 144 : deep_map_offset_data = create_tmp_var_raw (size_type_node,
13403 : "map_offset_data");
13404 144 : deep_map_offset = create_tmp_var_raw (size_type_node, "map_offset");
13405 144 : gimple_add_tmp_var (deep_map_offset_data);
13406 144 : gimple_add_tmp_var (deep_map_offset);
13407 144 : gimplify_assign (deep_map_offset_data, build_int_cst (size_type_node,
13408 : 0), &ilist);
13409 144 : gimplify_assign (deep_map_offset, build_int_cst (size_type_node,
13410 144 : map_cnt), &ilist);
13411 : }
13412 :
13413 32260 : vec<constructor_elt, va_gc> *vsize;
13414 32260 : vec<constructor_elt, va_gc> *vkind;
13415 32260 : vec_alloc (vsize, map_cnt);
13416 32260 : vec_alloc (vkind, map_cnt);
13417 32260 : unsigned int map_idx = 0;
13418 :
13419 162084 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
13420 129824 : switch (OMP_CLAUSE_CODE (c))
13421 : {
13422 : tree ovar, nc, s, purpose, var, x, type;
13423 : unsigned int talign;
13424 :
13425 : default:
13426 125860 : break;
13427 :
13428 87928 : case OMP_CLAUSE_MAP:
13429 87928 : case OMP_CLAUSE_TO:
13430 87928 : case OMP_CLAUSE_FROM:
13431 87928 : oacc_firstprivate_map:
13432 87928 : nc = c;
13433 87928 : ovar = OMP_CLAUSE_DECL (c);
13434 87928 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13435 87928 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
13436 72955 : || (OMP_CLAUSE_MAP_KIND (c)
13437 : == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
13438 : break;
13439 84096 : if (deep_map_cnt)
13440 : {
13441 428 : unsigned HOST_WIDE_INT tkind2;
13442 428 : switch (OMP_CLAUSE_CODE (c))
13443 : {
13444 424 : case OMP_CLAUSE_MAP:
13445 424 : tkind2 = OMP_CLAUSE_MAP_KIND (c);
13446 424 : if (OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (c)
13447 424 : && (((tkind2 & GOMP_MAP_FLAG_SPECIAL_BITS)
13448 : & ~GOMP_MAP_IMPLICIT)
13449 : == 0))
13450 : {
13451 : /* If this is an implicit map, and the GOMP_MAP_IMPLICIT
13452 : bits are not interfered by other special bit
13453 : encodings, then turn the GOMP_IMPLICIT_BIT flag on
13454 : for the runtime to see. */
13455 120 : tkind2 |= GOMP_MAP_IMPLICIT;
13456 : }
13457 : break;
13458 : case OMP_CLAUSE_FIRSTPRIVATE: tkind2 = GOMP_MAP_TO; break;
13459 : case OMP_CLAUSE_TO: tkind2 = GOMP_MAP_TO; break;
13460 2 : case OMP_CLAUSE_FROM: tkind2 = GOMP_MAP_FROM; break;
13461 0 : default: gcc_unreachable ();
13462 : }
13463 428 : lang_hooks.decls.omp_deep_mapping (stmt, c, tkind2,
13464 : deep_map_data,
13465 428 : TREE_VEC_ELT (t, 1),
13466 428 : TREE_VEC_ELT (t, 2),
13467 : deep_map_offset_data,
13468 : deep_map_offset, &ilist);
13469 : }
13470 84096 : if (!DECL_P (ovar))
13471 : {
13472 36882 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13473 36882 : && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
13474 : {
13475 0 : nc = OMP_CLAUSE_CHAIN (c);
13476 0 : gcc_checking_assert (OMP_CLAUSE_DECL (nc)
13477 : == get_base_address (ovar));
13478 0 : ovar = OMP_CLAUSE_DECL (nc);
13479 : }
13480 : else
13481 : {
13482 36882 : tree x = build_sender_ref (c, ctx);
13483 36882 : tree v = ovar;
13484 36882 : if (in_reduction_clauses
13485 163 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13486 37045 : && OMP_CLAUSE_MAP_IN_REDUCTION (c))
13487 : {
13488 104 : v = unshare_expr (v);
13489 104 : tree *p = &v;
13490 104 : while (handled_component_p (*p)
13491 : || TREE_CODE (*p) == INDIRECT_REF
13492 : || TREE_CODE (*p) == ADDR_EXPR
13493 : || TREE_CODE (*p) == MEM_REF
13494 256 : || TREE_CODE (*p) == NON_LVALUE_EXPR)
13495 152 : p = &TREE_OPERAND (*p, 0);
13496 104 : tree d = *p;
13497 104 : if (is_variable_sized (d))
13498 : {
13499 16 : gcc_assert (DECL_HAS_VALUE_EXPR_P (d));
13500 16 : d = DECL_VALUE_EXPR (d);
13501 16 : gcc_assert (TREE_CODE (d) == INDIRECT_REF);
13502 16 : d = TREE_OPERAND (d, 0);
13503 16 : gcc_assert (DECL_P (d));
13504 : }
13505 104 : splay_tree_key key
13506 104 : = (splay_tree_key) &DECL_CONTEXT (d);
13507 104 : tree nd = (tree) splay_tree_lookup (ctx->field_map,
13508 104 : key)->value;
13509 104 : if (d == *p)
13510 88 : *p = nd;
13511 : else
13512 16 : *p = build_fold_indirect_ref (nd);
13513 : }
13514 36882 : v = build_fold_addr_expr_with_type (v, ptr_type_node);
13515 36882 : v = lower_omp_map_iterator_expr (v, c, stmt);
13516 36882 : gimplify_assign (x, v, &ilist);
13517 36882 : nc = NULL_TREE;
13518 : }
13519 : }
13520 : else
13521 : {
13522 47214 : if (DECL_SIZE (ovar)
13523 47214 : && !poly_int_tree_p (DECL_SIZE (ovar)))
13524 : {
13525 7 : tree ovar2 = DECL_VALUE_EXPR (ovar);
13526 7 : gcc_assert (TREE_CODE (ovar2) == INDIRECT_REF);
13527 7 : ovar2 = TREE_OPERAND (ovar2, 0);
13528 7 : gcc_assert (DECL_P (ovar2));
13529 : ovar = ovar2;
13530 : }
13531 47214 : if (!maybe_lookup_field (c, ctx)
13532 51178 : && !(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13533 3964 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
13534 3964 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)))
13535 3964 : continue;
13536 : }
13537 :
13538 80132 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar));
13539 80132 : if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign)
13540 6409 : talign = DECL_ALIGN_UNIT (ovar);
13541 :
13542 80132 : var = NULL_TREE;
13543 80132 : if (nc)
13544 : {
13545 43250 : if (in_reduction_clauses
13546 897 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13547 44147 : && OMP_CLAUSE_MAP_IN_REDUCTION (c))
13548 : {
13549 268 : tree d = ovar;
13550 268 : if (is_variable_sized (d))
13551 : {
13552 0 : gcc_assert (DECL_HAS_VALUE_EXPR_P (d));
13553 0 : d = DECL_VALUE_EXPR (d);
13554 0 : gcc_assert (TREE_CODE (d) == INDIRECT_REF);
13555 0 : d = TREE_OPERAND (d, 0);
13556 0 : gcc_assert (DECL_P (d));
13557 : }
13558 268 : splay_tree_key key
13559 268 : = (splay_tree_key) &DECL_CONTEXT (d);
13560 268 : tree nd = (tree) splay_tree_lookup (ctx->field_map,
13561 268 : key)->value;
13562 268 : if (d == ovar)
13563 : var = nd;
13564 : else
13565 0 : var = build_fold_indirect_ref (nd);
13566 : }
13567 : else
13568 42982 : var = lookup_decl_in_outer_ctx (ovar, ctx);
13569 : }
13570 43250 : if (nc
13571 43250 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13572 33286 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
13573 33079 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
13574 261 : && is_omp_target (stmt))
13575 : {
13576 237 : x = build_sender_ref (c, ctx);
13577 237 : gimplify_assign (x, build_fold_addr_expr (var), &ilist);
13578 : }
13579 79895 : else if (nc)
13580 : {
13581 43013 : x = build_sender_ref (nc, ctx);
13582 :
13583 43013 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13584 33049 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
13585 5184 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
13586 48197 : && TREE_CODE (TREE_TYPE (ovar)) == ARRAY_TYPE)
13587 : {
13588 425 : gcc_assert (offloaded);
13589 425 : tree avar = build_fold_addr_expr (var);
13590 425 : if (!OMP_CLAUSE_ITERATORS (c))
13591 : {
13592 425 : tree tmp = create_tmp_var (TREE_TYPE (TREE_TYPE (x)));
13593 425 : mark_addressable (tmp);
13594 425 : gimplify_assign (tmp, avar, &ilist);
13595 425 : avar = tmp;
13596 : }
13597 425 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (x)));
13598 425 : avar = build_fold_addr_expr (avar);
13599 425 : avar = lower_omp_map_iterator_expr (avar, c, stmt);
13600 425 : gimplify_assign (x, avar, &ilist);
13601 : }
13602 42588 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
13603 : {
13604 3896 : gcc_assert (is_gimple_omp_oacc (ctx->stmt));
13605 3896 : if (!omp_privatize_by_reference (var))
13606 : {
13607 3441 : if (is_gimple_reg (var)
13608 3441 : && OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
13609 2578 : suppress_warning (var);
13610 3441 : var = build_fold_addr_expr (var);
13611 : }
13612 : else
13613 455 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
13614 3896 : gimplify_assign (x, var, &ilist);
13615 : }
13616 38692 : else if (is_gimple_reg (var))
13617 : {
13618 5283 : gcc_assert (offloaded);
13619 5283 : tree avar = create_tmp_var (TREE_TYPE (var));
13620 5283 : mark_addressable (avar);
13621 5283 : enum gomp_map_kind map_kind = OMP_CLAUSE_MAP_KIND (c);
13622 5283 : if (GOMP_MAP_COPY_TO_P (map_kind)
13623 : || map_kind == GOMP_MAP_POINTER
13624 475 : || map_kind == GOMP_MAP_TO_PSET
13625 2 : || map_kind == GOMP_MAP_FORCE_DEVICEPTR)
13626 : {
13627 : /* If we need to initialize a temporary
13628 : with VAR because it is not addressable, and
13629 : the variable hasn't been initialized yet, then
13630 : we'll get a warning for the store to avar.
13631 : Don't warn in that case, the mapping might
13632 : be implicit. */
13633 5281 : suppress_warning (var, OPT_Wuninitialized);
13634 5281 : gimplify_assign (avar, var, &ilist);
13635 : }
13636 5283 : avar = build_fold_addr_expr (avar);
13637 5283 : gimplify_assign (x, avar, &ilist);
13638 5283 : if ((GOMP_MAP_COPY_FROM_P (map_kind)
13639 1486 : || map_kind == GOMP_MAP_FORCE_DEVICEPTR)
13640 9080 : && !TYPE_READONLY (TREE_TYPE (var)))
13641 : {
13642 3797 : x = unshare_expr (x);
13643 3797 : x = build_simple_mem_ref (x);
13644 3797 : gimplify_assign (var, x, &olist);
13645 : }
13646 : }
13647 : else
13648 : {
13649 : /* While MAP is handled explicitly by the FE,
13650 : for 'target update', only the identified is passed. */
13651 33409 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FROM
13652 28285 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TO)
13653 34353 : && (omp_is_allocatable_or_ptr (var)
13654 120 : && omp_check_optional_argument (var, false)))
13655 0 : var = build_fold_indirect_ref (var);
13656 33409 : else if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FROM
13657 28285 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_TO)
13658 34353 : || (!omp_is_allocatable_or_ptr (var)
13659 5948 : && !omp_check_optional_argument (var, false)))
13660 33289 : var = build_fold_addr_expr (var);
13661 33409 : gimplify_assign (x, var, &ilist);
13662 : }
13663 : }
13664 80132 : s = NULL_TREE;
13665 80132 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
13666 : {
13667 3896 : gcc_checking_assert (is_gimple_omp_oacc (ctx->stmt));
13668 3896 : s = TREE_TYPE (ovar);
13669 3896 : if (TREE_CODE (s) == REFERENCE_TYPE
13670 3896 : || omp_check_optional_argument (ovar, false))
13671 449 : s = TREE_TYPE (s);
13672 3896 : s = TYPE_SIZE_UNIT (s);
13673 : }
13674 : else
13675 76236 : s = OMP_CLAUSE_SIZE (c);
13676 80132 : if (s == NULL_TREE)
13677 153 : s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
13678 80132 : s = fold_convert (size_type_node, s);
13679 80132 : s = lower_omp_map_iterator_size (s, c, stmt);
13680 80132 : purpose = size_int (map_idx++);
13681 80132 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
13682 80132 : if (TREE_CODE (s) != INTEGER_CST)
13683 16154 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
13684 :
13685 80132 : unsigned HOST_WIDE_INT tkind, tkind_zero;
13686 80132 : switch (OMP_CLAUSE_CODE (c))
13687 : {
13688 68562 : case OMP_CLAUSE_MAP:
13689 68562 : tkind = OMP_CLAUSE_MAP_KIND (c);
13690 68562 : tkind_zero = tkind;
13691 68562 : if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c))
13692 5276 : switch (tkind)
13693 : {
13694 : case GOMP_MAP_ALLOC:
13695 : case GOMP_MAP_IF_PRESENT:
13696 : case GOMP_MAP_TO:
13697 : case GOMP_MAP_FROM:
13698 : case GOMP_MAP_TOFROM:
13699 : case GOMP_MAP_ALWAYS_TO:
13700 : case GOMP_MAP_ALWAYS_FROM:
13701 : case GOMP_MAP_ALWAYS_TOFROM:
13702 : case GOMP_MAP_ALWAYS_PRESENT_TO:
13703 : case GOMP_MAP_ALWAYS_PRESENT_FROM:
13704 : case GOMP_MAP_ALWAYS_PRESENT_TOFROM:
13705 : case GOMP_MAP_RELEASE:
13706 : case GOMP_MAP_FORCE_TO:
13707 : case GOMP_MAP_FORCE_FROM:
13708 : case GOMP_MAP_FORCE_TOFROM:
13709 : case GOMP_MAP_FORCE_PRESENT:
13710 68562 : tkind_zero = GOMP_MAP_ZERO_LEN_ARRAY_SECTION;
13711 : break;
13712 21 : case GOMP_MAP_DELETE:
13713 21 : tkind_zero = GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION;
13714 : default:
13715 : break;
13716 : }
13717 68562 : if (tkind_zero != tkind)
13718 : {
13719 5276 : if (integer_zerop (s))
13720 : tkind = tkind_zero;
13721 4550 : else if (integer_nonzerop (s))
13722 68562 : tkind_zero = tkind;
13723 : }
13724 68562 : if (tkind_zero == tkind
13725 67101 : && OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (c)
13726 68562 : && (((tkind & GOMP_MAP_FLAG_SPECIAL_BITS)
13727 5232 : & ~GOMP_MAP_IMPLICIT)
13728 : == 0))
13729 : {
13730 : /* If this is an implicit map, and the GOMP_MAP_IMPLICIT
13731 : bits are not interfered by other special bit encodings,
13732 : then turn the GOMP_IMPLICIT_BIT flag on for the runtime
13733 : to see. */
13734 5189 : tkind |= GOMP_MAP_IMPLICIT;
13735 5189 : tkind_zero = tkind;
13736 : }
13737 : break;
13738 3896 : case OMP_CLAUSE_FIRSTPRIVATE:
13739 3896 : gcc_checking_assert (is_gimple_omp_oacc (ctx->stmt));
13740 : tkind = GOMP_MAP_TO;
13741 : tkind_zero = tkind;
13742 : break;
13743 1651 : case OMP_CLAUSE_TO:
13744 1651 : tkind
13745 1651 : = (OMP_CLAUSE_MOTION_PRESENT (c)
13746 1651 : ? GOMP_MAP_ALWAYS_PRESENT_TO : GOMP_MAP_TO);
13747 : tkind_zero = tkind;
13748 : break;
13749 6023 : case OMP_CLAUSE_FROM:
13750 6023 : tkind
13751 6023 : = (OMP_CLAUSE_MOTION_PRESENT (c)
13752 6023 : ? GOMP_MAP_ALWAYS_PRESENT_FROM : GOMP_MAP_FROM);
13753 : tkind_zero = tkind;
13754 : break;
13755 0 : default:
13756 0 : gcc_unreachable ();
13757 : }
13758 68562 : gcc_checking_assert (tkind
13759 : < (HOST_WIDE_INT_C (1U) << talign_shift));
13760 80132 : gcc_checking_assert (tkind_zero
13761 : < (HOST_WIDE_INT_C (1U) << talign_shift));
13762 80132 : talign = ceil_log2 (talign);
13763 80132 : tkind |= talign << talign_shift;
13764 80132 : tkind_zero |= talign << talign_shift;
13765 80132 : gcc_checking_assert (tkind
13766 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
13767 80132 : gcc_checking_assert (tkind_zero
13768 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
13769 80132 : if (tkind == tkind_zero)
13770 78671 : x = build_int_cstu (tkind_type, tkind);
13771 : else
13772 : {
13773 1461 : TREE_STATIC (TREE_VEC_ELT (t, 2)) = 0;
13774 1461 : x = build3 (COND_EXPR, tkind_type,
13775 : fold_build2 (EQ_EXPR, boolean_type_node,
13776 : unshare_expr (s), size_zero_node),
13777 1461 : build_int_cstu (tkind_type, tkind_zero),
13778 1461 : build_int_cstu (tkind_type, tkind));
13779 : }
13780 80132 : CONSTRUCTOR_APPEND_ELT (vkind, purpose, x);
13781 80132 : if (nc && nc != c)
13782 0 : c = nc;
13783 : break;
13784 :
13785 15294 : case OMP_CLAUSE_FIRSTPRIVATE:
13786 15294 : omp_has_device_addr_descr:
13787 15294 : if (is_gimple_omp_oacc (ctx->stmt))
13788 3896 : goto oacc_firstprivate_map;
13789 11398 : ovar = OMP_CLAUSE_DECL (c);
13790 11398 : if (omp_privatize_by_reference (ovar))
13791 822 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
13792 : else
13793 10576 : talign = DECL_ALIGN_UNIT (ovar);
13794 11398 : var = lookup_decl_in_outer_ctx (ovar, ctx);
13795 11398 : x = build_sender_ref (ovar, ctx);
13796 11398 : tkind = GOMP_MAP_FIRSTPRIVATE;
13797 11398 : type = TREE_TYPE (ovar);
13798 11398 : if (omp_privatize_by_reference (ovar))
13799 822 : type = TREE_TYPE (type);
13800 11398 : if ((INTEGRAL_TYPE_P (type)
13801 10996 : && TYPE_PRECISION (type) <= POINTER_SIZE)
13802 11408 : || TREE_CODE (type) == POINTER_TYPE)
13803 : {
13804 11141 : tkind = GOMP_MAP_FIRSTPRIVATE_INT;
13805 11141 : tree t = var;
13806 11141 : if (omp_privatize_by_reference (var))
13807 767 : t = build_simple_mem_ref (var);
13808 10374 : else if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
13809 9310 : suppress_warning (var);
13810 11141 : if (TREE_CODE (type) != POINTER_TYPE)
13811 10986 : t = fold_convert (pointer_sized_int_node, t);
13812 11141 : t = fold_convert (TREE_TYPE (x), t);
13813 11141 : gimplify_assign (x, t, &ilist);
13814 : }
13815 257 : else if (omp_privatize_by_reference (var))
13816 55 : gimplify_assign (x, var, &ilist);
13817 202 : else if (is_gimple_reg (var))
13818 : {
13819 47 : tree avar = create_tmp_var (TREE_TYPE (var));
13820 47 : mark_addressable (avar);
13821 47 : if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
13822 40 : suppress_warning (var);
13823 47 : gimplify_assign (avar, var, &ilist);
13824 47 : avar = build_fold_addr_expr (avar);
13825 47 : gimplify_assign (x, avar, &ilist);
13826 : }
13827 : else
13828 : {
13829 155 : var = build_fold_addr_expr (var);
13830 155 : gimplify_assign (x, var, &ilist);
13831 : }
13832 257 : if (tkind == GOMP_MAP_FIRSTPRIVATE_INT)
13833 11141 : s = size_int (0);
13834 257 : else if (omp_privatize_by_reference (ovar))
13835 55 : s = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
13836 : else
13837 202 : s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
13838 11398 : s = fold_convert (size_type_node, s);
13839 11398 : purpose = size_int (map_idx++);
13840 11398 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
13841 11398 : if (TREE_CODE (s) != INTEGER_CST)
13842 35 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
13843 :
13844 11398 : gcc_checking_assert (tkind
13845 : < (HOST_WIDE_INT_C (1U) << talign_shift));
13846 11398 : talign = ceil_log2 (talign);
13847 11398 : tkind |= talign << talign_shift;
13848 11398 : gcc_checking_assert (tkind
13849 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
13850 11398 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
13851 : build_int_cstu (tkind_type, tkind));
13852 : /* Fortran array descriptors: firstprivate of data + attach. */
13853 11398 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
13854 11379 : && lang_hooks.decls.omp_array_data (ovar, true)
13855 11429 : && lang_hooks.decls.omp_array_data_privatize (ovar))
13856 : {
13857 23 : tree not_null_lb, null_lb, after_lb;
13858 23 : tree var1, var2, size1, size2;
13859 23 : tree present = omp_check_optional_argument (ovar, true);
13860 23 : if (present)
13861 : {
13862 1 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
13863 1 : not_null_lb = create_artificial_label (clause_loc);
13864 1 : null_lb = create_artificial_label (clause_loc);
13865 1 : after_lb = create_artificial_label (clause_loc);
13866 1 : gimple_seq seq = NULL;
13867 1 : present = force_gimple_operand (present, &seq, true,
13868 : NULL_TREE);
13869 1 : gimple_seq_add_seq (&ilist, seq);
13870 1 : gimple_seq_add_stmt (&ilist,
13871 1 : gimple_build_cond_from_tree (present,
13872 : not_null_lb, null_lb));
13873 1 : gimple_seq_add_stmt (&ilist,
13874 1 : gimple_build_label (not_null_lb));
13875 : }
13876 23 : var1 = lang_hooks.decls.omp_array_data (var, false);
13877 23 : size1 = lang_hooks.decls.omp_array_size (var, &ilist);
13878 23 : var2 = build_fold_addr_expr (x);
13879 23 : if (!POINTER_TYPE_P (TREE_TYPE (var)))
13880 0 : var = build_fold_addr_expr (var);
13881 23 : size2 = fold_build2 (POINTER_DIFF_EXPR, ssizetype,
13882 : build_fold_addr_expr (var1), var);
13883 23 : size2 = fold_convert (sizetype, size2);
13884 23 : if (present)
13885 : {
13886 1 : tree tmp = create_tmp_var (TREE_TYPE (var1));
13887 1 : gimplify_assign (tmp, var1, &ilist);
13888 1 : var1 = tmp;
13889 1 : tmp = create_tmp_var (TREE_TYPE (var2));
13890 1 : gimplify_assign (tmp, var2, &ilist);
13891 1 : var2 = tmp;
13892 1 : tmp = create_tmp_var (TREE_TYPE (size1));
13893 1 : gimplify_assign (tmp, size1, &ilist);
13894 1 : size1 = tmp;
13895 1 : tmp = create_tmp_var (TREE_TYPE (size2));
13896 1 : gimplify_assign (tmp, size2, &ilist);
13897 1 : size2 = tmp;
13898 1 : gimple_seq_add_stmt (&ilist, gimple_build_goto (after_lb));
13899 1 : gimple_seq_add_stmt (&ilist, gimple_build_label (null_lb));
13900 1 : gimplify_assign (var1, null_pointer_node, &ilist);
13901 1 : gimplify_assign (var2, null_pointer_node, &ilist);
13902 1 : gimplify_assign (size1, size_zero_node, &ilist);
13903 1 : gimplify_assign (size2, size_zero_node, &ilist);
13904 1 : gimple_seq_add_stmt (&ilist, gimple_build_label (after_lb));
13905 : }
13906 23 : x = build_sender_ref ((splay_tree_key) &DECL_NAME (ovar), ctx);
13907 23 : gimplify_assign (x, var1, &ilist);
13908 23 : tkind = GOMP_MAP_FIRSTPRIVATE;
13909 23 : talign = DECL_ALIGN_UNIT (ovar);
13910 23 : talign = ceil_log2 (talign);
13911 23 : tkind |= talign << talign_shift;
13912 23 : gcc_checking_assert (tkind
13913 : <= tree_to_uhwi (
13914 : TYPE_MAX_VALUE (tkind_type)));
13915 23 : purpose = size_int (map_idx++);
13916 23 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, size1);
13917 23 : if (TREE_CODE (size1) != INTEGER_CST)
13918 23 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
13919 23 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
13920 : build_int_cstu (tkind_type, tkind));
13921 23 : x = build_sender_ref ((splay_tree_key) &DECL_UID (ovar), ctx);
13922 23 : gimplify_assign (x, var2, &ilist);
13923 23 : tkind = GOMP_MAP_ATTACH;
13924 23 : purpose = size_int (map_idx++);
13925 23 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, size2);
13926 23 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
13927 : build_int_cstu (tkind_type, tkind));
13928 : }
13929 : break;
13930 :
13931 2430 : case OMP_CLAUSE_USE_DEVICE_PTR:
13932 2430 : case OMP_CLAUSE_USE_DEVICE_ADDR:
13933 2430 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
13934 2430 : case OMP_CLAUSE_IS_DEVICE_PTR:
13935 2430 : ovar = OMP_CLAUSE_DECL (c);
13936 2430 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
13937 : {
13938 235 : if (lang_hooks.decls.omp_array_data (ovar, true))
13939 19 : goto omp_has_device_addr_descr;
13940 233 : while (TREE_CODE (ovar) == INDIRECT_REF
13941 233 : || TREE_CODE (ovar) == ARRAY_REF)
13942 17 : ovar = TREE_OPERAND (ovar, 0);
13943 : }
13944 2411 : var = lookup_decl_in_outer_ctx (ovar, ctx);
13945 :
13946 2411 : if (lang_hooks.decls.omp_array_data (ovar, true))
13947 : {
13948 601 : tkind = ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
13949 601 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
13950 601 : ? GOMP_MAP_USE_DEVICE_PTR : GOMP_MAP_FIRSTPRIVATE_INT);
13951 601 : x = build_sender_ref ((splay_tree_key) &DECL_NAME (ovar), ctx);
13952 : }
13953 1810 : else if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
13954 1810 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
13955 : {
13956 1453 : tkind = GOMP_MAP_USE_DEVICE_PTR;
13957 1453 : x = build_sender_ref ((splay_tree_key) &DECL_UID (ovar), ctx);
13958 : }
13959 : else
13960 : {
13961 357 : tkind = GOMP_MAP_FIRSTPRIVATE_INT;
13962 357 : x = build_sender_ref (ovar, ctx);
13963 : }
13964 :
13965 2411 : if (is_gimple_omp_oacc (ctx->stmt))
13966 : {
13967 147 : gcc_assert (tkind == GOMP_MAP_USE_DEVICE_PTR);
13968 :
13969 147 : if (OMP_CLAUSE_USE_DEVICE_PTR_IF_PRESENT (c))
13970 57 : tkind = GOMP_MAP_USE_DEVICE_PTR_IF_PRESENT;
13971 : }
13972 :
13973 2411 : type = TREE_TYPE (ovar);
13974 2411 : if (lang_hooks.decls.omp_array_data (ovar, true))
13975 601 : var = lang_hooks.decls.omp_array_data (var, false);
13976 1810 : else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
13977 542 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
13978 1484 : && !omp_privatize_by_reference (ovar)
13979 512 : && !omp_is_allocatable_or_ptr (ovar))
13980 3247 : || TREE_CODE (type) == ARRAY_TYPE)
13981 401 : var = build_fold_addr_expr (var);
13982 : else
13983 : {
13984 1409 : if (omp_privatize_by_reference (ovar)
13985 370 : || omp_check_optional_argument (ovar, false)
13986 1779 : || omp_is_allocatable_or_ptr (ovar))
13987 : {
13988 1212 : type = TREE_TYPE (type);
13989 1212 : if (POINTER_TYPE_P (type)
13990 1212 : && TREE_CODE (type) != ARRAY_TYPE
13991 1571 : && ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_USE_DEVICE_ADDR
13992 36 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
13993 28 : && !omp_is_allocatable_or_ptr (ovar))
13994 337 : || (omp_privatize_by_reference (ovar)
13995 337 : && omp_is_allocatable_or_ptr (ovar))))
13996 357 : var = build_simple_mem_ref (var);
13997 1212 : var = fold_convert (TREE_TYPE (x), var);
13998 : }
13999 : }
14000 2411 : tree present;
14001 2411 : present = omp_check_optional_argument (ovar, true);
14002 2411 : if (present)
14003 : {
14004 878 : tree null_label = create_artificial_label (UNKNOWN_LOCATION);
14005 878 : tree notnull_label = create_artificial_label (UNKNOWN_LOCATION);
14006 878 : tree opt_arg_label = create_artificial_label (UNKNOWN_LOCATION);
14007 878 : tree new_x = unshare_expr (x);
14008 878 : gimplify_expr (&present, &ilist, NULL, is_gimple_val,
14009 : fb_rvalue);
14010 878 : gcond *cond = gimple_build_cond_from_tree (present,
14011 : notnull_label,
14012 : null_label);
14013 878 : gimple_seq_add_stmt (&ilist, cond);
14014 878 : gimple_seq_add_stmt (&ilist, gimple_build_label (null_label));
14015 878 : gimplify_assign (new_x, null_pointer_node, &ilist);
14016 878 : gimple_seq_add_stmt (&ilist, gimple_build_goto (opt_arg_label));
14017 878 : gimple_seq_add_stmt (&ilist,
14018 878 : gimple_build_label (notnull_label));
14019 878 : gimplify_assign (x, var, &ilist);
14020 878 : gimple_seq_add_stmt (&ilist,
14021 878 : gimple_build_label (opt_arg_label));
14022 : }
14023 : else
14024 1533 : gimplify_assign (x, var, &ilist);
14025 2411 : s = size_int (0);
14026 2411 : purpose = size_int (map_idx++);
14027 2411 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
14028 2411 : gcc_checking_assert (tkind
14029 : < (HOST_WIDE_INT_C (1U) << talign_shift));
14030 2411 : gcc_checking_assert (tkind
14031 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
14032 2411 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
14033 : build_int_cstu (tkind_type, tkind));
14034 2411 : break;
14035 :
14036 68 : case OMP_CLAUSE_USES_ALLOCATORS:
14037 68 : tree allocator = OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c);
14038 68 : tree memspace = OMP_CLAUSE_USES_ALLOCATORS_MEMSPACE (c);
14039 68 : tree traits = OMP_CLAUSE_USES_ALLOCATORS_TRAITS (c);
14040 :
14041 68 : tree ntraits, traits_var;
14042 68 : if (traits == NULL_TREE)
14043 : {
14044 27 : ntraits = integer_zero_node;
14045 27 : traits_var = null_pointer_node;
14046 : }
14047 41 : else if (DECL_INITIAL (traits))
14048 : {
14049 29 : location_t loc = OMP_CLAUSE_LOCATION (c);
14050 29 : ntraits = array_type_nelts_top (TREE_TYPE (traits));
14051 29 : tree t = DECL_INITIAL (traits);
14052 29 : t = get_initialized_tmp_var (t, &ilist, NULL);
14053 29 : traits_var = build_fold_addr_expr_loc (loc, t);
14054 : }
14055 : else
14056 : {
14057 : /* This happens for VLAs, which probably aren't useful
14058 : because they can't be const initialized in the same
14059 : scope.... is there something else? */
14060 12 : location_t loc = OMP_CLAUSE_LOCATION (c);
14061 12 : gcc_assert (TREE_CODE (TREE_TYPE (traits)) == ARRAY_TYPE);
14062 12 : ntraits = array_type_nelts_top (TREE_TYPE (traits));
14063 12 : traits_var = build_fold_addr_expr_loc (loc, traits);
14064 : }
14065 :
14066 68 : if (memspace == NULL_TREE)
14067 55 : memspace = build_int_cst (pointer_sized_int_node, 0);
14068 : else
14069 13 : memspace = fold_convert (pointer_sized_int_node, memspace);
14070 :
14071 68 : tree arr_type
14072 68 : = build_array_type_nelts (pointer_sized_int_node, 3);
14073 68 : tree uses_allocators_descr
14074 68 : = create_tmp_var (arr_type, "uses_allocator_descr");
14075 68 : tree ua_descr[3];
14076 272 : for (int i = 0; i < 3; i++)
14077 204 : ua_descr[i] = build4 (ARRAY_REF, pointer_sized_int_node,
14078 : uses_allocators_descr,
14079 204 : build_int_cst (size_type_node, i),
14080 : NULL_TREE, NULL_TREE);
14081 68 : gimplify_assign (ua_descr[0],
14082 : fold_convert (pointer_sized_int_node, memspace),
14083 : &ilist);
14084 68 : gimplify_assign (ua_descr[1],
14085 : fold_convert (pointer_sized_int_node, ntraits),
14086 : &ilist);
14087 68 : gimplify_assign (ua_descr[2],
14088 : fold_convert (pointer_sized_int_node, traits_var),
14089 : &ilist);
14090 :
14091 68 : x = build_sender_ref (allocator, ctx);
14092 68 : tree ptr = build_fold_addr_expr (uses_allocators_descr);
14093 68 : gimplify_assign (x, fold_convert (TREE_TYPE (x), ptr), &ilist);
14094 :
14095 68 : s = size_int (0);
14096 68 : purpose = size_int (map_idx++);
14097 68 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
14098 68 : tkind = GOMP_MAP_USES_ALLOCATORS;
14099 68 : gcc_checking_assert (tkind
14100 : < (HOST_WIDE_INT_C (1U) << talign_shift));
14101 68 : talign = TYPE_ALIGN_UNIT (pointer_sized_int_node);
14102 68 : talign = ceil_log2 (talign);
14103 68 : tkind |= talign << talign_shift;
14104 68 : gcc_checking_assert (tkind
14105 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
14106 68 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
14107 : build_int_cstu (tkind_type, tkind));
14108 68 : break;
14109 : }
14110 :
14111 32260 : gcc_assert (map_idx == map_cnt);
14112 :
14113 32260 : if (!deep_map_cnt)
14114 : {
14115 32116 : DECL_INITIAL (TREE_VEC_ELT (t, 1))
14116 32116 : = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), vsize);
14117 32116 : DECL_INITIAL (TREE_VEC_ELT (t, 2))
14118 64232 : = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 2)), vkind);
14119 : }
14120 96780 : for (int i = 1; i <= 2; i++)
14121 64520 : if (deep_map_cnt || !TREE_STATIC (TREE_VEC_ELT (t, i)))
14122 : {
14123 9657 : tree tmp = TREE_VEC_ELT (t, i);
14124 9657 : if (deep_map_cnt)
14125 : {
14126 288 : const char *prefix = (i == 1 ? ".omp_data_sizes0"
14127 : : ".omp_data_kinds0");
14128 144 : tree type = (i == 1) ? size_type_node : tkind_type;
14129 288 : type = build_array_type_nelts (type, map_cnt);
14130 288 : tree var = create_tmp_var (type, prefix);
14131 288 : DECL_NAMELESS (var) = 1;
14132 288 : TREE_ADDRESSABLE (var) = 1;
14133 288 : TREE_STATIC (var) = TREE_STATIC (tmp);
14134 288 : DECL_INITIAL (var) = build_constructor (type, i == 1
14135 : ? vsize : vkind);
14136 288 : tmp = var;
14137 288 : TREE_STATIC (TREE_VEC_ELT (t, i)) = 0;
14138 : }
14139 :
14140 9657 : gimple_seq initlist = NULL;
14141 9657 : force_gimple_operand (build1 (DECL_EXPR, void_type_node, tmp),
14142 : &initlist, true, NULL_TREE);
14143 9657 : gimple_seq_add_seq (&ilist, initlist);
14144 :
14145 9657 : if (deep_map_cnt)
14146 : {
14147 288 : tree tmp2;
14148 288 : tree call = builtin_decl_explicit (BUILT_IN_MEMCPY);
14149 288 : tmp2 = TYPE_SIZE_UNIT (TREE_TYPE (tmp));
14150 576 : call = build_call_expr_loc (input_location, call, 3,
14151 288 : TREE_VEC_ELT (t, i),
14152 : build_fold_addr_expr (tmp), tmp2);
14153 288 : gimplify_and_add (call, &ilist);
14154 : }
14155 :
14156 9657 : if (!TREE_STATIC (tmp))
14157 : {
14158 9439 : tree clobber = build_clobber (TREE_TYPE (tmp));
14159 9439 : gimple_seq_add_stmt (&olist,
14160 9439 : gimple_build_assign (tmp, clobber));
14161 : }
14162 9657 : if (deep_map_cnt)
14163 : {
14164 288 : tmp = TREE_VEC_ELT (t, i);
14165 288 : tree call = builtin_decl_explicit (BUILT_IN_FREE);
14166 288 : call = build_call_expr_loc (input_location, call, 1, tmp);
14167 288 : gimplify_and_add (call, &olist);
14168 288 : tree clobber = build_clobber (TREE_TYPE (tmp));
14169 288 : gimple_seq_add_stmt (&olist,
14170 288 : gimple_build_assign (tmp, clobber));
14171 : }
14172 : }
14173 54863 : else if (omp_maybe_offloaded_ctx (ctx->outer))
14174 : {
14175 3296 : tree id = get_identifier ("omp declare target");
14176 3296 : tree decl = TREE_VEC_ELT (t, i);
14177 3296 : DECL_ATTRIBUTES (decl)
14178 3296 : = tree_cons (id, NULL_TREE, DECL_ATTRIBUTES (decl));
14179 3296 : varpool_node *node = varpool_node::get (decl);
14180 3296 : if (node)
14181 : {
14182 0 : node->offloadable = 1;
14183 0 : if (ENABLE_OFFLOADING)
14184 : {
14185 : g->have_offload = true;
14186 : vec_safe_push (offload_vars, t);
14187 : }
14188 : }
14189 : }
14190 :
14191 32260 : if (deep_map_cnt)
14192 : {
14193 144 : tree call = builtin_decl_explicit (BUILT_IN_FREE);
14194 144 : call = build_call_expr_loc (input_location, call, 1,
14195 144 : TREE_VEC_ELT (t, 0));
14196 144 : gimplify_and_add (call, &olist);
14197 :
14198 144 : gimplify_expr (&TREE_VEC_ELT (t, 1), &ilist, NULL, is_gimple_val,
14199 : fb_rvalue);
14200 : }
14201 :
14202 32260 : tree clobber = build_clobber (TREE_TYPE (ctx->sender_decl));
14203 32260 : gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
14204 : clobber));
14205 : }
14206 :
14207 : /* Once all the expansions are done, sequence all the different
14208 : fragments inside gimple_omp_body. */
14209 :
14210 37147 : new_body = NULL;
14211 :
14212 37147 : if (offloaded
14213 21187 : && ctx->record_type)
14214 : {
14215 16537 : t = ctx->sender_decl;
14216 16537 : if (!deep_map_cnt)
14217 16430 : t = build_fold_addr_expr_loc (loc, t);
14218 : /* fixup_child_record_type might have changed receiver_decl's type. */
14219 16537 : t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
14220 16537 : if (!AGGREGATE_TYPE_P (TREE_TYPE (ctx->sender_decl)))
14221 107 : gimplify_assign (ctx->receiver_decl, t, &new_body);
14222 : else
14223 16430 : gimple_seq_add_stmt (&new_body,
14224 16430 : gimple_build_assign (ctx->receiver_decl, t));
14225 : }
14226 37147 : gimple_seq_add_seq (&new_body, fplist);
14227 :
14228 37147 : if (offloaded || data_region)
14229 : {
14230 141038 : tree prev = NULL_TREE;
14231 : bool by_ref;
14232 141038 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
14233 115830 : switch (OMP_CLAUSE_CODE (c))
14234 : {
14235 : tree var, x, new_var, *allocate_ptr;
14236 : default:
14237 : break;
14238 15294 : case OMP_CLAUSE_FIRSTPRIVATE:
14239 15294 : omp_firstprivatize_data_region:
14240 15294 : if (is_gimple_omp_oacc (ctx->stmt))
14241 : break;
14242 11398 : var = OMP_CLAUSE_DECL (c);
14243 11398 : new_var = lookup_decl (var, ctx);
14244 11398 : allocate_ptr = alloc_map.get (new_var);
14245 11398 : by_ref = omp_privatize_by_reference (var);
14246 11398 : if (allocate_ptr)
14247 : {
14248 63 : if (is_variable_sized (var, by_ref))
14249 : /* Handle this in the next pass when the size is
14250 : available. */
14251 : break;
14252 :
14253 48 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14254 48 : gcc_assert (allocate_seq);
14255 48 : gimple_seq_add_seq (&new_body, *allocate_seq);
14256 :
14257 48 : if (by_ref)
14258 : {
14259 4 : x = fold_convert (TREE_TYPE (new_var), *allocate_ptr);
14260 4 : gimplify_assign (new_var, x, &new_body);
14261 4 : new_var = build_fold_indirect_ref (new_var);
14262 : }
14263 : else
14264 44 : new_var = build_simple_mem_ref (*allocate_ptr);
14265 : }
14266 11383 : if (by_ref || is_gimple_reg_type (TREE_TYPE (var)))
14267 : {
14268 11242 : tree type;
14269 11242 : type = TREE_TYPE (var);
14270 11242 : if (by_ref)
14271 809 : type = TREE_TYPE (type);
14272 11242 : if ((INTEGRAL_TYPE_P (type)
14273 10996 : && TYPE_PRECISION (type) <= POINTER_SIZE)
14274 11252 : || TREE_CODE (type) == POINTER_TYPE)
14275 : {
14276 11141 : x = build_receiver_ref (var, false, ctx);
14277 11141 : if (TREE_CODE (type) != POINTER_TYPE)
14278 10986 : x = fold_convert (pointer_sized_int_node, x);
14279 11141 : x = fold_convert (type, x);
14280 11141 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14281 : fb_rvalue);
14282 11141 : if (by_ref && !allocate_ptr)
14283 : {
14284 766 : tree v = create_tmp_var_raw (type, get_name (var));
14285 766 : gimple_add_tmp_var (v);
14286 766 : TREE_ADDRESSABLE (v) = 1;
14287 766 : gimple_seq_add_stmt (&new_body,
14288 766 : gimple_build_assign (v, x));
14289 766 : x = build_fold_addr_expr (v);
14290 : }
14291 11141 : gimplify_assign (new_var, x, &new_body);
14292 : }
14293 : else
14294 : {
14295 101 : x = build_receiver_ref (var, allocate_ptr || !by_ref, ctx);
14296 101 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14297 : fb_rvalue);
14298 101 : gimplify_assign (new_var, x, &new_body);
14299 : }
14300 : }
14301 141 : else if (is_variable_sized (var))
14302 : {
14303 4 : tree pvar = DECL_VALUE_EXPR (var);
14304 4 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14305 4 : pvar = TREE_OPERAND (pvar, 0);
14306 4 : gcc_assert (DECL_P (pvar));
14307 4 : tree new_var = lookup_decl (pvar, ctx);
14308 4 : x = build_receiver_ref (var, false, ctx);
14309 4 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14310 4 : gimple_seq_add_stmt (&new_body,
14311 4 : gimple_build_assign (new_var, x));
14312 : }
14313 137 : else if (allocate_ptr)
14314 : {
14315 19 : x = build_receiver_ref (var, true, ctx);
14316 19 : new_var = unshare_expr (new_var);
14317 19 : x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
14318 19 : gimplify_and_add (x, &new_body);
14319 : }
14320 : break;
14321 226 : case OMP_CLAUSE_PRIVATE:
14322 226 : if (is_gimple_omp_oacc (ctx->stmt))
14323 : break;
14324 146 : var = OMP_CLAUSE_DECL (c);
14325 146 : new_var = lookup_decl (var, ctx);
14326 146 : allocate_ptr = alloc_map.get (new_var);
14327 146 : by_ref = omp_privatize_by_reference (var);
14328 146 : if (allocate_ptr)
14329 : {
14330 55 : if (is_variable_sized (var, by_ref))
14331 : /* Handle this in the next pass when the size is
14332 : available. */
14333 : break;
14334 :
14335 43 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14336 43 : gcc_assert (allocate_seq);
14337 43 : gimple_seq_add_seq (&new_body, *allocate_seq);
14338 :
14339 43 : if (!by_ref)
14340 42 : new_var = build_simple_mem_ref (*allocate_ptr);
14341 : }
14342 133 : if (by_ref)
14343 : {
14344 12 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14345 12 : if (!allocate_ptr)
14346 : {
14347 11 : x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
14348 11 : if (TREE_CONSTANT (x))
14349 : {
14350 3 : x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
14351 : get_name (var));
14352 3 : gimple_add_tmp_var (x);
14353 3 : TREE_ADDRESSABLE (x) = 1;
14354 3 : x = build_fold_addr_expr_loc (clause_loc, x);
14355 : }
14356 : else
14357 : break;
14358 :
14359 3 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
14360 3 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14361 : fb_rvalue);
14362 : }
14363 : else
14364 1 : x = *allocate_ptr;
14365 :
14366 4 : gimple_seq_add_stmt (&new_body,
14367 4 : gimple_build_assign (new_var, x));
14368 : }
14369 : break;
14370 2430 : case OMP_CLAUSE_USE_DEVICE_PTR:
14371 2430 : case OMP_CLAUSE_USE_DEVICE_ADDR:
14372 2430 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
14373 2430 : case OMP_CLAUSE_IS_DEVICE_PTR:
14374 2430 : gimple_seq assign_body;
14375 2430 : bool is_array_data;
14376 2430 : bool do_optional_check;
14377 2430 : assign_body = NULL;
14378 2430 : do_optional_check = false;
14379 2430 : var = OMP_CLAUSE_DECL (c);
14380 2430 : is_array_data = lang_hooks.decls.omp_array_data (var, true) != NULL;
14381 2430 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR && is_array_data)
14382 19 : goto omp_firstprivatize_data_region;
14383 :
14384 2411 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
14385 2411 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
14386 4108 : x = build_sender_ref (is_array_data
14387 601 : ? (splay_tree_key) &DECL_NAME (var)
14388 1453 : : (splay_tree_key) &DECL_UID (var), ctx);
14389 : else
14390 : {
14391 357 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
14392 : {
14393 233 : while (TREE_CODE (var) == INDIRECT_REF
14394 233 : || TREE_CODE (var) == ARRAY_REF)
14395 17 : var = TREE_OPERAND (var, 0);
14396 : }
14397 357 : x = build_receiver_ref (var, false, ctx);
14398 : }
14399 :
14400 2411 : if (is_array_data)
14401 : {
14402 601 : bool is_ref = omp_privatize_by_reference (var);
14403 601 : do_optional_check = true;
14404 : /* First, we copy the descriptor data from the host; then
14405 : we update its data to point to the target address. */
14406 601 : new_var = lookup_decl (var, ctx);
14407 601 : new_var = DECL_VALUE_EXPR (new_var);
14408 601 : tree v = new_var;
14409 601 : tree v2 = var;
14410 601 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_PTR
14411 601 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR)
14412 601 : v2 = maybe_lookup_decl_in_outer_ctx (var, ctx);
14413 :
14414 601 : if (is_ref)
14415 : {
14416 396 : v2 = build_fold_indirect_ref (v2);
14417 396 : v = create_tmp_var_raw (TREE_TYPE (v2), get_name (var));
14418 396 : gimple_add_tmp_var (v);
14419 396 : TREE_ADDRESSABLE (v) = 1;
14420 396 : gimplify_assign (v, v2, &assign_body);
14421 396 : tree rhs = build_fold_addr_expr (v);
14422 396 : gimple_seq_add_stmt (&assign_body,
14423 396 : gimple_build_assign (new_var, rhs));
14424 : }
14425 : else
14426 205 : gimplify_assign (new_var, v2, &assign_body);
14427 :
14428 601 : v2 = lang_hooks.decls.omp_array_data (unshare_expr (v), false);
14429 601 : gcc_assert (v2);
14430 601 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14431 1202 : gimple_seq_add_stmt (&assign_body,
14432 601 : gimple_build_assign (v2, x));
14433 : }
14434 1810 : else if (is_variable_sized (var))
14435 : {
14436 12 : tree pvar = DECL_VALUE_EXPR (var);
14437 12 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14438 12 : pvar = TREE_OPERAND (pvar, 0);
14439 12 : gcc_assert (DECL_P (pvar));
14440 12 : new_var = lookup_decl (pvar, ctx);
14441 12 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14442 24 : gimple_seq_add_stmt (&assign_body,
14443 12 : gimple_build_assign (new_var, x));
14444 : }
14445 1798 : else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
14446 536 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
14447 1476 : && !omp_privatize_by_reference (var)
14448 504 : && !omp_is_allocatable_or_ptr (var))
14449 3231 : || TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
14450 : {
14451 389 : new_var = lookup_decl (var, ctx);
14452 389 : new_var = DECL_VALUE_EXPR (new_var);
14453 389 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
14454 389 : new_var = TREE_OPERAND (new_var, 0);
14455 389 : gcc_assert (DECL_P (new_var));
14456 389 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14457 778 : gimple_seq_add_stmt (&assign_body,
14458 389 : gimple_build_assign (new_var, x));
14459 : }
14460 : else
14461 : {
14462 1409 : tree type = TREE_TYPE (var);
14463 1409 : new_var = lookup_decl (var, ctx);
14464 1409 : if (omp_privatize_by_reference (var))
14465 : {
14466 1039 : type = TREE_TYPE (type);
14467 1039 : if (POINTER_TYPE_P (type)
14468 1039 : && TREE_CODE (type) != ARRAY_TYPE
14469 1398 : && ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_USE_DEVICE_ADDR
14470 36 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
14471 331 : || (omp_privatize_by_reference (var)
14472 331 : && omp_is_allocatable_or_ptr (var))))
14473 : {
14474 357 : tree v = create_tmp_var_raw (type, get_name (var));
14475 357 : gimple_add_tmp_var (v);
14476 357 : TREE_ADDRESSABLE (v) = 1;
14477 357 : x = fold_convert (type, x);
14478 357 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val,
14479 : fb_rvalue);
14480 357 : gimple_seq_add_stmt (&assign_body,
14481 357 : gimple_build_assign (v, x));
14482 357 : x = build_fold_addr_expr (v);
14483 357 : do_optional_check = true;
14484 : }
14485 : }
14486 1409 : new_var = DECL_VALUE_EXPR (new_var);
14487 1409 : x = fold_convert (TREE_TYPE (new_var), x);
14488 1409 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14489 1409 : gimple_seq_add_stmt (&assign_body,
14490 1409 : gimple_build_assign (new_var, x));
14491 : }
14492 2411 : tree present;
14493 1002 : present = ((do_optional_check
14494 958 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
14495 952 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR)
14496 2342 : ? omp_check_optional_argument (OMP_CLAUSE_DECL (c), true)
14497 : : NULL_TREE);
14498 2411 : if (present)
14499 : {
14500 480 : tree null_label = create_artificial_label (UNKNOWN_LOCATION);
14501 480 : tree notnull_label = create_artificial_label (UNKNOWN_LOCATION);
14502 480 : tree opt_arg_label = create_artificial_label (UNKNOWN_LOCATION);
14503 480 : glabel *null_glabel = gimple_build_label (null_label);
14504 480 : glabel *notnull_glabel = gimple_build_label (notnull_label);
14505 480 : ggoto *opt_arg_ggoto = gimple_build_goto (opt_arg_label);
14506 480 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14507 : fb_rvalue);
14508 480 : gimplify_expr (&present, &new_body, NULL, is_gimple_val,
14509 : fb_rvalue);
14510 480 : gcond *cond = gimple_build_cond_from_tree (present,
14511 : notnull_label,
14512 : null_label);
14513 480 : gimple_seq_add_stmt (&new_body, cond);
14514 480 : gimple_seq_add_stmt (&new_body, null_glabel);
14515 480 : gimplify_assign (new_var, null_pointer_node, &new_body);
14516 480 : gimple_seq_add_stmt (&new_body, opt_arg_ggoto);
14517 480 : gimple_seq_add_stmt (&new_body, notnull_glabel);
14518 480 : gimple_seq_add_seq (&new_body, assign_body);
14519 480 : gimple_seq_add_stmt (&new_body,
14520 480 : gimple_build_label (opt_arg_label));
14521 : }
14522 : else
14523 1931 : gimple_seq_add_seq (&new_body, assign_body);
14524 : break;
14525 : }
14526 : /* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass,
14527 : so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
14528 : are already handled. Similarly OMP_CLAUSE_PRIVATE for VLAs
14529 : or references to VLAs. */
14530 141038 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
14531 115830 : switch (OMP_CLAUSE_CODE (c))
14532 : {
14533 : tree var, new_var, *allocate_ptr;
14534 : default:
14535 100575 : break;
14536 63520 : case OMP_CLAUSE_MAP:
14537 63520 : if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
14538 63520 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
14539 : {
14540 3664 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14541 3664 : poly_int64 offset = 0;
14542 3664 : gcc_assert (prev);
14543 3664 : var = OMP_CLAUSE_DECL (c);
14544 3664 : if (DECL_P (var)
14545 3664 : && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
14546 1125 : && is_global_var (maybe_lookup_decl_in_outer_ctx (var,
14547 : ctx))
14548 3762 : && varpool_node::get_create (var)->offloadable)
14549 : break;
14550 3662 : if (TREE_CODE (var) == INDIRECT_REF
14551 3662 : && TREE_CODE (TREE_OPERAND (var, 0)) == COMPONENT_REF)
14552 0 : var = TREE_OPERAND (var, 0);
14553 3662 : if (TREE_CODE (var) == COMPONENT_REF)
14554 : {
14555 0 : var = get_addr_base_and_unit_offset (var, &offset);
14556 0 : gcc_assert (var != NULL_TREE && DECL_P (var));
14557 : }
14558 3662 : else if (DECL_SIZE (var)
14559 3662 : && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST)
14560 : {
14561 710 : tree var2 = DECL_VALUE_EXPR (var);
14562 710 : gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
14563 710 : var2 = TREE_OPERAND (var2, 0);
14564 710 : gcc_assert (DECL_P (var2));
14565 : var = var2;
14566 : }
14567 3662 : tree new_var = lookup_decl (var, ctx), x;
14568 3662 : tree type = TREE_TYPE (new_var);
14569 3662 : bool is_ref;
14570 3662 : if (TREE_CODE (OMP_CLAUSE_DECL (c)) == INDIRECT_REF
14571 3662 : && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c), 0))
14572 : == COMPONENT_REF))
14573 : {
14574 0 : type = TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c), 0));
14575 0 : is_ref = true;
14576 0 : new_var = build2 (MEM_REF, type,
14577 : build_fold_addr_expr (new_var),
14578 : build_int_cst (build_pointer_type (type),
14579 : offset));
14580 : }
14581 3662 : else if (TREE_CODE (OMP_CLAUSE_DECL (c)) == COMPONENT_REF)
14582 : {
14583 0 : type = TREE_TYPE (OMP_CLAUSE_DECL (c));
14584 0 : is_ref = TREE_CODE (type) == REFERENCE_TYPE;
14585 0 : new_var = build2 (MEM_REF, type,
14586 : build_fold_addr_expr (new_var),
14587 : build_int_cst (build_pointer_type (type),
14588 : offset));
14589 : }
14590 : else
14591 3662 : is_ref = omp_privatize_by_reference (var);
14592 3662 : if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
14593 : is_ref = false;
14594 6695 : bool ref_to_array = false;
14595 6695 : bool ref_to_ptr = false;
14596 3233 : if (is_ref)
14597 : {
14598 200 : type = TREE_TYPE (type);
14599 200 : if (TREE_CODE (type) == ARRAY_TYPE)
14600 : {
14601 194 : type = build_pointer_type (type);
14602 194 : ref_to_array = true;
14603 : }
14604 : }
14605 3462 : else if (TREE_CODE (type) == ARRAY_TYPE)
14606 : {
14607 413 : tree decl2 = DECL_VALUE_EXPR (new_var);
14608 413 : gcc_assert (TREE_CODE (decl2) == MEM_REF);
14609 413 : decl2 = TREE_OPERAND (decl2, 0);
14610 413 : gcc_assert (DECL_P (decl2));
14611 413 : new_var = decl2;
14612 413 : type = TREE_TYPE (new_var);
14613 : }
14614 3049 : else if (TREE_CODE (type) == REFERENCE_TYPE
14615 3049 : && TREE_CODE (TREE_TYPE (type)) == POINTER_TYPE)
14616 : {
14617 156 : type = TREE_TYPE (type);
14618 156 : ref_to_ptr = true;
14619 : }
14620 3662 : x = build_receiver_ref (prev, false, ctx);
14621 3662 : x = fold_convert_loc (clause_loc, type, x);
14622 3662 : if (!integer_zerop (OMP_CLAUSE_SIZE (c)))
14623 : {
14624 762 : tree bias = OMP_CLAUSE_SIZE (c);
14625 762 : if (DECL_P (bias))
14626 364 : bias = lookup_decl (bias, ctx);
14627 762 : bias = fold_convert_loc (clause_loc, sizetype, bias);
14628 762 : bias = fold_build1_loc (clause_loc, NEGATE_EXPR, sizetype,
14629 : bias);
14630 762 : x = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
14631 762 : TREE_TYPE (x), x, bias);
14632 : }
14633 3662 : if (ref_to_array)
14634 194 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
14635 3662 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14636 3662 : if ((is_ref && !ref_to_array)
14637 3656 : || ref_to_ptr)
14638 : {
14639 162 : tree t = create_tmp_var_raw (type, get_name (var));
14640 162 : gimple_add_tmp_var (t);
14641 162 : TREE_ADDRESSABLE (t) = 1;
14642 162 : gimple_seq_add_stmt (&new_body,
14643 162 : gimple_build_assign (t, x));
14644 162 : x = build_fold_addr_expr_loc (clause_loc, t);
14645 : }
14646 3662 : gimple_seq_add_stmt (&new_body,
14647 3662 : gimple_build_assign (new_var, x));
14648 3662 : prev = NULL_TREE;
14649 : }
14650 59856 : else if (OMP_CLAUSE_CHAIN (c)
14651 43250 : && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c))
14652 : == OMP_CLAUSE_MAP
14653 97677 : && (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c))
14654 : == GOMP_MAP_FIRSTPRIVATE_POINTER
14655 34586 : || (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c))
14656 : == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
14657 : prev = c;
14658 : break;
14659 226 : case OMP_CLAUSE_PRIVATE:
14660 226 : var = OMP_CLAUSE_DECL (c);
14661 226 : by_ref = omp_privatize_by_reference (var);
14662 226 : new_var = lookup_decl (var, ctx);
14663 226 : allocate_ptr = alloc_map.get (new_var);
14664 226 : if (is_variable_sized (var, by_ref)
14665 226 : || (!allocate_ptr && by_ref && !is_gimple_omp_oacc (ctx->stmt)))
14666 : {
14667 24 : if (allocate_ptr)
14668 : {
14669 12 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14670 12 : gcc_assert (allocate_seq);
14671 12 : gimple_seq_add_seq (&new_body, *allocate_seq);
14672 : }
14673 24 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14674 24 : tree pvar = var;
14675 24 : if (!by_ref)
14676 : {
14677 3 : pvar = DECL_VALUE_EXPR (var);
14678 3 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14679 3 : pvar = TREE_OPERAND (pvar, 0);
14680 : }
14681 24 : gcc_assert (DECL_P (pvar));
14682 24 : tree new_pvar = lookup_decl (pvar, ctx);
14683 24 : tree x;
14684 24 : if (!allocate_ptr)
14685 : {
14686 12 : tree atmp = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
14687 12 : tree ty = TREE_TYPE (new_var);
14688 12 : if (by_ref)
14689 11 : ty = TREE_TYPE (ty);
14690 12 : x = TYPE_SIZE_UNIT (ty);
14691 12 : if (TREE_CONSTANT (x))
14692 : break;
14693 9 : tree al = size_int (TYPE_ALIGN (ty));
14694 9 : x = build_call_expr_loc (clause_loc, atmp, 2, x, al);
14695 9 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_pvar), x);
14696 : }
14697 : else
14698 12 : x = *allocate_ptr;
14699 21 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14700 21 : gimple_seq_add_stmt (&new_body,
14701 21 : gimple_build_assign (new_pvar, x));
14702 : }
14703 : break;
14704 15275 : case OMP_CLAUSE_FIRSTPRIVATE:
14705 15275 : var = OMP_CLAUSE_DECL (c);
14706 15275 : by_ref = omp_privatize_by_reference (var);
14707 15275 : if (is_variable_sized (var, by_ref))
14708 : {
14709 35 : tree new_var = lookup_decl (var, ctx);
14710 35 : tree *allocate_ptr = alloc_map.get (new_var);
14711 35 : if (!allocate_ptr)
14712 : break;
14713 15 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14714 15 : gcc_assert (allocate_seq);
14715 15 : gimple_seq_add_seq (&new_body, *allocate_seq);
14716 :
14717 15 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14718 15 : tree pvar = var;
14719 15 : if (!by_ref)
14720 : {
14721 2 : pvar = DECL_VALUE_EXPR (var);
14722 2 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14723 2 : pvar = TREE_OPERAND (pvar, 0);
14724 : }
14725 15 : gcc_assert (DECL_P (pvar));
14726 15 : tree new_pvar = lookup_decl (pvar, ctx);
14727 15 : tree x = fold_convert_loc (clause_loc, TREE_TYPE (new_pvar),
14728 15 : *allocate_ptr);
14729 15 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14730 15 : gimple_seq_add_stmt (&new_body,
14731 15 : gimple_build_assign (new_pvar, x));
14732 :
14733 15 : x = build_receiver_ref (var, true, ctx);
14734 15 : new_var = unshare_expr (new_var);
14735 15 : if (by_ref)
14736 13 : new_var = build_fold_indirect_ref (new_var);
14737 15 : x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
14738 15 : gimplify_and_add (x, &new_body);
14739 : }
14740 : }
14741 :
14742 25208 : gimple_seq fork_seq = NULL;
14743 25208 : gimple_seq join_seq = NULL;
14744 :
14745 25208 : if (offloaded && is_gimple_omp_oacc (ctx->stmt))
14746 : {
14747 : /* If there are reductions on the offloaded region itself, treat
14748 : them as a dummy GANG loop. */
14749 9395 : tree level = build_int_cst (integer_type_node, GOMP_DIM_GANG);
14750 :
14751 9395 : gcall *private_marker = lower_oacc_private_marker (ctx);
14752 :
14753 9395 : if (private_marker)
14754 30 : gimple_call_set_arg (private_marker, 2, level);
14755 :
14756 9395 : lower_oacc_reductions (gimple_location (ctx->stmt), clauses, level,
14757 : false, NULL, private_marker, NULL, &fork_seq,
14758 : &join_seq, ctx);
14759 : }
14760 :
14761 25208 : gimple_seq_add_seq (&new_body, fork_seq);
14762 25208 : gimple_seq_add_seq (&new_body, tgt_body);
14763 25208 : gimple_seq_add_seq (&new_body, join_seq);
14764 25208 : gimple_seq_add_seq (&new_body, alloc_dlist);
14765 :
14766 25208 : if (offloaded)
14767 : {
14768 21187 : new_body = maybe_catch_exception (new_body);
14769 21187 : gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
14770 : }
14771 25208 : gimple_omp_set_body (stmt, new_body);
14772 : }
14773 :
14774 37147 : gsi_insert_seq_before (gsi_p, gimple_omp_target_iterator_loops (stmt),
14775 : GSI_SAME_STMT);
14776 37147 : gimple_omp_target_set_iterator_loops (stmt, NULL);
14777 58334 : bind = gimple_build_bind (NULL, NULL,
14778 21187 : tgt_bind ? gimple_bind_block (tgt_bind)
14779 : : NULL_TREE);
14780 73902 : gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
14781 37147 : gimple_bind_add_seq (bind, ilist);
14782 37147 : gimple_bind_add_stmt (bind, stmt);
14783 37147 : gimple_bind_add_seq (bind, olist);
14784 :
14785 37147 : pop_gimplify_context (NULL);
14786 :
14787 37147 : if (dep_bind)
14788 : {
14789 392 : gimple_bind_add_seq (dep_bind, dep_ilist);
14790 392 : gimple_bind_add_stmt (dep_bind, bind);
14791 392 : gimple_bind_add_seq (dep_bind, dep_olist);
14792 392 : pop_gimplify_context (dep_bind);
14793 : }
14794 37147 : }
14795 :
14796 : /* Generate code to implement the action-clauses (destroy, init, use) of an
14797 : OpenMP interop construct. */
14798 :
14799 : static void
14800 492 : lower_omp_interop_action_clauses (gimple_seq *seq, vec<tree> &objs,
14801 : vec<tree> *interop_types = NULL,
14802 : vec<tree> *prefer_types = NULL)
14803 : {
14804 492 : if (objs.length () == 0)
14805 229 : return;
14806 :
14807 263 : enum omp_clause_code action = OMP_CLAUSE_CODE (objs[0]);
14808 263 : if (action == OMP_CLAUSE_INIT)
14809 402 : gcc_checking_assert (objs.length () == interop_types->length ()
14810 : && objs.length () == prefer_types->length ());
14811 : else
14812 129 : gcc_assert (prefer_types == NULL && interop_types == NULL);
14813 :
14814 263 : tree ret_objs = NULL_TREE, ret_interop_types = NULL_TREE,
14815 263 : ret_prefer_types = NULL_TREE;
14816 :
14817 : /* Build an array of interop objects. */
14818 :
14819 263 : tree type_obj_pref = build_array_type_nelts (ptr_type_node, objs.length ());
14820 263 : ret_objs = create_tmp_var (type_obj_pref, "interopobjs");
14821 :
14822 263 : bool have_pref_type = false;
14823 263 : if (action == OMP_CLAUSE_INIT)
14824 : {
14825 592 : for (tree pref_type : prefer_types)
14826 260 : if (pref_type != NULL_TREE)
14827 : {
14828 : have_pref_type = true;
14829 : break;
14830 : }
14831 134 : tree type_tgtsync
14832 134 : = build_array_type_nelts (integer_type_node, objs.length ());
14833 134 : ret_interop_types = create_tmp_var (type_tgtsync, "tgt_tgtsync");
14834 134 : if (have_pref_type)
14835 70 : ret_prefer_types = create_tmp_var (type_obj_pref, "pref_type");
14836 : else
14837 : {
14838 64 : ret_prefer_types = null_pointer_node;
14839 64 : prefer_types->truncate (0);
14840 : }
14841 : }
14842 :
14843 856 : for (size_t i = 0; !objs.is_empty (); i++)
14844 : {
14845 593 : tree offset = build_int_cst (integer_type_node, i);
14846 593 : tree init = build4 (ARRAY_REF, ptr_type_node, ret_objs, offset, NULL_TREE,
14847 : NULL_TREE);
14848 593 : tree obj = OMP_CLAUSE_DECL (objs.pop ());
14849 593 : if (TREE_CODE (TREE_TYPE (obj)) == REFERENCE_TYPE)
14850 6 : obj = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (obj)), obj);
14851 593 : if (action != OMP_CLAUSE_USE
14852 593 : && TREE_CODE (TREE_TYPE (obj)) != POINTER_TYPE)
14853 : /* For modifying actions, we need a pointer. */
14854 483 : obj = build_fold_addr_expr (obj);
14855 110 : else if (action == OMP_CLAUSE_USE
14856 110 : && TREE_CODE (TREE_TYPE (obj)) == POINTER_TYPE)
14857 : /* For use action, we need the value. */
14858 2 : obj = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (obj)), obj);
14859 593 : init = build2 (MODIFY_EXPR, ptr_type_node, init,
14860 : fold_convert (ptr_type_node, obj));
14861 593 : gimplify_and_add (init, seq);
14862 :
14863 593 : if (action == OMP_CLAUSE_INIT)
14864 : {
14865 384 : init = build4 (ARRAY_REF, integer_type_node, ret_interop_types,
14866 : offset, NULL_TREE, NULL_TREE);
14867 384 : init = build2 (MODIFY_EXPR, integer_type_node, init,
14868 384 : interop_types->pop ());
14869 384 : gimplify_and_add (init, seq);
14870 :
14871 384 : if (have_pref_type)
14872 : {
14873 194 : tree prefer_type = prefer_types->pop ();
14874 194 : tree pref = (prefer_type == NULL_TREE
14875 194 : ? null_pointer_node
14876 194 : : build_fold_addr_expr (prefer_type));
14877 194 : init = build4 (ARRAY_REF, ptr_type_node, ret_prefer_types, offset,
14878 : NULL_TREE, NULL_TREE);
14879 194 : init = build2 (MODIFY_EXPR, ptr_type_node, init, pref);
14880 194 : gimplify_and_add (init, seq);
14881 : }
14882 : }
14883 : }
14884 263 : if (action == OMP_CLAUSE_INIT)
14885 : {
14886 134 : if (have_pref_type)
14887 70 : ret_prefer_types = build_fold_addr_expr (ret_prefer_types);
14888 134 : ret_interop_types = build_fold_addr_expr (ret_interop_types);
14889 : }
14890 263 : ret_objs = build_fold_addr_expr (ret_objs);
14891 :
14892 660 : gcc_assert (objs.is_empty ()
14893 : && (!interop_types || interop_types->is_empty ())
14894 : && (!prefer_types || prefer_types->is_empty ()));
14895 :
14896 263 : objs.safe_push (ret_objs);
14897 263 : if (action == OMP_CLAUSE_INIT)
14898 : {
14899 134 : interop_types->safe_push (ret_interop_types);
14900 134 : prefer_types->safe_push (ret_prefer_types);
14901 : }
14902 : }
14903 :
14904 : /* Lower code for an OpenMP interop directive. */
14905 :
14906 : static void
14907 164 : lower_omp_interop (gimple_stmt_iterator *gsi_p, omp_context *ctx)
14908 : {
14909 164 : push_gimplify_context ();
14910 :
14911 164 : tree block = make_node (BLOCK);
14912 164 : gbind *bind = gimple_build_bind (NULL, NULL, block);
14913 164 : gimple_seq bind_body = NULL;
14914 :
14915 : /* Emit call to GOMP_interop:
14916 : void
14917 : GOMP_interop (int device_num, int n_init, omp_interop_t **init,
14918 : const void *target_targetsync, const void *prefer_type,
14919 : int n_use, omp_interop_t *use, int n_destroy,
14920 : omp_interop_t **destroy, unsigned int flags,
14921 : void **depend) */
14922 :
14923 164 : tree flags = NULL_TREE;
14924 164 : tree depend = null_pointer_node;
14925 164 : tree device_num = NULL_TREE;
14926 :
14927 164 : auto_vec<tree> init_objs, use_objs, destroy_objs, prefer_type,
14928 164 : target_targetsync;
14929 164 : gimple_seq dep_ilist = NULL, dep_olist = NULL;
14930 164 : tree clauses = gimple_omp_interop_clauses (gsi_stmt (*gsi_p));
14931 868 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
14932 : {
14933 704 : switch (OMP_CLAUSE_CODE (c))
14934 : {
14935 384 : case OMP_CLAUSE_INIT:
14936 384 : {
14937 384 : init_objs.safe_push (c);
14938 384 : int target_targetsync_bits = 0;
14939 384 : if (OMP_CLAUSE_INIT_TARGET (c))
14940 229 : target_targetsync_bits |= GOMP_INTEROP_TARGET;
14941 384 : if (OMP_CLAUSE_INIT_TARGETSYNC (c))
14942 204 : target_targetsync_bits |= GOMP_INTEROP_TARGETSYNC;
14943 384 : tree t = build_int_cst (integer_type_node, target_targetsync_bits);
14944 384 : target_targetsync.safe_push (t);
14945 384 : prefer_type.safe_push (OMP_CLAUSE_INIT_PREFER_TYPE (c));
14946 : }
14947 384 : break;
14948 106 : case OMP_CLAUSE_USE:
14949 106 : use_objs.safe_push (c);
14950 106 : break;
14951 103 : case OMP_CLAUSE_DESTROY:
14952 103 : destroy_objs.safe_push (c);
14953 103 : break;
14954 50 : case OMP_CLAUSE_NOWAIT:
14955 50 : flags = build_int_cst (integer_type_node, GOMP_INTEROP_FLAG_NOWAIT);
14956 50 : break;
14957 35 : case OMP_CLAUSE_DEPEND:
14958 35 : {
14959 35 : tree *cp = gimple_omp_interop_clauses_ptr (gsi_stmt (*gsi_p));
14960 35 : lower_depend_clauses (cp, &dep_ilist, &dep_olist);
14961 35 : depend = OMP_CLAUSE_DECL (*cp);
14962 : }
14963 35 : break;
14964 26 : case OMP_CLAUSE_DEVICE:
14965 26 : device_num = OMP_CLAUSE_DEVICE_ID (c);
14966 26 : break;
14967 0 : default:
14968 0 : gcc_unreachable ();
14969 : }
14970 : }
14971 :
14972 164 : if (flags == NULL_TREE)
14973 114 : flags = build_int_cst (integer_type_node, 0);
14974 :
14975 164 : if (device_num == NULL_TREE)
14976 138 : device_num = build_int_cst (integer_type_node, GOMP_DEVICE_DEFAULT_OMP_61);
14977 :
14978 298 : tree n_init = build_int_cst (integer_type_node, init_objs.length ());
14979 234 : tree n_use = build_int_cst (integer_type_node, use_objs.length ());
14980 223 : tree n_destroy = build_int_cst (integer_type_node, destroy_objs.length ());
14981 :
14982 164 : lower_omp_interop_action_clauses (&bind_body, init_objs, &target_targetsync,
14983 : &prefer_type);
14984 164 : lower_omp_interop_action_clauses (&bind_body, use_objs);
14985 164 : lower_omp_interop_action_clauses (&bind_body, destroy_objs);
14986 :
14987 164 : gimple_seq_add_seq (&bind_body, dep_ilist);
14988 164 : tree fn = builtin_decl_explicit (BUILT_IN_GOMP_INTEROP);
14989 298 : tree init_arg = init_objs.length () ? init_objs.pop () : null_pointer_node;
14990 164 : tree target_targetsync_arg = target_targetsync.length ()
14991 134 : ? target_targetsync.pop ()
14992 164 : : null_pointer_node;
14993 164 : tree prefer_type_arg
14994 298 : = prefer_type.length () ? prefer_type.pop () : null_pointer_node;
14995 234 : tree use_arg = use_objs.length () ? use_objs.pop () : null_pointer_node;
14996 164 : tree destroy_arg
14997 223 : = destroy_objs.length () ? destroy_objs.pop () : null_pointer_node;
14998 164 : gcall *call
14999 164 : = gimple_build_call (fn, 11, device_num, n_init, init_arg,
15000 : target_targetsync_arg, prefer_type_arg, n_use, use_arg,
15001 : n_destroy, destroy_arg, flags, depend);
15002 164 : gimple_seq_add_stmt (&bind_body, call);
15003 164 : gimple_seq_add_seq (&bind_body, dep_olist);
15004 :
15005 164 : gsi_replace (gsi_p, bind, true);
15006 164 : gimple_bind_set_body (bind, bind_body);
15007 164 : pop_gimplify_context (bind);
15008 164 : gimple_bind_append_vars (bind, ctx->block_vars);
15009 164 : BLOCK_VARS (block) = ctx->block_vars;
15010 164 : }
15011 :
15012 : /* Expand code for an OpenMP teams directive. */
15013 :
15014 : static void
15015 5920 : lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx)
15016 : {
15017 5920 : gomp_teams *teams_stmt = as_a <gomp_teams *> (gsi_stmt (*gsi_p));
15018 5920 : push_gimplify_context ();
15019 :
15020 5920 : tree block = make_node (BLOCK);
15021 5920 : gbind *bind = gimple_build_bind (NULL, NULL, block);
15022 5920 : gsi_replace (gsi_p, bind, true);
15023 5920 : gimple_seq bind_body = NULL;
15024 5920 : gimple_seq dlist = NULL;
15025 5920 : gimple_seq olist = NULL;
15026 :
15027 5920 : tree num_teams = omp_find_clause (gimple_omp_teams_clauses (teams_stmt),
15028 5920 : OMP_CLAUSE_NUM_TEAMS);
15029 5920 : tree num_teams_lower = NULL_TREE;
15030 5920 : if (num_teams == NULL_TREE)
15031 5300 : num_teams = build_int_cst (unsigned_type_node, 0);
15032 : else
15033 : {
15034 620 : num_teams_lower = OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (num_teams);
15035 620 : if (num_teams_lower)
15036 : {
15037 148 : num_teams_lower = fold_convert (unsigned_type_node, num_teams_lower);
15038 148 : gimplify_expr (&num_teams_lower, &bind_body, NULL, is_gimple_val,
15039 : fb_rvalue);
15040 : }
15041 620 : num_teams = OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (num_teams);
15042 620 : num_teams = fold_convert (unsigned_type_node, num_teams);
15043 620 : gimplify_expr (&num_teams, &bind_body, NULL, is_gimple_val, fb_rvalue);
15044 : }
15045 5920 : if (num_teams_lower == NULL_TREE)
15046 5772 : num_teams_lower = num_teams;
15047 5920 : tree thread_limit = omp_find_clause (gimple_omp_teams_clauses (teams_stmt),
15048 5920 : OMP_CLAUSE_THREAD_LIMIT);
15049 5920 : if (thread_limit == NULL_TREE)
15050 5446 : thread_limit = build_int_cst (unsigned_type_node, 0);
15051 : else
15052 : {
15053 474 : thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit);
15054 474 : thread_limit = fold_convert (unsigned_type_node, thread_limit);
15055 474 : gimplify_expr (&thread_limit, &bind_body, NULL, is_gimple_val,
15056 : fb_rvalue);
15057 : }
15058 5920 : location_t loc = gimple_location (teams_stmt);
15059 5920 : tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS4);
15060 5920 : tree rettype = TREE_TYPE (TREE_TYPE (decl));
15061 5920 : tree first = create_tmp_var (rettype);
15062 5920 : gimple_seq_add_stmt (&bind_body,
15063 5920 : gimple_build_assign (first, build_one_cst (rettype)));
15064 5920 : tree llabel = create_artificial_label (loc);
15065 5920 : gimple_seq_add_stmt (&bind_body, gimple_build_label (llabel));
15066 5920 : gimple *call
15067 5920 : = gimple_build_call (decl, 4, num_teams_lower, num_teams, thread_limit,
15068 : first);
15069 5920 : gimple_set_location (call, loc);
15070 5920 : tree temp = create_tmp_var (rettype);
15071 5920 : gimple_call_set_lhs (call, temp);
15072 5920 : gimple_seq_add_stmt (&bind_body, call);
15073 :
15074 5920 : tree tlabel = create_artificial_label (loc);
15075 5920 : tree flabel = create_artificial_label (loc);
15076 5920 : gimple *cond = gimple_build_cond (NE_EXPR, temp, build_zero_cst (rettype),
15077 : tlabel, flabel);
15078 5920 : gimple_seq_add_stmt (&bind_body, cond);
15079 5920 : gimple_seq_add_stmt (&bind_body, gimple_build_label (tlabel));
15080 5920 : gimple_seq_add_stmt (&bind_body,
15081 5920 : gimple_build_assign (first, build_zero_cst (rettype)));
15082 :
15083 5920 : lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt),
15084 : &bind_body, &dlist, ctx, NULL);
15085 5920 : lower_omp (gimple_omp_body_ptr (teams_stmt), ctx);
15086 5920 : lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt), &olist,
15087 : NULL, ctx);
15088 5920 : gimple_seq_add_stmt (&bind_body, teams_stmt);
15089 :
15090 5920 : gimple_seq_add_seq (&bind_body, gimple_omp_body (teams_stmt));
15091 5920 : gimple_omp_set_body (teams_stmt, NULL);
15092 5920 : gimple_seq_add_seq (&bind_body, olist);
15093 5920 : gimple_seq_add_seq (&bind_body, dlist);
15094 5920 : gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true));
15095 5920 : gimple_seq_add_stmt (&bind_body, gimple_build_goto (llabel));
15096 5920 : gimple_seq_add_stmt (&bind_body, gimple_build_label (flabel));
15097 5920 : gimple_bind_set_body (bind, bind_body);
15098 :
15099 5920 : pop_gimplify_context (bind);
15100 :
15101 5920 : gimple_bind_append_vars (bind, ctx->block_vars);
15102 5920 : BLOCK_VARS (block) = ctx->block_vars;
15103 5920 : if (BLOCK_VARS (block))
15104 689 : TREE_USED (block) = 1;
15105 5920 : }
15106 :
15107 : /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
15108 : regimplified. If DATA is non-NULL, lower_omp_1 is outside
15109 : of OMP context, but with make_addressable_vars set. */
15110 :
15111 : static tree
15112 3203355 : lower_omp_regimplify_p (tree *tp, int *walk_subtrees,
15113 : void *data)
15114 : {
15115 3203355 : tree t = *tp;
15116 :
15117 : /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
15118 3203355 : if ((VAR_P (t) || TREE_CODE (t) == PARM_DECL || TREE_CODE (t) == RESULT_DECL)
15119 1491636 : && data == NULL
15120 1431186 : && DECL_HAS_VALUE_EXPR_P (t))
15121 : return t;
15122 :
15123 3098385 : if (make_addressable_vars
15124 423128 : && DECL_P (t)
15125 3303708 : && bitmap_bit_p (make_addressable_vars, DECL_UID (t)))
15126 : return t;
15127 :
15128 : /* If a global variable has been privatized, TREE_CONSTANT on
15129 : ADDR_EXPR might be wrong. */
15130 3095796 : if (data == NULL && TREE_CODE (t) == ADDR_EXPR)
15131 108230 : recompute_tree_invariant_for_addr_expr (t);
15132 :
15133 3095796 : *walk_subtrees = !IS_TYPE_OR_DECL_P (t);
15134 3095796 : return NULL_TREE;
15135 : }
15136 :
15137 : /* Data to be communicated between lower_omp_regimplify_operands and
15138 : lower_omp_regimplify_operands_p. */
15139 :
15140 : struct lower_omp_regimplify_operands_data
15141 : {
15142 : omp_context *ctx;
15143 : vec<tree> *decls;
15144 : };
15145 :
15146 : /* Helper function for lower_omp_regimplify_operands. Find
15147 : omp_member_access_dummy_var vars and adjust temporarily their
15148 : DECL_VALUE_EXPRs if needed. */
15149 :
15150 : static tree
15151 423655 : lower_omp_regimplify_operands_p (tree *tp, int *walk_subtrees,
15152 : void *data)
15153 : {
15154 423655 : tree t = omp_member_access_dummy_var (*tp);
15155 423655 : if (t)
15156 : {
15157 9 : struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
15158 9 : lower_omp_regimplify_operands_data *ldata
15159 : = (lower_omp_regimplify_operands_data *) wi->info;
15160 9 : tree o = maybe_lookup_decl (t, ldata->ctx);
15161 9 : if (o == NULL_TREE)
15162 5 : o = maybe_lookup_decl_in_outer_ctx (t, ldata->ctx);
15163 9 : if (o != t)
15164 : {
15165 9 : ldata->decls->safe_push (DECL_VALUE_EXPR (*tp));
15166 9 : ldata->decls->safe_push (*tp);
15167 9 : tree v = unshare_and_remap (DECL_VALUE_EXPR (*tp), t, o);
15168 9 : SET_DECL_VALUE_EXPR (*tp, v);
15169 : }
15170 : }
15171 423655 : *walk_subtrees = !IS_TYPE_OR_DECL_P (*tp);
15172 423655 : return NULL_TREE;
15173 : }
15174 :
15175 : /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
15176 : of omp_member_access_dummy_var vars during regimplification. */
15177 :
15178 : static void
15179 107404 : lower_omp_regimplify_operands (omp_context *ctx, gimple *stmt,
15180 : gimple_stmt_iterator *gsi_p)
15181 : {
15182 107404 : auto_vec<tree, 10> decls;
15183 107404 : if (ctx)
15184 : {
15185 104882 : struct walk_stmt_info wi;
15186 104882 : memset (&wi, '\0', sizeof (wi));
15187 104882 : struct lower_omp_regimplify_operands_data data;
15188 104882 : data.ctx = ctx;
15189 104882 : data.decls = &decls;
15190 104882 : wi.info = &data;
15191 104882 : walk_gimple_op (stmt, lower_omp_regimplify_operands_p, &wi);
15192 : }
15193 107404 : gimple_regimplify_operands (stmt, gsi_p);
15194 322221 : while (!decls.is_empty ())
15195 : {
15196 9 : tree t = decls.pop ();
15197 9 : tree v = decls.pop ();
15198 9 : SET_DECL_VALUE_EXPR (t, v);
15199 : }
15200 107404 : }
15201 :
15202 : static void
15203 2501990 : lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx)
15204 : {
15205 2501990 : gimple *stmt = gsi_stmt (*gsi_p);
15206 2501990 : struct walk_stmt_info wi;
15207 2501990 : gcall *call_stmt;
15208 :
15209 2501990 : if (gimple_has_location (stmt))
15210 2055460 : input_location = gimple_location (stmt);
15211 :
15212 2501990 : if (make_addressable_vars)
15213 154214 : memset (&wi, '\0', sizeof (wi));
15214 :
15215 : /* If we have issued syntax errors, avoid doing any heavy lifting.
15216 : Just replace the OMP directives with a NOP to avoid
15217 : confusing RTL expansion. */
15218 2501990 : if (seen_error () && is_gimple_omp (stmt))
15219 : {
15220 11337 : gsi_replace (gsi_p, gimple_build_nop (), true);
15221 11337 : return;
15222 : }
15223 :
15224 2490653 : switch (gimple_code (stmt))
15225 : {
15226 155454 : case GIMPLE_COND:
15227 155454 : {
15228 155454 : gcond *cond_stmt = as_a <gcond *> (stmt);
15229 88585 : if ((ctx || make_addressable_vars)
15230 159062 : && (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
15231 : lower_omp_regimplify_p,
15232 : ctx ? NULL : &wi, NULL)
15233 135399 : || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
15234 : lower_omp_regimplify_p,
15235 : ctx ? NULL : &wi, NULL)))
15236 1247 : lower_omp_regimplify_operands (ctx, cond_stmt, gsi_p);
15237 : }
15238 : break;
15239 20 : case GIMPLE_CATCH:
15240 20 : lower_omp (gimple_catch_handler_ptr (as_a <gcatch *> (stmt)), ctx);
15241 20 : break;
15242 0 : case GIMPLE_EH_FILTER:
15243 0 : lower_omp (gimple_eh_filter_failure_ptr (stmt), ctx);
15244 0 : break;
15245 25908 : case GIMPLE_TRY:
15246 25908 : lower_omp (gimple_try_eval_ptr (stmt), ctx);
15247 25903 : lower_omp (gimple_try_cleanup_ptr (stmt), ctx);
15248 25903 : break;
15249 0 : case GIMPLE_ASSUME:
15250 0 : lower_omp (gimple_assume_body_ptr (stmt), ctx);
15251 0 : break;
15252 5 : case GIMPLE_TRANSACTION:
15253 5 : lower_omp (gimple_transaction_body_ptr (as_a <gtransaction *> (stmt)),
15254 : ctx);
15255 5 : break;
15256 167961 : case GIMPLE_BIND:
15257 167961 : if (ctx && is_gimple_omp_oacc (ctx->stmt))
15258 : {
15259 17785 : tree vars = gimple_bind_vars (as_a <gbind *> (stmt));
15260 17785 : oacc_privatization_scan_decl_chain (ctx, vars);
15261 : }
15262 167961 : lower_omp (gimple_bind_body_ptr (as_a <gbind *> (stmt)), ctx);
15263 167954 : maybe_remove_omp_member_access_dummy_vars (as_a <gbind *> (stmt));
15264 167954 : break;
15265 19953 : case GIMPLE_OMP_PARALLEL:
15266 19953 : case GIMPLE_OMP_TASK:
15267 19953 : ctx = maybe_lookup_ctx (stmt);
15268 19953 : gcc_assert (ctx);
15269 19953 : if (ctx->cancellable)
15270 129 : ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
15271 19953 : lower_omp_taskreg (gsi_p, ctx);
15272 19953 : break;
15273 46859 : case GIMPLE_OMP_FOR:
15274 46859 : ctx = maybe_lookup_ctx (stmt);
15275 46859 : gcc_assert (ctx);
15276 46859 : if (ctx->cancellable)
15277 36 : ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
15278 46859 : lower_omp_for (gsi_p, ctx);
15279 46859 : break;
15280 386 : case GIMPLE_OMP_SECTIONS:
15281 386 : ctx = maybe_lookup_ctx (stmt);
15282 386 : gcc_assert (ctx);
15283 386 : if (ctx->cancellable)
15284 19 : ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
15285 386 : lower_omp_sections (gsi_p, ctx);
15286 386 : break;
15287 145 : case GIMPLE_OMP_SCOPE:
15288 145 : ctx = maybe_lookup_ctx (stmt);
15289 145 : gcc_assert (ctx);
15290 145 : lower_omp_scope (gsi_p, ctx);
15291 145 : break;
15292 659 : case GIMPLE_OMP_DISPATCH:
15293 659 : ctx = maybe_lookup_ctx (stmt);
15294 659 : gcc_assert (ctx);
15295 659 : lower_omp_dispatch (gsi_p, ctx);
15296 659 : break;
15297 164 : case GIMPLE_OMP_INTEROP:
15298 164 : ctx = maybe_lookup_ctx (stmt);
15299 164 : gcc_assert (ctx);
15300 164 : lower_omp_interop (gsi_p, ctx);
15301 164 : break;
15302 1127 : case GIMPLE_OMP_SINGLE:
15303 1127 : ctx = maybe_lookup_ctx (stmt);
15304 1127 : gcc_assert (ctx);
15305 1127 : lower_omp_single (gsi_p, ctx);
15306 1127 : break;
15307 658 : case GIMPLE_OMP_STRUCTURED_BLOCK:
15308 : /* We have already done error checking at this point, so these nodes
15309 : can be completely removed and replaced with their body. */
15310 658 : ctx = maybe_lookup_ctx (stmt);
15311 658 : gcc_assert (ctx);
15312 658 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
15313 658 : gsi_replace_with_seq (gsi_p, gimple_omp_body (stmt), true);
15314 658 : break;
15315 1118 : case GIMPLE_OMP_MASTER:
15316 1118 : case GIMPLE_OMP_MASKED:
15317 1118 : ctx = maybe_lookup_ctx (stmt);
15318 1118 : gcc_assert (ctx);
15319 1118 : lower_omp_master (gsi_p, ctx);
15320 1118 : break;
15321 536 : case GIMPLE_OMP_TASKGROUP:
15322 536 : ctx = maybe_lookup_ctx (stmt);
15323 536 : gcc_assert (ctx);
15324 536 : lower_omp_taskgroup (gsi_p, ctx);
15325 536 : break;
15326 1124 : case GIMPLE_OMP_ORDERED:
15327 1124 : ctx = maybe_lookup_ctx (stmt);
15328 1124 : gcc_assert (ctx);
15329 1124 : lower_omp_ordered (gsi_p, ctx);
15330 1124 : break;
15331 1276 : case GIMPLE_OMP_SCAN:
15332 1276 : ctx = maybe_lookup_ctx (stmt);
15333 1276 : gcc_assert (ctx);
15334 1276 : lower_omp_scan (gsi_p, ctx);
15335 1276 : break;
15336 311 : case GIMPLE_OMP_CRITICAL:
15337 311 : ctx = maybe_lookup_ctx (stmt);
15338 311 : gcc_assert (ctx);
15339 311 : lower_omp_critical (gsi_p, ctx);
15340 311 : break;
15341 2478 : case GIMPLE_OMP_ATOMIC_LOAD:
15342 83 : if ((ctx || make_addressable_vars)
15343 2478 : && walk_tree (gimple_omp_atomic_load_rhs_ptr (
15344 : as_a <gomp_atomic_load *> (stmt)),
15345 : lower_omp_regimplify_p, ctx ? NULL : &wi, NULL))
15346 1389 : lower_omp_regimplify_operands (ctx, stmt, gsi_p);
15347 : break;
15348 37147 : case GIMPLE_OMP_TARGET:
15349 37147 : ctx = maybe_lookup_ctx (stmt);
15350 37147 : gcc_assert (ctx);
15351 37147 : lower_omp_target (gsi_p, ctx);
15352 37147 : break;
15353 8438 : case GIMPLE_OMP_TEAMS:
15354 8438 : ctx = maybe_lookup_ctx (stmt);
15355 8438 : gcc_assert (ctx);
15356 8438 : if (gimple_omp_teams_host (as_a <gomp_teams *> (stmt)))
15357 2518 : lower_omp_taskreg (gsi_p, ctx);
15358 : else
15359 5920 : lower_omp_teams (gsi_p, ctx);
15360 : break;
15361 134309 : case GIMPLE_CALL:
15362 134309 : tree fndecl;
15363 134309 : call_stmt = as_a <gcall *> (stmt);
15364 134309 : fndecl = gimple_call_fndecl (call_stmt);
15365 134309 : if (fndecl
15366 134309 : && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL))
15367 38552 : switch (DECL_FUNCTION_CODE (fndecl))
15368 : {
15369 703 : case BUILT_IN_GOMP_BARRIER:
15370 703 : if (ctx == NULL)
15371 : break;
15372 : /* FALLTHRU */
15373 935 : case BUILT_IN_GOMP_CANCEL:
15374 935 : case BUILT_IN_GOMP_CANCELLATION_POINT:
15375 935 : omp_context *cctx;
15376 935 : cctx = ctx;
15377 935 : if (gimple_code (cctx->stmt) == GIMPLE_OMP_SECTION)
15378 43 : cctx = cctx->outer;
15379 935 : gcc_assert (gimple_call_lhs (call_stmt) == NULL_TREE);
15380 935 : if (!cctx->cancellable)
15381 : {
15382 655 : if (DECL_FUNCTION_CODE (fndecl)
15383 : == BUILT_IN_GOMP_CANCELLATION_POINT)
15384 : {
15385 0 : stmt = gimple_build_nop ();
15386 0 : gsi_replace (gsi_p, stmt, false);
15387 : }
15388 : break;
15389 : }
15390 280 : if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_GOMP_BARRIER)
15391 : {
15392 14 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL);
15393 14 : gimple_call_set_fndecl (call_stmt, fndecl);
15394 14 : gimple_call_set_fntype (call_stmt, TREE_TYPE (fndecl));
15395 : }
15396 280 : tree lhs;
15397 280 : lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl)));
15398 280 : gimple_call_set_lhs (call_stmt, lhs);
15399 280 : tree fallthru_label;
15400 280 : fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
15401 280 : gimple *g;
15402 280 : g = gimple_build_label (fallthru_label);
15403 280 : gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
15404 280 : g = gimple_build_cond (NE_EXPR, lhs,
15405 280 : fold_convert (TREE_TYPE (lhs),
15406 : boolean_false_node),
15407 : cctx->cancel_label, fallthru_label);
15408 280 : gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
15409 280 : break;
15410 : default:
15411 : break;
15412 : }
15413 134274 : goto regimplify;
15414 :
15415 : case GIMPLE_ASSIGN:
15416 1452289 : for (omp_context *up = ctx; up; up = up->outer)
15417 : {
15418 683369 : if (gimple_code (up->stmt) == GIMPLE_OMP_ORDERED
15419 : || gimple_code (up->stmt) == GIMPLE_OMP_CRITICAL
15420 : || gimple_code (up->stmt) == GIMPLE_OMP_TASKGROUP
15421 : || gimple_code (up->stmt) == GIMPLE_OMP_SCOPE
15422 : || gimple_code (up->stmt) == GIMPLE_OMP_SECTION
15423 : || gimple_code (up->stmt) == GIMPLE_OMP_SCAN
15424 : || (gimple_code (up->stmt) == GIMPLE_OMP_TARGET
15425 229893 : && (gimple_omp_target_kind (up->stmt)
15426 : == GF_OMP_TARGET_KIND_DATA)))
15427 90715 : continue;
15428 592654 : else if (!up->lastprivate_conditional_map)
15429 : break;
15430 4089 : tree lhs = get_base_address (gimple_assign_lhs (stmt));
15431 4089 : if (TREE_CODE (lhs) == MEM_REF
15432 26 : && DECL_P (TREE_OPERAND (lhs, 0))
15433 4112 : && TREE_CODE (TREE_TYPE (TREE_OPERAND (lhs,
15434 : 0))) == REFERENCE_TYPE)
15435 19 : lhs = TREE_OPERAND (lhs, 0);
15436 4089 : if (DECL_P (lhs))
15437 3046 : if (tree *v = up->lastprivate_conditional_map->get (lhs))
15438 : {
15439 376 : tree clauses;
15440 376 : if (up->combined_into_simd_safelen1)
15441 : {
15442 54 : up = up->outer;
15443 54 : if (gimple_code (up->stmt) == GIMPLE_OMP_SCAN)
15444 3 : up = up->outer;
15445 : }
15446 376 : if (gimple_code (up->stmt) == GIMPLE_OMP_FOR)
15447 292 : clauses = gimple_omp_for_clauses (up->stmt);
15448 : else
15449 84 : clauses = gimple_omp_sections_clauses (up->stmt);
15450 376 : tree c = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_);
15451 376 : if (!OMP_CLAUSE__CONDTEMP__ITER (c))
15452 342 : c = omp_find_clause (OMP_CLAUSE_CHAIN (c),
15453 : OMP_CLAUSE__CONDTEMP_);
15454 376 : gcc_assert (OMP_CLAUSE__CONDTEMP__ITER (c));
15455 376 : gimple *g = gimple_build_assign (*v, OMP_CLAUSE_DECL (c));
15456 376 : gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
15457 : }
15458 : }
15459 : /* FALLTHRU */
15460 :
15461 2018926 : default:
15462 2018926 : regimplify:
15463 1065012 : if ((ctx || make_addressable_vars)
15464 2094777 : && walk_gimple_op (stmt, lower_omp_regimplify_p,
15465 : ctx ? NULL : &wi))
15466 : {
15467 : /* Just remove clobbers, this should happen only if we have
15468 : "privatized" local addressable variables in SIMD regions,
15469 : the clobber isn't needed in that case and gimplifying address
15470 : of the ARRAY_REF into a pointer and creating MEM_REF based
15471 : clobber would create worse code than we get with the clobber
15472 : dropped. */
15473 104923 : if (gimple_clobber_p (stmt))
15474 : {
15475 155 : gsi_replace (gsi_p, gimple_build_nop (), true);
15476 155 : break;
15477 : }
15478 104768 : lower_omp_regimplify_operands (ctx, stmt, gsi_p);
15479 : }
15480 : break;
15481 : }
15482 : }
15483 :
15484 : static void
15485 410422 : lower_omp (gimple_seq *body, omp_context *ctx)
15486 : {
15487 410422 : location_t saved_location = input_location;
15488 410422 : gimple_stmt_iterator gsi;
15489 3274672 : for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
15490 2501990 : lower_omp_1 (&gsi, ctx);
15491 : /* During gimplification, we haven't folded statements inside offloading
15492 : or taskreg regions (gimplify.cc:maybe_fold_stmt); do that now. */
15493 410406 : if (target_nesting_level || taskreg_nesting_level)
15494 656469 : for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
15495 443336 : fold_stmt (&gsi);
15496 410406 : input_location = saved_location;
15497 410406 : }
15498 :
15499 : /* Main entry point. */
15500 :
15501 : static unsigned int
15502 3008128 : execute_lower_omp (void)
15503 : {
15504 3008128 : gimple_seq body;
15505 3008128 : int i;
15506 3008128 : omp_context *ctx;
15507 :
15508 : /* This pass always runs, to provide PROP_gimple_lomp.
15509 : But often, there is nothing to do. */
15510 3008128 : if (flag_openacc == 0 && flag_openmp == 0
15511 2953565 : && flag_openmp_simd == 0)
15512 : return 0;
15513 :
15514 56820 : all_contexts = splay_tree_new (splay_tree_compare_pointers, 0,
15515 : delete_omp_context);
15516 :
15517 56820 : body = gimple_body (current_function_decl);
15518 :
15519 56820 : scan_omp (&body, NULL);
15520 56820 : gcc_assert (taskreg_nesting_level == 0);
15521 82819 : FOR_EACH_VEC_ELT (taskreg_contexts, i, ctx)
15522 25999 : finish_taskreg_scan (ctx);
15523 56820 : taskreg_contexts.release ();
15524 :
15525 56820 : if (all_contexts->root)
15526 : {
15527 25785 : if (make_addressable_vars)
15528 622 : push_gimplify_context ();
15529 25785 : lower_omp (&body, NULL);
15530 25781 : if (make_addressable_vars)
15531 622 : pop_gimplify_context (NULL);
15532 : }
15533 :
15534 56816 : if (all_contexts)
15535 : {
15536 56816 : splay_tree_delete (all_contexts);
15537 56816 : all_contexts = NULL;
15538 : }
15539 56816 : BITMAP_FREE (make_addressable_vars);
15540 56816 : BITMAP_FREE (global_nonaddressable_vars);
15541 :
15542 : /* If current function is a method, remove artificial dummy VAR_DECL created
15543 : for non-static data member privatization, they aren't needed for
15544 : debuginfo nor anything else, have been already replaced everywhere in the
15545 : IL and cause problems with LTO. */
15546 56816 : if (DECL_ARGUMENTS (current_function_decl)
15547 32459 : && DECL_ARTIFICIAL (DECL_ARGUMENTS (current_function_decl))
15548 66345 : && (TREE_CODE (TREE_TYPE (DECL_ARGUMENTS (current_function_decl)))
15549 : == POINTER_TYPE))
15550 8967 : remove_member_access_dummy_vars (DECL_INITIAL (current_function_decl));
15551 :
15552 57756 : for (auto task_stmt : task_cpyfns)
15553 480 : finalize_task_copyfn (task_stmt);
15554 56816 : task_cpyfns.release ();
15555 56816 : return 0;
15556 : }
15557 :
15558 : namespace {
15559 :
15560 : const pass_data pass_data_lower_omp =
15561 : {
15562 : GIMPLE_PASS, /* type */
15563 : "omplower", /* name */
15564 : OPTGROUP_OMP, /* optinfo_flags */
15565 : TV_NONE, /* tv_id */
15566 : PROP_gimple_any, /* properties_required */
15567 : PROP_gimple_lomp | PROP_gimple_lomp_dev, /* properties_provided */
15568 : 0, /* properties_destroyed */
15569 : 0, /* todo_flags_start */
15570 : 0, /* todo_flags_finish */
15571 : };
15572 :
15573 : class pass_lower_omp : public gimple_opt_pass
15574 : {
15575 : public:
15576 292371 : pass_lower_omp (gcc::context *ctxt)
15577 584742 : : gimple_opt_pass (pass_data_lower_omp, ctxt)
15578 : {}
15579 :
15580 : /* opt_pass methods: */
15581 3008128 : unsigned int execute (function *) final override
15582 : {
15583 3008128 : return execute_lower_omp ();
15584 : }
15585 :
15586 : }; // class pass_lower_omp
15587 :
15588 : } // anon namespace
15589 :
15590 : gimple_opt_pass *
15591 292371 : make_pass_lower_omp (gcc::context *ctxt)
15592 : {
15593 292371 : return new pass_lower_omp (ctxt);
15594 : }
15595 :
15596 : /* The following is a utility to diagnose structured block violations.
15597 : It is not part of the "omplower" pass, as that's invoked too late. It
15598 : should be invoked by the respective front ends after gimplification. */
15599 :
15600 : static splay_tree all_labels;
15601 :
15602 : /* Check for mismatched contexts and generate an error if needed. Return
15603 : true if an error is detected. */
15604 :
15605 : static bool
15606 559571 : diagnose_sb_0 (gimple_stmt_iterator *gsi_p,
15607 : gimple *branch_ctx, gimple *label_ctx)
15608 : {
15609 559571 : gcc_checking_assert (!branch_ctx || is_gimple_omp (branch_ctx));
15610 559571 : gcc_checking_assert (!label_ctx || is_gimple_omp (label_ctx));
15611 :
15612 559571 : if (label_ctx == branch_ctx)
15613 : return false;
15614 :
15615 180 : const char* kind = NULL;
15616 :
15617 180 : if (flag_openacc)
15618 : {
15619 7 : if ((branch_ctx && is_gimple_omp_oacc (branch_ctx))
15620 16 : || (label_ctx && is_gimple_omp_oacc (label_ctx)))
15621 : {
15622 : gcc_checking_assert (kind == NULL);
15623 : kind = "OpenACC";
15624 : }
15625 : }
15626 : if (kind == NULL)
15627 : {
15628 164 : gcc_checking_assert (flag_openmp || flag_openmp_simd);
15629 : kind = "OpenMP";
15630 : }
15631 :
15632 : /* Previously we kept track of the label's entire context in diagnose_sb_[12]
15633 : so we could traverse it and issue a correct "exit" or "enter" error
15634 : message upon a structured block violation.
15635 :
15636 : We built the context by building a list with tree_cons'ing, but there is
15637 : no easy counterpart in gimple tuples. It seems like far too much work
15638 : for issuing exit/enter error messages. If someone really misses the
15639 : distinct error message... patches welcome. */
15640 :
15641 : #if 0
15642 : /* Try to avoid confusing the user by producing and error message
15643 : with correct "exit" or "enter" verbiage. We prefer "exit"
15644 : unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
15645 : if (branch_ctx == NULL)
15646 : exit_p = false;
15647 : else
15648 : {
15649 : while (label_ctx)
15650 : {
15651 : if (TREE_VALUE (label_ctx) == branch_ctx)
15652 : {
15653 : exit_p = false;
15654 : break;
15655 : }
15656 : label_ctx = TREE_CHAIN (label_ctx);
15657 : }
15658 : }
15659 :
15660 : if (exit_p)
15661 : error ("invalid exit from %s structured block", kind);
15662 : else
15663 : error ("invalid entry to %s structured block", kind);
15664 : #endif
15665 :
15666 : /* If it's obvious we have an invalid entry, be specific about the error. */
15667 180 : if (branch_ctx == NULL)
15668 53 : error ("invalid entry to %s structured block", kind);
15669 : else
15670 : {
15671 : /* Otherwise, be vague and lazy, but efficient. */
15672 127 : error ("invalid branch to/from %s structured block", kind);
15673 : }
15674 :
15675 180 : gsi_replace (gsi_p, gimple_build_nop (), false);
15676 180 : return true;
15677 : }
15678 :
15679 : /* Pass 1: Create a minimal tree of structured blocks, and record
15680 : where each label is found. */
15681 :
15682 : static tree
15683 3342816 : diagnose_sb_1 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
15684 : struct walk_stmt_info *wi)
15685 : {
15686 3342816 : gimple *context = (gimple *) wi->info;
15687 3342816 : gimple *inner_context;
15688 3342816 : gimple *stmt = gsi_stmt (*gsi_p);
15689 :
15690 3342816 : *handled_ops_p = true;
15691 :
15692 3342816 : switch (gimple_code (stmt))
15693 : {
15694 318998 : WALK_SUBSTMTS;
15695 :
15696 83608 : case GIMPLE_OMP_PARALLEL:
15697 83608 : case GIMPLE_OMP_TASK:
15698 83608 : case GIMPLE_OMP_SCOPE:
15699 83608 : case GIMPLE_OMP_SECTIONS:
15700 83608 : case GIMPLE_OMP_SINGLE:
15701 83608 : case GIMPLE_OMP_SECTION:
15702 83608 : case GIMPLE_OMP_STRUCTURED_BLOCK:
15703 83608 : case GIMPLE_OMP_MASTER:
15704 83608 : case GIMPLE_OMP_MASKED:
15705 83608 : case GIMPLE_OMP_ORDERED:
15706 83608 : case GIMPLE_OMP_SCAN:
15707 83608 : case GIMPLE_OMP_CRITICAL:
15708 83608 : case GIMPLE_OMP_TARGET:
15709 83608 : case GIMPLE_OMP_TEAMS:
15710 83608 : case GIMPLE_OMP_TASKGROUP:
15711 : /* The minimal context here is just the current OMP construct. */
15712 83608 : inner_context = stmt;
15713 83608 : wi->info = inner_context;
15714 83608 : walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
15715 83608 : wi->info = context;
15716 83608 : break;
15717 :
15718 52723 : case GIMPLE_OMP_FOR:
15719 52723 : inner_context = stmt;
15720 52723 : wi->info = inner_context;
15721 : /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
15722 : walk them. */
15723 52723 : walk_gimple_seq (gimple_omp_for_pre_body (stmt),
15724 : diagnose_sb_1, NULL, wi);
15725 52723 : walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
15726 52723 : wi->info = context;
15727 52723 : break;
15728 :
15729 556415 : case GIMPLE_LABEL:
15730 1112830 : splay_tree_insert (all_labels,
15731 556415 : (splay_tree_key) gimple_label_label (
15732 556415 : as_a <glabel *> (stmt)),
15733 : (splay_tree_value) context);
15734 556415 : break;
15735 :
15736 : default:
15737 : break;
15738 : }
15739 :
15740 3342816 : return NULL_TREE;
15741 : }
15742 :
15743 : /* Pass 2: Check each branch and see if its context differs from that of
15744 : the destination label's context. */
15745 :
15746 : static tree
15747 3342816 : diagnose_sb_2 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
15748 : struct walk_stmt_info *wi)
15749 : {
15750 3342816 : gimple *context = (gimple *) wi->info;
15751 3342816 : splay_tree_node n;
15752 3342816 : gimple *stmt = gsi_stmt (*gsi_p);
15753 :
15754 3342816 : *handled_ops_p = true;
15755 :
15756 3342816 : switch (gimple_code (stmt))
15757 : {
15758 318998 : WALK_SUBSTMTS;
15759 :
15760 83608 : case GIMPLE_OMP_PARALLEL:
15761 83608 : case GIMPLE_OMP_TASK:
15762 83608 : case GIMPLE_OMP_SCOPE:
15763 83608 : case GIMPLE_OMP_SECTIONS:
15764 83608 : case GIMPLE_OMP_SINGLE:
15765 83608 : case GIMPLE_OMP_SECTION:
15766 83608 : case GIMPLE_OMP_STRUCTURED_BLOCK:
15767 83608 : case GIMPLE_OMP_MASTER:
15768 83608 : case GIMPLE_OMP_MASKED:
15769 83608 : case GIMPLE_OMP_ORDERED:
15770 83608 : case GIMPLE_OMP_SCAN:
15771 83608 : case GIMPLE_OMP_CRITICAL:
15772 83608 : case GIMPLE_OMP_TARGET:
15773 83608 : case GIMPLE_OMP_TEAMS:
15774 83608 : case GIMPLE_OMP_TASKGROUP:
15775 83608 : wi->info = stmt;
15776 83608 : walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
15777 83608 : wi->info = context;
15778 83608 : break;
15779 :
15780 52723 : case GIMPLE_OMP_FOR:
15781 52723 : wi->info = stmt;
15782 : /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
15783 : walk them. */
15784 52723 : walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt),
15785 : diagnose_sb_2, NULL, wi);
15786 52723 : walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
15787 52723 : wi->info = context;
15788 52723 : break;
15789 :
15790 205914 : case GIMPLE_COND:
15791 205914 : {
15792 205914 : gcond *cond_stmt = as_a <gcond *> (stmt);
15793 205914 : tree lab = gimple_cond_true_label (cond_stmt);
15794 205914 : if (lab)
15795 : {
15796 205914 : n = splay_tree_lookup (all_labels,
15797 : (splay_tree_key) lab);
15798 411828 : diagnose_sb_0 (gsi_p, context,
15799 205914 : n ? (gimple *) n->value : NULL);
15800 : }
15801 205914 : lab = gimple_cond_false_label (cond_stmt);
15802 205914 : if (lab)
15803 : {
15804 205914 : n = splay_tree_lookup (all_labels,
15805 : (splay_tree_key) lab);
15806 411828 : diagnose_sb_0 (gsi_p, context,
15807 205914 : n ? (gimple *) n->value : NULL);
15808 : }
15809 : }
15810 : break;
15811 :
15812 111979 : case GIMPLE_GOTO:
15813 111979 : {
15814 111979 : tree lab = gimple_goto_dest (stmt);
15815 111979 : if (TREE_CODE (lab) != LABEL_DECL)
15816 : break;
15817 :
15818 111979 : n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
15819 111979 : diagnose_sb_0 (gsi_p, context, n ? (gimple *) n->value : NULL);
15820 : }
15821 111979 : break;
15822 :
15823 424 : case GIMPLE_SWITCH:
15824 424 : {
15825 424 : gswitch *switch_stmt = as_a <gswitch *> (stmt);
15826 424 : unsigned int i;
15827 1724 : for (i = 0; i < gimple_switch_num_labels (switch_stmt); ++i)
15828 : {
15829 1316 : tree lab = CASE_LABEL (gimple_switch_label (switch_stmt, i));
15830 1316 : n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
15831 1316 : if (n && diagnose_sb_0 (gsi_p, context, (gimple *) n->value))
15832 : break;
15833 : }
15834 : }
15835 : break;
15836 :
15837 730 : case GIMPLE_ASM:
15838 730 : {
15839 730 : gasm *asm_stmt = as_a <gasm *> (stmt);
15840 745 : for (unsigned i = 0; i < gimple_asm_nlabels (asm_stmt); ++i)
15841 : {
15842 21 : tree lab = TREE_VALUE (gimple_asm_label_op (asm_stmt, i));
15843 21 : n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
15844 21 : if (n && diagnose_sb_0 (gsi_p, context, (gimple *) n->value))
15845 : break;
15846 : }
15847 : }
15848 : break;
15849 :
15850 34427 : case GIMPLE_RETURN:
15851 34427 : diagnose_sb_0 (gsi_p, context, NULL);
15852 34427 : break;
15853 :
15854 : default:
15855 : break;
15856 : }
15857 :
15858 3342816 : return NULL_TREE;
15859 : }
15860 :
15861 : static unsigned int
15862 56829 : diagnose_omp_structured_block_errors (void)
15863 : {
15864 56829 : struct walk_stmt_info wi;
15865 56829 : gimple_seq body = gimple_body (current_function_decl);
15866 :
15867 56829 : all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0);
15868 :
15869 56829 : memset (&wi, 0, sizeof (wi));
15870 56829 : walk_gimple_seq (body, diagnose_sb_1, NULL, &wi);
15871 :
15872 56829 : memset (&wi, 0, sizeof (wi));
15873 56829 : wi.want_locations = true;
15874 56829 : walk_gimple_seq_mod (&body, diagnose_sb_2, NULL, &wi);
15875 :
15876 56829 : gimple_set_body (current_function_decl, body);
15877 :
15878 56829 : splay_tree_delete (all_labels);
15879 56829 : all_labels = NULL;
15880 :
15881 56829 : return 0;
15882 : }
15883 :
15884 : namespace {
15885 :
15886 : const pass_data pass_data_diagnose_omp_blocks =
15887 : {
15888 : GIMPLE_PASS, /* type */
15889 : "*diagnose_omp_blocks", /* name */
15890 : OPTGROUP_OMP, /* optinfo_flags */
15891 : TV_NONE, /* tv_id */
15892 : PROP_gimple_any, /* properties_required */
15893 : 0, /* properties_provided */
15894 : 0, /* properties_destroyed */
15895 : 0, /* todo_flags_start */
15896 : 0, /* todo_flags_finish */
15897 : };
15898 :
15899 : class pass_diagnose_omp_blocks : public gimple_opt_pass
15900 : {
15901 : public:
15902 292371 : pass_diagnose_omp_blocks (gcc::context *ctxt)
15903 584742 : : gimple_opt_pass (pass_data_diagnose_omp_blocks, ctxt)
15904 : {}
15905 :
15906 : /* opt_pass methods: */
15907 3008142 : bool gate (function *) final override
15908 : {
15909 3008142 : return flag_openacc || flag_openmp || flag_openmp_simd;
15910 : }
15911 56829 : unsigned int execute (function *) final override
15912 : {
15913 56829 : return diagnose_omp_structured_block_errors ();
15914 : }
15915 :
15916 : }; // class pass_diagnose_omp_blocks
15917 :
15918 : } // anon namespace
15919 :
15920 : gimple_opt_pass *
15921 292371 : make_pass_diagnose_omp_blocks (gcc::context *ctxt)
15922 : {
15923 292371 : return new pass_diagnose_omp_blocks (ctxt);
15924 : }
15925 :
15926 :
15927 : #include "gt-omp-low.h"
|
