Line data Source code
1 : /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 : calls to the runtime library (libgomp), data marshalling to implement data
3 : sharing and copying clauses, offloading to accelerators, and more.
4 :
5 : Contributed by Diego Novillo <dnovillo@redhat.com>
6 :
7 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
8 :
9 : This file is part of GCC.
10 :
11 : GCC is free software; you can redistribute it and/or modify it under
12 : the terms of the GNU General Public License as published by the Free
13 : Software Foundation; either version 3, or (at your option) any later
14 : version.
15 :
16 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 : for more details.
20 :
21 : You should have received a copy of the GNU General Public License
22 : along with GCC; see the file COPYING3. If not see
23 : <http://www.gnu.org/licenses/>. */
24 :
25 : #include "config.h"
26 : #include "system.h"
27 : #include "coretypes.h"
28 : #include "backend.h"
29 : #include "target.h"
30 : #include "tree.h"
31 : #include "gimple.h"
32 : #include "tree-pass.h"
33 : #include "ssa.h"
34 : #include "cgraph.h"
35 : #include "pretty-print.h"
36 : #include "diagnostic-core.h"
37 : #include "fold-const.h"
38 : #include "stor-layout.h"
39 : #include "internal-fn.h"
40 : #include "gimple-iterator.h"
41 : #include "gimple-fold.h"
42 : #include "gimplify.h"
43 : #include "gimplify-me.h"
44 : #include "gimple-walk.h"
45 : #include "tree-iterator.h"
46 : #include "tree-inline.h"
47 : #include "langhooks.h"
48 : #include "tree-dfa.h"
49 : #include "tree-ssa.h"
50 : #include "splay-tree.h"
51 : #include "omp-general.h"
52 : #include "omp-low.h"
53 : #include "gimple-low.h"
54 : #include "alloc-pool.h"
55 : #include "symbol-summary.h"
56 : #include "tree-nested.h"
57 : #include "context.h"
58 : #include "gomp-constants.h"
59 : #include "gimple-pretty-print.h"
60 : #include "stringpool.h"
61 : #include "attribs.h"
62 : #include "omp-offload.h"
63 :
64 : /* Lowering of OMP parallel and workshare constructs proceeds in two
65 : phases. The first phase scans the function looking for OMP statements
66 : and then for variables that must be replaced to satisfy data sharing
67 : clauses. The second phase expands code for the constructs, as well as
68 : re-gimplifying things when variables have been replaced with complex
69 : expressions.
70 :
71 : Final code generation is done by pass_expand_omp. The flowgraph is
72 : scanned for regions which are then moved to a new
73 : function, to be invoked by the thread library, or offloaded. */
74 :
75 : /* Context structure. Used to store information about each parallel
76 : directive in the code. */
77 :
78 : struct omp_context
79 : {
80 : /* This field must be at the beginning, as we do "inheritance": Some
81 : callback functions for tree-inline.cc (e.g., omp_copy_decl)
82 : receive a copy_body_data pointer that is up-casted to an
83 : omp_context pointer. */
84 : copy_body_data cb;
85 :
86 : /* The tree of contexts corresponding to the encountered constructs. */
87 : struct omp_context *outer;
88 : gimple *stmt;
89 :
90 : /* Map variables to fields in a structure that allows communication
91 : between sending and receiving threads. */
92 : splay_tree field_map;
93 : tree record_type;
94 : tree sender_decl;
95 : tree receiver_decl;
96 :
97 : /* These are used just by task contexts, if task firstprivate fn is
98 : needed. srecord_type is used to communicate from the thread
99 : that encountered the task construct to task firstprivate fn,
100 : record_type is allocated by GOMP_task, initialized by task firstprivate
101 : fn and passed to the task body fn. */
102 : splay_tree sfield_map;
103 : tree srecord_type;
104 :
105 : /* A chain of variables to add to the top-level block surrounding the
106 : construct. In the case of a parallel, this is in the child function. */
107 : tree block_vars;
108 :
109 : /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
110 : barriers should jump to during omplower pass. */
111 : tree cancel_label;
112 :
113 : /* The sibling GIMPLE_OMP_FOR simd with _simt_ clause or NULL
114 : otherwise. */
115 : gimple *simt_stmt;
116 :
117 : /* For task reductions registered in this context, a vector containing
118 : the length of the private copies block (if constant, otherwise NULL)
119 : and then offsets (if constant, otherwise NULL) for each entry. */
120 : vec<tree> task_reductions;
121 :
122 : /* A hash map from the reduction clauses to the registered array
123 : elts. */
124 : hash_map<tree, unsigned> *task_reduction_map;
125 :
126 : /* And a hash map from the lastprivate(conditional:) variables to their
127 : corresponding tracking loop iteration variables. */
128 : hash_map<tree, tree> *lastprivate_conditional_map;
129 :
130 : /* And a hash map from the allocate variables to their corresponding
131 : allocators. */
132 : hash_map<tree, tree> *allocate_map;
133 :
134 : /* A tree_list of the reduction clauses in this context. This is
135 : only used for checking the consistency of OpenACC reduction
136 : clauses in scan_omp_for and is not guaranteed to contain a valid
137 : value outside of this function. */
138 : tree local_reduction_clauses;
139 :
140 : /* A tree_list of the reduction clauses in outer contexts. This is
141 : only used for checking the consistency of OpenACC reduction
142 : clauses in scan_omp_for and is not guaranteed to contain a valid
143 : value outside of this function. */
144 : tree outer_reduction_clauses;
145 :
146 : /* Nesting depth of this context. Used to beautify error messages re
147 : invalid gotos. The outermost ctx is depth 1, with depth 0 being
148 : reserved for the main body of the function. */
149 : int depth;
150 :
151 : /* True if this parallel directive is nested within another. */
152 : bool is_nested;
153 :
154 : /* True if this construct can be cancelled. */
155 : bool cancellable;
156 :
157 : /* True if lower_omp_1 should look up lastprivate conditional in parent
158 : context. */
159 : bool combined_into_simd_safelen1;
160 :
161 : /* True if there is nested scan context with inclusive clause. */
162 : bool scan_inclusive;
163 :
164 : /* True if there is nested scan context with exclusive clause. */
165 : bool scan_exclusive;
166 :
167 : /* True in the second simd loop of for simd with inscan reductions. */
168 : bool for_simd_scan_phase;
169 :
170 : /* True if there is order(concurrent) clause on the construct. */
171 : bool order_concurrent;
172 :
173 : /* True if there is bind clause on the construct (i.e. a loop construct). */
174 : bool loop_p;
175 :
176 : /* Only used for omp target contexts. True if a teams construct is
177 : strictly nested in it. */
178 : bool teams_nested_p;
179 :
180 : /* Only used for omp target contexts. True if an OpenMP construct other
181 : than teams is strictly nested in it. */
182 : bool nonteams_nested_p;
183 :
184 : /* Candidates for adjusting OpenACC privatization level. */
185 : vec<tree> oacc_privatization_candidates;
186 : };
187 :
188 : static splay_tree all_contexts;
189 : static int taskreg_nesting_level;
190 : static int target_nesting_level;
191 : static bitmap make_addressable_vars;
192 : static bitmap global_nonaddressable_vars;
193 : static vec<omp_context *> taskreg_contexts;
194 : static vec<gomp_task *> task_cpyfns;
195 :
196 : static void scan_omp (gimple_seq *, omp_context *);
197 : static tree scan_omp_1_op (tree *, int *, void *);
198 : static bool omp_maybe_offloaded_ctx (omp_context *ctx);
199 :
200 : #define WALK_SUBSTMTS \
201 : case GIMPLE_BIND: \
202 : case GIMPLE_TRY: \
203 : case GIMPLE_CATCH: \
204 : case GIMPLE_EH_FILTER: \
205 : case GIMPLE_ASSUME: \
206 : case GIMPLE_TRANSACTION: \
207 : /* The sub-statements for these should be walked. */ \
208 : *handled_ops_p = false; \
209 : break;
210 :
211 : /* Return whether CTX represents an OpenACC 'parallel' or 'serial' construct.
212 : (This doesn't include OpenACC 'kernels' decomposed parts.) */
213 :
214 : static bool
215 33677 : is_oacc_parallel_or_serial (omp_context *ctx)
216 : {
217 33677 : enum gimple_code outer_type = gimple_code (ctx->stmt);
218 33677 : return ((outer_type == GIMPLE_OMP_TARGET)
219 33677 : && ((gimple_omp_target_kind (ctx->stmt)
220 : == GF_OMP_TARGET_KIND_OACC_PARALLEL)
221 15335 : || (gimple_omp_target_kind (ctx->stmt)
222 33677 : == GF_OMP_TARGET_KIND_OACC_SERIAL)));
223 : }
224 :
225 : /* Return whether CTX represents an OpenACC 'kernels' construct.
226 : (This doesn't include OpenACC 'kernels' decomposed parts.) */
227 :
228 : static bool
229 49900 : is_oacc_kernels (omp_context *ctx)
230 : {
231 49900 : enum gimple_code outer_type = gimple_code (ctx->stmt);
232 49900 : return ((outer_type == GIMPLE_OMP_TARGET)
233 49900 : && (gimple_omp_target_kind (ctx->stmt)
234 49900 : == GF_OMP_TARGET_KIND_OACC_KERNELS));
235 : }
236 :
237 : /* Return whether CTX represents an OpenACC 'kernels' decomposed part. */
238 :
239 : static bool
240 21645 : is_oacc_kernels_decomposed_part (omp_context *ctx)
241 : {
242 21645 : enum gimple_code outer_type = gimple_code (ctx->stmt);
243 21645 : return ((outer_type == GIMPLE_OMP_TARGET)
244 21645 : && ((gimple_omp_target_kind (ctx->stmt)
245 : == GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED)
246 14552 : || (gimple_omp_target_kind (ctx->stmt)
247 : == GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE)
248 14552 : || (gimple_omp_target_kind (ctx->stmt)
249 21645 : == GF_OMP_TARGET_KIND_OACC_DATA_KERNELS)));
250 : }
251 :
252 : /* Return true if STMT corresponds to an OpenMP target region. */
253 : static bool
254 42990 : is_omp_target (gimple *stmt)
255 : {
256 42990 : if (gimple_code (stmt) == GIMPLE_OMP_TARGET)
257 : {
258 6511 : int kind = gimple_omp_target_kind (stmt);
259 6511 : return (kind == GF_OMP_TARGET_KIND_REGION
260 6511 : || kind == GF_OMP_TARGET_KIND_DATA
261 6511 : || kind == GF_OMP_TARGET_KIND_ENTER_DATA
262 11147 : || kind == GF_OMP_TARGET_KIND_EXIT_DATA);
263 : }
264 : return false;
265 : }
266 :
267 : /* If DECL is the artificial dummy VAR_DECL created for non-static
268 : data member privatization, return the underlying "this" parameter,
269 : otherwise return NULL. */
270 :
271 : tree
272 540868 : omp_member_access_dummy_var (tree decl)
273 : {
274 540868 : if (!VAR_P (decl)
275 331755 : || !DECL_ARTIFICIAL (decl)
276 162893 : || !DECL_IGNORED_P (decl)
277 157465 : || !DECL_HAS_VALUE_EXPR_P (decl)
278 553691 : || !lang_hooks.decls.omp_disregard_value_expr (decl, false))
279 538608 : return NULL_TREE;
280 :
281 2260 : tree v = DECL_VALUE_EXPR (decl);
282 2260 : if (TREE_CODE (v) != COMPONENT_REF)
283 : return NULL_TREE;
284 :
285 16164 : while (1)
286 9197 : switch (TREE_CODE (v))
287 : {
288 6967 : case COMPONENT_REF:
289 6967 : case MEM_REF:
290 6967 : case INDIRECT_REF:
291 6967 : CASE_CONVERT:
292 6967 : case POINTER_PLUS_EXPR:
293 6967 : v = TREE_OPERAND (v, 0);
294 6967 : continue;
295 2211 : case PARM_DECL:
296 2211 : if (DECL_CONTEXT (v) == current_function_decl
297 2211 : && DECL_ARTIFICIAL (v)
298 4422 : && TREE_CODE (TREE_TYPE (v)) == POINTER_TYPE)
299 : return v;
300 : return NULL_TREE;
301 : default:
302 : return NULL_TREE;
303 : }
304 : }
305 :
306 : /* Helper for unshare_and_remap, called through walk_tree. */
307 :
308 : static tree
309 1579 : unshare_and_remap_1 (tree *tp, int *walk_subtrees, void *data)
310 : {
311 1579 : tree *pair = (tree *) data;
312 1579 : if (*tp == pair[0])
313 : {
314 233 : *tp = unshare_expr (pair[1]);
315 233 : *walk_subtrees = 0;
316 : }
317 1346 : else if (IS_TYPE_OR_DECL_P (*tp))
318 244 : *walk_subtrees = 0;
319 1579 : return NULL_TREE;
320 : }
321 :
322 : /* Return unshare_expr (X) with all occurrences of FROM
323 : replaced with TO. */
324 :
325 : static tree
326 150 : unshare_and_remap (tree x, tree from, tree to)
327 : {
328 150 : tree pair[2] = { from, to };
329 150 : x = unshare_expr (x);
330 150 : walk_tree (&x, unshare_and_remap_1, pair, NULL);
331 150 : return x;
332 : }
333 :
334 : /* Convenience function for calling scan_omp_1_op on tree operands. */
335 :
336 : static inline tree
337 339357 : scan_omp_op (tree *tp, omp_context *ctx)
338 : {
339 339357 : struct walk_stmt_info wi;
340 :
341 339357 : memset (&wi, 0, sizeof (wi));
342 339357 : wi.info = ctx;
343 339357 : wi.want_locations = true;
344 :
345 339357 : return walk_tree (tp, scan_omp_1_op, &wi, NULL);
346 : }
347 :
348 : static void lower_omp (gimple_seq *, omp_context *);
349 : static tree lookup_decl_in_outer_ctx (tree, omp_context *);
350 : static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *);
351 :
352 : /* Return true if CTX is for an omp parallel. */
353 :
354 : static inline bool
355 777399 : is_parallel_ctx (omp_context *ctx)
356 : {
357 61299 : return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL;
358 : }
359 :
360 :
361 : /* Return true if CTX is for an omp task. */
362 :
363 : static inline bool
364 236839 : is_task_ctx (omp_context *ctx)
365 : {
366 38944 : return gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
367 : }
368 :
369 :
370 : /* Return true if CTX is for an omp taskloop. */
371 :
372 : static inline bool
373 8193 : is_taskloop_ctx (omp_context *ctx)
374 : {
375 8193 : return gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
376 8193 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_TASKLOOP;
377 : }
378 :
379 :
380 : /* Return true if CTX is for a host omp teams. */
381 :
382 : static inline bool
383 522004 : is_host_teams_ctx (omp_context *ctx)
384 : {
385 522004 : return gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
386 522004 : && gimple_omp_teams_host (as_a <gomp_teams *> (ctx->stmt));
387 : }
388 :
389 : /* Return true if CTX is for an omp parallel or omp task or host omp teams
390 : (the last one is strictly not a task region in OpenMP speak, but we
391 : need to treat it similarly). */
392 :
393 : static inline bool
394 664575 : is_taskreg_ctx (omp_context *ctx)
395 : {
396 664575 : return is_parallel_ctx (ctx) || is_task_ctx (ctx) || is_host_teams_ctx (ctx);
397 : }
398 :
399 : /* Return true if EXPR is variable sized. If IS_REF is true, then
400 : EXPR is assumed to be a reference and the object it refers
401 : to is checked instead. */
402 :
403 : static inline bool
404 907490 : is_variable_sized (const_tree expr, bool is_ref = false)
405 : {
406 907490 : if (is_ref)
407 2178 : expr = TREE_TYPE (expr);
408 907490 : return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr)));
409 : }
410 :
411 : /* Lookup variables. The "maybe" form
412 : allows for the variable form to not have been entered, otherwise we
413 : assert that the variable must have been entered. */
414 :
415 : static inline tree
416 717555 : lookup_decl (tree var, omp_context *ctx)
417 : {
418 1435011 : tree *n = ctx->cb.decl_map->get (var);
419 717555 : return *n;
420 : }
421 :
422 : static inline tree
423 720027 : maybe_lookup_decl (const_tree var, omp_context *ctx)
424 : {
425 720027 : tree *n = ctx->cb.decl_map->get (const_cast<tree> (var));
426 720027 : return n ? *n : NULL_TREE;
427 : }
428 :
429 : static inline tree
430 127629 : lookup_field (tree var, omp_context *ctx)
431 : {
432 127629 : splay_tree_node n;
433 255258 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
434 127629 : return (tree) n->value;
435 : }
436 :
437 : static inline tree
438 169719 : lookup_sfield (splay_tree_key key, omp_context *ctx)
439 : {
440 169719 : splay_tree_node n;
441 169719 : n = splay_tree_lookup (ctx->sfield_map
442 : ? ctx->sfield_map : ctx->field_map, key);
443 169719 : return (tree) n->value;
444 : }
445 :
446 : static inline tree
447 967 : lookup_sfield (tree var, omp_context *ctx)
448 : {
449 967 : return lookup_sfield ((splay_tree_key) var, ctx);
450 : }
451 :
452 : static inline tree
453 221918 : maybe_lookup_field (splay_tree_key key, omp_context *ctx)
454 : {
455 221918 : splay_tree_node n;
456 221918 : n = splay_tree_lookup (ctx->field_map, key);
457 221918 : return n ? (tree) n->value : NULL_TREE;
458 : }
459 :
460 : static inline tree
461 221918 : maybe_lookup_field (tree var, omp_context *ctx)
462 : {
463 2555 : return maybe_lookup_field ((splay_tree_key) var, ctx);
464 : }
465 :
466 : /* Return true if DECL should be copied by pointer. SHARED_CTX is
467 : the parallel context if DECL is to be shared. */
468 :
469 : static bool
470 233057 : use_pointer_for_field (tree decl, omp_context *shared_ctx)
471 : {
472 454538 : if (AGGREGATE_TYPE_P (TREE_TYPE (decl))
473 216274 : || TYPE_ATOMIC (TREE_TYPE (decl))
474 449331 : || POLY_INT_CST_P (DECL_SIZE (decl)))
475 : return true;
476 :
477 : /* We can only use copy-in/copy-out semantics for shared variables
478 : when we know the value is not accessible from an outer scope. */
479 216244 : if (shared_ctx)
480 : {
481 29571 : gcc_assert (!is_gimple_omp_oacc (shared_ctx->stmt));
482 :
483 : /* ??? Trivially accessible from anywhere. But why would we even
484 : be passing an address in this case? Should we simply assert
485 : this to be false, or should we have a cleanup pass that removes
486 : these from the list of mappings? */
487 29571 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, shared_ctx)))
488 : return true;
489 :
490 : /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
491 : without analyzing the expression whether or not its location
492 : is accessible to anyone else. In the case of nested parallel
493 : regions it certainly may be. */
494 29571 : if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl))
495 : return true;
496 :
497 : /* Do not use copy-in/copy-out for variables that have their
498 : address taken. */
499 28657 : if (is_global_var (decl))
500 : {
501 : /* For file scope vars, track whether we've seen them as
502 : non-addressable initially and in that case, keep the same
503 : answer for the duration of the pass, even when they are made
504 : addressable later on e.g. through reduction expansion. Global
505 : variables which weren't addressable before the pass will not
506 : have their privatized copies address taken. See PR91216. */
507 1232 : if (!TREE_ADDRESSABLE (decl))
508 : {
509 479 : if (!global_nonaddressable_vars)
510 74 : global_nonaddressable_vars = BITMAP_ALLOC (NULL);
511 479 : bitmap_set_bit (global_nonaddressable_vars, DECL_UID (decl));
512 : }
513 753 : else if (!global_nonaddressable_vars
514 986 : || !bitmap_bit_p (global_nonaddressable_vars,
515 233 : DECL_UID (decl)))
516 731 : return true;
517 : }
518 27425 : else if (TREE_ADDRESSABLE (decl))
519 : return true;
520 :
521 : /* lower_send_shared_vars only uses copy-in, but not copy-out
522 : for these. */
523 18514 : if (TREE_READONLY (decl)
524 18514 : || ((TREE_CODE (decl) == RESULT_DECL
525 11868 : || TREE_CODE (decl) == PARM_DECL)
526 673 : && DECL_BY_REFERENCE (decl)))
527 : return false;
528 :
529 : /* Disallow copy-in/out in nested parallel if
530 : decl is shared in outer parallel, otherwise
531 : each thread could store the shared variable
532 : in its own copy-in location, making the
533 : variable no longer really shared. */
534 11636 : if (shared_ctx->is_nested)
535 : {
536 2470 : omp_context *up;
537 :
538 5975 : for (up = shared_ctx->outer; up; up = up->outer)
539 5702 : if ((is_taskreg_ctx (up)
540 3643 : || (gimple_code (up->stmt) == GIMPLE_OMP_TARGET
541 411 : && is_gimple_omp_offloaded (up->stmt)))
542 6113 : && maybe_lookup_decl (decl, up))
543 : break;
544 :
545 2470 : if (up)
546 : {
547 2197 : tree c;
548 :
549 2197 : if (gimple_code (up->stmt) == GIMPLE_OMP_TARGET)
550 : {
551 309 : for (c = gimple_omp_target_clauses (up->stmt);
552 2031 : c; c = OMP_CLAUSE_CHAIN (c))
553 1932 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
554 1932 : && OMP_CLAUSE_DECL (c) == decl)
555 : break;
556 : }
557 : else
558 1888 : for (c = gimple_omp_taskreg_clauses (up->stmt);
559 6909 : c; c = OMP_CLAUSE_CHAIN (c))
560 6454 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
561 6454 : && OMP_CLAUSE_DECL (c) == decl)
562 : break;
563 :
564 2197 : if (c)
565 1643 : goto maybe_mark_addressable_and_ret;
566 : }
567 : }
568 :
569 : /* For tasks avoid using copy-in/out. As tasks can be
570 : deferred or executed in different thread, when GOMP_task
571 : returns, the task hasn't necessarily terminated. */
572 9993 : if (is_task_ctx (shared_ctx))
573 : {
574 2195 : tree outer;
575 552 : maybe_mark_addressable_and_ret:
576 2195 : outer = maybe_lookup_decl_in_outer_ctx (decl, shared_ctx);
577 2195 : if (is_gimple_reg (outer) && !omp_member_access_dummy_var (outer))
578 : {
579 : /* Taking address of OUTER in lower_send_shared_vars
580 : might need regimplification of everything that uses the
581 : variable. */
582 984 : if (!make_addressable_vars)
583 491 : make_addressable_vars = BITMAP_ALLOC (NULL);
584 984 : bitmap_set_bit (make_addressable_vars, DECL_UID (outer));
585 984 : TREE_ADDRESSABLE (outer) = 1;
586 : }
587 2195 : return true;
588 : }
589 : }
590 :
591 : return false;
592 : }
593 :
594 : /* Construct a new automatic decl similar to VAR. */
595 :
596 : static tree
597 230114 : omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx)
598 : {
599 230114 : tree copy = copy_var_decl (var, name, type);
600 :
601 230114 : DECL_CONTEXT (copy) = current_function_decl;
602 :
603 230114 : if (ctx)
604 : {
605 229833 : DECL_CHAIN (copy) = ctx->block_vars;
606 229833 : ctx->block_vars = copy;
607 : }
608 : else
609 281 : record_vars (copy);
610 :
611 : /* If VAR is listed in make_addressable_vars, it wasn't
612 : originally addressable, but was only later made so.
613 : We don't need to take address of privatizations
614 : from that var. */
615 230114 : if (TREE_ADDRESSABLE (var)
616 230114 : && ((make_addressable_vars
617 3469 : && bitmap_bit_p (make_addressable_vars, DECL_UID (var)))
618 39791 : || (global_nonaddressable_vars
619 515 : && bitmap_bit_p (global_nonaddressable_vars, DECL_UID (var)))))
620 2079 : TREE_ADDRESSABLE (copy) = 0;
621 :
622 230114 : return copy;
623 : }
624 :
625 : static tree
626 230114 : omp_copy_decl_1 (tree var, omp_context *ctx)
627 : {
628 230114 : return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx);
629 : }
630 :
631 : /* Build tree nodes to access the field for VAR on the receiver side. */
632 :
633 : static tree
634 123171 : build_receiver_ref (tree var, bool by_ref, omp_context *ctx)
635 : {
636 123171 : tree x, field = lookup_field (var, ctx);
637 :
638 : /* If the receiver record type was remapped in the child function,
639 : remap the field into the new record type. */
640 123171 : x = maybe_lookup_field (field, ctx);
641 123171 : if (x != NULL)
642 11245 : field = x;
643 :
644 123171 : x = build_simple_mem_ref (ctx->receiver_decl);
645 123171 : TREE_THIS_NOTRAP (x) = 1;
646 123171 : x = omp_build_component_ref (x, field);
647 123171 : if (by_ref)
648 : {
649 36260 : x = build_simple_mem_ref (x);
650 36260 : TREE_THIS_NOTRAP (x) = 1;
651 : }
652 :
653 123171 : return x;
654 : }
655 :
656 : /* Build tree nodes to access VAR in the scope outer to CTX. In the case
657 : of a parallel, this is a component reference; for workshare constructs
658 : this is some variable. */
659 :
660 : static tree
661 114639 : build_outer_var_ref (tree var, omp_context *ctx,
662 : enum omp_clause_code code = OMP_CLAUSE_ERROR)
663 : {
664 114639 : tree x;
665 114639 : omp_context *outer = ctx->outer;
666 115028 : for (; outer; outer = outer->outer)
667 : {
668 82594 : if (gimple_code (outer->stmt) == GIMPLE_OMP_TASKGROUP)
669 373 : continue;
670 82237 : if (gimple_code (outer->stmt) == GIMPLE_OMP_SCOPE
671 82221 : && !maybe_lookup_decl (var, outer))
672 16 : continue;
673 : break;
674 : }
675 :
676 114639 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx)))
677 : x = var;
678 106498 : else if (is_variable_sized (var))
679 : {
680 47 : x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0);
681 47 : x = build_outer_var_ref (x, ctx, code);
682 47 : x = build_simple_mem_ref (x);
683 : }
684 106451 : else if (is_taskreg_ctx (ctx))
685 : {
686 63850 : bool by_ref = use_pointer_for_field (var, NULL);
687 63850 : x = build_receiver_ref (var, by_ref, ctx);
688 : }
689 42601 : else if ((gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
690 41228 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
691 14133 : || ctx->loop_p
692 13360 : || code == OMP_CLAUSE_ALLOCATE
693 55705 : || (code == OMP_CLAUSE_PRIVATE
694 40 : && (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
695 : || gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS
696 : || gimple_code (ctx->stmt) == GIMPLE_OMP_SINGLE)))
697 : {
698 : /* #pragma omp simd isn't a worksharing construct, and can reference
699 : even private vars in its linear etc. clauses.
700 : Similarly for OMP_CLAUSE_PRIVATE with outer ref, that can refer
701 : to private vars in all worksharing constructs. */
702 29517 : x = NULL_TREE;
703 29517 : if (outer && is_taskreg_ctx (outer))
704 1002 : x = lookup_decl (var, outer);
705 28515 : else if (outer)
706 22321 : x = maybe_lookup_decl_in_outer_ctx (var, ctx);
707 23323 : if (x == NULL_TREE)
708 : x = var;
709 : }
710 13084 : else if (code == OMP_CLAUSE_LASTPRIVATE && is_taskloop_ctx (ctx))
711 : {
712 689 : gcc_assert (outer);
713 689 : splay_tree_node n
714 1378 : = splay_tree_lookup (outer->field_map,
715 689 : (splay_tree_key) &DECL_UID (var));
716 689 : if (n == NULL)
717 : {
718 610 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, outer)))
719 : x = var;
720 : else
721 384 : x = lookup_decl (var, outer);
722 : }
723 : else
724 : {
725 79 : tree field = (tree) n->value;
726 : /* If the receiver record type was remapped in the child function,
727 : remap the field into the new record type. */
728 79 : x = maybe_lookup_field (field, outer);
729 79 : if (x != NULL)
730 0 : field = x;
731 :
732 79 : x = build_simple_mem_ref (outer->receiver_decl);
733 79 : x = omp_build_component_ref (x, field);
734 79 : if (use_pointer_for_field (var, outer))
735 59 : x = build_simple_mem_ref (x);
736 : }
737 : }
738 12395 : else if (outer)
739 12203 : x = lookup_decl (var, outer);
740 192 : else if (omp_privatize_by_reference (var))
741 : /* This can happen with orphaned constructs. If var is reference, it is
742 : possible it is shared and as such valid. */
743 : x = var;
744 94 : else if (omp_member_access_dummy_var (var))
745 : x = var;
746 : else
747 0 : gcc_unreachable ();
748 :
749 114639 : if (x == var)
750 : {
751 16326 : tree t = omp_member_access_dummy_var (var);
752 16326 : if (t)
753 : {
754 220 : x = DECL_VALUE_EXPR (var);
755 220 : tree o = maybe_lookup_decl_in_outer_ctx (t, ctx);
756 220 : if (o != t)
757 54 : x = unshare_and_remap (x, t, o);
758 : else
759 166 : x = unshare_expr (x);
760 : }
761 : }
762 :
763 114639 : if (omp_privatize_by_reference (var))
764 3092 : x = build_simple_mem_ref (x);
765 :
766 114639 : return x;
767 : }
768 :
769 : /* Build tree nodes to access the field for VAR on the sender side. */
770 :
771 : static tree
772 168673 : build_sender_ref (splay_tree_key key, omp_context *ctx)
773 : {
774 168673 : tree field = lookup_sfield (key, ctx);
775 168673 : tree tmp = ctx->sender_decl;
776 168673 : if (POINTER_TYPE_P (TREE_TYPE (tmp)))
777 451 : tmp = build_fold_indirect_ref (tmp);
778 168673 : return omp_build_component_ref (tmp, field);
779 : }
780 :
781 : static tree
782 164519 : build_sender_ref (tree var, omp_context *ctx)
783 : {
784 0 : return build_sender_ref ((splay_tree_key) var, ctx);
785 : }
786 :
787 : /* Add a new field for VAR inside the structure CTX. If BY_REF is true,
788 : use a pointer to the VAR rather than VAR itself.
789 : MASK is a bit mask of other options. Bits are interpreted as:
790 : 1: Install VAR in ctx->field_map.
791 : 2: Install VAR in ctx->sfield_map.
792 : 4: VAR is an array, convert it to a pointer.
793 : 8: Use DECL_UID (VAR) instead of VAR as key.
794 : 16: Use DECL_NAME (VAR) instead of VAR as key.
795 : 32: Don't dereference omp_is_reference types.
796 : KEY_EXPR allows specifying something other than VAR as the lookup key.
797 : If specified, it also overrides the 8 and 16 MASK bits. */
798 :
799 : static void
800 131520 : install_var_field (tree var, bool by_ref, int mask, omp_context *ctx,
801 : tree key_expr = NULL_TREE)
802 : {
803 131520 : tree field, type, sfield = NULL_TREE;
804 131520 : splay_tree_key key = (splay_tree_key) var;
805 :
806 131520 : if (key_expr)
807 : /* Allow caller to explicitly set the expression used as the key. */
808 44775 : key = (splay_tree_key) key_expr;
809 : else
810 : {
811 86745 : if ((mask & 16) != 0)
812 : {
813 624 : key = (splay_tree_key) &DECL_NAME (var);
814 624 : gcc_checking_assert (key != (splay_tree_key) var);
815 : }
816 86745 : if ((mask & 8) != 0)
817 : {
818 1619 : key = (splay_tree_key) &DECL_UID (var);
819 1619 : gcc_checking_assert (key != (splay_tree_key) var);
820 : }
821 : }
822 131520 : gcc_assert ((mask & 1) == 0
823 : || !splay_tree_lookup (ctx->field_map, key));
824 131520 : gcc_assert ((mask & 2) == 0 || !ctx->sfield_map
825 : || !splay_tree_lookup (ctx->sfield_map, key));
826 131520 : gcc_assert ((mask & 3) == 3
827 : || !is_gimple_omp_oacc (ctx->stmt));
828 :
829 131520 : type = TREE_TYPE (var);
830 131520 : if ((mask & 16) != 0)
831 624 : type = lang_hooks.decls.omp_array_data (var, true);
832 :
833 : /* Prevent redeclaring the var in the split-off function with a restrict
834 : pointer type. Note that we only clear type itself, restrict qualifiers in
835 : the pointed-to type will be ignored by points-to analysis. */
836 131520 : if (POINTER_TYPE_P (type)
837 131520 : && TYPE_RESTRICT (type))
838 7058 : type = build_qualified_type (type, TYPE_QUALS (type) & ~TYPE_QUAL_RESTRICT);
839 :
840 131520 : if (mask & 4)
841 : {
842 425 : gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
843 425 : type = build_pointer_type (build_pointer_type (type));
844 : }
845 131095 : else if (by_ref)
846 64157 : type = build_pointer_type (type);
847 66938 : else if ((mask & (32 | 3)) == 1
848 66938 : && omp_privatize_by_reference (var))
849 162 : type = TREE_TYPE (type);
850 :
851 131520 : field = build_decl (DECL_SOURCE_LOCATION (var),
852 131520 : FIELD_DECL, DECL_NAME (var), type);
853 :
854 : /* Remember what variable this field was created for. This does have a
855 : side effect of making dwarf2out ignore this member, so for helpful
856 : debugging we clear it later in delete_omp_context. */
857 131520 : DECL_ABSTRACT_ORIGIN (field) = var;
858 131520 : if ((mask & 16) == 0 && type == TREE_TYPE (var))
859 : {
860 62399 : SET_DECL_ALIGN (field, DECL_ALIGN (var));
861 62399 : DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
862 62399 : TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
863 : }
864 : else
865 69121 : SET_DECL_ALIGN (field, TYPE_ALIGN (type));
866 :
867 131520 : if ((mask & 3) == 3)
868 : {
869 130468 : insert_field_into_struct (ctx->record_type, field);
870 130468 : if (ctx->srecord_type)
871 : {
872 680 : sfield = build_decl (DECL_SOURCE_LOCATION (var),
873 680 : FIELD_DECL, DECL_NAME (var), type);
874 680 : DECL_ABSTRACT_ORIGIN (sfield) = var;
875 680 : SET_DECL_ALIGN (sfield, DECL_ALIGN (field));
876 680 : DECL_USER_ALIGN (sfield) = DECL_USER_ALIGN (field);
877 680 : TREE_THIS_VOLATILE (sfield) = TREE_THIS_VOLATILE (field);
878 680 : insert_field_into_struct (ctx->srecord_type, sfield);
879 : }
880 : }
881 : else
882 : {
883 1052 : if (ctx->srecord_type == NULL_TREE)
884 : {
885 544 : tree t;
886 :
887 544 : ctx->srecord_type = lang_hooks.types.make_type (RECORD_TYPE);
888 544 : ctx->sfield_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
889 2027 : for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t))
890 : {
891 1483 : sfield = build_decl (DECL_SOURCE_LOCATION (t),
892 1483 : FIELD_DECL, DECL_NAME (t), TREE_TYPE (t));
893 1483 : DECL_ABSTRACT_ORIGIN (sfield) = DECL_ABSTRACT_ORIGIN (t);
894 1483 : insert_field_into_struct (ctx->srecord_type, sfield);
895 2966 : splay_tree_insert (ctx->sfield_map,
896 1483 : (splay_tree_key) DECL_ABSTRACT_ORIGIN (t),
897 : (splay_tree_value) sfield);
898 : }
899 : }
900 1052 : sfield = field;
901 1052 : insert_field_into_struct ((mask & 1) ? ctx->record_type
902 : : ctx->srecord_type, field);
903 : }
904 :
905 131520 : if (mask & 1)
906 131133 : splay_tree_insert (ctx->field_map, key, (splay_tree_value) field);
907 131520 : if ((mask & 2) && ctx->sfield_map)
908 1067 : splay_tree_insert (ctx->sfield_map, key, (splay_tree_value) sfield);
909 131520 : }
910 :
911 : static tree
912 229683 : install_var_local (tree var, omp_context *ctx)
913 : {
914 229683 : tree new_var = omp_copy_decl_1 (var, ctx);
915 229683 : insert_decl_map (&ctx->cb, var, new_var);
916 229683 : return new_var;
917 : }
918 :
919 : /* Adjust the replacement for DECL in CTX for the new context. This means
920 : copying the DECL_VALUE_EXPR, and fixing up the type. */
921 :
922 : static void
923 204831 : fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug)
924 : {
925 204831 : tree new_decl, size;
926 :
927 204831 : new_decl = lookup_decl (decl, ctx);
928 :
929 204831 : TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), &ctx->cb);
930 :
931 408529 : if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug)
932 205020 : && DECL_HAS_VALUE_EXPR_P (decl))
933 : {
934 1322 : tree ve = DECL_VALUE_EXPR (decl);
935 1322 : walk_tree (&ve, copy_tree_body_r, &ctx->cb, NULL);
936 1322 : SET_DECL_VALUE_EXPR (new_decl, ve);
937 1322 : DECL_HAS_VALUE_EXPR_P (new_decl) = 1;
938 : }
939 :
940 204831 : if (!TREE_CONSTANT (DECL_SIZE (new_decl)))
941 : {
942 1133 : size = remap_decl (DECL_SIZE (decl), &ctx->cb);
943 1133 : if (size == error_mark_node)
944 316 : size = TYPE_SIZE (TREE_TYPE (new_decl));
945 1133 : DECL_SIZE (new_decl) = size;
946 :
947 1133 : size = remap_decl (DECL_SIZE_UNIT (decl), &ctx->cb);
948 1133 : if (size == error_mark_node)
949 303 : size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl));
950 1133 : DECL_SIZE_UNIT (new_decl) = size;
951 : }
952 204831 : }
953 :
954 : /* The callback for remap_decl. Search all containing contexts for a
955 : mapping of the variable; this avoids having to duplicate the splay
956 : tree ahead of time. We know a mapping doesn't already exist in the
957 : given context. Create new mappings to implement default semantics. */
958 :
959 : static tree
960 464607 : omp_copy_decl (tree var, copy_body_data *cb)
961 : {
962 464607 : omp_context *ctx = (omp_context *) cb;
963 464607 : tree new_var;
964 :
965 464607 : if (TREE_CODE (var) == LABEL_DECL)
966 : {
967 189033 : if (FORCED_LABEL (var) || DECL_NONLOCAL (var))
968 : return var;
969 189027 : new_var = create_artificial_label (DECL_SOURCE_LOCATION (var));
970 189027 : DECL_CONTEXT (new_var) = current_function_decl;
971 189027 : insert_decl_map (&ctx->cb, var, new_var);
972 189027 : return new_var;
973 : }
974 :
975 357419 : while (!is_taskreg_ctx (ctx))
976 : {
977 338179 : ctx = ctx->outer;
978 338179 : if (ctx == NULL)
979 : return var;
980 279911 : new_var = maybe_lookup_decl (var, ctx);
981 279911 : if (new_var)
982 : return new_var;
983 : }
984 :
985 19240 : if (is_global_var (var) || decl_function_context (var) != ctx->cb.src_fn)
986 13383 : return var;
987 :
988 5857 : return error_mark_node;
989 : }
990 :
991 : /* Create a new context, with OUTER_CTX being the surrounding context. */
992 :
993 : static omp_context *
994 137898 : new_omp_context (gimple *stmt, omp_context *outer_ctx)
995 : {
996 137898 : omp_context *ctx = XCNEW (omp_context);
997 :
998 137898 : splay_tree_insert (all_contexts, (splay_tree_key) stmt,
999 : (splay_tree_value) ctx);
1000 137898 : ctx->stmt = stmt;
1001 :
1002 137898 : if (outer_ctx)
1003 : {
1004 74563 : ctx->outer = outer_ctx;
1005 74563 : ctx->cb = outer_ctx->cb;
1006 74563 : ctx->cb.block = NULL;
1007 74563 : ctx->depth = outer_ctx->depth + 1;
1008 : }
1009 : else
1010 : {
1011 63335 : ctx->cb.src_fn = current_function_decl;
1012 63335 : ctx->cb.dst_fn = current_function_decl;
1013 63335 : ctx->cb.src_node = cgraph_node::get (current_function_decl);
1014 63335 : gcc_checking_assert (ctx->cb.src_node);
1015 63335 : ctx->cb.dst_node = ctx->cb.src_node;
1016 63335 : ctx->cb.src_cfun = cfun;
1017 63335 : ctx->cb.copy_decl = omp_copy_decl;
1018 63335 : ctx->cb.eh_lp_nr = 0;
1019 63335 : ctx->cb.transform_call_graph_edges = CB_CGE_MOVE;
1020 63335 : ctx->cb.adjust_array_error_bounds = true;
1021 63335 : ctx->cb.dont_remap_vla_if_no_change = true;
1022 63335 : ctx->depth = 1;
1023 : }
1024 :
1025 137898 : ctx->cb.decl_map = new hash_map<tree, tree>;
1026 :
1027 137898 : return ctx;
1028 : }
1029 :
1030 : static gimple_seq maybe_catch_exception (gimple_seq);
1031 :
1032 : /* Finalize task copyfn. */
1033 :
1034 : static void
1035 504 : finalize_task_copyfn (gomp_task *task_stmt)
1036 : {
1037 504 : struct function *child_cfun;
1038 504 : tree child_fn;
1039 504 : gimple_seq seq = NULL, new_seq;
1040 504 : gbind *bind;
1041 :
1042 504 : child_fn = gimple_omp_task_copy_fn (task_stmt);
1043 504 : if (child_fn == NULL_TREE)
1044 0 : return;
1045 :
1046 504 : child_cfun = DECL_STRUCT_FUNCTION (child_fn);
1047 504 : DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
1048 :
1049 504 : push_cfun (child_cfun);
1050 504 : bind = gimplify_body (child_fn, false);
1051 504 : gimple_seq_add_stmt (&seq, bind);
1052 504 : new_seq = maybe_catch_exception (seq);
1053 504 : if (new_seq != seq)
1054 : {
1055 332 : bind = gimple_build_bind (NULL, new_seq, NULL);
1056 332 : seq = NULL;
1057 332 : gimple_seq_add_stmt (&seq, bind);
1058 : }
1059 504 : gimple_set_body (child_fn, seq);
1060 504 : pop_cfun ();
1061 :
1062 : /* Inform the callgraph about the new function. */
1063 504 : cgraph_node *node = cgraph_node::get_create (child_fn);
1064 504 : node->parallelized_function = 1;
1065 504 : cgraph_node::add_new_function (child_fn, false);
1066 : }
1067 :
1068 : /* Destroy a omp_context data structures. Called through the splay tree
1069 : value delete callback. */
1070 :
1071 : static void
1072 137892 : delete_omp_context (splay_tree_value value)
1073 : {
1074 137892 : omp_context *ctx = (omp_context *) value;
1075 :
1076 275784 : delete ctx->cb.decl_map;
1077 :
1078 137892 : if (ctx->field_map)
1079 69230 : splay_tree_delete (ctx->field_map);
1080 137892 : if (ctx->sfield_map)
1081 544 : splay_tree_delete (ctx->sfield_map);
1082 :
1083 : /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1084 : it produces corrupt debug information. */
1085 137892 : if (ctx->record_type)
1086 : {
1087 53503 : tree t;
1088 222842 : for (t = TYPE_FIELDS (ctx->record_type); t ; t = DECL_CHAIN (t))
1089 169339 : DECL_ABSTRACT_ORIGIN (t) = NULL;
1090 : }
1091 137892 : if (ctx->srecord_type)
1092 : {
1093 544 : tree t;
1094 3094 : for (t = TYPE_FIELDS (ctx->srecord_type); t ; t = DECL_CHAIN (t))
1095 2550 : DECL_ABSTRACT_ORIGIN (t) = NULL;
1096 : }
1097 :
1098 137892 : if (ctx->task_reduction_map)
1099 : {
1100 1173 : ctx->task_reductions.release ();
1101 2346 : delete ctx->task_reduction_map;
1102 : }
1103 :
1104 138165 : delete ctx->lastprivate_conditional_map;
1105 139280 : delete ctx->allocate_map;
1106 :
1107 137892 : XDELETE (ctx);
1108 137892 : }
1109 :
1110 : /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1111 : context. */
1112 :
1113 : static void
1114 36892 : fixup_child_record_type (omp_context *ctx)
1115 : {
1116 36892 : tree f, type = ctx->record_type;
1117 :
1118 36892 : if (!ctx->receiver_decl)
1119 : return;
1120 : /* ??? It isn't sufficient to just call remap_type here, because
1121 : variably_modified_type_p doesn't work the way we expect for
1122 : record types. Testing each field for whether it needs remapping
1123 : and creating a new record by hand works, however. */
1124 161986 : for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
1125 126375 : if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
1126 : break;
1127 36892 : if (f)
1128 : {
1129 1281 : tree name, new_fields = NULL;
1130 :
1131 1281 : type = lang_hooks.types.make_type (RECORD_TYPE);
1132 1281 : name = DECL_NAME (TYPE_NAME (ctx->record_type));
1133 1281 : name = build_decl (DECL_SOURCE_LOCATION (ctx->receiver_decl),
1134 : TYPE_DECL, name, type);
1135 1281 : TYPE_NAME (type) = name;
1136 :
1137 13520 : for (f = TYPE_FIELDS (ctx->record_type); f ; f = DECL_CHAIN (f))
1138 : {
1139 12239 : tree new_f = copy_node (f);
1140 12239 : DECL_CONTEXT (new_f) = type;
1141 12239 : TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &ctx->cb);
1142 12239 : DECL_CHAIN (new_f) = new_fields;
1143 12239 : walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &ctx->cb, NULL);
1144 12239 : walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r,
1145 : &ctx->cb, NULL);
1146 12239 : walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
1147 : &ctx->cb, NULL);
1148 12239 : new_fields = new_f;
1149 :
1150 : /* Arrange to be able to look up the receiver field
1151 : given the sender field. */
1152 12239 : splay_tree_insert (ctx->field_map, (splay_tree_key) f,
1153 : (splay_tree_value) new_f);
1154 : }
1155 1281 : TYPE_FIELDS (type) = nreverse (new_fields);
1156 1281 : layout_type (type);
1157 : }
1158 :
1159 : /* In a target region we never modify any of the pointers in *.omp_data_i,
1160 : so attempt to help the optimizers. */
1161 36892 : if (is_gimple_omp_offloaded (ctx->stmt))
1162 17627 : type = build_qualified_type (type, TYPE_QUAL_CONST);
1163 :
1164 36892 : TREE_TYPE (ctx->receiver_decl)
1165 110676 : = build_qualified_type (flexible_array_type_p (type)
1166 107 : ? build_pointer_type (type)
1167 36785 : : build_reference_type (type), TYPE_QUAL_RESTRICT);
1168 : }
1169 :
1170 : /* Instantiate decls as necessary in CTX to satisfy the data sharing
1171 : specified by CLAUSES. */
1172 :
1173 : static void
1174 130045 : scan_sharing_clauses (tree clauses, omp_context *ctx)
1175 : {
1176 130045 : tree c, decl;
1177 130045 : bool scan_array_reductions = false;
1178 130045 : bool flex_array_ptr = false;
1179 :
1180 551008 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1181 420963 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_ALLOCATE
1182 420963 : && (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) == NULL_TREE
1183 : /* omp_default_mem_alloc is 1 */
1184 1297 : || !integer_onep (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c))
1185 572 : || OMP_CLAUSE_ALLOCATE_ALIGN (c) != NULL_TREE))
1186 : {
1187 : /* The allocate clauses that appear on a target construct or on
1188 : constructs in a target region must specify an allocator expression
1189 : unless a requires directive with the dynamic_allocators clause
1190 : is present in the same compilation unit. */
1191 1865 : if (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) == NULL_TREE
1192 1096 : && ((omp_requires_mask & OMP_REQUIRES_DYNAMIC_ALLOCATORS) == 0)
1193 2949 : && omp_maybe_offloaded_ctx (ctx))
1194 14 : error_at (OMP_CLAUSE_LOCATION (c), "%<allocate%> clause must"
1195 : " specify an allocator here");
1196 1865 : if ((omp_requires_mask & OMP_REQUIRES_DYNAMIC_ALLOCATORS) == 0
1197 1853 : && OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) != NULL_TREE
1198 769 : && DECL_P (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c))
1199 2374 : && !DECL_ARTIFICIAL (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c)))
1200 : {
1201 279 : tree alloc2 = OMP_CLAUSE_ALLOCATE_ALLOCATOR (c);
1202 279 : if (TREE_CODE (alloc2) == MEM_REF
1203 279 : || TREE_CODE (alloc2) == INDIRECT_REF)
1204 0 : alloc2 = TREE_OPERAND (alloc2, 0);
1205 279 : omp_context *ctx2 = ctx;
1206 740 : for (; ctx2; ctx2 = ctx2->outer)
1207 480 : if (is_gimple_omp_offloaded (ctx2->stmt))
1208 : break;
1209 279 : if (ctx2 != NULL)
1210 : {
1211 19 : tree c2 = gimple_omp_target_clauses (ctx2->stmt);
1212 108 : for (; c2; c2 = OMP_CLAUSE_CHAIN (c2))
1213 106 : if (OMP_CLAUSE_CODE (c2) == OMP_CLAUSE_USES_ALLOCATORS
1214 123 : && operand_equal_p (
1215 17 : alloc2, OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c2)))
1216 : break;
1217 19 : if (c2 == NULL_TREE)
1218 4 : error_at (EXPR_LOC_OR_LOC (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c),
1219 : OMP_CLAUSE_LOCATION (c)),
1220 : "allocator %qE in %<allocate%> clause inside a "
1221 : "target region must be specified in an "
1222 : "%<uses_allocators%> clause on the %<target%> "
1223 : "directive", alloc2);
1224 : }
1225 : }
1226 1865 : if (ctx->allocate_map == NULL)
1227 1388 : ctx->allocate_map = new hash_map<tree, tree>;
1228 1865 : tree val = integer_zero_node;
1229 1865 : if (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c))
1230 769 : val = OMP_CLAUSE_ALLOCATE_ALLOCATOR (c);
1231 1865 : if (OMP_CLAUSE_ALLOCATE_ALIGN (c))
1232 295 : val = build_tree_list (val, OMP_CLAUSE_ALLOCATE_ALIGN (c));
1233 1865 : ctx->allocate_map->put (OMP_CLAUSE_DECL (c), val);
1234 : }
1235 :
1236 551008 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1237 : {
1238 420963 : bool by_ref;
1239 :
1240 420963 : switch (OMP_CLAUSE_CODE (c))
1241 : {
1242 73306 : case OMP_CLAUSE_PRIVATE:
1243 73306 : decl = OMP_CLAUSE_DECL (c);
1244 73306 : if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
1245 165 : goto do_private;
1246 73141 : else if (!is_variable_sized (decl))
1247 72778 : install_var_local (decl, ctx);
1248 : break;
1249 :
1250 46884 : case OMP_CLAUSE_SHARED:
1251 46884 : decl = OMP_CLAUSE_DECL (c);
1252 46884 : if (ctx->allocate_map && ctx->allocate_map->get (decl))
1253 37 : ctx->allocate_map->remove (decl);
1254 : /* Ignore shared directives in teams construct inside of
1255 : target construct. */
1256 46884 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
1257 46884 : && !is_host_teams_ctx (ctx))
1258 : {
1259 : /* Global variables don't need to be copied,
1260 : the receiver side will use them directly. */
1261 11862 : tree odecl = maybe_lookup_decl_in_outer_ctx (decl, ctx);
1262 11862 : if (is_global_var (odecl))
1263 : break;
1264 8133 : insert_decl_map (&ctx->cb, decl, odecl);
1265 8133 : break;
1266 : }
1267 35022 : gcc_assert (is_taskreg_ctx (ctx));
1268 35022 : gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl))
1269 : || !is_variable_sized (decl));
1270 : /* Global variables don't need to be copied,
1271 : the receiver side will use them directly. */
1272 35022 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1273 : break;
1274 28957 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
1275 : {
1276 86 : use_pointer_for_field (decl, ctx);
1277 86 : break;
1278 : }
1279 28871 : by_ref = use_pointer_for_field (decl, NULL);
1280 55234 : if ((! TREE_READONLY (decl) && !OMP_CLAUSE_SHARED_READONLY (c))
1281 19334 : || TREE_ADDRESSABLE (decl)
1282 18977 : || by_ref
1283 47848 : || omp_privatize_by_reference (decl))
1284 : {
1285 12048 : by_ref = use_pointer_for_field (decl, ctx);
1286 12048 : install_var_field (decl, by_ref, 3, ctx);
1287 12048 : install_var_local (decl, ctx);
1288 12048 : break;
1289 : }
1290 : /* We don't need to copy const scalar vars back. */
1291 16823 : OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE);
1292 16823 : goto do_private;
1293 :
1294 14566 : case OMP_CLAUSE_REDUCTION:
1295 : /* Collect 'reduction' clauses on OpenACC compute construct. */
1296 14566 : if (is_gimple_omp_oacc (ctx->stmt)
1297 14566 : && is_gimple_omp_offloaded (ctx->stmt))
1298 : {
1299 : /* No 'reduction' clauses on OpenACC 'kernels'. */
1300 790 : gcc_checking_assert (!is_oacc_kernels (ctx));
1301 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
1302 790 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
1303 :
1304 790 : ctx->local_reduction_clauses
1305 790 : = tree_cons (NULL, c, ctx->local_reduction_clauses);
1306 : }
1307 : /* FALLTHRU */
1308 :
1309 16640 : case OMP_CLAUSE_IN_REDUCTION:
1310 16640 : decl = OMP_CLAUSE_DECL (c);
1311 16640 : if (ctx->allocate_map
1312 16640 : && ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1313 389 : && (OMP_CLAUSE_REDUCTION_INSCAN (c)
1314 389 : || OMP_CLAUSE_REDUCTION_TASK (c)))
1315 397 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
1316 389 : || is_task_ctx (ctx)))
1317 : {
1318 : /* For now. */
1319 17 : if (ctx->allocate_map->get (decl))
1320 17 : ctx->allocate_map->remove (decl);
1321 : }
1322 16640 : if (TREE_CODE (decl) == MEM_REF)
1323 : {
1324 2366 : tree t = TREE_OPERAND (decl, 0);
1325 2366 : if (TREE_CODE (t) == POINTER_PLUS_EXPR)
1326 436 : t = TREE_OPERAND (t, 0);
1327 2366 : if (INDIRECT_REF_P (t)
1328 2272 : || TREE_CODE (t) == ADDR_EXPR)
1329 1378 : t = TREE_OPERAND (t, 0);
1330 2366 : if (is_omp_target (ctx->stmt))
1331 : {
1332 96 : if (is_variable_sized (t))
1333 : {
1334 32 : gcc_assert (DECL_HAS_VALUE_EXPR_P (t));
1335 32 : t = DECL_VALUE_EXPR (t);
1336 32 : gcc_assert (INDIRECT_REF_P (t));
1337 32 : t = TREE_OPERAND (t, 0);
1338 32 : gcc_assert (DECL_P (t));
1339 : }
1340 96 : tree at = t;
1341 96 : if (ctx->outer)
1342 96 : scan_omp_op (&at, ctx->outer);
1343 96 : tree nt = omp_copy_decl_1 (at, ctx->outer);
1344 192 : splay_tree_insert (ctx->field_map,
1345 96 : (splay_tree_key) &DECL_CONTEXT (t),
1346 : (splay_tree_value) nt);
1347 96 : if (at != t)
1348 96 : splay_tree_insert (ctx->field_map,
1349 48 : (splay_tree_key) &DECL_CONTEXT (at),
1350 : (splay_tree_value) nt);
1351 96 : break;
1352 : }
1353 2270 : install_var_local (t, ctx);
1354 2270 : if (is_taskreg_ctx (ctx)
1355 1822 : && (!is_global_var (maybe_lookup_decl_in_outer_ctx (t, ctx))
1356 387 : || (is_task_ctx (ctx)
1357 310 : && (TREE_CODE (TREE_TYPE (t)) == POINTER_TYPE
1358 234 : || (TREE_CODE (TREE_TYPE (t)) == REFERENCE_TYPE
1359 0 : && (TREE_CODE (TREE_TYPE (TREE_TYPE (t)))
1360 : == POINTER_TYPE)))))
1361 1511 : && !is_variable_sized (t)
1362 3722 : && (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
1363 617 : || (!OMP_CLAUSE_REDUCTION_TASK (c)
1364 578 : && !is_task_ctx (ctx))))
1365 : {
1366 1335 : by_ref = use_pointer_for_field (t, NULL);
1367 1335 : if (is_task_ctx (ctx)
1368 835 : && TREE_CODE (TREE_TYPE (t)) == REFERENCE_TYPE
1369 1495 : && TREE_CODE (TREE_TYPE (TREE_TYPE (t))) == POINTER_TYPE)
1370 : {
1371 32 : install_var_field (t, false, 1, ctx);
1372 32 : install_var_field (t, by_ref, 2, ctx);
1373 : }
1374 : else
1375 1303 : install_var_field (t, by_ref, 3, ctx);
1376 : }
1377 : break;
1378 : }
1379 14274 : if (is_omp_target (ctx->stmt))
1380 : {
1381 335 : tree at = decl;
1382 335 : if (ctx->outer)
1383 54 : scan_omp_op (&at, ctx->outer);
1384 335 : tree nt = omp_copy_decl_1 (at, ctx->outer);
1385 670 : splay_tree_insert (ctx->field_map,
1386 335 : (splay_tree_key) &DECL_CONTEXT (decl),
1387 : (splay_tree_value) nt);
1388 335 : if (at != decl)
1389 24 : splay_tree_insert (ctx->field_map,
1390 12 : (splay_tree_key) &DECL_CONTEXT (at),
1391 : (splay_tree_value) nt);
1392 335 : break;
1393 : }
1394 13939 : if (is_task_ctx (ctx)
1395 26808 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1396 12869 : && OMP_CLAUSE_REDUCTION_TASK (c)
1397 381 : && is_parallel_ctx (ctx)))
1398 : {
1399 : /* Global variables don't need to be copied,
1400 : the receiver side will use them directly. */
1401 1150 : if (!is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1402 : {
1403 462 : by_ref = use_pointer_for_field (decl, ctx);
1404 462 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION)
1405 304 : install_var_field (decl, by_ref, 3, ctx);
1406 : }
1407 1150 : install_var_local (decl, ctx);
1408 1150 : break;
1409 : }
1410 12789 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1411 12789 : && OMP_CLAUSE_REDUCTION_TASK (c))
1412 : {
1413 301 : install_var_local (decl, ctx);
1414 301 : break;
1415 : }
1416 12488 : goto do_private;
1417 :
1418 22747 : case OMP_CLAUSE_LASTPRIVATE:
1419 : /* Let the corresponding firstprivate clause create
1420 : the variable. */
1421 22747 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1422 : break;
1423 : /* FALLTHRU */
1424 :
1425 59407 : case OMP_CLAUSE_FIRSTPRIVATE:
1426 59407 : case OMP_CLAUSE_LINEAR:
1427 59407 : decl = OMP_CLAUSE_DECL (c);
1428 89271 : do_private:
1429 89271 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1430 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR
1431 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1432 45834 : && is_gimple_omp_offloaded (ctx->stmt))
1433 : {
1434 16274 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1435 16274 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR
1436 239 : && lang_hooks.decls.omp_array_data (decl, true)))
1437 : {
1438 : /* OpenACC firstprivate clauses are later processed with same
1439 : code path as map clauses in lower_omp_target, so follow
1440 : the same convention of using the whole clause expression
1441 : as splay-tree key. */
1442 15905 : tree k = (is_oacc_parallel_or_serial (ctx) ? c : NULL_TREE);
1443 15905 : by_ref = !omp_privatize_by_reference (decl);
1444 15905 : install_var_field (decl, by_ref, 3, ctx, k);
1445 : }
1446 369 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1447 : {
1448 220 : if (INDIRECT_REF_P (decl))
1449 0 : decl = TREE_OPERAND (decl, 0);
1450 220 : install_var_field (decl, true, 3, ctx);
1451 : }
1452 149 : else if (TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1453 8 : install_var_field (decl, true, 3, ctx);
1454 : else
1455 141 : install_var_field (decl, false, 3, ctx);
1456 : }
1457 89271 : if (is_variable_sized (decl))
1458 : {
1459 60 : if (is_task_ctx (ctx))
1460 : {
1461 14 : if (ctx->allocate_map
1462 14 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
1463 : {
1464 : /* For now. */
1465 0 : if (ctx->allocate_map->get (decl))
1466 0 : ctx->allocate_map->remove (decl);
1467 : }
1468 14 : install_var_field (decl, false, 1, ctx);
1469 : }
1470 : break;
1471 : }
1472 89211 : else if (is_taskreg_ctx (ctx))
1473 : {
1474 38408 : bool global
1475 38408 : = is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx));
1476 38408 : by_ref = use_pointer_for_field (decl, NULL);
1477 :
1478 38408 : if (is_task_ctx (ctx)
1479 38408 : && (global || by_ref || omp_privatize_by_reference (decl)))
1480 : {
1481 619 : if (ctx->allocate_map
1482 619 : && ctx->allocate_map->get (decl))
1483 59 : install_var_field (decl, by_ref, 32 | 1, ctx);
1484 : else
1485 560 : install_var_field (decl, false, 1, ctx);
1486 619 : if (!global)
1487 355 : install_var_field (decl, by_ref, 2, ctx);
1488 : }
1489 37789 : else if (!global)
1490 36520 : install_var_field (decl, by_ref, 3, ctx);
1491 : }
1492 89211 : install_var_local (decl, ctx);
1493 : /* For descr arrays on target: firstprivatize data + attach ptr. */
1494 89211 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1495 45399 : && is_gimple_omp_offloaded (ctx->stmt)
1496 15880 : && !is_gimple_omp_oacc (ctx->stmt)
1497 11632 : && lang_hooks.decls.omp_array_data (decl, true)
1498 89242 : && lang_hooks.decls.omp_array_data_privatize (decl))
1499 : {
1500 23 : install_var_field (decl, false, 16 | 3, ctx);
1501 23 : install_var_field (decl, true, 8 | 3, ctx);
1502 : }
1503 : break;
1504 :
1505 2118 : case OMP_CLAUSE_USE_DEVICE_PTR:
1506 2118 : case OMP_CLAUSE_USE_DEVICE_ADDR:
1507 2118 : decl = OMP_CLAUSE_DECL (c);
1508 :
1509 : /* Fortran array descriptors. */
1510 2118 : if (lang_hooks.decls.omp_array_data (decl, true))
1511 601 : install_var_field (decl, false, 19, ctx);
1512 1517 : else if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
1513 1312 : && !omp_privatize_by_reference (decl)
1514 513 : && !omp_is_allocatable_or_ptr (decl))
1515 2454 : || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1516 407 : install_var_field (decl, true, 11, ctx);
1517 : else
1518 1110 : install_var_field (decl, false, 11, ctx);
1519 2118 : if (DECL_SIZE (decl)
1520 2118 : && !poly_int_tree_p (DECL_SIZE (decl)))
1521 : {
1522 6 : tree decl2 = DECL_VALUE_EXPR (decl);
1523 6 : gcc_assert (INDIRECT_REF_P (decl2));
1524 6 : decl2 = TREE_OPERAND (decl2, 0);
1525 6 : gcc_assert (DECL_P (decl2));
1526 6 : install_var_local (decl2, ctx);
1527 : }
1528 2118 : install_var_local (decl, ctx);
1529 2118 : break;
1530 :
1531 239 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
1532 239 : decl = OMP_CLAUSE_DECL (c);
1533 239 : while (INDIRECT_REF_P (decl)
1534 256 : || TREE_CODE (decl) == ARRAY_REF)
1535 17 : decl = TREE_OPERAND (decl, 0);
1536 239 : goto do_private;
1537 :
1538 149 : case OMP_CLAUSE_IS_DEVICE_PTR:
1539 149 : decl = OMP_CLAUSE_DECL (c);
1540 149 : goto do_private;
1541 :
1542 20123 : case OMP_CLAUSE__LOOPTEMP_:
1543 20123 : case OMP_CLAUSE__REDUCTEMP_:
1544 20123 : gcc_assert (is_taskreg_ctx (ctx));
1545 20123 : decl = OMP_CLAUSE_DECL (c);
1546 20123 : install_var_field (decl, false, 3, ctx);
1547 20123 : install_var_local (decl, ctx);
1548 20123 : break;
1549 :
1550 896 : case OMP_CLAUSE_COPYPRIVATE:
1551 896 : case OMP_CLAUSE_COPYIN:
1552 896 : decl = OMP_CLAUSE_DECL (c);
1553 896 : by_ref = use_pointer_for_field (decl, NULL);
1554 896 : install_var_field (decl, by_ref, 3, ctx);
1555 896 : break;
1556 :
1557 55395 : case OMP_CLAUSE_FINAL:
1558 55395 : case OMP_CLAUSE_IF:
1559 55395 : case OMP_CLAUSE_SELF:
1560 55395 : case OMP_CLAUSE_NUM_THREADS:
1561 55395 : case OMP_CLAUSE_NUM_TEAMS:
1562 55395 : case OMP_CLAUSE_THREAD_LIMIT:
1563 55395 : case OMP_CLAUSE_DEVICE:
1564 55395 : case OMP_CLAUSE_SCHEDULE:
1565 55395 : case OMP_CLAUSE_DIST_SCHEDULE:
1566 55395 : case OMP_CLAUSE_DEPEND:
1567 55395 : case OMP_CLAUSE_PRIORITY:
1568 55395 : case OMP_CLAUSE_GRAINSIZE:
1569 55395 : case OMP_CLAUSE_NUM_TASKS:
1570 55395 : case OMP_CLAUSE_NUM_GANGS:
1571 55395 : case OMP_CLAUSE_NUM_WORKERS:
1572 55395 : case OMP_CLAUSE_VECTOR_LENGTH:
1573 55395 : case OMP_CLAUSE_DETACH:
1574 55395 : case OMP_CLAUSE_FILTER:
1575 55395 : if (ctx->outer)
1576 17618 : scan_omp_op (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer);
1577 : break;
1578 :
1579 88352 : case OMP_CLAUSE_TO:
1580 88352 : case OMP_CLAUSE_FROM:
1581 88352 : case OMP_CLAUSE_MAP:
1582 88352 : if (ctx->outer)
1583 15385 : scan_omp_op (&OMP_CLAUSE_SIZE (c), ctx->outer);
1584 88352 : decl = OMP_CLAUSE_DECL (c);
1585 : /* If requested, make 'decl' addressable. */
1586 88352 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1587 88352 : && OMP_CLAUSE_MAP_DECL_MAKE_ADDRESSABLE (c))
1588 : {
1589 433 : gcc_checking_assert (DECL_P (decl));
1590 :
1591 433 : bool decl_addressable = TREE_ADDRESSABLE (decl);
1592 433 : if (!decl_addressable)
1593 : {
1594 219 : if (!make_addressable_vars)
1595 131 : make_addressable_vars = BITMAP_ALLOC (NULL);
1596 219 : bitmap_set_bit (make_addressable_vars, DECL_UID (decl));
1597 219 : TREE_ADDRESSABLE (decl) = 1;
1598 : }
1599 :
1600 433 : if (dump_enabled_p ())
1601 : {
1602 421 : location_t loc = OMP_CLAUSE_LOCATION (c);
1603 421 : const dump_user_location_t d_u_loc
1604 421 : = dump_user_location_t::from_location_t (loc);
1605 : /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */
1606 : #if __GNUC__ >= 10
1607 421 : # pragma GCC diagnostic push
1608 421 : # pragma GCC diagnostic ignored "-Wformat"
1609 : #endif
1610 421 : if (!decl_addressable)
1611 207 : dump_printf_loc (MSG_NOTE, d_u_loc,
1612 : "variable %<%T%>"
1613 : " made addressable\n",
1614 : decl);
1615 : else
1616 214 : dump_printf_loc (MSG_NOTE, d_u_loc,
1617 : "variable %<%T%>"
1618 : " already made addressable\n",
1619 : decl);
1620 : #if __GNUC__ >= 10
1621 421 : # pragma GCC diagnostic pop
1622 : #endif
1623 : }
1624 :
1625 : /* Done. */
1626 433 : OMP_CLAUSE_MAP_DECL_MAKE_ADDRESSABLE (c) = 0;
1627 : }
1628 : /* Global variables with "omp declare target" attribute
1629 : don't need to be copied, the receiver side will use them
1630 : directly. However, global variables with "omp declare target link"
1631 : attribute need to be copied. Or when ALWAYS modifier is used. */
1632 88352 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1633 80567 : && DECL_P (decl)
1634 44377 : && ((OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER
1635 40555 : && (OMP_CLAUSE_MAP_KIND (c)
1636 : != GOMP_MAP_FIRSTPRIVATE_REFERENCE)
1637 40126 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ATTACH
1638 39892 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_DETACH)
1639 4550 : || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1640 41030 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_TO
1641 40895 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_FROM
1642 40866 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_TOFROM
1643 40378 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_TO
1644 40348 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_FROM
1645 40326 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_TOFROM
1646 40312 : && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_TO_PSET
1647 36321 : && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1648 7560 : && varpool_node::get_create (decl)->offloadable
1649 93618 : && !lookup_attribute ("omp declare target link",
1650 5266 : DECL_ATTRIBUTES (decl)))
1651 : break;
1652 83114 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1653 83114 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER)
1654 : {
1655 : /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
1656 : not offloaded; there is nothing to map for those. */
1657 10489 : if (!is_gimple_omp_offloaded (ctx->stmt)
1658 4978 : && !POINTER_TYPE_P (TREE_TYPE (decl))
1659 10753 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
1660 : break;
1661 : }
1662 82850 : if (!flex_array_ptr)
1663 82622 : flex_array_ptr = lang_hooks.decls.omp_deep_mapping_p (ctx->stmt, c);
1664 82850 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1665 75065 : && DECL_P (decl)
1666 38875 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
1667 38641 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
1668 83149 : && is_omp_target (ctx->stmt))
1669 : {
1670 : /* If this is an offloaded region, an attach operation should
1671 : only exist when the pointer variable is mapped in a prior
1672 : clause. An exception is if we have a reference (to pointer):
1673 : in that case we should have mapped "*decl" in a previous
1674 : mapping instead of "decl". Skip the assertion in that case.
1675 : If we had an error, we may not have attempted to sort clauses
1676 : properly, so avoid the test. */
1677 261 : if (TREE_CODE (TREE_TYPE (decl)) != REFERENCE_TYPE
1678 229 : && is_gimple_omp_offloaded (ctx->stmt)
1679 331 : && !seen_error ())
1680 62 : gcc_assert
1681 : (maybe_lookup_decl (decl, ctx)
1682 : || (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1683 : && lookup_attribute ("omp declare target",
1684 : DECL_ATTRIBUTES (decl))));
1685 :
1686 : /* By itself, attach/detach is generated as part of pointer
1687 : variable mapping and should not create new variables in the
1688 : offloaded region, however sender refs for it must be created
1689 : for its address to be passed to the runtime. */
1690 261 : tree field
1691 261 : = build_decl (OMP_CLAUSE_LOCATION (c),
1692 : FIELD_DECL, NULL_TREE, ptr_type_node);
1693 261 : SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
1694 261 : insert_field_into_struct (ctx->record_type, field);
1695 : /* To not clash with a map of the pointer variable itself,
1696 : attach/detach maps have their field looked up by the *clause*
1697 : tree expression, not the decl. */
1698 261 : gcc_assert (!splay_tree_lookup (ctx->field_map,
1699 : (splay_tree_key) c));
1700 261 : splay_tree_insert (ctx->field_map, (splay_tree_key) c,
1701 : (splay_tree_value) field);
1702 261 : break;
1703 : }
1704 82589 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1705 82589 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
1706 70984 : || (OMP_CLAUSE_MAP_KIND (c)
1707 : == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
1708 : {
1709 4249 : if (TREE_CODE (decl) == COMPONENT_REF
1710 4249 : || (INDIRECT_REF_P (decl)
1711 0 : && TREE_CODE (TREE_OPERAND (decl, 0)) == COMPONENT_REF
1712 0 : && (((TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0)))
1713 : == REFERENCE_TYPE)
1714 0 : || (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0)))
1715 : == POINTER_TYPE)))))
1716 : break;
1717 4249 : if (DECL_SIZE (decl)
1718 4249 : && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
1719 : {
1720 710 : tree decl2 = DECL_VALUE_EXPR (decl);
1721 710 : gcc_assert (INDIRECT_REF_P (decl2));
1722 710 : decl2 = TREE_OPERAND (decl2, 0);
1723 710 : gcc_assert (DECL_P (decl2));
1724 710 : install_var_local (decl2, ctx);
1725 : }
1726 4249 : install_var_local (decl, ctx);
1727 4249 : break;
1728 : }
1729 78340 : if (DECL_P (decl))
1730 : {
1731 40527 : if (DECL_SIZE (decl)
1732 40527 : && !poly_int_tree_p (DECL_SIZE (decl)))
1733 : {
1734 0 : tree decl2 = DECL_VALUE_EXPR (decl);
1735 0 : gcc_assert (INDIRECT_REF_P (decl2));
1736 0 : decl2 = TREE_OPERAND (decl2, 0);
1737 0 : gcc_assert (DECL_P (decl2));
1738 0 : install_var_field (decl2, true, 3, ctx, c);
1739 0 : install_var_local (decl2, ctx);
1740 0 : install_var_local (decl, ctx);
1741 : }
1742 : else
1743 : {
1744 40527 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1745 34365 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
1746 5204 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
1747 45731 : && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1748 425 : install_var_field (decl, true, 7, ctx, c);
1749 : else
1750 40102 : install_var_field (decl, true, 3, ctx, c);
1751 40527 : if (is_gimple_omp_offloaded (ctx->stmt)
1752 40527 : && !(is_gimple_omp_oacc (ctx->stmt)
1753 13857 : && OMP_CLAUSE_MAP_IN_REDUCTION (c)))
1754 23843 : install_var_local (decl, ctx);
1755 : }
1756 : }
1757 : else
1758 : {
1759 37813 : tree base = get_base_address (decl);
1760 37813 : tree nc = OMP_CLAUSE_CHAIN (c);
1761 37813 : if (DECL_P (base)
1762 9267 : && nc != NULL_TREE
1763 6678 : && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP
1764 5516 : && OMP_CLAUSE_DECL (nc) == base
1765 1184 : && OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_POINTER
1766 38481 : && integer_zerop (OMP_CLAUSE_SIZE (nc)))
1767 : {
1768 0 : OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) = 1;
1769 0 : OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc) = 1;
1770 : }
1771 : else
1772 : {
1773 37813 : if (ctx->outer)
1774 : {
1775 7152 : scan_omp_op (&OMP_CLAUSE_DECL (c), ctx->outer);
1776 7152 : decl = OMP_CLAUSE_DECL (c);
1777 : }
1778 37813 : gcc_assert (!splay_tree_lookup (ctx->field_map,
1779 : (splay_tree_key) decl));
1780 37813 : tree field
1781 37813 : = build_decl (OMP_CLAUSE_LOCATION (c),
1782 : FIELD_DECL, NULL_TREE, ptr_type_node);
1783 37813 : SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
1784 37813 : insert_field_into_struct (ctx->record_type, field);
1785 37813 : splay_tree_insert (ctx->field_map, (splay_tree_key) c,
1786 : (splay_tree_value) field);
1787 : }
1788 : }
1789 : break;
1790 :
1791 3973 : case OMP_CLAUSE_ORDER:
1792 3973 : ctx->order_concurrent = true;
1793 3973 : break;
1794 :
1795 1907 : case OMP_CLAUSE_BIND:
1796 1907 : ctx->loop_p = true;
1797 1907 : break;
1798 :
1799 : case OMP_CLAUSE_NOWAIT:
1800 : case OMP_CLAUSE_ORDERED:
1801 : case OMP_CLAUSE_COLLAPSE:
1802 : case OMP_CLAUSE_UNTIED:
1803 : case OMP_CLAUSE_MERGEABLE:
1804 : case OMP_CLAUSE_PROC_BIND:
1805 : case OMP_CLAUSE_SAFELEN:
1806 : case OMP_CLAUSE_SIMDLEN:
1807 : case OMP_CLAUSE_THREADS:
1808 : case OMP_CLAUSE_SIMD:
1809 : case OMP_CLAUSE_NOGROUP:
1810 : case OMP_CLAUSE_DEFAULTMAP:
1811 : case OMP_CLAUSE_ASYNC:
1812 : case OMP_CLAUSE_WAIT:
1813 : case OMP_CLAUSE_GANG:
1814 : case OMP_CLAUSE_WORKER:
1815 : case OMP_CLAUSE_VECTOR:
1816 : case OMP_CLAUSE_INDEPENDENT:
1817 : case OMP_CLAUSE_AUTO:
1818 : case OMP_CLAUSE_SEQ:
1819 : case OMP_CLAUSE_TILE:
1820 : case OMP_CLAUSE__SIMT_:
1821 : case OMP_CLAUSE_DEFAULT:
1822 : case OMP_CLAUSE_NONTEMPORAL:
1823 : case OMP_CLAUSE_IF_PRESENT:
1824 : case OMP_CLAUSE_FINALIZE:
1825 : case OMP_CLAUSE_TASK_REDUCTION:
1826 : case OMP_CLAUSE_ALLOCATE:
1827 : case OMP_CLAUSE_DEVICE_TYPE:
1828 : break;
1829 :
1830 112 : case OMP_CLAUSE_USES_ALLOCATORS:
1831 112 : decl = OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c);
1832 112 : gcc_assert (DECL_P (decl));
1833 112 : gcc_assert (is_gimple_omp_offloaded (ctx->stmt));
1834 112 : install_var_field (decl, false, 3, ctx);
1835 112 : install_var_local (decl, ctx);
1836 112 : break;
1837 :
1838 162 : case OMP_CLAUSE_ALIGNED:
1839 162 : decl = OMP_CLAUSE_DECL (c);
1840 162 : if (is_global_var (decl)
1841 162 : && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1842 86 : install_var_local (decl, ctx);
1843 : break;
1844 :
1845 361 : case OMP_CLAUSE__CONDTEMP_:
1846 361 : decl = OMP_CLAUSE_DECL (c);
1847 361 : if (is_parallel_ctx (ctx))
1848 : {
1849 118 : install_var_field (decl, false, 3, ctx);
1850 118 : install_var_local (decl, ctx);
1851 : }
1852 243 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
1853 243 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
1854 368 : && !OMP_CLAUSE__CONDTEMP__ITER (c))
1855 125 : install_var_local (decl, ctx);
1856 : break;
1857 :
1858 : case OMP_CLAUSE_INIT:
1859 : case OMP_CLAUSE_USE:
1860 : case OMP_CLAUSE_DESTROY:
1861 : case OMP_CLAUSE_MESSAGE:
1862 : break;
1863 :
1864 0 : case OMP_CLAUSE__CACHE_:
1865 0 : case OMP_CLAUSE_NOHOST:
1866 0 : default:
1867 0 : gcc_unreachable ();
1868 : }
1869 : }
1870 :
1871 551008 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1872 : {
1873 420963 : switch (OMP_CLAUSE_CODE (c))
1874 : {
1875 22747 : case OMP_CLAUSE_LASTPRIVATE:
1876 : /* Let the corresponding firstprivate clause create
1877 : the variable. */
1878 22747 : if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
1879 7245 : scan_array_reductions = true;
1880 22747 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1881 : break;
1882 : /* FALLTHRU */
1883 :
1884 149924 : case OMP_CLAUSE_FIRSTPRIVATE:
1885 149924 : case OMP_CLAUSE_PRIVATE:
1886 149924 : case OMP_CLAUSE_LINEAR:
1887 149924 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
1888 149924 : case OMP_CLAUSE_IS_DEVICE_PTR:
1889 149924 : decl = OMP_CLAUSE_DECL (c);
1890 149924 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1891 : {
1892 256 : while (INDIRECT_REF_P (decl)
1893 256 : || TREE_CODE (decl) == ARRAY_REF)
1894 17 : decl = TREE_OPERAND (decl, 0);
1895 : }
1896 :
1897 149924 : if (is_variable_sized (decl))
1898 : {
1899 423 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
1900 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR
1901 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
1902 53 : && is_gimple_omp_offloaded (ctx->stmt))
1903 : {
1904 12 : tree decl2 = DECL_VALUE_EXPR (decl);
1905 12 : gcc_assert (INDIRECT_REF_P (decl2));
1906 12 : decl2 = TREE_OPERAND (decl2, 0);
1907 12 : gcc_assert (DECL_P (decl2));
1908 12 : install_var_local (decl2, ctx);
1909 12 : fixup_remapped_decl (decl2, ctx, false);
1910 : }
1911 423 : install_var_local (decl, ctx);
1912 : }
1913 299848 : fixup_remapped_decl (decl, ctx,
1914 149924 : OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE
1915 149924 : && OMP_CLAUSE_PRIVATE_DEBUG (c));
1916 149924 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
1917 149924 : && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
1918 : scan_array_reductions = true;
1919 : break;
1920 :
1921 16640 : case OMP_CLAUSE_REDUCTION:
1922 16640 : case OMP_CLAUSE_IN_REDUCTION:
1923 16640 : decl = OMP_CLAUSE_DECL (c);
1924 16640 : if (TREE_CODE (decl) != MEM_REF && !is_omp_target (ctx->stmt))
1925 : {
1926 13939 : if (is_variable_sized (decl))
1927 0 : install_var_local (decl, ctx);
1928 13939 : fixup_remapped_decl (decl, ctx, false);
1929 : }
1930 16640 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1931 2852 : scan_array_reductions = true;
1932 : break;
1933 :
1934 530 : case OMP_CLAUSE_TASK_REDUCTION:
1935 530 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1936 2852 : scan_array_reductions = true;
1937 : break;
1938 :
1939 30061 : case OMP_CLAUSE_SHARED:
1940 : /* Ignore shared directives in teams construct inside of
1941 : target construct. */
1942 30061 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
1943 30061 : && !is_host_teams_ctx (ctx))
1944 : break;
1945 18199 : decl = OMP_CLAUSE_DECL (c);
1946 18199 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1947 : break;
1948 12134 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
1949 : {
1950 86 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl,
1951 : ctx->outer)))
1952 : break;
1953 79 : bool by_ref = use_pointer_for_field (decl, ctx);
1954 79 : install_var_field (decl, by_ref, 11, ctx);
1955 79 : break;
1956 : }
1957 12048 : fixup_remapped_decl (decl, ctx, false);
1958 12048 : break;
1959 :
1960 80567 : case OMP_CLAUSE_MAP:
1961 80567 : if (!is_gimple_omp_offloaded (ctx->stmt))
1962 : break;
1963 53828 : decl = OMP_CLAUSE_DECL (c);
1964 53828 : if (DECL_P (decl)
1965 33904 : && ((OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER
1966 30250 : && (OMP_CLAUSE_MAP_KIND (c)
1967 : != GOMP_MAP_FIRSTPRIVATE_REFERENCE))
1968 4083 : || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1969 31024 : && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1970 61211 : && varpool_node::get_create (decl)->offloadable)
1971 : break;
1972 48146 : if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
1973 45934 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
1974 2212 : && is_omp_target (ctx->stmt)
1975 50149 : && !is_gimple_omp_offloaded (ctx->stmt))
1976 : break;
1977 48146 : if (DECL_P (decl))
1978 : {
1979 28222 : if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
1980 25169 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER)
1981 6705 : && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
1982 29847 : && !COMPLETE_TYPE_P (TREE_TYPE (decl)))
1983 : {
1984 24 : tree new_decl = lookup_decl (decl, ctx);
1985 24 : TREE_TYPE (new_decl)
1986 48 : = remap_type (TREE_TYPE (decl), &ctx->cb);
1987 : }
1988 28198 : else if (DECL_SIZE (decl)
1989 28198 : && !poly_int_tree_p (DECL_SIZE (decl)))
1990 : {
1991 710 : tree decl2 = DECL_VALUE_EXPR (decl);
1992 710 : gcc_assert (INDIRECT_REF_P (decl2));
1993 710 : decl2 = TREE_OPERAND (decl2, 0);
1994 710 : gcc_assert (DECL_P (decl2));
1995 710 : fixup_remapped_decl (decl2, ctx, false);
1996 710 : fixup_remapped_decl (decl, ctx, true);
1997 : }
1998 : else
1999 27488 : fixup_remapped_decl (decl, ctx, false);
2000 : }
2001 : break;
2002 :
2003 : case OMP_CLAUSE_COPYPRIVATE:
2004 : case OMP_CLAUSE_COPYIN:
2005 : case OMP_CLAUSE_DEFAULT:
2006 : case OMP_CLAUSE_IF:
2007 : case OMP_CLAUSE_SELF:
2008 : case OMP_CLAUSE_NUM_THREADS:
2009 : case OMP_CLAUSE_NUM_TEAMS:
2010 : case OMP_CLAUSE_THREAD_LIMIT:
2011 : case OMP_CLAUSE_DEVICE:
2012 : case OMP_CLAUSE_SCHEDULE:
2013 : case OMP_CLAUSE_DIST_SCHEDULE:
2014 : case OMP_CLAUSE_NOWAIT:
2015 : case OMP_CLAUSE_ORDERED:
2016 : case OMP_CLAUSE_COLLAPSE:
2017 : case OMP_CLAUSE_UNTIED:
2018 : case OMP_CLAUSE_FINAL:
2019 : case OMP_CLAUSE_MERGEABLE:
2020 : case OMP_CLAUSE_PROC_BIND:
2021 : case OMP_CLAUSE_SAFELEN:
2022 : case OMP_CLAUSE_SIMDLEN:
2023 : case OMP_CLAUSE_ALIGNED:
2024 : case OMP_CLAUSE_DEPEND:
2025 : case OMP_CLAUSE_DETACH:
2026 : case OMP_CLAUSE_ALLOCATE:
2027 : case OMP_CLAUSE__LOOPTEMP_:
2028 : case OMP_CLAUSE__REDUCTEMP_:
2029 : case OMP_CLAUSE_TO:
2030 : case OMP_CLAUSE_FROM:
2031 : case OMP_CLAUSE_PRIORITY:
2032 : case OMP_CLAUSE_GRAINSIZE:
2033 : case OMP_CLAUSE_NUM_TASKS:
2034 : case OMP_CLAUSE_THREADS:
2035 : case OMP_CLAUSE_SIMD:
2036 : case OMP_CLAUSE_NOGROUP:
2037 : case OMP_CLAUSE_DEFAULTMAP:
2038 : case OMP_CLAUSE_ORDER:
2039 : case OMP_CLAUSE_BIND:
2040 : case OMP_CLAUSE_USE_DEVICE_PTR:
2041 : case OMP_CLAUSE_USE_DEVICE_ADDR:
2042 : case OMP_CLAUSE_NONTEMPORAL:
2043 : case OMP_CLAUSE_ASYNC:
2044 : case OMP_CLAUSE_WAIT:
2045 : case OMP_CLAUSE_NUM_GANGS:
2046 : case OMP_CLAUSE_NUM_WORKERS:
2047 : case OMP_CLAUSE_VECTOR_LENGTH:
2048 : case OMP_CLAUSE_GANG:
2049 : case OMP_CLAUSE_WORKER:
2050 : case OMP_CLAUSE_VECTOR:
2051 : case OMP_CLAUSE_INDEPENDENT:
2052 : case OMP_CLAUSE_AUTO:
2053 : case OMP_CLAUSE_SEQ:
2054 : case OMP_CLAUSE_TILE:
2055 : case OMP_CLAUSE__SIMT_:
2056 : case OMP_CLAUSE_IF_PRESENT:
2057 : case OMP_CLAUSE_FINALIZE:
2058 : case OMP_CLAUSE_FILTER:
2059 : case OMP_CLAUSE__CONDTEMP_:
2060 : case OMP_CLAUSE_INIT:
2061 : case OMP_CLAUSE_USE:
2062 : case OMP_CLAUSE_DESTROY:
2063 : case OMP_CLAUSE_DEVICE_TYPE:
2064 : case OMP_CLAUSE_USES_ALLOCATORS:
2065 : case OMP_CLAUSE_MESSAGE:
2066 : break;
2067 :
2068 0 : case OMP_CLAUSE__CACHE_:
2069 0 : case OMP_CLAUSE_NOHOST:
2070 0 : default:
2071 0 : gcc_unreachable ();
2072 : }
2073 : }
2074 :
2075 130045 : gcc_checking_assert (!scan_array_reductions
2076 : || !is_gimple_omp_oacc (ctx->stmt));
2077 : if (scan_array_reductions)
2078 : {
2079 34654 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2080 29452 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
2081 27058 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
2082 26434 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
2083 30156 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
2084 : {
2085 2196 : omp_context *rctx = ctx;
2086 2196 : if (is_omp_target (ctx->stmt))
2087 60 : rctx = ctx->outer;
2088 2196 : scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), rctx);
2089 2196 : scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), rctx);
2090 : }
2091 27256 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
2092 27256 : && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
2093 7245 : scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
2094 20011 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
2095 20011 : && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
2096 656 : scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
2097 : }
2098 130045 : if (flex_array_ptr)
2099 : {
2100 144 : tree field = build_range_type (size_type_node,
2101 : build_int_cstu (size_type_node, 0),
2102 : NULL_TREE);
2103 144 : field = build_array_type (ptr_type_node, field);
2104 144 : field = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE, field);
2105 144 : SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
2106 144 : DECL_CONTEXT (field) = ctx->record_type;
2107 144 : DECL_CHAIN (field) = TYPE_FIELDS (ctx->record_type);
2108 144 : TYPE_FIELDS (ctx->record_type) = field;
2109 : }
2110 130045 : }
2111 :
2112 : /* Create a new name for omp child function. Returns an identifier. */
2113 :
2114 : static tree
2115 51515 : create_omp_child_function_name (bool task_copy)
2116 : {
2117 102486 : return clone_function_name_numbered (current_function_decl,
2118 51515 : task_copy ? "_omp_cpyfn" : "_omp_fn");
2119 : }
2120 :
2121 : /* Return true if CTX may belong to offloaded code: either if current function
2122 : is offloaded, or any enclosing context corresponds to a target region. */
2123 :
2124 : static bool
2125 123099 : omp_maybe_offloaded_ctx (omp_context *ctx)
2126 : {
2127 123099 : if (cgraph_node::get (current_function_decl)->offloadable)
2128 : return true;
2129 203688 : for (; ctx; ctx = ctx->outer)
2130 120376 : if (is_gimple_omp_offloaded (ctx->stmt))
2131 : return true;
2132 : return false;
2133 : }
2134 :
2135 : /* Build a decl for the omp child function. It'll not contain a body
2136 : yet, just the bare decl. If HOST_ONLY, do not create a device version. */
2137 :
2138 : static void
2139 51515 : create_omp_child_function (omp_context *ctx, bool task_copy, bool host_only)
2140 : {
2141 51515 : tree decl, type, name, t;
2142 :
2143 51515 : name = create_omp_child_function_name (task_copy);
2144 51515 : if (task_copy)
2145 544 : type = build_function_type_list (void_type_node, ptr_type_node,
2146 : ptr_type_node, NULL_TREE);
2147 : else
2148 50971 : type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE);
2149 :
2150 51515 : decl = build_decl (gimple_location (ctx->stmt), FUNCTION_DECL, name, type);
2151 :
2152 51515 : gcc_checking_assert (!is_gimple_omp_oacc (ctx->stmt)
2153 : || !task_copy);
2154 40129 : if (!task_copy)
2155 50971 : ctx->cb.dst_fn = decl;
2156 : else
2157 544 : gimple_omp_task_set_copy_fn (ctx->stmt, decl);
2158 :
2159 51515 : TREE_STATIC (decl) = 1;
2160 51515 : TREE_USED (decl) = 1;
2161 51515 : DECL_ARTIFICIAL (decl) = 1;
2162 51515 : DECL_IGNORED_P (decl) = 0;
2163 51515 : TREE_PUBLIC (decl) = 0;
2164 51515 : DECL_UNINLINABLE (decl) = 1;
2165 51515 : DECL_EXTERNAL (decl) = 0;
2166 51515 : DECL_CONTEXT (decl) = NULL_TREE;
2167 51515 : DECL_INITIAL (decl) = make_node (BLOCK);
2168 51515 : BLOCK_SUPERCONTEXT (DECL_INITIAL (decl)) = decl;
2169 51515 : DECL_ATTRIBUTES (decl) = DECL_ATTRIBUTES (current_function_decl);
2170 : /* Remove omp declare simd attribute from the new attributes. */
2171 51515 : if (tree a = lookup_attribute ("omp declare simd", DECL_ATTRIBUTES (decl)))
2172 : {
2173 9 : while (tree a2 = lookup_attribute ("omp declare simd", TREE_CHAIN (a)))
2174 : a = a2;
2175 9 : a = TREE_CHAIN (a);
2176 22 : for (tree *p = &DECL_ATTRIBUTES (decl); *p != a;)
2177 13 : if (is_attribute_p ("omp declare simd", get_attribute_name (*p)))
2178 9 : *p = TREE_CHAIN (*p);
2179 : else
2180 : {
2181 4 : tree chain = TREE_CHAIN (*p);
2182 4 : *p = copy_node (*p);
2183 4 : p = &TREE_CHAIN (*p);
2184 4 : *p = chain;
2185 : }
2186 : }
2187 103030 : DECL_FUNCTION_SPECIFIC_OPTIMIZATION (decl)
2188 51515 : = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (current_function_decl);
2189 103030 : DECL_FUNCTION_SPECIFIC_TARGET (decl)
2190 51515 : = DECL_FUNCTION_SPECIFIC_TARGET (current_function_decl);
2191 103030 : DECL_FUNCTION_VERSIONED (decl)
2192 51515 : = DECL_FUNCTION_VERSIONED (current_function_decl);
2193 :
2194 51515 : if (omp_maybe_offloaded_ctx (ctx) && !host_only)
2195 : {
2196 31046 : cgraph_node::get_create (decl)->offloadable = 1;
2197 31046 : if (ENABLE_OFFLOADING)
2198 : g->have_offload = true;
2199 : }
2200 :
2201 51515 : if (cgraph_node::get_create (decl)->offloadable)
2202 : {
2203 31046 : const char *target_attr = (is_gimple_omp_offloaded (ctx->stmt)
2204 31046 : ? "omp target entrypoint"
2205 6216 : : "omp declare target");
2206 31046 : if (lookup_attribute ("omp declare target",
2207 31046 : DECL_ATTRIBUTES (current_function_decl)))
2208 : {
2209 3032 : if (is_gimple_omp_offloaded (ctx->stmt))
2210 1620 : DECL_ATTRIBUTES (decl)
2211 1620 : = remove_attribute ("omp declare target",
2212 1620 : copy_list (DECL_ATTRIBUTES (decl)));
2213 : else
2214 : target_attr = NULL;
2215 : }
2216 1620 : if (target_attr
2217 29634 : && is_gimple_omp_offloaded (ctx->stmt)
2218 24830 : && lookup_attribute ("noclone", DECL_ATTRIBUTES (decl)) == NULL_TREE)
2219 19323 : DECL_ATTRIBUTES (decl) = tree_cons (get_identifier ("noclone"),
2220 19323 : NULL_TREE, DECL_ATTRIBUTES (decl));
2221 31046 : if (target_attr)
2222 29634 : DECL_ATTRIBUTES (decl)
2223 59268 : = tree_cons (get_identifier (target_attr),
2224 29634 : NULL_TREE, DECL_ATTRIBUTES (decl));
2225 : }
2226 :
2227 51515 : t = build_decl (DECL_SOURCE_LOCATION (decl),
2228 : RESULT_DECL, NULL_TREE, void_type_node);
2229 51515 : DECL_ARTIFICIAL (t) = 1;
2230 51515 : DECL_IGNORED_P (t) = 1;
2231 51515 : DECL_CONTEXT (t) = decl;
2232 51515 : DECL_RESULT (decl) = t;
2233 :
2234 51515 : tree data_name = get_identifier (".omp_data_i");
2235 51515 : t = build_decl (DECL_SOURCE_LOCATION (decl), PARM_DECL, data_name,
2236 : ptr_type_node);
2237 51515 : DECL_ARTIFICIAL (t) = 1;
2238 51515 : DECL_NAMELESS (t) = 1;
2239 51515 : DECL_ARG_TYPE (t) = ptr_type_node;
2240 51515 : DECL_CONTEXT (t) = current_function_decl;
2241 51515 : TREE_USED (t) = 1;
2242 51515 : TREE_READONLY (t) = 1;
2243 51515 : DECL_ARGUMENTS (decl) = t;
2244 51515 : if (!task_copy)
2245 50971 : ctx->receiver_decl = t;
2246 : else
2247 : {
2248 544 : t = build_decl (DECL_SOURCE_LOCATION (decl),
2249 : PARM_DECL, get_identifier (".omp_data_o"),
2250 : ptr_type_node);
2251 544 : DECL_ARTIFICIAL (t) = 1;
2252 544 : DECL_NAMELESS (t) = 1;
2253 544 : DECL_ARG_TYPE (t) = ptr_type_node;
2254 544 : DECL_CONTEXT (t) = current_function_decl;
2255 544 : TREE_USED (t) = 1;
2256 544 : TREE_ADDRESSABLE (t) = 1;
2257 544 : DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2258 544 : DECL_ARGUMENTS (decl) = t;
2259 : }
2260 :
2261 : /* Allocate memory for the function structure. The call to
2262 : allocate_struct_function clobbers CFUN, so we need to restore
2263 : it afterward. */
2264 51515 : push_struct_function (decl);
2265 51515 : cfun->function_end_locus = gimple_location (ctx->stmt);
2266 51515 : init_tree_ssa (cfun);
2267 51515 : pop_cfun ();
2268 51515 : }
2269 :
2270 : /* Callback for walk_gimple_seq. Check if combined parallel
2271 : contains gimple_omp_for_combined_into_p OMP_FOR. */
2272 :
2273 : tree
2274 35681 : omp_find_combined_for (gimple_stmt_iterator *gsi_p,
2275 : bool *handled_ops_p,
2276 : struct walk_stmt_info *wi)
2277 : {
2278 35681 : gimple *stmt = gsi_stmt (*gsi_p);
2279 :
2280 35681 : *handled_ops_p = true;
2281 35681 : switch (gimple_code (stmt))
2282 : {
2283 19989 : WALK_SUBSTMTS;
2284 :
2285 13519 : case GIMPLE_OMP_FOR:
2286 13519 : if (gimple_omp_for_combined_into_p (stmt)
2287 13519 : && gimple_omp_for_kind (stmt)
2288 7966 : == *(const enum gf_mask *) (wi->info))
2289 : {
2290 7966 : wi->info = stmt;
2291 7966 : return integer_zero_node;
2292 : }
2293 : break;
2294 : default:
2295 : break;
2296 : }
2297 : return NULL;
2298 : }
2299 :
2300 : /* Add _LOOPTEMP_/_REDUCTEMP_ clauses on OpenMP parallel or task. */
2301 :
2302 : static void
2303 14104 : add_taskreg_looptemp_clauses (enum gf_mask msk, gimple *stmt,
2304 : omp_context *outer_ctx)
2305 : {
2306 14104 : struct walk_stmt_info wi;
2307 :
2308 14104 : memset (&wi, 0, sizeof (wi));
2309 14104 : wi.val_only = true;
2310 14104 : wi.info = (void *) &msk;
2311 14104 : walk_gimple_seq (gimple_omp_body (stmt), omp_find_combined_for, NULL, &wi);
2312 14104 : if (wi.info != (void *) &msk)
2313 : {
2314 7966 : gomp_for *for_stmt = as_a <gomp_for *> ((gimple *) wi.info);
2315 7966 : struct omp_for_data fd;
2316 7966 : omp_extract_for_data (for_stmt, &fd, NULL);
2317 : /* We need two temporaries with fd.loop.v type (istart/iend)
2318 : and then (fd.collapse - 1) temporaries with the same
2319 : type for count2 ... countN-1 vars if not constant. */
2320 7966 : size_t count = 2, i;
2321 7966 : tree type = fd.iter_type;
2322 7966 : if (fd.collapse > 1
2323 2648 : && TREE_CODE (fd.loop.n2) != INTEGER_CST)
2324 : {
2325 1466 : count += fd.collapse - 1;
2326 : /* If there are lastprivate clauses on the inner
2327 : GIMPLE_OMP_FOR, add one more temporaries for the total number
2328 : of iterations (product of count1 ... countN-1). */
2329 1466 : if (omp_find_clause (gimple_omp_for_clauses (for_stmt),
2330 : OMP_CLAUSE_LASTPRIVATE)
2331 1466 : || (msk == GF_OMP_FOR_KIND_FOR
2332 1335 : && omp_find_clause (gimple_omp_parallel_clauses (stmt),
2333 : OMP_CLAUSE_LASTPRIVATE)))
2334 : {
2335 726 : tree temp = create_tmp_var (type);
2336 726 : tree c = build_omp_clause (UNKNOWN_LOCATION,
2337 : OMP_CLAUSE__LOOPTEMP_);
2338 726 : insert_decl_map (&outer_ctx->cb, temp, temp);
2339 726 : OMP_CLAUSE_DECL (c) = temp;
2340 726 : OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
2341 726 : gimple_omp_taskreg_set_clauses (stmt, c);
2342 : }
2343 1466 : if (fd.non_rect
2344 24 : && fd.last_nonrect == fd.first_nonrect + 1)
2345 12 : if (tree v = gimple_omp_for_index (for_stmt, fd.last_nonrect))
2346 12 : if (!TYPE_UNSIGNED (TREE_TYPE (v)))
2347 : {
2348 12 : v = gimple_omp_for_index (for_stmt, fd.first_nonrect);
2349 12 : tree type2 = TREE_TYPE (v);
2350 12 : count++;
2351 48 : for (i = 0; i < 3; i++)
2352 : {
2353 36 : tree temp = create_tmp_var (type2);
2354 36 : tree c = build_omp_clause (UNKNOWN_LOCATION,
2355 : OMP_CLAUSE__LOOPTEMP_);
2356 36 : insert_decl_map (&outer_ctx->cb, temp, temp);
2357 36 : OMP_CLAUSE_DECL (c) = temp;
2358 36 : OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
2359 36 : gimple_omp_taskreg_set_clauses (stmt, c);
2360 : }
2361 : }
2362 : }
2363 26743 : for (i = 0; i < count; i++)
2364 : {
2365 18777 : tree temp = create_tmp_var (type);
2366 18777 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
2367 18777 : insert_decl_map (&outer_ctx->cb, temp, temp);
2368 18777 : OMP_CLAUSE_DECL (c) = temp;
2369 18777 : OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
2370 18777 : gimple_omp_taskreg_set_clauses (stmt, c);
2371 : }
2372 : }
2373 14104 : if (msk == GF_OMP_FOR_KIND_TASKLOOP
2374 15671 : && omp_find_clause (gimple_omp_task_clauses (stmt),
2375 : OMP_CLAUSE_REDUCTION))
2376 : {
2377 517 : tree type = build_pointer_type (pointer_sized_int_node);
2378 517 : tree temp = create_tmp_var (type);
2379 517 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
2380 517 : insert_decl_map (&outer_ctx->cb, temp, temp);
2381 517 : OMP_CLAUSE_DECL (c) = temp;
2382 517 : OMP_CLAUSE_CHAIN (c) = gimple_omp_task_clauses (stmt);
2383 517 : gimple_omp_task_set_clauses (stmt, c);
2384 : }
2385 14104 : }
2386 :
2387 : /* Scan an OpenMP parallel directive. */
2388 :
2389 : static void
2390 18243 : scan_omp_parallel (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2391 : {
2392 18243 : omp_context *ctx;
2393 18243 : tree name;
2394 18243 : gomp_parallel *stmt = as_a <gomp_parallel *> (gsi_stmt (*gsi));
2395 :
2396 : /* Ignore parallel directives with empty bodies, unless there
2397 : are copyin clauses. */
2398 18243 : if (optimize > 0
2399 12515 : && empty_body_p (gimple_omp_body (stmt))
2400 18290 : && omp_find_clause (gimple_omp_parallel_clauses (stmt),
2401 : OMP_CLAUSE_COPYIN) == NULL)
2402 : {
2403 41 : gsi_replace (gsi, gimple_build_nop (), false);
2404 41 : return;
2405 : }
2406 :
2407 18202 : if (gimple_omp_parallel_combined_p (stmt))
2408 12537 : add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR, stmt, outer_ctx);
2409 18202 : for (tree c = omp_find_clause (gimple_omp_parallel_clauses (stmt),
2410 : OMP_CLAUSE_REDUCTION);
2411 21036 : c; c = omp_find_clause (OMP_CLAUSE_CHAIN (c), OMP_CLAUSE_REDUCTION))
2412 3921 : if (OMP_CLAUSE_REDUCTION_TASK (c))
2413 : {
2414 67 : tree type = build_pointer_type (pointer_sized_int_node);
2415 67 : tree temp = create_tmp_var (type);
2416 67 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
2417 67 : if (outer_ctx)
2418 18 : insert_decl_map (&outer_ctx->cb, temp, temp);
2419 67 : OMP_CLAUSE_DECL (c) = temp;
2420 67 : OMP_CLAUSE_CHAIN (c) = gimple_omp_parallel_clauses (stmt);
2421 67 : gimple_omp_parallel_set_clauses (stmt, c);
2422 67 : break;
2423 : }
2424 3854 : else if (OMP_CLAUSE_CHAIN (c) == NULL_TREE)
2425 : break;
2426 :
2427 18202 : ctx = new_omp_context (stmt, outer_ctx);
2428 18202 : taskreg_contexts.safe_push (ctx);
2429 18202 : if (taskreg_nesting_level > 1)
2430 6222 : ctx->is_nested = true;
2431 18202 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2432 18202 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2433 18202 : name = create_tmp_var_name (".omp_data_s");
2434 18202 : name = build_decl (gimple_location (stmt),
2435 : TYPE_DECL, name, ctx->record_type);
2436 18202 : DECL_ARTIFICIAL (name) = 1;
2437 18202 : DECL_NAMELESS (name) = 1;
2438 18202 : TYPE_NAME (ctx->record_type) = name;
2439 18202 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
2440 18202 : create_omp_child_function (ctx, false, false);
2441 18202 : gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn);
2442 :
2443 18202 : scan_sharing_clauses (gimple_omp_parallel_clauses (stmt), ctx);
2444 18202 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
2445 :
2446 18202 : if (TYPE_FIELDS (ctx->record_type) == NULL)
2447 3584 : ctx->record_type = ctx->receiver_decl = NULL;
2448 : }
2449 :
2450 : /* Scan an OpenMP task directive. */
2451 :
2452 : static void
2453 5374 : scan_omp_task (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2454 : {
2455 5374 : omp_context *ctx;
2456 5374 : tree name, t;
2457 5374 : gomp_task *stmt = as_a <gomp_task *> (gsi_stmt (*gsi));
2458 :
2459 : /* Ignore task directives with empty bodies, unless they have depend
2460 : clause. */
2461 5374 : if (optimize > 0
2462 2552 : && gimple_omp_body (stmt)
2463 2509 : && empty_body_p (gimple_omp_body (stmt))
2464 5428 : && !omp_find_clause (gimple_omp_task_clauses (stmt), OMP_CLAUSE_DEPEND))
2465 : {
2466 19 : gsi_replace (gsi, gimple_build_nop (), false);
2467 131 : return;
2468 : }
2469 :
2470 5355 : if (gimple_omp_task_taskloop_p (stmt))
2471 1567 : add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP, stmt, outer_ctx);
2472 :
2473 5355 : ctx = new_omp_context (stmt, outer_ctx);
2474 :
2475 5355 : if (gimple_omp_task_taskwait_p (stmt))
2476 : {
2477 93 : scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
2478 93 : return;
2479 : }
2480 :
2481 5262 : taskreg_contexts.safe_push (ctx);
2482 5262 : if (taskreg_nesting_level > 1)
2483 2326 : ctx->is_nested = true;
2484 5262 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2485 5262 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2486 5262 : name = create_tmp_var_name (".omp_data_s");
2487 5262 : name = build_decl (gimple_location (stmt),
2488 : TYPE_DECL, name, ctx->record_type);
2489 5262 : DECL_ARTIFICIAL (name) = 1;
2490 5262 : DECL_NAMELESS (name) = 1;
2491 5262 : TYPE_NAME (ctx->record_type) = name;
2492 5262 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
2493 5262 : create_omp_child_function (ctx, false, false);
2494 5262 : gimple_omp_task_set_child_fn (stmt, ctx->cb.dst_fn);
2495 :
2496 5262 : scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
2497 :
2498 5262 : if (ctx->srecord_type)
2499 : {
2500 544 : name = create_tmp_var_name (".omp_data_a");
2501 544 : name = build_decl (gimple_location (stmt),
2502 : TYPE_DECL, name, ctx->srecord_type);
2503 544 : DECL_ARTIFICIAL (name) = 1;
2504 544 : DECL_NAMELESS (name) = 1;
2505 544 : TYPE_NAME (ctx->srecord_type) = name;
2506 544 : TYPE_ARTIFICIAL (ctx->srecord_type) = 1;
2507 544 : create_omp_child_function (ctx, true, false);
2508 : }
2509 :
2510 5262 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
2511 :
2512 5262 : if (TYPE_FIELDS (ctx->record_type) == NULL)
2513 : {
2514 1837 : ctx->record_type = ctx->receiver_decl = NULL;
2515 1837 : t = build_int_cst (long_integer_type_node, 0);
2516 1837 : gimple_omp_task_set_arg_size (stmt, t);
2517 1837 : t = build_int_cst (long_integer_type_node, 1);
2518 1837 : gimple_omp_task_set_arg_align (stmt, t);
2519 : }
2520 : }
2521 :
2522 : /* Helper function for finish_taskreg_scan, called through walk_tree.
2523 : If maybe_lookup_decl_in_outer_context returns non-NULL for some
2524 : tree, replace it in the expression. */
2525 :
2526 : static tree
2527 276 : finish_taskreg_remap (tree *tp, int *walk_subtrees, void *data)
2528 : {
2529 276 : if (VAR_P (*tp))
2530 : {
2531 96 : omp_context *ctx = (omp_context *) data;
2532 96 : tree t = maybe_lookup_decl_in_outer_ctx (*tp, ctx);
2533 96 : if (t != *tp)
2534 : {
2535 9 : if (DECL_HAS_VALUE_EXPR_P (t))
2536 0 : t = unshare_expr (DECL_VALUE_EXPR (t));
2537 9 : *tp = t;
2538 : }
2539 96 : *walk_subtrees = 0;
2540 : }
2541 180 : else if (IS_TYPE_OR_DECL_P (*tp))
2542 0 : *walk_subtrees = 0;
2543 276 : return NULL_TREE;
2544 : }
2545 :
2546 : /* If any decls have been made addressable during scan_omp,
2547 : adjust their fields if needed, and layout record types
2548 : of parallel/task constructs. */
2549 :
2550 : static void
2551 26135 : finish_taskreg_scan (omp_context *ctx)
2552 : {
2553 26135 : if (ctx->record_type == NULL_TREE)
2554 : return;
2555 :
2556 : /* If any make_addressable_vars were needed, verify all
2557 : OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK,TEAMS}
2558 : statements if use_pointer_for_field hasn't changed
2559 : because of that. If it did, update field types now. */
2560 19265 : if (make_addressable_vars)
2561 : {
2562 2211 : tree c;
2563 :
2564 12357 : for (c = gimple_omp_taskreg_clauses (ctx->stmt);
2565 12357 : c; c = OMP_CLAUSE_CHAIN (c))
2566 10146 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
2567 10146 : && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
2568 : {
2569 2620 : tree decl = OMP_CLAUSE_DECL (c);
2570 :
2571 : /* Global variables don't need to be copied,
2572 : the receiver side will use them directly. */
2573 2620 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
2574 282 : continue;
2575 2338 : if (!bitmap_bit_p (make_addressable_vars, DECL_UID (decl))
2576 2338 : || !use_pointer_for_field (decl, ctx))
2577 1787 : continue;
2578 551 : tree field = lookup_field (decl, ctx);
2579 551 : if (TREE_CODE (TREE_TYPE (field)) == POINTER_TYPE
2580 551 : && TREE_TYPE (TREE_TYPE (field)) == TREE_TYPE (decl))
2581 497 : continue;
2582 54 : TREE_TYPE (field) = build_pointer_type (TREE_TYPE (decl));
2583 54 : TREE_THIS_VOLATILE (field) = 0;
2584 54 : DECL_USER_ALIGN (field) = 0;
2585 54 : SET_DECL_ALIGN (field, TYPE_ALIGN (TREE_TYPE (field)));
2586 54 : if (TYPE_ALIGN (ctx->record_type) < DECL_ALIGN (field))
2587 16 : SET_TYPE_ALIGN (ctx->record_type, DECL_ALIGN (field));
2588 54 : if (ctx->srecord_type)
2589 : {
2590 0 : tree sfield = lookup_sfield (decl, ctx);
2591 0 : TREE_TYPE (sfield) = TREE_TYPE (field);
2592 0 : TREE_THIS_VOLATILE (sfield) = 0;
2593 0 : DECL_USER_ALIGN (sfield) = 0;
2594 0 : SET_DECL_ALIGN (sfield, DECL_ALIGN (field));
2595 0 : if (TYPE_ALIGN (ctx->srecord_type) < DECL_ALIGN (sfield))
2596 0 : SET_TYPE_ALIGN (ctx->srecord_type, DECL_ALIGN (sfield));
2597 : }
2598 : }
2599 : }
2600 :
2601 19265 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL)
2602 : {
2603 14618 : tree clauses = gimple_omp_parallel_clauses (ctx->stmt);
2604 14618 : tree c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
2605 14618 : if (c)
2606 : {
2607 : /* Move the _reductemp_ clause first. GOMP_parallel_reductions
2608 : expects to find it at the start of data. */
2609 67 : tree f = lookup_field (OMP_CLAUSE_DECL (c), ctx);
2610 67 : tree *p = &TYPE_FIELDS (ctx->record_type);
2611 153 : while (*p)
2612 153 : if (*p == f)
2613 : {
2614 67 : *p = DECL_CHAIN (*p);
2615 67 : break;
2616 : }
2617 : else
2618 86 : p = &DECL_CHAIN (*p);
2619 67 : DECL_CHAIN (f) = TYPE_FIELDS (ctx->record_type);
2620 67 : TYPE_FIELDS (ctx->record_type) = f;
2621 : }
2622 14618 : layout_type (ctx->record_type);
2623 14618 : fixup_child_record_type (ctx);
2624 : }
2625 4647 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
2626 : {
2627 1222 : layout_type (ctx->record_type);
2628 1222 : fixup_child_record_type (ctx);
2629 : }
2630 : else
2631 : {
2632 3425 : location_t loc = gimple_location (ctx->stmt);
2633 3425 : tree *p, vla_fields = NULL_TREE, *q = &vla_fields;
2634 3425 : tree detach_clause
2635 3425 : = omp_find_clause (gimple_omp_task_clauses (ctx->stmt),
2636 : OMP_CLAUSE_DETACH);
2637 : /* Move VLA fields to the end. */
2638 3425 : p = &TYPE_FIELDS (ctx->record_type);
2639 15329 : while (*p)
2640 11904 : if (!TYPE_SIZE_UNIT (TREE_TYPE (*p))
2641 11904 : || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p))))
2642 : {
2643 96 : *q = *p;
2644 96 : *p = TREE_CHAIN (*p);
2645 96 : TREE_CHAIN (*q) = NULL_TREE;
2646 96 : q = &TREE_CHAIN (*q);
2647 : }
2648 : else
2649 11808 : p = &DECL_CHAIN (*p);
2650 3425 : *p = vla_fields;
2651 3425 : if (gimple_omp_task_taskloop_p (ctx->stmt))
2652 : {
2653 : /* Move fields corresponding to first and second _looptemp_
2654 : clause first. There are filled by GOMP_taskloop
2655 : and thus need to be in specific positions. */
2656 1567 : tree clauses = gimple_omp_task_clauses (ctx->stmt);
2657 1567 : tree c1 = omp_find_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
2658 1567 : tree c2 = omp_find_clause (OMP_CLAUSE_CHAIN (c1),
2659 : OMP_CLAUSE__LOOPTEMP_);
2660 1567 : tree c3 = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
2661 1567 : tree f1 = lookup_field (OMP_CLAUSE_DECL (c1), ctx);
2662 1567 : tree f2 = lookup_field (OMP_CLAUSE_DECL (c2), ctx);
2663 2084 : tree f3 = c3 ? lookup_field (OMP_CLAUSE_DECL (c3), ctx) : NULL_TREE;
2664 1567 : p = &TYPE_FIELDS (ctx->record_type);
2665 8873 : while (*p)
2666 7306 : if (*p == f1 || *p == f2 || *p == f3)
2667 3651 : *p = DECL_CHAIN (*p);
2668 : else
2669 3655 : p = &DECL_CHAIN (*p);
2670 1567 : DECL_CHAIN (f1) = f2;
2671 1567 : if (c3)
2672 : {
2673 517 : DECL_CHAIN (f2) = f3;
2674 517 : DECL_CHAIN (f3) = TYPE_FIELDS (ctx->record_type);
2675 : }
2676 : else
2677 1050 : DECL_CHAIN (f2) = TYPE_FIELDS (ctx->record_type);
2678 1567 : TYPE_FIELDS (ctx->record_type) = f1;
2679 1567 : if (ctx->srecord_type)
2680 : {
2681 389 : f1 = lookup_sfield (OMP_CLAUSE_DECL (c1), ctx);
2682 389 : f2 = lookup_sfield (OMP_CLAUSE_DECL (c2), ctx);
2683 389 : if (c3)
2684 186 : f3 = lookup_sfield (OMP_CLAUSE_DECL (c3), ctx);
2685 389 : p = &TYPE_FIELDS (ctx->srecord_type);
2686 2099 : while (*p)
2687 1710 : if (*p == f1 || *p == f2 || *p == f3)
2688 964 : *p = DECL_CHAIN (*p);
2689 : else
2690 746 : p = &DECL_CHAIN (*p);
2691 389 : DECL_CHAIN (f1) = f2;
2692 389 : DECL_CHAIN (f2) = TYPE_FIELDS (ctx->srecord_type);
2693 389 : if (c3)
2694 : {
2695 186 : DECL_CHAIN (f2) = f3;
2696 186 : DECL_CHAIN (f3) = TYPE_FIELDS (ctx->srecord_type);
2697 : }
2698 : else
2699 203 : DECL_CHAIN (f2) = TYPE_FIELDS (ctx->srecord_type);
2700 389 : TYPE_FIELDS (ctx->srecord_type) = f1;
2701 : }
2702 : }
2703 3425 : if (detach_clause)
2704 : {
2705 189 : tree c, field;
2706 :
2707 : /* Look for a firstprivate clause with the detach event handle. */
2708 540 : for (c = gimple_omp_taskreg_clauses (ctx->stmt);
2709 540 : c; c = OMP_CLAUSE_CHAIN (c))
2710 : {
2711 540 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FIRSTPRIVATE)
2712 347 : continue;
2713 193 : if (maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_DECL (c), ctx)
2714 193 : == OMP_CLAUSE_DECL (detach_clause))
2715 : break;
2716 : }
2717 :
2718 189 : gcc_assert (c);
2719 189 : field = lookup_field (OMP_CLAUSE_DECL (c), ctx);
2720 :
2721 : /* Move field corresponding to the detach clause first.
2722 : This is filled by GOMP_task and needs to be in a
2723 : specific position. */
2724 189 : p = &TYPE_FIELDS (ctx->record_type);
2725 518 : while (*p)
2726 329 : if (*p == field)
2727 189 : *p = DECL_CHAIN (*p);
2728 : else
2729 140 : p = &DECL_CHAIN (*p);
2730 189 : DECL_CHAIN (field) = TYPE_FIELDS (ctx->record_type);
2731 189 : TYPE_FIELDS (ctx->record_type) = field;
2732 189 : if (ctx->srecord_type)
2733 : {
2734 3 : field = lookup_sfield (OMP_CLAUSE_DECL (c), ctx);
2735 3 : p = &TYPE_FIELDS (ctx->srecord_type);
2736 9 : while (*p)
2737 6 : if (*p == field)
2738 3 : *p = DECL_CHAIN (*p);
2739 : else
2740 3 : p = &DECL_CHAIN (*p);
2741 3 : DECL_CHAIN (field) = TYPE_FIELDS (ctx->srecord_type);
2742 3 : TYPE_FIELDS (ctx->srecord_type) = field;
2743 : }
2744 : }
2745 3425 : layout_type (ctx->record_type);
2746 3425 : fixup_child_record_type (ctx);
2747 3425 : if (ctx->srecord_type)
2748 544 : layout_type (ctx->srecord_type);
2749 3425 : tree t = fold_convert_loc (loc, long_integer_type_node,
2750 3425 : TYPE_SIZE_UNIT (ctx->record_type));
2751 3425 : if (TREE_CODE (t) != INTEGER_CST)
2752 : {
2753 21 : t = unshare_expr (t);
2754 21 : walk_tree (&t, finish_taskreg_remap, ctx, NULL);
2755 : }
2756 3425 : gimple_omp_task_set_arg_size (ctx->stmt, t);
2757 3425 : t = build_int_cst (long_integer_type_node,
2758 3425 : TYPE_ALIGN_UNIT (ctx->record_type));
2759 3425 : gimple_omp_task_set_arg_align (ctx->stmt, t);
2760 : }
2761 : }
2762 :
2763 : /* Find the enclosing offload context. */
2764 :
2765 : static omp_context *
2766 9665 : enclosing_target_ctx (omp_context *ctx)
2767 : {
2768 43359 : for (; ctx; ctx = ctx->outer)
2769 41745 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET)
2770 : break;
2771 :
2772 22111 : return ctx;
2773 : }
2774 :
2775 : /* Return whether CTX's parent compute construct is an OpenACC 'kernels'
2776 : construct.
2777 : (This doesn't include OpenACC 'kernels' decomposed parts.) */
2778 :
2779 : static bool
2780 11325 : ctx_in_oacc_kernels_region (omp_context *ctx)
2781 : {
2782 39137 : for (;ctx != NULL; ctx = ctx->outer)
2783 : {
2784 29472 : gimple *stmt = ctx->stmt;
2785 29472 : if (gimple_code (stmt) == GIMPLE_OMP_TARGET
2786 29472 : && gimple_omp_target_kind (stmt) == GF_OMP_TARGET_KIND_OACC_KERNELS)
2787 : return true;
2788 : }
2789 :
2790 : return false;
2791 : }
2792 :
2793 : /* Check the parallelism clauses inside a OpenACC 'kernels' region.
2794 : (This doesn't include OpenACC 'kernels' decomposed parts.)
2795 : Until kernels handling moves to use the same loop indirection
2796 : scheme as parallel, we need to do this checking early. */
2797 :
2798 : static unsigned
2799 7151 : check_oacc_kernel_gwv (gomp_for *stmt, omp_context *ctx)
2800 : {
2801 7151 : bool checking = true;
2802 7151 : unsigned outer_mask = 0;
2803 7151 : unsigned this_mask = 0;
2804 7151 : bool has_seq = false, has_auto = false;
2805 :
2806 7151 : if (ctx->outer)
2807 4819 : outer_mask = check_oacc_kernel_gwv (NULL, ctx->outer);
2808 7151 : if (!stmt)
2809 : {
2810 4819 : checking = false;
2811 4819 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR)
2812 : return outer_mask;
2813 1730 : stmt = as_a <gomp_for *> (ctx->stmt);
2814 : }
2815 :
2816 11330 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
2817 : {
2818 7268 : switch (OMP_CLAUSE_CODE (c))
2819 : {
2820 645 : case OMP_CLAUSE_GANG:
2821 645 : this_mask |= GOMP_DIM_MASK (GOMP_DIM_GANG);
2822 645 : break;
2823 439 : case OMP_CLAUSE_WORKER:
2824 439 : this_mask |= GOMP_DIM_MASK (GOMP_DIM_WORKER);
2825 439 : break;
2826 342 : case OMP_CLAUSE_VECTOR:
2827 342 : this_mask |= GOMP_DIM_MASK (GOMP_DIM_VECTOR);
2828 342 : break;
2829 63 : case OMP_CLAUSE_SEQ:
2830 63 : has_seq = true;
2831 63 : break;
2832 94 : case OMP_CLAUSE_AUTO:
2833 94 : has_auto = true;
2834 94 : break;
2835 : default:
2836 : break;
2837 : }
2838 : }
2839 :
2840 4062 : if (checking)
2841 : {
2842 2332 : if (has_seq && (this_mask || has_auto))
2843 36 : error_at (gimple_location (stmt), "%<seq%> overrides other"
2844 : " OpenACC loop specifiers");
2845 2296 : else if (has_auto && this_mask)
2846 30 : error_at (gimple_location (stmt), "%<auto%> conflicts with other"
2847 : " OpenACC loop specifiers");
2848 :
2849 2332 : if (this_mask & outer_mask)
2850 15 : error_at (gimple_location (stmt), "inner loop uses same"
2851 : " OpenACC parallelism as containing loop");
2852 : }
2853 :
2854 4062 : return outer_mask | this_mask;
2855 : }
2856 :
2857 : /* Scan a GIMPLE_OMP_FOR. */
2858 :
2859 : static omp_context *
2860 52725 : scan_omp_for (gomp_for *stmt, omp_context *outer_ctx)
2861 : {
2862 52725 : omp_context *ctx;
2863 52725 : size_t i;
2864 52725 : tree clauses = gimple_omp_for_clauses (stmt);
2865 :
2866 52725 : ctx = new_omp_context (stmt, outer_ctx);
2867 :
2868 52725 : if (is_gimple_omp_oacc (stmt))
2869 : {
2870 12446 : omp_context *tgt = enclosing_target_ctx (outer_ctx);
2871 :
2872 12446 : if (!(tgt && is_oacc_kernels (tgt)))
2873 32251 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2874 : {
2875 22137 : tree c_op0;
2876 22137 : switch (OMP_CLAUSE_CODE (c))
2877 : {
2878 1952 : case OMP_CLAUSE_GANG:
2879 1952 : c_op0 = OMP_CLAUSE_GANG_EXPR (c);
2880 1952 : break;
2881 :
2882 1426 : case OMP_CLAUSE_WORKER:
2883 1426 : c_op0 = OMP_CLAUSE_WORKER_EXPR (c);
2884 1426 : break;
2885 :
2886 1627 : case OMP_CLAUSE_VECTOR:
2887 1627 : c_op0 = OMP_CLAUSE_VECTOR_EXPR (c);
2888 1627 : break;
2889 :
2890 17132 : default:
2891 17132 : continue;
2892 : }
2893 :
2894 5005 : if (c_op0)
2895 : {
2896 : /* By construction, this is impossible for OpenACC 'kernels'
2897 : decomposed parts. */
2898 210 : gcc_assert (!(tgt && is_oacc_kernels_decomposed_part (tgt)));
2899 :
2900 210 : error_at (OMP_CLAUSE_LOCATION (c),
2901 : "argument not permitted on %qs clause",
2902 210 : omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
2903 210 : if (tgt)
2904 180 : inform (gimple_location (tgt->stmt),
2905 : "enclosing parent compute construct");
2906 30 : else if (oacc_get_fn_attrib (current_function_decl))
2907 30 : inform (DECL_SOURCE_LOCATION (current_function_decl),
2908 : "enclosing routine");
2909 : else
2910 0 : gcc_unreachable ();
2911 : }
2912 : }
2913 :
2914 12446 : if (tgt && is_oacc_kernels (tgt))
2915 2332 : check_oacc_kernel_gwv (stmt, ctx);
2916 :
2917 : /* Collect all variables named in reductions on this loop. Ensure
2918 : that, if this loop has a reduction on some variable v, and there is
2919 : a reduction on v somewhere in an outer context, then there is a
2920 : reduction on v on all intervening loops as well. */
2921 12446 : tree local_reduction_clauses = NULL;
2922 39374 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
2923 : {
2924 26928 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
2925 4712 : local_reduction_clauses
2926 4712 : = tree_cons (NULL, c, local_reduction_clauses);
2927 : }
2928 12446 : if (ctx->outer_reduction_clauses == NULL && ctx->outer != NULL)
2929 11920 : ctx->outer_reduction_clauses
2930 11920 : = chainon (unshare_expr (ctx->outer->local_reduction_clauses),
2931 : ctx->outer->outer_reduction_clauses);
2932 12446 : tree outer_reduction_clauses = ctx->outer_reduction_clauses;
2933 12446 : tree local_iter = local_reduction_clauses;
2934 17158 : for (; local_iter; local_iter = TREE_CHAIN (local_iter))
2935 : {
2936 4712 : tree local_clause = TREE_VALUE (local_iter);
2937 4712 : tree local_var = OMP_CLAUSE_DECL (local_clause);
2938 4712 : tree_code local_op = OMP_CLAUSE_REDUCTION_CODE (local_clause);
2939 4712 : bool have_outer_reduction = false;
2940 4712 : tree ctx_iter = outer_reduction_clauses;
2941 5826 : for (; ctx_iter; ctx_iter = TREE_CHAIN (ctx_iter))
2942 : {
2943 3325 : tree outer_clause = TREE_VALUE (ctx_iter);
2944 3325 : tree outer_var = OMP_CLAUSE_DECL (outer_clause);
2945 3325 : tree_code outer_op = OMP_CLAUSE_REDUCTION_CODE (outer_clause);
2946 3325 : if (outer_var == local_var && outer_op != local_op)
2947 : {
2948 535 : if (warning_at (OMP_CLAUSE_LOCATION (local_clause),
2949 535 : OPT_Wopenmp, "conflicting reduction "
2950 : "operations for %qE",
2951 : local_var))
2952 535 : inform (OMP_CLAUSE_LOCATION (outer_clause),
2953 : "location of the previous reduction for %qE",
2954 : outer_var);
2955 : }
2956 3325 : if (outer_var == local_var)
2957 : {
2958 : have_outer_reduction = true;
2959 : break;
2960 : }
2961 : }
2962 4712 : if (have_outer_reduction)
2963 : {
2964 : /* There is a reduction on outer_var both on this loop and on
2965 : some enclosing loop. Walk up the context tree until such a
2966 : loop with a reduction on outer_var is found, and complain
2967 : about all intervening loops that do not have such a
2968 : reduction. */
2969 2211 : struct omp_context *curr_loop = ctx->outer;
2970 2211 : bool found = false;
2971 2826 : while (curr_loop != NULL)
2972 : {
2973 2826 : tree curr_iter = curr_loop->local_reduction_clauses;
2974 3460 : for (; curr_iter; curr_iter = TREE_CHAIN (curr_iter))
2975 : {
2976 2845 : tree curr_clause = TREE_VALUE (curr_iter);
2977 2845 : tree curr_var = OMP_CLAUSE_DECL (curr_clause);
2978 2845 : if (curr_var == local_var)
2979 : {
2980 : found = true;
2981 : break;
2982 : }
2983 : }
2984 2826 : if (!found)
2985 615 : warning_at (gimple_location (curr_loop->stmt), OPT_Wopenmp,
2986 : "nested loop in reduction needs "
2987 : "reduction clause for %qE",
2988 : local_var);
2989 : else
2990 : break;
2991 615 : curr_loop = curr_loop->outer;
2992 : }
2993 : }
2994 : }
2995 12446 : ctx->local_reduction_clauses = local_reduction_clauses;
2996 12446 : ctx->outer_reduction_clauses
2997 12446 : = chainon (unshare_expr (ctx->local_reduction_clauses),
2998 : ctx->outer_reduction_clauses);
2999 :
3000 12446 : if (tgt && is_oacc_kernels (tgt))
3001 : {
3002 : /* Strip out reductions, as they are not handled yet. */
3003 : tree *prev_ptr = &clauses;
3004 :
3005 7123 : while (tree probe = *prev_ptr)
3006 : {
3007 4791 : tree *next_ptr = &OMP_CLAUSE_CHAIN (probe);
3008 :
3009 4791 : if (OMP_CLAUSE_CODE (probe) == OMP_CLAUSE_REDUCTION)
3010 718 : *prev_ptr = *next_ptr;
3011 : else
3012 : prev_ptr = next_ptr;
3013 : }
3014 :
3015 2332 : gimple_omp_for_set_clauses (stmt, clauses);
3016 : }
3017 : }
3018 :
3019 52725 : scan_sharing_clauses (clauses, ctx);
3020 :
3021 52725 : scan_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
3022 180213 : for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
3023 : {
3024 74763 : scan_omp_op (gimple_omp_for_index_ptr (stmt, i), ctx);
3025 74763 : scan_omp_op (gimple_omp_for_initial_ptr (stmt, i), ctx);
3026 74763 : scan_omp_op (gimple_omp_for_final_ptr (stmt, i), ctx);
3027 74763 : scan_omp_op (gimple_omp_for_incr_ptr (stmt, i), ctx);
3028 : }
3029 52725 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3030 52725 : return ctx;
3031 : }
3032 :
3033 : /* Duplicate #pragma omp simd, one for SIMT, another one for SIMD. */
3034 :
3035 : static void
3036 0 : scan_omp_simd (gimple_stmt_iterator *gsi, gomp_for *stmt,
3037 : omp_context *outer_ctx)
3038 : {
3039 0 : gbind *bind = gimple_build_bind (NULL, NULL, NULL);
3040 0 : gsi_replace (gsi, bind, false);
3041 0 : gimple_seq seq = NULL;
3042 0 : gimple *g = gimple_build_call_internal (IFN_GOMP_USE_SIMT, 0);
3043 0 : tree cond = create_tmp_var_raw (integer_type_node);
3044 0 : DECL_CONTEXT (cond) = current_function_decl;
3045 0 : DECL_SEEN_IN_BIND_EXPR_P (cond) = 1;
3046 0 : gimple_bind_set_vars (bind, cond);
3047 0 : gimple_call_set_lhs (g, cond);
3048 0 : gimple_seq_add_stmt (&seq, g);
3049 0 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
3050 0 : tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
3051 0 : tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
3052 0 : g = gimple_build_cond (NE_EXPR, cond, integer_zero_node, lab1, lab2);
3053 0 : gimple_seq_add_stmt (&seq, g);
3054 0 : g = gimple_build_label (lab1);
3055 0 : gimple_seq_add_stmt (&seq, g);
3056 0 : gimple_seq new_seq = copy_gimple_seq_and_replace_locals (stmt);
3057 0 : gomp_for *new_stmt = as_a <gomp_for *> (new_seq);
3058 0 : tree clause = build_omp_clause (gimple_location (stmt), OMP_CLAUSE__SIMT_);
3059 0 : OMP_CLAUSE_CHAIN (clause) = gimple_omp_for_clauses (new_stmt);
3060 0 : gimple_omp_for_set_clauses (new_stmt, clause);
3061 0 : gimple_seq_add_stmt (&seq, new_stmt);
3062 0 : g = gimple_build_goto (lab3);
3063 0 : gimple_seq_add_stmt (&seq, g);
3064 0 : g = gimple_build_label (lab2);
3065 0 : gimple_seq_add_stmt (&seq, g);
3066 0 : gimple_seq_add_stmt (&seq, stmt);
3067 0 : g = gimple_build_label (lab3);
3068 0 : gimple_seq_add_stmt (&seq, g);
3069 0 : gimple_bind_set_body (bind, seq);
3070 0 : update_stmt (bind);
3071 0 : scan_omp_for (new_stmt, outer_ctx);
3072 0 : scan_omp_for (stmt, outer_ctx)->simt_stmt = new_stmt;
3073 0 : }
3074 :
3075 : static tree omp_find_scan (gimple_stmt_iterator *, bool *,
3076 : struct walk_stmt_info *);
3077 : static omp_context *maybe_lookup_ctx (gimple *);
3078 :
3079 : /* Duplicate #pragma omp simd, one for the scan input phase loop and one
3080 : for scan phase loop. */
3081 :
3082 : static void
3083 83 : scan_omp_simd_scan (gimple_stmt_iterator *gsi, gomp_for *stmt,
3084 : omp_context *outer_ctx)
3085 : {
3086 : /* The only change between inclusive and exclusive scan will be
3087 : within the first simd loop, so just use inclusive in the
3088 : worksharing loop. */
3089 83 : outer_ctx->scan_inclusive = true;
3090 83 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_INCLUSIVE);
3091 83 : OMP_CLAUSE_DECL (c) = integer_zero_node;
3092 :
3093 83 : gomp_scan *input_stmt = gimple_build_omp_scan (NULL, NULL_TREE);
3094 83 : gomp_scan *scan_stmt = gimple_build_omp_scan (NULL, c);
3095 83 : gsi_replace (gsi, input_stmt, false);
3096 83 : gimple_seq input_body = NULL;
3097 83 : gimple_seq_add_stmt (&input_body, stmt);
3098 83 : gsi_insert_after (gsi, scan_stmt, GSI_NEW_STMT);
3099 :
3100 83 : gimple_stmt_iterator input1_gsi = gsi_none ();
3101 83 : struct walk_stmt_info wi;
3102 83 : memset (&wi, 0, sizeof (wi));
3103 83 : wi.val_only = true;
3104 83 : wi.info = (void *) &input1_gsi;
3105 83 : walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), omp_find_scan, NULL, &wi);
3106 83 : gcc_assert (!gsi_end_p (input1_gsi));
3107 :
3108 83 : gimple *input_stmt1 = gsi_stmt (input1_gsi);
3109 83 : gsi_next (&input1_gsi);
3110 83 : gimple *scan_stmt1 = gsi_stmt (input1_gsi);
3111 83 : gcc_assert (scan_stmt1 && gimple_code (scan_stmt1) == GIMPLE_OMP_SCAN);
3112 83 : c = gimple_omp_scan_clauses (as_a <gomp_scan *> (scan_stmt1));
3113 166 : if (c && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_EXCLUSIVE)
3114 : std::swap (input_stmt1, scan_stmt1);
3115 :
3116 83 : gimple_seq input_body1 = gimple_omp_body (input_stmt1);
3117 83 : gimple_omp_set_body (input_stmt1, NULL);
3118 :
3119 83 : gimple_seq scan_body = copy_gimple_seq_and_replace_locals (stmt);
3120 83 : gomp_for *new_stmt = as_a <gomp_for *> (scan_body);
3121 :
3122 83 : gimple_omp_set_body (input_stmt1, input_body1);
3123 83 : gimple_omp_set_body (scan_stmt1, NULL);
3124 :
3125 83 : gimple_stmt_iterator input2_gsi = gsi_none ();
3126 83 : memset (&wi, 0, sizeof (wi));
3127 83 : wi.val_only = true;
3128 83 : wi.info = (void *) &input2_gsi;
3129 83 : walk_gimple_seq_mod (gimple_omp_body_ptr (new_stmt), omp_find_scan,
3130 : NULL, &wi);
3131 83 : gcc_assert (!gsi_end_p (input2_gsi));
3132 :
3133 83 : gimple *input_stmt2 = gsi_stmt (input2_gsi);
3134 83 : gsi_next (&input2_gsi);
3135 83 : gimple *scan_stmt2 = gsi_stmt (input2_gsi);
3136 83 : gcc_assert (scan_stmt2 && gimple_code (scan_stmt2) == GIMPLE_OMP_SCAN);
3137 166 : if (c && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_EXCLUSIVE)
3138 : std::swap (input_stmt2, scan_stmt2);
3139 :
3140 83 : gimple_omp_set_body (input_stmt2, NULL);
3141 :
3142 83 : gimple_omp_set_body (input_stmt, input_body);
3143 83 : gimple_omp_set_body (scan_stmt, scan_body);
3144 :
3145 83 : omp_context *ctx = new_omp_context (input_stmt, outer_ctx);
3146 83 : scan_omp (gimple_omp_body_ptr (input_stmt), ctx);
3147 :
3148 83 : ctx = new_omp_context (scan_stmt, outer_ctx);
3149 83 : scan_omp (gimple_omp_body_ptr (scan_stmt), ctx);
3150 :
3151 83 : maybe_lookup_ctx (new_stmt)->for_simd_scan_phase = true;
3152 83 : }
3153 :
3154 : /* Scan an OpenMP sections directive. */
3155 :
3156 : static void
3157 589 : scan_omp_sections (gomp_sections *stmt, omp_context *outer_ctx)
3158 : {
3159 589 : omp_context *ctx;
3160 :
3161 589 : ctx = new_omp_context (stmt, outer_ctx);
3162 589 : scan_sharing_clauses (gimple_omp_sections_clauses (stmt), ctx);
3163 589 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3164 589 : }
3165 :
3166 : /* Scan an OpenMP single directive. */
3167 :
3168 : static void
3169 1236 : scan_omp_single (gomp_single *stmt, omp_context *outer_ctx)
3170 : {
3171 1236 : omp_context *ctx;
3172 1236 : tree name;
3173 :
3174 1236 : ctx = new_omp_context (stmt, outer_ctx);
3175 1236 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
3176 1236 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
3177 1236 : name = create_tmp_var_name (".omp_copy_s");
3178 1236 : name = build_decl (gimple_location (stmt),
3179 : TYPE_DECL, name, ctx->record_type);
3180 1236 : TYPE_NAME (ctx->record_type) = name;
3181 :
3182 1236 : scan_sharing_clauses (gimple_omp_single_clauses (stmt), ctx);
3183 1236 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3184 :
3185 1236 : if (TYPE_FIELDS (ctx->record_type) == NULL)
3186 1089 : ctx->record_type = NULL;
3187 : else
3188 147 : layout_type (ctx->record_type);
3189 1236 : }
3190 :
3191 : /* Scan a GIMPLE_OMP_TARGET. */
3192 :
3193 : static void
3194 41865 : scan_omp_target (gomp_target *stmt, omp_context *outer_ctx)
3195 : {
3196 41865 : omp_context *ctx;
3197 41865 : tree name;
3198 41865 : bool offloaded = is_gimple_omp_offloaded (stmt);
3199 41865 : tree clauses = gimple_omp_target_clauses (stmt);
3200 :
3201 41865 : ctx = new_omp_context (stmt, outer_ctx);
3202 41865 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
3203 41865 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
3204 41865 : name = create_tmp_var_name (".omp_data_t");
3205 41865 : name = build_decl (gimple_location (stmt),
3206 : TYPE_DECL, name, ctx->record_type);
3207 41865 : DECL_ARTIFICIAL (name) = 1;
3208 41865 : DECL_NAMELESS (name) = 1;
3209 41865 : TYPE_NAME (ctx->record_type) = name;
3210 41865 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
3211 :
3212 41865 : if (offloaded)
3213 : {
3214 24836 : tree c = omp_find_clause (clauses, OMP_CLAUSE_DEVICE_TYPE);
3215 24836 : bool host_only
3216 24836 : = c && OMP_CLAUSE_DEVICE_TYPE_KIND (c) == OMP_CLAUSE_DEVICE_TYPE_HOST;
3217 24836 : create_omp_child_function (ctx, false, host_only);
3218 24836 : gimple_omp_target_set_child_fn (stmt, ctx->cb.dst_fn);
3219 : }
3220 :
3221 41865 : scan_sharing_clauses (clauses, ctx);
3222 41865 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3223 :
3224 41865 : if (TYPE_FIELDS (ctx->record_type) == NULL)
3225 7768 : ctx->record_type = ctx->receiver_decl = NULL;
3226 : else
3227 : {
3228 34097 : TYPE_FIELDS (ctx->record_type)
3229 34097 : = nreverse (TYPE_FIELDS (ctx->record_type));
3230 34097 : if (flag_checking)
3231 : {
3232 34097 : unsigned int align = DECL_ALIGN (TYPE_FIELDS (ctx->record_type));
3233 34097 : for (tree field = TYPE_FIELDS (ctx->record_type);
3234 131392 : field;
3235 97295 : field = DECL_CHAIN (field))
3236 97295 : gcc_assert (DECL_ALIGN (field) == align);
3237 : }
3238 34097 : layout_type (ctx->record_type);
3239 34097 : if (offloaded)
3240 17627 : fixup_child_record_type (ctx);
3241 : }
3242 :
3243 41865 : if (ctx->teams_nested_p && ctx->nonteams_nested_p)
3244 : {
3245 4 : error_at (gimple_location (stmt),
3246 : "%<target%> construct with nested %<teams%> construct "
3247 : "contains directives outside of the %<teams%> construct");
3248 4 : gimple_omp_set_body (stmt, gimple_build_bind (NULL, NULL, NULL));
3249 : }
3250 41865 : }
3251 :
3252 : /* Scan an OpenMP teams directive. */
3253 :
3254 : static void
3255 8797 : scan_omp_teams (gomp_teams *stmt, omp_context *outer_ctx)
3256 : {
3257 8797 : omp_context *ctx = new_omp_context (stmt, outer_ctx);
3258 :
3259 8797 : if (!gimple_omp_teams_host (stmt))
3260 : {
3261 6126 : scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
3262 6126 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3263 6126 : return;
3264 : }
3265 2671 : taskreg_contexts.safe_push (ctx);
3266 2671 : gcc_assert (taskreg_nesting_level == 1);
3267 2671 : ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
3268 2671 : ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
3269 2671 : tree name = create_tmp_var_name (".omp_data_s");
3270 2671 : name = build_decl (gimple_location (stmt),
3271 : TYPE_DECL, name, ctx->record_type);
3272 2671 : DECL_ARTIFICIAL (name) = 1;
3273 2671 : DECL_NAMELESS (name) = 1;
3274 2671 : TYPE_NAME (ctx->record_type) = name;
3275 2671 : TYPE_ARTIFICIAL (ctx->record_type) = 1;
3276 2671 : create_omp_child_function (ctx, false, false);
3277 2671 : gimple_omp_teams_set_child_fn (stmt, ctx->cb.dst_fn);
3278 :
3279 2671 : scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
3280 2671 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
3281 :
3282 2671 : if (TYPE_FIELDS (ctx->record_type) == NULL)
3283 1449 : ctx->record_type = ctx->receiver_decl = NULL;
3284 : }
3285 :
3286 : /* Check nesting restrictions. */
3287 : static bool
3288 157188 : check_omp_nesting_restrictions (gimple *stmt, omp_context *ctx)
3289 : {
3290 157188 : tree c;
3291 :
3292 : /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
3293 : inside an OpenACC CTX. */
3294 157188 : if (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
3295 157188 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE)
3296 : /* ..., except for the atomic codes that OpenACC shares with OpenMP. */
3297 : ;
3298 282499 : else if (!(is_gimple_omp (stmt)
3299 139139 : && is_gimple_omp_oacc (stmt)))
3300 : {
3301 113096 : if (oacc_get_fn_attrib (cfun->decl) != NULL)
3302 : {
3303 48 : error_at (gimple_location (stmt),
3304 : "non-OpenACC construct inside of OpenACC routine");
3305 48 : return false;
3306 : }
3307 : else
3308 236650 : for (omp_context *octx = ctx; octx != NULL; octx = octx->outer)
3309 247380 : if (is_gimple_omp (octx->stmt)
3310 123778 : && is_gimple_omp_oacc (octx->stmt))
3311 : {
3312 176 : error_at (gimple_location (stmt),
3313 : "non-OpenACC construct inside of OpenACC region");
3314 176 : return false;
3315 : }
3316 : }
3317 :
3318 156964 : if (ctx != NULL)
3319 : {
3320 84256 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
3321 84256 : && gimple_omp_target_kind (ctx->stmt) == GF_OMP_TARGET_KIND_REGION)
3322 : {
3323 8335 : c = omp_find_clause (gimple_omp_target_clauses (ctx->stmt),
3324 : OMP_CLAUSE_DEVICE);
3325 8335 : if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
3326 : {
3327 9 : error_at (gimple_location (stmt),
3328 : "OpenMP constructs are not allowed in target region "
3329 : "with %<ancestor%>");
3330 9 : return false;
3331 : }
3332 :
3333 8326 : if (gimple_code (stmt) == GIMPLE_OMP_TEAMS && !ctx->teams_nested_p)
3334 6122 : ctx->teams_nested_p = true;
3335 : else
3336 2204 : ctx->nonteams_nested_p = true;
3337 : }
3338 84247 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_SCAN
3339 172 : && ctx->outer
3340 84419 : && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR)
3341 : ctx = ctx->outer;
3342 84247 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3343 28325 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
3344 86056 : && !ctx->loop_p)
3345 : {
3346 1589 : c = NULL_TREE;
3347 1589 : if (ctx->order_concurrent
3348 1589 : && (gimple_code (stmt) == GIMPLE_OMP_ORDERED
3349 280 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
3350 190 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE))
3351 : {
3352 210 : error_at (gimple_location (stmt),
3353 : "OpenMP constructs other than %<parallel%>, %<loop%>"
3354 : " or %<simd%> may not be nested inside a region with"
3355 : " the %<order(concurrent)%> clause");
3356 210 : return false;
3357 : }
3358 1379 : if (gimple_code (stmt) == GIMPLE_OMP_ORDERED)
3359 : {
3360 133 : c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
3361 133 : if (omp_find_clause (c, OMP_CLAUSE_SIMD))
3362 : {
3363 133 : if (omp_find_clause (c, OMP_CLAUSE_THREADS)
3364 133 : && (ctx->outer == NULL
3365 29 : || !gimple_omp_for_combined_into_p (ctx->stmt)
3366 25 : || gimple_code (ctx->outer->stmt) != GIMPLE_OMP_FOR
3367 25 : || (gimple_omp_for_kind (ctx->outer->stmt)
3368 : != GF_OMP_FOR_KIND_FOR)
3369 25 : || !gimple_omp_for_combined_p (ctx->outer->stmt)))
3370 : {
3371 8 : error_at (gimple_location (stmt),
3372 : "%<ordered simd threads%> must be closely "
3373 : "nested inside of %<%s simd%> region",
3374 8 : lang_GNU_Fortran () ? "do" : "for");
3375 8 : return false;
3376 : }
3377 : return true;
3378 : }
3379 : }
3380 : else if (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
3381 : || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE
3382 : || gimple_code (stmt) == GIMPLE_OMP_SCAN
3383 : || gimple_code (stmt) == GIMPLE_OMP_STRUCTURED_BLOCK)
3384 : return true;
3385 : else if (gimple_code (stmt) == GIMPLE_OMP_FOR
3386 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
3387 : return true;
3388 60 : error_at (gimple_location (stmt),
3389 : "OpenMP constructs other than "
3390 : "%<ordered simd%>, %<simd%>, %<loop%> or %<atomic%> may "
3391 : "not be nested inside %<simd%> region");
3392 60 : return false;
3393 : }
3394 82658 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
3395 : {
3396 5873 : if ((gimple_code (stmt) != GIMPLE_OMP_FOR
3397 5555 : || (gimple_omp_for_kind (stmt) != GF_OMP_FOR_KIND_DISTRIBUTE
3398 25 : && omp_find_clause (gimple_omp_for_clauses (stmt),
3399 : OMP_CLAUSE_BIND) == NULL_TREE))
3400 5889 : && gimple_code (stmt) != GIMPLE_OMP_PARALLEL)
3401 : {
3402 132 : error_at (gimple_location (stmt),
3403 : "only %<distribute%>, %<parallel%> or %<loop%> "
3404 : "regions are allowed to be strictly nested inside "
3405 : "%<teams%> region");
3406 132 : return false;
3407 : }
3408 : }
3409 76785 : else if (ctx->order_concurrent
3410 1944 : && gimple_code (stmt) != GIMPLE_OMP_PARALLEL
3411 1465 : && (gimple_code (stmt) != GIMPLE_OMP_FOR
3412 1191 : || gimple_omp_for_kind (stmt) != GF_OMP_FOR_KIND_SIMD)
3413 289 : && gimple_code (stmt) != GIMPLE_OMP_SCAN
3414 77074 : && gimple_code (stmt) != GIMPLE_OMP_STRUCTURED_BLOCK)
3415 : {
3416 285 : if (ctx->loop_p)
3417 135 : error_at (gimple_location (stmt),
3418 : "OpenMP constructs other than %<parallel%>, %<loop%> or "
3419 : "%<simd%> may not be nested inside a %<loop%> region");
3420 : else
3421 150 : error_at (gimple_location (stmt),
3422 : "OpenMP constructs other than %<parallel%>, %<loop%> or "
3423 : "%<simd%> may not be nested inside a region with "
3424 : "the %<order(concurrent)%> clause");
3425 285 : return false;
3426 : }
3427 : }
3428 154949 : switch (gimple_code (stmt))
3429 : {
3430 52844 : case GIMPLE_OMP_FOR:
3431 52844 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_SIMD)
3432 : return true;
3433 42083 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
3434 : {
3435 8247 : if (ctx != NULL && gimple_code (ctx->stmt) != GIMPLE_OMP_TEAMS)
3436 : {
3437 0 : error_at (gimple_location (stmt),
3438 : "%<distribute%> region must be strictly nested "
3439 : "inside %<teams%> construct");
3440 0 : return false;
3441 : }
3442 : return true;
3443 : }
3444 : /* We split taskloop into task and nested taskloop in it. */
3445 33836 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_TASKLOOP)
3446 : return true;
3447 : /* For now, hope this will change and loop bind(parallel) will not
3448 : be allowed in lots of contexts. */
3449 30702 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR
3450 48893 : && omp_find_clause (gimple_omp_for_clauses (stmt), OMP_CLAUSE_BIND))
3451 : return true;
3452 30073 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
3453 : {
3454 12511 : bool ok = false;
3455 :
3456 12511 : if (ctx)
3457 11964 : switch (gimple_code (ctx->stmt))
3458 : {
3459 4049 : case GIMPLE_OMP_FOR:
3460 4049 : ok = (gimple_omp_for_kind (ctx->stmt)
3461 : == GF_OMP_FOR_KIND_OACC_LOOP);
3462 4049 : break;
3463 :
3464 7887 : case GIMPLE_OMP_TARGET:
3465 7887 : switch (gimple_omp_target_kind (ctx->stmt))
3466 : {
3467 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
3468 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
3469 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
3470 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
3471 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
3472 : ok = true;
3473 : break;
3474 :
3475 : default:
3476 : break;
3477 : }
3478 :
3479 : default:
3480 : break;
3481 : }
3482 547 : else if (oacc_get_fn_attrib (current_function_decl))
3483 : ok = true;
3484 4049 : if (!ok)
3485 : {
3486 65 : error_at (gimple_location (stmt),
3487 : "OpenACC loop directive must be associated with"
3488 : " an OpenACC compute region");
3489 65 : return false;
3490 : }
3491 : }
3492 : /* FALLTHRU */
3493 34169 : case GIMPLE_CALL:
3494 34169 : if (is_gimple_call (stmt)
3495 34169 : && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3496 : == BUILT_IN_GOMP_CANCEL
3497 3057 : || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3498 : == BUILT_IN_GOMP_CANCELLATION_POINT))
3499 : {
3500 1907 : const char *bad = NULL;
3501 1907 : const char *kind = NULL;
3502 1907 : const char *construct
3503 1907 : = (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3504 : == BUILT_IN_GOMP_CANCEL)
3505 1907 : ? "cancel"
3506 803 : : "cancellation point";
3507 1907 : if (ctx == NULL)
3508 : {
3509 41 : error_at (gimple_location (stmt), "orphaned %qs construct",
3510 : construct);
3511 41 : return false;
3512 : }
3513 1866 : switch (tree_fits_shwi_p (gimple_call_arg (stmt, 0))
3514 1866 : ? tree_to_shwi (gimple_call_arg (stmt, 0))
3515 : : 0)
3516 : {
3517 528 : case 1:
3518 528 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_PARALLEL)
3519 : bad = "parallel";
3520 208 : else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3521 : == BUILT_IN_GOMP_CANCEL
3522 208 : && !integer_zerop (gimple_call_arg (stmt, 1)))
3523 164 : ctx->cancellable = true;
3524 164 : kind = "parallel";
3525 164 : break;
3526 427 : case 2:
3527 427 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
3528 427 : || gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR)
3529 : bad = "for";
3530 117 : else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3531 : == BUILT_IN_GOMP_CANCEL
3532 117 : && !integer_zerop (gimple_call_arg (stmt, 1)))
3533 : {
3534 106 : ctx->cancellable = true;
3535 106 : if (omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3536 : OMP_CLAUSE_NOWAIT))
3537 5 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3538 : "%<cancel for%> inside "
3539 : "%<nowait%> for construct");
3540 106 : if (omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3541 : OMP_CLAUSE_ORDERED))
3542 5 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3543 : "%<cancel for%> inside "
3544 : "%<ordered%> for construct");
3545 : }
3546 5 : kind = "for";
3547 5 : break;
3548 383 : case 4:
3549 383 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_SECTIONS
3550 383 : && gimple_code (ctx->stmt) != GIMPLE_OMP_SECTION)
3551 : bad = "sections";
3552 93 : else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3553 : == BUILT_IN_GOMP_CANCEL
3554 93 : && !integer_zerop (gimple_call_arg (stmt, 1)))
3555 : {
3556 73 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
3557 : {
3558 0 : ctx->cancellable = true;
3559 0 : if (omp_find_clause (gimple_omp_sections_clauses
3560 0 : (ctx->stmt),
3561 : OMP_CLAUSE_NOWAIT))
3562 0 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3563 : "%<cancel sections%> inside "
3564 : "%<nowait%> sections construct");
3565 : }
3566 : else
3567 : {
3568 73 : gcc_assert (ctx->outer
3569 : && gimple_code (ctx->outer->stmt)
3570 : == GIMPLE_OMP_SECTIONS);
3571 73 : ctx->outer->cancellable = true;
3572 73 : if (omp_find_clause (gimple_omp_sections_clauses
3573 73 : (ctx->outer->stmt),
3574 : OMP_CLAUSE_NOWAIT))
3575 10 : warning_at (gimple_location (stmt), OPT_Wopenmp,
3576 : "%<cancel sections%> inside "
3577 : "%<nowait%> sections construct");
3578 : }
3579 : }
3580 10 : kind = "sections";
3581 10 : break;
3582 528 : case 8:
3583 528 : if (!is_task_ctx (ctx)
3584 528 : && (!is_taskloop_ctx (ctx)
3585 54 : || ctx->outer == NULL
3586 54 : || !is_task_ctx (ctx->outer)))
3587 : bad = "task";
3588 : else
3589 : {
3590 248 : for (omp_context *octx = ctx->outer;
3591 439 : octx; octx = octx->outer)
3592 : {
3593 425 : switch (gimple_code (octx->stmt))
3594 : {
3595 : case GIMPLE_OMP_TASKGROUP:
3596 : break;
3597 40 : case GIMPLE_OMP_TARGET:
3598 40 : if (gimple_omp_target_kind (octx->stmt)
3599 : != GF_OMP_TARGET_KIND_REGION)
3600 20 : continue;
3601 : /* FALLTHRU */
3602 99 : case GIMPLE_OMP_PARALLEL:
3603 99 : case GIMPLE_OMP_TEAMS:
3604 99 : error_at (gimple_location (stmt),
3605 : "%<%s taskgroup%> construct not closely "
3606 : "nested inside of %<taskgroup%> region",
3607 : construct);
3608 99 : return false;
3609 104 : case GIMPLE_OMP_TASK:
3610 104 : if (gimple_omp_task_taskloop_p (octx->stmt)
3611 94 : && octx->outer
3612 198 : && is_taskloop_ctx (octx->outer))
3613 : {
3614 94 : tree clauses
3615 94 : = gimple_omp_for_clauses (octx->outer->stmt);
3616 94 : if (!omp_find_clause (clauses, OMP_CLAUSE_NOGROUP))
3617 : break;
3618 : }
3619 60 : continue;
3620 111 : default:
3621 111 : continue;
3622 171 : }
3623 : break;
3624 : }
3625 149 : ctx->cancellable = true;
3626 : }
3627 149 : kind = "taskgroup";
3628 149 : break;
3629 0 : default:
3630 0 : error_at (gimple_location (stmt), "invalid arguments");
3631 0 : return false;
3632 : }
3633 328 : if (bad)
3634 : {
3635 1200 : error_at (gimple_location (stmt),
3636 : "%<%s %s%> construct not closely nested inside of %qs",
3637 : construct, kind, bad);
3638 1200 : return false;
3639 : }
3640 : }
3641 : /* FALLTHRU */
3642 : case GIMPLE_OMP_SECTIONS:
3643 : case GIMPLE_OMP_SINGLE:
3644 55645 : for (; ctx != NULL; ctx = ctx->outer)
3645 37534 : switch (gimple_code (ctx->stmt))
3646 : {
3647 6873 : case GIMPLE_OMP_FOR:
3648 6873 : if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
3649 6873 : && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
3650 : break;
3651 : /* FALLTHRU */
3652 778 : case GIMPLE_OMP_SECTIONS:
3653 778 : case GIMPLE_OMP_SINGLE:
3654 778 : case GIMPLE_OMP_ORDERED:
3655 778 : case GIMPLE_OMP_MASTER:
3656 778 : case GIMPLE_OMP_MASKED:
3657 778 : case GIMPLE_OMP_TASK:
3658 778 : case GIMPLE_OMP_CRITICAL:
3659 778 : if (is_gimple_call (stmt))
3660 : {
3661 707 : if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
3662 : != BUILT_IN_GOMP_BARRIER)
3663 : return true;
3664 23 : error_at (gimple_location (stmt),
3665 : "barrier region may not be closely nested inside "
3666 : "of work-sharing, %<loop%>, %<critical%>, "
3667 : "%<ordered%>, %<master%>, %<masked%>, explicit "
3668 : "%<task%> or %<taskloop%> region");
3669 23 : return false;
3670 : }
3671 71 : error_at (gimple_location (stmt),
3672 : "work-sharing region may not be closely nested inside "
3673 : "of work-sharing, %<loop%>, %<critical%>, %<ordered%>, "
3674 : "%<master%>, %<masked%>, explicit %<task%> or "
3675 : "%<taskloop%> region");
3676 71 : return false;
3677 : case GIMPLE_OMP_PARALLEL:
3678 : case GIMPLE_OMP_TEAMS:
3679 : return true;
3680 13437 : case GIMPLE_OMP_TARGET:
3681 13437 : if (gimple_omp_target_kind (ctx->stmt)
3682 : == GF_OMP_TARGET_KIND_REGION)
3683 : return true;
3684 : break;
3685 : default:
3686 : break;
3687 : }
3688 : break;
3689 : case GIMPLE_OMP_MASTER:
3690 : case GIMPLE_OMP_MASKED:
3691 1523 : for (; ctx != NULL; ctx = ctx->outer)
3692 1014 : switch (gimple_code (ctx->stmt))
3693 : {
3694 14 : case GIMPLE_OMP_FOR:
3695 14 : if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
3696 14 : && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
3697 : break;
3698 : /* FALLTHRU */
3699 30 : case GIMPLE_OMP_SECTIONS:
3700 30 : case GIMPLE_OMP_SINGLE:
3701 30 : case GIMPLE_OMP_TASK:
3702 60 : error_at (gimple_location (stmt),
3703 : "%qs region may not be closely nested inside "
3704 : "of work-sharing, %<loop%>, explicit %<task%> or "
3705 : "%<taskloop%> region",
3706 30 : gimple_code (stmt) == GIMPLE_OMP_MASTER
3707 : ? "master" : "masked");
3708 30 : return false;
3709 : case GIMPLE_OMP_PARALLEL:
3710 : case GIMPLE_OMP_TEAMS:
3711 : return true;
3712 10 : case GIMPLE_OMP_TARGET:
3713 10 : if (gimple_omp_target_kind (ctx->stmt)
3714 : == GF_OMP_TARGET_KIND_REGION)
3715 : return true;
3716 : break;
3717 : default:
3718 : break;
3719 : }
3720 : break;
3721 : case GIMPLE_OMP_SCOPE:
3722 302 : for (; ctx != NULL; ctx = ctx->outer)
3723 183 : switch (gimple_code (ctx->stmt))
3724 : {
3725 7 : case GIMPLE_OMP_FOR:
3726 7 : if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
3727 7 : && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
3728 : break;
3729 : /* FALLTHRU */
3730 25 : case GIMPLE_OMP_SECTIONS:
3731 25 : case GIMPLE_OMP_SINGLE:
3732 25 : case GIMPLE_OMP_TASK:
3733 25 : case GIMPLE_OMP_CRITICAL:
3734 25 : case GIMPLE_OMP_ORDERED:
3735 25 : case GIMPLE_OMP_MASTER:
3736 25 : case GIMPLE_OMP_MASKED:
3737 25 : error_at (gimple_location (stmt),
3738 : "%<scope%> region may not be closely nested inside "
3739 : "of work-sharing, %<loop%>, explicit %<task%>, "
3740 : "%<taskloop%>, %<critical%>, %<ordered%>, %<master%>, "
3741 : "or %<masked%> region");
3742 25 : return false;
3743 : case GIMPLE_OMP_PARALLEL:
3744 : case GIMPLE_OMP_TEAMS:
3745 : return true;
3746 5 : case GIMPLE_OMP_TARGET:
3747 5 : if (gimple_omp_target_kind (ctx->stmt)
3748 : == GF_OMP_TARGET_KIND_REGION)
3749 : return true;
3750 : break;
3751 : default:
3752 : break;
3753 : }
3754 : break;
3755 5378 : case GIMPLE_OMP_TASK:
3756 22942 : for (c = gimple_omp_task_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
3757 17568 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS)
3758 : {
3759 4 : enum omp_clause_doacross_kind kind = OMP_CLAUSE_DOACROSS_KIND (c);
3760 4 : error_at (OMP_CLAUSE_LOCATION (c),
3761 : "%<%s(%s)%> is only allowed in %<omp ordered%>",
3762 4 : OMP_CLAUSE_DOACROSS_DEPEND (c) ? "depend" : "doacross",
3763 : kind == OMP_CLAUSE_DOACROSS_SOURCE ? "source" : "sink");
3764 4 : return false;
3765 : }
3766 : break;
3767 1631 : case GIMPLE_OMP_ORDERED:
3768 2873 : for (c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
3769 2873 : c; c = OMP_CLAUSE_CHAIN (c))
3770 : {
3771 1258 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DOACROSS)
3772 : {
3773 169 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
3774 : {
3775 4 : error_at (OMP_CLAUSE_LOCATION (c),
3776 : "invalid depend kind in omp %<ordered%> %<depend%>");
3777 4 : return false;
3778 : }
3779 165 : gcc_assert (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_THREADS
3780 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SIMD);
3781 165 : continue;
3782 : }
3783 :
3784 1089 : tree oclause;
3785 : /* Look for containing ordered(N) loop. */
3786 1089 : if (ctx == NULL
3787 1077 : || gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
3788 1089 : || (oclause
3789 1077 : = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3790 : OMP_CLAUSE_ORDERED)) == NULL_TREE)
3791 : {
3792 12 : error_at (OMP_CLAUSE_LOCATION (c),
3793 : "%<ordered%> construct with %<depend%> clause "
3794 : "must be closely nested inside an %<ordered%> loop");
3795 12 : return false;
3796 : }
3797 : }
3798 1615 : c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
3799 1615 : if (omp_find_clause (c, OMP_CLAUSE_SIMD))
3800 : {
3801 : /* ordered simd must be closely nested inside of simd region,
3802 : and simd region must not encounter constructs other than
3803 : ordered simd, therefore ordered simd may be either orphaned,
3804 : or ctx->stmt must be simd. The latter case is handled already
3805 : earlier. */
3806 34 : if (ctx != NULL)
3807 : {
3808 10 : error_at (gimple_location (stmt),
3809 : "%<ordered%> %<simd%> must be closely nested inside "
3810 : "%<simd%> region");
3811 10 : return false;
3812 : }
3813 : }
3814 1605 : for (; ctx != NULL; ctx = ctx->outer)
3815 1474 : switch (gimple_code (ctx->stmt))
3816 : {
3817 38 : case GIMPLE_OMP_CRITICAL:
3818 38 : case GIMPLE_OMP_TASK:
3819 38 : case GIMPLE_OMP_ORDERED:
3820 38 : ordered_in_taskloop:
3821 38 : error_at (gimple_location (stmt),
3822 : "%<ordered%> region may not be closely nested inside "
3823 : "of %<critical%>, %<ordered%>, explicit %<task%> or "
3824 : "%<taskloop%> region");
3825 38 : return false;
3826 1416 : case GIMPLE_OMP_FOR:
3827 1416 : if (gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_TASKLOOP)
3828 10 : goto ordered_in_taskloop;
3829 1406 : tree o;
3830 1406 : o = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3831 : OMP_CLAUSE_ORDERED);
3832 1406 : if (o == NULL)
3833 : {
3834 22 : error_at (gimple_location (stmt),
3835 : "%<ordered%> region must be closely nested inside "
3836 : "a loop region with an %<ordered%> clause");
3837 22 : return false;
3838 : }
3839 1384 : if (!gimple_omp_ordered_standalone_p (stmt))
3840 : {
3841 375 : if (OMP_CLAUSE_ORDERED_DOACROSS (o))
3842 : {
3843 10 : error_at (gimple_location (stmt),
3844 : "%<ordered%> construct without %<doacross%> or "
3845 : "%<depend%> clauses must not have the same "
3846 : "binding region as %<ordered%> construct with "
3847 : "those clauses");
3848 10 : return false;
3849 : }
3850 365 : else if (OMP_CLAUSE_ORDERED_EXPR (o))
3851 : {
3852 23 : tree co
3853 23 : = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
3854 : OMP_CLAUSE_COLLAPSE);
3855 23 : HOST_WIDE_INT
3856 23 : o_n = tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (o));
3857 23 : HOST_WIDE_INT c_n = 1;
3858 23 : if (co)
3859 5 : c_n = tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (co));
3860 23 : if (o_n != c_n)
3861 : {
3862 10 : error_at (gimple_location (stmt),
3863 : "%<ordered%> construct without %<doacross%> "
3864 : "or %<depend%> clauses binds to loop where "
3865 : "%<collapse%> argument %wd is different from "
3866 : "%<ordered%> argument %wd", c_n, o_n);
3867 10 : return false;
3868 : }
3869 : }
3870 : }
3871 : return true;
3872 10 : case GIMPLE_OMP_TARGET:
3873 10 : if (gimple_omp_target_kind (ctx->stmt)
3874 : != GF_OMP_TARGET_KIND_REGION)
3875 : break;
3876 : /* FALLTHRU */
3877 30 : case GIMPLE_OMP_PARALLEL:
3878 30 : case GIMPLE_OMP_TEAMS:
3879 30 : error_at (gimple_location (stmt),
3880 : "%<ordered%> region must be closely nested inside "
3881 : "a loop region with an %<ordered%> clause");
3882 30 : return false;
3883 : default:
3884 : break;
3885 : }
3886 : break;
3887 462 : case GIMPLE_OMP_CRITICAL:
3888 462 : {
3889 462 : tree this_stmt_name
3890 462 : = gimple_omp_critical_name (as_a <gomp_critical *> (stmt));
3891 888 : for (; ctx != NULL; ctx = ctx->outer)
3892 432 : if (gomp_critical *other_crit
3893 451 : = dyn_cast <gomp_critical *> (ctx->stmt))
3894 25 : if (this_stmt_name == gimple_omp_critical_name (other_crit))
3895 : {
3896 6 : error_at (gimple_location (stmt),
3897 : "%<critical%> region may not be nested inside "
3898 : "a %<critical%> region with the same name");
3899 6 : return false;
3900 : }
3901 : }
3902 : break;
3903 8841 : case GIMPLE_OMP_TEAMS:
3904 8841 : if (ctx == NULL)
3905 : break;
3906 6170 : else if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET
3907 6170 : || (gimple_omp_target_kind (ctx->stmt)
3908 : != GF_OMP_TARGET_KIND_REGION))
3909 : {
3910 : /* Teams construct can appear either strictly nested inside of
3911 : target construct with no intervening stmts, or can be encountered
3912 : only by initial task (so must not appear inside any OpenMP
3913 : construct. */
3914 44 : error_at (gimple_location (stmt),
3915 : "%<teams%> construct must be closely nested inside of "
3916 : "%<target%> construct or not nested in any OpenMP "
3917 : "construct");
3918 44 : return false;
3919 : }
3920 : break;
3921 42248 : case GIMPLE_OMP_TARGET:
3922 189464 : for (c = gimple_omp_target_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
3923 147220 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS)
3924 : {
3925 4 : enum omp_clause_doacross_kind kind = OMP_CLAUSE_DOACROSS_KIND (c);
3926 4 : error_at (OMP_CLAUSE_LOCATION (c),
3927 : "%<depend(%s)%> is only allowed in %<omp ordered%>",
3928 : kind == OMP_CLAUSE_DOACROSS_SOURCE ? "source" : "sink");
3929 4 : return false;
3930 : }
3931 42244 : if (is_gimple_omp_offloaded (stmt)
3932 42244 : && oacc_get_fn_attrib (cfun->decl) != NULL)
3933 : {
3934 4 : error_at (gimple_location (stmt),
3935 : "OpenACC region inside of OpenACC routine, nested "
3936 : "parallelism not supported yet");
3937 4 : return false;
3938 : }
3939 49654 : for (; ctx != NULL; ctx = ctx->outer)
3940 : {
3941 7820 : if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET)
3942 : {
3943 768 : if (is_gimple_omp (stmt)
3944 768 : && is_gimple_omp_oacc (stmt)
3945 242 : && is_gimple_omp (ctx->stmt))
3946 : {
3947 242 : error_at (gimple_location (stmt),
3948 : "OpenACC construct inside of non-OpenACC region");
3949 242 : return false;
3950 : }
3951 526 : continue;
3952 : }
3953 :
3954 7052 : const char *stmt_name, *ctx_stmt_name;
3955 7052 : switch (gimple_omp_target_kind (stmt))
3956 : {
3957 : case GF_OMP_TARGET_KIND_REGION: stmt_name = "target"; break;
3958 399 : case GF_OMP_TARGET_KIND_DATA: stmt_name = "target data"; break;
3959 1688 : case GF_OMP_TARGET_KIND_UPDATE: stmt_name = "target update"; break;
3960 12 : case GF_OMP_TARGET_KIND_ENTER_DATA:
3961 12 : stmt_name = "target enter data"; break;
3962 18 : case GF_OMP_TARGET_KIND_EXIT_DATA:
3963 18 : stmt_name = "target exit data"; break;
3964 1442 : case GF_OMP_TARGET_KIND_OACC_PARALLEL: stmt_name = "parallel"; break;
3965 1621 : case GF_OMP_TARGET_KIND_OACC_KERNELS: stmt_name = "kernels"; break;
3966 183 : case GF_OMP_TARGET_KIND_OACC_SERIAL: stmt_name = "serial"; break;
3967 386 : case GF_OMP_TARGET_KIND_OACC_DATA: stmt_name = "data"; break;
3968 256 : case GF_OMP_TARGET_KIND_OACC_UPDATE: stmt_name = "update"; break;
3969 94 : case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
3970 94 : stmt_name = "enter data"; break;
3971 86 : case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
3972 86 : stmt_name = "exit data"; break;
3973 64 : case GF_OMP_TARGET_KIND_OACC_DECLARE: stmt_name = "declare"; break;
3974 128 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA: stmt_name = "host_data";
3975 128 : break;
3976 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
3977 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
3978 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
3979 : /* OpenACC 'kernels' decomposed parts. */
3980 1621 : stmt_name = "kernels"; break;
3981 0 : default: gcc_unreachable ();
3982 : }
3983 7052 : switch (gimple_omp_target_kind (ctx->stmt))
3984 : {
3985 : case GF_OMP_TARGET_KIND_REGION: ctx_stmt_name = "target"; break;
3986 2712 : case GF_OMP_TARGET_KIND_DATA: ctx_stmt_name = "target data"; break;
3987 35 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
3988 35 : ctx_stmt_name = "parallel"; break;
3989 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
3990 1037 : ctx_stmt_name = "kernels"; break;
3991 35 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
3992 35 : ctx_stmt_name = "serial"; break;
3993 3117 : case GF_OMP_TARGET_KIND_OACC_DATA: ctx_stmt_name = "data"; break;
3994 8 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
3995 8 : ctx_stmt_name = "host_data"; break;
3996 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
3997 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
3998 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
3999 : /* OpenACC 'kernels' decomposed parts. */
4000 1037 : ctx_stmt_name = "kernels"; break;
4001 0 : default: gcc_unreachable ();
4002 : }
4003 :
4004 : /* OpenACC/OpenMP mismatch? */
4005 14104 : if (is_gimple_omp_oacc (stmt)
4006 7052 : != is_gimple_omp_oacc (ctx->stmt))
4007 : {
4008 56 : error_at (gimple_location (stmt),
4009 : "%s %qs construct inside of %s %qs region",
4010 : (is_gimple_omp_oacc (stmt)
4011 : ? "OpenACC" : "OpenMP"), stmt_name,
4012 : (is_gimple_omp_oacc (ctx->stmt)
4013 : ? "OpenACC" : "OpenMP"), ctx_stmt_name);
4014 28 : return false;
4015 : }
4016 7024 : if (is_gimple_omp_offloaded (ctx->stmt))
4017 : {
4018 : /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
4019 185 : if (is_gimple_omp_oacc (ctx->stmt))
4020 : {
4021 105 : error_at (gimple_location (stmt),
4022 : "%qs construct inside of %qs region",
4023 : stmt_name, ctx_stmt_name);
4024 105 : return false;
4025 : }
4026 : else
4027 : {
4028 80 : if ((gimple_omp_target_kind (ctx->stmt)
4029 : == GF_OMP_TARGET_KIND_REGION)
4030 80 : && (gimple_omp_target_kind (stmt)
4031 : == GF_OMP_TARGET_KIND_REGION))
4032 : {
4033 58 : c = omp_find_clause (gimple_omp_target_clauses (stmt),
4034 : OMP_CLAUSE_DEVICE);
4035 58 : if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
4036 : break;
4037 : }
4038 49 : warning_at (gimple_location (stmt), OPT_Wopenmp,
4039 : "%qs construct inside of %qs region",
4040 : stmt_name, ctx_stmt_name);
4041 : }
4042 : }
4043 : }
4044 : break;
4045 : default:
4046 : break;
4047 : }
4048 : return true;
4049 : }
4050 :
4051 :
4052 : /* Helper function scan_omp.
4053 :
4054 : Callback for walk_tree or operators in walk_gimple_stmt used to
4055 : scan for OMP directives in TP. */
4056 :
4057 : static tree
4058 9832877 : scan_omp_1_op (tree *tp, int *walk_subtrees, void *data)
4059 : {
4060 9832877 : struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
4061 9832877 : omp_context *ctx = (omp_context *) wi->info;
4062 9832877 : tree t = *tp;
4063 9832877 : tree tmp;
4064 :
4065 9832877 : switch (TREE_CODE (t))
4066 : {
4067 4162241 : case VAR_DECL:
4068 4162241 : case PARM_DECL:
4069 4162241 : case LABEL_DECL:
4070 4162241 : case RESULT_DECL:
4071 4162241 : if (ctx)
4072 : {
4073 2131652 : tmp = NULL_TREE;
4074 2131652 : if (TREE_CODE (t) == VAR_DECL
4075 3818894 : && (tmp = lookup_attribute ("omp allocate var",
4076 1687242 : DECL_ATTRIBUTES (t))) != NULL_TREE)
4077 61 : t = TREE_VALUE (TREE_VALUE (tmp));
4078 2131652 : tree repl = remap_decl (t, &ctx->cb);
4079 2131652 : gcc_checking_assert (TREE_CODE (repl) != ERROR_MARK);
4080 2131652 : if (tmp != NULL_TREE && t != repl)
4081 31 : *tp = build_fold_addr_expr (repl);
4082 2131621 : else if (tmp == NULL_TREE)
4083 2131591 : *tp = repl;
4084 : }
4085 : break;
4086 :
4087 193682 : case INDIRECT_REF:
4088 193682 : case MEM_REF:
4089 193682 : if (ctx
4090 78127 : && TREE_CODE (TREE_OPERAND (t, 0)) == VAR_DECL
4091 252751 : && ((tmp = lookup_attribute ("omp allocate var",
4092 59069 : DECL_ATTRIBUTES (TREE_OPERAND (t, 0))))
4093 : != NULL_TREE))
4094 : {
4095 127 : tmp = TREE_VALUE (TREE_VALUE (tmp));
4096 127 : tree repl = remap_decl (tmp, &ctx->cb);
4097 127 : gcc_checking_assert (TREE_CODE (repl) != ERROR_MARK);
4098 127 : if (tmp != repl)
4099 118 : *tp = repl;
4100 : break;
4101 : }
4102 5670509 : gcc_fallthrough ();
4103 :
4104 5670509 : default:
4105 5670509 : if (ctx && TYPE_P (t))
4106 5 : *tp = remap_type (t, &ctx->cb);
4107 5670504 : else if (!DECL_P (t))
4108 : {
4109 4704126 : *walk_subtrees = 1;
4110 4704126 : if (ctx)
4111 : {
4112 1430117 : tree tem = remap_type (TREE_TYPE (t), &ctx->cb);
4113 1430117 : if (tem != TREE_TYPE (t))
4114 : {
4115 9946 : if (TREE_CODE (t) == INTEGER_CST)
4116 4202 : *tp = wide_int_to_tree (tem, wi::to_wide (t));
4117 : else
4118 5744 : TREE_TYPE (t) = tem;
4119 : }
4120 : }
4121 : }
4122 : break;
4123 : }
4124 :
4125 9832877 : return NULL_TREE;
4126 : }
4127 :
4128 : /* Return true if FNDECL is a setjmp or a longjmp. */
4129 :
4130 : static bool
4131 3042 : setjmp_or_longjmp_p (const_tree fndecl)
4132 : {
4133 3042 : if (fndecl_built_in_p (fndecl, BUILT_IN_SETJMP, BUILT_IN_LONGJMP))
4134 : return true;
4135 :
4136 3042 : tree declname = DECL_NAME (fndecl);
4137 3042 : if (!declname
4138 3042 : || (DECL_CONTEXT (fndecl) != NULL_TREE
4139 2347 : && TREE_CODE (DECL_CONTEXT (fndecl)) != TRANSLATION_UNIT_DECL)
4140 5833 : || !TREE_PUBLIC (fndecl))
4141 : return false;
4142 :
4143 2629 : const char *name = IDENTIFIER_POINTER (declname);
4144 2629 : return !strcmp (name, "setjmp") || !strcmp (name, "longjmp");
4145 : }
4146 :
4147 : /* Helper function for scan_omp.
4148 :
4149 : Callback for walk_gimple_stmt used to scan for OMP directives in
4150 : the current statement in GSI. */
4151 :
4152 : static tree
4153 3417779 : scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
4154 : struct walk_stmt_info *wi)
4155 : {
4156 3417779 : gimple *stmt = gsi_stmt (*gsi);
4157 3417779 : omp_context *ctx = (omp_context *) wi->info;
4158 :
4159 3417779 : if (gimple_has_location (stmt))
4160 2788885 : input_location = gimple_location (stmt);
4161 :
4162 : /* Check the nesting restrictions. */
4163 3417779 : bool remove = false;
4164 3417779 : if (is_gimple_omp (stmt))
4165 152967 : remove = !check_omp_nesting_restrictions (stmt, ctx);
4166 3264812 : else if (is_gimple_call (stmt))
4167 : {
4168 233079 : tree fndecl = gimple_call_fndecl (stmt);
4169 233079 : if (fndecl)
4170 : {
4171 217244 : if (ctx
4172 68122 : && gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
4173 14714 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
4174 3042 : && setjmp_or_longjmp_p (fndecl)
4175 217252 : && !ctx->loop_p)
4176 : {
4177 4 : remove = true;
4178 4 : error_at (gimple_location (stmt),
4179 : "setjmp/longjmp inside %<simd%> construct");
4180 : }
4181 217240 : else if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
4182 63866 : switch (DECL_FUNCTION_CODE (fndecl))
4183 : {
4184 4221 : case BUILT_IN_GOMP_BARRIER:
4185 4221 : case BUILT_IN_GOMP_CANCEL:
4186 4221 : case BUILT_IN_GOMP_CANCELLATION_POINT:
4187 4221 : case BUILT_IN_GOMP_TASKYIELD:
4188 4221 : case BUILT_IN_GOMP_TASKWAIT:
4189 4221 : case BUILT_IN_GOMP_TASKGROUP_START:
4190 4221 : case BUILT_IN_GOMP_TASKGROUP_END:
4191 4221 : remove = !check_omp_nesting_restrictions (stmt, ctx);
4192 4221 : break;
4193 : default:
4194 : break;
4195 : }
4196 153374 : else if (ctx)
4197 : {
4198 46120 : omp_context *octx = ctx;
4199 46120 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_SCAN && ctx->outer)
4200 46120 : octx = ctx->outer;
4201 46120 : if (octx->order_concurrent && omp_runtime_api_call (fndecl))
4202 : {
4203 240 : remove = true;
4204 240 : error_at (gimple_location (stmt),
4205 : "OpenMP runtime API call %qD in a region with "
4206 : "%<order(concurrent)%> clause", fndecl);
4207 : }
4208 46120 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
4209 3339 : && omp_runtime_api_call (fndecl)
4210 339 : && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
4211 : != strlen ("omp_get_num_teams"))
4212 185 : || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
4213 : "omp_get_num_teams") != 0)
4214 169 : && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
4215 : != strlen ("omp_get_num_teams_dim"))
4216 6 : || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
4217 : "omp_get_num_teams_dim") != 0)
4218 163 : && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
4219 : != strlen ("omp_get_team_num"))
4220 124 : || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
4221 : "omp_get_team_num") != 0)
4222 46159 : && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
4223 : != strlen ("omp_get_team_num_dim"))
4224 12 : || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
4225 : "omp_get_team_num_dim") != 0))
4226 : {
4227 27 : remove = true;
4228 27 : error_at (gimple_location (stmt),
4229 : "OpenMP runtime API call %qD strictly nested in a "
4230 : "%<teams%> region", fndecl);
4231 : }
4232 46120 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
4233 13374 : && (gimple_omp_target_kind (ctx->stmt)
4234 : == GF_OMP_TARGET_KIND_REGION)
4235 51108 : && omp_runtime_api_call (fndecl))
4236 : {
4237 411 : tree tgt_clauses = gimple_omp_target_clauses (ctx->stmt);
4238 411 : tree c = omp_find_clause (tgt_clauses, OMP_CLAUSE_DEVICE);
4239 411 : if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
4240 5 : error_at (gimple_location (stmt),
4241 : "OpenMP runtime API call %qD in a region with "
4242 : "%<device(ancestor)%> clause", fndecl);
4243 : }
4244 : }
4245 : }
4246 : }
4247 203312 : if (remove)
4248 : {
4249 3326 : stmt = gimple_build_nop ();
4250 3326 : gsi_replace (gsi, stmt, false);
4251 : }
4252 :
4253 3417779 : *handled_ops_p = true;
4254 :
4255 3417779 : switch (gimple_code (stmt))
4256 : {
4257 18243 : case GIMPLE_OMP_PARALLEL:
4258 18243 : taskreg_nesting_level++;
4259 18243 : scan_omp_parallel (gsi, ctx);
4260 18243 : taskreg_nesting_level--;
4261 18243 : break;
4262 :
4263 5374 : case GIMPLE_OMP_TASK:
4264 5374 : taskreg_nesting_level++;
4265 5374 : scan_omp_task (gsi, ctx);
4266 5374 : taskreg_nesting_level--;
4267 5374 : break;
4268 :
4269 52808 : case GIMPLE_OMP_FOR:
4270 52808 : if ((gimple_omp_for_kind (as_a <gomp_for *> (stmt))
4271 : == GF_OMP_FOR_KIND_SIMD)
4272 10813 : && gimple_omp_for_combined_into_p (stmt)
4273 60503 : && gimple_code (ctx->stmt) != GIMPLE_OMP_SCAN)
4274 : {
4275 7529 : tree clauses = gimple_omp_for_clauses (as_a <gomp_for *> (stmt));
4276 7529 : tree c = omp_find_clause (clauses, OMP_CLAUSE_REDUCTION);
4277 7529 : if (c && OMP_CLAUSE_REDUCTION_INSCAN (c) && !seen_error ())
4278 : {
4279 83 : scan_omp_simd_scan (gsi, as_a <gomp_for *> (stmt), ctx);
4280 83 : break;
4281 : }
4282 : }
4283 52725 : if ((gimple_omp_for_kind (as_a <gomp_for *> (stmt))
4284 : == GF_OMP_FOR_KIND_SIMD)
4285 10730 : && omp_maybe_offloaded_ctx (ctx)
4286 3643 : && omp_max_simt_vf ()
4287 52725 : && gimple_omp_for_collapse (stmt) == 1)
4288 0 : scan_omp_simd (gsi, as_a <gomp_for *> (stmt), ctx);
4289 : else
4290 52725 : scan_omp_for (as_a <gomp_for *> (stmt), ctx);
4291 : break;
4292 :
4293 212 : case GIMPLE_OMP_SCOPE:
4294 212 : ctx = new_omp_context (stmt, ctx);
4295 212 : scan_sharing_clauses (gimple_omp_scope_clauses (stmt), ctx);
4296 212 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4297 212 : break;
4298 :
4299 852 : case GIMPLE_OMP_DISPATCH:
4300 852 : ctx = new_omp_context (stmt, ctx);
4301 852 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4302 852 : break;
4303 :
4304 612 : case GIMPLE_OMP_INTEROP:
4305 612 : ctx = new_omp_context (stmt, ctx);
4306 612 : break;
4307 :
4308 589 : case GIMPLE_OMP_SECTIONS:
4309 589 : scan_omp_sections (as_a <gomp_sections *> (stmt), ctx);
4310 589 : break;
4311 :
4312 1236 : case GIMPLE_OMP_SINGLE:
4313 1236 : scan_omp_single (as_a <gomp_single *> (stmt), ctx);
4314 1236 : break;
4315 :
4316 1292 : case GIMPLE_OMP_SCAN:
4317 1292 : if (tree clauses = gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt)))
4318 : {
4319 610 : if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_INCLUSIVE)
4320 308 : ctx->scan_inclusive = true;
4321 302 : else if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_EXCLUSIVE)
4322 302 : ctx->scan_exclusive = true;
4323 : }
4324 : /* FALLTHRU */
4325 6223 : case GIMPLE_OMP_SECTION:
4326 6223 : case GIMPLE_OMP_STRUCTURED_BLOCK:
4327 6223 : case GIMPLE_OMP_MASTER:
4328 6223 : case GIMPLE_OMP_ORDERED:
4329 6223 : case GIMPLE_OMP_CRITICAL:
4330 6223 : ctx = new_omp_context (stmt, ctx);
4331 6223 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4332 6223 : break;
4333 :
4334 457 : case GIMPLE_OMP_MASKED:
4335 457 : ctx = new_omp_context (stmt, ctx);
4336 457 : scan_sharing_clauses (gimple_omp_masked_clauses (stmt), ctx);
4337 457 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4338 457 : break;
4339 :
4340 607 : case GIMPLE_OMP_TASKGROUP:
4341 607 : ctx = new_omp_context (stmt, ctx);
4342 607 : scan_sharing_clauses (gimple_omp_taskgroup_clauses (stmt), ctx);
4343 607 : scan_omp (gimple_omp_body_ptr (stmt), ctx);
4344 607 : break;
4345 :
4346 41865 : case GIMPLE_OMP_TARGET:
4347 41865 : if (is_gimple_omp_offloaded (stmt))
4348 : {
4349 24836 : taskreg_nesting_level++;
4350 24836 : scan_omp_target (as_a <gomp_target *> (stmt), ctx);
4351 24836 : taskreg_nesting_level--;
4352 : }
4353 : else
4354 17029 : scan_omp_target (as_a <gomp_target *> (stmt), ctx);
4355 : break;
4356 :
4357 8797 : case GIMPLE_OMP_TEAMS:
4358 8797 : if (gimple_omp_teams_host (as_a <gomp_teams *> (stmt)))
4359 : {
4360 2671 : taskreg_nesting_level++;
4361 2671 : scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
4362 2671 : taskreg_nesting_level--;
4363 : }
4364 : else
4365 6126 : scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
4366 : break;
4367 :
4368 282832 : case GIMPLE_BIND:
4369 282832 : {
4370 282832 : tree var;
4371 :
4372 282832 : *handled_ops_p = false;
4373 282832 : if (ctx)
4374 152930 : for (var = gimple_bind_vars (as_a <gbind *> (stmt));
4375 580368 : var ;
4376 427438 : var = DECL_CHAIN (var))
4377 427438 : insert_decl_map (&ctx->cb, var, var);
4378 : }
4379 : break;
4380 2997072 : default:
4381 2997072 : *handled_ops_p = false;
4382 2997072 : break;
4383 : }
4384 :
4385 3417779 : return NULL_TREE;
4386 : }
4387 :
4388 :
4389 : /* Scan all the statements starting at the current statement. CTX
4390 : contains context information about the OMP directives and
4391 : clauses found during the scan. */
4392 :
4393 : static void
4394 259403 : scan_omp (gimple_seq *body_p, omp_context *ctx)
4395 : {
4396 259403 : location_t saved_location;
4397 259403 : struct walk_stmt_info wi;
4398 :
4399 259403 : memset (&wi, 0, sizeof (wi));
4400 259403 : wi.info = ctx;
4401 259403 : wi.want_locations = true;
4402 :
4403 259403 : saved_location = input_location;
4404 259403 : walk_gimple_seq_mod (body_p, scan_omp_1_stmt, scan_omp_1_op, &wi);
4405 259403 : input_location = saved_location;
4406 259403 : }
4407 :
4408 : /* Re-gimplification and code generation routines. */
4409 :
4410 : /* Remove omp_member_access_dummy_var variables from gimple_bind_vars
4411 : of BIND if in a method. */
4412 :
4413 : static void
4414 259143 : maybe_remove_omp_member_access_dummy_vars (gbind *bind)
4415 : {
4416 259143 : if (DECL_ARGUMENTS (current_function_decl)
4417 98195 : && DECL_ARTIFICIAL (DECL_ARGUMENTS (current_function_decl))
4418 269893 : && (TREE_CODE (TREE_TYPE (DECL_ARGUMENTS (current_function_decl)))
4419 : == POINTER_TYPE))
4420 : {
4421 3520 : tree vars = gimple_bind_vars (bind);
4422 19340 : for (tree *pvar = &vars; *pvar; )
4423 15820 : if (omp_member_access_dummy_var (*pvar))
4424 719 : *pvar = DECL_CHAIN (*pvar);
4425 : else
4426 15101 : pvar = &DECL_CHAIN (*pvar);
4427 3520 : gimple_bind_set_vars (bind, vars);
4428 : }
4429 259143 : }
4430 :
4431 : /* Remove omp_member_access_dummy_var variables from BLOCK_VARS of
4432 : block and its subblocks. */
4433 :
4434 : static void
4435 12877 : remove_member_access_dummy_vars (tree block)
4436 : {
4437 22439 : for (tree *pvar = &BLOCK_VARS (block); *pvar; )
4438 9562 : if (omp_member_access_dummy_var (*pvar))
4439 108 : *pvar = DECL_CHAIN (*pvar);
4440 : else
4441 9454 : pvar = &DECL_CHAIN (*pvar);
4442 :
4443 16764 : for (block = BLOCK_SUBBLOCKS (block); block; block = BLOCK_CHAIN (block))
4444 3887 : remove_member_access_dummy_vars (block);
4445 12877 : }
4446 :
4447 : /* If a context was created for STMT when it was scanned, return it. */
4448 :
4449 : static omp_context *
4450 121530 : maybe_lookup_ctx (gimple *stmt)
4451 : {
4452 121530 : splay_tree_node n;
4453 121530 : n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt);
4454 121530 : return n ? (omp_context *) n->value : NULL;
4455 : }
4456 :
4457 :
4458 : /* Find the mapping for DECL in CTX or the immediately enclosing
4459 : context that has a mapping for DECL.
4460 :
4461 : If CTX is a nested parallel directive, we may have to use the decl
4462 : mappings created in CTX's parent context. Suppose that we have the
4463 : following parallel nesting (variable UIDs showed for clarity):
4464 :
4465 : iD.1562 = 0;
4466 : #omp parallel shared(iD.1562) -> outer parallel
4467 : iD.1562 = iD.1562 + 1;
4468 :
4469 : #omp parallel shared (iD.1562) -> inner parallel
4470 : iD.1562 = iD.1562 - 1;
4471 :
4472 : Each parallel structure will create a distinct .omp_data_s structure
4473 : for copying iD.1562 in/out of the directive:
4474 :
4475 : outer parallel .omp_data_s.1.i -> iD.1562
4476 : inner parallel .omp_data_s.2.i -> iD.1562
4477 :
4478 : A shared variable mapping will produce a copy-out operation before
4479 : the parallel directive and a copy-in operation after it. So, in
4480 : this case we would have:
4481 :
4482 : iD.1562 = 0;
4483 : .omp_data_o.1.i = iD.1562;
4484 : #omp parallel shared(iD.1562) -> outer parallel
4485 : .omp_data_i.1 = &.omp_data_o.1
4486 : .omp_data_i.1->i = .omp_data_i.1->i + 1;
4487 :
4488 : .omp_data_o.2.i = iD.1562; -> **
4489 : #omp parallel shared(iD.1562) -> inner parallel
4490 : .omp_data_i.2 = &.omp_data_o.2
4491 : .omp_data_i.2->i = .omp_data_i.2->i - 1;
4492 :
4493 :
4494 : ** This is a problem. The symbol iD.1562 cannot be referenced
4495 : inside the body of the outer parallel region. But since we are
4496 : emitting this copy operation while expanding the inner parallel
4497 : directive, we need to access the CTX structure of the outer
4498 : parallel directive to get the correct mapping:
4499 :
4500 : .omp_data_o.2.i = .omp_data_i.1->i
4501 :
4502 : Since there may be other workshare or parallel directives enclosing
4503 : the parallel directive, it may be necessary to walk up the context
4504 : parent chain. This is not a problem in general because nested
4505 : parallelism happens only rarely. */
4506 :
4507 : static tree
4508 126457 : lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
4509 : {
4510 126457 : tree t;
4511 126457 : omp_context *up;
4512 :
4513 183235 : for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
4514 56778 : t = maybe_lookup_decl (decl, up);
4515 :
4516 126457 : gcc_assert (!ctx->is_nested || t || is_global_var (decl));
4517 :
4518 98095 : return t ? t : decl;
4519 : }
4520 :
4521 :
4522 : /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
4523 : in outer contexts. */
4524 :
4525 : static tree
4526 357741 : maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
4527 : {
4528 357741 : tree t = NULL;
4529 357741 : omp_context *up;
4530 :
4531 629162 : for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
4532 271421 : t = maybe_lookup_decl (decl, up);
4533 :
4534 357741 : return t ? t : decl;
4535 : }
4536 :
4537 :
4538 : /* Construct the initialization value for reduction operation OP. */
4539 :
4540 : tree
4541 13138 : omp_reduction_init_op (location_t loc, enum tree_code op, tree type)
4542 : {
4543 13138 : switch (op)
4544 : {
4545 10803 : case PLUS_EXPR:
4546 10803 : case MINUS_EXPR:
4547 10803 : case BIT_IOR_EXPR:
4548 10803 : case BIT_XOR_EXPR:
4549 10803 : case TRUTH_OR_EXPR:
4550 10803 : case TRUTH_ORIF_EXPR:
4551 10803 : case TRUTH_XOR_EXPR:
4552 10803 : case NE_EXPR:
4553 10803 : return build_zero_cst (type);
4554 :
4555 1431 : case MULT_EXPR:
4556 1431 : case TRUTH_AND_EXPR:
4557 1431 : case TRUTH_ANDIF_EXPR:
4558 1431 : case EQ_EXPR:
4559 1431 : return fold_convert_loc (loc, type, integer_one_node);
4560 :
4561 208 : case BIT_AND_EXPR:
4562 208 : return fold_convert_loc (loc, type, integer_minus_one_node);
4563 :
4564 388 : case MAX_EXPR:
4565 388 : if (SCALAR_FLOAT_TYPE_P (type))
4566 : {
4567 158 : REAL_VALUE_TYPE min;
4568 158 : if (HONOR_INFINITIES (type))
4569 158 : real_arithmetic (&min, NEGATE_EXPR, &dconstinf, NULL);
4570 : else
4571 0 : real_maxval (&min, 1, TYPE_MODE (type));
4572 158 : return build_real (type, min);
4573 : }
4574 230 : else if (POINTER_TYPE_P (type))
4575 : {
4576 2 : wide_int min
4577 2 : = wi::min_value (TYPE_PRECISION (type), TYPE_SIGN (type));
4578 2 : return wide_int_to_tree (type, min);
4579 2 : }
4580 : else
4581 : {
4582 228 : gcc_assert (INTEGRAL_TYPE_P (type));
4583 228 : return TYPE_MIN_VALUE (type);
4584 : }
4585 :
4586 308 : case MIN_EXPR:
4587 308 : if (SCALAR_FLOAT_TYPE_P (type))
4588 : {
4589 108 : REAL_VALUE_TYPE max;
4590 108 : if (HONOR_INFINITIES (type))
4591 108 : max = dconstinf;
4592 : else
4593 0 : real_maxval (&max, 0, TYPE_MODE (type));
4594 108 : return build_real (type, max);
4595 : }
4596 200 : else if (POINTER_TYPE_P (type))
4597 : {
4598 2 : wide_int max
4599 2 : = wi::max_value (TYPE_PRECISION (type), TYPE_SIGN (type));
4600 2 : return wide_int_to_tree (type, max);
4601 2 : }
4602 : else
4603 : {
4604 198 : gcc_assert (INTEGRAL_TYPE_P (type));
4605 198 : return TYPE_MAX_VALUE (type);
4606 : }
4607 :
4608 0 : default:
4609 0 : gcc_unreachable ();
4610 : }
4611 : }
4612 :
4613 : /* Construct the initialization value for reduction CLAUSE. */
4614 :
4615 : tree
4616 10517 : omp_reduction_init (tree clause, tree type)
4617 : {
4618 10517 : return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause),
4619 10517 : OMP_CLAUSE_REDUCTION_CODE (clause), type);
4620 : }
4621 :
4622 : /* Return alignment to be assumed for var in CLAUSE, which should be
4623 : OMP_CLAUSE_ALIGNED. */
4624 :
4625 : static tree
4626 140 : omp_clause_aligned_alignment (tree clause)
4627 : {
4628 140 : if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause))
4629 123 : return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause);
4630 :
4631 : /* Otherwise return implementation defined alignment. */
4632 17 : unsigned int al = 1;
4633 17 : opt_scalar_mode mode_iter;
4634 17 : auto_vector_modes modes;
4635 17 : targetm.vectorize.autovectorize_vector_modes (&modes, true);
4636 17 : static enum mode_class classes[]
4637 : = { MODE_INT, MODE_VECTOR_INT, MODE_FLOAT, MODE_VECTOR_FLOAT };
4638 51 : for (int i = 0; i < 4; i += 2)
4639 : /* The for loop above dictates that we only walk through scalar classes. */
4640 255 : FOR_EACH_MODE_IN_CLASS (mode_iter, classes[i])
4641 : {
4642 221 : scalar_mode mode = mode_iter.require ();
4643 221 : machine_mode vmode = targetm.vectorize.preferred_simd_mode (mode);
4644 221 : if (GET_MODE_CLASS (vmode) != classes[i + 1])
4645 323 : continue;
4646 : machine_mode alt_vmode;
4647 476 : for (unsigned int j = 0; j < modes.length (); ++j)
4648 527 : if (related_vector_mode (modes[j], mode).exists (&alt_vmode)
4649 578 : && known_ge (GET_MODE_SIZE (alt_vmode), GET_MODE_SIZE (vmode)))
4650 119 : vmode = alt_vmode;
4651 :
4652 119 : tree type = lang_hooks.types.type_for_mode (mode, 1);
4653 119 : if (type == NULL_TREE || TYPE_MODE (type) != mode)
4654 17 : continue;
4655 102 : type = build_vector_type_for_mode (type, vmode);
4656 102 : if (TYPE_MODE (type) != vmode)
4657 0 : continue;
4658 102 : if (TYPE_ALIGN_UNIT (type) > al)
4659 17 : al = TYPE_ALIGN_UNIT (type);
4660 : }
4661 17 : return build_int_cst (integer_type_node, al);
4662 17 : }
4663 :
4664 :
4665 : /* This structure is part of the interface between lower_rec_simd_input_clauses
4666 : and lower_rec_input_clauses. */
4667 :
4668 : class omplow_simd_context {
4669 : public:
4670 77364 : omplow_simd_context () { memset (this, 0, sizeof (*this)); }
4671 : tree idx;
4672 : tree lane;
4673 : tree lastlane;
4674 : vec<tree, va_heap> simt_eargs;
4675 : gimple_seq simt_dlist;
4676 : poly_uint64 max_vf;
4677 : bool is_simt;
4678 : };
4679 :
4680 : /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
4681 : privatization. */
4682 :
4683 : static bool
4684 10228 : lower_rec_simd_input_clauses (tree new_var, omp_context *ctx,
4685 : omplow_simd_context *sctx, tree &ivar,
4686 : tree &lvar, tree *rvar = NULL,
4687 : tree *rvar2 = NULL)
4688 : {
4689 10228 : if (known_eq (sctx->max_vf, 0U))
4690 : {
4691 4790 : sctx->max_vf = (sctx->is_simt ? omp_max_simt_vf ()
4692 4790 : : omp_max_vf (omp_maybe_offloaded_ctx (ctx)));
4693 4790 : if (maybe_gt (sctx->max_vf, 1U))
4694 : {
4695 3452 : tree c = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
4696 : OMP_CLAUSE_SAFELEN);
4697 3452 : if (c)
4698 : {
4699 89 : poly_uint64 safe_len;
4700 89 : if (!poly_int_tree_p (OMP_CLAUSE_SAFELEN_EXPR (c), &safe_len)
4701 89 : || maybe_lt (safe_len, 1U))
4702 6 : sctx->max_vf = 1;
4703 : else
4704 83 : sctx->max_vf = lower_bound (sctx->max_vf, safe_len);
4705 : }
4706 : }
4707 4790 : if (sctx->is_simt && !known_eq (sctx->max_vf, 1U))
4708 : {
4709 0 : for (tree c = gimple_omp_for_clauses (ctx->stmt); c;
4710 0 : c = OMP_CLAUSE_CHAIN (c))
4711 : {
4712 0 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
4713 0 : continue;
4714 :
4715 0 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
4716 : {
4717 : /* UDR reductions are not supported yet for SIMT, disable
4718 : SIMT. */
4719 0 : sctx->max_vf = 1;
4720 0 : break;
4721 : }
4722 :
4723 0 : if (truth_value_p (OMP_CLAUSE_REDUCTION_CODE (c))
4724 0 : && !INTEGRAL_TYPE_P (TREE_TYPE (new_var)))
4725 : {
4726 : /* Doing boolean operations on non-integral types is
4727 : for conformance only, it's not worth supporting this
4728 : for SIMT. */
4729 0 : sctx->max_vf = 1;
4730 0 : break;
4731 : }
4732 : }
4733 : }
4734 4790 : if (maybe_gt (sctx->max_vf, 1U))
4735 : {
4736 3433 : sctx->idx = create_tmp_var (unsigned_type_node);
4737 3433 : sctx->lane = create_tmp_var (unsigned_type_node);
4738 : }
4739 : }
4740 10228 : if (known_eq (sctx->max_vf, 1U))
4741 : return false;
4742 :
4743 7174 : if (sctx->is_simt)
4744 : {
4745 0 : if (is_gimple_reg (new_var))
4746 : {
4747 0 : ivar = lvar = new_var;
4748 0 : return true;
4749 : }
4750 0 : tree type = TREE_TYPE (new_var), ptype = build_pointer_type (type);
4751 0 : ivar = lvar = create_tmp_var (type);
4752 0 : TREE_ADDRESSABLE (ivar) = 1;
4753 0 : DECL_ATTRIBUTES (ivar) = tree_cons (get_identifier ("omp simt private"),
4754 0 : NULL, DECL_ATTRIBUTES (ivar));
4755 0 : sctx->simt_eargs.safe_push (build1 (ADDR_EXPR, ptype, ivar));
4756 0 : tree clobber = build_clobber (type);
4757 0 : gimple *g = gimple_build_assign (ivar, clobber);
4758 0 : gimple_seq_add_stmt (&sctx->simt_dlist, g);
4759 : }
4760 : else
4761 : {
4762 7174 : tree atype = build_array_type_nelts (TREE_TYPE (new_var), sctx->max_vf);
4763 7174 : tree avar = create_tmp_var_raw (atype);
4764 7174 : if (TREE_ADDRESSABLE (new_var))
4765 912 : TREE_ADDRESSABLE (avar) = 1;
4766 7174 : DECL_ATTRIBUTES (avar)
4767 7174 : = tree_cons (get_identifier ("omp simd array"), NULL,
4768 7174 : DECL_ATTRIBUTES (avar));
4769 7174 : gimple_add_tmp_var (avar);
4770 7174 : tree iavar = avar;
4771 7174 : if (rvar && !ctx->for_simd_scan_phase)
4772 : {
4773 : /* For inscan reductions, create another array temporary,
4774 : which will hold the reduced value. */
4775 232 : iavar = create_tmp_var_raw (atype);
4776 232 : if (TREE_ADDRESSABLE (new_var))
4777 33 : TREE_ADDRESSABLE (iavar) = 1;
4778 232 : DECL_ATTRIBUTES (iavar)
4779 232 : = tree_cons (get_identifier ("omp simd array"), NULL,
4780 : tree_cons (get_identifier ("omp simd inscan"), NULL,
4781 232 : DECL_ATTRIBUTES (iavar)));
4782 232 : gimple_add_tmp_var (iavar);
4783 232 : ctx->cb.decl_map->put (avar, iavar);
4784 232 : if (sctx->lastlane == NULL_TREE)
4785 184 : sctx->lastlane = create_tmp_var (unsigned_type_node);
4786 232 : *rvar = build4 (ARRAY_REF, TREE_TYPE (new_var), iavar,
4787 : sctx->lastlane, NULL_TREE, NULL_TREE);
4788 232 : TREE_THIS_NOTRAP (*rvar) = 1;
4789 :
4790 232 : if (ctx->scan_exclusive)
4791 : {
4792 : /* And for exclusive scan yet another one, which will
4793 : hold the value during the scan phase. */
4794 114 : tree savar = create_tmp_var_raw (atype);
4795 114 : if (TREE_ADDRESSABLE (new_var))
4796 17 : TREE_ADDRESSABLE (savar) = 1;
4797 114 : DECL_ATTRIBUTES (savar)
4798 114 : = tree_cons (get_identifier ("omp simd array"), NULL,
4799 : tree_cons (get_identifier ("omp simd inscan "
4800 : "exclusive"), NULL,
4801 114 : DECL_ATTRIBUTES (savar)));
4802 114 : gimple_add_tmp_var (savar);
4803 114 : ctx->cb.decl_map->put (iavar, savar);
4804 114 : *rvar2 = build4 (ARRAY_REF, TREE_TYPE (new_var), savar,
4805 : sctx->idx, NULL_TREE, NULL_TREE);
4806 114 : TREE_THIS_NOTRAP (*rvar2) = 1;
4807 : }
4808 : }
4809 7174 : ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), iavar, sctx->idx,
4810 : NULL_TREE, NULL_TREE);
4811 7174 : lvar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, sctx->lane,
4812 : NULL_TREE, NULL_TREE);
4813 7174 : TREE_THIS_NOTRAP (ivar) = 1;
4814 7174 : TREE_THIS_NOTRAP (lvar) = 1;
4815 : }
4816 7174 : if (DECL_P (new_var))
4817 : {
4818 7020 : SET_DECL_VALUE_EXPR (new_var, lvar);
4819 7020 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
4820 : }
4821 : return true;
4822 : }
4823 :
4824 : /* Helper function of lower_rec_input_clauses. For a reference
4825 : in simd reduction, add an underlying variable it will reference. */
4826 :
4827 : static void
4828 64 : handle_simd_reference (location_t loc, tree new_vard, gimple_seq *ilist)
4829 : {
4830 64 : tree z = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard)));
4831 64 : if (TREE_CONSTANT (z))
4832 : {
4833 64 : z = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard)),
4834 : get_name (new_vard));
4835 64 : gimple_add_tmp_var (z);
4836 64 : TREE_ADDRESSABLE (z) = 1;
4837 64 : z = build_fold_addr_expr_loc (loc, z);
4838 64 : gimplify_assign (new_vard, z, ilist);
4839 : }
4840 64 : }
4841 :
4842 : /* Helper function for lower_rec_input_clauses. Emit into ilist sequence
4843 : code to emit (type) (tskred_temp[idx]). */
4844 :
4845 : static tree
4846 1128 : task_reduction_read (gimple_seq *ilist, tree tskred_temp, tree type,
4847 : unsigned idx)
4848 : {
4849 1128 : unsigned HOST_WIDE_INT sz
4850 1128 : = tree_to_uhwi (TYPE_SIZE_UNIT (pointer_sized_int_node));
4851 1128 : tree r = build2 (MEM_REF, pointer_sized_int_node,
4852 1128 : tskred_temp, build_int_cst (TREE_TYPE (tskred_temp),
4853 1128 : idx * sz));
4854 1128 : tree v = create_tmp_var (pointer_sized_int_node);
4855 1128 : gimple *g = gimple_build_assign (v, r);
4856 1128 : gimple_seq_add_stmt (ilist, g);
4857 1128 : if (!useless_type_conversion_p (type, pointer_sized_int_node))
4858 : {
4859 880 : v = create_tmp_var (type);
4860 880 : g = gimple_build_assign (v, NOP_EXPR, gimple_assign_lhs (g));
4861 880 : gimple_seq_add_stmt (ilist, g);
4862 : }
4863 1128 : return v;
4864 : }
4865 :
4866 : /* Lower early initialization of privatized variable NEW_VAR
4867 : if it needs an allocator (has allocate clause). */
4868 :
4869 : static bool
4870 144780 : lower_private_allocate (tree var, tree new_var, tree &allocator,
4871 : tree &allocate_ptr, gimple_seq *ilist,
4872 : omp_context *ctx, bool is_ref, tree size)
4873 : {
4874 144780 : if (allocator)
4875 : return false;
4876 144686 : gcc_assert (allocate_ptr == NULL_TREE);
4877 144686 : if (ctx->allocate_map
4878 3108 : && (DECL_P (new_var) || (TYPE_P (new_var) && size)))
4879 3105 : if (tree *allocatorp = ctx->allocate_map->get (var))
4880 1161 : allocator = *allocatorp;
4881 144686 : if (allocator == NULL_TREE)
4882 : return false;
4883 1161 : if (!is_ref && omp_privatize_by_reference (var))
4884 : {
4885 0 : allocator = NULL_TREE;
4886 0 : return false;
4887 : }
4888 :
4889 1161 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET && DECL_P (allocator))
4890 : {
4891 4 : allocator = lookup_decl (allocator, ctx);
4892 4 : gcc_checking_assert (allocator != NULL_TREE);
4893 : }
4894 :
4895 1161 : unsigned HOST_WIDE_INT ialign = 0;
4896 1161 : if (TREE_CODE (allocator) == TREE_LIST)
4897 : {
4898 248 : ialign = tree_to_uhwi (TREE_VALUE (allocator));
4899 248 : allocator = TREE_PURPOSE (allocator);
4900 : }
4901 1161 : if (TREE_CODE (allocator) != INTEGER_CST)
4902 443 : allocator = build_outer_var_ref (allocator, ctx, OMP_CLAUSE_ALLOCATE);
4903 1161 : allocator = fold_convert (pointer_sized_int_node, allocator);
4904 1161 : if (TREE_CODE (allocator) != INTEGER_CST)
4905 : {
4906 443 : tree var = create_tmp_var (TREE_TYPE (allocator));
4907 443 : gimplify_assign (var, allocator, ilist);
4908 443 : allocator = var;
4909 : }
4910 :
4911 1161 : tree ptr_type, align, sz = size;
4912 1161 : if (TYPE_P (new_var))
4913 : {
4914 225 : ptr_type = build_pointer_type (new_var);
4915 225 : ialign = MAX (ialign, TYPE_ALIGN_UNIT (new_var));
4916 : }
4917 936 : else if (is_ref)
4918 : {
4919 114 : ptr_type = build_pointer_type (TREE_TYPE (TREE_TYPE (new_var)));
4920 114 : ialign = MAX (ialign, TYPE_ALIGN_UNIT (TREE_TYPE (ptr_type)));
4921 : }
4922 : else
4923 : {
4924 822 : ptr_type = build_pointer_type (TREE_TYPE (new_var));
4925 822 : ialign = MAX (ialign, DECL_ALIGN_UNIT (new_var));
4926 822 : if (sz == NULL_TREE)
4927 720 : sz = fold_convert (size_type_node, DECL_SIZE_UNIT (new_var));
4928 : }
4929 1161 : align = build_int_cst (size_type_node, ialign);
4930 1161 : if (TREE_CODE (sz) != INTEGER_CST)
4931 : {
4932 54 : tree szvar = create_tmp_var (size_type_node);
4933 54 : gimplify_assign (szvar, sz, ilist);
4934 54 : sz = szvar;
4935 : }
4936 1161 : allocate_ptr = create_tmp_var (ptr_type);
4937 1161 : tree a = builtin_decl_explicit (BUILT_IN_GOMP_ALLOC);
4938 1161 : gimple *g = gimple_build_call (a, 3, align, sz, allocator);
4939 1161 : gimple_call_set_lhs (g, allocate_ptr);
4940 1161 : gimple_seq_add_stmt (ilist, g);
4941 1161 : if (!is_ref)
4942 : {
4943 822 : tree x = build_simple_mem_ref (allocate_ptr);
4944 822 : TREE_THIS_NOTRAP (x) = 1;
4945 822 : SET_DECL_VALUE_EXPR (new_var, x);
4946 822 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
4947 : }
4948 : return true;
4949 : }
4950 :
4951 : /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
4952 : from the receiver (aka child) side and initializers for REFERENCE_TYPE
4953 : private variables. Initialization statements go in ILIST, while calls
4954 : to destructors go in DLIST. */
4955 :
4956 : static void
4957 77364 : lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist,
4958 : omp_context *ctx, struct omp_for_data *fd)
4959 : {
4960 77364 : tree c, copyin_seq, x, ptr;
4961 77364 : bool copyin_by_ref = false;
4962 77364 : bool lastprivate_firstprivate = false;
4963 77364 : bool reduction_omp_orig_ref = false;
4964 77364 : int pass;
4965 77364 : bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
4966 77364 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD);
4967 77364 : omplow_simd_context sctx = omplow_simd_context ();
4968 77364 : tree simt_lane = NULL_TREE, simtrec = NULL_TREE;
4969 77364 : tree ivar = NULL_TREE, lvar = NULL_TREE, uid = NULL_TREE;
4970 77364 : gimple_seq llist[4] = { };
4971 77364 : tree nonconst_simd_if = NULL_TREE;
4972 :
4973 77364 : copyin_seq = NULL;
4974 77364 : sctx.is_simt = is_simd && omp_find_clause (clauses, OMP_CLAUSE__SIMT_);
4975 :
4976 : /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
4977 : with data sharing clauses referencing variable sized vars. That
4978 : is unnecessarily hard to support and very unlikely to result in
4979 : vectorized code anyway. */
4980 9358 : if (is_simd)
4981 61475 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
4982 52117 : switch (OMP_CLAUSE_CODE (c))
4983 : {
4984 7582 : case OMP_CLAUSE_LINEAR:
4985 7582 : if (OMP_CLAUSE_LINEAR_ARRAY (c))
4986 216 : sctx.max_vf = 1;
4987 : /* FALLTHRU */
4988 25158 : case OMP_CLAUSE_PRIVATE:
4989 25158 : case OMP_CLAUSE_FIRSTPRIVATE:
4990 25158 : case OMP_CLAUSE_LASTPRIVATE:
4991 25158 : if (is_variable_sized (OMP_CLAUSE_DECL (c)))
4992 0 : sctx.max_vf = 1;
4993 25158 : else if (omp_privatize_by_reference (OMP_CLAUSE_DECL (c)))
4994 : {
4995 279 : tree rtype = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
4996 279 : if (!TREE_CONSTANT (TYPE_SIZE_UNIT (rtype)))
4997 96 : sctx.max_vf = 1;
4998 : }
4999 : break;
5000 2578 : case OMP_CLAUSE_REDUCTION:
5001 2578 : case OMP_CLAUSE_IN_REDUCTION:
5002 2578 : if (TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF
5003 2578 : || is_variable_sized (OMP_CLAUSE_DECL (c)))
5004 180 : sctx.max_vf = 1;
5005 2398 : else if (omp_privatize_by_reference (OMP_CLAUSE_DECL (c)))
5006 : {
5007 144 : tree rtype = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
5008 144 : if (!TREE_CONSTANT (TYPE_SIZE_UNIT (rtype)))
5009 0 : sctx.max_vf = 1;
5010 : }
5011 : break;
5012 759 : case OMP_CLAUSE_IF:
5013 759 : if (integer_zerop (OMP_CLAUSE_IF_EXPR (c)))
5014 124 : sctx.max_vf = 1;
5015 635 : else if (TREE_CODE (OMP_CLAUSE_IF_EXPR (c)) != INTEGER_CST)
5016 625 : nonconst_simd_if = OMP_CLAUSE_IF_EXPR (c);
5017 : break;
5018 626 : case OMP_CLAUSE_SIMDLEN:
5019 626 : if (integer_onep (OMP_CLAUSE_SIMDLEN_EXPR (c)))
5020 110 : sctx.max_vf = 1;
5021 : break;
5022 184 : case OMP_CLAUSE__CONDTEMP_:
5023 : /* FIXME: lastprivate(conditional:) not handled for SIMT yet. */
5024 184 : if (sctx.is_simt)
5025 0 : sctx.max_vf = 1;
5026 : break;
5027 22812 : default:
5028 22812 : continue;
5029 22812 : }
5030 :
5031 : /* Add a placeholder for simduid. */
5032 77364 : if (sctx.is_simt && maybe_ne (sctx.max_vf, 1U))
5033 0 : sctx.simt_eargs.safe_push (NULL_TREE);
5034 :
5035 : unsigned task_reduction_cnt = 0;
5036 : unsigned task_reduction_cntorig = 0;
5037 : unsigned task_reduction_cnt_full = 0;
5038 : unsigned task_reduction_cntorig_full = 0;
5039 : unsigned task_reduction_other_cnt = 0;
5040 235163 : tree tskred_atype = NULL_TREE, tskred_avar = NULL_TREE;
5041 : tree tskred_base = NULL_TREE, tskred_temp = NULL_TREE;
5042 : /* Do all the fixed sized types in the first pass, and the variable sized
5043 : types in the second pass. This makes sure that the scalar arguments to
5044 : the variable sized types are processed before we use them in the
5045 : variable sized operations. For task reductions we use 4 passes, in the
5046 : first two we ignore them, in the third one gather arguments for
5047 : GOMP_task_reduction_remap call and in the last pass actually handle
5048 : the task reductions. */
5049 392962 : for (pass = 0; pass < ((task_reduction_cnt || task_reduction_other_cnt)
5050 235163 : ? 4 : 2); ++pass)
5051 : {
5052 157803 : if (pass == 2 && task_reduction_cnt)
5053 : {
5054 871 : tskred_atype
5055 871 : = build_array_type_nelts (ptr_type_node, task_reduction_cnt
5056 871 : + task_reduction_cntorig);
5057 871 : tskred_avar = create_tmp_var_raw (tskred_atype);
5058 871 : gimple_add_tmp_var (tskred_avar);
5059 871 : TREE_ADDRESSABLE (tskred_avar) = 1;
5060 871 : task_reduction_cnt_full = task_reduction_cnt;
5061 871 : task_reduction_cntorig_full = task_reduction_cntorig;
5062 : }
5063 156932 : else if (pass == 3 && task_reduction_cnt)
5064 : {
5065 871 : x = builtin_decl_explicit (BUILT_IN_GOMP_TASK_REDUCTION_REMAP);
5066 871 : gimple *g
5067 871 : = gimple_build_call (x, 3, size_int (task_reduction_cnt),
5068 871 : size_int (task_reduction_cntorig),
5069 : build_fold_addr_expr (tskred_avar));
5070 871 : gimple_seq_add_stmt (ilist, g);
5071 : }
5072 157803 : if (pass == 3 && task_reduction_other_cnt)
5073 : {
5074 : /* For reduction clauses, build
5075 : tskred_base = (void *) tskred_temp[2]
5076 : + omp_get_thread_num () * tskred_temp[1]
5077 : or if tskred_temp[1] is known to be constant, that constant
5078 : directly. This is the start of the private reduction copy block
5079 : for the current thread. */
5080 846 : tree v = create_tmp_var (integer_type_node);
5081 846 : x = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
5082 846 : gimple *g = gimple_build_call (x, 0);
5083 846 : gimple_call_set_lhs (g, v);
5084 846 : gimple_seq_add_stmt (ilist, g);
5085 846 : c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
5086 846 : tskred_temp = OMP_CLAUSE_DECL (c);
5087 846 : if (is_taskreg_ctx (ctx))
5088 557 : tskred_temp = lookup_decl (tskred_temp, ctx);
5089 846 : tree v2 = create_tmp_var (sizetype);
5090 846 : g = gimple_build_assign (v2, NOP_EXPR, v);
5091 846 : gimple_seq_add_stmt (ilist, g);
5092 846 : if (ctx->task_reductions[0])
5093 829 : v = fold_convert (sizetype, ctx->task_reductions[0]);
5094 : else
5095 17 : v = task_reduction_read (ilist, tskred_temp, sizetype, 1);
5096 846 : tree v3 = create_tmp_var (sizetype);
5097 846 : g = gimple_build_assign (v3, MULT_EXPR, v2, v);
5098 846 : gimple_seq_add_stmt (ilist, g);
5099 846 : v = task_reduction_read (ilist, tskred_temp, ptr_type_node, 2);
5100 846 : tskred_base = create_tmp_var (ptr_type_node);
5101 846 : g = gimple_build_assign (tskred_base, POINTER_PLUS_EXPR, v, v3);
5102 846 : gimple_seq_add_stmt (ilist, g);
5103 : }
5104 157803 : task_reduction_cnt = 0;
5105 157803 : task_reduction_cntorig = 0;
5106 157803 : task_reduction_other_cnt = 0;
5107 762782 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
5108 : {
5109 604983 : enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c);
5110 604983 : tree var, new_var;
5111 604983 : bool by_ref;
5112 604983 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
5113 604983 : bool task_reduction_p = false;
5114 604983 : bool task_reduction_needs_orig_p = false;
5115 604983 : tree cond = NULL_TREE;
5116 604983 : tree allocator, allocate_ptr;
5117 :
5118 604983 : switch (c_kind)
5119 : {
5120 136166 : case OMP_CLAUSE_PRIVATE:
5121 136166 : if (OMP_CLAUSE_PRIVATE_DEBUG (c))
5122 436275 : continue;
5123 : break;
5124 60138 : case OMP_CLAUSE_SHARED:
5125 : /* Ignore shared directives in teams construct inside
5126 : of target construct. */
5127 60138 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
5128 60138 : && !is_host_teams_ctx (ctx))
5129 23506 : continue;
5130 36632 : if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL)
5131 : {
5132 12080 : gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c)
5133 : || is_global_var (OMP_CLAUSE_DECL (c)));
5134 12080 : continue;
5135 : }
5136 : case OMP_CLAUSE_FIRSTPRIVATE:
5137 : case OMP_CLAUSE_COPYIN:
5138 : break;
5139 15618 : case OMP_CLAUSE_LINEAR:
5140 15618 : if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c)
5141 15618 : && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
5142 : lastprivate_firstprivate = true;
5143 : break;
5144 37306 : case OMP_CLAUSE_REDUCTION:
5145 37306 : case OMP_CLAUSE_IN_REDUCTION:
5146 37306 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
5147 29058 : || is_task_ctx (ctx)
5148 64024 : || OMP_CLAUSE_REDUCTION_TASK (c))
5149 : {
5150 12812 : task_reduction_p = true;
5151 12812 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
5152 : {
5153 4564 : task_reduction_other_cnt++;
5154 4564 : if (pass == 2)
5155 1141 : continue;
5156 : }
5157 : else
5158 8248 : task_reduction_cnt++;
5159 11671 : if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5160 : {
5161 696 : var = OMP_CLAUSE_DECL (c);
5162 : /* If var is a global variable that isn't privatized
5163 : in outer contexts, we don't need to look up the
5164 : original address, it is always the address of the
5165 : global variable itself. */
5166 696 : if (!DECL_P (var)
5167 272 : || omp_privatize_by_reference (var)
5168 923 : || !is_global_var
5169 227 : (maybe_lookup_decl_in_outer_ctx (var, ctx)))
5170 : {
5171 594 : task_reduction_needs_orig_p = true;
5172 594 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5173 492 : task_reduction_cntorig++;
5174 : }
5175 : }
5176 : }
5177 24494 : else if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5178 358115 : reduction_omp_orig_ref = true;
5179 : break;
5180 3384 : case OMP_CLAUSE__REDUCTEMP_:
5181 3384 : if (!is_taskreg_ctx (ctx))
5182 1156 : continue;
5183 : /* FALLTHRU */
5184 111492 : case OMP_CLAUSE__LOOPTEMP_:
5185 : /* Handle _looptemp_/_reductemp_ clauses only on
5186 : parallel/task. */
5187 111492 : if (fd)
5188 69234 : continue;
5189 : break;
5190 43724 : case OMP_CLAUSE_LASTPRIVATE:
5191 43724 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
5192 : {
5193 2066 : lastprivate_firstprivate = true;
5194 2066 : if (pass != 0 || is_taskloop_ctx (ctx))
5195 1347 : continue;
5196 : }
5197 : /* Even without corresponding firstprivate, if
5198 : decl is Fortran allocatable, it needs outer var
5199 : reference. */
5200 41658 : else if (pass == 0
5201 62451 : && lang_hooks.decls.omp_private_outer_ref
5202 20793 : (OMP_CLAUSE_DECL (c)))
5203 : lastprivate_firstprivate = true;
5204 : break;
5205 280 : case OMP_CLAUSE_ALIGNED:
5206 280 : if (pass != 1)
5207 140 : continue;
5208 140 : var = OMP_CLAUSE_DECL (c);
5209 140 : if (TREE_CODE (TREE_TYPE (var)) == POINTER_TYPE
5210 140 : && !is_global_var (var))
5211 : {
5212 76 : new_var = maybe_lookup_decl (var, ctx);
5213 76 : if (new_var == NULL_TREE)
5214 6 : new_var = maybe_lookup_decl_in_outer_ctx (var, ctx);
5215 76 : x = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
5216 76 : tree alarg = omp_clause_aligned_alignment (c);
5217 76 : alarg = fold_convert_loc (clause_loc, size_type_node, alarg);
5218 76 : x = build_call_expr_loc (clause_loc, x, 2, new_var, alarg);
5219 76 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
5220 76 : x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
5221 76 : gimplify_and_add (x, ilist);
5222 : }
5223 64 : else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
5224 64 : && is_global_var (var))
5225 : {
5226 64 : tree ptype = build_pointer_type (TREE_TYPE (var)), t, t2;
5227 64 : new_var = lookup_decl (var, ctx);
5228 64 : t = maybe_lookup_decl_in_outer_ctx (var, ctx);
5229 64 : t = build_fold_addr_expr_loc (clause_loc, t);
5230 64 : t2 = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
5231 64 : tree alarg = omp_clause_aligned_alignment (c);
5232 64 : alarg = fold_convert_loc (clause_loc, size_type_node, alarg);
5233 64 : t = build_call_expr_loc (clause_loc, t2, 2, t, alarg);
5234 64 : t = fold_convert_loc (clause_loc, ptype, t);
5235 64 : x = create_tmp_var (ptype);
5236 64 : t = build2 (MODIFY_EXPR, ptype, x, t);
5237 64 : gimplify_and_add (t, ilist);
5238 64 : t = build_simple_mem_ref_loc (clause_loc, x);
5239 64 : SET_DECL_VALUE_EXPR (new_var, t);
5240 64 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5241 : }
5242 140 : continue;
5243 1514 : case OMP_CLAUSE__CONDTEMP_:
5244 1514 : if (is_parallel_ctx (ctx)
5245 1514 : || (is_simd && !OMP_CLAUSE__CONDTEMP__ITER (c)))
5246 : break;
5247 1094 : continue;
5248 136108 : default:
5249 136108 : continue;
5250 137342 : }
5251 :
5252 358115 : if (task_reduction_p != (pass >= 2))
5253 15632 : continue;
5254 :
5255 342483 : allocator = NULL_TREE;
5256 342483 : allocate_ptr = NULL_TREE;
5257 342483 : new_var = var = OMP_CLAUSE_DECL (c);
5258 342483 : if ((c_kind == OMP_CLAUSE_REDUCTION
5259 342483 : || c_kind == OMP_CLAUSE_IN_REDUCTION)
5260 29705 : && TREE_CODE (var) == MEM_REF)
5261 : {
5262 4356 : var = TREE_OPERAND (var, 0);
5263 4356 : if (TREE_CODE (var) == POINTER_PLUS_EXPR)
5264 773 : var = TREE_OPERAND (var, 0);
5265 4356 : if (TREE_CODE (var) == INDIRECT_REF
5266 4168 : || TREE_CODE (var) == ADDR_EXPR)
5267 2506 : var = TREE_OPERAND (var, 0);
5268 4356 : if (is_variable_sized (var))
5269 : {
5270 179 : gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
5271 179 : var = DECL_VALUE_EXPR (var);
5272 179 : gcc_assert (TREE_CODE (var) == INDIRECT_REF);
5273 179 : var = TREE_OPERAND (var, 0);
5274 179 : gcc_assert (DECL_P (var));
5275 : }
5276 4356 : new_var = var;
5277 : }
5278 29705 : if (c_kind == OMP_CLAUSE_IN_REDUCTION && is_omp_target (ctx->stmt))
5279 : {
5280 844 : splay_tree_key key = (splay_tree_key) &DECL_CONTEXT (var);
5281 844 : new_var = (tree) splay_tree_lookup (ctx->field_map, key)->value;
5282 : }
5283 341639 : else if (c_kind != OMP_CLAUSE_COPYIN)
5284 340611 : new_var = lookup_decl (var, ctx);
5285 :
5286 342483 : if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN)
5287 : {
5288 24904 : if (pass != 0)
5289 12451 : continue;
5290 : }
5291 : /* C/C++ array section reductions. */
5292 317579 : else if ((c_kind == OMP_CLAUSE_REDUCTION
5293 : || c_kind == OMP_CLAUSE_IN_REDUCTION)
5294 317579 : && var != OMP_CLAUSE_DECL (c))
5295 : {
5296 4356 : if (pass == 0)
5297 873 : continue;
5298 :
5299 3483 : tree bias = TREE_OPERAND (OMP_CLAUSE_DECL (c), 1);
5300 3483 : tree orig_var = TREE_OPERAND (OMP_CLAUSE_DECL (c), 0);
5301 :
5302 3483 : if (TREE_CODE (orig_var) == POINTER_PLUS_EXPR)
5303 : {
5304 723 : tree b = TREE_OPERAND (orig_var, 1);
5305 723 : if (is_omp_target (ctx->stmt))
5306 : b = NULL_TREE;
5307 : else
5308 723 : b = maybe_lookup_decl (b, ctx);
5309 723 : if (b == NULL)
5310 : {
5311 17 : b = TREE_OPERAND (orig_var, 1);
5312 17 : b = maybe_lookup_decl_in_outer_ctx (b, ctx);
5313 : }
5314 723 : if (integer_zerop (bias))
5315 : bias = b;
5316 : else
5317 : {
5318 0 : bias = fold_convert_loc (clause_loc,
5319 0 : TREE_TYPE (b), bias);
5320 0 : bias = fold_build2_loc (clause_loc, PLUS_EXPR,
5321 0 : TREE_TYPE (b), b, bias);
5322 : }
5323 723 : orig_var = TREE_OPERAND (orig_var, 0);
5324 : }
5325 3483 : if (pass == 2)
5326 : {
5327 1149 : tree out = maybe_lookup_decl_in_outer_ctx (var, ctx);
5328 1149 : if (is_global_var (out)
5329 241 : && TREE_CODE (TREE_TYPE (out)) != POINTER_TYPE
5330 1336 : && (TREE_CODE (TREE_TYPE (out)) != REFERENCE_TYPE
5331 0 : || (TREE_CODE (TREE_TYPE (TREE_TYPE (out)))
5332 : != POINTER_TYPE)))
5333 : x = var;
5334 962 : else if (is_omp_target (ctx->stmt))
5335 : x = out;
5336 : else
5337 : {
5338 866 : bool by_ref = use_pointer_for_field (var, NULL);
5339 866 : x = build_receiver_ref (var, by_ref, ctx);
5340 866 : if (TREE_CODE (TREE_TYPE (var)) == REFERENCE_TYPE
5341 866 : && (TREE_CODE (TREE_TYPE (TREE_TYPE (var)))
5342 : == POINTER_TYPE))
5343 32 : x = build_fold_addr_expr (x);
5344 : }
5345 1149 : if (TREE_CODE (orig_var) == INDIRECT_REF)
5346 40 : x = build_simple_mem_ref (x);
5347 1109 : else if (TREE_CODE (orig_var) == ADDR_EXPR)
5348 : {
5349 646 : if (var == TREE_OPERAND (orig_var, 0))
5350 583 : x = build_fold_addr_expr (x);
5351 : }
5352 1149 : bias = fold_convert (sizetype, bias);
5353 1149 : x = fold_convert (ptr_type_node, x);
5354 1149 : x = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
5355 1149 : TREE_TYPE (x), x, bias);
5356 1149 : unsigned cnt = task_reduction_cnt - 1;
5357 1149 : if (!task_reduction_needs_orig_p)
5358 1061 : cnt += (task_reduction_cntorig_full
5359 1061 : - task_reduction_cntorig);
5360 : else
5361 88 : cnt = task_reduction_cntorig - 1;
5362 1149 : tree r = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5363 1149 : size_int (cnt), NULL_TREE, NULL_TREE);
5364 1149 : gimplify_assign (r, x, ilist);
5365 1149 : continue;
5366 1149 : }
5367 :
5368 2334 : if (TREE_CODE (orig_var) == INDIRECT_REF
5369 2240 : || TREE_CODE (orig_var) == ADDR_EXPR)
5370 1362 : orig_var = TREE_OPERAND (orig_var, 0);
5371 2334 : tree d = OMP_CLAUSE_DECL (c);
5372 2334 : tree type = TREE_TYPE (d);
5373 2334 : gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
5374 2334 : tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
5375 2334 : tree sz = v;
5376 2334 : const char *name = get_name (orig_var);
5377 2334 : if (pass != 3 && !TREE_CONSTANT (v))
5378 : {
5379 93 : tree t;
5380 93 : if (is_omp_target (ctx->stmt))
5381 : t = NULL_TREE;
5382 : else
5383 93 : t = maybe_lookup_decl (v, ctx);
5384 93 : if (t)
5385 88 : v = t;
5386 : else
5387 5 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
5388 93 : gimplify_expr (&v, ilist, NULL, is_gimple_val, fb_rvalue);
5389 186 : t = fold_build2_loc (clause_loc, PLUS_EXPR,
5390 93 : TREE_TYPE (v), v,
5391 93 : build_int_cst (TREE_TYPE (v), 1));
5392 93 : sz = fold_build2_loc (clause_loc, MULT_EXPR,
5393 93 : TREE_TYPE (v), t,
5394 93 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5395 : }
5396 2334 : if (pass == 3)
5397 : {
5398 1461 : tree xv = create_tmp_var (ptr_type_node);
5399 1461 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5400 : {
5401 1149 : unsigned cnt = task_reduction_cnt - 1;
5402 1149 : if (!task_reduction_needs_orig_p)
5403 1061 : cnt += (task_reduction_cntorig_full
5404 1061 : - task_reduction_cntorig);
5405 : else
5406 88 : cnt = task_reduction_cntorig - 1;
5407 1149 : x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5408 1149 : size_int (cnt), NULL_TREE, NULL_TREE);
5409 :
5410 1149 : gimple *g = gimple_build_assign (xv, x);
5411 1149 : gimple_seq_add_stmt (ilist, g);
5412 : }
5413 : else
5414 : {
5415 312 : unsigned int idx = *ctx->task_reduction_map->get (c);
5416 312 : tree off;
5417 312 : if (ctx->task_reductions[1 + idx])
5418 81 : off = fold_convert (sizetype,
5419 : ctx->task_reductions[1 + idx]);
5420 : else
5421 231 : off = task_reduction_read (ilist, tskred_temp, sizetype,
5422 231 : 7 + 3 * idx + 1);
5423 312 : gimple *g = gimple_build_assign (xv, POINTER_PLUS_EXPR,
5424 : tskred_base, off);
5425 312 : gimple_seq_add_stmt (ilist, g);
5426 : }
5427 1461 : x = fold_convert (build_pointer_type (boolean_type_node),
5428 : xv);
5429 1461 : if (TREE_CONSTANT (v))
5430 1065 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (x), x,
5431 : TYPE_SIZE_UNIT (type));
5432 : else
5433 : {
5434 396 : tree t;
5435 396 : if (is_omp_target (ctx->stmt))
5436 : t = NULL_TREE;
5437 : else
5438 336 : t = maybe_lookup_decl (v, ctx);
5439 336 : if (t)
5440 316 : v = t;
5441 : else
5442 80 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
5443 396 : gimplify_expr (&v, ilist, NULL, is_gimple_val,
5444 : fb_rvalue);
5445 792 : t = fold_build2_loc (clause_loc, PLUS_EXPR,
5446 396 : TREE_TYPE (v), v,
5447 396 : build_int_cst (TREE_TYPE (v), 1));
5448 396 : t = fold_build2_loc (clause_loc, MULT_EXPR,
5449 396 : TREE_TYPE (v), t,
5450 396 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5451 396 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (x), x, t);
5452 : }
5453 1461 : cond = create_tmp_var (TREE_TYPE (x));
5454 1461 : gimplify_assign (cond, x, ilist);
5455 1461 : x = xv;
5456 : }
5457 873 : else if (lower_private_allocate (var, type, allocator,
5458 : allocate_ptr, ilist, ctx,
5459 : true,
5460 873 : TREE_CONSTANT (v)
5461 780 : ? TYPE_SIZE_UNIT (type)
5462 : : sz))
5463 225 : x = allocate_ptr;
5464 648 : else if (TREE_CONSTANT (v))
5465 : {
5466 560 : x = create_tmp_var_raw (type, name);
5467 560 : gimple_add_tmp_var (x);
5468 560 : TREE_ADDRESSABLE (x) = 1;
5469 560 : x = build_fold_addr_expr_loc (clause_loc, x);
5470 : }
5471 : else
5472 : {
5473 88 : tree atmp
5474 88 : = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
5475 88 : tree al = size_int (TYPE_ALIGN (TREE_TYPE (type)));
5476 88 : x = build_call_expr_loc (clause_loc, atmp, 2, sz, al);
5477 : }
5478 :
5479 2334 : tree ptype = build_pointer_type (TREE_TYPE (type));
5480 2334 : x = fold_convert_loc (clause_loc, ptype, x);
5481 2334 : tree y = create_tmp_var (ptype, name);
5482 2334 : gimplify_assign (y, x, ilist);
5483 2334 : x = y;
5484 2334 : tree yb = y;
5485 :
5486 2334 : if (!integer_zerop (bias))
5487 : {
5488 1512 : bias = fold_convert_loc (clause_loc, pointer_sized_int_node,
5489 : bias);
5490 1512 : yb = fold_convert_loc (clause_loc, pointer_sized_int_node,
5491 : x);
5492 1512 : yb = fold_build2_loc (clause_loc, MINUS_EXPR,
5493 : pointer_sized_int_node, yb, bias);
5494 1512 : x = fold_convert_loc (clause_loc, TREE_TYPE (x), yb);
5495 1512 : yb = create_tmp_var (ptype, name);
5496 1512 : gimplify_assign (yb, x, ilist);
5497 1512 : x = yb;
5498 : }
5499 :
5500 2334 : d = TREE_OPERAND (d, 0);
5501 2334 : if (TREE_CODE (d) == POINTER_PLUS_EXPR)
5502 420 : d = TREE_OPERAND (d, 0);
5503 2334 : if (TREE_CODE (d) == ADDR_EXPR)
5504 : {
5505 1268 : if (orig_var != var)
5506 : {
5507 91 : gcc_assert (is_variable_sized (orig_var));
5508 91 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var),
5509 : x);
5510 91 : gimplify_assign (new_var, x, ilist);
5511 91 : tree new_orig_var = lookup_decl (orig_var, ctx);
5512 91 : tree t = build_fold_indirect_ref (new_var);
5513 91 : DECL_IGNORED_P (new_var) = 0;
5514 91 : TREE_THIS_NOTRAP (t) = 1;
5515 91 : SET_DECL_VALUE_EXPR (new_orig_var, t);
5516 91 : DECL_HAS_VALUE_EXPR_P (new_orig_var) = 1;
5517 : }
5518 : else
5519 : {
5520 1177 : x = build2 (MEM_REF, TREE_TYPE (new_var), x,
5521 : build_int_cst (ptype, 0));
5522 1177 : SET_DECL_VALUE_EXPR (new_var, x);
5523 1177 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5524 : }
5525 : }
5526 : else
5527 : {
5528 1066 : gcc_assert (orig_var == var);
5529 1066 : if (TREE_CODE (d) == INDIRECT_REF)
5530 : {
5531 94 : x = create_tmp_var (ptype, name);
5532 94 : TREE_ADDRESSABLE (x) = 1;
5533 94 : gimplify_assign (x, yb, ilist);
5534 94 : x = build_fold_addr_expr_loc (clause_loc, x);
5535 : }
5536 1066 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
5537 1066 : gimplify_assign (new_var, x, ilist);
5538 : }
5539 : /* GOMP_taskgroup_reduction_register memsets the whole
5540 : array to zero. If the initializer is zero, we don't
5541 : need to initialize it again, just mark it as ever
5542 : used unconditionally, i.e. cond = true. */
5543 2334 : if (cond
5544 1461 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) == NULL_TREE
5545 3267 : && initializer_zerop (omp_reduction_init (c,
5546 933 : TREE_TYPE (type))))
5547 : {
5548 624 : gimple *g = gimple_build_assign (build_simple_mem_ref (cond),
5549 : boolean_true_node);
5550 624 : gimple_seq_add_stmt (ilist, g);
5551 624 : continue;
5552 624 : }
5553 1710 : tree end = create_artificial_label (UNKNOWN_LOCATION);
5554 1710 : if (cond)
5555 : {
5556 837 : gimple *g;
5557 837 : if (!is_parallel_ctx (ctx))
5558 : {
5559 786 : tree condv = create_tmp_var (boolean_type_node);
5560 786 : g = gimple_build_assign (condv,
5561 : build_simple_mem_ref (cond));
5562 786 : gimple_seq_add_stmt (ilist, g);
5563 786 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
5564 786 : g = gimple_build_cond (NE_EXPR, condv,
5565 : boolean_false_node, end, lab1);
5566 786 : gimple_seq_add_stmt (ilist, g);
5567 786 : gimple_seq_add_stmt (ilist, gimple_build_label (lab1));
5568 : }
5569 837 : g = gimple_build_assign (build_simple_mem_ref (cond),
5570 : boolean_true_node);
5571 837 : gimple_seq_add_stmt (ilist, g);
5572 : }
5573 :
5574 1710 : tree y1 = create_tmp_var (ptype);
5575 1710 : gimplify_assign (y1, y, ilist);
5576 1710 : tree i2 = NULL_TREE, y2 = NULL_TREE;
5577 1710 : tree body2 = NULL_TREE, end2 = NULL_TREE;
5578 1710 : tree y3 = NULL_TREE, y4 = NULL_TREE;
5579 1710 : if (task_reduction_needs_orig_p)
5580 : {
5581 112 : y3 = create_tmp_var (ptype);
5582 112 : tree ref;
5583 112 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5584 88 : ref = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5585 88 : size_int (task_reduction_cnt_full
5586 : + task_reduction_cntorig - 1),
5587 : NULL_TREE, NULL_TREE);
5588 : else
5589 : {
5590 24 : unsigned int idx = *ctx->task_reduction_map->get (c);
5591 24 : ref = task_reduction_read (ilist, tskred_temp, ptype,
5592 24 : 7 + 3 * idx);
5593 : }
5594 112 : gimplify_assign (y3, ref, ilist);
5595 : }
5596 1598 : else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) || is_simd)
5597 : {
5598 668 : if (pass != 3)
5599 : {
5600 252 : y2 = create_tmp_var (ptype);
5601 252 : gimplify_assign (y2, y, ilist);
5602 : }
5603 668 : if (is_simd || OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5604 : {
5605 188 : tree ref = build_outer_var_ref (var, ctx);
5606 : /* For ref build_outer_var_ref already performs this. */
5607 188 : if (TREE_CODE (d) == INDIRECT_REF)
5608 0 : gcc_assert (omp_privatize_by_reference (var));
5609 188 : else if (TREE_CODE (d) == ADDR_EXPR)
5610 96 : ref = build_fold_addr_expr (ref);
5611 92 : else if (omp_privatize_by_reference (var))
5612 8 : ref = build_fold_addr_expr (ref);
5613 188 : ref = fold_convert_loc (clause_loc, ptype, ref);
5614 188 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)
5615 188 : && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
5616 : {
5617 8 : y3 = create_tmp_var (ptype);
5618 8 : gimplify_assign (y3, unshare_expr (ref), ilist);
5619 : }
5620 188 : if (is_simd)
5621 : {
5622 180 : y4 = create_tmp_var (ptype);
5623 180 : gimplify_assign (y4, ref, dlist);
5624 : }
5625 : }
5626 : }
5627 1710 : tree i = create_tmp_var (TREE_TYPE (v));
5628 1710 : gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), ilist);
5629 1710 : tree body = create_artificial_label (UNKNOWN_LOCATION);
5630 1710 : gimple_seq_add_stmt (ilist, gimple_build_label (body));
5631 1710 : if (y2)
5632 : {
5633 252 : i2 = create_tmp_var (TREE_TYPE (v));
5634 252 : gimplify_assign (i2, build_int_cst (TREE_TYPE (v), 0), dlist);
5635 252 : body2 = create_artificial_label (UNKNOWN_LOCATION);
5636 252 : end2 = create_artificial_label (UNKNOWN_LOCATION);
5637 252 : gimple_seq_add_stmt (dlist, gimple_build_label (body2));
5638 : }
5639 1710 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
5640 : {
5641 600 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
5642 600 : tree decl_placeholder
5643 600 : = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
5644 600 : SET_DECL_VALUE_EXPR (decl_placeholder,
5645 : build_simple_mem_ref (y1));
5646 600 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
5647 600 : SET_DECL_VALUE_EXPR (placeholder,
5648 : y3 ? build_simple_mem_ref (y3)
5649 : : error_mark_node);
5650 600 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
5651 600 : x = lang_hooks.decls.omp_clause_default_ctor
5652 720 : (c, build_simple_mem_ref (y1),
5653 120 : y3 ? build_simple_mem_ref (y3) : NULL_TREE);
5654 600 : if (x)
5655 152 : gimplify_and_add (x, ilist);
5656 600 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
5657 : {
5658 576 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
5659 576 : lower_omp (&tseq, ctx);
5660 576 : gimple_seq_add_seq (ilist, tseq);
5661 : }
5662 600 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
5663 600 : if (is_simd)
5664 : {
5665 0 : SET_DECL_VALUE_EXPR (decl_placeholder,
5666 : build_simple_mem_ref (y2));
5667 0 : SET_DECL_VALUE_EXPR (placeholder,
5668 : build_simple_mem_ref (y4));
5669 0 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
5670 0 : lower_omp (&tseq, ctx);
5671 0 : gimple_seq_add_seq (dlist, tseq);
5672 0 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
5673 : }
5674 600 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
5675 600 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 0;
5676 600 : if (y2)
5677 : {
5678 72 : x = lang_hooks.decls.omp_clause_dtor
5679 72 : (c, build_simple_mem_ref (y2));
5680 72 : if (x)
5681 40 : gimplify_and_add (x, dlist);
5682 : }
5683 : }
5684 : else
5685 : {
5686 1110 : x = omp_reduction_init (c, TREE_TYPE (type));
5687 1110 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
5688 :
5689 : /* reduction(-:var) sums up the partial results, so it
5690 : acts identically to reduction(+:var). */
5691 1110 : if (code == MINUS_EXPR)
5692 0 : code = PLUS_EXPR;
5693 :
5694 1110 : gimplify_assign (build_simple_mem_ref (y1), x, ilist);
5695 1110 : if (is_simd)
5696 : {
5697 180 : x = build2 (code, TREE_TYPE (type),
5698 : build_simple_mem_ref (y4),
5699 : build_simple_mem_ref (y2));
5700 180 : gimplify_assign (build_simple_mem_ref (y4), x, dlist);
5701 : }
5702 : }
5703 1710 : gimple *g
5704 1710 : = gimple_build_assign (y1, POINTER_PLUS_EXPR, y1,
5705 1710 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5706 1710 : gimple_seq_add_stmt (ilist, g);
5707 1710 : if (y3)
5708 : {
5709 120 : g = gimple_build_assign (y3, POINTER_PLUS_EXPR, y3,
5710 120 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5711 120 : gimple_seq_add_stmt (ilist, g);
5712 : }
5713 1710 : g = gimple_build_assign (i, PLUS_EXPR, i,
5714 1710 : build_int_cst (TREE_TYPE (i), 1));
5715 1710 : gimple_seq_add_stmt (ilist, g);
5716 1710 : g = gimple_build_cond (LE_EXPR, i, v, body, end);
5717 1710 : gimple_seq_add_stmt (ilist, g);
5718 1710 : gimple_seq_add_stmt (ilist, gimple_build_label (end));
5719 1710 : if (y2)
5720 : {
5721 252 : g = gimple_build_assign (y2, POINTER_PLUS_EXPR, y2,
5722 252 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5723 252 : gimple_seq_add_stmt (dlist, g);
5724 252 : if (y4)
5725 : {
5726 180 : g = gimple_build_assign
5727 180 : (y4, POINTER_PLUS_EXPR, y4,
5728 180 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5729 180 : gimple_seq_add_stmt (dlist, g);
5730 : }
5731 252 : g = gimple_build_assign (i2, PLUS_EXPR, i2,
5732 252 : build_int_cst (TREE_TYPE (i2), 1));
5733 252 : gimple_seq_add_stmt (dlist, g);
5734 252 : g = gimple_build_cond (LE_EXPR, i2, v, body2, end2);
5735 252 : gimple_seq_add_stmt (dlist, g);
5736 252 : gimple_seq_add_stmt (dlist, gimple_build_label (end2));
5737 : }
5738 1710 : if (allocator)
5739 : {
5740 225 : tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
5741 225 : g = gimple_build_call (f, 2, allocate_ptr, allocator);
5742 225 : gimple_seq_add_stmt (dlist, g);
5743 : }
5744 1710 : continue;
5745 1710 : }
5746 313223 : else if (pass == 2)
5747 : {
5748 913 : tree out = maybe_lookup_decl_in_outer_ctx (var, ctx);
5749 913 : if (is_global_var (out))
5750 : x = var;
5751 361 : else if (is_omp_target (ctx->stmt))
5752 : x = out;
5753 : else
5754 : {
5755 301 : bool by_ref = use_pointer_for_field (var, ctx);
5756 301 : x = build_receiver_ref (var, by_ref, ctx);
5757 : }
5758 913 : if (!omp_privatize_by_reference (var))
5759 823 : x = build_fold_addr_expr (x);
5760 913 : x = fold_convert (ptr_type_node, x);
5761 913 : unsigned cnt = task_reduction_cnt - 1;
5762 913 : if (!task_reduction_needs_orig_p)
5763 878 : cnt += task_reduction_cntorig_full - task_reduction_cntorig;
5764 : else
5765 35 : cnt = task_reduction_cntorig - 1;
5766 913 : tree r = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5767 913 : size_int (cnt), NULL_TREE, NULL_TREE);
5768 913 : gimplify_assign (r, x, ilist);
5769 913 : continue;
5770 913 : }
5771 312310 : else if (pass == 3)
5772 : {
5773 1742 : tree type = TREE_TYPE (new_var);
5774 1742 : if (!omp_privatize_by_reference (var))
5775 1636 : type = build_pointer_type (type);
5776 1742 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
5777 : {
5778 913 : unsigned cnt = task_reduction_cnt - 1;
5779 913 : if (!task_reduction_needs_orig_p)
5780 878 : cnt += (task_reduction_cntorig_full
5781 878 : - task_reduction_cntorig);
5782 : else
5783 35 : cnt = task_reduction_cntorig - 1;
5784 913 : x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
5785 913 : size_int (cnt), NULL_TREE, NULL_TREE);
5786 : }
5787 : else
5788 : {
5789 829 : unsigned int idx = *ctx->task_reduction_map->get (c);
5790 829 : tree off;
5791 829 : if (ctx->task_reductions[1 + idx])
5792 829 : off = fold_convert (sizetype,
5793 : ctx->task_reductions[1 + idx]);
5794 : else
5795 0 : off = task_reduction_read (ilist, tskred_temp, sizetype,
5796 0 : 7 + 3 * idx + 1);
5797 829 : x = fold_build2 (POINTER_PLUS_EXPR, ptr_type_node,
5798 : tskred_base, off);
5799 : }
5800 1742 : x = fold_convert (type, x);
5801 1742 : tree t;
5802 1742 : if (omp_privatize_by_reference (var))
5803 : {
5804 106 : gimplify_assign (new_var, x, ilist);
5805 106 : t = new_var;
5806 106 : new_var = build_simple_mem_ref (new_var);
5807 : }
5808 : else
5809 : {
5810 1636 : t = create_tmp_var (type);
5811 1636 : gimplify_assign (t, x, ilist);
5812 1636 : SET_DECL_VALUE_EXPR (new_var, build_simple_mem_ref (t));
5813 1636 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5814 : }
5815 1742 : t = fold_convert (build_pointer_type (boolean_type_node), t);
5816 1742 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t,
5817 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
5818 1742 : cond = create_tmp_var (TREE_TYPE (t));
5819 1742 : gimplify_assign (cond, t, ilist);
5820 : }
5821 310568 : else if (is_variable_sized (var))
5822 : {
5823 : /* For variable sized types, we need to allocate the
5824 : actual storage here. Call alloca and store the
5825 : result in the pointer decl that we created elsewhere. */
5826 262 : if (pass == 0)
5827 134 : continue;
5828 :
5829 128 : if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx))
5830 : {
5831 114 : tree tmp;
5832 :
5833 114 : ptr = DECL_VALUE_EXPR (new_var);
5834 114 : gcc_assert (TREE_CODE (ptr) == INDIRECT_REF);
5835 114 : ptr = TREE_OPERAND (ptr, 0);
5836 114 : gcc_assert (DECL_P (ptr));
5837 114 : x = TYPE_SIZE_UNIT (TREE_TYPE (new_var));
5838 :
5839 114 : if (lower_private_allocate (var, new_var, allocator,
5840 : allocate_ptr, ilist, ctx,
5841 : false, x))
5842 8 : tmp = allocate_ptr;
5843 : else
5844 : {
5845 : /* void *tmp = __builtin_alloca */
5846 106 : tree atmp
5847 106 : = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
5848 106 : gcall *stmt
5849 106 : = gimple_build_call (atmp, 2, x,
5850 106 : size_int (DECL_ALIGN (var)));
5851 106 : cfun->calls_alloca = 1;
5852 106 : tmp = create_tmp_var_raw (ptr_type_node);
5853 106 : gimple_add_tmp_var (tmp);
5854 106 : gimple_call_set_lhs (stmt, tmp);
5855 :
5856 106 : gimple_seq_add_stmt (ilist, stmt);
5857 : }
5858 :
5859 114 : x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp);
5860 114 : gimplify_assign (ptr, x, ilist);
5861 : }
5862 : }
5863 310306 : else if (omp_privatize_by_reference (var)
5864 310306 : && (c_kind != OMP_CLAUSE_FIRSTPRIVATE
5865 2016 : || !OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c)))
5866 : {
5867 : /* For references that are being privatized for Fortran,
5868 : allocate new backing storage for the new pointer
5869 : variable. This allows us to avoid changing all the
5870 : code that expects a pointer to something that expects
5871 : a direct variable. */
5872 4895 : if (pass == 0)
5873 2550 : continue;
5874 :
5875 2345 : x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
5876 2345 : if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx))
5877 : {
5878 146 : x = build_receiver_ref (var, false, ctx);
5879 146 : if (ctx->allocate_map)
5880 16 : if (tree *allocatep = ctx->allocate_map->get (var))
5881 : {
5882 16 : allocator = *allocatep;
5883 16 : if (TREE_CODE (allocator) == TREE_LIST)
5884 0 : allocator = TREE_PURPOSE (allocator);
5885 16 : if (TREE_CODE (allocator) != INTEGER_CST)
5886 8 : allocator = build_outer_var_ref (allocator, ctx);
5887 16 : allocator = fold_convert (pointer_sized_int_node,
5888 : allocator);
5889 16 : allocate_ptr = unshare_expr (x);
5890 : }
5891 146 : if (allocator == NULL_TREE)
5892 130 : x = build_fold_addr_expr_loc (clause_loc, x);
5893 : }
5894 2199 : else if (lower_private_allocate (var, new_var, allocator,
5895 : allocate_ptr,
5896 : ilist, ctx, true, x))
5897 86 : x = allocate_ptr;
5898 2113 : else if (TREE_CONSTANT (x))
5899 : {
5900 : /* For reduction in SIMD loop, defer adding the
5901 : initialization of the reference, because if we decide
5902 : to use SIMD array for it, the initialization could cause
5903 : expansion ICE. Ditto for other privatization clauses. */
5904 1404 : if (is_simd)
5905 : x = NULL_TREE;
5906 : else
5907 : {
5908 1077 : x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
5909 : get_name (var));
5910 1077 : gimple_add_tmp_var (x);
5911 1077 : TREE_ADDRESSABLE (x) = 1;
5912 1077 : x = build_fold_addr_expr_loc (clause_loc, x);
5913 : }
5914 : }
5915 : else
5916 : {
5917 709 : tree atmp
5918 709 : = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
5919 709 : tree rtype = TREE_TYPE (TREE_TYPE (new_var));
5920 709 : tree al = size_int (TYPE_ALIGN (rtype));
5921 709 : x = build_call_expr_loc (clause_loc, atmp, 2, x, al);
5922 : }
5923 :
5924 2018 : if (x)
5925 : {
5926 2018 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
5927 2018 : gimplify_assign (new_var, x, ilist);
5928 : }
5929 :
5930 2345 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
5931 : }
5932 305411 : else if ((c_kind == OMP_CLAUSE_REDUCTION
5933 : || c_kind == OMP_CLAUSE_IN_REDUCTION)
5934 305411 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
5935 : {
5936 1980 : if (pass == 0)
5937 990 : continue;
5938 : }
5939 303431 : else if (pass != 0)
5940 151459 : continue;
5941 :
5942 169630 : switch (OMP_CLAUSE_CODE (c))
5943 : {
5944 11939 : case OMP_CLAUSE_SHARED:
5945 : /* Ignore shared directives in teams construct inside
5946 : target construct. */
5947 11939 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
5948 11939 : && !is_host_teams_ctx (ctx))
5949 0 : continue;
5950 : /* Shared global vars are just accessed directly. */
5951 11939 : if (is_global_var (new_var))
5952 : break;
5953 : /* For taskloop firstprivate/lastprivate, represented
5954 : as firstprivate and shared clause on the task, new_var
5955 : is the firstprivate var. */
5956 11785 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
5957 : break;
5958 : /* Set up the DECL_VALUE_EXPR for shared variables now. This
5959 : needs to be delayed until after fixup_child_record_type so
5960 : that we get the correct type during the dereference. */
5961 11473 : by_ref = use_pointer_for_field (var, ctx);
5962 11473 : x = build_receiver_ref (var, by_ref, ctx);
5963 11473 : SET_DECL_VALUE_EXPR (new_var, x);
5964 11473 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5965 :
5966 : /* ??? If VAR is not passed by reference, and the variable
5967 : hasn't been initialized yet, then we'll get a warning for
5968 : the store into the omp_data_s structure. Ideally, we'd be
5969 : able to notice this and not store anything at all, but
5970 : we're generating code too early. Suppress the warning. */
5971 11473 : if (!by_ref)
5972 5242 : suppress_warning (var, OPT_Wuninitialized);
5973 : break;
5974 :
5975 210 : case OMP_CLAUSE__CONDTEMP_:
5976 210 : if (is_parallel_ctx (ctx))
5977 : {
5978 103 : x = build_receiver_ref (var, false, ctx);
5979 103 : SET_DECL_VALUE_EXPR (new_var, x);
5980 103 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
5981 : }
5982 107 : else if (is_simd && !OMP_CLAUSE__CONDTEMP__ITER (c))
5983 : {
5984 107 : x = build_zero_cst (TREE_TYPE (var));
5985 107 : goto do_private;
5986 : }
5987 : break;
5988 :
5989 21301 : case OMP_CLAUSE_LASTPRIVATE:
5990 21301 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
5991 : break;
5992 : /* FALLTHRU */
5993 :
5994 87472 : case OMP_CLAUSE_PRIVATE:
5995 87472 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_PRIVATE)
5996 20793 : x = build_outer_var_ref (var, ctx);
5997 66679 : else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
5998 : {
5999 163 : if (is_task_ctx (ctx))
6000 12 : x = build_receiver_ref (var, false, ctx);
6001 : else
6002 151 : x = build_outer_var_ref (var, ctx, OMP_CLAUSE_PRIVATE);
6003 : }
6004 : else
6005 : x = NULL;
6006 94119 : do_private:
6007 94119 : tree nx;
6008 94119 : bool copy_ctor;
6009 94119 : copy_ctor = false;
6010 94119 : lower_private_allocate (var, new_var, allocator, allocate_ptr,
6011 : ilist, ctx, false, NULL_TREE);
6012 94119 : nx = unshare_expr (new_var);
6013 94119 : if (is_simd
6014 24202 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6015 101505 : && OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c))
6016 22 : copy_ctor = true;
6017 94119 : if (copy_ctor)
6018 22 : nx = lang_hooks.decls.omp_clause_copy_ctor (c, nx, x);
6019 : else
6020 94097 : nx = lang_hooks.decls.omp_clause_default_ctor (c, nx, x);
6021 94119 : if (is_simd)
6022 : {
6023 24202 : tree y = lang_hooks.decls.omp_clause_dtor (c, new_var);
6024 23520 : if ((TREE_ADDRESSABLE (new_var) || nx || y
6025 23504 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6026 7010 : && (gimple_omp_for_collapse (ctx->stmt) != 1
6027 1375 : || (gimple_omp_for_index (ctx->stmt, 0)
6028 : != new_var)))
6029 16796 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE__CONDTEMP_
6030 16689 : || omp_privatize_by_reference (var))
6031 31067 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
6032 : ivar, lvar))
6033 : {
6034 5981 : if (omp_privatize_by_reference (var))
6035 : {
6036 52 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6037 52 : tree new_vard = TREE_OPERAND (new_var, 0);
6038 52 : gcc_assert (DECL_P (new_vard));
6039 52 : SET_DECL_VALUE_EXPR (new_vard,
6040 : build_fold_addr_expr (lvar));
6041 52 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6042 : }
6043 :
6044 5981 : if (nx)
6045 : {
6046 30 : tree iv = unshare_expr (ivar);
6047 30 : if (copy_ctor)
6048 22 : x = lang_hooks.decls.omp_clause_copy_ctor (c, iv,
6049 : x);
6050 : else
6051 8 : x = lang_hooks.decls.omp_clause_default_ctor (c,
6052 : iv,
6053 : x);
6054 : }
6055 5951 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__CONDTEMP_)
6056 : {
6057 37 : x = build2 (MODIFY_EXPR, TREE_TYPE (ivar),
6058 : unshare_expr (ivar), x);
6059 37 : nx = x;
6060 : }
6061 5981 : if (nx && x)
6062 67 : gimplify_and_add (x, &llist[0]);
6063 5981 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6064 5981 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
6065 : {
6066 37 : tree v = new_var;
6067 37 : if (!DECL_P (v))
6068 : {
6069 4 : gcc_assert (TREE_CODE (v) == MEM_REF);
6070 4 : v = TREE_OPERAND (v, 0);
6071 4 : gcc_assert (DECL_P (v));
6072 : }
6073 37 : v = *ctx->lastprivate_conditional_map->get (v);
6074 37 : tree t = create_tmp_var (TREE_TYPE (v));
6075 37 : tree z = build_zero_cst (TREE_TYPE (v));
6076 37 : tree orig_v
6077 37 : = build_outer_var_ref (var, ctx,
6078 : OMP_CLAUSE_LASTPRIVATE);
6079 37 : gimple_seq_add_stmt (dlist,
6080 37 : gimple_build_assign (t, z));
6081 37 : gcc_assert (DECL_HAS_VALUE_EXPR_P (v));
6082 37 : tree civar = DECL_VALUE_EXPR (v);
6083 37 : gcc_assert (TREE_CODE (civar) == ARRAY_REF);
6084 37 : civar = unshare_expr (civar);
6085 37 : TREE_OPERAND (civar, 1) = sctx.idx;
6086 37 : x = build2 (MODIFY_EXPR, TREE_TYPE (t), t,
6087 : unshare_expr (civar));
6088 74 : x = build2 (COMPOUND_EXPR, TREE_TYPE (orig_v), x,
6089 37 : build2 (MODIFY_EXPR, TREE_TYPE (orig_v),
6090 : orig_v, unshare_expr (ivar)));
6091 37 : tree cond = build2 (LT_EXPR, boolean_type_node, t,
6092 : civar);
6093 37 : x = build3 (COND_EXPR, void_type_node, cond, x,
6094 : void_node);
6095 37 : gimple_seq tseq = NULL;
6096 37 : gimplify_and_add (x, &tseq);
6097 37 : if (ctx->outer)
6098 27 : lower_omp (&tseq, ctx->outer);
6099 37 : gimple_seq_add_seq (&llist[1], tseq);
6100 : }
6101 5981 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6102 5981 : && ctx->for_simd_scan_phase)
6103 : {
6104 9 : x = unshare_expr (ivar);
6105 9 : tree orig_v
6106 9 : = build_outer_var_ref (var, ctx,
6107 : OMP_CLAUSE_LASTPRIVATE);
6108 9 : x = lang_hooks.decls.omp_clause_assign_op (c, x,
6109 : orig_v);
6110 9 : gimplify_and_add (x, &llist[0]);
6111 : }
6112 5981 : if (y)
6113 : {
6114 30 : y = lang_hooks.decls.omp_clause_dtor (c, ivar);
6115 30 : if (y)
6116 30 : gimplify_and_add (y, &llist[1]);
6117 : }
6118 : break;
6119 : }
6120 18221 : if (omp_privatize_by_reference (var))
6121 : {
6122 28 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6123 28 : tree new_vard = TREE_OPERAND (new_var, 0);
6124 28 : gcc_assert (DECL_P (new_vard));
6125 28 : tree type = TREE_TYPE (TREE_TYPE (new_vard));
6126 28 : x = TYPE_SIZE_UNIT (type);
6127 28 : if (TREE_CONSTANT (x))
6128 : {
6129 28 : x = create_tmp_var_raw (type, get_name (var));
6130 28 : gimple_add_tmp_var (x);
6131 28 : TREE_ADDRESSABLE (x) = 1;
6132 28 : x = build_fold_addr_expr_loc (clause_loc, x);
6133 28 : x = fold_convert_loc (clause_loc,
6134 28 : TREE_TYPE (new_vard), x);
6135 28 : gimplify_assign (new_vard, x, ilist);
6136 : }
6137 : }
6138 : }
6139 88138 : if (nx)
6140 488 : gimplify_and_add (nx, ilist);
6141 88138 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6142 15170 : && is_simd
6143 89901 : && ctx->for_simd_scan_phase)
6144 : {
6145 5 : tree orig_v = build_outer_var_ref (var, ctx,
6146 : OMP_CLAUSE_LASTPRIVATE);
6147 5 : x = lang_hooks.decls.omp_clause_assign_op (c, new_var,
6148 : orig_v);
6149 5 : gimplify_and_add (x, ilist);
6150 : }
6151 : /* FALLTHRU */
6152 :
6153 119032 : do_dtor:
6154 119032 : x = lang_hooks.decls.omp_clause_dtor (c, new_var);
6155 119032 : if (x)
6156 1134 : gimplify_and_add (x, dlist);
6157 119032 : if (allocator)
6158 : {
6159 864 : if (!is_gimple_val (allocator))
6160 : {
6161 22 : tree avar = create_tmp_var (TREE_TYPE (allocator));
6162 22 : gimplify_assign (avar, allocator, dlist);
6163 22 : allocator = avar;
6164 : }
6165 864 : if (!is_gimple_val (allocate_ptr))
6166 : {
6167 50 : tree apvar = create_tmp_var (TREE_TYPE (allocate_ptr));
6168 50 : gimplify_assign (apvar, allocate_ptr, dlist);
6169 50 : allocate_ptr = apvar;
6170 : }
6171 864 : tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
6172 864 : gimple *g
6173 864 : = gimple_build_call (f, 2, allocate_ptr, allocator);
6174 864 : gimple_seq_add_stmt (dlist, g);
6175 : }
6176 : break;
6177 :
6178 7809 : case OMP_CLAUSE_LINEAR:
6179 7809 : if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c))
6180 1269 : goto do_firstprivate;
6181 6540 : if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
6182 : x = NULL;
6183 : else
6184 3601 : x = build_outer_var_ref (var, ctx);
6185 6540 : goto do_private;
6186 :
6187 28731 : case OMP_CLAUSE_FIRSTPRIVATE:
6188 28731 : if (is_task_ctx (ctx))
6189 : {
6190 4631 : if ((omp_privatize_by_reference (var)
6191 146 : && !OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c))
6192 4631 : || is_variable_sized (var))
6193 160 : goto do_dtor;
6194 4471 : else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var,
6195 : ctx))
6196 4471 : || use_pointer_for_field (var, NULL))
6197 : {
6198 421 : x = build_receiver_ref (var, false, ctx);
6199 421 : if (ctx->allocate_map)
6200 34 : if (tree *allocatep = ctx->allocate_map->get (var))
6201 : {
6202 34 : allocator = *allocatep;
6203 34 : if (TREE_CODE (allocator) == TREE_LIST)
6204 4 : allocator = TREE_PURPOSE (allocator);
6205 34 : if (TREE_CODE (allocator) != INTEGER_CST)
6206 14 : allocator = build_outer_var_ref (allocator, ctx);
6207 34 : allocator = fold_convert (pointer_sized_int_node,
6208 : allocator);
6209 34 : allocate_ptr = unshare_expr (x);
6210 34 : x = build_simple_mem_ref (x);
6211 34 : TREE_THIS_NOTRAP (x) = 1;
6212 : }
6213 421 : SET_DECL_VALUE_EXPR (new_var, x);
6214 421 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
6215 421 : goto do_dtor;
6216 : }
6217 : }
6218 28150 : if (OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c)
6219 28150 : && omp_privatize_by_reference (var))
6220 : {
6221 0 : x = build_outer_var_ref (var, ctx);
6222 0 : gcc_assert (TREE_CODE (x) == MEM_REF
6223 : && integer_zerop (TREE_OPERAND (x, 1)));
6224 0 : x = TREE_OPERAND (x, 0);
6225 0 : x = lang_hooks.decls.omp_clause_copy_ctor
6226 0 : (c, unshare_expr (new_var), x);
6227 0 : gimplify_and_add (x, ilist);
6228 0 : goto do_dtor;
6229 : }
6230 29419 : do_firstprivate:
6231 29419 : lower_private_allocate (var, new_var, allocator, allocate_ptr,
6232 : ilist, ctx, false, NULL_TREE);
6233 29419 : x = build_outer_var_ref (var, ctx);
6234 29419 : if (is_simd)
6235 : {
6236 1063 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
6237 1063 : && gimple_omp_for_combined_into_p (ctx->stmt))
6238 : {
6239 668 : tree t = OMP_CLAUSE_LINEAR_STEP (c);
6240 668 : if (DECL_P (t))
6241 114 : t = build_outer_var_ref (t, ctx);
6242 668 : tree stept = TREE_TYPE (t);
6243 668 : tree ct = omp_find_clause (clauses,
6244 : OMP_CLAUSE__LOOPTEMP_);
6245 668 : gcc_assert (ct);
6246 668 : tree l = OMP_CLAUSE_DECL (ct);
6247 668 : tree n1 = fd->loop.n1;
6248 668 : tree step = fd->loop.step;
6249 668 : tree itype = TREE_TYPE (l);
6250 668 : if (POINTER_TYPE_P (itype))
6251 0 : itype = signed_type_for (itype);
6252 668 : l = fold_build2 (MINUS_EXPR, itype, l, n1);
6253 668 : if (TYPE_UNSIGNED (itype)
6254 668 : && fd->loop.cond_code == GT_EXPR)
6255 0 : l = fold_build2 (TRUNC_DIV_EXPR, itype,
6256 : fold_build1 (NEGATE_EXPR, itype, l),
6257 : fold_build1 (NEGATE_EXPR,
6258 : itype, step));
6259 : else
6260 668 : l = fold_build2 (TRUNC_DIV_EXPR, itype, l, step);
6261 668 : t = fold_build2 (MULT_EXPR, stept,
6262 : fold_convert (stept, l), t);
6263 :
6264 668 : if (OMP_CLAUSE_LINEAR_ARRAY (c))
6265 : {
6266 108 : if (omp_privatize_by_reference (var))
6267 : {
6268 72 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6269 72 : tree new_vard = TREE_OPERAND (new_var, 0);
6270 72 : gcc_assert (DECL_P (new_vard));
6271 72 : tree type = TREE_TYPE (TREE_TYPE (new_vard));
6272 72 : nx = TYPE_SIZE_UNIT (type);
6273 72 : if (TREE_CONSTANT (nx))
6274 : {
6275 24 : nx = create_tmp_var_raw (type,
6276 : get_name (var));
6277 24 : gimple_add_tmp_var (nx);
6278 24 : TREE_ADDRESSABLE (nx) = 1;
6279 24 : nx = build_fold_addr_expr_loc (clause_loc,
6280 : nx);
6281 24 : nx = fold_convert_loc (clause_loc,
6282 24 : TREE_TYPE (new_vard),
6283 : nx);
6284 24 : gimplify_assign (new_vard, nx, ilist);
6285 : }
6286 : }
6287 :
6288 108 : x = lang_hooks.decls.omp_clause_linear_ctor
6289 108 : (c, new_var, x, t);
6290 108 : gimplify_and_add (x, ilist);
6291 108 : goto do_dtor;
6292 : }
6293 :
6294 560 : if (POINTER_TYPE_P (TREE_TYPE (x)))
6295 11 : x = fold_build_pointer_plus (x, t);
6296 : else
6297 549 : x = fold_build2 (PLUS_EXPR, TREE_TYPE (x), x,
6298 : fold_convert (TREE_TYPE (x), t));
6299 : }
6300 :
6301 955 : if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LINEAR
6302 934 : || TREE_ADDRESSABLE (new_var)
6303 799 : || omp_privatize_by_reference (var))
6304 1217 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
6305 : ivar, lvar))
6306 : {
6307 152 : if (omp_privatize_by_reference (var))
6308 : {
6309 22 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6310 22 : tree new_vard = TREE_OPERAND (new_var, 0);
6311 22 : gcc_assert (DECL_P (new_vard));
6312 22 : SET_DECL_VALUE_EXPR (new_vard,
6313 : build_fold_addr_expr (lvar));
6314 22 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6315 : }
6316 152 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR)
6317 : {
6318 131 : tree iv = create_tmp_var (TREE_TYPE (new_var));
6319 131 : x = lang_hooks.decls.omp_clause_copy_ctor (c, iv, x);
6320 131 : gimplify_and_add (x, ilist);
6321 131 : gimple_stmt_iterator gsi
6322 131 : = gsi_start (*gimple_omp_body_ptr (ctx->stmt));
6323 131 : gassign *g
6324 131 : = gimple_build_assign (unshare_expr (lvar), iv);
6325 131 : gsi_insert_before_without_update (&gsi, g,
6326 : GSI_SAME_STMT);
6327 131 : tree t = OMP_CLAUSE_LINEAR_STEP (c);
6328 131 : enum tree_code code = PLUS_EXPR;
6329 131 : if (POINTER_TYPE_P (TREE_TYPE (new_var)))
6330 : code = POINTER_PLUS_EXPR;
6331 131 : g = gimple_build_assign (iv, code, iv, t);
6332 131 : gsi_insert_before_without_update (&gsi, g,
6333 : GSI_SAME_STMT);
6334 131 : break;
6335 : }
6336 21 : x = lang_hooks.decls.omp_clause_copy_ctor
6337 21 : (c, unshare_expr (ivar), x);
6338 21 : gimplify_and_add (x, &llist[0]);
6339 21 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6340 21 : if (x)
6341 21 : gimplify_and_add (x, &llist[1]);
6342 : break;
6343 : }
6344 803 : if (omp_privatize_by_reference (var))
6345 : {
6346 105 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6347 105 : tree new_vard = TREE_OPERAND (new_var, 0);
6348 105 : gcc_assert (DECL_P (new_vard));
6349 105 : tree type = TREE_TYPE (TREE_TYPE (new_vard));
6350 105 : nx = TYPE_SIZE_UNIT (type);
6351 105 : if (TREE_CONSTANT (nx))
6352 : {
6353 57 : nx = create_tmp_var_raw (type, get_name (var));
6354 57 : gimple_add_tmp_var (nx);
6355 57 : TREE_ADDRESSABLE (nx) = 1;
6356 57 : nx = build_fold_addr_expr_loc (clause_loc, nx);
6357 57 : nx = fold_convert_loc (clause_loc,
6358 57 : TREE_TYPE (new_vard), nx);
6359 57 : gimplify_assign (new_vard, nx, ilist);
6360 : }
6361 : }
6362 : }
6363 29159 : x = lang_hooks.decls.omp_clause_copy_ctor
6364 29159 : (c, unshare_expr (new_var), x);
6365 29155 : gimplify_and_add (x, ilist);
6366 29155 : goto do_dtor;
6367 :
6368 19358 : case OMP_CLAUSE__LOOPTEMP_:
6369 19358 : case OMP_CLAUSE__REDUCTEMP_:
6370 19358 : gcc_assert (is_taskreg_ctx (ctx));
6371 19358 : x = build_outer_var_ref (var, ctx);
6372 19358 : x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
6373 19358 : gimplify_and_add (x, ilist);
6374 19358 : break;
6375 :
6376 514 : case OMP_CLAUSE_COPYIN:
6377 514 : by_ref = use_pointer_for_field (var, NULL);
6378 514 : x = build_receiver_ref (var, by_ref, ctx);
6379 514 : x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x);
6380 514 : append_to_statement_list (x, ©in_seq);
6381 514 : copyin_by_ref |= by_ref;
6382 514 : break;
6383 :
6384 13089 : case OMP_CLAUSE_REDUCTION:
6385 13089 : case OMP_CLAUSE_IN_REDUCTION:
6386 : /* OpenACC reductions are initialized using the
6387 : GOACC_REDUCTION internal function. */
6388 13089 : if (is_gimple_omp_oacc (ctx->stmt))
6389 : break;
6390 9107 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
6391 : {
6392 1458 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
6393 1458 : gimple *tseq;
6394 1458 : tree ptype = TREE_TYPE (placeholder);
6395 1458 : if (cond)
6396 : {
6397 295 : x = error_mark_node;
6398 295 : if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)
6399 295 : && !task_reduction_needs_orig_p)
6400 28 : x = var;
6401 267 : else if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
6402 : {
6403 45 : tree pptype = build_pointer_type (ptype);
6404 45 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
6405 35 : x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
6406 35 : size_int (task_reduction_cnt_full
6407 : + task_reduction_cntorig - 1),
6408 : NULL_TREE, NULL_TREE);
6409 : else
6410 : {
6411 10 : unsigned int idx
6412 10 : = *ctx->task_reduction_map->get (c);
6413 10 : x = task_reduction_read (ilist, tskred_temp,
6414 10 : pptype, 7 + 3 * idx);
6415 : }
6416 45 : x = fold_convert (pptype, x);
6417 45 : x = build_simple_mem_ref (x);
6418 : }
6419 : }
6420 : else
6421 : {
6422 1163 : lower_private_allocate (var, new_var, allocator,
6423 : allocate_ptr, ilist, ctx, false,
6424 : NULL_TREE);
6425 1163 : x = build_outer_var_ref (var, ctx);
6426 :
6427 1163 : if (omp_privatize_by_reference (var)
6428 1163 : && !useless_type_conversion_p (ptype, TREE_TYPE (x)))
6429 49 : x = build_fold_addr_expr_loc (clause_loc, x);
6430 : }
6431 1458 : SET_DECL_VALUE_EXPR (placeholder, x);
6432 1458 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
6433 1458 : tree new_vard = new_var;
6434 1458 : if (omp_privatize_by_reference (var))
6435 : {
6436 203 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6437 203 : new_vard = TREE_OPERAND (new_var, 0);
6438 203 : gcc_assert (DECL_P (new_vard));
6439 : }
6440 1458 : tree rvar = NULL_TREE, *rvarp = NULL, rvar2 = NULL_TREE;
6441 1458 : if (is_simd
6442 311 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6443 1769 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6444 : rvarp = &rvar;
6445 1458 : if (is_simd
6446 1458 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
6447 : ivar, lvar, rvarp,
6448 : &rvar2))
6449 : {
6450 174 : if (new_vard == new_var)
6451 : {
6452 128 : gcc_assert (DECL_VALUE_EXPR (new_var) == lvar);
6453 128 : SET_DECL_VALUE_EXPR (new_var, ivar);
6454 : }
6455 : else
6456 : {
6457 46 : SET_DECL_VALUE_EXPR (new_vard,
6458 : build_fold_addr_expr (ivar));
6459 46 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6460 : }
6461 174 : x = lang_hooks.decls.omp_clause_default_ctor
6462 174 : (c, unshare_expr (ivar),
6463 : build_outer_var_ref (var, ctx));
6464 174 : if (rvarp && ctx->for_simd_scan_phase)
6465 : {
6466 16 : if (x)
6467 8 : gimplify_and_add (x, &llist[0]);
6468 16 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6469 16 : if (x)
6470 8 : gimplify_and_add (x, &llist[1]);
6471 469 : break;
6472 : }
6473 78 : else if (rvarp)
6474 : {
6475 78 : if (x)
6476 : {
6477 38 : gimplify_and_add (x, &llist[0]);
6478 :
6479 38 : tree ivar2 = unshare_expr (lvar);
6480 38 : TREE_OPERAND (ivar2, 1) = sctx.idx;
6481 38 : x = lang_hooks.decls.omp_clause_default_ctor
6482 38 : (c, ivar2, build_outer_var_ref (var, ctx));
6483 38 : gimplify_and_add (x, &llist[0]);
6484 :
6485 38 : if (rvar2)
6486 : {
6487 16 : x = lang_hooks.decls.omp_clause_default_ctor
6488 16 : (c, unshare_expr (rvar2),
6489 : build_outer_var_ref (var, ctx));
6490 16 : gimplify_and_add (x, &llist[0]);
6491 : }
6492 :
6493 : /* For types that need construction, add another
6494 : private var which will be default constructed
6495 : and optionally initialized with
6496 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT, as in the
6497 : loop we want to assign this value instead of
6498 : constructing and destructing it in each
6499 : iteration. */
6500 38 : tree nv = create_tmp_var_raw (TREE_TYPE (ivar));
6501 38 : gimple_add_tmp_var (nv);
6502 60 : ctx->cb.decl_map->put (TREE_OPERAND (rvar2
6503 : ? rvar2
6504 : : ivar, 0),
6505 : nv);
6506 38 : x = lang_hooks.decls.omp_clause_default_ctor
6507 38 : (c, nv, build_outer_var_ref (var, ctx));
6508 38 : gimplify_and_add (x, ilist);
6509 :
6510 38 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6511 : {
6512 19 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6513 19 : x = DECL_VALUE_EXPR (new_vard);
6514 19 : tree vexpr = nv;
6515 19 : if (new_vard != new_var)
6516 11 : vexpr = build_fold_addr_expr (nv);
6517 19 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
6518 19 : lower_omp (&tseq, ctx);
6519 19 : SET_DECL_VALUE_EXPR (new_vard, x);
6520 19 : gimple_seq_add_seq (ilist, tseq);
6521 19 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6522 : }
6523 :
6524 38 : x = lang_hooks.decls.omp_clause_dtor (c, nv);
6525 38 : if (x)
6526 38 : gimplify_and_add (x, dlist);
6527 : }
6528 :
6529 78 : tree ref = build_outer_var_ref (var, ctx);
6530 78 : x = unshare_expr (ivar);
6531 78 : x = lang_hooks.decls.omp_clause_assign_op (c, x,
6532 : ref);
6533 78 : gimplify_and_add (x, &llist[0]);
6534 :
6535 78 : ref = build_outer_var_ref (var, ctx);
6536 78 : x = lang_hooks.decls.omp_clause_assign_op (c, ref,
6537 : rvar);
6538 78 : gimplify_and_add (x, &llist[3]);
6539 :
6540 78 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6541 78 : if (new_vard == new_var)
6542 48 : SET_DECL_VALUE_EXPR (new_var, lvar);
6543 : else
6544 30 : SET_DECL_VALUE_EXPR (new_vard,
6545 : build_fold_addr_expr (lvar));
6546 :
6547 78 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6548 78 : if (x)
6549 38 : gimplify_and_add (x, &llist[1]);
6550 :
6551 78 : tree ivar2 = unshare_expr (lvar);
6552 78 : TREE_OPERAND (ivar2, 1) = sctx.idx;
6553 78 : x = lang_hooks.decls.omp_clause_dtor (c, ivar2);
6554 78 : if (x)
6555 38 : gimplify_and_add (x, &llist[1]);
6556 :
6557 78 : if (rvar2)
6558 : {
6559 36 : x = lang_hooks.decls.omp_clause_dtor (c, rvar2);
6560 36 : if (x)
6561 16 : gimplify_and_add (x, &llist[1]);
6562 : }
6563 : break;
6564 : }
6565 80 : if (x)
6566 34 : gimplify_and_add (x, &llist[0]);
6567 80 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6568 : {
6569 76 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6570 76 : lower_omp (&tseq, ctx);
6571 76 : gimple_seq_add_seq (&llist[0], tseq);
6572 : }
6573 80 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6574 80 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
6575 80 : lower_omp (&tseq, ctx);
6576 80 : gimple_seq_add_seq (&llist[1], tseq);
6577 80 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
6578 80 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6579 80 : if (new_vard == new_var)
6580 70 : SET_DECL_VALUE_EXPR (new_var, lvar);
6581 : else
6582 10 : SET_DECL_VALUE_EXPR (new_vard,
6583 : build_fold_addr_expr (lvar));
6584 80 : x = lang_hooks.decls.omp_clause_dtor (c, ivar);
6585 80 : if (x)
6586 38 : gimplify_and_add (x, &llist[1]);
6587 : break;
6588 : }
6589 : /* If this is a reference to constant size reduction var
6590 : with placeholder, we haven't emitted the initializer
6591 : for it because it is undesirable if SIMD arrays are used.
6592 : But if they aren't used, we need to emit the deferred
6593 : initialization now. */
6594 1284 : else if (omp_privatize_by_reference (var) && is_simd)
6595 46 : handle_simd_reference (clause_loc, new_vard, ilist);
6596 :
6597 1284 : tree lab2 = NULL_TREE;
6598 1284 : if (cond)
6599 : {
6600 295 : gimple *g;
6601 295 : if (!is_parallel_ctx (ctx))
6602 : {
6603 278 : tree condv = create_tmp_var (boolean_type_node);
6604 278 : tree m = build_simple_mem_ref (cond);
6605 278 : g = gimple_build_assign (condv, m);
6606 278 : gimple_seq_add_stmt (ilist, g);
6607 278 : tree lab1
6608 278 : = create_artificial_label (UNKNOWN_LOCATION);
6609 278 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
6610 278 : g = gimple_build_cond (NE_EXPR, condv,
6611 : boolean_false_node,
6612 : lab2, lab1);
6613 278 : gimple_seq_add_stmt (ilist, g);
6614 278 : gimple_seq_add_stmt (ilist,
6615 278 : gimple_build_label (lab1));
6616 : }
6617 295 : g = gimple_build_assign (build_simple_mem_ref (cond),
6618 : boolean_true_node);
6619 295 : gimple_seq_add_stmt (ilist, g);
6620 : }
6621 295 : x = lang_hooks.decls.omp_clause_default_ctor
6622 1284 : (c, unshare_expr (new_var),
6623 : cond ? NULL_TREE
6624 989 : : build_outer_var_ref (var, ctx));
6625 1284 : if (x)
6626 286 : gimplify_and_add (x, ilist);
6627 :
6628 1284 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6629 1284 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6630 : {
6631 158 : if (ctx->for_simd_scan_phase)
6632 989 : goto do_dtor;
6633 142 : if (x || (!is_simd
6634 32 : && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)))
6635 : {
6636 70 : tree nv = create_tmp_var_raw (TREE_TYPE (new_var));
6637 70 : gimple_add_tmp_var (nv);
6638 70 : ctx->cb.decl_map->put (new_vard, nv);
6639 70 : x = lang_hooks.decls.omp_clause_default_ctor
6640 70 : (c, nv, build_outer_var_ref (var, ctx));
6641 70 : if (x)
6642 70 : gimplify_and_add (x, ilist);
6643 70 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6644 : {
6645 35 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6646 35 : tree vexpr = nv;
6647 35 : if (new_vard != new_var)
6648 19 : vexpr = build_fold_addr_expr (nv);
6649 35 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
6650 35 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6651 35 : lower_omp (&tseq, ctx);
6652 35 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
6653 35 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
6654 35 : gimple_seq_add_seq (ilist, tseq);
6655 : }
6656 70 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6657 70 : if (is_simd && ctx->scan_exclusive)
6658 : {
6659 16 : tree nv2
6660 16 : = create_tmp_var_raw (TREE_TYPE (new_var));
6661 16 : gimple_add_tmp_var (nv2);
6662 16 : ctx->cb.decl_map->put (nv, nv2);
6663 16 : x = lang_hooks.decls.omp_clause_default_ctor
6664 16 : (c, nv2, build_outer_var_ref (var, ctx));
6665 16 : gimplify_and_add (x, ilist);
6666 16 : x = lang_hooks.decls.omp_clause_dtor (c, nv2);
6667 16 : if (x)
6668 16 : gimplify_and_add (x, dlist);
6669 : }
6670 70 : x = lang_hooks.decls.omp_clause_dtor (c, nv);
6671 70 : if (x)
6672 70 : gimplify_and_add (x, dlist);
6673 : }
6674 72 : else if (is_simd
6675 40 : && ctx->scan_exclusive
6676 92 : && TREE_ADDRESSABLE (TREE_TYPE (new_var)))
6677 : {
6678 0 : tree nv2 = create_tmp_var_raw (TREE_TYPE (new_var));
6679 0 : gimple_add_tmp_var (nv2);
6680 0 : ctx->cb.decl_map->put (new_vard, nv2);
6681 0 : x = lang_hooks.decls.omp_clause_dtor (c, nv2);
6682 0 : if (x)
6683 0 : gimplify_and_add (x, dlist);
6684 : }
6685 142 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6686 142 : goto do_dtor;
6687 : }
6688 :
6689 1126 : if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
6690 : {
6691 1078 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
6692 1078 : if (c_kind == OMP_CLAUSE_IN_REDUCTION
6693 1078 : && is_omp_target (ctx->stmt))
6694 : {
6695 12 : tree d = maybe_lookup_decl_in_outer_ctx (var, ctx);
6696 12 : tree oldv = NULL_TREE;
6697 12 : gcc_assert (d);
6698 12 : if (DECL_HAS_VALUE_EXPR_P (d))
6699 6 : oldv = DECL_VALUE_EXPR (d);
6700 12 : SET_DECL_VALUE_EXPR (d, new_vard);
6701 12 : DECL_HAS_VALUE_EXPR_P (d) = 1;
6702 12 : lower_omp (&tseq, ctx);
6703 12 : if (oldv)
6704 6 : SET_DECL_VALUE_EXPR (d, oldv);
6705 : else
6706 : {
6707 6 : SET_DECL_VALUE_EXPR (d, NULL_TREE);
6708 6 : DECL_HAS_VALUE_EXPR_P (d) = 0;
6709 : }
6710 : }
6711 : else
6712 1066 : lower_omp (&tseq, ctx);
6713 1078 : gimple_seq_add_seq (ilist, tseq);
6714 : }
6715 1126 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
6716 1126 : if (is_simd)
6717 : {
6718 43 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
6719 43 : lower_omp (&tseq, ctx);
6720 43 : gimple_seq_add_seq (dlist, tseq);
6721 43 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
6722 : }
6723 1126 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
6724 1126 : if (cond)
6725 : {
6726 295 : if (lab2)
6727 278 : gimple_seq_add_stmt (ilist, gimple_build_label (lab2));
6728 : break;
6729 : }
6730 831 : goto do_dtor;
6731 : }
6732 : else
6733 : {
6734 7649 : x = omp_reduction_init (c, TREE_TYPE (new_var));
6735 7649 : gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE);
6736 7649 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
6737 :
6738 7649 : if (cond)
6739 : {
6740 1447 : gimple *g;
6741 1447 : tree lab2 = NULL_TREE;
6742 : /* GOMP_taskgroup_reduction_register memsets the whole
6743 : array to zero. If the initializer is zero, we don't
6744 : need to initialize it again, just mark it as ever
6745 : used unconditionally, i.e. cond = true. */
6746 1447 : if (initializer_zerop (x))
6747 : {
6748 1266 : g = gimple_build_assign (build_simple_mem_ref (cond),
6749 : boolean_true_node);
6750 1266 : gimple_seq_add_stmt (ilist, g);
6751 3243 : break;
6752 : }
6753 :
6754 : /* Otherwise, emit
6755 : if (!cond) { cond = true; new_var = x; } */
6756 181 : if (!is_parallel_ctx (ctx))
6757 : {
6758 177 : tree condv = create_tmp_var (boolean_type_node);
6759 177 : tree m = build_simple_mem_ref (cond);
6760 177 : g = gimple_build_assign (condv, m);
6761 177 : gimple_seq_add_stmt (ilist, g);
6762 177 : tree lab1
6763 177 : = create_artificial_label (UNKNOWN_LOCATION);
6764 177 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
6765 177 : g = gimple_build_cond (NE_EXPR, condv,
6766 : boolean_false_node,
6767 : lab2, lab1);
6768 177 : gimple_seq_add_stmt (ilist, g);
6769 177 : gimple_seq_add_stmt (ilist,
6770 177 : gimple_build_label (lab1));
6771 : }
6772 181 : g = gimple_build_assign (build_simple_mem_ref (cond),
6773 : boolean_true_node);
6774 181 : gimple_seq_add_stmt (ilist, g);
6775 181 : gimplify_assign (new_var, x, ilist);
6776 181 : if (lab2)
6777 177 : gimple_seq_add_stmt (ilist, gimple_build_label (lab2));
6778 : break;
6779 : }
6780 :
6781 : /* reduction(-:var) sums up the partial results, so it
6782 : acts identically to reduction(+:var). */
6783 6202 : if (code == MINUS_EXPR)
6784 33 : code = PLUS_EXPR;
6785 :
6786 6202 : bool is_truth_op
6787 6202 : = (code == TRUTH_ANDIF_EXPR || code == TRUTH_ORIF_EXPR);
6788 6202 : tree new_vard = new_var;
6789 6202 : if (is_simd && omp_privatize_by_reference (var))
6790 : {
6791 52 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
6792 52 : new_vard = TREE_OPERAND (new_var, 0);
6793 52 : gcc_assert (DECL_P (new_vard));
6794 : }
6795 6202 : tree rvar = NULL_TREE, *rvarp = NULL, rvar2 = NULL_TREE;
6796 6202 : if (is_simd
6797 2087 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6798 8289 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6799 : rvarp = &rvar;
6800 6202 : if (is_simd
6801 6202 : && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
6802 : ivar, lvar, rvarp,
6803 : &rvar2))
6804 : {
6805 867 : if (new_vard != new_var)
6806 : {
6807 34 : SET_DECL_VALUE_EXPR (new_vard,
6808 : build_fold_addr_expr (lvar));
6809 34 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
6810 : }
6811 :
6812 867 : tree ref = build_outer_var_ref (var, ctx);
6813 :
6814 867 : if (rvarp)
6815 : {
6816 193 : if (ctx->for_simd_scan_phase)
6817 : break;
6818 154 : gimplify_assign (ivar, ref, &llist[0]);
6819 154 : ref = build_outer_var_ref (var, ctx);
6820 154 : gimplify_assign (ref, rvar, &llist[3]);
6821 154 : break;
6822 : }
6823 :
6824 674 : gimplify_assign (unshare_expr (ivar), x, &llist[0]);
6825 :
6826 674 : if (sctx.is_simt)
6827 : {
6828 0 : if (!simt_lane)
6829 0 : simt_lane = create_tmp_var (unsigned_type_node);
6830 0 : x = build_call_expr_internal_loc
6831 0 : (UNKNOWN_LOCATION, IFN_GOMP_SIMT_XCHG_BFLY,
6832 0 : TREE_TYPE (ivar), 2, ivar, simt_lane);
6833 : /* Make sure x is evaluated unconditionally. */
6834 0 : tree bfly_var = create_tmp_var (TREE_TYPE (ivar));
6835 0 : gimplify_assign (bfly_var, x, &llist[2]);
6836 0 : x = build2 (code, TREE_TYPE (ivar), ivar, bfly_var);
6837 0 : gimplify_assign (ivar, x, &llist[2]);
6838 : }
6839 674 : tree ivar2 = ivar;
6840 674 : tree ref2 = ref;
6841 674 : if (is_truth_op)
6842 : {
6843 99 : tree zero = build_zero_cst (TREE_TYPE (ivar));
6844 99 : ivar2 = fold_build2_loc (clause_loc, NE_EXPR,
6845 : boolean_type_node, ivar,
6846 : zero);
6847 99 : ref2 = fold_build2_loc (clause_loc, NE_EXPR,
6848 : boolean_type_node, ref,
6849 : zero);
6850 : }
6851 674 : x = build2 (code, TREE_TYPE (ref), ref2, ivar2);
6852 674 : if (is_truth_op)
6853 99 : x = fold_convert (TREE_TYPE (ref), x);
6854 674 : ref = build_outer_var_ref (var, ctx);
6855 674 : gimplify_assign (ref, x, &llist[1]);
6856 :
6857 : }
6858 : else
6859 : {
6860 5335 : lower_private_allocate (var, new_var, allocator,
6861 : allocate_ptr, ilist, ctx,
6862 : false, NULL_TREE);
6863 5335 : if (omp_privatize_by_reference (var) && is_simd)
6864 18 : handle_simd_reference (clause_loc, new_vard, ilist);
6865 5335 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
6866 5335 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
6867 : break;
6868 4998 : gimplify_assign (new_var, x, ilist);
6869 4998 : if (is_simd)
6870 : {
6871 1028 : tree ref = build_outer_var_ref (var, ctx);
6872 1028 : tree new_var2 = new_var;
6873 1028 : tree ref2 = ref;
6874 1028 : if (is_truth_op)
6875 : {
6876 24 : tree zero = build_zero_cst (TREE_TYPE (new_var));
6877 24 : new_var2
6878 24 : = fold_build2_loc (clause_loc, NE_EXPR,
6879 : boolean_type_node, new_var,
6880 : zero);
6881 24 : ref2 = fold_build2_loc (clause_loc, NE_EXPR,
6882 : boolean_type_node, ref,
6883 : zero);
6884 : }
6885 1028 : x = build2 (code, TREE_TYPE (ref2), ref2, new_var2);
6886 1028 : if (is_truth_op)
6887 24 : x = fold_convert (TREE_TYPE (new_var), x);
6888 1028 : ref = build_outer_var_ref (var, ctx);
6889 1028 : gimplify_assign (ref, x, dlist);
6890 : }
6891 4998 : if (allocator)
6892 61 : goto do_dtor;
6893 : }
6894 : }
6895 5611 : break;
6896 :
6897 0 : default:
6898 0 : gcc_unreachable ();
6899 : }
6900 : }
6901 : }
6902 77360 : if (tskred_avar)
6903 : {
6904 871 : tree clobber = build_clobber (TREE_TYPE (tskred_avar));
6905 871 : gimple_seq_add_stmt (ilist, gimple_build_assign (tskred_avar, clobber));
6906 : }
6907 :
6908 77360 : if (known_eq (sctx.max_vf, 1U))
6909 : {
6910 1779 : sctx.is_simt = false;
6911 1779 : if (ctx->lastprivate_conditional_map)
6912 : {
6913 52 : if (gimple_omp_for_combined_into_p (ctx->stmt))
6914 : {
6915 : /* Signal to lower_omp_1 that it should use parent context. */
6916 43 : ctx->combined_into_simd_safelen1 = true;
6917 370 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
6918 327 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
6919 327 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
6920 : {
6921 57 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
6922 57 : omp_context *outer = ctx->outer;
6923 57 : if (gimple_code (outer->stmt) == GIMPLE_OMP_SCAN)
6924 6 : outer = outer->outer;
6925 57 : tree *v = ctx->lastprivate_conditional_map->get (o);
6926 57 : tree po = lookup_decl (OMP_CLAUSE_DECL (c), outer);
6927 57 : tree *pv = outer->lastprivate_conditional_map->get (po);
6928 57 : *v = *pv;
6929 : }
6930 : }
6931 : else
6932 : {
6933 : /* When not vectorized, treat lastprivate(conditional:) like
6934 : normal lastprivate, as there will be just one simd lane
6935 : writing the privatized variable. */
6936 9 : delete ctx->lastprivate_conditional_map;
6937 9 : ctx->lastprivate_conditional_map = NULL;
6938 : }
6939 : }
6940 : }
6941 :
6942 77360 : if (nonconst_simd_if)
6943 : {
6944 625 : if (sctx.lane == NULL_TREE)
6945 : {
6946 570 : sctx.idx = create_tmp_var (unsigned_type_node);
6947 570 : sctx.lane = create_tmp_var (unsigned_type_node);
6948 : }
6949 : /* FIXME: For now. */
6950 625 : sctx.is_simt = false;
6951 : }
6952 :
6953 77360 : if (sctx.lane || sctx.is_simt)
6954 : {
6955 4003 : uid = create_tmp_var (ptr_type_node, "simduid");
6956 : /* Don't want uninit warnings on simduid, it is always uninitialized,
6957 : but we use it not for the value, but for the DECL_UID only. */
6958 4003 : suppress_warning (uid, OPT_Wuninitialized);
6959 4003 : c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SIMDUID_);
6960 4003 : OMP_CLAUSE__SIMDUID__DECL (c) = uid;
6961 4003 : OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
6962 4003 : gimple_omp_for_set_clauses (ctx->stmt, c);
6963 : }
6964 : /* Emit calls denoting privatized variables and initializing a pointer to
6965 : structure that holds private variables as fields after ompdevlow pass. */
6966 77360 : if (sctx.is_simt)
6967 : {
6968 0 : sctx.simt_eargs[0] = uid;
6969 0 : gimple *g
6970 0 : = gimple_build_call_internal_vec (IFN_GOMP_SIMT_ENTER, sctx.simt_eargs);
6971 0 : gimple_call_set_lhs (g, uid);
6972 0 : gimple_seq_add_stmt (ilist, g);
6973 0 : sctx.simt_eargs.release ();
6974 :
6975 0 : simtrec = create_tmp_var (ptr_type_node, ".omp_simt");
6976 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_ENTER_ALLOC, 1, uid);
6977 0 : gimple_call_set_lhs (g, simtrec);
6978 0 : gimple_seq_add_stmt (ilist, g);
6979 : }
6980 77360 : if (sctx.lane)
6981 : {
6982 7381 : gimple *g = gimple_build_call_internal (IFN_GOMP_SIMD_LANE,
6983 : 2 + (nonconst_simd_if != NULL),
6984 : uid, integer_zero_node,
6985 : nonconst_simd_if);
6986 4003 : gimple_call_set_lhs (g, sctx.lane);
6987 4003 : gimple_stmt_iterator gsi = gsi_start (*gimple_omp_body_ptr (ctx->stmt));
6988 4003 : gsi_insert_before_without_update (&gsi, g, GSI_SAME_STMT);
6989 4003 : g = gimple_build_assign (sctx.lane, INTEGER_CST,
6990 : build_int_cst (unsigned_type_node, 0));
6991 4003 : gimple_seq_add_stmt (ilist, g);
6992 4003 : if (sctx.lastlane)
6993 : {
6994 184 : g = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
6995 : 2, uid, sctx.lane);
6996 184 : gimple_call_set_lhs (g, sctx.lastlane);
6997 184 : gimple_seq_add_stmt (dlist, g);
6998 184 : gimple_seq_add_seq (dlist, llist[3]);
6999 : }
7000 : /* Emit reductions across SIMT lanes in log_2(simt_vf) steps. */
7001 4003 : if (llist[2])
7002 : {
7003 0 : tree simt_vf = create_tmp_var (unsigned_type_node);
7004 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_VF, 0);
7005 0 : gimple_call_set_lhs (g, simt_vf);
7006 0 : gimple_seq_add_stmt (dlist, g);
7007 :
7008 0 : tree t = build_int_cst (unsigned_type_node, 1);
7009 0 : g = gimple_build_assign (simt_lane, INTEGER_CST, t);
7010 0 : gimple_seq_add_stmt (dlist, g);
7011 :
7012 0 : t = build_int_cst (unsigned_type_node, 0);
7013 0 : g = gimple_build_assign (sctx.idx, INTEGER_CST, t);
7014 0 : gimple_seq_add_stmt (dlist, g);
7015 :
7016 0 : tree body = create_artificial_label (UNKNOWN_LOCATION);
7017 0 : tree header = create_artificial_label (UNKNOWN_LOCATION);
7018 0 : tree end = create_artificial_label (UNKNOWN_LOCATION);
7019 0 : gimple_seq_add_stmt (dlist, gimple_build_goto (header));
7020 0 : gimple_seq_add_stmt (dlist, gimple_build_label (body));
7021 :
7022 0 : gimple_seq_add_seq (dlist, llist[2]);
7023 :
7024 0 : g = gimple_build_assign (simt_lane, LSHIFT_EXPR, simt_lane, integer_one_node);
7025 0 : gimple_seq_add_stmt (dlist, g);
7026 :
7027 0 : gimple_seq_add_stmt (dlist, gimple_build_label (header));
7028 0 : g = gimple_build_cond (LT_EXPR, simt_lane, simt_vf, body, end);
7029 0 : gimple_seq_add_stmt (dlist, g);
7030 :
7031 0 : gimple_seq_add_stmt (dlist, gimple_build_label (end));
7032 : }
7033 12009 : for (int i = 0; i < 2; i++)
7034 8006 : if (llist[i])
7035 : {
7036 1716 : tree vf = create_tmp_var (unsigned_type_node);
7037 1716 : g = gimple_build_call_internal (IFN_GOMP_SIMD_VF, 1, uid);
7038 1716 : gimple_call_set_lhs (g, vf);
7039 1716 : gimple_seq *seq = i == 0 ? ilist : dlist;
7040 1716 : gimple_seq_add_stmt (seq, g);
7041 1716 : tree t = build_int_cst (unsigned_type_node, 0);
7042 1716 : g = gimple_build_assign (sctx.idx, INTEGER_CST, t);
7043 1716 : gimple_seq_add_stmt (seq, g);
7044 1716 : tree body = create_artificial_label (UNKNOWN_LOCATION);
7045 1716 : tree header = create_artificial_label (UNKNOWN_LOCATION);
7046 1716 : tree end = create_artificial_label (UNKNOWN_LOCATION);
7047 1716 : gimple_seq_add_stmt (seq, gimple_build_goto (header));
7048 1716 : gimple_seq_add_stmt (seq, gimple_build_label (body));
7049 1716 : gimple_seq_add_seq (seq, llist[i]);
7050 1716 : t = build_int_cst (unsigned_type_node, 1);
7051 1716 : g = gimple_build_assign (sctx.idx, PLUS_EXPR, sctx.idx, t);
7052 1716 : gimple_seq_add_stmt (seq, g);
7053 1716 : gimple_seq_add_stmt (seq, gimple_build_label (header));
7054 1716 : g = gimple_build_cond (LT_EXPR, sctx.idx, vf, body, end);
7055 1716 : gimple_seq_add_stmt (seq, g);
7056 1716 : gimple_seq_add_stmt (seq, gimple_build_label (end));
7057 : }
7058 : }
7059 77360 : if (sctx.is_simt)
7060 : {
7061 0 : gimple_seq_add_seq (dlist, sctx.simt_dlist);
7062 0 : gimple *g
7063 0 : = gimple_build_call_internal (IFN_GOMP_SIMT_EXIT, 1, simtrec);
7064 0 : gimple_seq_add_stmt (dlist, g);
7065 : }
7066 :
7067 : /* The copyin sequence is not to be executed by the main thread, since
7068 : that would result in self-copies. Perhaps not visible to scalars,
7069 : but it certainly is to C++ operator=. */
7070 77360 : if (copyin_seq)
7071 : {
7072 440 : x = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM),
7073 : 0);
7074 440 : x = build2 (NE_EXPR, boolean_type_node, x,
7075 440 : build_int_cst (TREE_TYPE (x), 0));
7076 440 : x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL);
7077 440 : gimplify_and_add (x, ilist);
7078 : }
7079 :
7080 : /* If any copyin variable is passed by reference, we must ensure the
7081 : master thread doesn't modify it before it is copied over in all
7082 : threads. Similarly for variables in both firstprivate and
7083 : lastprivate clauses we need to ensure the lastprivate copying
7084 : happens after firstprivate copying in all threads. And similarly
7085 : for UDRs if initializer expression refers to omp_orig. */
7086 77360 : if (copyin_by_ref || lastprivate_firstprivate
7087 75420 : || (reduction_omp_orig_ref
7088 101 : && !ctx->scan_inclusive
7089 101 : && !ctx->scan_exclusive))
7090 : {
7091 : /* Don't add any barrier for #pragma omp simd or
7092 : #pragma omp distribute. */
7093 2041 : if (!is_task_ctx (ctx)
7094 2041 : && (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
7095 1486 : || gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_FOR))
7096 : {
7097 904 : int barrier_kind;
7098 904 : switch (gimple_code (ctx->stmt))
7099 : {
7100 : case GIMPLE_OMP_SECTIONS:
7101 : case GIMPLE_OMP_SCOPE:
7102 : case GIMPLE_OMP_FOR:
7103 : barrier_kind = GOMP_BARRIER_IMPLICIT_WORKSHARE;
7104 : break;
7105 540 : case GIMPLE_OMP_PARALLEL:
7106 540 : case GIMPLE_OMP_TEAMS:
7107 540 : barrier_kind = GOMP_BARRIER_IMPLICIT_PARALLEL;
7108 540 : break;
7109 0 : default:
7110 0 : gcc_unreachable ();
7111 : }
7112 904 : gimple_seq_add_stmt (ilist,
7113 : omp_build_barrier (NULL_TREE, barrier_kind));
7114 : }
7115 : }
7116 :
7117 : /* If max_vf is non-zero, then we can use only a vectorization factor
7118 : up to the max_vf we chose. So stick it into the safelen clause. */
7119 77360 : if (maybe_ne (sctx.max_vf, 0U))
7120 : {
7121 5212 : tree c = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
7122 : OMP_CLAUSE_SAFELEN);
7123 5212 : poly_uint64 safe_len;
7124 5212 : if (c == NULL_TREE
7125 5212 : || (poly_int_tree_p (OMP_CLAUSE_SAFELEN_EXPR (c), &safe_len)
7126 613 : && maybe_gt (safe_len, sctx.max_vf)))
7127 : {
7128 5122 : c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_SAFELEN);
7129 5122 : OMP_CLAUSE_SAFELEN_EXPR (c) = build_int_cst (integer_type_node,
7130 5122 : sctx.max_vf);
7131 5122 : OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
7132 5122 : gimple_omp_for_set_clauses (ctx->stmt, c);
7133 : }
7134 : }
7135 77360 : }
7136 :
7137 : /* Create temporary variables for lastprivate(conditional:) implementation
7138 : in context CTX with CLAUSES. */
7139 :
7140 : static void
7141 47352 : lower_lastprivate_conditional_clauses (tree *clauses, omp_context *ctx)
7142 : {
7143 47352 : tree iter_type = NULL_TREE;
7144 47352 : tree cond_ptr = NULL_TREE;
7145 47352 : tree iter_var = NULL_TREE;
7146 47352 : bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
7147 47352 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD);
7148 47352 : tree next = *clauses;
7149 227306 : for (tree c = *clauses; c; c = OMP_CLAUSE_CHAIN (c))
7150 179954 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7151 179954 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
7152 : {
7153 386 : if (is_simd)
7154 : {
7155 107 : tree cc = omp_find_clause (next, OMP_CLAUSE__CONDTEMP_);
7156 107 : gcc_assert (cc);
7157 107 : if (iter_type == NULL_TREE)
7158 : {
7159 77 : iter_type = TREE_TYPE (OMP_CLAUSE_DECL (cc));
7160 77 : iter_var = create_tmp_var_raw (iter_type);
7161 77 : DECL_CONTEXT (iter_var) = current_function_decl;
7162 77 : DECL_SEEN_IN_BIND_EXPR_P (iter_var) = 1;
7163 77 : DECL_CHAIN (iter_var) = ctx->block_vars;
7164 77 : ctx->block_vars = iter_var;
7165 77 : tree c3
7166 77 : = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__CONDTEMP_);
7167 77 : OMP_CLAUSE__CONDTEMP__ITER (c3) = 1;
7168 77 : OMP_CLAUSE_DECL (c3) = iter_var;
7169 77 : OMP_CLAUSE_CHAIN (c3) = *clauses;
7170 77 : *clauses = c3;
7171 77 : ctx->lastprivate_conditional_map = new hash_map<tree, tree>;
7172 : }
7173 107 : next = OMP_CLAUSE_CHAIN (cc);
7174 107 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
7175 107 : tree v = lookup_decl (OMP_CLAUSE_DECL (cc), ctx);
7176 107 : ctx->lastprivate_conditional_map->put (o, v);
7177 107 : continue;
7178 107 : }
7179 279 : if (iter_type == NULL)
7180 : {
7181 205 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR)
7182 : {
7183 193 : struct omp_for_data fd;
7184 193 : omp_extract_for_data (as_a <gomp_for *> (ctx->stmt), &fd,
7185 : NULL);
7186 193 : iter_type = unsigned_type_for (fd.iter_type);
7187 : }
7188 12 : else if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
7189 12 : iter_type = unsigned_type_node;
7190 205 : tree c2 = omp_find_clause (*clauses, OMP_CLAUSE__CONDTEMP_);
7191 205 : if (c2)
7192 : {
7193 103 : cond_ptr
7194 103 : = lookup_decl_in_outer_ctx (OMP_CLAUSE_DECL (c2), ctx);
7195 103 : OMP_CLAUSE_DECL (c2) = cond_ptr;
7196 : }
7197 : else
7198 : {
7199 102 : cond_ptr = create_tmp_var_raw (build_pointer_type (iter_type));
7200 102 : DECL_CONTEXT (cond_ptr) = current_function_decl;
7201 102 : DECL_SEEN_IN_BIND_EXPR_P (cond_ptr) = 1;
7202 102 : DECL_CHAIN (cond_ptr) = ctx->block_vars;
7203 102 : ctx->block_vars = cond_ptr;
7204 102 : c2 = build_omp_clause (UNKNOWN_LOCATION,
7205 : OMP_CLAUSE__CONDTEMP_);
7206 102 : OMP_CLAUSE_DECL (c2) = cond_ptr;
7207 102 : OMP_CLAUSE_CHAIN (c2) = *clauses;
7208 102 : *clauses = c2;
7209 : }
7210 205 : iter_var = create_tmp_var_raw (iter_type);
7211 205 : DECL_CONTEXT (iter_var) = current_function_decl;
7212 205 : DECL_SEEN_IN_BIND_EXPR_P (iter_var) = 1;
7213 205 : DECL_CHAIN (iter_var) = ctx->block_vars;
7214 205 : ctx->block_vars = iter_var;
7215 205 : tree c3
7216 205 : = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__CONDTEMP_);
7217 205 : OMP_CLAUSE__CONDTEMP__ITER (c3) = 1;
7218 205 : OMP_CLAUSE_DECL (c3) = iter_var;
7219 205 : OMP_CLAUSE_CHAIN (c3) = OMP_CLAUSE_CHAIN (c2);
7220 205 : OMP_CLAUSE_CHAIN (c2) = c3;
7221 205 : ctx->lastprivate_conditional_map = new hash_map<tree, tree>;
7222 : }
7223 279 : tree v = create_tmp_var_raw (iter_type);
7224 279 : DECL_CONTEXT (v) = current_function_decl;
7225 279 : DECL_SEEN_IN_BIND_EXPR_P (v) = 1;
7226 279 : DECL_CHAIN (v) = ctx->block_vars;
7227 279 : ctx->block_vars = v;
7228 279 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
7229 279 : ctx->lastprivate_conditional_map->put (o, v);
7230 : }
7231 47352 : }
7232 :
7233 :
7234 : /* Generate code to implement the LASTPRIVATE clauses. This is used for
7235 : both parallel and workshare constructs. PREDICATE may be NULL if it's
7236 : always true. BODY_P is the sequence to insert early initialization
7237 : if needed, STMT_LIST is where the non-conditional lastprivate handling
7238 : goes into and CSTMT_LIST is a sequence that needs to be run in a critical
7239 : section. */
7240 :
7241 : static void
7242 47352 : lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *body_p,
7243 : gimple_seq *stmt_list, gimple_seq *cstmt_list,
7244 : omp_context *ctx)
7245 : {
7246 47352 : tree x, c, label = NULL, orig_clauses = clauses;
7247 47352 : bool par_clauses = false;
7248 47352 : tree simduid = NULL, lastlane = NULL, simtcond = NULL, simtlast = NULL;
7249 47352 : unsigned HOST_WIDE_INT conditional_off = 0;
7250 47352 : gimple_seq post_stmt_list = NULL;
7251 :
7252 : /* Early exit if there are no lastprivate or linear clauses. */
7253 196544 : for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
7254 161684 : if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LASTPRIVATE
7255 161684 : || (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LINEAR
7256 6766 : && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses)))
7257 : break;
7258 47352 : if (clauses == NULL)
7259 : {
7260 : /* If this was a workshare clause, see if it had been combined
7261 : with its parallel. In that case, look for the clauses on the
7262 : parallel statement itself. */
7263 34860 : if (is_parallel_ctx (ctx))
7264 30857 : return;
7265 :
7266 34860 : ctx = ctx->outer;
7267 34860 : if (ctx == NULL || !is_parallel_ctx (ctx))
7268 : return;
7269 :
7270 12271 : clauses = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
7271 : OMP_CLAUSE_LASTPRIVATE);
7272 12271 : if (clauses == NULL)
7273 : return;
7274 : par_clauses = true;
7275 : }
7276 :
7277 16495 : bool maybe_simt = false;
7278 16495 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
7279 16495 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
7280 : {
7281 7427 : maybe_simt = omp_find_clause (orig_clauses, OMP_CLAUSE__SIMT_);
7282 7427 : simduid = omp_find_clause (orig_clauses, OMP_CLAUSE__SIMDUID_);
7283 7427 : if (simduid)
7284 3551 : simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
7285 : }
7286 :
7287 16495 : if (predicate)
7288 : {
7289 16372 : gcond *stmt;
7290 16372 : tree label_true, arm1, arm2;
7291 16372 : enum tree_code pred_code = TREE_CODE (predicate);
7292 :
7293 16372 : label = create_artificial_label (UNKNOWN_LOCATION);
7294 16372 : label_true = create_artificial_label (UNKNOWN_LOCATION);
7295 16372 : if (TREE_CODE_CLASS (pred_code) == tcc_comparison)
7296 : {
7297 16372 : arm1 = TREE_OPERAND (predicate, 0);
7298 16372 : arm2 = TREE_OPERAND (predicate, 1);
7299 16372 : gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
7300 16372 : gimplify_expr (&arm2, stmt_list, NULL, is_gimple_val, fb_rvalue);
7301 : }
7302 : else
7303 : {
7304 0 : arm1 = predicate;
7305 0 : gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
7306 0 : arm2 = boolean_false_node;
7307 0 : pred_code = NE_EXPR;
7308 : }
7309 16372 : if (maybe_simt)
7310 : {
7311 0 : c = build2 (pred_code, boolean_type_node, arm1, arm2);
7312 0 : c = fold_convert (integer_type_node, c);
7313 0 : simtcond = create_tmp_var (integer_type_node);
7314 0 : gimplify_assign (simtcond, c, stmt_list);
7315 0 : gcall *g = gimple_build_call_internal (IFN_GOMP_SIMT_VOTE_ANY,
7316 : 1, simtcond);
7317 0 : c = create_tmp_var (integer_type_node);
7318 0 : gimple_call_set_lhs (g, c);
7319 0 : gimple_seq_add_stmt (stmt_list, g);
7320 0 : stmt = gimple_build_cond (NE_EXPR, c, integer_zero_node,
7321 : label_true, label);
7322 : }
7323 : else
7324 16372 : stmt = gimple_build_cond (pred_code, arm1, arm2, label_true, label);
7325 16372 : gimple_seq_add_stmt (stmt_list, stmt);
7326 16372 : gimple_seq_add_stmt (stmt_list, gimple_build_label (label_true));
7327 : }
7328 :
7329 16495 : tree cond_ptr = NULL_TREE;
7330 61602 : for (c = clauses; c ;)
7331 : {
7332 56370 : tree var, new_var;
7333 56370 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
7334 56370 : gimple_seq *this_stmt_list = stmt_list;
7335 56370 : tree lab2 = NULL_TREE;
7336 :
7337 56370 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7338 21824 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c)
7339 386 : && ctx->lastprivate_conditional_map
7340 56743 : && !ctx->combined_into_simd_safelen1)
7341 : {
7342 316 : gcc_assert (body_p);
7343 316 : if (simduid)
7344 37 : goto next;
7345 279 : if (cond_ptr == NULL_TREE)
7346 : {
7347 205 : cond_ptr = omp_find_clause (orig_clauses, OMP_CLAUSE__CONDTEMP_);
7348 205 : cond_ptr = OMP_CLAUSE_DECL (cond_ptr);
7349 : }
7350 279 : tree type = TREE_TYPE (TREE_TYPE (cond_ptr));
7351 279 : tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
7352 279 : tree v = *ctx->lastprivate_conditional_map->get (o);
7353 279 : gimplify_assign (v, build_zero_cst (type), body_p);
7354 279 : this_stmt_list = cstmt_list;
7355 279 : tree mem;
7356 279 : if (POINTER_TYPE_P (TREE_TYPE (cond_ptr)))
7357 : {
7358 136 : mem = build2 (MEM_REF, type, cond_ptr,
7359 136 : build_int_cst (TREE_TYPE (cond_ptr),
7360 136 : conditional_off));
7361 136 : conditional_off += tree_to_uhwi (TYPE_SIZE_UNIT (type));
7362 : }
7363 : else
7364 143 : mem = build4 (ARRAY_REF, type, cond_ptr,
7365 143 : size_int (conditional_off++), NULL_TREE, NULL_TREE);
7366 279 : tree mem2 = copy_node (mem);
7367 279 : gimple_seq seq = NULL;
7368 279 : mem = force_gimple_operand (mem, &seq, true, NULL_TREE);
7369 279 : gimple_seq_add_seq (this_stmt_list, seq);
7370 279 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
7371 279 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
7372 279 : gimple *g = gimple_build_cond (GT_EXPR, v, mem, lab1, lab2);
7373 279 : gimple_seq_add_stmt (this_stmt_list, g);
7374 279 : gimple_seq_add_stmt (this_stmt_list, gimple_build_label (lab1));
7375 279 : gimplify_assign (mem2, v, this_stmt_list);
7376 : }
7377 56054 : else if (predicate
7378 55808 : && ctx->combined_into_simd_safelen1
7379 116 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7380 57 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c)
7381 56111 : && ctx->lastprivate_conditional_map)
7382 : this_stmt_list = &post_stmt_list;
7383 :
7384 56333 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7385 56333 : || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
7386 4870 : && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c)))
7387 : {
7388 26657 : var = OMP_CLAUSE_DECL (c);
7389 26657 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7390 21787 : && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)
7391 27688 : && is_taskloop_ctx (ctx))
7392 : {
7393 312 : gcc_checking_assert (ctx->outer && is_task_ctx (ctx->outer));
7394 312 : new_var = lookup_decl (var, ctx->outer);
7395 : }
7396 : else
7397 : {
7398 26345 : new_var = lookup_decl (var, ctx);
7399 : /* Avoid uninitialized warnings for lastprivate and
7400 : for linear iterators. */
7401 26345 : if (predicate
7402 52497 : && (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7403 4870 : || OMP_CLAUSE_LINEAR_NO_COPYIN (c)))
7404 24883 : suppress_warning (new_var, OPT_Wuninitialized);
7405 : }
7406 :
7407 26657 : if (!maybe_simt && simduid && DECL_HAS_VALUE_EXPR_P (new_var))
7408 : {
7409 5864 : tree val = DECL_VALUE_EXPR (new_var);
7410 5864 : if (TREE_CODE (val) == ARRAY_REF
7411 5816 : && VAR_P (TREE_OPERAND (val, 0))
7412 11680 : && lookup_attribute ("omp simd array",
7413 5816 : DECL_ATTRIBUTES (TREE_OPERAND (val,
7414 : 0))))
7415 : {
7416 5816 : if (lastlane == NULL)
7417 : {
7418 2759 : lastlane = create_tmp_var (unsigned_type_node);
7419 2759 : gcall *g
7420 2759 : = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
7421 : 2, simduid,
7422 2759 : TREE_OPERAND (val, 1));
7423 2759 : gimple_call_set_lhs (g, lastlane);
7424 2759 : gimple_seq_add_stmt (this_stmt_list, g);
7425 : }
7426 5816 : new_var = build4 (ARRAY_REF, TREE_TYPE (val),
7427 5816 : TREE_OPERAND (val, 0), lastlane,
7428 : NULL_TREE, NULL_TREE);
7429 5816 : TREE_THIS_NOTRAP (new_var) = 1;
7430 : }
7431 : }
7432 20793 : else if (maybe_simt)
7433 : {
7434 0 : tree val = (DECL_HAS_VALUE_EXPR_P (new_var)
7435 0 : ? DECL_VALUE_EXPR (new_var)
7436 : : new_var);
7437 0 : if (simtlast == NULL)
7438 : {
7439 0 : simtlast = create_tmp_var (unsigned_type_node);
7440 0 : gcall *g = gimple_build_call_internal
7441 0 : (IFN_GOMP_SIMT_LAST_LANE, 1, simtcond);
7442 0 : gimple_call_set_lhs (g, simtlast);
7443 0 : gimple_seq_add_stmt (this_stmt_list, g);
7444 : }
7445 0 : x = build_call_expr_internal_loc
7446 0 : (UNKNOWN_LOCATION, IFN_GOMP_SIMT_XCHG_IDX,
7447 0 : TREE_TYPE (val), 2, val, simtlast);
7448 0 : new_var = unshare_expr (new_var);
7449 0 : gimplify_assign (new_var, x, this_stmt_list);
7450 0 : new_var = unshare_expr (new_var);
7451 : }
7452 :
7453 26657 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7454 26657 : && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
7455 : {
7456 7159 : lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
7457 7159 : gimple_seq_add_seq (this_stmt_list,
7458 7159 : OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c));
7459 7159 : OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL;
7460 : }
7461 19498 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
7462 19498 : && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
7463 : {
7464 426 : lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
7465 426 : gimple_seq_add_seq (this_stmt_list,
7466 426 : OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c));
7467 426 : OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c) = NULL;
7468 : }
7469 :
7470 26657 : x = NULL_TREE;
7471 26657 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
7472 21787 : && OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c)
7473 26717 : && is_taskloop_ctx (ctx))
7474 : {
7475 76 : tree ovar = maybe_lookup_decl_in_outer_ctx (var,
7476 38 : ctx->outer->outer);
7477 38 : if (is_global_var (ovar))
7478 7 : x = ovar;
7479 : }
7480 7 : if (!x)
7481 26650 : x = build_outer_var_ref (var, ctx, OMP_CLAUSE_LASTPRIVATE);
7482 26657 : if (omp_privatize_by_reference (var))
7483 657 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
7484 26657 : x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var);
7485 26657 : gimplify_and_add (x, this_stmt_list);
7486 :
7487 26657 : if (lab2)
7488 279 : gimple_seq_add_stmt (this_stmt_list, gimple_build_label (lab2));
7489 : }
7490 :
7491 56370 : next:
7492 56370 : c = OMP_CLAUSE_CHAIN (c);
7493 56370 : if (c == NULL && !par_clauses)
7494 : {
7495 : /* If this was a workshare clause, see if it had been combined
7496 : with its parallel. In that case, continue looking for the
7497 : clauses also on the parallel statement itself. */
7498 12492 : if (is_parallel_ctx (ctx))
7499 : break;
7500 :
7501 12492 : ctx = ctx->outer;
7502 12492 : if (ctx == NULL || !is_parallel_ctx (ctx))
7503 : break;
7504 :
7505 1229 : c = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
7506 : OMP_CLAUSE_LASTPRIVATE);
7507 1229 : par_clauses = true;
7508 : }
7509 : }
7510 :
7511 16495 : if (label)
7512 16372 : gimple_seq_add_stmt (stmt_list, gimple_build_label (label));
7513 16495 : gimple_seq_add_seq (stmt_list, post_stmt_list);
7514 : }
7515 :
7516 : /* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL
7517 : (which might be a placeholder). INNER is true if this is an inner
7518 : axis of a multi-axis loop. FORK and JOIN are (optional) fork and
7519 : join markers. Generate the before-loop forking sequence in
7520 : FORK_SEQ and the after-loop joining sequence to JOIN_SEQ. The
7521 : general form of these sequences is
7522 :
7523 : GOACC_REDUCTION_SETUP
7524 : GOACC_FORK
7525 : GOACC_REDUCTION_INIT
7526 : ...
7527 : GOACC_REDUCTION_FINI
7528 : GOACC_JOIN
7529 : GOACC_REDUCTION_TEARDOWN. */
7530 :
7531 : static void
7532 25946 : lower_oacc_reductions (location_t loc, tree clauses, tree level, bool inner,
7533 : gcall *fork, gcall *private_marker, gcall *join,
7534 : gimple_seq *fork_seq, gimple_seq *join_seq,
7535 : omp_context *ctx)
7536 : {
7537 25946 : gimple_seq before_fork = NULL;
7538 25946 : gimple_seq after_fork = NULL;
7539 25946 : gimple_seq before_join = NULL;
7540 25946 : gimple_seq after_join = NULL;
7541 25946 : tree init_code = NULL_TREE, fini_code = NULL_TREE,
7542 25946 : setup_code = NULL_TREE, teardown_code = NULL_TREE;
7543 25946 : unsigned offset = 0;
7544 :
7545 96516 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
7546 70570 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
7547 : {
7548 : /* No 'reduction' clauses on OpenACC 'kernels'. */
7549 7751 : gcc_checking_assert (!is_oacc_kernels (ctx));
7550 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
7551 7751 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
7552 :
7553 7751 : tree orig = OMP_CLAUSE_DECL (c);
7554 7751 : tree orig_clause;
7555 7751 : tree var = maybe_lookup_decl (orig, ctx);
7556 7751 : tree ref_to_res = NULL_TREE;
7557 7751 : tree incoming, outgoing, v1, v2, v3;
7558 7751 : bool is_private = false;
7559 :
7560 7751 : enum tree_code rcode = OMP_CLAUSE_REDUCTION_CODE (c);
7561 7751 : if (rcode == MINUS_EXPR)
7562 : rcode = PLUS_EXPR;
7563 6609 : else if (rcode == TRUTH_ANDIF_EXPR)
7564 : rcode = BIT_AND_EXPR;
7565 6431 : else if (rcode == TRUTH_ORIF_EXPR)
7566 433 : rcode = BIT_IOR_EXPR;
7567 7751 : tree op = build_int_cst (unsigned_type_node, rcode);
7568 :
7569 7751 : if (!var)
7570 4 : var = orig;
7571 :
7572 7751 : incoming = outgoing = var;
7573 :
7574 7751 : if (!inner)
7575 : {
7576 : /* See if an outer construct also reduces this variable. */
7577 : omp_context *outer = ctx;
7578 :
7579 7385 : while (omp_context *probe = outer->outer)
7580 : {
7581 4708 : enum gimple_code type = gimple_code (probe->stmt);
7582 4708 : tree cls;
7583 :
7584 4708 : switch (type)
7585 : {
7586 2220 : case GIMPLE_OMP_FOR:
7587 2220 : cls = gimple_omp_for_clauses (probe->stmt);
7588 2220 : break;
7589 :
7590 2488 : case GIMPLE_OMP_TARGET:
7591 : /* No 'reduction' clauses inside OpenACC 'kernels'
7592 : regions. */
7593 2488 : gcc_checking_assert (!is_oacc_kernels (probe));
7594 :
7595 2488 : if (!is_gimple_omp_offloaded (probe->stmt))
7596 26 : goto do_lookup;
7597 :
7598 2462 : cls = gimple_omp_target_clauses (probe->stmt);
7599 2462 : break;
7600 :
7601 0 : default:
7602 0 : goto do_lookup;
7603 : }
7604 :
7605 14249 : outer = probe;
7606 14249 : for (; cls; cls = OMP_CLAUSE_CHAIN (cls))
7607 11632 : if (OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_REDUCTION
7608 11632 : && orig == OMP_CLAUSE_DECL (cls))
7609 : {
7610 1772 : incoming = outgoing = lookup_decl (orig, probe);
7611 1772 : goto has_outer_reduction;
7612 : }
7613 9860 : else if ((OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_FIRSTPRIVATE
7614 8971 : || OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_PRIVATE)
7615 11164 : && orig == OMP_CLAUSE_DECL (cls))
7616 : {
7617 293 : is_private = true;
7618 293 : goto do_lookup;
7619 : }
7620 : }
7621 :
7622 2677 : do_lookup:
7623 : /* This is the outermost construct with this reduction,
7624 : see if there's a mapping for it. */
7625 2996 : orig_clause = NULL_TREE;
7626 2996 : if (gimple_code (outer->stmt) == GIMPLE_OMP_TARGET)
7627 7557 : for (tree cls = gimple_omp_target_clauses (outer->stmt);
7628 7557 : cls; cls = OMP_CLAUSE_CHAIN (cls))
7629 7250 : if (OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_MAP
7630 4823 : && orig == OMP_CLAUSE_DECL (cls)
7631 9805 : && maybe_lookup_field (cls, outer))
7632 : {
7633 : orig_clause = cls;
7634 : break;
7635 : }
7636 2996 : if (orig_clause != NULL_TREE && !is_private)
7637 : {
7638 2553 : ref_to_res = build_receiver_ref (orig_clause, false, outer);
7639 2553 : if (omp_privatize_by_reference (orig))
7640 79 : ref_to_res = build_simple_mem_ref (ref_to_res);
7641 :
7642 2553 : tree type = TREE_TYPE (var);
7643 2553 : if (POINTER_TYPE_P (type))
7644 79 : type = TREE_TYPE (type);
7645 :
7646 2553 : outgoing = var;
7647 2553 : incoming = omp_reduction_init_op (loc, rcode, type);
7648 : }
7649 : else
7650 : {
7651 : /* Try to look at enclosing contexts for reduction var,
7652 : use original if no mapping found. */
7653 443 : tree t = NULL_TREE;
7654 443 : omp_context *c = ctx->outer;
7655 895 : while (c && !t)
7656 : {
7657 452 : t = maybe_lookup_decl (orig, c);
7658 452 : c = c->outer;
7659 : }
7660 443 : incoming = outgoing = (t ? t : orig);
7661 : }
7662 :
7663 2553 : has_outer_reduction:;
7664 : }
7665 :
7666 4325 : if (!ref_to_res)
7667 5198 : ref_to_res = integer_zero_node;
7668 :
7669 7751 : if (omp_privatize_by_reference (orig))
7670 : {
7671 159 : tree type = TREE_TYPE (var);
7672 159 : const char *id = IDENTIFIER_POINTER (DECL_NAME (var));
7673 :
7674 159 : if (!inner)
7675 : {
7676 110 : tree x = create_tmp_var (TREE_TYPE (type), id);
7677 110 : gimplify_assign (var, build_fold_addr_expr (x), fork_seq);
7678 : }
7679 :
7680 159 : v1 = create_tmp_var (type, id);
7681 159 : v2 = create_tmp_var (type, id);
7682 159 : v3 = create_tmp_var (type, id);
7683 :
7684 159 : gimplify_assign (v1, var, fork_seq);
7685 159 : gimplify_assign (v2, var, fork_seq);
7686 159 : gimplify_assign (v3, var, fork_seq);
7687 :
7688 159 : var = build_simple_mem_ref (var);
7689 159 : v1 = build_simple_mem_ref (v1);
7690 159 : v2 = build_simple_mem_ref (v2);
7691 159 : v3 = build_simple_mem_ref (v3);
7692 159 : outgoing = build_simple_mem_ref (outgoing);
7693 :
7694 159 : if (!TREE_CONSTANT (incoming))
7695 80 : incoming = build_simple_mem_ref (incoming);
7696 : }
7697 : else
7698 : /* Note that 'var' might be a mem ref. */
7699 : v1 = v2 = v3 = var;
7700 :
7701 : /* Determine position in reduction buffer, which may be used
7702 : by target. The parser has ensured that this is not a
7703 : variable-sized type. */
7704 7751 : fixed_size_mode mode
7705 7751 : = as_a <fixed_size_mode> (TYPE_MODE (TREE_TYPE (var)));
7706 7751 : unsigned align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT;
7707 7751 : offset = (offset + align - 1) & ~(align - 1);
7708 7751 : tree off = build_int_cst (sizetype, offset);
7709 7751 : offset += GET_MODE_SIZE (mode);
7710 :
7711 7751 : if (!init_code)
7712 : {
7713 6969 : init_code = build_int_cst (integer_type_node,
7714 : IFN_GOACC_REDUCTION_INIT);
7715 6969 : fini_code = build_int_cst (integer_type_node,
7716 : IFN_GOACC_REDUCTION_FINI);
7717 6969 : setup_code = build_int_cst (integer_type_node,
7718 : IFN_GOACC_REDUCTION_SETUP);
7719 6969 : teardown_code = build_int_cst (integer_type_node,
7720 : IFN_GOACC_REDUCTION_TEARDOWN);
7721 : }
7722 :
7723 7751 : tree setup_call
7724 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7725 7751 : TREE_TYPE (var), 6, setup_code,
7726 : unshare_expr (ref_to_res),
7727 : unshare_expr (incoming),
7728 : level, op, off);
7729 7751 : tree init_call
7730 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7731 7751 : TREE_TYPE (var), 6, init_code,
7732 : unshare_expr (ref_to_res),
7733 : unshare_expr (v1), level, op, off);
7734 7751 : tree fini_call
7735 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7736 7751 : TREE_TYPE (var), 6, fini_code,
7737 : unshare_expr (ref_to_res),
7738 : unshare_expr (v2), level, op, off);
7739 7751 : tree teardown_call
7740 15502 : = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
7741 7751 : TREE_TYPE (var), 6, teardown_code,
7742 : ref_to_res, unshare_expr (v3),
7743 : level, op, off);
7744 :
7745 7751 : gimplify_assign (unshare_expr (v1), setup_call, &before_fork);
7746 7751 : gimplify_assign (unshare_expr (v2), init_call, &after_fork);
7747 7751 : gimplify_assign (unshare_expr (v3), fini_call, &before_join);
7748 7751 : gimplify_assign (unshare_expr (outgoing), teardown_call, &after_join);
7749 : }
7750 :
7751 : /* Now stitch things together. */
7752 25946 : gimple_seq_add_seq (fork_seq, before_fork);
7753 25946 : if (private_marker)
7754 259 : gimple_seq_add_stmt (fork_seq, private_marker);
7755 25946 : if (fork)
7756 16551 : gimple_seq_add_stmt (fork_seq, fork);
7757 25946 : gimple_seq_add_seq (fork_seq, after_fork);
7758 :
7759 25946 : gimple_seq_add_seq (join_seq, before_join);
7760 25946 : if (join)
7761 16551 : gimple_seq_add_stmt (join_seq, join);
7762 25946 : gimple_seq_add_seq (join_seq, after_join);
7763 25946 : }
7764 :
7765 : /* Generate code to implement the REDUCTION clauses, append it
7766 : to STMT_SEQP. CLIST if non-NULL is a pointer to a sequence
7767 : that should be emitted also inside of the critical section,
7768 : in that case clear *CLIST afterwards, otherwise leave it as is
7769 : and let the caller emit it itself. */
7770 :
7771 : static void
7772 72141 : lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp,
7773 : gimple_seq *clist, omp_context *ctx)
7774 : {
7775 72141 : gimple_seq sub_seq = NULL;
7776 72141 : gimple *stmt;
7777 72141 : tree x, c;
7778 72141 : int count = 0;
7779 :
7780 : /* OpenACC loop reductions are handled elsewhere. */
7781 72141 : if (is_gimple_omp_oacc (ctx->stmt))
7782 71082 : return;
7783 :
7784 : /* SIMD reductions are handled in lower_rec_input_clauses. */
7785 60816 : if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
7786 60816 : && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
7787 : return;
7788 :
7789 : /* inscan reductions are handled elsewhere. */
7790 51458 : if (ctx->scan_inclusive || ctx->scan_exclusive)
7791 : return;
7792 :
7793 : /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
7794 : update in that case, otherwise use a lock. */
7795 242781 : for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c))
7796 192362 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
7797 192362 : && !OMP_CLAUSE_REDUCTION_TASK (c))
7798 : {
7799 4635 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)
7800 4635 : || TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF)
7801 : {
7802 : /* Never use OMP_ATOMIC for array reductions or UDRs. */
7803 : count = -1;
7804 : break;
7805 : }
7806 3773 : count++;
7807 : }
7808 :
7809 51281 : if (count == 0)
7810 : return;
7811 :
7812 15006 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
7813 : {
7814 13947 : tree var, ref, new_var, orig_var;
7815 13947 : enum tree_code code;
7816 13947 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
7817 :
7818 22447 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
7819 13947 : || OMP_CLAUSE_REDUCTION_TASK (c))
7820 8500 : continue;
7821 :
7822 5447 : enum omp_clause_code ccode = OMP_CLAUSE_REDUCTION;
7823 5447 : orig_var = var = OMP_CLAUSE_DECL (c);
7824 5447 : if (TREE_CODE (var) == MEM_REF)
7825 : {
7826 689 : var = TREE_OPERAND (var, 0);
7827 689 : if (TREE_CODE (var) == POINTER_PLUS_EXPR)
7828 50 : var = TREE_OPERAND (var, 0);
7829 689 : if (TREE_CODE (var) == ADDR_EXPR)
7830 304 : var = TREE_OPERAND (var, 0);
7831 : else
7832 : {
7833 : /* If this is a pointer or referenced based array
7834 : section, the var could be private in the outer
7835 : context e.g. on orphaned loop construct. Pretend this
7836 : is private variable's outer reference. */
7837 385 : ccode = OMP_CLAUSE_PRIVATE;
7838 385 : if (INDIRECT_REF_P (var))
7839 54 : var = TREE_OPERAND (var, 0);
7840 : }
7841 689 : orig_var = var;
7842 689 : if (is_variable_sized (var))
7843 : {
7844 25 : gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
7845 25 : var = DECL_VALUE_EXPR (var);
7846 25 : gcc_assert (INDIRECT_REF_P (var));
7847 25 : var = TREE_OPERAND (var, 0);
7848 25 : gcc_assert (DECL_P (var));
7849 : }
7850 : }
7851 5447 : new_var = lookup_decl (var, ctx);
7852 5447 : if (var == OMP_CLAUSE_DECL (c)
7853 5447 : && omp_privatize_by_reference (var))
7854 152 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
7855 5447 : ref = build_outer_var_ref (var, ctx, ccode);
7856 5447 : code = OMP_CLAUSE_REDUCTION_CODE (c);
7857 :
7858 : /* reduction(-:var) sums up the partial results, so it acts
7859 : identically to reduction(+:var). */
7860 5447 : if (code == MINUS_EXPR)
7861 57 : code = PLUS_EXPR;
7862 :
7863 5447 : bool is_truth_op = (code == TRUTH_ANDIF_EXPR || code == TRUTH_ORIF_EXPR);
7864 5447 : if (count == 1)
7865 : {
7866 3324 : tree addr = build_fold_addr_expr_loc (clause_loc, ref);
7867 :
7868 3324 : addr = save_expr (addr);
7869 3324 : ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr);
7870 3324 : tree new_var2 = new_var;
7871 3324 : tree ref2 = ref;
7872 3324 : if (is_truth_op)
7873 : {
7874 784 : tree zero = build_zero_cst (TREE_TYPE (new_var));
7875 784 : new_var2 = fold_build2_loc (clause_loc, NE_EXPR,
7876 : boolean_type_node, new_var, zero);
7877 784 : ref2 = fold_build2_loc (clause_loc, NE_EXPR, boolean_type_node,
7878 : ref, zero);
7879 : }
7880 3324 : x = fold_build2_loc (clause_loc, code, TREE_TYPE (new_var2), ref2,
7881 : new_var2);
7882 3324 : if (is_truth_op)
7883 784 : x = fold_convert (TREE_TYPE (new_var), x);
7884 3324 : x = build2 (OMP_ATOMIC, void_type_node, addr, x);
7885 3324 : OMP_ATOMIC_MEMORY_ORDER (x) = OMP_MEMORY_ORDER_RELAXED;
7886 3324 : gimplify_and_add (x, stmt_seqp);
7887 3324 : return;
7888 : }
7889 2123 : else if (TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF)
7890 : {
7891 689 : tree d = OMP_CLAUSE_DECL (c);
7892 689 : tree type = TREE_TYPE (d);
7893 689 : tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
7894 689 : tree i = create_tmp_var (TREE_TYPE (v));
7895 689 : tree ptype = build_pointer_type (TREE_TYPE (type));
7896 689 : tree bias = TREE_OPERAND (d, 1);
7897 689 : d = TREE_OPERAND (d, 0);
7898 689 : if (TREE_CODE (d) == POINTER_PLUS_EXPR)
7899 : {
7900 50 : tree b = TREE_OPERAND (d, 1);
7901 50 : b = maybe_lookup_decl (b, ctx);
7902 50 : if (b == NULL)
7903 : {
7904 2 : b = TREE_OPERAND (d, 1);
7905 2 : b = maybe_lookup_decl_in_outer_ctx (b, ctx);
7906 : }
7907 50 : if (integer_zerop (bias))
7908 : bias = b;
7909 : else
7910 : {
7911 0 : bias = fold_convert_loc (clause_loc, TREE_TYPE (b), bias);
7912 0 : bias = fold_build2_loc (clause_loc, PLUS_EXPR,
7913 0 : TREE_TYPE (b), b, bias);
7914 : }
7915 50 : d = TREE_OPERAND (d, 0);
7916 : }
7917 : /* For ref build_outer_var_ref already performs this, so
7918 : only new_var needs a dereference. */
7919 689 : if (INDIRECT_REF_P (d))
7920 : {
7921 54 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
7922 54 : gcc_assert (omp_privatize_by_reference (var)
7923 : && var == orig_var);
7924 : }
7925 635 : else if (TREE_CODE (d) == ADDR_EXPR)
7926 : {
7927 304 : if (orig_var == var)
7928 : {
7929 279 : new_var = build_fold_addr_expr (new_var);
7930 279 : ref = build_fold_addr_expr (ref);
7931 : }
7932 : }
7933 : else
7934 : {
7935 331 : gcc_assert (orig_var == var);
7936 331 : if (omp_privatize_by_reference (var))
7937 79 : ref = build_fold_addr_expr (ref);
7938 : }
7939 689 : if (DECL_P (v))
7940 : {
7941 93 : tree t = maybe_lookup_decl (v, ctx);
7942 93 : if (t)
7943 88 : v = t;
7944 : else
7945 5 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
7946 93 : gimplify_expr (&v, stmt_seqp, NULL, is_gimple_val, fb_rvalue);
7947 : }
7948 689 : if (!integer_zerop (bias))
7949 : {
7950 402 : bias = fold_convert_loc (clause_loc, sizetype, bias);
7951 804 : new_var = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
7952 402 : TREE_TYPE (new_var), new_var,
7953 : unshare_expr (bias));
7954 402 : ref = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
7955 402 : TREE_TYPE (ref), ref, bias);
7956 : }
7957 689 : new_var = fold_convert_loc (clause_loc, ptype, new_var);
7958 689 : ref = fold_convert_loc (clause_loc, ptype, ref);
7959 689 : tree m = create_tmp_var (ptype);
7960 689 : gimplify_assign (m, new_var, stmt_seqp);
7961 689 : new_var = m;
7962 689 : m = create_tmp_var (ptype);
7963 689 : gimplify_assign (m, ref, stmt_seqp);
7964 689 : ref = m;
7965 689 : gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), stmt_seqp);
7966 689 : tree body = create_artificial_label (UNKNOWN_LOCATION);
7967 689 : tree end = create_artificial_label (UNKNOWN_LOCATION);
7968 689 : gimple_seq_add_stmt (&sub_seq, gimple_build_label (body));
7969 689 : tree priv = build_simple_mem_ref_loc (clause_loc, new_var);
7970 689 : tree out = build_simple_mem_ref_loc (clause_loc, ref);
7971 689 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
7972 : {
7973 72 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
7974 72 : tree decl_placeholder
7975 72 : = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
7976 72 : SET_DECL_VALUE_EXPR (placeholder, out);
7977 72 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
7978 72 : SET_DECL_VALUE_EXPR (decl_placeholder, priv);
7979 72 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
7980 72 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
7981 72 : gimple_seq_add_seq (&sub_seq,
7982 72 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
7983 72 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
7984 72 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
7985 72 : OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c) = NULL;
7986 : }
7987 : else
7988 : {
7989 617 : tree out2 = out;
7990 617 : tree priv2 = priv;
7991 617 : if (is_truth_op)
7992 : {
7993 0 : tree zero = build_zero_cst (TREE_TYPE (out));
7994 0 : out2 = fold_build2_loc (clause_loc, NE_EXPR,
7995 : boolean_type_node, out, zero);
7996 0 : priv2 = fold_build2_loc (clause_loc, NE_EXPR,
7997 : boolean_type_node, priv, zero);
7998 : }
7999 617 : x = build2 (code, TREE_TYPE (out2), out2, priv2);
8000 617 : if (is_truth_op)
8001 0 : x = fold_convert (TREE_TYPE (out), x);
8002 617 : out = unshare_expr (out);
8003 617 : gimplify_assign (out, x, &sub_seq);
8004 : }
8005 689 : gimple *g = gimple_build_assign (new_var, POINTER_PLUS_EXPR, new_var,
8006 689 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
8007 689 : gimple_seq_add_stmt (&sub_seq, g);
8008 689 : g = gimple_build_assign (ref, POINTER_PLUS_EXPR, ref,
8009 689 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
8010 689 : gimple_seq_add_stmt (&sub_seq, g);
8011 689 : g = gimple_build_assign (i, PLUS_EXPR, i,
8012 689 : build_int_cst (TREE_TYPE (i), 1));
8013 689 : gimple_seq_add_stmt (&sub_seq, g);
8014 689 : g = gimple_build_cond (LE_EXPR, i, v, body, end);
8015 689 : gimple_seq_add_stmt (&sub_seq, g);
8016 689 : gimple_seq_add_stmt (&sub_seq, gimple_build_label (end));
8017 : }
8018 1434 : else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
8019 : {
8020 788 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
8021 :
8022 788 : if (omp_privatize_by_reference (var)
8023 845 : && !useless_type_conversion_p (TREE_TYPE (placeholder),
8024 57 : TREE_TYPE (ref)))
8025 49 : ref = build_fold_addr_expr_loc (clause_loc, ref);
8026 788 : SET_DECL_VALUE_EXPR (placeholder, ref);
8027 788 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
8028 788 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
8029 788 : gimple_seq_add_seq (&sub_seq, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
8030 788 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
8031 788 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
8032 : }
8033 : else
8034 : {
8035 646 : tree new_var2 = new_var;
8036 646 : tree ref2 = ref;
8037 646 : if (is_truth_op)
8038 : {
8039 78 : tree zero = build_zero_cst (TREE_TYPE (new_var));
8040 78 : new_var2 = fold_build2_loc (clause_loc, NE_EXPR,
8041 : boolean_type_node, new_var, zero);
8042 78 : ref2 = fold_build2_loc (clause_loc, NE_EXPR, boolean_type_node,
8043 : ref, zero);
8044 : }
8045 646 : x = build2 (code, TREE_TYPE (ref), ref2, new_var2);
8046 646 : if (is_truth_op)
8047 78 : x = fold_convert (TREE_TYPE (new_var), x);
8048 646 : ref = build_outer_var_ref (var, ctx);
8049 646 : gimplify_assign (ref, x, &sub_seq);
8050 : }
8051 : }
8052 :
8053 1059 : stmt
8054 1059 : = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_START),
8055 : 0);
8056 1059 : gimple_seq_add_stmt (stmt_seqp, stmt);
8057 :
8058 1059 : gimple_seq_add_seq (stmt_seqp, sub_seq);
8059 :
8060 1059 : if (clist)
8061 : {
8062 173 : gimple_seq_add_seq (stmt_seqp, *clist);
8063 173 : *clist = NULL;
8064 : }
8065 :
8066 1059 : stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_END),
8067 : 0);
8068 1059 : gimple_seq_add_stmt (stmt_seqp, stmt);
8069 : }
8070 :
8071 :
8072 : /* Generate code to implement the COPYPRIVATE clauses. */
8073 :
8074 : static void
8075 115 : lower_copyprivate_clauses (tree clauses, gimple_seq *slist, gimple_seq *rlist,
8076 : omp_context *ctx)
8077 : {
8078 115 : tree c;
8079 :
8080 463 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
8081 : {
8082 348 : tree var, new_var, ref, x;
8083 348 : bool by_ref;
8084 348 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
8085 :
8086 348 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE)
8087 6 : continue;
8088 :
8089 342 : var = OMP_CLAUSE_DECL (c);
8090 342 : by_ref = use_pointer_for_field (var, NULL);
8091 :
8092 342 : ref = build_sender_ref (var, ctx);
8093 342 : x = new_var = lookup_decl_in_outer_ctx (var, ctx);
8094 342 : if (by_ref)
8095 : {
8096 74 : x = build_fold_addr_expr_loc (clause_loc, new_var);
8097 74 : x = fold_convert_loc (clause_loc, TREE_TYPE (ref), x);
8098 : }
8099 342 : gimplify_assign (ref, x, slist);
8100 :
8101 342 : ref = build_receiver_ref (var, false, ctx);
8102 342 : if (by_ref)
8103 : {
8104 74 : ref = fold_convert_loc (clause_loc,
8105 74 : build_pointer_type (TREE_TYPE (new_var)),
8106 : ref);
8107 74 : ref = build_fold_indirect_ref_loc (clause_loc, ref);
8108 : }
8109 342 : if (omp_privatize_by_reference (var))
8110 : {
8111 171 : ref = fold_convert_loc (clause_loc, TREE_TYPE (new_var), ref);
8112 171 : ref = build_simple_mem_ref_loc (clause_loc, ref);
8113 171 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
8114 : }
8115 342 : x = lang_hooks.decls.omp_clause_assign_op (c, new_var, ref);
8116 342 : gimplify_and_add (x, rlist);
8117 : }
8118 115 : }
8119 :
8120 :
8121 : /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
8122 : and REDUCTION from the sender (aka parent) side. */
8123 :
8124 : static void
8125 22482 : lower_send_clauses (tree clauses, gimple_seq *ilist, gimple_seq *olist,
8126 : omp_context *ctx)
8127 : {
8128 22482 : tree c, t;
8129 22482 : int ignored_looptemp = 0;
8130 22482 : bool is_taskloop = false;
8131 :
8132 : /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
8133 : by GOMP_taskloop. */
8134 22482 : if (is_task_ctx (ctx) && gimple_omp_task_taskloop_p (ctx->stmt))
8135 : {
8136 : ignored_looptemp = 2;
8137 : is_taskloop = true;
8138 : }
8139 :
8140 117247 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
8141 : {
8142 94765 : tree val, ref, x, var;
8143 94765 : bool by_ref, do_in = false, do_out = false;
8144 94765 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
8145 :
8146 94765 : switch (OMP_CLAUSE_CODE (c))
8147 : {
8148 4660 : case OMP_CLAUSE_PRIVATE:
8149 4660 : if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
8150 : break;
8151 4505 : continue;
8152 : case OMP_CLAUSE_FIRSTPRIVATE:
8153 : case OMP_CLAUSE_COPYIN:
8154 : case OMP_CLAUSE_LASTPRIVATE:
8155 : case OMP_CLAUSE_IN_REDUCTION:
8156 : case OMP_CLAUSE__REDUCTEMP_:
8157 : break;
8158 4811 : case OMP_CLAUSE_REDUCTION:
8159 4811 : if (is_task_ctx (ctx) || OMP_CLAUSE_REDUCTION_TASK (c))
8160 745 : continue;
8161 : break;
8162 17603 : case OMP_CLAUSE_SHARED:
8163 17603 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
8164 : break;
8165 17291 : continue;
8166 18801 : case OMP_CLAUSE__LOOPTEMP_:
8167 18801 : if (ignored_looptemp)
8168 : {
8169 2660 : ignored_looptemp--;
8170 2660 : continue;
8171 : }
8172 : break;
8173 11985 : default:
8174 11985 : continue;
8175 : }
8176 :
8177 57579 : val = OMP_CLAUSE_DECL (c);
8178 57579 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
8179 53513 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION)
8180 59219 : && TREE_CODE (val) == MEM_REF)
8181 : {
8182 1578 : val = TREE_OPERAND (val, 0);
8183 1578 : if (TREE_CODE (val) == POINTER_PLUS_EXPR)
8184 351 : val = TREE_OPERAND (val, 0);
8185 1578 : if (INDIRECT_REF_P (val)
8186 1494 : || TREE_CODE (val) == ADDR_EXPR)
8187 906 : val = TREE_OPERAND (val, 0);
8188 1578 : if (is_variable_sized (val))
8189 56 : continue;
8190 : }
8191 :
8192 : /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
8193 : outer taskloop region. */
8194 57523 : omp_context *ctx_for_o = ctx;
8195 57523 : if (is_taskloop
8196 3911 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
8197 57835 : && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
8198 312 : ctx_for_o = ctx->outer;
8199 :
8200 57523 : var = lookup_decl_in_outer_ctx (val, ctx_for_o);
8201 :
8202 57523 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN
8203 57009 : && is_global_var (var)
8204 59828 : && (val == OMP_CLAUSE_DECL (c)
8205 273 : || !is_task_ctx (ctx)
8206 241 : || (TREE_CODE (TREE_TYPE (val)) != POINTER_TYPE
8207 187 : && (TREE_CODE (TREE_TYPE (val)) != REFERENCE_TYPE
8208 0 : || (TREE_CODE (TREE_TYPE (TREE_TYPE (val)))
8209 : != POINTER_TYPE)))))
8210 2251 : continue;
8211 :
8212 55272 : t = omp_member_access_dummy_var (var);
8213 55272 : if (t)
8214 : {
8215 249 : var = DECL_VALUE_EXPR (var);
8216 249 : tree o = maybe_lookup_decl_in_outer_ctx (t, ctx_for_o);
8217 249 : if (o != t)
8218 69 : var = unshare_and_remap (var, t, o);
8219 : else
8220 180 : var = unshare_expr (var);
8221 : }
8222 :
8223 55272 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED)
8224 : {
8225 : /* Handle taskloop firstprivate/lastprivate, where the
8226 : lastprivate on GIMPLE_OMP_TASK is represented as
8227 : OMP_CLAUSE_SHARED_FIRSTPRIVATE. */
8228 79 : tree f = lookup_sfield ((splay_tree_key) &DECL_UID (val), ctx);
8229 79 : x = omp_build_component_ref (ctx->sender_decl, f);
8230 79 : if (use_pointer_for_field (val, ctx))
8231 59 : var = build_fold_addr_expr (var);
8232 79 : gimplify_assign (x, var, ilist);
8233 79 : DECL_ABSTRACT_ORIGIN (f) = NULL;
8234 79 : continue;
8235 79 : }
8236 :
8237 55247 : if (((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
8238 51872 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IN_REDUCTION)
8239 4457 : || val == OMP_CLAUSE_DECL (c))
8240 109083 : && is_variable_sized (val))
8241 54 : continue;
8242 55139 : by_ref = use_pointer_for_field (val, NULL);
8243 :
8244 55139 : switch (OMP_CLAUSE_CODE (c))
8245 : {
8246 26849 : case OMP_CLAUSE_FIRSTPRIVATE:
8247 26849 : if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c)
8248 7262 : && !by_ref
8249 34059 : && is_task_ctx (ctx))
8250 2331 : suppress_warning (var);
8251 : do_in = true;
8252 : break;
8253 :
8254 : case OMP_CLAUSE_PRIVATE:
8255 : case OMP_CLAUSE_COPYIN:
8256 : case OMP_CLAUSE__LOOPTEMP_:
8257 : case OMP_CLAUSE__REDUCTEMP_:
8258 : do_in = true;
8259 : break;
8260 :
8261 6467 : case OMP_CLAUSE_LASTPRIVATE:
8262 6467 : if (by_ref || omp_privatize_by_reference (val))
8263 : {
8264 384 : if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
8265 239 : continue;
8266 : do_in = true;
8267 : }
8268 : else
8269 : {
8270 6083 : do_out = true;
8271 6083 : if (lang_hooks.decls.omp_private_outer_ref (val))
8272 : do_in = true;
8273 : }
8274 : break;
8275 :
8276 4457 : case OMP_CLAUSE_REDUCTION:
8277 4457 : case OMP_CLAUSE_IN_REDUCTION:
8278 4457 : do_in = true;
8279 4457 : if (val == OMP_CLAUSE_DECL (c))
8280 : {
8281 3154 : if (is_task_ctx (ctx))
8282 301 : by_ref = use_pointer_for_field (val, ctx);
8283 : else
8284 2853 : do_out = !(by_ref || omp_privatize_by_reference (val));
8285 : }
8286 : else
8287 1303 : by_ref = TREE_CODE (TREE_TYPE (val)) == ARRAY_TYPE;
8288 : break;
8289 :
8290 0 : default:
8291 0 : gcc_unreachable ();
8292 : }
8293 :
8294 1604 : if (do_in)
8295 : {
8296 48841 : ref = build_sender_ref (val, ctx);
8297 48841 : x = by_ref ? build_fold_addr_expr_loc (clause_loc, var) : var;
8298 48841 : gimplify_assign (ref, x, ilist);
8299 48841 : if (is_task_ctx (ctx))
8300 6232 : DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref, 1)) = NULL;
8301 : }
8302 :
8303 54900 : if (do_out)
8304 : {
8305 8393 : ref = build_sender_ref (val, ctx);
8306 8393 : gimplify_assign (var, ref, olist);
8307 : }
8308 : }
8309 22482 : }
8310 :
8311 : /* Generate code to implement SHARED from the sender (aka parent)
8312 : side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
8313 : list things that got automatically shared. */
8314 :
8315 : static void
8316 22482 : lower_send_shared_vars (gimple_seq *ilist, gimple_seq *olist, omp_context *ctx)
8317 : {
8318 22482 : tree var, ovar, nvar, t, f, x, record_type;
8319 :
8320 22482 : if (ctx->record_type == NULL)
8321 4464 : return;
8322 :
8323 18018 : record_type = ctx->srecord_type ? ctx->srecord_type : ctx->record_type;
8324 86978 : for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
8325 : {
8326 68960 : ovar = DECL_ABSTRACT_ORIGIN (f);
8327 68960 : if (!ovar || TREE_CODE (ovar) == FIELD_DECL)
8328 6311 : continue;
8329 :
8330 62649 : nvar = maybe_lookup_decl (ovar, ctx);
8331 113621 : if (!nvar
8332 62236 : || !DECL_HAS_VALUE_EXPR_P (nvar)
8333 74524 : || (ctx->allocate_map
8334 291 : && ctx->allocate_map->get (ovar)))
8335 50972 : continue;
8336 :
8337 : /* If CTX is a nested parallel directive. Find the immediately
8338 : enclosing parallel or workshare construct that contains a
8339 : mapping for OVAR. */
8340 11677 : var = lookup_decl_in_outer_ctx (ovar, ctx);
8341 :
8342 11677 : t = omp_member_access_dummy_var (var);
8343 11677 : if (t)
8344 : {
8345 49 : var = DECL_VALUE_EXPR (var);
8346 49 : tree o = maybe_lookup_decl_in_outer_ctx (t, ctx);
8347 49 : if (o != t)
8348 18 : var = unshare_and_remap (var, t, o);
8349 : else
8350 31 : var = unshare_expr (var);
8351 : }
8352 :
8353 11677 : if (use_pointer_for_field (ovar, ctx))
8354 : {
8355 6437 : x = build_sender_ref (ovar, ctx);
8356 6437 : if (TREE_CODE (TREE_TYPE (f)) == ARRAY_TYPE
8357 6437 : && TREE_TYPE (f) == TREE_TYPE (ovar))
8358 : {
8359 103 : gcc_assert (is_parallel_ctx (ctx)
8360 : && DECL_ARTIFICIAL (ovar));
8361 : /* _condtemp_ clause. */
8362 103 : var = build_constructor (TREE_TYPE (x), NULL);
8363 : }
8364 : else
8365 6334 : var = build_fold_addr_expr (var);
8366 6437 : gimplify_assign (x, var, ilist);
8367 : }
8368 : else
8369 : {
8370 5240 : x = build_sender_ref (ovar, ctx);
8371 5240 : gimplify_assign (x, var, ilist);
8372 :
8373 5240 : if (!TREE_READONLY (var)
8374 : /* We don't need to receive a new reference to a result
8375 : or parm decl. In fact we may not store to it as we will
8376 : invalidate any pending RSO and generate wrong gimple
8377 : during inlining. */
8378 5240 : && !((TREE_CODE (var) == RESULT_DECL
8379 3338 : || TREE_CODE (var) == PARM_DECL)
8380 182 : && DECL_BY_REFERENCE (var)))
8381 : {
8382 3267 : x = build_sender_ref (ovar, ctx);
8383 3267 : gimplify_assign (var, x, olist);
8384 : }
8385 : }
8386 : }
8387 : }
8388 :
8389 : /* Emit an OpenACC head marker call, encapulating the partitioning and
8390 : other information that must be processed by the target compiler.
8391 : Return the maximum number of dimensions the associated loop might
8392 : be partitioned over. */
8393 :
8394 : static unsigned
8395 9665 : lower_oacc_head_mark (location_t loc, tree ddvar, tree clauses,
8396 : gimple_seq *seq, omp_context *ctx)
8397 : {
8398 9665 : unsigned levels = 0;
8399 9665 : unsigned tag = 0;
8400 9665 : tree gang_static = NULL_TREE;
8401 9665 : auto_vec<tree, 5> args;
8402 :
8403 19330 : args.quick_push (build_int_cst
8404 9665 : (integer_type_node, IFN_UNIQUE_OACC_HEAD_MARK));
8405 9665 : args.quick_push (ddvar);
8406 30854 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
8407 : {
8408 21189 : switch (OMP_CLAUSE_CODE (c))
8409 : {
8410 1856 : case OMP_CLAUSE_GANG:
8411 1856 : tag |= OLF_DIM_GANG;
8412 1856 : gang_static = OMP_CLAUSE_GANG_STATIC_EXPR (c);
8413 : /* static:* is represented by -1, and we can ignore it, as
8414 : scheduling is always static. */
8415 1856 : if (gang_static && integer_minus_onep (gang_static))
8416 70 : gang_static = NULL_TREE;
8417 1856 : levels++;
8418 1856 : break;
8419 :
8420 1330 : case OMP_CLAUSE_WORKER:
8421 1330 : tag |= OLF_DIM_WORKER;
8422 1330 : levels++;
8423 1330 : break;
8424 :
8425 1523 : case OMP_CLAUSE_VECTOR:
8426 1523 : tag |= OLF_DIM_VECTOR;
8427 1523 : levels++;
8428 1523 : break;
8429 :
8430 314 : case OMP_CLAUSE_SEQ:
8431 314 : tag |= OLF_SEQ;
8432 314 : break;
8433 :
8434 409 : case OMP_CLAUSE_AUTO:
8435 409 : tag |= OLF_AUTO;
8436 409 : break;
8437 :
8438 260 : case OMP_CLAUSE_INDEPENDENT:
8439 260 : tag |= OLF_INDEPENDENT;
8440 260 : break;
8441 :
8442 136 : case OMP_CLAUSE_TILE:
8443 136 : tag |= OLF_TILE;
8444 136 : break;
8445 :
8446 3986 : case OMP_CLAUSE_REDUCTION:
8447 3986 : tag |= OLF_REDUCTION;
8448 3986 : break;
8449 :
8450 11375 : default:
8451 11375 : continue;
8452 : }
8453 : }
8454 :
8455 9665 : if (gang_static)
8456 : {
8457 146 : if (DECL_P (gang_static))
8458 16 : gang_static = build_outer_var_ref (gang_static, ctx);
8459 146 : tag |= OLF_GANG_STATIC;
8460 : }
8461 :
8462 9665 : omp_context *tgt = enclosing_target_ctx (ctx);
8463 9665 : if (!tgt || is_oacc_parallel_or_serial (tgt))
8464 : ;
8465 62 : else if (is_oacc_kernels (tgt))
8466 : /* Not using this loops handling inside OpenACC 'kernels' regions. */
8467 0 : gcc_unreachable ();
8468 62 : else if (is_oacc_kernels_decomposed_part (tgt))
8469 : ;
8470 : else
8471 0 : gcc_unreachable ();
8472 :
8473 : /* In a parallel region, loops are implicitly INDEPENDENT. */
8474 9665 : if (!tgt || is_oacc_parallel_or_serial (tgt))
8475 9603 : tag |= OLF_INDEPENDENT;
8476 :
8477 : /* Loops inside OpenACC 'kernels' decomposed parts' regions are expected to
8478 : have an explicit 'seq' or 'independent' clause, and no 'auto' clause. */
8479 9665 : if (tgt && is_oacc_kernels_decomposed_part (tgt))
8480 : {
8481 62 : gcc_assert (tag & (OLF_SEQ | OLF_INDEPENDENT));
8482 62 : gcc_assert (!(tag & OLF_AUTO));
8483 : }
8484 :
8485 9665 : if (tag & OLF_TILE)
8486 : /* Tiling could use all 3 levels. */
8487 : levels = 3;
8488 : else
8489 : {
8490 : /* A loop lacking SEQ, GANG, WORKER and/or VECTOR could be AUTO.
8491 : Ensure at least one level, or 2 for possible auto
8492 : partitioning */
8493 9529 : bool maybe_auto = !(tag & (((GOMP_DIM_MASK (GOMP_DIM_MAX) - 1)
8494 : << OLF_DIM_BASE) | OLF_SEQ));
8495 :
8496 9529 : if (levels < 1u + maybe_auto)
8497 : levels = 1u + maybe_auto;
8498 : }
8499 :
8500 9665 : args.quick_push (build_int_cst (integer_type_node, levels));
8501 9665 : args.quick_push (build_int_cst (integer_type_node, tag));
8502 9665 : if (gang_static)
8503 146 : args.quick_push (gang_static);
8504 :
8505 9665 : gcall *call = gimple_build_call_internal_vec (IFN_UNIQUE, args);
8506 9665 : gimple_set_location (call, loc);
8507 9665 : gimple_set_lhs (call, ddvar);
8508 9665 : gimple_seq_add_stmt (seq, call);
8509 :
8510 9665 : return levels;
8511 9665 : }
8512 :
8513 : /* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the
8514 : partitioning level of the enclosed region. */
8515 :
8516 : static void
8517 42767 : lower_oacc_loop_marker (location_t loc, tree ddvar, bool head,
8518 : tree tofollow, gimple_seq *seq)
8519 : {
8520 42767 : int marker_kind = (head ? IFN_UNIQUE_OACC_HEAD_MARK
8521 : : IFN_UNIQUE_OACC_TAIL_MARK);
8522 42767 : tree marker = build_int_cst (integer_type_node, marker_kind);
8523 42767 : int nargs = 2 + (tofollow != NULL_TREE);
8524 42767 : gcall *call = gimple_build_call_internal (IFN_UNIQUE, nargs,
8525 : marker, ddvar, tofollow);
8526 42767 : gimple_set_location (call, loc);
8527 42767 : gimple_set_lhs (call, ddvar);
8528 42767 : gimple_seq_add_stmt (seq, call);
8529 42767 : }
8530 :
8531 : /* Generate the before and after OpenACC loop sequences. CLAUSES are
8532 : the loop clauses, from which we extract reductions. Initialize
8533 : HEAD and TAIL. */
8534 :
8535 : static void
8536 9665 : lower_oacc_head_tail (location_t loc, tree clauses, gcall *private_marker,
8537 : gimple_seq *head, gimple_seq *tail, omp_context *ctx)
8538 : {
8539 9665 : bool inner = false;
8540 9665 : tree ddvar = create_tmp_var (integer_type_node, ".data_dep");
8541 9665 : gimple_seq_add_stmt (head, gimple_build_assign (ddvar, integer_zero_node));
8542 :
8543 9665 : unsigned count = lower_oacc_head_mark (loc, ddvar, clauses, head, ctx);
8544 :
8545 9665 : if (private_marker)
8546 : {
8547 229 : gimple_set_location (private_marker, loc);
8548 229 : gimple_call_set_lhs (private_marker, ddvar);
8549 229 : gimple_call_set_arg (private_marker, 1, ddvar);
8550 : }
8551 :
8552 9665 : tree fork_kind = build_int_cst (unsigned_type_node, IFN_UNIQUE_OACC_FORK);
8553 9665 : tree join_kind = build_int_cst (unsigned_type_node, IFN_UNIQUE_OACC_JOIN);
8554 :
8555 9665 : gcc_assert (count);
8556 26216 : for (unsigned done = 1; count; count--, done++)
8557 : {
8558 16551 : gimple_seq fork_seq = NULL;
8559 16551 : gimple_seq join_seq = NULL;
8560 :
8561 16551 : tree place = build_int_cst (integer_type_node, -1);
8562 16551 : gcall *fork = gimple_build_call_internal (IFN_UNIQUE, 3,
8563 : fork_kind, ddvar, place);
8564 16551 : gimple_set_location (fork, loc);
8565 16551 : gimple_set_lhs (fork, ddvar);
8566 :
8567 16551 : gcall *join = gimple_build_call_internal (IFN_UNIQUE, 3,
8568 : join_kind, ddvar, place);
8569 16551 : gimple_set_location (join, loc);
8570 16551 : gimple_set_lhs (join, ddvar);
8571 :
8572 : /* Mark the beginning of this level sequence. */
8573 16551 : if (inner)
8574 6886 : lower_oacc_loop_marker (loc, ddvar, true,
8575 6886 : build_int_cst (integer_type_node, count),
8576 : &fork_seq);
8577 16551 : lower_oacc_loop_marker (loc, ddvar, false,
8578 16551 : build_int_cst (integer_type_node, done),
8579 : &join_seq);
8580 :
8581 23437 : lower_oacc_reductions (loc, clauses, place, inner,
8582 : fork, (count == 1) ? private_marker : NULL,
8583 : join, &fork_seq, &join_seq, ctx);
8584 :
8585 : /* Append this level to head. */
8586 16551 : gimple_seq_add_seq (head, fork_seq);
8587 : /* Prepend it to tail. */
8588 16551 : gimple_seq_add_seq (&join_seq, *tail);
8589 16551 : *tail = join_seq;
8590 :
8591 16551 : inner = true;
8592 : }
8593 :
8594 : /* Mark the end of the sequence. */
8595 9665 : lower_oacc_loop_marker (loc, ddvar, true, NULL_TREE, head);
8596 9665 : lower_oacc_loop_marker (loc, ddvar, false, NULL_TREE, tail);
8597 9665 : }
8598 :
8599 : /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
8600 : catch handler and return it. This prevents programs from violating the
8601 : structured block semantics with throws. */
8602 :
8603 : static gimple_seq
8604 95335 : maybe_catch_exception (gimple_seq body)
8605 : {
8606 95335 : gimple *g;
8607 95335 : tree decl;
8608 :
8609 95335 : if (!flag_exceptions)
8610 : return body;
8611 :
8612 42704 : if (lang_hooks.eh_protect_cleanup_actions != NULL)
8613 42676 : decl = lang_hooks.eh_protect_cleanup_actions ();
8614 : else
8615 28 : decl = builtin_decl_explicit (BUILT_IN_TRAP);
8616 :
8617 42704 : g = gimple_build_eh_must_not_throw (decl);
8618 42704 : g = gimple_build_try (body, gimple_seq_alloc_with_stmt (g),
8619 : GIMPLE_TRY_CATCH);
8620 :
8621 42704 : return gimple_seq_alloc_with_stmt (g);
8622 : }
8623 :
8624 :
8625 : /* Routines to lower OMP directives into OMP-GIMPLE. */
8626 :
8627 : /* If ctx is a worksharing context inside of a cancellable parallel
8628 : region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8629 : and conditional branch to parallel's cancel_label to handle
8630 : cancellation in the implicit barrier. */
8631 :
8632 : static void
8633 48640 : maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple *omp_return,
8634 : gimple_seq *body)
8635 : {
8636 48640 : gcc_assert (gimple_code (omp_return) == GIMPLE_OMP_RETURN);
8637 48640 : if (gimple_omp_return_nowait_p (omp_return))
8638 : return;
8639 19644 : for (omp_context *outer = ctx->outer; outer; outer = outer->outer)
8640 16027 : if (gimple_code (outer->stmt) == GIMPLE_OMP_PARALLEL
8641 16027 : && outer->cancellable)
8642 : {
8643 64 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_CANCEL);
8644 64 : tree c_bool_type = TREE_TYPE (TREE_TYPE (fndecl));
8645 64 : tree lhs = create_tmp_var (c_bool_type);
8646 64 : gimple_omp_return_set_lhs (omp_return, lhs);
8647 64 : tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
8648 64 : gimple *g = gimple_build_cond (NE_EXPR, lhs,
8649 : fold_convert (c_bool_type,
8650 : boolean_false_node),
8651 : outer->cancel_label, fallthru_label);
8652 64 : gimple_seq_add_stmt (body, g);
8653 64 : gimple_seq_add_stmt (body, gimple_build_label (fallthru_label));
8654 : }
8655 15963 : else if (gimple_code (outer->stmt) != GIMPLE_OMP_TASKGROUP
8656 15963 : && gimple_code (outer->stmt) != GIMPLE_OMP_SCOPE)
8657 : return;
8658 : }
8659 :
8660 : /* Find the first task_reduction or reduction clause or return NULL
8661 : if there are none. */
8662 :
8663 : static inline tree
8664 48895 : omp_task_reductions_find_first (tree clauses, enum tree_code code,
8665 : enum omp_clause_code ccode)
8666 : {
8667 64277 : while (1)
8668 : {
8669 56586 : clauses = omp_find_clause (clauses, ccode);
8670 56586 : if (clauses == NULL_TREE)
8671 : return NULL_TREE;
8672 9153 : if (ccode != OMP_CLAUSE_REDUCTION
8673 9153 : || code == OMP_TASKLOOP
8674 9153 : || OMP_CLAUSE_REDUCTION_TASK (clauses))
8675 : return clauses;
8676 7691 : clauses = OMP_CLAUSE_CHAIN (clauses);
8677 : }
8678 : }
8679 :
8680 : static void lower_omp_task_reductions (omp_context *, enum tree_code, tree,
8681 : gimple_seq *, gimple_seq *);
8682 :
8683 : /* Lower the OpenMP sections directive in the current statement in GSI_P.
8684 : CTX is the enclosing OMP context for the current statement. */
8685 :
8686 : static void
8687 386 : lower_omp_sections (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8688 : {
8689 386 : tree block, control;
8690 386 : gimple_stmt_iterator tgsi;
8691 386 : gomp_sections *stmt;
8692 386 : gimple *t;
8693 386 : gbind *new_stmt, *bind;
8694 386 : gimple_seq ilist, dlist, olist, tred_dlist = NULL, clist = NULL, new_body;
8695 :
8696 386 : stmt = as_a <gomp_sections *> (gsi_stmt (*gsi_p));
8697 :
8698 386 : push_gimplify_context ();
8699 :
8700 386 : dlist = NULL;
8701 386 : ilist = NULL;
8702 :
8703 386 : tree rclauses
8704 386 : = omp_task_reductions_find_first (gimple_omp_sections_clauses (stmt),
8705 : OMP_SECTIONS, OMP_CLAUSE_REDUCTION);
8706 386 : tree rtmp = NULL_TREE;
8707 386 : if (rclauses)
8708 : {
8709 8 : tree type = build_pointer_type (pointer_sized_int_node);
8710 8 : tree temp = create_tmp_var (type);
8711 8 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
8712 8 : OMP_CLAUSE_DECL (c) = temp;
8713 8 : OMP_CLAUSE_CHAIN (c) = gimple_omp_sections_clauses (stmt);
8714 8 : gimple_omp_sections_set_clauses (stmt, c);
8715 8 : lower_omp_task_reductions (ctx, OMP_SECTIONS,
8716 : gimple_omp_sections_clauses (stmt),
8717 : &ilist, &tred_dlist);
8718 8 : rclauses = c;
8719 8 : rtmp = make_ssa_name (type);
8720 8 : gimple_seq_add_stmt (&ilist, gimple_build_assign (rtmp, temp));
8721 : }
8722 :
8723 386 : tree *clauses_ptr = gimple_omp_sections_clauses_ptr (stmt);
8724 386 : lower_lastprivate_conditional_clauses (clauses_ptr, ctx);
8725 :
8726 386 : lower_rec_input_clauses (gimple_omp_sections_clauses (stmt),
8727 : &ilist, &dlist, ctx, NULL);
8728 :
8729 386 : control = create_tmp_var (unsigned_type_node, ".section");
8730 386 : gimple_omp_sections_set_control (stmt, control);
8731 :
8732 386 : new_body = gimple_omp_body (stmt);
8733 386 : gimple_omp_set_body (stmt, NULL);
8734 386 : tgsi = gsi_start (new_body);
8735 1249 : for (; !gsi_end_p (tgsi); gsi_next (&tgsi))
8736 : {
8737 863 : omp_context *sctx;
8738 863 : gimple *sec_start;
8739 :
8740 863 : sec_start = gsi_stmt (tgsi);
8741 863 : sctx = maybe_lookup_ctx (sec_start);
8742 863 : gcc_assert (sctx);
8743 :
8744 863 : lower_omp (gimple_omp_body_ptr (sec_start), sctx);
8745 863 : gsi_insert_seq_after (&tgsi, gimple_omp_body (sec_start),
8746 : GSI_CONTINUE_LINKING);
8747 863 : gimple_omp_set_body (sec_start, NULL);
8748 :
8749 863 : if (gsi_one_before_end_p (tgsi))
8750 : {
8751 386 : gimple_seq l = NULL;
8752 386 : lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt), NULL,
8753 : &ilist, &l, &clist, ctx);
8754 386 : gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING);
8755 386 : gimple_omp_section_set_last (sec_start);
8756 : }
8757 :
8758 863 : gsi_insert_after (&tgsi, gimple_build_omp_return (false),
8759 : GSI_CONTINUE_LINKING);
8760 : }
8761 :
8762 386 : block = make_node (BLOCK);
8763 386 : bind = gimple_build_bind (NULL, new_body, block);
8764 :
8765 386 : olist = NULL;
8766 386 : lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist,
8767 : &clist, ctx);
8768 386 : if (clist)
8769 : {
8770 10 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_START);
8771 10 : gcall *g = gimple_build_call (fndecl, 0);
8772 10 : gimple_seq_add_stmt (&olist, g);
8773 10 : gimple_seq_add_seq (&olist, clist);
8774 10 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_END);
8775 10 : g = gimple_build_call (fndecl, 0);
8776 10 : gimple_seq_add_stmt (&olist, g);
8777 : }
8778 :
8779 386 : block = make_node (BLOCK);
8780 386 : new_stmt = gimple_build_bind (NULL, NULL, block);
8781 386 : gsi_replace (gsi_p, new_stmt, true);
8782 :
8783 386 : pop_gimplify_context (new_stmt);
8784 386 : gimple_bind_append_vars (new_stmt, ctx->block_vars);
8785 386 : BLOCK_VARS (block) = gimple_bind_vars (bind);
8786 386 : if (BLOCK_VARS (block))
8787 0 : TREE_USED (block) = 1;
8788 :
8789 386 : new_body = NULL;
8790 386 : gimple_seq_add_seq (&new_body, ilist);
8791 386 : gimple_seq_add_stmt (&new_body, stmt);
8792 386 : gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ());
8793 386 : gimple_seq_add_stmt (&new_body, bind);
8794 :
8795 386 : t = gimple_build_omp_continue (control, control);
8796 386 : gimple_seq_add_stmt (&new_body, t);
8797 :
8798 386 : gimple_seq_add_seq (&new_body, olist);
8799 386 : if (ctx->cancellable)
8800 19 : gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
8801 386 : gimple_seq_add_seq (&new_body, dlist);
8802 :
8803 386 : new_body = maybe_catch_exception (new_body);
8804 :
8805 386 : bool nowait = omp_find_clause (gimple_omp_sections_clauses (stmt),
8806 386 : OMP_CLAUSE_NOWAIT) != NULL_TREE;
8807 386 : t = gimple_build_omp_return (nowait);
8808 386 : gimple_seq_add_stmt (&new_body, t);
8809 386 : gimple_seq_add_seq (&new_body, tred_dlist);
8810 386 : maybe_add_implicit_barrier_cancel (ctx, t, &new_body);
8811 :
8812 386 : if (rclauses)
8813 8 : OMP_CLAUSE_DECL (rclauses) = rtmp;
8814 :
8815 386 : gimple_bind_set_body (new_stmt, new_body);
8816 386 : }
8817 :
8818 :
8819 : /* A subroutine of lower_omp_single. Expand the simple form of
8820 : a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8821 :
8822 : if (GOMP_single_start ())
8823 : BODY;
8824 : [ GOMP_barrier (); ] -> unless 'nowait' is present.
8825 :
8826 : FIXME. It may be better to delay expanding the logic of this until
8827 : pass_expand_omp. The expanded logic may make the job more difficult
8828 : to a synchronization analysis pass. */
8829 :
8830 : static void
8831 1012 : lower_omp_single_simple (gomp_single *single_stmt, gimple_seq *pre_p)
8832 : {
8833 1012 : location_t loc = gimple_location (single_stmt);
8834 1012 : tree tlabel = create_artificial_label (loc);
8835 1012 : tree flabel = create_artificial_label (loc);
8836 1012 : gimple *call, *cond;
8837 1012 : tree lhs, decl;
8838 :
8839 2020 : decl = builtin_decl_explicit (flag_openmp_ompt
8840 : ? BUILT_IN_GOMP_SINGLE_START_WITH_END
8841 : : BUILT_IN_GOMP_SINGLE_START);
8842 1012 : lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)));
8843 1012 : call = gimple_build_call (decl, 0);
8844 1012 : gimple_call_set_lhs (call, lhs);
8845 1012 : gimple_seq_add_stmt (pre_p, call);
8846 :
8847 1012 : cond = gimple_build_cond (EQ_EXPR, lhs,
8848 1012 : fold_convert_loc (loc, TREE_TYPE (lhs),
8849 : boolean_true_node),
8850 : tlabel, flabel);
8851 1012 : gimple_seq_add_stmt (pre_p, cond);
8852 1012 : gimple_seq_add_stmt (pre_p, gimple_build_label (tlabel));
8853 1012 : gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
8854 1012 : gimple_seq_add_stmt (pre_p, gimple_build_label (flabel));
8855 1012 : }
8856 :
8857 :
8858 : /* A subroutine of lower_omp_single. Expand the simple form of
8859 : a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8860 :
8861 : #pragma omp single copyprivate (a, b, c)
8862 :
8863 : Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
8864 :
8865 : {
8866 : if ((copyout_p = GOMP_single_copy_start ()) == NULL)
8867 : {
8868 : BODY;
8869 : copyout.a = a;
8870 : copyout.b = b;
8871 : copyout.c = c;
8872 : GOMP_single_copy_end (©out);
8873 : }
8874 : else
8875 : {
8876 : a = copyout_p->a;
8877 : b = copyout_p->b;
8878 : c = copyout_p->c;
8879 : }
8880 : GOMP_barrier ();
8881 : }
8882 :
8883 : FIXME. It may be better to delay expanding the logic of this until
8884 : pass_expand_omp. The expanded logic may make the job more difficult
8885 : to a synchronization analysis pass. */
8886 :
8887 : static void
8888 115 : lower_omp_single_copy (gomp_single *single_stmt, gimple_seq *pre_p,
8889 : omp_context *ctx)
8890 : {
8891 115 : tree ptr_type, t, l0, l1, l2, bfn_decl;
8892 115 : gimple_seq copyin_seq;
8893 115 : location_t loc = gimple_location (single_stmt);
8894 :
8895 115 : ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o");
8896 :
8897 115 : ptr_type = build_pointer_type (ctx->record_type);
8898 115 : ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i");
8899 :
8900 115 : l0 = create_artificial_label (loc);
8901 115 : l1 = create_artificial_label (loc);
8902 115 : l2 = create_artificial_label (loc);
8903 :
8904 115 : bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START);
8905 115 : t = build_call_expr_loc (loc, bfn_decl, 0);
8906 115 : t = fold_convert_loc (loc, ptr_type, t);
8907 115 : gimplify_assign (ctx->receiver_decl, t, pre_p);
8908 :
8909 115 : t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl,
8910 : build_int_cst (ptr_type, 0));
8911 115 : t = build3 (COND_EXPR, void_type_node, t,
8912 : build_and_jump (&l0), build_and_jump (&l1));
8913 115 : gimplify_and_add (t, pre_p);
8914 :
8915 115 : gimple_seq_add_stmt (pre_p, gimple_build_label (l0));
8916 :
8917 115 : gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
8918 :
8919 115 : copyin_seq = NULL;
8920 115 : lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt), pre_p,
8921 : ©in_seq, ctx);
8922 :
8923 115 : t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
8924 115 : bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END);
8925 115 : t = build_call_expr_loc (loc, bfn_decl, 1, t);
8926 115 : gimplify_and_add (t, pre_p);
8927 :
8928 115 : t = build_and_jump (&l2);
8929 115 : gimplify_and_add (t, pre_p);
8930 :
8931 115 : gimple_seq_add_stmt (pre_p, gimple_build_label (l1));
8932 :
8933 115 : gimple_seq_add_seq (pre_p, copyin_seq);
8934 :
8935 115 : gimple_seq_add_stmt (pre_p, gimple_build_label (l2));
8936 115 : }
8937 :
8938 :
8939 : /* Expand code for an OpenMP single directive. */
8940 :
8941 : static void
8942 1127 : lower_omp_single (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8943 : {
8944 1127 : tree block;
8945 1127 : gomp_single *single_stmt = as_a <gomp_single *> (gsi_stmt (*gsi_p));
8946 1127 : gbind *bind;
8947 1127 : gimple_seq bind_body, bind_body_tail = NULL, dlist;
8948 :
8949 1127 : push_gimplify_context ();
8950 :
8951 1127 : block = make_node (BLOCK);
8952 1127 : bind = gimple_build_bind (NULL, NULL, block);
8953 1127 : gsi_replace (gsi_p, bind, true);
8954 1127 : bind_body = NULL;
8955 1127 : dlist = NULL;
8956 1127 : lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt),
8957 : &bind_body, &dlist, ctx, NULL);
8958 1127 : lower_omp (gimple_omp_body_ptr (single_stmt), ctx);
8959 :
8960 1127 : gimple_seq_add_stmt (&bind_body, single_stmt);
8961 :
8962 1127 : if (ctx->record_type)
8963 115 : lower_omp_single_copy (single_stmt, &bind_body, ctx);
8964 : else
8965 1012 : lower_omp_single_simple (single_stmt, &bind_body);
8966 :
8967 1127 : gimple_omp_set_body (single_stmt, NULL);
8968 :
8969 1127 : gimple_seq_add_seq (&bind_body, dlist);
8970 :
8971 1127 : bind_body = maybe_catch_exception (bind_body);
8972 :
8973 1127 : bool nowait = omp_find_clause (gimple_omp_single_clauses (single_stmt),
8974 1127 : OMP_CLAUSE_NOWAIT) != NULL_TREE;
8975 1127 : gimple *g = gimple_build_omp_return (nowait);
8976 1127 : gimple_seq_add_stmt (&bind_body_tail, g);
8977 1127 : maybe_add_implicit_barrier_cancel (ctx, g, &bind_body_tail);
8978 :
8979 1127 : if (flag_openmp_ompt && !ctx->record_type)
8980 : {
8981 : /* Insert call to GOMP_single_end. */
8982 4 : tree decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_END);
8983 4 : gimple *stmt = gimple_build_call (decl, 0);
8984 4 : gimple_seq_add_stmt (&bind_body_tail, stmt);
8985 : }
8986 :
8987 1127 : if (ctx->record_type)
8988 : {
8989 115 : gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
8990 115 : tree clobber = build_clobber (ctx->record_type);
8991 115 : gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
8992 : clobber), GSI_SAME_STMT);
8993 : }
8994 1127 : gimple_seq_add_seq (&bind_body, bind_body_tail);
8995 1127 : gimple_bind_set_body (bind, bind_body);
8996 :
8997 1127 : pop_gimplify_context (bind);
8998 :
8999 1127 : gimple_bind_append_vars (bind, ctx->block_vars);
9000 1127 : BLOCK_VARS (block) = ctx->block_vars;
9001 1127 : if (BLOCK_VARS (block))
9002 26 : TREE_USED (block) = 1;
9003 1127 : }
9004 :
9005 :
9006 : /* Lower code for an OMP scope directive. */
9007 :
9008 : static void
9009 161 : lower_omp_scope (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9010 : {
9011 161 : tree block;
9012 161 : gimple *scope_stmt = gsi_stmt (*gsi_p);
9013 161 : gbind *bind;
9014 161 : gimple_seq bind_body, bind_body_tail = NULL, dlist;
9015 161 : gimple_seq tred_dlist = NULL;
9016 :
9017 161 : push_gimplify_context ();
9018 :
9019 161 : block = make_node (BLOCK);
9020 161 : bind = gimple_build_bind (NULL, NULL, block);
9021 161 : gsi_replace (gsi_p, bind, true);
9022 161 : bind_body = NULL;
9023 161 : dlist = NULL;
9024 :
9025 161 : tree rclauses
9026 161 : = omp_task_reductions_find_first (gimple_omp_scope_clauses (scope_stmt),
9027 : OMP_SCOPE, OMP_CLAUSE_REDUCTION);
9028 161 : if (rclauses)
9029 : {
9030 54 : tree type = build_pointer_type (pointer_sized_int_node);
9031 54 : tree temp = create_tmp_var (type);
9032 54 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
9033 54 : OMP_CLAUSE_DECL (c) = temp;
9034 54 : OMP_CLAUSE_CHAIN (c) = gimple_omp_scope_clauses (scope_stmt);
9035 54 : gimple_omp_scope_set_clauses (scope_stmt, c);
9036 54 : lower_omp_task_reductions (ctx, OMP_SCOPE,
9037 : gimple_omp_scope_clauses (scope_stmt),
9038 : &bind_body, &tred_dlist);
9039 54 : rclauses = c;
9040 54 : tree fndecl = builtin_decl_explicit (
9041 54 : flag_openmp_ompt ? BUILT_IN_GOMP_SCOPE_START_WITH_END
9042 : : BUILT_IN_GOMP_SCOPE_START);
9043 54 : gimple *stmt = gimple_build_call (fndecl, 1, temp);
9044 54 : gimple_seq_add_stmt (&bind_body, stmt);
9045 : }
9046 107 : else if (flag_openmp_ompt)
9047 : {
9048 4 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_SCOPE_START_WITH_END);
9049 4 : gimple *stmt = gimple_build_call (fndecl, 1, null_pointer_node);
9050 4 : gimple_seq_add_stmt (&bind_body, stmt);
9051 : }
9052 :
9053 161 : lower_rec_input_clauses (gimple_omp_scope_clauses (scope_stmt),
9054 : &bind_body, &dlist, ctx, NULL);
9055 161 : lower_omp (gimple_omp_body_ptr (scope_stmt), ctx);
9056 :
9057 161 : gimple_seq_add_stmt (&bind_body, scope_stmt);
9058 :
9059 161 : gimple_seq_add_seq (&bind_body, gimple_omp_body (scope_stmt));
9060 :
9061 161 : gimple_omp_set_body (scope_stmt, NULL);
9062 :
9063 161 : gimple_seq clist = NULL;
9064 161 : lower_reduction_clauses (gimple_omp_scope_clauses (scope_stmt),
9065 : &bind_body, &clist, ctx);
9066 161 : if (clist)
9067 : {
9068 0 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_START);
9069 0 : gcall *g = gimple_build_call (fndecl, 0);
9070 0 : gimple_seq_add_stmt (&bind_body, g);
9071 0 : gimple_seq_add_seq (&bind_body, clist);
9072 0 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_END);
9073 0 : g = gimple_build_call (fndecl, 0);
9074 0 : gimple_seq_add_stmt (&bind_body, g);
9075 : }
9076 :
9077 161 : gimple_seq_add_seq (&bind_body, dlist);
9078 :
9079 161 : bind_body = maybe_catch_exception (bind_body);
9080 :
9081 161 : if (flag_openmp_ompt)
9082 : {
9083 8 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_SCOPE_END);
9084 8 : gcall *g = gimple_build_call (fndecl, 0);
9085 8 : gimple_seq_add_stmt (&bind_body_tail, g);
9086 : }
9087 :
9088 161 : bool nowait = omp_find_clause (gimple_omp_scope_clauses (scope_stmt),
9089 161 : OMP_CLAUSE_NOWAIT) != NULL_TREE;
9090 161 : gimple *g = gimple_build_omp_return (nowait);
9091 161 : gimple_seq_add_stmt (&bind_body_tail, g);
9092 161 : gimple_seq_add_seq (&bind_body_tail, tred_dlist);
9093 161 : maybe_add_implicit_barrier_cancel (ctx, g, &bind_body_tail);
9094 161 : if (ctx->record_type)
9095 : {
9096 0 : gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
9097 0 : tree clobber = build_clobber (ctx->record_type);
9098 0 : gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
9099 : clobber), GSI_SAME_STMT);
9100 : }
9101 161 : gimple_seq_add_seq (&bind_body, bind_body_tail);
9102 :
9103 161 : gimple_bind_set_body (bind, bind_body);
9104 :
9105 161 : pop_gimplify_context (bind);
9106 :
9107 161 : gimple_bind_append_vars (bind, ctx->block_vars);
9108 161 : BLOCK_VARS (block) = ctx->block_vars;
9109 161 : if (BLOCK_VARS (block))
9110 121 : TREE_USED (block) = 1;
9111 161 : }
9112 :
9113 : /* Lower code for an OMP dispatch directive. */
9114 :
9115 : static void
9116 659 : lower_omp_dispatch (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9117 : {
9118 659 : tree block;
9119 659 : gimple *stmt = gsi_stmt (*gsi_p);
9120 659 : gbind *bind;
9121 :
9122 659 : push_gimplify_context ();
9123 :
9124 659 : block = make_node (BLOCK);
9125 659 : bind = gimple_build_bind (NULL, NULL, block);
9126 659 : gsi_replace (gsi_p, bind, true);
9127 :
9128 659 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
9129 659 : gimple_bind_set_body (bind, maybe_catch_exception (gimple_omp_body (stmt)));
9130 :
9131 659 : pop_gimplify_context (bind);
9132 :
9133 659 : gimple_bind_append_vars (bind, ctx->block_vars);
9134 659 : BLOCK_VARS (block) = ctx->block_vars;
9135 659 : }
9136 :
9137 : /* Expand code for an OpenMP master or masked directive. */
9138 :
9139 : static void
9140 1118 : lower_omp_master (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9141 : {
9142 1118 : tree block, lab = NULL, x, bfn_decl;
9143 1118 : gimple *stmt = gsi_stmt (*gsi_p);
9144 1118 : gbind *bind;
9145 1118 : location_t loc = gimple_location (stmt);
9146 1118 : gimple_seq tseq;
9147 1118 : tree filter = integer_zero_node;
9148 :
9149 1118 : push_gimplify_context ();
9150 :
9151 1118 : if (gimple_code (stmt) == GIMPLE_OMP_MASKED)
9152 : {
9153 387 : filter = omp_find_clause (gimple_omp_masked_clauses (stmt),
9154 : OMP_CLAUSE_FILTER);
9155 387 : if (filter)
9156 252 : filter = fold_convert (integer_type_node,
9157 : OMP_CLAUSE_FILTER_EXPR (filter));
9158 : else
9159 135 : filter = integer_zero_node;
9160 : }
9161 1118 : block = make_node (BLOCK);
9162 1118 : bind = gimple_build_bind (NULL, NULL, block);
9163 1118 : gsi_replace (gsi_p, bind, true);
9164 1118 : gimple_bind_add_stmt (bind, stmt);
9165 :
9166 1118 : bfn_decl = builtin_decl_explicit (
9167 1118 : flag_openmp_ompt ? BUILT_IN_GOMP_HAS_MASKED_THREAD_NUM_WITH_END
9168 : : BUILT_IN_GOMP_HAS_MASKED_THREAD_NUM);
9169 1118 : x = build_call_expr_loc (loc, bfn_decl, 1, filter);
9170 1118 : x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab));
9171 1118 : tseq = NULL;
9172 1118 : gimplify_and_add (x, &tseq);
9173 1118 : gimple_bind_add_seq (bind, tseq);
9174 :
9175 1118 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
9176 1118 : gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
9177 1118 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
9178 1118 : gimple_omp_set_body (stmt, NULL);
9179 :
9180 1118 : if (flag_openmp_ompt)
9181 : {
9182 : /* Insert call to GOMP_masked_end at the end of the body. */
9183 4 : tree decl = builtin_decl_explicit (BUILT_IN_GOMP_MASKED_END);
9184 4 : gcall *g = gimple_build_call (decl, 0);
9185 4 : gimple_bind_add_stmt (bind, g);
9186 : }
9187 :
9188 1118 : gimple_bind_add_stmt (bind, gimple_build_label (lab));
9189 :
9190 1118 : gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
9191 :
9192 1118 : pop_gimplify_context (bind);
9193 :
9194 1118 : gimple_bind_append_vars (bind, ctx->block_vars);
9195 1118 : BLOCK_VARS (block) = ctx->block_vars;
9196 1118 : }
9197 :
9198 : /* Helper function for lower_omp_task_reductions. For a specific PASS
9199 : find out the current clause it should be processed, or return false
9200 : if all have been processed already. */
9201 :
9202 : static inline bool
9203 8000 : omp_task_reduction_iterate (int pass, enum tree_code code,
9204 : enum omp_clause_code ccode, tree *c, tree *decl,
9205 : tree *type, tree *next)
9206 : {
9207 11364 : for (; *c; *c = omp_find_clause (OMP_CLAUSE_CHAIN (*c), ccode))
9208 : {
9209 6728 : if (ccode == OMP_CLAUSE_REDUCTION
9210 6672 : && code != OMP_TASKLOOP
9211 6672 : && !OMP_CLAUSE_REDUCTION_TASK (*c))
9212 56 : continue;
9213 6616 : *decl = OMP_CLAUSE_DECL (*c);
9214 6616 : *type = TREE_TYPE (*decl);
9215 6616 : if (TREE_CODE (*decl) == MEM_REF)
9216 : {
9217 1940 : if (pass != 1)
9218 970 : continue;
9219 : }
9220 : else
9221 : {
9222 4676 : if (omp_privatize_by_reference (*decl))
9223 152 : *type = TREE_TYPE (*type);
9224 4676 : if (pass != (!TREE_CONSTANT (TYPE_SIZE_UNIT (*type))))
9225 2338 : continue;
9226 : }
9227 3308 : *next = omp_find_clause (OMP_CLAUSE_CHAIN (*c), ccode);
9228 3308 : return true;
9229 : }
9230 4692 : *decl = NULL_TREE;
9231 4692 : *type = NULL_TREE;
9232 4692 : *next = NULL_TREE;
9233 4692 : return false;
9234 : }
9235 :
9236 : /* Lower task_reduction and reduction clauses (the latter unless CODE is
9237 : OMP_TASKGROUP only with task modifier). Register mapping of those in
9238 : START sequence and reducing them and unregister them in the END sequence. */
9239 :
9240 : static void
9241 1382 : lower_omp_task_reductions (omp_context *ctx, enum tree_code code, tree clauses,
9242 : gimple_seq *start, gimple_seq *end)
9243 : {
9244 846 : enum omp_clause_code ccode
9245 : = (code == OMP_TASKGROUP
9246 1382 : ? OMP_CLAUSE_TASK_REDUCTION : OMP_CLAUSE_REDUCTION);
9247 1382 : tree cancellable = NULL_TREE;
9248 1382 : clauses = omp_task_reductions_find_first (clauses, code, ccode);
9249 1382 : if (clauses == NULL_TREE)
9250 209 : return;
9251 1173 : if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE)
9252 : {
9253 324 : for (omp_context *outer = ctx->outer; outer; outer = outer->outer)
9254 238 : if (gimple_code (outer->stmt) == GIMPLE_OMP_PARALLEL
9255 238 : && outer->cancellable)
9256 : {
9257 14 : cancellable = error_mark_node;
9258 14 : break;
9259 : }
9260 224 : else if (gimple_code (outer->stmt) != GIMPLE_OMP_TASKGROUP
9261 224 : && gimple_code (outer->stmt) != GIMPLE_OMP_SCOPE)
9262 : break;
9263 : }
9264 1173 : tree record_type = lang_hooks.types.make_type (RECORD_TYPE);
9265 1173 : tree *last = &TYPE_FIELDS (record_type);
9266 1173 : unsigned cnt = 0;
9267 1173 : if (cancellable)
9268 : {
9269 14 : tree field = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE,
9270 : ptr_type_node);
9271 14 : tree ifield = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE,
9272 : integer_type_node);
9273 14 : *last = field;
9274 14 : DECL_CHAIN (field) = ifield;
9275 14 : last = &DECL_CHAIN (ifield);
9276 14 : DECL_CONTEXT (field) = record_type;
9277 14 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (field))
9278 0 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (field));
9279 14 : DECL_CONTEXT (ifield) = record_type;
9280 14 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (ifield))
9281 0 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (ifield));
9282 : }
9283 3519 : for (int pass = 0; pass < 2; pass++)
9284 : {
9285 2346 : tree decl, type, next;
9286 2346 : for (tree c = clauses;
9287 4000 : omp_task_reduction_iterate (pass, code, ccode,
9288 1654 : &c, &decl, &type, &next); c = next)
9289 : {
9290 1654 : ++cnt;
9291 1654 : tree new_type = type;
9292 1654 : if (ctx->outer)
9293 1165 : new_type = remap_type (type, &ctx->outer->cb);
9294 1654 : tree field
9295 1654 : = build_decl (OMP_CLAUSE_LOCATION (c), FIELD_DECL,
9296 1654 : DECL_P (decl) ? DECL_NAME (decl) : NULL_TREE,
9297 : new_type);
9298 1654 : if (DECL_P (decl) && type == TREE_TYPE (decl))
9299 : {
9300 1131 : SET_DECL_ALIGN (field, DECL_ALIGN (decl));
9301 1131 : DECL_USER_ALIGN (field) = DECL_USER_ALIGN (decl);
9302 1131 : TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (decl);
9303 : }
9304 : else
9305 523 : SET_DECL_ALIGN (field, TYPE_ALIGN (type));
9306 1654 : DECL_CONTEXT (field) = record_type;
9307 1654 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (field))
9308 1195 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (field));
9309 1654 : *last = field;
9310 1654 : last = &DECL_CHAIN (field);
9311 1654 : tree bfield
9312 1654 : = build_decl (OMP_CLAUSE_LOCATION (c), FIELD_DECL, NULL_TREE,
9313 : boolean_type_node);
9314 1654 : DECL_CONTEXT (bfield) = record_type;
9315 1654 : if (TYPE_ALIGN (record_type) < DECL_ALIGN (bfield))
9316 0 : SET_TYPE_ALIGN (record_type, DECL_ALIGN (bfield));
9317 1654 : *last = bfield;
9318 1654 : last = &DECL_CHAIN (bfield);
9319 : }
9320 : }
9321 1173 : *last = NULL_TREE;
9322 1173 : layout_type (record_type);
9323 :
9324 : /* Build up an array which registers with the runtime all the reductions
9325 : and deregisters them at the end. Format documented in libgomp/task.c. */
9326 1173 : tree atype = build_array_type_nelts (pointer_sized_int_node, 7 + cnt * 3);
9327 1173 : tree avar = create_tmp_var_raw (atype);
9328 1173 : gimple_add_tmp_var (avar);
9329 1173 : TREE_ADDRESSABLE (avar) = 1;
9330 1173 : tree r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_zero_node,
9331 : NULL_TREE, NULL_TREE);
9332 1173 : tree t = build_int_cst (pointer_sized_int_node, cnt);
9333 1173 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9334 1173 : gimple_seq seq = NULL;
9335 1173 : tree sz = fold_convert (pointer_sized_int_node,
9336 : TYPE_SIZE_UNIT (record_type));
9337 1173 : int cachesz = 64;
9338 1173 : sz = fold_build2 (PLUS_EXPR, pointer_sized_int_node, sz,
9339 : build_int_cst (pointer_sized_int_node, cachesz - 1));
9340 1173 : sz = fold_build2 (BIT_AND_EXPR, pointer_sized_int_node, sz,
9341 : build_int_cst (pointer_sized_int_node, ~(cachesz - 1)));
9342 1173 : ctx->task_reductions.create (1 + cnt);
9343 1173 : ctx->task_reduction_map = new hash_map<tree, unsigned>;
9344 2346 : ctx->task_reductions.quick_push (TREE_CODE (sz) == INTEGER_CST
9345 1213 : ? sz : NULL_TREE);
9346 1173 : sz = force_gimple_operand (sz, &seq, true, NULL_TREE);
9347 1173 : gimple_seq_add_seq (start, seq);
9348 1173 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_one_node,
9349 : NULL_TREE, NULL_TREE);
9350 1173 : gimple_seq_add_stmt (start, gimple_build_assign (r, sz));
9351 1173 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (2),
9352 : NULL_TREE, NULL_TREE);
9353 1173 : t = build_int_cst (pointer_sized_int_node,
9354 1173 : MAX (TYPE_ALIGN_UNIT (record_type), (unsigned) cachesz));
9355 1173 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9356 1173 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (3),
9357 : NULL_TREE, NULL_TREE);
9358 1173 : t = build_int_cst (pointer_sized_int_node, -1);
9359 1173 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9360 1173 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (4),
9361 : NULL_TREE, NULL_TREE);
9362 1173 : t = build_int_cst (pointer_sized_int_node, 0);
9363 1173 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9364 :
9365 : /* In end, build a loop that iterates from 0 to < omp_get_num_threads ()
9366 : and for each task reduction checks a bool right after the private variable
9367 : within that thread's chunk; if the bool is clear, it hasn't been
9368 : initialized and thus isn't going to be reduced nor destructed, otherwise
9369 : reduce and destruct it. */
9370 1173 : tree idx = create_tmp_var (size_type_node);
9371 1173 : gimple_seq_add_stmt (end, gimple_build_assign (idx, size_zero_node));
9372 1173 : tree num_thr_sz = create_tmp_var (size_type_node);
9373 1173 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
9374 1173 : tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
9375 1173 : tree lab3 = NULL_TREE, lab7 = NULL_TREE;
9376 1173 : gimple *g;
9377 1173 : if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE)
9378 : {
9379 : /* For worksharing constructs or scope, only perform it in the master
9380 : thread, with the exception of cancelled implicit barriers - then only
9381 : handle the current thread. */
9382 289 : tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
9383 289 : t = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
9384 289 : tree thr_num = create_tmp_var (integer_type_node);
9385 289 : g = gimple_build_call (t, 0);
9386 289 : gimple_call_set_lhs (g, thr_num);
9387 289 : gimple_seq_add_stmt (end, g);
9388 289 : if (cancellable)
9389 : {
9390 14 : tree c;
9391 14 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9392 14 : tree lab6 = create_artificial_label (UNKNOWN_LOCATION);
9393 14 : lab3 = create_artificial_label (UNKNOWN_LOCATION);
9394 14 : if (code == OMP_FOR)
9395 4 : c = gimple_omp_for_clauses (ctx->stmt);
9396 10 : else if (code == OMP_SECTIONS)
9397 0 : c = gimple_omp_sections_clauses (ctx->stmt);
9398 : else /* if (code == OMP_SCOPE) */
9399 10 : c = gimple_omp_scope_clauses (ctx->stmt);
9400 14 : c = OMP_CLAUSE_DECL (omp_find_clause (c, OMP_CLAUSE__REDUCTEMP_));
9401 14 : cancellable = c;
9402 14 : g = gimple_build_cond (NE_EXPR, c, build_zero_cst (TREE_TYPE (c)),
9403 : lab5, lab6);
9404 14 : gimple_seq_add_stmt (end, g);
9405 14 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9406 14 : g = gimple_build_assign (idx, NOP_EXPR, thr_num);
9407 14 : gimple_seq_add_stmt (end, g);
9408 14 : g = gimple_build_assign (num_thr_sz, PLUS_EXPR, idx,
9409 14 : build_one_cst (TREE_TYPE (idx)));
9410 14 : gimple_seq_add_stmt (end, g);
9411 14 : gimple_seq_add_stmt (end, gimple_build_goto (lab3));
9412 14 : gimple_seq_add_stmt (end, gimple_build_label (lab6));
9413 : }
9414 289 : g = gimple_build_cond (NE_EXPR, thr_num, integer_zero_node, lab2, lab4);
9415 289 : gimple_seq_add_stmt (end, g);
9416 289 : gimple_seq_add_stmt (end, gimple_build_label (lab4));
9417 : }
9418 1173 : if (code != OMP_PARALLEL)
9419 : {
9420 1113 : t = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
9421 1113 : tree num_thr = create_tmp_var (integer_type_node);
9422 1113 : g = gimple_build_call (t, 0);
9423 1113 : gimple_call_set_lhs (g, num_thr);
9424 1113 : gimple_seq_add_stmt (end, g);
9425 1113 : g = gimple_build_assign (num_thr_sz, NOP_EXPR, num_thr);
9426 1113 : gimple_seq_add_stmt (end, g);
9427 1113 : if (cancellable)
9428 14 : gimple_seq_add_stmt (end, gimple_build_label (lab3));
9429 : }
9430 : else
9431 : {
9432 60 : tree c = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
9433 : OMP_CLAUSE__REDUCTEMP_);
9434 60 : t = fold_convert (pointer_sized_int_node, OMP_CLAUSE_DECL (c));
9435 60 : t = fold_convert (size_type_node, t);
9436 60 : gimplify_assign (num_thr_sz, t, end);
9437 : }
9438 1173 : t = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (2),
9439 : NULL_TREE, NULL_TREE);
9440 1173 : tree data = create_tmp_var (pointer_sized_int_node);
9441 1173 : gimple_seq_add_stmt (end, gimple_build_assign (data, t));
9442 1173 : if (code == OMP_TASKLOOP)
9443 : {
9444 497 : lab7 = create_artificial_label (UNKNOWN_LOCATION);
9445 497 : g = gimple_build_cond (NE_EXPR, data,
9446 : build_zero_cst (pointer_sized_int_node),
9447 : lab1, lab7);
9448 497 : gimple_seq_add_stmt (end, g);
9449 : }
9450 1173 : gimple_seq_add_stmt (end, gimple_build_label (lab1));
9451 1173 : tree ptr;
9452 1173 : if (TREE_CODE (TYPE_SIZE_UNIT (record_type)) == INTEGER_CST)
9453 1133 : ptr = create_tmp_var (build_pointer_type (record_type));
9454 : else
9455 40 : ptr = create_tmp_var (ptr_type_node);
9456 1173 : gimple_seq_add_stmt (end, gimple_build_assign (ptr, NOP_EXPR, data));
9457 :
9458 1173 : tree field = TYPE_FIELDS (record_type);
9459 1173 : cnt = 0;
9460 1173 : if (cancellable)
9461 14 : field = DECL_CHAIN (DECL_CHAIN (field));
9462 3519 : for (int pass = 0; pass < 2; pass++)
9463 : {
9464 2346 : tree decl, type, next;
9465 2346 : for (tree c = clauses;
9466 4000 : omp_task_reduction_iterate (pass, code, ccode,
9467 1654 : &c, &decl, &type, &next); c = next)
9468 : {
9469 1654 : tree var = decl, ref;
9470 1654 : if (TREE_CODE (decl) == MEM_REF)
9471 : {
9472 485 : var = TREE_OPERAND (var, 0);
9473 485 : if (TREE_CODE (var) == POINTER_PLUS_EXPR)
9474 108 : var = TREE_OPERAND (var, 0);
9475 485 : tree v = var;
9476 485 : if (TREE_CODE (var) == ADDR_EXPR)
9477 310 : var = TREE_OPERAND (var, 0);
9478 175 : else if (INDIRECT_REF_P (var))
9479 4 : var = TREE_OPERAND (var, 0);
9480 485 : tree orig_var = var;
9481 485 : if (is_variable_sized (var))
9482 : {
9483 31 : gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
9484 31 : var = DECL_VALUE_EXPR (var);
9485 31 : gcc_assert (INDIRECT_REF_P (var));
9486 31 : var = TREE_OPERAND (var, 0);
9487 31 : gcc_assert (DECL_P (var));
9488 : }
9489 485 : t = ref = maybe_lookup_decl_in_outer_ctx (var, ctx);
9490 485 : if (orig_var != var)
9491 31 : gcc_assert (TREE_CODE (v) == ADDR_EXPR);
9492 454 : else if (TREE_CODE (v) == ADDR_EXPR)
9493 279 : t = build_fold_addr_expr (t);
9494 175 : else if (INDIRECT_REF_P (v))
9495 4 : t = build_fold_indirect_ref (t);
9496 485 : if (TREE_CODE (TREE_OPERAND (decl, 0)) == POINTER_PLUS_EXPR)
9497 : {
9498 108 : tree b = TREE_OPERAND (TREE_OPERAND (decl, 0), 1);
9499 108 : b = maybe_lookup_decl_in_outer_ctx (b, ctx);
9500 108 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t, b);
9501 : }
9502 485 : if (!integer_zerop (TREE_OPERAND (decl, 1)))
9503 219 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t,
9504 : fold_convert (size_type_node,
9505 : TREE_OPERAND (decl, 1)));
9506 : }
9507 : else
9508 : {
9509 1169 : t = ref = maybe_lookup_decl_in_outer_ctx (var, ctx);
9510 1169 : if (!omp_privatize_by_reference (decl))
9511 1131 : t = build_fold_addr_expr (t);
9512 : }
9513 1654 : t = fold_convert (pointer_sized_int_node, t);
9514 1654 : seq = NULL;
9515 1654 : t = force_gimple_operand (t, &seq, true, NULL_TREE);
9516 1654 : gimple_seq_add_seq (start, seq);
9517 1654 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar,
9518 1654 : size_int (7 + cnt * 3), NULL_TREE, NULL_TREE);
9519 1654 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9520 1654 : t = unshare_expr (byte_position (field));
9521 1654 : t = fold_convert (pointer_sized_int_node, t);
9522 1654 : ctx->task_reduction_map->put (c, cnt);
9523 3308 : ctx->task_reductions.quick_push (TREE_CODE (t) == INTEGER_CST
9524 2026 : ? t : NULL_TREE);
9525 1654 : seq = NULL;
9526 1654 : t = force_gimple_operand (t, &seq, true, NULL_TREE);
9527 1654 : gimple_seq_add_seq (start, seq);
9528 1654 : r = build4 (ARRAY_REF, pointer_sized_int_node, avar,
9529 1654 : size_int (7 + cnt * 3 + 1), NULL_TREE, NULL_TREE);
9530 1654 : gimple_seq_add_stmt (start, gimple_build_assign (r, t));
9531 :
9532 1654 : tree bfield = DECL_CHAIN (field);
9533 1654 : tree cond;
9534 1654 : if (code == OMP_PARALLEL
9535 1654 : || code == OMP_FOR
9536 : || code == OMP_SECTIONS
9537 : || code == OMP_SCOPE)
9538 : /* In parallel, worksharing or scope all threads unconditionally
9539 : initialize all their task reduction private variables. */
9540 556 : cond = boolean_true_node;
9541 1098 : else if (TREE_TYPE (ptr) == ptr_type_node)
9542 : {
9543 261 : cond = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr,
9544 : unshare_expr (byte_position (bfield)));
9545 261 : seq = NULL;
9546 261 : cond = force_gimple_operand (cond, &seq, true, NULL_TREE);
9547 261 : gimple_seq_add_seq (end, seq);
9548 261 : tree pbool = build_pointer_type (TREE_TYPE (bfield));
9549 261 : cond = build2 (MEM_REF, TREE_TYPE (bfield), cond,
9550 : build_int_cst (pbool, 0));
9551 : }
9552 : else
9553 837 : cond = build3 (COMPONENT_REF, TREE_TYPE (bfield),
9554 : build_simple_mem_ref (ptr), bfield, NULL_TREE);
9555 1654 : tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
9556 1654 : tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
9557 1654 : tree condv = create_tmp_var (boolean_type_node);
9558 1654 : gimple_seq_add_stmt (end, gimple_build_assign (condv, cond));
9559 1654 : g = gimple_build_cond (NE_EXPR, condv, boolean_false_node,
9560 : lab3, lab4);
9561 1654 : gimple_seq_add_stmt (end, g);
9562 1654 : gimple_seq_add_stmt (end, gimple_build_label (lab3));
9563 1654 : if (cancellable && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) == NULL_TREE)
9564 : {
9565 : /* If this reduction doesn't need destruction and parallel
9566 : has been cancelled, there is nothing to do for this
9567 : reduction, so jump around the merge operation. */
9568 18 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9569 18 : g = gimple_build_cond (NE_EXPR, cancellable,
9570 18 : build_zero_cst (TREE_TYPE (cancellable)),
9571 : lab4, lab5);
9572 18 : gimple_seq_add_stmt (end, g);
9573 18 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9574 : }
9575 :
9576 1654 : tree new_var;
9577 1654 : if (TREE_TYPE (ptr) == ptr_type_node)
9578 : {
9579 428 : new_var = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr,
9580 : unshare_expr (byte_position (field)));
9581 428 : seq = NULL;
9582 428 : new_var = force_gimple_operand (new_var, &seq, true, NULL_TREE);
9583 428 : gimple_seq_add_seq (end, seq);
9584 428 : tree pbool = build_pointer_type (TREE_TYPE (field));
9585 428 : new_var = build2 (MEM_REF, TREE_TYPE (field), new_var,
9586 : build_int_cst (pbool, 0));
9587 : }
9588 : else
9589 1226 : new_var = build3 (COMPONENT_REF, TREE_TYPE (field),
9590 : build_simple_mem_ref (ptr), field, NULL_TREE);
9591 :
9592 1654 : enum tree_code rcode = OMP_CLAUSE_REDUCTION_CODE (c);
9593 1654 : if (TREE_CODE (decl) != MEM_REF
9594 1654 : && omp_privatize_by_reference (decl))
9595 38 : ref = build_simple_mem_ref (ref);
9596 : /* reduction(-:var) sums up the partial results, so it acts
9597 : identically to reduction(+:var). */
9598 1654 : if (rcode == MINUS_EXPR)
9599 0 : rcode = PLUS_EXPR;
9600 1654 : if (TREE_CODE (decl) == MEM_REF)
9601 : {
9602 485 : tree type = TREE_TYPE (new_var);
9603 485 : tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
9604 485 : tree i = create_tmp_var (TREE_TYPE (v));
9605 485 : tree ptype = build_pointer_type (TREE_TYPE (type));
9606 485 : if (DECL_P (v))
9607 : {
9608 142 : v = maybe_lookup_decl_in_outer_ctx (v, ctx);
9609 142 : tree vv = create_tmp_var (TREE_TYPE (v));
9610 142 : gimplify_assign (vv, v, start);
9611 142 : v = vv;
9612 : }
9613 485 : ref = build4 (ARRAY_REF, pointer_sized_int_node, avar,
9614 485 : size_int (7 + cnt * 3), NULL_TREE, NULL_TREE);
9615 485 : new_var = build_fold_addr_expr (new_var);
9616 485 : new_var = fold_convert (ptype, new_var);
9617 485 : ref = fold_convert (ptype, ref);
9618 485 : tree m = create_tmp_var (ptype);
9619 485 : gimplify_assign (m, new_var, end);
9620 485 : new_var = m;
9621 485 : m = create_tmp_var (ptype);
9622 485 : gimplify_assign (m, ref, end);
9623 485 : ref = m;
9624 485 : gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), end);
9625 485 : tree body = create_artificial_label (UNKNOWN_LOCATION);
9626 485 : tree endl = create_artificial_label (UNKNOWN_LOCATION);
9627 485 : gimple_seq_add_stmt (end, gimple_build_label (body));
9628 485 : tree priv = build_simple_mem_ref (new_var);
9629 485 : tree out = build_simple_mem_ref (ref);
9630 485 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
9631 : {
9632 161 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
9633 161 : tree decl_placeholder
9634 161 : = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
9635 161 : tree lab6 = NULL_TREE;
9636 161 : if (cancellable)
9637 : {
9638 : /* If this reduction needs destruction and parallel
9639 : has been cancelled, jump around the merge operation
9640 : to the destruction. */
9641 4 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9642 4 : lab6 = create_artificial_label (UNKNOWN_LOCATION);
9643 4 : tree zero = build_zero_cst (TREE_TYPE (cancellable));
9644 4 : g = gimple_build_cond (NE_EXPR, cancellable, zero,
9645 : lab6, lab5);
9646 4 : gimple_seq_add_stmt (end, g);
9647 4 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9648 : }
9649 161 : SET_DECL_VALUE_EXPR (placeholder, out);
9650 161 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
9651 161 : SET_DECL_VALUE_EXPR (decl_placeholder, priv);
9652 161 : DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
9653 161 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
9654 322 : gimple_seq_add_seq (end,
9655 161 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
9656 161 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
9657 161 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
9658 : {
9659 56 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
9660 56 : OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c) = NULL;
9661 : }
9662 161 : if (cancellable)
9663 4 : gimple_seq_add_stmt (end, gimple_build_label (lab6));
9664 161 : tree x = lang_hooks.decls.omp_clause_dtor (c, priv);
9665 161 : if (x)
9666 : {
9667 137 : gimple_seq tseq = NULL;
9668 137 : gimplify_stmt (&x, &tseq);
9669 137 : gimple_seq_add_seq (end, tseq);
9670 : }
9671 : }
9672 : else
9673 : {
9674 324 : tree x = build2 (rcode, TREE_TYPE (out), out, priv);
9675 324 : out = unshare_expr (out);
9676 324 : gimplify_assign (out, x, end);
9677 : }
9678 485 : gimple *g
9679 485 : = gimple_build_assign (new_var, POINTER_PLUS_EXPR, new_var,
9680 485 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
9681 485 : gimple_seq_add_stmt (end, g);
9682 485 : g = gimple_build_assign (ref, POINTER_PLUS_EXPR, ref,
9683 485 : TYPE_SIZE_UNIT (TREE_TYPE (type)));
9684 485 : gimple_seq_add_stmt (end, g);
9685 485 : g = gimple_build_assign (i, PLUS_EXPR, i,
9686 485 : build_int_cst (TREE_TYPE (i), 1));
9687 485 : gimple_seq_add_stmt (end, g);
9688 485 : g = gimple_build_cond (LE_EXPR, i, v, body, endl);
9689 485 : gimple_seq_add_stmt (end, g);
9690 485 : gimple_seq_add_stmt (end, gimple_build_label (endl));
9691 : }
9692 1169 : else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
9693 : {
9694 127 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
9695 127 : tree oldv = NULL_TREE;
9696 127 : tree lab6 = NULL_TREE;
9697 127 : if (cancellable)
9698 : {
9699 : /* If this reduction needs destruction and parallel
9700 : has been cancelled, jump around the merge operation
9701 : to the destruction. */
9702 4 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
9703 4 : lab6 = create_artificial_label (UNKNOWN_LOCATION);
9704 4 : tree zero = build_zero_cst (TREE_TYPE (cancellable));
9705 4 : g = gimple_build_cond (NE_EXPR, cancellable, zero,
9706 : lab6, lab5);
9707 4 : gimple_seq_add_stmt (end, g);
9708 4 : gimple_seq_add_stmt (end, gimple_build_label (lab5));
9709 : }
9710 127 : if (omp_privatize_by_reference (decl)
9711 137 : && !useless_type_conversion_p (TREE_TYPE (placeholder),
9712 10 : TREE_TYPE (ref)))
9713 0 : ref = build_fold_addr_expr_loc (OMP_CLAUSE_LOCATION (c), ref);
9714 127 : ref = build_fold_addr_expr_loc (OMP_CLAUSE_LOCATION (c), ref);
9715 127 : tree refv = create_tmp_var (TREE_TYPE (ref));
9716 127 : gimplify_assign (refv, ref, end);
9717 127 : ref = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), refv);
9718 127 : SET_DECL_VALUE_EXPR (placeholder, ref);
9719 127 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
9720 127 : tree d = maybe_lookup_decl (decl, ctx);
9721 127 : gcc_assert (d);
9722 127 : if (DECL_HAS_VALUE_EXPR_P (d))
9723 7 : oldv = DECL_VALUE_EXPR (d);
9724 127 : if (omp_privatize_by_reference (var))
9725 : {
9726 10 : tree v = fold_convert (TREE_TYPE (d),
9727 : build_fold_addr_expr (new_var));
9728 10 : SET_DECL_VALUE_EXPR (d, v);
9729 : }
9730 : else
9731 117 : SET_DECL_VALUE_EXPR (d, new_var);
9732 127 : DECL_HAS_VALUE_EXPR_P (d) = 1;
9733 127 : lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
9734 127 : if (oldv)
9735 7 : SET_DECL_VALUE_EXPR (d, oldv);
9736 : else
9737 : {
9738 120 : SET_DECL_VALUE_EXPR (d, NULL_TREE);
9739 120 : DECL_HAS_VALUE_EXPR_P (d) = 0;
9740 : }
9741 127 : gimple_seq_add_seq (end, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
9742 127 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
9743 127 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
9744 24 : OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
9745 127 : if (cancellable)
9746 4 : gimple_seq_add_stmt (end, gimple_build_label (lab6));
9747 127 : tree x = lang_hooks.decls.omp_clause_dtor (c, new_var);
9748 127 : if (x)
9749 : {
9750 29 : gimple_seq tseq = NULL;
9751 29 : gimplify_stmt (&x, &tseq);
9752 29 : gimple_seq_add_seq (end, tseq);
9753 : }
9754 : }
9755 : else
9756 : {
9757 1042 : tree x = build2 (rcode, TREE_TYPE (ref), ref, new_var);
9758 1042 : ref = unshare_expr (ref);
9759 1042 : gimplify_assign (ref, x, end);
9760 : }
9761 1654 : gimple_seq_add_stmt (end, gimple_build_label (lab4));
9762 1654 : ++cnt;
9763 1654 : field = DECL_CHAIN (bfield);
9764 : }
9765 : }
9766 :
9767 1173 : if (code == OMP_TASKGROUP)
9768 : {
9769 327 : t = builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_REDUCTION_REGISTER);
9770 327 : g = gimple_build_call (t, 1, build_fold_addr_expr (avar));
9771 327 : gimple_seq_add_stmt (start, g);
9772 : }
9773 : else
9774 : {
9775 846 : tree c;
9776 846 : if (code == OMP_FOR)
9777 227 : c = gimple_omp_for_clauses (ctx->stmt);
9778 619 : else if (code == OMP_SECTIONS)
9779 8 : c = gimple_omp_sections_clauses (ctx->stmt);
9780 611 : else if (code == OMP_SCOPE)
9781 54 : c = gimple_omp_scope_clauses (ctx->stmt);
9782 : else
9783 557 : c = gimple_omp_taskreg_clauses (ctx->stmt);
9784 846 : c = omp_find_clause (c, OMP_CLAUSE__REDUCTEMP_);
9785 846 : t = fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (c)),
9786 : build_fold_addr_expr (avar));
9787 846 : gimplify_assign (OMP_CLAUSE_DECL (c), t, start);
9788 : }
9789 :
9790 1173 : gimple_seq_add_stmt (end, gimple_build_assign (data, PLUS_EXPR, data, sz));
9791 1173 : gimple_seq_add_stmt (end, gimple_build_assign (idx, PLUS_EXPR, idx,
9792 : size_one_node));
9793 1173 : g = gimple_build_cond (NE_EXPR, idx, num_thr_sz, lab1, lab2);
9794 1173 : gimple_seq_add_stmt (end, g);
9795 1173 : gimple_seq_add_stmt (end, gimple_build_label (lab2));
9796 1173 : if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE)
9797 : {
9798 289 : enum built_in_function bfn
9799 : = BUILT_IN_GOMP_WORKSHARE_TASK_REDUCTION_UNREGISTER;
9800 289 : t = builtin_decl_explicit (bfn);
9801 289 : tree c_bool_type = TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (t)));
9802 289 : tree arg;
9803 289 : if (cancellable)
9804 : {
9805 14 : arg = create_tmp_var (c_bool_type);
9806 14 : gimple_seq_add_stmt (end, gimple_build_assign (arg, NOP_EXPR,
9807 : cancellable));
9808 : }
9809 : else
9810 275 : arg = build_int_cst (c_bool_type, 0);
9811 289 : g = gimple_build_call (t, 1, arg);
9812 289 : }
9813 : else
9814 : {
9815 884 : t = builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_REDUCTION_UNREGISTER);
9816 884 : g = gimple_build_call (t, 1, build_fold_addr_expr (avar));
9817 : }
9818 1173 : gimple_seq_add_stmt (end, g);
9819 1173 : if (lab7)
9820 497 : gimple_seq_add_stmt (end, gimple_build_label (lab7));
9821 1173 : t = build_constructor (atype, NULL);
9822 1173 : TREE_THIS_VOLATILE (t) = 1;
9823 1173 : gimple_seq_add_stmt (end, gimple_build_assign (avar, t));
9824 : }
9825 :
9826 : /* Expand code for an OpenMP taskgroup directive. */
9827 :
9828 : static void
9829 536 : lower_omp_taskgroup (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9830 : {
9831 536 : gimple *stmt = gsi_stmt (*gsi_p);
9832 536 : gcall *x;
9833 536 : gbind *bind;
9834 536 : gimple_seq dseq = NULL;
9835 536 : tree block = make_node (BLOCK);
9836 :
9837 536 : bind = gimple_build_bind (NULL, NULL, block);
9838 536 : gsi_replace (gsi_p, bind, true);
9839 536 : gimple_bind_add_stmt (bind, stmt);
9840 :
9841 536 : push_gimplify_context ();
9842 :
9843 536 : x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START),
9844 : 0);
9845 536 : gimple_bind_add_stmt (bind, x);
9846 :
9847 536 : lower_omp_task_reductions (ctx, OMP_TASKGROUP,
9848 : gimple_omp_taskgroup_clauses (stmt),
9849 : gimple_bind_body_ptr (bind), &dseq);
9850 :
9851 536 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
9852 536 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
9853 536 : gimple_omp_set_body (stmt, NULL);
9854 :
9855 536 : gimple_bind_add_seq (bind, dseq);
9856 :
9857 536 : pop_gimplify_context (bind);
9858 :
9859 536 : gimple_bind_append_vars (bind, ctx->block_vars);
9860 536 : BLOCK_VARS (block) = ctx->block_vars;
9861 536 : }
9862 :
9863 :
9864 : /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */
9865 :
9866 : static void
9867 0 : lower_omp_ordered_clauses (gimple_stmt_iterator *gsi_p, gomp_ordered *ord_stmt,
9868 : omp_context *ctx)
9869 : {
9870 0 : struct omp_for_data fd;
9871 0 : if (!ctx->outer || gimple_code (ctx->outer->stmt) != GIMPLE_OMP_FOR)
9872 0 : return;
9873 :
9874 0 : unsigned int len = gimple_omp_for_collapse (ctx->outer->stmt);
9875 0 : struct omp_for_data_loop *loops = XALLOCAVEC (struct omp_for_data_loop, len);
9876 0 : omp_extract_for_data (as_a <gomp_for *> (ctx->outer->stmt), &fd, loops);
9877 0 : if (!fd.ordered)
9878 : return;
9879 :
9880 0 : tree *list_p = gimple_omp_ordered_clauses_ptr (ord_stmt);
9881 0 : tree c = gimple_omp_ordered_clauses (ord_stmt);
9882 0 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS
9883 0 : && OMP_CLAUSE_DOACROSS_KIND (c) == OMP_CLAUSE_DOACROSS_SINK)
9884 : {
9885 : /* Merge depend clauses from multiple adjacent
9886 : #pragma omp ordered depend(sink:...) constructs
9887 : into one #pragma omp ordered depend(sink:...), so that
9888 : we can optimize them together. */
9889 0 : gimple_stmt_iterator gsi = *gsi_p;
9890 0 : gsi_next (&gsi);
9891 0 : while (!gsi_end_p (gsi))
9892 : {
9893 0 : gimple *stmt = gsi_stmt (gsi);
9894 0 : if (is_gimple_debug (stmt)
9895 0 : || gimple_code (stmt) == GIMPLE_NOP)
9896 : {
9897 0 : gsi_next (&gsi);
9898 0 : continue;
9899 : }
9900 0 : if (gimple_code (stmt) != GIMPLE_OMP_ORDERED)
9901 : break;
9902 0 : gomp_ordered *ord_stmt2 = as_a <gomp_ordered *> (stmt);
9903 0 : c = gimple_omp_ordered_clauses (ord_stmt2);
9904 0 : if (c == NULL_TREE
9905 0 : || OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DOACROSS
9906 0 : || OMP_CLAUSE_DOACROSS_KIND (c) != OMP_CLAUSE_DOACROSS_SINK)
9907 : break;
9908 0 : while (*list_p)
9909 0 : list_p = &OMP_CLAUSE_CHAIN (*list_p);
9910 0 : *list_p = c;
9911 0 : gsi_remove (&gsi, true);
9912 : }
9913 : }
9914 :
9915 : /* Canonicalize sink dependence clauses into one folded clause if
9916 : possible.
9917 :
9918 : The basic algorithm is to create a sink vector whose first
9919 : element is the GCD of all the first elements, and whose remaining
9920 : elements are the minimum of the subsequent columns.
9921 :
9922 : We ignore dependence vectors whose first element is zero because
9923 : such dependencies are known to be executed by the same thread.
9924 :
9925 : We take into account the direction of the loop, so a minimum
9926 : becomes a maximum if the loop is iterating forwards. We also
9927 : ignore sink clauses where the loop direction is unknown, or where
9928 : the offsets are clearly invalid because they are not a multiple
9929 : of the loop increment.
9930 :
9931 : For example:
9932 :
9933 : #pragma omp for ordered(2)
9934 : for (i=0; i < N; ++i)
9935 : for (j=0; j < M; ++j)
9936 : {
9937 : #pragma omp ordered \
9938 : depend(sink:i-8,j-2) \
9939 : depend(sink:i,j-1) \ // Completely ignored because i+0.
9940 : depend(sink:i-4,j-3) \
9941 : depend(sink:i-6,j-4)
9942 : #pragma omp ordered depend(source)
9943 : }
9944 :
9945 : Folded clause is:
9946 :
9947 : depend(sink:-gcd(8,4,6),-min(2,3,4))
9948 : -or-
9949 : depend(sink:-2,-2)
9950 : */
9951 :
9952 : /* FIXME: Computing GCD's where the first element is zero is
9953 : non-trivial in the presence of collapsed loops. Do this later. */
9954 0 : if (fd.collapse > 1)
9955 : return;
9956 :
9957 0 : wide_int *folded_deps = XALLOCAVEC (wide_int, 2 * len - 1);
9958 :
9959 : /* wide_int is not a POD so it must be default-constructed. */
9960 0 : for (unsigned i = 0; i != 2 * len - 1; ++i)
9961 0 : new (static_cast<void*>(folded_deps + i)) wide_int ();
9962 :
9963 : tree folded_dep = NULL_TREE;
9964 : /* TRUE if the first dimension's offset is negative. */
9965 : bool neg_offset_p = false;
9966 :
9967 : list_p = gimple_omp_ordered_clauses_ptr (ord_stmt);
9968 : unsigned int i;
9969 0 : while ((c = *list_p) != NULL)
9970 : {
9971 0 : bool remove = false;
9972 :
9973 0 : gcc_assert (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS);
9974 0 : if (OMP_CLAUSE_DOACROSS_KIND (c) != OMP_CLAUSE_DOACROSS_SINK)
9975 0 : goto next_ordered_clause;
9976 :
9977 0 : tree vec;
9978 0 : for (vec = OMP_CLAUSE_DECL (c), i = 0;
9979 0 : vec && TREE_CODE (vec) == TREE_LIST;
9980 0 : vec = TREE_CHAIN (vec), ++i)
9981 : {
9982 0 : gcc_assert (i < len);
9983 :
9984 : /* omp_extract_for_data has canonicalized the condition. */
9985 0 : gcc_assert (fd.loops[i].cond_code == LT_EXPR
9986 : || fd.loops[i].cond_code == GT_EXPR);
9987 0 : bool forward = fd.loops[i].cond_code == LT_EXPR;
9988 0 : bool maybe_lexically_later = true;
9989 :
9990 : /* While the committee makes up its mind, bail if we have any
9991 : non-constant steps. */
9992 0 : if (TREE_CODE (fd.loops[i].step) != INTEGER_CST)
9993 0 : goto lower_omp_ordered_ret;
9994 :
9995 0 : tree itype = TREE_TYPE (TREE_VALUE (vec));
9996 0 : if (POINTER_TYPE_P (itype))
9997 0 : itype = sizetype;
9998 0 : wide_int offset = wide_int::from (wi::to_wide (TREE_PURPOSE (vec)),
9999 0 : TYPE_PRECISION (itype),
10000 0 : TYPE_SIGN (itype));
10001 :
10002 : /* Ignore invalid offsets that are not multiples of the step. */
10003 0 : if (!wi::multiple_of_p (wi::abs (offset),
10004 0 : wi::abs (wi::to_wide (fd.loops[i].step)),
10005 : UNSIGNED))
10006 : {
10007 0 : warning_at (OMP_CLAUSE_LOCATION (c), OPT_Wopenmp,
10008 : "ignoring %<sink%> clause with offset that is not "
10009 : "a multiple of the loop step");
10010 0 : remove = true;
10011 0 : goto next_ordered_clause;
10012 : }
10013 :
10014 : /* Calculate the first dimension. The first dimension of
10015 : the folded dependency vector is the GCD of the first
10016 : elements, while ignoring any first elements whose offset
10017 : is 0. */
10018 0 : if (i == 0)
10019 : {
10020 : /* Ignore dependence vectors whose first dimension is 0. */
10021 0 : if (offset == 0)
10022 : {
10023 0 : remove = true;
10024 0 : goto next_ordered_clause;
10025 : }
10026 : else
10027 : {
10028 0 : if (!TYPE_UNSIGNED (itype) && (forward ^ wi::neg_p (offset)))
10029 : {
10030 0 : error_at (OMP_CLAUSE_LOCATION (c),
10031 : "first offset must be in opposite direction "
10032 : "of loop iterations");
10033 0 : goto lower_omp_ordered_ret;
10034 : }
10035 0 : if (forward)
10036 0 : offset = -offset;
10037 0 : neg_offset_p = forward;
10038 : /* Initialize the first time around. */
10039 0 : if (folded_dep == NULL_TREE)
10040 : {
10041 0 : folded_dep = c;
10042 0 : folded_deps[0] = offset;
10043 : }
10044 : else
10045 0 : folded_deps[0] = wi::gcd (folded_deps[0],
10046 0 : offset, UNSIGNED);
10047 : }
10048 : }
10049 : /* Calculate minimum for the remaining dimensions. */
10050 : else
10051 : {
10052 0 : folded_deps[len + i - 1] = offset;
10053 0 : if (folded_dep == c)
10054 0 : folded_deps[i] = offset;
10055 0 : else if (maybe_lexically_later
10056 0 : && !wi::eq_p (folded_deps[i], offset))
10057 : {
10058 0 : if (forward ^ wi::gts_p (folded_deps[i], offset))
10059 : {
10060 : unsigned int j;
10061 0 : folded_dep = c;
10062 0 : for (j = 1; j <= i; j++)
10063 0 : folded_deps[j] = folded_deps[len + j - 1];
10064 : }
10065 : else
10066 0 : maybe_lexically_later = false;
10067 : }
10068 : }
10069 0 : }
10070 0 : gcc_assert (i == len);
10071 :
10072 : remove = true;
10073 :
10074 0 : next_ordered_clause:
10075 0 : if (remove)
10076 0 : *list_p = OMP_CLAUSE_CHAIN (c);
10077 : else
10078 0 : list_p = &OMP_CLAUSE_CHAIN (c);
10079 : }
10080 :
10081 0 : if (folded_dep)
10082 : {
10083 0 : if (neg_offset_p)
10084 0 : folded_deps[0] = -folded_deps[0];
10085 :
10086 0 : tree itype = TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep)));
10087 0 : if (POINTER_TYPE_P (itype))
10088 0 : itype = sizetype;
10089 :
10090 0 : TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep))
10091 0 : = wide_int_to_tree (itype, folded_deps[0]);
10092 0 : OMP_CLAUSE_CHAIN (folded_dep) = gimple_omp_ordered_clauses (ord_stmt);
10093 0 : *gimple_omp_ordered_clauses_ptr (ord_stmt) = folded_dep;
10094 : }
10095 :
10096 0 : lower_omp_ordered_ret:
10097 :
10098 : /* Ordered without clauses is #pragma omp threads, while we want
10099 : a nop instead if we remove all clauses. */
10100 0 : if (gimple_omp_ordered_clauses (ord_stmt) == NULL_TREE)
10101 0 : gsi_replace (gsi_p, gimple_build_nop (), true);
10102 : }
10103 :
10104 :
10105 : /* Expand code for an OpenMP ordered directive. */
10106 :
10107 : static void
10108 1124 : lower_omp_ordered (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10109 : {
10110 1124 : tree block;
10111 1124 : gimple *stmt = gsi_stmt (*gsi_p), *g;
10112 1124 : gomp_ordered *ord_stmt = as_a <gomp_ordered *> (stmt);
10113 1124 : gcall *x;
10114 1124 : gbind *bind;
10115 1124 : bool simd = omp_find_clause (gimple_omp_ordered_clauses (ord_stmt),
10116 1124 : OMP_CLAUSE_SIMD);
10117 : /* FIXME: this should check presence of OMP_CLAUSE__SIMT_ on the enclosing
10118 : loop. */
10119 1124 : bool maybe_simt
10120 1124 : = simd && omp_maybe_offloaded_ctx (ctx) && omp_max_simt_vf () > 1;
10121 1124 : bool threads = omp_find_clause (gimple_omp_ordered_clauses (ord_stmt),
10122 1124 : OMP_CLAUSE_THREADS);
10123 :
10124 1124 : if (gimple_omp_ordered_standalone_p (ord_stmt))
10125 : {
10126 : /* FIXME: This is needs to be moved to the expansion to verify various
10127 : conditions only testable on cfg with dominators computed, and also
10128 : all the depend clauses to be merged still might need to be available
10129 : for the runtime checks. */
10130 : if (0)
10131 : lower_omp_ordered_clauses (gsi_p, ord_stmt, ctx);
10132 1124 : return;
10133 : }
10134 :
10135 411 : push_gimplify_context ();
10136 :
10137 411 : block = make_node (BLOCK);
10138 411 : bind = gimple_build_bind (NULL, NULL, block);
10139 411 : gsi_replace (gsi_p, bind, true);
10140 411 : gimple_bind_add_stmt (bind, stmt);
10141 :
10142 411 : if (simd)
10143 : {
10144 79 : x = gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START, 1,
10145 79 : build_int_cst (NULL_TREE, threads));
10146 79 : cfun->has_simduid_loops = true;
10147 : }
10148 : else
10149 332 : x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START),
10150 : 0);
10151 411 : gimple_bind_add_stmt (bind, x);
10152 :
10153 411 : tree counter = NULL_TREE, test = NULL_TREE, body = NULL_TREE;
10154 411 : if (maybe_simt)
10155 : {
10156 0 : counter = create_tmp_var (integer_type_node);
10157 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_LANE, 0);
10158 0 : gimple_call_set_lhs (g, counter);
10159 0 : gimple_bind_add_stmt (bind, g);
10160 :
10161 0 : body = create_artificial_label (UNKNOWN_LOCATION);
10162 0 : test = create_artificial_label (UNKNOWN_LOCATION);
10163 0 : gimple_bind_add_stmt (bind, gimple_build_label (body));
10164 :
10165 0 : tree simt_pred = create_tmp_var (integer_type_node);
10166 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_ORDERED_PRED, 1, counter);
10167 0 : gimple_call_set_lhs (g, simt_pred);
10168 0 : gimple_bind_add_stmt (bind, g);
10169 :
10170 0 : tree t = create_artificial_label (UNKNOWN_LOCATION);
10171 0 : g = gimple_build_cond (EQ_EXPR, simt_pred, integer_zero_node, t, test);
10172 0 : gimple_bind_add_stmt (bind, g);
10173 :
10174 0 : gimple_bind_add_stmt (bind, gimple_build_label (t));
10175 : }
10176 411 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
10177 411 : gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
10178 411 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10179 411 : gimple_omp_set_body (stmt, NULL);
10180 :
10181 411 : if (maybe_simt)
10182 : {
10183 0 : gimple_bind_add_stmt (bind, gimple_build_label (test));
10184 0 : g = gimple_build_assign (counter, MINUS_EXPR, counter, integer_one_node);
10185 0 : gimple_bind_add_stmt (bind, g);
10186 :
10187 0 : tree c = build2 (GE_EXPR, boolean_type_node, counter, integer_zero_node);
10188 0 : tree nonneg = create_tmp_var (integer_type_node);
10189 0 : gimple_seq tseq = NULL;
10190 0 : gimplify_assign (nonneg, fold_convert (integer_type_node, c), &tseq);
10191 0 : gimple_bind_add_seq (bind, tseq);
10192 :
10193 0 : g = gimple_build_call_internal (IFN_GOMP_SIMT_VOTE_ANY, 1, nonneg);
10194 0 : gimple_call_set_lhs (g, nonneg);
10195 0 : gimple_bind_add_stmt (bind, g);
10196 :
10197 0 : tree end = create_artificial_label (UNKNOWN_LOCATION);
10198 0 : g = gimple_build_cond (NE_EXPR, nonneg, integer_zero_node, body, end);
10199 0 : gimple_bind_add_stmt (bind, g);
10200 :
10201 0 : gimple_bind_add_stmt (bind, gimple_build_label (end));
10202 : }
10203 411 : if (simd)
10204 79 : x = gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END, 1,
10205 79 : build_int_cst (NULL_TREE, threads));
10206 : else
10207 332 : x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END),
10208 : 0);
10209 411 : gimple_bind_add_stmt (bind, x);
10210 :
10211 411 : gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10212 :
10213 411 : pop_gimplify_context (bind);
10214 :
10215 411 : gimple_bind_append_vars (bind, ctx->block_vars);
10216 411 : BLOCK_VARS (block) = gimple_bind_vars (bind);
10217 : }
10218 :
10219 :
10220 : /* Expand code for an OpenMP scan directive and the structured block
10221 : before the scan directive. */
10222 :
10223 : static void
10224 1544 : lower_omp_scan (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10225 : {
10226 1544 : gimple *stmt = gsi_stmt (*gsi_p);
10227 1544 : bool has_clauses
10228 1544 : = gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt)) != NULL;
10229 1544 : tree lane = NULL_TREE;
10230 1544 : gimple_seq before = NULL;
10231 1544 : omp_context *octx = ctx->outer;
10232 1544 : gcc_assert (octx);
10233 1544 : if (octx->scan_exclusive && !has_clauses)
10234 : {
10235 536 : gimple_stmt_iterator gsi2 = *gsi_p;
10236 536 : gsi_next (&gsi2);
10237 536 : gimple *stmt2 = gsi_stmt (gsi2);
10238 : /* For exclusive scan, swap GIMPLE_OMP_SCAN without clauses
10239 : with following GIMPLE_OMP_SCAN with clauses, so that input_phase,
10240 : the one with exclusive clause(s), comes first. */
10241 536 : if (stmt2
10242 274 : && gimple_code (stmt2) == GIMPLE_OMP_SCAN
10243 804 : && gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt2)) != NULL)
10244 : {
10245 268 : gsi_remove (gsi_p, false);
10246 268 : gsi_insert_after (gsi_p, stmt, GSI_SAME_STMT);
10247 268 : ctx = maybe_lookup_ctx (stmt2);
10248 268 : gcc_assert (ctx);
10249 268 : lower_omp_scan (gsi_p, ctx);
10250 268 : return;
10251 : }
10252 : }
10253 :
10254 1276 : bool input_phase = has_clauses ^ octx->scan_inclusive;
10255 1276 : bool is_simd = (gimple_code (octx->stmt) == GIMPLE_OMP_FOR
10256 1276 : && gimple_omp_for_kind (octx->stmt) == GF_OMP_FOR_KIND_SIMD);
10257 1276 : bool is_for = (gimple_code (octx->stmt) == GIMPLE_OMP_FOR
10258 1276 : && gimple_omp_for_kind (octx->stmt) == GF_OMP_FOR_KIND_FOR
10259 1630 : && !gimple_omp_for_combined_p (octx->stmt));
10260 1276 : bool is_for_simd = is_simd && gimple_omp_for_combined_into_p (octx->stmt);
10261 332 : if (is_for_simd && octx->for_simd_scan_phase)
10262 : is_simd = false;
10263 1110 : if (is_simd)
10264 756 : if (tree c = omp_find_clause (gimple_omp_for_clauses (octx->stmt),
10265 : OMP_CLAUSE__SIMDUID_))
10266 : {
10267 368 : tree uid = OMP_CLAUSE__SIMDUID__DECL (c);
10268 368 : lane = create_tmp_var (unsigned_type_node);
10269 368 : tree t = build_int_cst (integer_type_node,
10270 552 : input_phase ? 1
10271 184 : : octx->scan_inclusive ? 2 : 3);
10272 368 : gimple *g
10273 368 : = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 2, uid, t);
10274 368 : gimple_call_set_lhs (g, lane);
10275 368 : gimple_seq_add_stmt (&before, g);
10276 : }
10277 :
10278 1276 : if (is_simd || is_for)
10279 : {
10280 5004 : for (tree c = gimple_omp_for_clauses (octx->stmt);
10281 5004 : c; c = OMP_CLAUSE_CHAIN (c))
10282 4060 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
10283 4060 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
10284 : {
10285 1168 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
10286 1168 : tree var = OMP_CLAUSE_DECL (c);
10287 1168 : tree new_var = lookup_decl (var, octx);
10288 1168 : tree val = new_var;
10289 1168 : tree var2 = NULL_TREE;
10290 1168 : tree var3 = NULL_TREE;
10291 1168 : tree var4 = NULL_TREE;
10292 1168 : tree lane0 = NULL_TREE;
10293 1168 : tree new_vard = new_var;
10294 1168 : if (omp_privatize_by_reference (var))
10295 : {
10296 184 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
10297 184 : val = new_var;
10298 : }
10299 1168 : if (DECL_HAS_VALUE_EXPR_P (new_vard))
10300 : {
10301 464 : val = DECL_VALUE_EXPR (new_vard);
10302 464 : if (new_vard != new_var)
10303 : {
10304 76 : gcc_assert (TREE_CODE (val) == ADDR_EXPR);
10305 76 : val = TREE_OPERAND (val, 0);
10306 : }
10307 464 : if (TREE_CODE (val) == ARRAY_REF
10308 464 : && VAR_P (TREE_OPERAND (val, 0)))
10309 : {
10310 464 : tree v = TREE_OPERAND (val, 0);
10311 464 : if (lookup_attribute ("omp simd array",
10312 464 : DECL_ATTRIBUTES (v)))
10313 : {
10314 464 : val = unshare_expr (val);
10315 464 : lane0 = TREE_OPERAND (val, 1);
10316 464 : TREE_OPERAND (val, 1) = lane;
10317 464 : var2 = lookup_decl (v, octx);
10318 464 : if (octx->scan_exclusive)
10319 228 : var4 = lookup_decl (var2, octx);
10320 464 : if (input_phase
10321 464 : && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
10322 120 : var3 = maybe_lookup_decl (var4 ? var4 : var2, octx);
10323 464 : if (!input_phase)
10324 : {
10325 232 : var2 = build4 (ARRAY_REF, TREE_TYPE (val),
10326 : var2, lane, NULL_TREE, NULL_TREE);
10327 232 : TREE_THIS_NOTRAP (var2) = 1;
10328 232 : if (octx->scan_exclusive)
10329 : {
10330 114 : var4 = build4 (ARRAY_REF, TREE_TYPE (val),
10331 : var4, lane, NULL_TREE,
10332 : NULL_TREE);
10333 114 : TREE_THIS_NOTRAP (var4) = 1;
10334 : }
10335 : }
10336 : else
10337 : var2 = val;
10338 : }
10339 : }
10340 464 : gcc_assert (var2);
10341 : }
10342 : else
10343 : {
10344 704 : var2 = build_outer_var_ref (var, octx);
10345 704 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
10346 : {
10347 220 : var3 = maybe_lookup_decl (new_vard, octx);
10348 220 : if (var3 == new_vard || var3 == NULL_TREE)
10349 : var3 = NULL_TREE;
10350 108 : else if (is_simd && octx->scan_exclusive && !input_phase)
10351 : {
10352 16 : var4 = maybe_lookup_decl (var3, octx);
10353 16 : if (var4 == var3 || var4 == NULL_TREE)
10354 : {
10355 0 : if (TREE_ADDRESSABLE (TREE_TYPE (new_var)))
10356 : {
10357 : var4 = var3;
10358 : var3 = NULL_TREE;
10359 : }
10360 : else
10361 16 : var4 = NULL_TREE;
10362 : }
10363 : }
10364 : }
10365 672 : if (is_simd
10366 484 : && octx->scan_exclusive
10367 244 : && !input_phase
10368 244 : && var4 == NULL_TREE)
10369 106 : var4 = create_tmp_var (TREE_TYPE (val));
10370 : }
10371 1168 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
10372 : {
10373 376 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
10374 376 : if (input_phase)
10375 : {
10376 188 : if (var3)
10377 : {
10378 : /* If we've added a separate identity element
10379 : variable, copy it over into val. */
10380 92 : tree x = lang_hooks.decls.omp_clause_assign_op (c, val,
10381 : var3);
10382 92 : gimplify_and_add (x, &before);
10383 : }
10384 96 : else if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
10385 : {
10386 : /* Otherwise, assign to it the identity element. */
10387 96 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
10388 96 : if (is_for)
10389 16 : tseq = copy_gimple_seq_and_replace_locals (tseq);
10390 96 : tree ref = build_outer_var_ref (var, octx);
10391 96 : tree x = (DECL_HAS_VALUE_EXPR_P (new_vard)
10392 96 : ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
10393 40 : if (x)
10394 : {
10395 40 : if (new_vard != new_var)
10396 8 : val = build_fold_addr_expr_loc (clause_loc, val);
10397 40 : SET_DECL_VALUE_EXPR (new_vard, val);
10398 : }
10399 96 : SET_DECL_VALUE_EXPR (placeholder, ref);
10400 96 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
10401 96 : lower_omp (&tseq, octx);
10402 96 : if (x)
10403 40 : SET_DECL_VALUE_EXPR (new_vard, x);
10404 96 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
10405 96 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
10406 96 : gimple_seq_add_seq (&before, tseq);
10407 96 : if (is_simd)
10408 80 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
10409 : }
10410 : }
10411 188 : else if (is_simd)
10412 : {
10413 156 : tree x;
10414 156 : if (octx->scan_exclusive)
10415 : {
10416 72 : tree v4 = unshare_expr (var4);
10417 72 : tree v2 = unshare_expr (var2);
10418 72 : x = lang_hooks.decls.omp_clause_assign_op (c, v4, v2);
10419 72 : gimplify_and_add (x, &before);
10420 : }
10421 156 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
10422 156 : x = (DECL_HAS_VALUE_EXPR_P (new_vard)
10423 156 : ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
10424 156 : tree vexpr = val;
10425 156 : if (x && new_vard != new_var)
10426 30 : vexpr = build_fold_addr_expr_loc (clause_loc, val);
10427 156 : if (x)
10428 78 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
10429 156 : SET_DECL_VALUE_EXPR (placeholder, var2);
10430 156 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
10431 156 : lower_omp (&tseq, octx);
10432 156 : gimple_seq_add_seq (&before, tseq);
10433 156 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
10434 156 : if (x)
10435 78 : SET_DECL_VALUE_EXPR (new_vard, x);
10436 156 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
10437 156 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
10438 156 : if (octx->scan_inclusive)
10439 : {
10440 84 : x = lang_hooks.decls.omp_clause_assign_op (c, val,
10441 : var2);
10442 84 : gimplify_and_add (x, &before);
10443 : }
10444 72 : else if (lane0 == NULL_TREE)
10445 : {
10446 36 : x = lang_hooks.decls.omp_clause_assign_op (c, val,
10447 : var4);
10448 36 : gimplify_and_add (x, &before);
10449 : }
10450 : }
10451 : }
10452 : else
10453 : {
10454 792 : if (input_phase)
10455 : {
10456 : /* input phase. Set val to initializer before
10457 : the body. */
10458 396 : tree x = omp_reduction_init (c, TREE_TYPE (new_var));
10459 396 : gimplify_assign (val, x, &before);
10460 : }
10461 396 : else if (is_simd)
10462 : {
10463 : /* scan phase. */
10464 318 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
10465 318 : if (code == MINUS_EXPR)
10466 0 : code = PLUS_EXPR;
10467 :
10468 318 : tree x = build2 (code, TREE_TYPE (var2),
10469 : unshare_expr (var2), unshare_expr (val));
10470 318 : if (octx->scan_inclusive)
10471 : {
10472 154 : gimplify_assign (unshare_expr (var2), x, &before);
10473 154 : gimplify_assign (val, var2, &before);
10474 : }
10475 : else
10476 : {
10477 164 : gimplify_assign (unshare_expr (var4),
10478 : unshare_expr (var2), &before);
10479 164 : gimplify_assign (var2, x, &before);
10480 164 : if (lane0 == NULL_TREE)
10481 86 : gimplify_assign (val, var4, &before);
10482 : }
10483 : }
10484 : }
10485 1168 : if (octx->scan_exclusive && !input_phase && lane0)
10486 : {
10487 114 : tree vexpr = unshare_expr (var4);
10488 114 : TREE_OPERAND (vexpr, 1) = lane0;
10489 114 : if (new_vard != new_var)
10490 16 : vexpr = build_fold_addr_expr_loc (clause_loc, vexpr);
10491 114 : SET_DECL_VALUE_EXPR (new_vard, vexpr);
10492 : }
10493 : }
10494 : }
10495 1276 : if (is_simd && !is_for_simd)
10496 : {
10497 590 : gsi_insert_seq_after (gsi_p, gimple_omp_body (stmt), GSI_SAME_STMT);
10498 590 : gsi_insert_seq_after (gsi_p, before, GSI_SAME_STMT);
10499 590 : gsi_replace (gsi_p, gimple_build_nop (), true);
10500 590 : return;
10501 : }
10502 686 : lower_omp (gimple_omp_body_ptr (stmt), octx);
10503 686 : if (before)
10504 : {
10505 260 : gimple_stmt_iterator gsi = gsi_start (*gimple_omp_body_ptr (stmt));
10506 260 : gsi_insert_seq_before (&gsi, before, GSI_SAME_STMT);
10507 : }
10508 : }
10509 :
10510 :
10511 : /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
10512 : substitution of a couple of function calls. But in the NAMED case,
10513 : requires that languages coordinate a symbol name. It is therefore
10514 : best put here in common code. */
10515 :
10516 : static GTY(()) hash_map<tree, tree> *critical_name_mutexes;
10517 :
10518 : static void
10519 311 : lower_omp_critical (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10520 : {
10521 311 : tree block;
10522 311 : tree name, lock, unlock;
10523 311 : gomp_critical *stmt = as_a <gomp_critical *> (gsi_stmt (*gsi_p));
10524 311 : gbind *bind;
10525 311 : location_t loc = gimple_location (stmt);
10526 311 : gimple_seq tbody;
10527 :
10528 311 : name = gimple_omp_critical_name (stmt);
10529 311 : if (name)
10530 : {
10531 88 : tree decl;
10532 :
10533 88 : if (!critical_name_mutexes)
10534 48 : critical_name_mutexes = hash_map<tree, tree>::create_ggc (10);
10535 :
10536 88 : tree *n = critical_name_mutexes->get (name);
10537 88 : if (n == NULL)
10538 : {
10539 70 : char *new_str;
10540 :
10541 70 : decl = create_tmp_var_raw (ptr_type_node);
10542 :
10543 70 : new_str = ACONCAT ((".gomp_critical_user_",
10544 : IDENTIFIER_POINTER (name), NULL));
10545 70 : DECL_NAME (decl) = get_identifier (new_str);
10546 70 : TREE_PUBLIC (decl) = 1;
10547 70 : TREE_STATIC (decl) = 1;
10548 70 : DECL_COMMON (decl) = 1;
10549 70 : DECL_ARTIFICIAL (decl) = 1;
10550 70 : DECL_IGNORED_P (decl) = 1;
10551 :
10552 70 : varpool_node::finalize_decl (decl);
10553 :
10554 70 : critical_name_mutexes->put (name, decl);
10555 : }
10556 : else
10557 18 : decl = *n;
10558 :
10559 : /* If '#pragma omp critical' is inside offloaded region or
10560 : inside function marked as offloadable, the symbol must be
10561 : marked as offloadable too. */
10562 88 : omp_context *octx;
10563 88 : if (cgraph_node::get (current_function_decl)->offloadable)
10564 1 : varpool_node::get_create (decl)->offloadable = 1;
10565 : else
10566 173 : for (octx = ctx->outer; octx; octx = octx->outer)
10567 87 : if (is_gimple_omp_offloaded (octx->stmt))
10568 : {
10569 1 : varpool_node::get_create (decl)->offloadable = 1;
10570 1 : break;
10571 : }
10572 :
10573 88 : lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START);
10574 88 : lock = build_call_expr_loc (loc, lock, 1,
10575 : build_fold_addr_expr_loc (loc, decl));
10576 :
10577 88 : unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END);
10578 88 : unlock = build_call_expr_loc (loc, unlock, 1,
10579 : build_fold_addr_expr_loc (loc, decl));
10580 : }
10581 : else
10582 : {
10583 223 : lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START);
10584 223 : lock = build_call_expr_loc (loc, lock, 0);
10585 :
10586 223 : unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END);
10587 223 : unlock = build_call_expr_loc (loc, unlock, 0);
10588 : }
10589 :
10590 311 : push_gimplify_context ();
10591 :
10592 311 : block = make_node (BLOCK);
10593 311 : bind = gimple_build_bind (NULL, NULL, block);
10594 311 : gsi_replace (gsi_p, bind, true);
10595 311 : gimple_bind_add_stmt (bind, stmt);
10596 :
10597 311 : tbody = gimple_bind_body (bind);
10598 311 : gimplify_and_add (lock, &tbody);
10599 311 : gimple_bind_set_body (bind, tbody);
10600 :
10601 311 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
10602 311 : gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
10603 311 : gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10604 311 : gimple_omp_set_body (stmt, NULL);
10605 :
10606 311 : tbody = gimple_bind_body (bind);
10607 311 : gimplify_and_add (unlock, &tbody);
10608 311 : gimple_bind_set_body (bind, tbody);
10609 :
10610 311 : gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10611 :
10612 311 : pop_gimplify_context (bind);
10613 311 : gimple_bind_append_vars (bind, ctx->block_vars);
10614 311 : BLOCK_VARS (block) = gimple_bind_vars (bind);
10615 311 : }
10616 :
10617 : /* A subroutine of lower_omp_for. Generate code to emit the predicate
10618 : for a lastprivate clause. Given a loop control predicate of (V
10619 : cond N2), we gate the clause on (!(V cond N2)). The lowered form
10620 : is appended to *DLIST, iterator initialization is appended to
10621 : *BODY_P. *CLIST is for lastprivate(conditional:) code that needs
10622 : to be emitted in a critical section. */
10623 :
10624 : static void
10625 46966 : lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p,
10626 : gimple_seq *dlist, gimple_seq *clist,
10627 : struct omp_context *ctx)
10628 : {
10629 46966 : tree clauses, cond, vinit;
10630 46966 : enum tree_code cond_code;
10631 46966 : gimple_seq stmts;
10632 :
10633 46966 : cond_code = fd->loop.cond_code;
10634 46966 : cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR;
10635 :
10636 : /* When possible, use a strict equality expression. This can let VRP
10637 : type optimizations deduce the value and remove a copy. */
10638 46966 : if (tree_fits_shwi_p (fd->loop.step))
10639 : {
10640 44727 : HOST_WIDE_INT step = tree_to_shwi (fd->loop.step);
10641 44727 : if (step == 1 || step == -1)
10642 46966 : cond_code = EQ_EXPR;
10643 : }
10644 :
10645 46966 : tree n2 = fd->loop.n2;
10646 46966 : if (fd->collapse > 1
10647 11251 : && TREE_CODE (n2) != INTEGER_CST
10648 53180 : && gimple_omp_for_combined_into_p (fd->for_stmt))
10649 : {
10650 2692 : struct omp_context *taskreg_ctx = NULL;
10651 2692 : if (gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR)
10652 : {
10653 1226 : gomp_for *gfor = as_a <gomp_for *> (ctx->outer->stmt);
10654 1226 : if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_FOR
10655 1226 : || gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_DISTRIBUTE)
10656 : {
10657 1194 : if (gimple_omp_for_combined_into_p (gfor))
10658 : {
10659 672 : gcc_assert (ctx->outer->outer
10660 : && is_parallel_ctx (ctx->outer->outer));
10661 : taskreg_ctx = ctx->outer->outer;
10662 : }
10663 : else
10664 : {
10665 522 : struct omp_for_data outer_fd;
10666 522 : omp_extract_for_data (gfor, &outer_fd, NULL);
10667 522 : n2 = fold_convert (TREE_TYPE (n2), outer_fd.loop.n2);
10668 : }
10669 : }
10670 32 : else if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_TASKLOOP)
10671 32 : taskreg_ctx = ctx->outer->outer;
10672 : }
10673 1466 : else if (is_taskreg_ctx (ctx->outer))
10674 : taskreg_ctx = ctx->outer;
10675 2692 : if (taskreg_ctx)
10676 : {
10677 2170 : int i;
10678 2170 : tree taskreg_clauses
10679 2170 : = gimple_omp_taskreg_clauses (taskreg_ctx->stmt);
10680 2170 : tree innerc = omp_find_clause (taskreg_clauses,
10681 : OMP_CLAUSE__LOOPTEMP_);
10682 2170 : gcc_assert (innerc);
10683 2170 : int count = fd->collapse;
10684 2170 : if (fd->non_rect
10685 24 : && fd->last_nonrect == fd->first_nonrect + 1)
10686 12 : if (tree v = gimple_omp_for_index (fd->for_stmt, fd->last_nonrect))
10687 12 : if (!TYPE_UNSIGNED (TREE_TYPE (v)))
10688 12 : count += 4;
10689 8582 : for (i = 0; i < count; i++)
10690 : {
10691 6412 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
10692 : OMP_CLAUSE__LOOPTEMP_);
10693 6412 : gcc_assert (innerc);
10694 : }
10695 2170 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
10696 : OMP_CLAUSE__LOOPTEMP_);
10697 2170 : if (innerc)
10698 1419 : n2 = fold_convert (TREE_TYPE (n2),
10699 : lookup_decl (OMP_CLAUSE_DECL (innerc),
10700 : taskreg_ctx));
10701 : }
10702 : }
10703 46966 : cond = build2 (cond_code, boolean_type_node, fd->loop.v, n2);
10704 :
10705 46966 : clauses = gimple_omp_for_clauses (fd->for_stmt);
10706 46966 : stmts = NULL;
10707 46966 : lower_lastprivate_clauses (clauses, cond, body_p, &stmts, clist, ctx);
10708 46966 : if (!gimple_seq_empty_p (stmts))
10709 : {
10710 16372 : gimple_seq_add_seq (&stmts, *dlist);
10711 16372 : *dlist = stmts;
10712 :
10713 : /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
10714 16372 : vinit = fd->loop.n1;
10715 16372 : if (cond_code == EQ_EXPR
10716 14385 : && tree_fits_shwi_p (fd->loop.n2)
10717 25943 : && ! integer_zerop (fd->loop.n2))
10718 8355 : vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0);
10719 : else
10720 8017 : vinit = unshare_expr (vinit);
10721 :
10722 : /* Initialize the iterator variable, so that threads that don't execute
10723 : any iterations don't execute the lastprivate clauses by accident. */
10724 16372 : gimplify_assign (fd->loop.v, vinit, body_p);
10725 : }
10726 46966 : }
10727 :
10728 : /* OpenACC privatization.
10729 :
10730 : Or, in other words, *sharing* at the respective OpenACC level of
10731 : parallelism.
10732 :
10733 : From a correctness perspective, a non-addressable variable can't be accessed
10734 : outside the current thread, so it can go in a (faster than shared memory)
10735 : register -- though that register may need to be broadcast in some
10736 : circumstances. A variable can only meaningfully be "shared" across workers
10737 : or vector lanes if its address is taken, e.g. by a call to an atomic
10738 : builtin.
10739 :
10740 : From an optimisation perspective, the answer might be fuzzier: maybe
10741 : sometimes, using shared memory directly would be faster than
10742 : broadcasting. */
10743 :
10744 : static void
10745 18997 : oacc_privatization_begin_diagnose_var (const dump_flags_t l_dump_flags,
10746 : const location_t loc, const tree c,
10747 : const tree decl)
10748 : {
10749 18997 : const dump_user_location_t d_u_loc
10750 18997 : = dump_user_location_t::from_location_t (loc);
10751 : /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */
10752 : #if __GNUC__ >= 10
10753 18997 : # pragma GCC diagnostic push
10754 18997 : # pragma GCC diagnostic ignored "-Wformat"
10755 : #endif
10756 18997 : dump_printf_loc (l_dump_flags, d_u_loc,
10757 : "variable %<%T%> ", decl);
10758 : #if __GNUC__ >= 10
10759 18997 : # pragma GCC diagnostic pop
10760 : #endif
10761 18997 : if (c)
10762 2797 : dump_printf (l_dump_flags,
10763 : "in %qs clause ",
10764 2797 : omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
10765 : else
10766 16200 : dump_printf (l_dump_flags,
10767 : "declared in block ");
10768 18997 : }
10769 :
10770 : static bool
10771 41114 : oacc_privatization_candidate_p (const location_t loc, const tree c,
10772 : const tree decl)
10773 : {
10774 41114 : dump_flags_t l_dump_flags = get_openacc_privatization_dump_flags ();
10775 :
10776 : /* There is some differentiation depending on block vs. clause. */
10777 41114 : bool block = !c;
10778 :
10779 41114 : bool res = true;
10780 :
10781 41114 : if (res && !VAR_P (decl))
10782 : {
10783 : /* A PARM_DECL (appearing in a 'private' clause) is expected to have been
10784 : privatized into a new VAR_DECL. */
10785 712 : gcc_checking_assert (TREE_CODE (decl) != PARM_DECL);
10786 :
10787 712 : res = false;
10788 :
10789 712 : if (dump_enabled_p ())
10790 : {
10791 683 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10792 683 : dump_printf (l_dump_flags,
10793 : "potentially has improper OpenACC privatization level: %qs\n",
10794 683 : get_tree_code_name (TREE_CODE (decl)));
10795 : }
10796 : }
10797 :
10798 41085 : if (res && block && TREE_STATIC (decl))
10799 : {
10800 53 : res = false;
10801 :
10802 53 : if (dump_enabled_p ())
10803 : {
10804 28 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10805 28 : dump_printf (l_dump_flags,
10806 : "isn%'t candidate for adjusting OpenACC privatization level: %s\n",
10807 : "static");
10808 : }
10809 : }
10810 :
10811 41089 : if (res && block && DECL_EXTERNAL (decl))
10812 : {
10813 12 : res = false;
10814 :
10815 12 : if (dump_enabled_p ())
10816 : {
10817 12 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10818 12 : dump_printf (l_dump_flags,
10819 : "isn%'t candidate for adjusting OpenACC privatization level: %s\n",
10820 : "external");
10821 : }
10822 : }
10823 :
10824 41114 : if (res && !TREE_ADDRESSABLE (decl))
10825 : {
10826 39245 : res = false;
10827 :
10828 39245 : if (dump_enabled_p ())
10829 : {
10830 17456 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10831 17456 : dump_printf (l_dump_flags,
10832 : "isn%'t candidate for adjusting OpenACC privatization level: %s\n",
10833 : "not addressable");
10834 : }
10835 : }
10836 :
10837 : /* If an artificial variable has been added to a bind, e.g.
10838 : a compiler-generated temporary structure used by the Fortran front-end, do
10839 : not consider it as a privatization candidate. Note that variables on
10840 : the stack are private per-thread by default: making them "gang-private"
10841 : for OpenACC actually means to share a single instance of a variable
10842 : amongst all workers and threads spawned within each gang.
10843 : At present, no compiler-generated artificial variables require such
10844 : sharing semantics, so this is safe. */
10845 :
10846 18548 : if (res && block && DECL_ARTIFICIAL (decl))
10847 : {
10848 548 : res = false;
10849 :
10850 548 : if (dump_enabled_p ())
10851 : {
10852 360 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10853 360 : dump_printf (l_dump_flags,
10854 : "isn%'t candidate for adjusting OpenACC privatization "
10855 : "level: %s\n", "artificial");
10856 : }
10857 : }
10858 :
10859 40926 : if (res)
10860 : {
10861 544 : if (dump_enabled_p ())
10862 : {
10863 458 : oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
10864 458 : dump_printf (l_dump_flags,
10865 : "is candidate for adjusting OpenACC privatization level\n");
10866 : }
10867 : }
10868 :
10869 41114 : if (dump_file && (dump_flags & TDF_DETAILS))
10870 : {
10871 0 : print_generic_decl (dump_file, decl, dump_flags);
10872 0 : fprintf (dump_file, "\n");
10873 : }
10874 :
10875 41114 : return res;
10876 : }
10877 :
10878 : /* Scan CLAUSES for candidates for adjusting OpenACC privatization level in
10879 : CTX. */
10880 :
10881 : static void
10882 11325 : oacc_privatization_scan_clause_chain (omp_context *ctx, tree clauses)
10883 : {
10884 35223 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
10885 23898 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE)
10886 : {
10887 12986 : tree decl = OMP_CLAUSE_DECL (c);
10888 :
10889 12986 : tree new_decl = lookup_decl (decl, ctx);
10890 :
10891 12986 : if (!oacc_privatization_candidate_p (OMP_CLAUSE_LOCATION (c), c,
10892 : new_decl))
10893 12654 : continue;
10894 :
10895 332 : gcc_checking_assert
10896 : (!ctx->oacc_privatization_candidates.contains (new_decl));
10897 332 : ctx->oacc_privatization_candidates.safe_push (new_decl);
10898 : }
10899 11325 : }
10900 :
10901 : /* Scan DECLS for candidates for adjusting OpenACC privatization level in
10902 : CTX. */
10903 :
10904 : static void
10905 23090 : oacc_privatization_scan_decl_chain (omp_context *ctx, tree decls)
10906 : {
10907 51218 : for (tree decl = decls; decl; decl = DECL_CHAIN (decl))
10908 : {
10909 28128 : tree new_decl = lookup_decl (decl, ctx);
10910 28128 : gcc_checking_assert (new_decl == decl);
10911 :
10912 28128 : if (!oacc_privatization_candidate_p (gimple_location (ctx->stmt), NULL,
10913 : new_decl))
10914 27916 : continue;
10915 :
10916 212 : gcc_checking_assert
10917 : (!ctx->oacc_privatization_candidates.contains (new_decl));
10918 212 : ctx->oacc_privatization_candidates.safe_push (new_decl);
10919 : }
10920 23090 : }
10921 :
10922 : /* Callback for walk_gimple_seq. Find #pragma omp scan statement. */
10923 :
10924 : static tree
10925 2233 : omp_find_scan (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
10926 : struct walk_stmt_info *wi)
10927 : {
10928 2233 : gimple *stmt = gsi_stmt (*gsi_p);
10929 :
10930 2233 : *handled_ops_p = true;
10931 2233 : switch (gimple_code (stmt))
10932 : {
10933 418 : WALK_SUBSTMTS;
10934 :
10935 166 : case GIMPLE_OMP_FOR:
10936 166 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_SIMD
10937 166 : && gimple_omp_for_combined_into_p (stmt))
10938 166 : *handled_ops_p = false;
10939 : break;
10940 :
10941 686 : case GIMPLE_OMP_SCAN:
10942 686 : *(gimple_stmt_iterator *) (wi->info) = *gsi_p;
10943 686 : return integer_zero_node;
10944 : default:
10945 : break;
10946 : }
10947 : return NULL;
10948 : }
10949 :
10950 : /* Helper function for lower_omp_for, add transformations for a worksharing
10951 : loop with scan directives inside of it.
10952 : For worksharing loop not combined with simd, transform:
10953 : #pragma omp for reduction(inscan,+:r) private(i)
10954 : for (i = 0; i < n; i = i + 1)
10955 : {
10956 : {
10957 : update (r);
10958 : }
10959 : #pragma omp scan inclusive(r)
10960 : {
10961 : use (r);
10962 : }
10963 : }
10964 :
10965 : into two worksharing loops + code to merge results:
10966 :
10967 : num_threads = omp_get_num_threads ();
10968 : thread_num = omp_get_thread_num ();
10969 : if (thread_num == 0) goto <D.2099>; else goto <D.2100>;
10970 : <D.2099>:
10971 : var2 = r;
10972 : goto <D.2101>;
10973 : <D.2100>:
10974 : // For UDRs this is UDR init, or if ctors are needed, copy from
10975 : // var3 that has been constructed to contain the neutral element.
10976 : var2 = 0;
10977 : <D.2101>:
10978 : ivar = 0;
10979 : // The _scantemp_ clauses will arrange for rpriva to be initialized to
10980 : // a shared array with num_threads elements and rprivb to a local array
10981 : // number of elements equal to the number of (contiguous) iterations the
10982 : // current thread will perform. controlb and controlp variables are
10983 : // temporaries to handle deallocation of rprivb at the end of second
10984 : // GOMP_FOR.
10985 : #pragma omp for _scantemp_(rpriva) _scantemp_(rprivb) _scantemp_(controlb) \
10986 : _scantemp_(controlp) reduction(inscan,+:r) private(i) nowait
10987 : for (i = 0; i < n; i = i + 1)
10988 : {
10989 : {
10990 : // For UDRs this is UDR init or copy from var3.
10991 : r = 0;
10992 : // This is the input phase from user code.
10993 : update (r);
10994 : }
10995 : {
10996 : // For UDRs this is UDR merge.
10997 : var2 = var2 + r;
10998 : // Rather than handing it over to the user, save to local thread's
10999 : // array.
11000 : rprivb[ivar] = var2;
11001 : // For exclusive scan, the above two statements are swapped.
11002 : ivar = ivar + 1;
11003 : }
11004 : }
11005 : // And remember the final value from this thread's into the shared
11006 : // rpriva array.
11007 : rpriva[(sizetype) thread_num] = var2;
11008 : // If more than one thread, compute using Work-Efficient prefix sum
11009 : // the inclusive parallel scan of the rpriva array.
11010 : if (num_threads > 1) goto <D.2102>; else goto <D.2103>;
11011 : <D.2102>:
11012 : GOMP_barrier ();
11013 : down = 0;
11014 : k = 1;
11015 : num_threadsu = (unsigned int) num_threads;
11016 : thread_numup1 = (unsigned int) thread_num + 1;
11017 : <D.2108>:
11018 : twok = k << 1;
11019 : if (twok > num_threadsu) goto <D.2110>; else goto <D.2111>;
11020 : <D.2110>:
11021 : down = 4294967295;
11022 : k = k >> 1;
11023 : if (k == num_threadsu) goto <D.2112>; else goto <D.2111>;
11024 : <D.2112>:
11025 : k = k >> 1;
11026 : <D.2111>:
11027 : twok = k << 1;
11028 : cplx = .MUL_OVERFLOW (thread_nump1, twok);
11029 : mul = REALPART_EXPR <cplx>;
11030 : ovf = IMAGPART_EXPR <cplx>;
11031 : if (ovf == 0) goto <D.2116>; else goto <D.2117>;
11032 : <D.2116>:
11033 : andv = k & down;
11034 : andvm1 = andv + 4294967295;
11035 : l = mul + andvm1;
11036 : if (l < num_threadsu) goto <D.2120>; else goto <D.2117>;
11037 : <D.2120>:
11038 : // For UDRs this is UDR merge, performed using var2 variable as temporary,
11039 : // i.e. var2 = rpriva[l - k]; UDR merge (var2, rpriva[l]); rpriva[l] = var2;
11040 : rpriva[l] = rpriva[l - k] + rpriva[l];
11041 : <D.2117>:
11042 : if (down == 0) goto <D.2121>; else goto <D.2122>;
11043 : <D.2121>:
11044 : k = k << 1;
11045 : goto <D.2123>;
11046 : <D.2122>:
11047 : k = k >> 1;
11048 : <D.2123>:
11049 : GOMP_barrier ();
11050 : if (k != 0) goto <D.2108>; else goto <D.2103>;
11051 : <D.2103>:
11052 : if (thread_num == 0) goto <D.2124>; else goto <D.2125>;
11053 : <D.2124>:
11054 : // For UDRs this is UDR init or copy from var3.
11055 : var2 = 0;
11056 : goto <D.2126>;
11057 : <D.2125>:
11058 : var2 = rpriva[thread_num - 1];
11059 : <D.2126>:
11060 : ivar = 0;
11061 : #pragma omp for _scantemp_(controlb) _scantemp_(controlp) \
11062 : reduction(inscan,+:r) private(i)
11063 : for (i = 0; i < n; i = i + 1)
11064 : {
11065 : {
11066 : // For UDRs, this is r = var2; UDR merge (r, rprivb[ivar]);
11067 : r = var2 + rprivb[ivar];
11068 : }
11069 : {
11070 : // This is the scan phase from user code.
11071 : use (r);
11072 : // Plus a bump of the iterator.
11073 : ivar = ivar + 1;
11074 : }
11075 : } */
11076 :
11077 : static void
11078 177 : lower_omp_for_scan (gimple_seq *body_p, gimple_seq *dlist, gomp_for *stmt,
11079 : struct omp_for_data *fd, omp_context *ctx)
11080 : {
11081 177 : bool is_for_simd = gimple_omp_for_combined_p (stmt);
11082 177 : gcc_assert (ctx->scan_inclusive || ctx->scan_exclusive);
11083 :
11084 177 : gimple_seq body = gimple_omp_body (stmt);
11085 177 : gimple_stmt_iterator input1_gsi = gsi_none ();
11086 177 : struct walk_stmt_info wi;
11087 177 : memset (&wi, 0, sizeof (wi));
11088 177 : wi.val_only = true;
11089 177 : wi.info = (void *) &input1_gsi;
11090 177 : walk_gimple_seq_mod (&body, omp_find_scan, NULL, &wi);
11091 177 : gcc_assert (!gsi_end_p (input1_gsi));
11092 :
11093 177 : gimple *input_stmt1 = gsi_stmt (input1_gsi);
11094 177 : gimple_stmt_iterator gsi = input1_gsi;
11095 177 : gsi_next (&gsi);
11096 177 : gimple_stmt_iterator scan1_gsi = gsi;
11097 177 : gimple *scan_stmt1 = gsi_stmt (gsi);
11098 177 : gcc_assert (scan_stmt1 && gimple_code (scan_stmt1) == GIMPLE_OMP_SCAN);
11099 :
11100 177 : gimple_seq input_body = gimple_omp_body (input_stmt1);
11101 177 : gimple_seq scan_body = gimple_omp_body (scan_stmt1);
11102 177 : gimple_omp_set_body (input_stmt1, NULL);
11103 177 : gimple_omp_set_body (scan_stmt1, NULL);
11104 177 : gimple_omp_set_body (stmt, NULL);
11105 :
11106 177 : gomp_for *new_stmt = as_a <gomp_for *> (gimple_copy (stmt));
11107 177 : gimple_seq new_body = copy_gimple_seq_and_replace_locals (body);
11108 177 : gimple_omp_set_body (stmt, body);
11109 177 : gimple_omp_set_body (input_stmt1, input_body);
11110 :
11111 177 : gimple_stmt_iterator input2_gsi = gsi_none ();
11112 177 : memset (&wi, 0, sizeof (wi));
11113 177 : wi.val_only = true;
11114 177 : wi.info = (void *) &input2_gsi;
11115 177 : walk_gimple_seq_mod (&new_body, omp_find_scan, NULL, &wi);
11116 177 : gcc_assert (!gsi_end_p (input2_gsi));
11117 :
11118 177 : gimple *input_stmt2 = gsi_stmt (input2_gsi);
11119 177 : gsi = input2_gsi;
11120 177 : gsi_next (&gsi);
11121 177 : gimple_stmt_iterator scan2_gsi = gsi;
11122 177 : gimple *scan_stmt2 = gsi_stmt (gsi);
11123 177 : gcc_assert (scan_stmt2 && gimple_code (scan_stmt2) == GIMPLE_OMP_SCAN);
11124 177 : gimple_omp_set_body (scan_stmt2, scan_body);
11125 :
11126 177 : gimple_stmt_iterator input3_gsi = gsi_none ();
11127 177 : gimple_stmt_iterator scan3_gsi = gsi_none ();
11128 177 : gimple_stmt_iterator input4_gsi = gsi_none ();
11129 177 : gimple_stmt_iterator scan4_gsi = gsi_none ();
11130 177 : gimple *input_stmt3 = NULL, *scan_stmt3 = NULL;
11131 177 : gimple *input_stmt4 = NULL, *scan_stmt4 = NULL;
11132 177 : omp_context *input_simd_ctx = NULL, *scan_simd_ctx = NULL;
11133 177 : if (is_for_simd)
11134 : {
11135 83 : memset (&wi, 0, sizeof (wi));
11136 83 : wi.val_only = true;
11137 83 : wi.info = (void *) &input3_gsi;
11138 83 : walk_gimple_seq_mod (&input_body, omp_find_scan, NULL, &wi);
11139 83 : gcc_assert (!gsi_end_p (input3_gsi));
11140 :
11141 83 : input_stmt3 = gsi_stmt (input3_gsi);
11142 83 : gsi = input3_gsi;
11143 83 : gsi_next (&gsi);
11144 83 : scan3_gsi = gsi;
11145 83 : scan_stmt3 = gsi_stmt (gsi);
11146 83 : gcc_assert (scan_stmt3 && gimple_code (scan_stmt3) == GIMPLE_OMP_SCAN);
11147 :
11148 83 : memset (&wi, 0, sizeof (wi));
11149 83 : wi.val_only = true;
11150 83 : wi.info = (void *) &input4_gsi;
11151 83 : walk_gimple_seq_mod (&scan_body, omp_find_scan, NULL, &wi);
11152 83 : gcc_assert (!gsi_end_p (input4_gsi));
11153 :
11154 83 : input_stmt4 = gsi_stmt (input4_gsi);
11155 83 : gsi = input4_gsi;
11156 83 : gsi_next (&gsi);
11157 83 : scan4_gsi = gsi;
11158 83 : scan_stmt4 = gsi_stmt (gsi);
11159 83 : gcc_assert (scan_stmt4 && gimple_code (scan_stmt4) == GIMPLE_OMP_SCAN);
11160 :
11161 83 : input_simd_ctx = maybe_lookup_ctx (input_stmt3)->outer;
11162 83 : scan_simd_ctx = maybe_lookup_ctx (input_stmt4)->outer;
11163 : }
11164 :
11165 177 : tree num_threads = create_tmp_var (integer_type_node);
11166 177 : tree thread_num = create_tmp_var (integer_type_node);
11167 177 : tree nthreads_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
11168 177 : tree threadnum_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
11169 177 : gimple *g = gimple_build_call (nthreads_decl, 0);
11170 177 : gimple_call_set_lhs (g, num_threads);
11171 177 : gimple_seq_add_stmt (body_p, g);
11172 177 : g = gimple_build_call (threadnum_decl, 0);
11173 177 : gimple_call_set_lhs (g, thread_num);
11174 177 : gimple_seq_add_stmt (body_p, g);
11175 :
11176 177 : tree ivar = create_tmp_var (sizetype);
11177 177 : tree new_clauses1 = NULL_TREE, new_clauses2 = NULL_TREE;
11178 177 : tree *cp1 = &new_clauses1, *cp2 = &new_clauses2;
11179 177 : tree k = create_tmp_var (unsigned_type_node);
11180 177 : tree l = create_tmp_var (unsigned_type_node);
11181 :
11182 177 : gimple_seq clist = NULL, mdlist = NULL;
11183 177 : gimple_seq thr01_list = NULL, thrn1_list = NULL;
11184 177 : gimple_seq thr02_list = NULL, thrn2_list = NULL;
11185 177 : gimple_seq scan1_list = NULL, input2_list = NULL;
11186 177 : gimple_seq last_list = NULL, reduc_list = NULL;
11187 686 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
11188 509 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
11189 509 : && OMP_CLAUSE_REDUCTION_INSCAN (c))
11190 : {
11191 209 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
11192 209 : tree var = OMP_CLAUSE_DECL (c);
11193 209 : tree new_var = lookup_decl (var, ctx);
11194 209 : tree var3 = NULL_TREE;
11195 209 : tree new_vard = new_var;
11196 209 : if (omp_privatize_by_reference (var))
11197 32 : new_var = build_simple_mem_ref_loc (clause_loc, new_var);
11198 209 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
11199 : {
11200 64 : var3 = maybe_lookup_decl (new_vard, ctx);
11201 64 : if (var3 == new_vard)
11202 177 : var3 = NULL_TREE;
11203 : }
11204 :
11205 209 : tree ptype = build_pointer_type (TREE_TYPE (new_var));
11206 209 : tree rpriva = create_tmp_var (ptype);
11207 209 : tree nc = build_omp_clause (clause_loc, OMP_CLAUSE__SCANTEMP_);
11208 209 : OMP_CLAUSE_DECL (nc) = rpriva;
11209 209 : *cp1 = nc;
11210 209 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11211 :
11212 209 : tree rprivb = create_tmp_var (ptype);
11213 209 : nc = build_omp_clause (clause_loc, OMP_CLAUSE__SCANTEMP_);
11214 209 : OMP_CLAUSE_DECL (nc) = rprivb;
11215 209 : OMP_CLAUSE__SCANTEMP__ALLOC (nc) = 1;
11216 209 : *cp1 = nc;
11217 209 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11218 :
11219 209 : tree var2 = create_tmp_var_raw (TREE_TYPE (new_var));
11220 209 : if (new_vard != new_var)
11221 32 : TREE_ADDRESSABLE (var2) = 1;
11222 209 : gimple_add_tmp_var (var2);
11223 :
11224 209 : tree x = fold_convert_loc (clause_loc, sizetype, thread_num);
11225 209 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11226 209 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11227 209 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11228 209 : tree rpriva_ref = build_simple_mem_ref_loc (clause_loc, x);
11229 :
11230 209 : x = fold_build2_loc (clause_loc, PLUS_EXPR, integer_type_node,
11231 : thread_num, integer_minus_one_node);
11232 209 : x = fold_convert_loc (clause_loc, sizetype, x);
11233 209 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11234 209 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11235 209 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11236 209 : tree rprivam1_ref = build_simple_mem_ref_loc (clause_loc, x);
11237 :
11238 209 : x = fold_convert_loc (clause_loc, sizetype, l);
11239 209 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11240 209 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11241 209 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11242 209 : tree rprival_ref = build_simple_mem_ref_loc (clause_loc, x);
11243 :
11244 209 : x = fold_build2_loc (clause_loc, MINUS_EXPR, unsigned_type_node, l, k);
11245 209 : x = fold_convert_loc (clause_loc, sizetype, x);
11246 209 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
11247 209 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11248 209 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
11249 209 : tree rprivalmk_ref = build_simple_mem_ref_loc (clause_loc, x);
11250 :
11251 209 : x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, ivar,
11252 209 : TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
11253 209 : x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rprivb), rprivb, x);
11254 209 : tree rprivb_ref = build_simple_mem_ref_loc (clause_loc, x);
11255 :
11256 209 : tree var4 = is_for_simd ? new_var : var2;
11257 99 : tree var5 = NULL_TREE, var6 = NULL_TREE;
11258 99 : if (is_for_simd)
11259 : {
11260 99 : var5 = lookup_decl (var, input_simd_ctx);
11261 99 : var6 = lookup_decl (var, scan_simd_ctx);
11262 99 : if (new_vard != new_var)
11263 : {
11264 16 : var5 = build_simple_mem_ref_loc (clause_loc, var5);
11265 16 : var6 = build_simple_mem_ref_loc (clause_loc, var6);
11266 : }
11267 : }
11268 209 : if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
11269 : {
11270 64 : tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
11271 64 : tree val = var2;
11272 :
11273 64 : x = lang_hooks.decls.omp_clause_default_ctor
11274 64 : (c, var2, build_outer_var_ref (var, ctx));
11275 64 : if (x)
11276 32 : gimplify_and_add (x, &clist);
11277 :
11278 64 : x = build_outer_var_ref (var, ctx);
11279 64 : x = lang_hooks.decls.omp_clause_assign_op (c, unshare_expr (var4),
11280 : x);
11281 64 : gimplify_and_add (x, &thr01_list);
11282 :
11283 64 : tree y = (DECL_HAS_VALUE_EXPR_P (new_vard)
11284 64 : ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
11285 64 : if (var3)
11286 : {
11287 32 : x = unshare_expr (var4);
11288 32 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var3);
11289 32 : gimplify_and_add (x, &thrn1_list);
11290 32 : x = unshare_expr (var4);
11291 32 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var3);
11292 32 : gimplify_and_add (x, &thr02_list);
11293 : }
11294 32 : else if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
11295 : {
11296 : /* Otherwise, assign to it the identity element. */
11297 32 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
11298 32 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11299 32 : if (!is_for_simd)
11300 : {
11301 16 : if (new_vard != new_var)
11302 4 : val = build_fold_addr_expr_loc (clause_loc, val);
11303 16 : SET_DECL_VALUE_EXPR (new_vard, val);
11304 16 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
11305 : }
11306 32 : SET_DECL_VALUE_EXPR (placeholder, error_mark_node);
11307 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11308 32 : lower_omp (&tseq, ctx);
11309 32 : gimple_seq_add_seq (&thrn1_list, tseq);
11310 32 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
11311 32 : lower_omp (&tseq, ctx);
11312 32 : gimple_seq_add_seq (&thr02_list, tseq);
11313 32 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
11314 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
11315 32 : OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
11316 32 : if (y)
11317 0 : SET_DECL_VALUE_EXPR (new_vard, y);
11318 : else
11319 : {
11320 32 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
11321 32 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
11322 : }
11323 : }
11324 :
11325 64 : x = unshare_expr (var4);
11326 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x, rprivam1_ref);
11327 64 : gimplify_and_add (x, &thrn2_list);
11328 :
11329 64 : if (is_for_simd)
11330 : {
11331 32 : x = unshare_expr (rprivb_ref);
11332 32 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var5);
11333 32 : gimplify_and_add (x, &scan1_list);
11334 : }
11335 : else
11336 : {
11337 32 : if (ctx->scan_exclusive)
11338 : {
11339 16 : x = unshare_expr (rprivb_ref);
11340 16 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var2);
11341 16 : gimplify_and_add (x, &scan1_list);
11342 : }
11343 :
11344 32 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
11345 32 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11346 32 : SET_DECL_VALUE_EXPR (placeholder, var2);
11347 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11348 32 : lower_omp (&tseq, ctx);
11349 32 : gimple_seq_add_seq (&scan1_list, tseq);
11350 :
11351 32 : if (ctx->scan_inclusive)
11352 : {
11353 16 : x = unshare_expr (rprivb_ref);
11354 16 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var2);
11355 16 : gimplify_and_add (x, &scan1_list);
11356 : }
11357 : }
11358 :
11359 64 : x = unshare_expr (rpriva_ref);
11360 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x,
11361 : unshare_expr (var4));
11362 64 : gimplify_and_add (x, &mdlist);
11363 :
11364 96 : x = unshare_expr (is_for_simd ? var6 : new_var);
11365 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x, var4);
11366 64 : gimplify_and_add (x, &input2_list);
11367 :
11368 64 : val = rprivb_ref;
11369 64 : if (new_vard != new_var)
11370 24 : val = build_fold_addr_expr_loc (clause_loc, val);
11371 :
11372 64 : gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
11373 64 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11374 64 : SET_DECL_VALUE_EXPR (new_vard, val);
11375 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
11376 64 : if (is_for_simd)
11377 : {
11378 32 : SET_DECL_VALUE_EXPR (placeholder, var6);
11379 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11380 : }
11381 : else
11382 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
11383 64 : lower_omp (&tseq, ctx);
11384 64 : if (y)
11385 0 : SET_DECL_VALUE_EXPR (new_vard, y);
11386 : else
11387 : {
11388 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
11389 64 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
11390 : }
11391 64 : if (!is_for_simd)
11392 : {
11393 32 : SET_DECL_VALUE_EXPR (placeholder, new_var);
11394 32 : DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
11395 32 : lower_omp (&tseq, ctx);
11396 : }
11397 64 : gimple_seq_add_seq (&input2_list, tseq);
11398 :
11399 64 : x = build_outer_var_ref (var, ctx);
11400 64 : x = lang_hooks.decls.omp_clause_assign_op (c, x, rpriva_ref);
11401 64 : gimplify_and_add (x, &last_list);
11402 :
11403 64 : x = lang_hooks.decls.omp_clause_assign_op (c, var2, rprivalmk_ref);
11404 64 : gimplify_and_add (x, &reduc_list);
11405 64 : tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
11406 64 : tseq = copy_gimple_seq_and_replace_locals (tseq);
11407 64 : val = rprival_ref;
11408 64 : if (new_vard != new_var)
11409 24 : val = build_fold_addr_expr_loc (clause_loc, val);
11410 64 : SET_DECL_VALUE_EXPR (new_vard, val);
11411 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
11412 64 : SET_DECL_VALUE_EXPR (placeholder, var2);
11413 64 : lower_omp (&tseq, ctx);
11414 64 : OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
11415 64 : SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
11416 64 : DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
11417 64 : if (y)
11418 0 : SET_DECL_VALUE_EXPR (new_vard, y);
11419 : else
11420 : {
11421 64 : DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
11422 64 : SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
11423 : }
11424 64 : gimple_seq_add_seq (&reduc_list, tseq);
11425 64 : x = lang_hooks.decls.omp_clause_assign_op (c, rprival_ref, var2);
11426 64 : gimplify_and_add (x, &reduc_list);
11427 :
11428 64 : x = lang_hooks.decls.omp_clause_dtor (c, var2);
11429 64 : if (x)
11430 32 : gimplify_and_add (x, dlist);
11431 : }
11432 : else
11433 : {
11434 145 : x = build_outer_var_ref (var, ctx);
11435 145 : gimplify_assign (unshare_expr (var4), x, &thr01_list);
11436 :
11437 145 : x = omp_reduction_init (c, TREE_TYPE (new_var));
11438 145 : gimplify_assign (unshare_expr (var4), unshare_expr (x),
11439 : &thrn1_list);
11440 145 : gimplify_assign (unshare_expr (var4), x, &thr02_list);
11441 :
11442 145 : gimplify_assign (unshare_expr (var4), rprivam1_ref, &thrn2_list);
11443 :
11444 145 : enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
11445 145 : if (code == MINUS_EXPR)
11446 0 : code = PLUS_EXPR;
11447 :
11448 145 : if (is_for_simd)
11449 67 : gimplify_assign (unshare_expr (rprivb_ref), var5, &scan1_list);
11450 : else
11451 : {
11452 78 : if (ctx->scan_exclusive)
11453 28 : gimplify_assign (unshare_expr (rprivb_ref), var2,
11454 : &scan1_list);
11455 78 : x = build2 (code, TREE_TYPE (new_var), var2, new_var);
11456 78 : gimplify_assign (var2, x, &scan1_list);
11457 78 : if (ctx->scan_inclusive)
11458 50 : gimplify_assign (unshare_expr (rprivb_ref), var2,
11459 : &scan1_list);
11460 : }
11461 :
11462 145 : gimplify_assign (unshare_expr (rpriva_ref), unshare_expr (var4),
11463 : &mdlist);
11464 :
11465 145 : x = build2 (code, TREE_TYPE (new_var), var4, rprivb_ref);
11466 223 : gimplify_assign (is_for_simd ? var6 : new_var, x, &input2_list);
11467 :
11468 145 : gimplify_assign (build_outer_var_ref (var, ctx), rpriva_ref,
11469 : &last_list);
11470 :
11471 145 : x = build2 (code, TREE_TYPE (new_var), rprivalmk_ref,
11472 : unshare_expr (rprival_ref));
11473 145 : gimplify_assign (rprival_ref, x, &reduc_list);
11474 : }
11475 : }
11476 :
11477 177 : g = gimple_build_assign (ivar, PLUS_EXPR, ivar, size_one_node);
11478 177 : gimple_seq_add_stmt (&scan1_list, g);
11479 177 : g = gimple_build_assign (ivar, PLUS_EXPR, ivar, size_one_node);
11480 271 : gimple_seq_add_stmt (gimple_omp_body_ptr (is_for_simd
11481 : ? scan_stmt4 : scan_stmt2), g);
11482 :
11483 177 : tree controlb = create_tmp_var (boolean_type_node);
11484 177 : tree controlp = create_tmp_var (ptr_type_node);
11485 177 : tree nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11486 177 : OMP_CLAUSE_DECL (nc) = controlb;
11487 177 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11488 177 : *cp1 = nc;
11489 177 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11490 177 : nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11491 177 : OMP_CLAUSE_DECL (nc) = controlp;
11492 177 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11493 177 : *cp1 = nc;
11494 177 : cp1 = &OMP_CLAUSE_CHAIN (nc);
11495 177 : nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11496 177 : OMP_CLAUSE_DECL (nc) = controlb;
11497 177 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11498 177 : *cp2 = nc;
11499 177 : cp2 = &OMP_CLAUSE_CHAIN (nc);
11500 177 : nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
11501 177 : OMP_CLAUSE_DECL (nc) = controlp;
11502 177 : OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
11503 177 : *cp2 = nc;
11504 177 : cp2 = &OMP_CLAUSE_CHAIN (nc);
11505 :
11506 177 : *cp1 = gimple_omp_for_clauses (stmt);
11507 177 : gimple_omp_for_set_clauses (stmt, new_clauses1);
11508 177 : *cp2 = gimple_omp_for_clauses (new_stmt);
11509 177 : gimple_omp_for_set_clauses (new_stmt, new_clauses2);
11510 :
11511 177 : if (is_for_simd)
11512 : {
11513 83 : gimple_seq_add_seq (gimple_omp_body_ptr (scan_stmt3), scan1_list);
11514 83 : gimple_seq_add_seq (gimple_omp_body_ptr (input_stmt4), input2_list);
11515 :
11516 83 : gsi_insert_seq_after (&input3_gsi, gimple_omp_body (input_stmt3),
11517 : GSI_SAME_STMT);
11518 83 : gsi_remove (&input3_gsi, true);
11519 83 : gsi_insert_seq_after (&scan3_gsi, gimple_omp_body (scan_stmt3),
11520 : GSI_SAME_STMT);
11521 83 : gsi_remove (&scan3_gsi, true);
11522 83 : gsi_insert_seq_after (&input4_gsi, gimple_omp_body (input_stmt4),
11523 : GSI_SAME_STMT);
11524 83 : gsi_remove (&input4_gsi, true);
11525 83 : gsi_insert_seq_after (&scan4_gsi, gimple_omp_body (scan_stmt4),
11526 : GSI_SAME_STMT);
11527 83 : gsi_remove (&scan4_gsi, true);
11528 : }
11529 : else
11530 : {
11531 94 : gimple_omp_set_body (scan_stmt1, scan1_list);
11532 94 : gimple_omp_set_body (input_stmt2, input2_list);
11533 : }
11534 :
11535 177 : gsi_insert_seq_after (&input1_gsi, gimple_omp_body (input_stmt1),
11536 : GSI_SAME_STMT);
11537 177 : gsi_remove (&input1_gsi, true);
11538 177 : gsi_insert_seq_after (&scan1_gsi, gimple_omp_body (scan_stmt1),
11539 : GSI_SAME_STMT);
11540 177 : gsi_remove (&scan1_gsi, true);
11541 177 : gsi_insert_seq_after (&input2_gsi, gimple_omp_body (input_stmt2),
11542 : GSI_SAME_STMT);
11543 177 : gsi_remove (&input2_gsi, true);
11544 177 : gsi_insert_seq_after (&scan2_gsi, gimple_omp_body (scan_stmt2),
11545 : GSI_SAME_STMT);
11546 177 : gsi_remove (&scan2_gsi, true);
11547 :
11548 177 : gimple_seq_add_seq (body_p, clist);
11549 :
11550 177 : tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
11551 177 : tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
11552 177 : tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
11553 177 : g = gimple_build_cond (EQ_EXPR, thread_num, integer_zero_node, lab1, lab2);
11554 177 : gimple_seq_add_stmt (body_p, g);
11555 177 : g = gimple_build_label (lab1);
11556 177 : gimple_seq_add_stmt (body_p, g);
11557 177 : gimple_seq_add_seq (body_p, thr01_list);
11558 177 : g = gimple_build_goto (lab3);
11559 177 : gimple_seq_add_stmt (body_p, g);
11560 177 : g = gimple_build_label (lab2);
11561 177 : gimple_seq_add_stmt (body_p, g);
11562 177 : gimple_seq_add_seq (body_p, thrn1_list);
11563 177 : g = gimple_build_label (lab3);
11564 177 : gimple_seq_add_stmt (body_p, g);
11565 :
11566 177 : g = gimple_build_assign (ivar, size_zero_node);
11567 177 : gimple_seq_add_stmt (body_p, g);
11568 :
11569 177 : gimple_seq_add_stmt (body_p, stmt);
11570 177 : gimple_seq_add_seq (body_p, body);
11571 177 : gimple_seq_add_stmt (body_p, gimple_build_omp_continue (fd->loop.v,
11572 : fd->loop.v));
11573 :
11574 177 : g = gimple_build_omp_return (true);
11575 177 : gimple_seq_add_stmt (body_p, g);
11576 177 : gimple_seq_add_seq (body_p, mdlist);
11577 :
11578 177 : lab1 = create_artificial_label (UNKNOWN_LOCATION);
11579 177 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
11580 177 : g = gimple_build_cond (GT_EXPR, num_threads, integer_one_node, lab1, lab2);
11581 177 : gimple_seq_add_stmt (body_p, g);
11582 177 : g = gimple_build_label (lab1);
11583 177 : gimple_seq_add_stmt (body_p, g);
11584 :
11585 177 : g = omp_build_barrier (NULL, GOMP_BARRIER_IMPLICIT_WORKSHARE);
11586 177 : gimple_seq_add_stmt (body_p, g);
11587 :
11588 177 : tree down = create_tmp_var (unsigned_type_node);
11589 177 : g = gimple_build_assign (down, build_zero_cst (unsigned_type_node));
11590 177 : gimple_seq_add_stmt (body_p, g);
11591 :
11592 177 : g = gimple_build_assign (k, build_one_cst (unsigned_type_node));
11593 177 : gimple_seq_add_stmt (body_p, g);
11594 :
11595 177 : tree num_threadsu = create_tmp_var (unsigned_type_node);
11596 177 : g = gimple_build_assign (num_threadsu, NOP_EXPR, num_threads);
11597 177 : gimple_seq_add_stmt (body_p, g);
11598 :
11599 177 : tree thread_numu = create_tmp_var (unsigned_type_node);
11600 177 : g = gimple_build_assign (thread_numu, NOP_EXPR, thread_num);
11601 177 : gimple_seq_add_stmt (body_p, g);
11602 :
11603 177 : tree thread_nump1 = create_tmp_var (unsigned_type_node);
11604 177 : g = gimple_build_assign (thread_nump1, PLUS_EXPR, thread_numu,
11605 : build_int_cst (unsigned_type_node, 1));
11606 177 : gimple_seq_add_stmt (body_p, g);
11607 :
11608 177 : lab3 = create_artificial_label (UNKNOWN_LOCATION);
11609 177 : g = gimple_build_label (lab3);
11610 177 : gimple_seq_add_stmt (body_p, g);
11611 :
11612 177 : tree twok = create_tmp_var (unsigned_type_node);
11613 177 : g = gimple_build_assign (twok, LSHIFT_EXPR, k, integer_one_node);
11614 177 : gimple_seq_add_stmt (body_p, g);
11615 :
11616 177 : tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
11617 177 : tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
11618 177 : tree lab6 = create_artificial_label (UNKNOWN_LOCATION);
11619 177 : g = gimple_build_cond (GT_EXPR, twok, num_threadsu, lab4, lab5);
11620 177 : gimple_seq_add_stmt (body_p, g);
11621 177 : g = gimple_build_label (lab4);
11622 177 : gimple_seq_add_stmt (body_p, g);
11623 177 : g = gimple_build_assign (down, build_all_ones_cst (unsigned_type_node));
11624 177 : gimple_seq_add_stmt (body_p, g);
11625 177 : g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
11626 177 : gimple_seq_add_stmt (body_p, g);
11627 :
11628 177 : g = gimple_build_cond (EQ_EXPR, k, num_threadsu, lab6, lab5);
11629 177 : gimple_seq_add_stmt (body_p, g);
11630 177 : g = gimple_build_label (lab6);
11631 177 : gimple_seq_add_stmt (body_p, g);
11632 :
11633 177 : g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
11634 177 : gimple_seq_add_stmt (body_p, g);
11635 :
11636 177 : g = gimple_build_label (lab5);
11637 177 : gimple_seq_add_stmt (body_p, g);
11638 :
11639 177 : g = gimple_build_assign (twok, LSHIFT_EXPR, k, integer_one_node);
11640 177 : gimple_seq_add_stmt (body_p, g);
11641 :
11642 177 : tree cplx = create_tmp_var (build_complex_type (unsigned_type_node, false));
11643 177 : g = gimple_build_call_internal (IFN_MUL_OVERFLOW, 2, thread_nump1, twok);
11644 177 : gimple_call_set_lhs (g, cplx);
11645 177 : gimple_seq_add_stmt (body_p, g);
11646 177 : tree mul = create_tmp_var (unsigned_type_node);
11647 177 : g = gimple_build_assign (mul, REALPART_EXPR,
11648 : build1 (REALPART_EXPR, unsigned_type_node, cplx));
11649 177 : gimple_seq_add_stmt (body_p, g);
11650 177 : tree ovf = create_tmp_var (unsigned_type_node);
11651 177 : g = gimple_build_assign (ovf, IMAGPART_EXPR,
11652 : build1 (IMAGPART_EXPR, unsigned_type_node, cplx));
11653 177 : gimple_seq_add_stmt (body_p, g);
11654 :
11655 177 : tree lab7 = create_artificial_label (UNKNOWN_LOCATION);
11656 177 : tree lab8 = create_artificial_label (UNKNOWN_LOCATION);
11657 177 : g = gimple_build_cond (EQ_EXPR, ovf, build_zero_cst (unsigned_type_node),
11658 : lab7, lab8);
11659 177 : gimple_seq_add_stmt (body_p, g);
11660 177 : g = gimple_build_label (lab7);
11661 177 : gimple_seq_add_stmt (body_p, g);
11662 :
11663 177 : tree andv = create_tmp_var (unsigned_type_node);
11664 177 : g = gimple_build_assign (andv, BIT_AND_EXPR, k, down);
11665 177 : gimple_seq_add_stmt (body_p, g);
11666 177 : tree andvm1 = create_tmp_var (unsigned_type_node);
11667 177 : g = gimple_build_assign (andvm1, PLUS_EXPR, andv,
11668 : build_minus_one_cst (unsigned_type_node));
11669 177 : gimple_seq_add_stmt (body_p, g);
11670 :
11671 177 : g = gimple_build_assign (l, PLUS_EXPR, mul, andvm1);
11672 177 : gimple_seq_add_stmt (body_p, g);
11673 :
11674 177 : tree lab9 = create_artificial_label (UNKNOWN_LOCATION);
11675 177 : g = gimple_build_cond (LT_EXPR, l, num_threadsu, lab9, lab8);
11676 177 : gimple_seq_add_stmt (body_p, g);
11677 177 : g = gimple_build_label (lab9);
11678 177 : gimple_seq_add_stmt (body_p, g);
11679 177 : gimple_seq_add_seq (body_p, reduc_list);
11680 177 : g = gimple_build_label (lab8);
11681 177 : gimple_seq_add_stmt (body_p, g);
11682 :
11683 177 : tree lab10 = create_artificial_label (UNKNOWN_LOCATION);
11684 177 : tree lab11 = create_artificial_label (UNKNOWN_LOCATION);
11685 177 : tree lab12 = create_artificial_label (UNKNOWN_LOCATION);
11686 177 : g = gimple_build_cond (EQ_EXPR, down, build_zero_cst (unsigned_type_node),
11687 : lab10, lab11);
11688 177 : gimple_seq_add_stmt (body_p, g);
11689 177 : g = gimple_build_label (lab10);
11690 177 : gimple_seq_add_stmt (body_p, g);
11691 177 : g = gimple_build_assign (k, LSHIFT_EXPR, k, integer_one_node);
11692 177 : gimple_seq_add_stmt (body_p, g);
11693 177 : g = gimple_build_goto (lab12);
11694 177 : gimple_seq_add_stmt (body_p, g);
11695 177 : g = gimple_build_label (lab11);
11696 177 : gimple_seq_add_stmt (body_p, g);
11697 177 : g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
11698 177 : gimple_seq_add_stmt (body_p, g);
11699 177 : g = gimple_build_label (lab12);
11700 177 : gimple_seq_add_stmt (body_p, g);
11701 :
11702 177 : g = omp_build_barrier (NULL, GOMP_BARRIER_IMPLICIT_WORKSHARE);
11703 177 : gimple_seq_add_stmt (body_p, g);
11704 :
11705 177 : g = gimple_build_cond (NE_EXPR, k, build_zero_cst (unsigned_type_node),
11706 : lab3, lab2);
11707 177 : gimple_seq_add_stmt (body_p, g);
11708 :
11709 177 : g = gimple_build_label (lab2);
11710 177 : gimple_seq_add_stmt (body_p, g);
11711 :
11712 177 : lab1 = create_artificial_label (UNKNOWN_LOCATION);
11713 177 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
11714 177 : lab3 = create_artificial_label (UNKNOWN_LOCATION);
11715 177 : g = gimple_build_cond (EQ_EXPR, thread_num, integer_zero_node, lab1, lab2);
11716 177 : gimple_seq_add_stmt (body_p, g);
11717 177 : g = gimple_build_label (lab1);
11718 177 : gimple_seq_add_stmt (body_p, g);
11719 177 : gimple_seq_add_seq (body_p, thr02_list);
11720 177 : g = gimple_build_goto (lab3);
11721 177 : gimple_seq_add_stmt (body_p, g);
11722 177 : g = gimple_build_label (lab2);
11723 177 : gimple_seq_add_stmt (body_p, g);
11724 177 : gimple_seq_add_seq (body_p, thrn2_list);
11725 177 : g = gimple_build_label (lab3);
11726 177 : gimple_seq_add_stmt (body_p, g);
11727 :
11728 177 : g = gimple_build_assign (ivar, size_zero_node);
11729 177 : gimple_seq_add_stmt (body_p, g);
11730 177 : gimple_seq_add_stmt (body_p, new_stmt);
11731 177 : gimple_seq_add_seq (body_p, new_body);
11732 :
11733 177 : gimple_seq new_dlist = NULL;
11734 177 : lab1 = create_artificial_label (UNKNOWN_LOCATION);
11735 177 : lab2 = create_artificial_label (UNKNOWN_LOCATION);
11736 177 : tree num_threadsm1 = create_tmp_var (integer_type_node);
11737 177 : g = gimple_build_assign (num_threadsm1, PLUS_EXPR, num_threads,
11738 : integer_minus_one_node);
11739 177 : gimple_seq_add_stmt (&new_dlist, g);
11740 177 : g = gimple_build_cond (EQ_EXPR, thread_num, num_threadsm1, lab1, lab2);
11741 177 : gimple_seq_add_stmt (&new_dlist, g);
11742 177 : g = gimple_build_label (lab1);
11743 177 : gimple_seq_add_stmt (&new_dlist, g);
11744 177 : gimple_seq_add_seq (&new_dlist, last_list);
11745 177 : g = gimple_build_label (lab2);
11746 177 : gimple_seq_add_stmt (&new_dlist, g);
11747 177 : gimple_seq_add_seq (&new_dlist, *dlist);
11748 177 : *dlist = new_dlist;
11749 177 : }
11750 :
11751 : /* Build an internal UNIQUE function with type IFN_UNIQUE_OACC_PRIVATE listing
11752 : the addresses of variables to be made private at the surrounding
11753 : parallelism level. Such functions appear in the gimple code stream in two
11754 : forms, e.g. for a partitioned loop:
11755 :
11756 : .data_dep.6 = .UNIQUE (OACC_HEAD_MARK, .data_dep.6, 1, 68);
11757 : .data_dep.6 = .UNIQUE (OACC_PRIVATE, .data_dep.6, -1, &w);
11758 : .data_dep.6 = .UNIQUE (OACC_FORK, .data_dep.6, -1);
11759 : .data_dep.6 = .UNIQUE (OACC_HEAD_MARK, .data_dep.6);
11760 :
11761 : or alternatively, OACC_PRIVATE can appear at the top level of a parallel,
11762 : not as part of a HEAD_MARK sequence:
11763 :
11764 : .UNIQUE (OACC_PRIVATE, 0, 0, &w);
11765 :
11766 : For such stand-alone appearances, the 3rd argument is always 0, denoting
11767 : gang partitioning. */
11768 :
11769 : static gcall *
11770 19728 : lower_oacc_private_marker (omp_context *ctx)
11771 : {
11772 19728 : if (ctx->oacc_privatization_candidates.length () == 0)
11773 : return NULL;
11774 :
11775 308 : auto_vec<tree, 5> args;
11776 :
11777 308 : args.quick_push (build_int_cst (integer_type_node, IFN_UNIQUE_OACC_PRIVATE));
11778 308 : args.quick_push (integer_zero_node);
11779 308 : args.quick_push (integer_minus_one_node);
11780 :
11781 308 : int i;
11782 308 : tree decl;
11783 696 : FOR_EACH_VEC_ELT (ctx->oacc_privatization_candidates, i, decl)
11784 : {
11785 388 : gcc_checking_assert (TREE_ADDRESSABLE (decl));
11786 388 : tree addr = build_fold_addr_expr (decl);
11787 388 : args.safe_push (addr);
11788 : }
11789 :
11790 308 : return gimple_build_call_internal_vec (IFN_UNIQUE, args);
11791 308 : }
11792 :
11793 : /* Lower code for an OMP loop directive. */
11794 :
11795 : static void
11796 46966 : lower_omp_for (gimple_stmt_iterator *gsi_p, omp_context *ctx)
11797 : {
11798 46966 : tree *rhs_p, block;
11799 46966 : struct omp_for_data fd, *fdp = NULL;
11800 46966 : gomp_for *stmt = as_a <gomp_for *> (gsi_stmt (*gsi_p));
11801 46966 : gbind *new_stmt;
11802 46966 : gimple_seq omp_for_body, body, dlist, tred_ilist = NULL, tred_dlist = NULL;
11803 46966 : gimple_seq cnt_list = NULL, clist = NULL;
11804 46966 : gimple_seq oacc_head = NULL, oacc_tail = NULL;
11805 46966 : size_t i;
11806 :
11807 46966 : push_gimplify_context ();
11808 :
11809 46966 : if (is_gimple_omp_oacc (ctx->stmt))
11810 11325 : oacc_privatization_scan_clause_chain (ctx, gimple_omp_for_clauses (stmt));
11811 :
11812 46966 : lower_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
11813 :
11814 46966 : block = make_node (BLOCK);
11815 46966 : new_stmt = gimple_build_bind (NULL, NULL, block);
11816 : /* Replace at gsi right away, so that 'stmt' is no member
11817 : of a sequence anymore as we're going to add to a different
11818 : one below. */
11819 46966 : gsi_replace (gsi_p, new_stmt, true);
11820 :
11821 : /* Move declaration of temporaries in the loop body before we make
11822 : it go away. */
11823 46966 : omp_for_body = gimple_omp_body (stmt);
11824 46966 : if (!gimple_seq_empty_p (omp_for_body)
11825 46966 : && gimple_code (gimple_seq_first_stmt (omp_for_body)) == GIMPLE_BIND)
11826 : {
11827 19621 : gbind *inner_bind
11828 19621 : = as_a <gbind *> (gimple_seq_first_stmt (omp_for_body));
11829 19621 : tree vars = gimple_bind_vars (inner_bind);
11830 19621 : if (is_gimple_omp_oacc (ctx->stmt))
11831 5305 : oacc_privatization_scan_decl_chain (ctx, vars);
11832 19621 : gimple_bind_append_vars (new_stmt, vars);
11833 : /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
11834 : keep them on the inner_bind and it's block. */
11835 19621 : gimple_bind_set_vars (inner_bind, NULL_TREE);
11836 19621 : if (gimple_bind_block (inner_bind))
11837 17062 : BLOCK_VARS (gimple_bind_block (inner_bind)) = NULL_TREE;
11838 : }
11839 :
11840 46966 : if (gimple_omp_for_combined_into_p (stmt))
11841 : {
11842 14604 : omp_extract_for_data (stmt, &fd, NULL);
11843 14604 : fdp = &fd;
11844 :
11845 : /* We need two temporaries with fd.loop.v type (istart/iend)
11846 : and then (fd.collapse - 1) temporaries with the same
11847 : type for count2 ... countN-1 vars if not constant. */
11848 14604 : size_t count = 2;
11849 14604 : tree type = fd.iter_type;
11850 14604 : if (fd.collapse > 1
11851 4867 : && TREE_CODE (fd.loop.n2) != INTEGER_CST)
11852 2692 : count += fd.collapse - 1;
11853 14604 : size_t count2 = 0;
11854 14604 : tree type2 = NULL_TREE;
11855 14604 : bool taskreg_for
11856 14604 : = (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR
11857 14604 : || gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_TASKLOOP);
11858 14604 : tree outerc = NULL, *pc = gimple_omp_for_clauses_ptr (stmt);
11859 14604 : tree simtc = NULL;
11860 14604 : tree clauses = *pc;
11861 14604 : if (fd.collapse > 1
11862 4867 : && fd.non_rect
11863 137 : && fd.last_nonrect == fd.first_nonrect + 1
11864 89 : && TREE_CODE (fd.loop.n2) != INTEGER_CST)
11865 64 : if (tree v = gimple_omp_for_index (stmt, fd.last_nonrect))
11866 64 : if (!TYPE_UNSIGNED (TREE_TYPE (v)))
11867 : {
11868 60 : v = gimple_omp_for_index (stmt, fd.first_nonrect);
11869 60 : type2 = TREE_TYPE (v);
11870 60 : count++;
11871 60 : count2 = 3;
11872 : }
11873 14604 : if (taskreg_for)
11874 7597 : outerc
11875 7597 : = omp_find_clause (gimple_omp_taskreg_clauses (ctx->outer->stmt),
11876 : OMP_CLAUSE__LOOPTEMP_);
11877 14604 : if (ctx->simt_stmt)
11878 0 : simtc = omp_find_clause (gimple_omp_for_clauses (ctx->simt_stmt),
11879 : OMP_CLAUSE__LOOPTEMP_);
11880 49221 : for (i = 0; i < count + count2; i++)
11881 : {
11882 34617 : tree temp;
11883 34617 : if (taskreg_for)
11884 : {
11885 18075 : gcc_assert (outerc);
11886 18075 : temp = lookup_decl (OMP_CLAUSE_DECL (outerc), ctx->outer);
11887 18075 : outerc = omp_find_clause (OMP_CLAUSE_CHAIN (outerc),
11888 : OMP_CLAUSE__LOOPTEMP_);
11889 : }
11890 : else
11891 : {
11892 : /* If there are 2 adjacent SIMD stmts, one with _simt_
11893 : clause, another without, make sure they have the same
11894 : decls in _looptemp_ clauses, because the outer stmt
11895 : they are combined into will look up just one inner_stmt. */
11896 16542 : if (ctx->simt_stmt)
11897 0 : temp = OMP_CLAUSE_DECL (simtc);
11898 : else
11899 32940 : temp = create_tmp_var (i >= count ? type2 : type);
11900 16542 : insert_decl_map (&ctx->outer->cb, temp, temp);
11901 : }
11902 34617 : *pc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
11903 34617 : OMP_CLAUSE_DECL (*pc) = temp;
11904 34617 : pc = &OMP_CLAUSE_CHAIN (*pc);
11905 34617 : if (ctx->simt_stmt)
11906 0 : simtc = omp_find_clause (OMP_CLAUSE_CHAIN (simtc),
11907 : OMP_CLAUSE__LOOPTEMP_);
11908 : }
11909 14604 : *pc = clauses;
11910 : }
11911 :
11912 : /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
11913 46966 : dlist = NULL;
11914 46966 : body = NULL;
11915 46966 : tree rclauses
11916 46966 : = omp_task_reductions_find_first (gimple_omp_for_clauses (stmt), OMP_FOR,
11917 : OMP_CLAUSE_REDUCTION);
11918 46966 : tree rtmp = NULL_TREE;
11919 46966 : if (rclauses)
11920 : {
11921 227 : tree type = build_pointer_type (pointer_sized_int_node);
11922 227 : tree temp = create_tmp_var (type);
11923 227 : tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
11924 227 : OMP_CLAUSE_DECL (c) = temp;
11925 227 : OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (stmt);
11926 227 : gimple_omp_for_set_clauses (stmt, c);
11927 227 : lower_omp_task_reductions (ctx, OMP_FOR,
11928 : gimple_omp_for_clauses (stmt),
11929 : &tred_ilist, &tred_dlist);
11930 227 : rclauses = c;
11931 227 : rtmp = make_ssa_name (type);
11932 227 : gimple_seq_add_stmt (&body, gimple_build_assign (rtmp, temp));
11933 : }
11934 :
11935 46966 : lower_lastprivate_conditional_clauses (gimple_omp_for_clauses_ptr (stmt),
11936 : ctx);
11937 :
11938 46966 : lower_rec_input_clauses (gimple_omp_for_clauses (stmt), &body, &dlist, ctx,
11939 : fdp);
11940 93705 : gimple_seq_add_seq (rclauses ? &tred_ilist : &body,
11941 : gimple_omp_for_pre_body (stmt));
11942 :
11943 46966 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
11944 :
11945 46966 : gcall *private_marker = NULL;
11946 46966 : if (is_gimple_omp_oacc (ctx->stmt)
11947 46966 : && !gimple_seq_empty_p (omp_for_body))
11948 10333 : private_marker = lower_oacc_private_marker (ctx);
11949 :
11950 : /* Lower the header expressions. At this point, we can assume that
11951 : the header is of the form:
11952 :
11953 : #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
11954 :
11955 : We just need to make sure that VAL1, VAL2 and VAL3 are lowered
11956 : using the .omp_data_s mapping, if needed. */
11957 115216 : for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
11958 : {
11959 68250 : rhs_p = gimple_omp_for_initial_ptr (stmt, i);
11960 68250 : if (TREE_CODE (*rhs_p) == TREE_VEC)
11961 : {
11962 639 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 1)))
11963 149 : TREE_VEC_ELT (*rhs_p, 1)
11964 298 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 1), &cnt_list);
11965 639 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 2)))
11966 161 : TREE_VEC_ELT (*rhs_p, 2)
11967 322 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 2), &cnt_list);
11968 : }
11969 67611 : else if (!is_gimple_min_invariant (*rhs_p))
11970 11465 : *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
11971 56146 : else if (TREE_CODE (*rhs_p) == ADDR_EXPR)
11972 4713 : recompute_tree_invariant_for_addr_expr (*rhs_p);
11973 :
11974 68250 : rhs_p = gimple_omp_for_final_ptr (stmt, i);
11975 68250 : if (TREE_CODE (*rhs_p) == TREE_VEC)
11976 : {
11977 519 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 1)))
11978 142 : TREE_VEC_ELT (*rhs_p, 1)
11979 284 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 1), &cnt_list);
11980 519 : if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 2)))
11981 157 : TREE_VEC_ELT (*rhs_p, 2)
11982 314 : = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 2), &cnt_list);
11983 : }
11984 67731 : else if (!is_gimple_min_invariant (*rhs_p))
11985 16851 : *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
11986 50880 : else if (TREE_CODE (*rhs_p) == ADDR_EXPR)
11987 4630 : recompute_tree_invariant_for_addr_expr (*rhs_p);
11988 :
11989 68250 : rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1);
11990 68250 : if (!is_gimple_min_invariant (*rhs_p))
11991 6318 : *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
11992 : }
11993 46966 : if (rclauses)
11994 227 : gimple_seq_add_seq (&tred_ilist, cnt_list);
11995 : else
11996 46739 : gimple_seq_add_seq (&body, cnt_list);
11997 :
11998 : /* Once lowered, extract the bounds and clauses. */
11999 46966 : omp_extract_for_data (stmt, &fd, NULL);
12000 :
12001 46966 : if (is_gimple_omp_oacc (ctx->stmt)
12002 46966 : && !ctx_in_oacc_kernels_region (ctx))
12003 9665 : lower_oacc_head_tail (gimple_location (stmt),
12004 : gimple_omp_for_clauses (stmt), private_marker,
12005 : &oacc_head, &oacc_tail, ctx);
12006 :
12007 : /* Add OpenACC partitioning and reduction markers just before the loop. */
12008 46966 : if (oacc_head)
12009 9665 : gimple_seq_add_seq (&body, oacc_head);
12010 :
12011 46966 : lower_omp_for_lastprivate (&fd, &body, &dlist, &clist, ctx);
12012 :
12013 46966 : if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
12014 85950 : for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
12015 70332 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
12016 70332 : && !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
12017 : {
12018 227 : OMP_CLAUSE_DECL (c) = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
12019 227 : if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c)))
12020 42 : OMP_CLAUSE_LINEAR_STEP (c)
12021 84 : = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c),
12022 : ctx);
12023 : }
12024 :
12025 46596 : if ((ctx->scan_inclusive || ctx->scan_exclusive)
12026 47234 : && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
12027 177 : lower_omp_for_scan (&body, &dlist, stmt, &fd, ctx);
12028 : else
12029 : {
12030 46789 : gimple_seq_add_stmt (&body, stmt);
12031 46789 : gimple_seq_add_seq (&body, gimple_omp_body (stmt));
12032 : }
12033 :
12034 46966 : gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v,
12035 : fd.loop.v));
12036 :
12037 : /* After the loop, add exit clauses. */
12038 46966 : lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, &clist, ctx);
12039 :
12040 46966 : if (clist)
12041 : {
12042 179 : tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_START);
12043 179 : gcall *g = gimple_build_call (fndecl, 0);
12044 179 : gimple_seq_add_stmt (&body, g);
12045 179 : gimple_seq_add_seq (&body, clist);
12046 179 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_END);
12047 179 : g = gimple_build_call (fndecl, 0);
12048 179 : gimple_seq_add_stmt (&body, g);
12049 : }
12050 :
12051 46966 : if (ctx->cancellable)
12052 84 : gimple_seq_add_stmt (&body, gimple_build_label (ctx->cancel_label));
12053 :
12054 46966 : gimple_seq_add_seq (&body, dlist);
12055 :
12056 46966 : if (rclauses)
12057 : {
12058 227 : gimple_seq_add_seq (&tred_ilist, body);
12059 227 : body = tred_ilist;
12060 : }
12061 :
12062 46966 : body = maybe_catch_exception (body);
12063 :
12064 : /* Region exit marker goes at the end of the loop body. */
12065 46966 : gimple *g = gimple_build_omp_return (fd.have_nowait);
12066 46966 : gimple_seq_add_stmt (&body, g);
12067 :
12068 46966 : gimple_seq_add_seq (&body, tred_dlist);
12069 :
12070 46966 : maybe_add_implicit_barrier_cancel (ctx, g, &body);
12071 :
12072 46966 : if (rclauses)
12073 227 : OMP_CLAUSE_DECL (rclauses) = rtmp;
12074 :
12075 : /* Add OpenACC joining and reduction markers just after the loop. */
12076 46966 : if (oacc_tail)
12077 9665 : gimple_seq_add_seq (&body, oacc_tail);
12078 :
12079 46966 : pop_gimplify_context (new_stmt);
12080 :
12081 46966 : gimple_bind_append_vars (new_stmt, ctx->block_vars);
12082 46966 : maybe_remove_omp_member_access_dummy_vars (new_stmt);
12083 46966 : BLOCK_VARS (block) = gimple_bind_vars (new_stmt);
12084 46966 : if (BLOCK_VARS (block))
12085 46323 : TREE_USED (block) = 1;
12086 :
12087 46966 : gimple_bind_set_body (new_stmt, body);
12088 46966 : gimple_omp_set_body (stmt, NULL);
12089 46966 : gimple_omp_for_set_pre_body (stmt, NULL);
12090 46966 : }
12091 :
12092 : /* Callback for walk_stmts. Check if the current statement only contains
12093 : GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
12094 :
12095 : static tree
12096 214603 : check_combined_parallel (gimple_stmt_iterator *gsi_p,
12097 : bool *handled_ops_p,
12098 : struct walk_stmt_info *wi)
12099 : {
12100 214603 : int *info = (int *) wi->info;
12101 214603 : gimple *stmt = gsi_stmt (*gsi_p);
12102 :
12103 214603 : *handled_ops_p = true;
12104 214603 : switch (gimple_code (stmt))
12105 : {
12106 11697 : WALK_SUBSTMTS;
12107 :
12108 : case GIMPLE_DEBUG:
12109 : break;
12110 1910 : case GIMPLE_OMP_FOR:
12111 1910 : case GIMPLE_OMP_SECTIONS:
12112 1910 : *info = *info == 0 ? 1 : -1;
12113 1910 : break;
12114 200957 : default:
12115 200957 : *info = -1;
12116 200957 : break;
12117 : }
12118 214603 : return NULL;
12119 : }
12120 :
12121 : struct omp_taskcopy_context
12122 : {
12123 : /* This field must be at the beginning, as we do "inheritance": Some
12124 : callback functions for tree-inline.cc (e.g., omp_copy_decl)
12125 : receive a copy_body_data pointer that is up-casted to an
12126 : omp_context pointer. */
12127 : copy_body_data cb;
12128 : omp_context *ctx;
12129 : };
12130 :
12131 : static tree
12132 366 : task_copyfn_copy_decl (tree var, copy_body_data *cb)
12133 : {
12134 366 : struct omp_taskcopy_context *tcctx = (struct omp_taskcopy_context *) cb;
12135 :
12136 366 : if (splay_tree_lookup (tcctx->ctx->sfield_map, (splay_tree_key) var))
12137 158 : return create_tmp_var (TREE_TYPE (var));
12138 :
12139 : return var;
12140 : }
12141 :
12142 : static tree
12143 50 : task_copyfn_remap_type (struct omp_taskcopy_context *tcctx, tree orig_type)
12144 : {
12145 50 : tree name, new_fields = NULL, type, f;
12146 :
12147 50 : type = lang_hooks.types.make_type (RECORD_TYPE);
12148 50 : name = DECL_NAME (TYPE_NAME (orig_type));
12149 50 : name = build_decl (gimple_location (tcctx->ctx->stmt),
12150 : TYPE_DECL, name, type);
12151 50 : TYPE_NAME (type) = name;
12152 :
12153 1140 : for (f = TYPE_FIELDS (orig_type); f ; f = TREE_CHAIN (f))
12154 : {
12155 1090 : tree new_f = copy_node (f);
12156 1090 : DECL_CONTEXT (new_f) = type;
12157 1090 : TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &tcctx->cb);
12158 1090 : TREE_CHAIN (new_f) = new_fields;
12159 1090 : walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &tcctx->cb, NULL);
12160 1090 : walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, &tcctx->cb, NULL);
12161 1090 : walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
12162 : &tcctx->cb, NULL);
12163 1090 : new_fields = new_f;
12164 1090 : tcctx->cb.decl_map->put (f, new_f);
12165 : }
12166 50 : TYPE_FIELDS (type) = nreverse (new_fields);
12167 50 : layout_type (type);
12168 50 : return type;
12169 : }
12170 :
12171 : /* Create task copyfn. */
12172 :
12173 : static void
12174 504 : create_task_copyfn (gomp_task *task_stmt, omp_context *ctx)
12175 : {
12176 504 : struct function *child_cfun;
12177 504 : tree child_fn, t, c, src, dst, f, sf, arg, sarg, decl;
12178 504 : tree record_type, srecord_type, bind, list;
12179 504 : bool record_needs_remap = false, srecord_needs_remap = false;
12180 504 : splay_tree_node n;
12181 504 : struct omp_taskcopy_context tcctx;
12182 504 : location_t loc = gimple_location (task_stmt);
12183 504 : size_t looptempno = 0;
12184 :
12185 504 : child_fn = gimple_omp_task_copy_fn (task_stmt);
12186 504 : task_cpyfns.safe_push (task_stmt);
12187 504 : child_cfun = DECL_STRUCT_FUNCTION (child_fn);
12188 504 : gcc_assert (child_cfun->cfg == NULL);
12189 504 : DECL_SAVED_TREE (child_fn) = alloc_stmt_list ();
12190 :
12191 : /* Reset DECL_CONTEXT on function arguments. */
12192 1512 : for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
12193 1008 : DECL_CONTEXT (t) = child_fn;
12194 :
12195 : /* Populate the function. */
12196 504 : push_gimplify_context ();
12197 504 : push_cfun (child_cfun);
12198 :
12199 504 : bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
12200 504 : TREE_SIDE_EFFECTS (bind) = 1;
12201 504 : list = NULL;
12202 504 : DECL_SAVED_TREE (child_fn) = bind;
12203 504 : DECL_SOURCE_LOCATION (child_fn) = gimple_location (task_stmt);
12204 :
12205 : /* Remap src and dst argument types if needed. */
12206 504 : record_type = ctx->record_type;
12207 504 : srecord_type = ctx->srecord_type;
12208 3102 : for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
12209 2623 : if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
12210 : {
12211 : record_needs_remap = true;
12212 : break;
12213 : }
12214 2841 : for (f = TYPE_FIELDS (srecord_type); f ; f = DECL_CHAIN (f))
12215 2362 : if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
12216 : {
12217 : srecord_needs_remap = true;
12218 : break;
12219 : }
12220 :
12221 504 : if (record_needs_remap || srecord_needs_remap)
12222 : {
12223 25 : memset (&tcctx, '\0', sizeof (tcctx));
12224 25 : tcctx.cb.src_fn = ctx->cb.src_fn;
12225 25 : tcctx.cb.dst_fn = child_fn;
12226 25 : tcctx.cb.src_node = cgraph_node::get (tcctx.cb.src_fn);
12227 25 : gcc_checking_assert (tcctx.cb.src_node);
12228 25 : tcctx.cb.dst_node = tcctx.cb.src_node;
12229 25 : tcctx.cb.src_cfun = ctx->cb.src_cfun;
12230 25 : tcctx.cb.copy_decl = task_copyfn_copy_decl;
12231 25 : tcctx.cb.eh_lp_nr = 0;
12232 25 : tcctx.cb.transform_call_graph_edges = CB_CGE_MOVE;
12233 25 : tcctx.cb.decl_map = new hash_map<tree, tree>;
12234 25 : tcctx.ctx = ctx;
12235 :
12236 25 : if (record_needs_remap)
12237 25 : record_type = task_copyfn_remap_type (&tcctx, record_type);
12238 25 : if (srecord_needs_remap)
12239 25 : srecord_type = task_copyfn_remap_type (&tcctx, srecord_type);
12240 : }
12241 : else
12242 479 : tcctx.cb.decl_map = NULL;
12243 :
12244 504 : arg = DECL_ARGUMENTS (child_fn);
12245 504 : TREE_TYPE (arg) = build_pointer_type (record_type);
12246 504 : sarg = DECL_CHAIN (arg);
12247 504 : TREE_TYPE (sarg) = build_pointer_type (srecord_type);
12248 :
12249 : /* First pass: initialize temporaries used in record_type and srecord_type
12250 : sizes and field offsets. */
12251 504 : if (tcctx.cb.decl_map)
12252 631 : for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
12253 606 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
12254 : {
12255 497 : tree *p;
12256 :
12257 497 : decl = OMP_CLAUSE_DECL (c);
12258 497 : p = tcctx.cb.decl_map->get (decl);
12259 497 : if (p == NULL)
12260 339 : continue;
12261 158 : n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
12262 158 : sf = (tree) n->value;
12263 158 : sf = *tcctx.cb.decl_map->get (sf);
12264 158 : src = build_simple_mem_ref_loc (loc, sarg);
12265 158 : src = omp_build_component_ref (src, sf);
12266 158 : t = build2 (MODIFY_EXPR, TREE_TYPE (*p), *p, src);
12267 158 : append_to_statement_list (t, &list);
12268 : }
12269 :
12270 : /* Second pass: copy shared var pointers and copy construct non-VLA
12271 : firstprivate vars. */
12272 5850 : for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
12273 5346 : switch (OMP_CLAUSE_CODE (c))
12274 : {
12275 583 : splay_tree_key key;
12276 583 : case OMP_CLAUSE_SHARED:
12277 583 : decl = OMP_CLAUSE_DECL (c);
12278 583 : key = (splay_tree_key) decl;
12279 583 : if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
12280 273 : key = (splay_tree_key) &DECL_UID (decl);
12281 583 : n = splay_tree_lookup (ctx->field_map, key);
12282 583 : if (n == NULL)
12283 : break;
12284 149 : f = (tree) n->value;
12285 149 : if (tcctx.cb.decl_map)
12286 12 : f = *tcctx.cb.decl_map->get (f);
12287 149 : n = splay_tree_lookup (ctx->sfield_map, key);
12288 149 : sf = (tree) n->value;
12289 149 : if (tcctx.cb.decl_map)
12290 12 : sf = *tcctx.cb.decl_map->get (sf);
12291 149 : src = build_simple_mem_ref_loc (loc, sarg);
12292 149 : src = omp_build_component_ref (src, sf);
12293 149 : dst = build_simple_mem_ref_loc (loc, arg);
12294 149 : dst = omp_build_component_ref (dst, f);
12295 149 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12296 149 : append_to_statement_list (t, &list);
12297 149 : break;
12298 460 : case OMP_CLAUSE_REDUCTION:
12299 460 : case OMP_CLAUSE_IN_REDUCTION:
12300 460 : decl = OMP_CLAUSE_DECL (c);
12301 460 : if (TREE_CODE (decl) == MEM_REF)
12302 : {
12303 160 : decl = TREE_OPERAND (decl, 0);
12304 160 : if (TREE_CODE (decl) == POINTER_PLUS_EXPR)
12305 38 : decl = TREE_OPERAND (decl, 0);
12306 160 : if (TREE_CODE (decl) == INDIRECT_REF
12307 128 : || TREE_CODE (decl) == ADDR_EXPR)
12308 95 : decl = TREE_OPERAND (decl, 0);
12309 : }
12310 460 : key = (splay_tree_key) decl;
12311 460 : n = splay_tree_lookup (ctx->field_map, key);
12312 460 : if (n == NULL)
12313 : break;
12314 101 : f = (tree) n->value;
12315 101 : if (tcctx.cb.decl_map)
12316 20 : f = *tcctx.cb.decl_map->get (f);
12317 101 : n = splay_tree_lookup (ctx->sfield_map, key);
12318 101 : sf = (tree) n->value;
12319 101 : if (tcctx.cb.decl_map)
12320 20 : sf = *tcctx.cb.decl_map->get (sf);
12321 101 : src = build_simple_mem_ref_loc (loc, sarg);
12322 101 : src = omp_build_component_ref (src, sf);
12323 101 : if (decl != OMP_CLAUSE_DECL (c)
12324 95 : && TREE_CODE (TREE_TYPE (decl)) == REFERENCE_TYPE
12325 173 : && TREE_CODE (TREE_TYPE (TREE_TYPE (decl))) == POINTER_TYPE)
12326 32 : src = build_simple_mem_ref_loc (loc, src);
12327 101 : dst = build_simple_mem_ref_loc (loc, arg);
12328 101 : dst = omp_build_component_ref (dst, f);
12329 101 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12330 101 : append_to_statement_list (t, &list);
12331 101 : break;
12332 754 : case OMP_CLAUSE__LOOPTEMP_:
12333 : /* Fields for first two _looptemp_ clauses are initialized by
12334 : GOMP_taskloop*, the rest are handled like firstprivate. */
12335 754 : if (looptempno < 2)
12336 : {
12337 748 : looptempno++;
12338 748 : break;
12339 : }
12340 : /* FALLTHRU */
12341 1731 : case OMP_CLAUSE__REDUCTEMP_:
12342 1731 : case OMP_CLAUSE_FIRSTPRIVATE:
12343 1731 : decl = OMP_CLAUSE_DECL (c);
12344 1731 : if (is_variable_sized (decl))
12345 : break;
12346 1717 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
12347 1717 : if (n == NULL)
12348 : break;
12349 1717 : f = (tree) n->value;
12350 1717 : if (tcctx.cb.decl_map)
12351 490 : f = *tcctx.cb.decl_map->get (f);
12352 1717 : n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
12353 1717 : if (n != NULL)
12354 : {
12355 1467 : sf = (tree) n->value;
12356 1467 : if (tcctx.cb.decl_map)
12357 490 : sf = *tcctx.cb.decl_map->get (sf);
12358 1467 : src = build_simple_mem_ref_loc (loc, sarg);
12359 1467 : src = omp_build_component_ref (src, sf);
12360 1467 : if (use_pointer_for_field (decl, NULL)
12361 1467 : || omp_privatize_by_reference (decl))
12362 317 : src = build_simple_mem_ref_loc (loc, src);
12363 : }
12364 : else
12365 : src = decl;
12366 1717 : dst = build_simple_mem_ref_loc (loc, arg);
12367 1717 : dst = omp_build_component_ref (dst, f);
12368 1717 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FIRSTPRIVATE)
12369 192 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12370 : else
12371 : {
12372 1525 : if (ctx->allocate_map)
12373 84 : if (tree *allocatorp = ctx->allocate_map->get (decl))
12374 : {
12375 50 : tree allocator = *allocatorp;
12376 50 : HOST_WIDE_INT ialign = 0;
12377 50 : if (TREE_CODE (allocator) == TREE_LIST)
12378 : {
12379 4 : ialign = tree_to_uhwi (TREE_VALUE (allocator));
12380 4 : allocator = TREE_PURPOSE (allocator);
12381 : }
12382 50 : if (TREE_CODE (allocator) != INTEGER_CST)
12383 : {
12384 22 : n = splay_tree_lookup (ctx->sfield_map,
12385 : (splay_tree_key) allocator);
12386 22 : allocator = (tree) n->value;
12387 22 : if (tcctx.cb.decl_map)
12388 0 : allocator = *tcctx.cb.decl_map->get (allocator);
12389 22 : tree a = build_simple_mem_ref_loc (loc, sarg);
12390 22 : allocator = omp_build_component_ref (a, allocator);
12391 : }
12392 50 : allocator = fold_convert (pointer_sized_int_node, allocator);
12393 50 : tree a = builtin_decl_explicit (BUILT_IN_GOMP_ALLOC);
12394 50 : tree align = build_int_cst (size_type_node,
12395 50 : MAX (ialign,
12396 : DECL_ALIGN_UNIT (decl)));
12397 50 : tree sz = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (dst)));
12398 50 : tree ptr = build_call_expr_loc (loc, a, 3, align, sz,
12399 : allocator);
12400 50 : ptr = fold_convert (TREE_TYPE (dst), ptr);
12401 50 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, ptr);
12402 50 : append_to_statement_list (t, &list);
12403 50 : dst = build_simple_mem_ref_loc (loc, dst);
12404 : }
12405 1525 : t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
12406 : }
12407 1717 : append_to_statement_list (t, &list);
12408 1717 : break;
12409 489 : case OMP_CLAUSE_PRIVATE:
12410 489 : if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c))
12411 : break;
12412 12 : decl = OMP_CLAUSE_DECL (c);
12413 12 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
12414 12 : f = (tree) n->value;
12415 12 : if (tcctx.cb.decl_map)
12416 0 : f = *tcctx.cb.decl_map->get (f);
12417 12 : n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
12418 12 : if (n != NULL)
12419 : {
12420 12 : sf = (tree) n->value;
12421 12 : if (tcctx.cb.decl_map)
12422 0 : sf = *tcctx.cb.decl_map->get (sf);
12423 12 : src = build_simple_mem_ref_loc (loc, sarg);
12424 12 : src = omp_build_component_ref (src, sf);
12425 12 : if (use_pointer_for_field (decl, NULL))
12426 12 : src = build_simple_mem_ref_loc (loc, src);
12427 : }
12428 : else
12429 : src = decl;
12430 12 : dst = build_simple_mem_ref_loc (loc, arg);
12431 12 : dst = omp_build_component_ref (dst, f);
12432 12 : t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
12433 12 : append_to_statement_list (t, &list);
12434 12 : break;
12435 : default:
12436 : break;
12437 : }
12438 :
12439 : /* Last pass: handle VLA firstprivates. */
12440 504 : if (tcctx.cb.decl_map)
12441 631 : for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
12442 606 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
12443 : {
12444 497 : tree ind, ptr, df;
12445 :
12446 497 : decl = OMP_CLAUSE_DECL (c);
12447 497 : if (!is_variable_sized (decl))
12448 483 : continue;
12449 14 : n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
12450 14 : if (n == NULL)
12451 0 : continue;
12452 14 : f = (tree) n->value;
12453 14 : f = *tcctx.cb.decl_map->get (f);
12454 14 : gcc_assert (DECL_HAS_VALUE_EXPR_P (decl));
12455 14 : ind = DECL_VALUE_EXPR (decl);
12456 14 : gcc_assert (TREE_CODE (ind) == INDIRECT_REF);
12457 14 : gcc_assert (DECL_P (TREE_OPERAND (ind, 0)));
12458 42 : n = splay_tree_lookup (ctx->sfield_map,
12459 14 : (splay_tree_key) TREE_OPERAND (ind, 0));
12460 14 : sf = (tree) n->value;
12461 14 : sf = *tcctx.cb.decl_map->get (sf);
12462 14 : src = build_simple_mem_ref_loc (loc, sarg);
12463 14 : src = omp_build_component_ref (src, sf);
12464 14 : src = build_simple_mem_ref_loc (loc, src);
12465 14 : dst = build_simple_mem_ref_loc (loc, arg);
12466 14 : dst = omp_build_component_ref (dst, f);
12467 14 : t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
12468 14 : append_to_statement_list (t, &list);
12469 42 : n = splay_tree_lookup (ctx->field_map,
12470 14 : (splay_tree_key) TREE_OPERAND (ind, 0));
12471 14 : df = (tree) n->value;
12472 14 : df = *tcctx.cb.decl_map->get (df);
12473 14 : ptr = build_simple_mem_ref_loc (loc, arg);
12474 14 : ptr = omp_build_component_ref (ptr, df);
12475 14 : t = build2 (MODIFY_EXPR, TREE_TYPE (ptr), ptr,
12476 : build_fold_addr_expr_loc (loc, dst));
12477 14 : append_to_statement_list (t, &list);
12478 : }
12479 :
12480 504 : t = build1 (RETURN_EXPR, void_type_node, NULL);
12481 504 : append_to_statement_list (t, &list);
12482 :
12483 504 : if (tcctx.cb.decl_map)
12484 25 : delete tcctx.cb.decl_map;
12485 504 : pop_gimplify_context (NULL);
12486 504 : BIND_EXPR_BODY (bind) = list;
12487 504 : pop_cfun ();
12488 504 : }
12489 :
12490 : static void
12491 1553 : lower_depend_clauses (tree *pclauses, gimple_seq *iseq, gimple_seq *oseq)
12492 : {
12493 1553 : tree c, clauses;
12494 1553 : gimple *g;
12495 1553 : size_t cnt[5] = { 0, 0, 0, 0, 0 }, idx = 2, i;
12496 :
12497 1553 : clauses = omp_find_clause (*pclauses, OMP_CLAUSE_DEPEND);
12498 1553 : gcc_assert (clauses);
12499 6314 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
12500 4886 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
12501 1713 : switch (OMP_CLAUSE_DEPEND_KIND (c))
12502 : {
12503 125 : case OMP_CLAUSE_DEPEND_LAST:
12504 : /* Lowering already done at gimplification. */
12505 125 : return;
12506 481 : case OMP_CLAUSE_DEPEND_IN:
12507 481 : cnt[2]++;
12508 481 : break;
12509 969 : case OMP_CLAUSE_DEPEND_OUT:
12510 969 : case OMP_CLAUSE_DEPEND_INOUT:
12511 969 : cnt[0]++;
12512 969 : break;
12513 30 : case OMP_CLAUSE_DEPEND_MUTEXINOUTSET:
12514 30 : cnt[1]++;
12515 30 : break;
12516 76 : case OMP_CLAUSE_DEPEND_DEPOBJ:
12517 76 : cnt[3]++;
12518 76 : break;
12519 32 : case OMP_CLAUSE_DEPEND_INOUTSET:
12520 32 : cnt[4]++;
12521 32 : break;
12522 0 : default:
12523 0 : gcc_unreachable ();
12524 : }
12525 1428 : if (cnt[1] || cnt[3] || cnt[4])
12526 135 : idx = 5;
12527 1428 : size_t total = cnt[0] + cnt[1] + cnt[2] + cnt[3] + cnt[4];
12528 1428 : size_t inoutidx = total + idx;
12529 1428 : tree type = build_array_type_nelts (ptr_type_node, total + idx + 2 * cnt[4]);
12530 1428 : tree array = create_tmp_var (type);
12531 1428 : TREE_ADDRESSABLE (array) = 1;
12532 1428 : tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE,
12533 : NULL_TREE);
12534 1428 : if (idx == 5)
12535 : {
12536 135 : g = gimple_build_assign (r, build_int_cst (ptr_type_node, 0));
12537 135 : gimple_seq_add_stmt (iseq, g);
12538 135 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE,
12539 : NULL_TREE);
12540 : }
12541 1428 : g = gimple_build_assign (r, build_int_cst (ptr_type_node, total));
12542 1428 : gimple_seq_add_stmt (iseq, g);
12543 7140 : for (i = 0; i < (idx == 5 ? 3 : 1); i++)
12544 : {
12545 1698 : r = build4 (ARRAY_REF, ptr_type_node, array,
12546 1698 : size_int (i + 1 + (idx == 5)), NULL_TREE, NULL_TREE);
12547 1698 : g = gimple_build_assign (r, build_int_cst (ptr_type_node, cnt[i]));
12548 1698 : gimple_seq_add_stmt (iseq, g);
12549 : }
12550 8568 : for (i = 0; i < 5; i++)
12551 : {
12552 7140 : if (cnt[i] == 0)
12553 5656 : continue;
12554 6563 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
12555 5079 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DEPEND)
12556 3352 : continue;
12557 : else
12558 : {
12559 1727 : switch (OMP_CLAUSE_DEPEND_KIND (c))
12560 : {
12561 547 : case OMP_CLAUSE_DEPEND_IN:
12562 547 : if (i != 2)
12563 139 : continue;
12564 : break;
12565 1025 : case OMP_CLAUSE_DEPEND_OUT:
12566 1025 : case OMP_CLAUSE_DEPEND_INOUT:
12567 1025 : if (i != 0)
12568 56 : continue;
12569 : break;
12570 44 : case OMP_CLAUSE_DEPEND_MUTEXINOUTSET:
12571 44 : if (i != 1)
12572 14 : continue;
12573 : break;
12574 79 : case OMP_CLAUSE_DEPEND_DEPOBJ:
12575 79 : if (i != 3)
12576 3 : continue;
12577 : break;
12578 32 : case OMP_CLAUSE_DEPEND_INOUTSET:
12579 32 : if (i != 4)
12580 0 : continue;
12581 : break;
12582 0 : default:
12583 0 : gcc_unreachable ();
12584 : }
12585 1588 : tree t = OMP_CLAUSE_DECL (c);
12586 1588 : if (i == 4)
12587 : {
12588 32 : t = build4 (ARRAY_REF, ptr_type_node, array,
12589 32 : size_int (inoutidx), NULL_TREE, NULL_TREE);
12590 32 : t = build_fold_addr_expr (t);
12591 32 : inoutidx += 2;
12592 : }
12593 1588 : t = fold_convert (ptr_type_node, t);
12594 1588 : gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
12595 1588 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
12596 : NULL_TREE, NULL_TREE);
12597 1588 : g = gimple_build_assign (r, t);
12598 1588 : gimple_seq_add_stmt (iseq, g);
12599 : }
12600 : }
12601 1428 : if (cnt[4])
12602 81 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
12603 49 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
12604 49 : && OMP_CLAUSE_DEPEND_KIND (c) == OMP_CLAUSE_DEPEND_INOUTSET)
12605 : {
12606 32 : tree t = OMP_CLAUSE_DECL (c);
12607 32 : t = fold_convert (ptr_type_node, t);
12608 32 : gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
12609 32 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
12610 : NULL_TREE, NULL_TREE);
12611 32 : g = gimple_build_assign (r, t);
12612 32 : gimple_seq_add_stmt (iseq, g);
12613 32 : t = build_int_cst (ptr_type_node, GOMP_DEPEND_INOUTSET);
12614 32 : r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
12615 : NULL_TREE, NULL_TREE);
12616 32 : g = gimple_build_assign (r, t);
12617 32 : gimple_seq_add_stmt (iseq, g);
12618 : }
12619 :
12620 1428 : c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND);
12621 1428 : OMP_CLAUSE_DEPEND_KIND (c) = OMP_CLAUSE_DEPEND_LAST;
12622 1428 : OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array);
12623 1428 : OMP_CLAUSE_CHAIN (c) = *pclauses;
12624 1428 : *pclauses = c;
12625 1428 : tree clobber = build_clobber (type);
12626 1428 : g = gimple_build_assign (array, clobber);
12627 1428 : gimple_seq_add_stmt (oseq, g);
12628 : }
12629 :
12630 : /* Lower the OpenMP parallel or task directive in the current statement
12631 : in GSI_P. CTX holds context information for the directive. */
12632 :
12633 : static void
12634 22570 : lower_omp_taskreg (gimple_stmt_iterator *gsi_p, omp_context *ctx)
12635 : {
12636 22570 : tree clauses;
12637 22570 : tree child_fn, t;
12638 22570 : gimple *stmt = gsi_stmt (*gsi_p);
12639 22570 : gbind *par_bind, *bind, *dep_bind = NULL;
12640 22570 : gimple_seq par_body;
12641 22570 : location_t loc = gimple_location (stmt);
12642 :
12643 22570 : clauses = gimple_omp_taskreg_clauses (stmt);
12644 22570 : if (gimple_code (stmt) == GIMPLE_OMP_TASK
12645 22570 : && gimple_omp_task_taskwait_p (stmt))
12646 : {
12647 84 : par_bind = NULL;
12648 84 : par_body = NULL;
12649 : }
12650 : else
12651 : {
12652 22486 : par_bind
12653 22486 : = as_a <gbind *> (gimple_seq_first_stmt (gimple_omp_body (stmt)));
12654 22486 : par_body = gimple_bind_body (par_bind);
12655 : }
12656 22570 : child_fn = ctx->cb.dst_fn;
12657 22570 : if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
12658 22570 : && !gimple_omp_parallel_combined_p (stmt))
12659 : {
12660 4257 : struct walk_stmt_info wi;
12661 4257 : int ws_num = 0;
12662 :
12663 4257 : memset (&wi, 0, sizeof (wi));
12664 4257 : wi.info = &ws_num;
12665 4257 : wi.val_only = true;
12666 4257 : walk_gimple_seq (par_body, check_combined_parallel, NULL, &wi);
12667 4257 : if (ws_num == 1)
12668 386 : gimple_omp_parallel_set_combined_p (stmt, true);
12669 : }
12670 22570 : gimple_seq dep_ilist = NULL;
12671 22570 : gimple_seq dep_olist = NULL;
12672 22570 : if (gimple_code (stmt) == GIMPLE_OMP_TASK
12673 22570 : && omp_find_clause (clauses, OMP_CLAUSE_DEPEND))
12674 : {
12675 1189 : push_gimplify_context ();
12676 1189 : dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12677 1189 : lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt),
12678 : &dep_ilist, &dep_olist);
12679 : }
12680 :
12681 22570 : if (gimple_code (stmt) == GIMPLE_OMP_TASK
12682 22570 : && gimple_omp_task_taskwait_p (stmt))
12683 : {
12684 84 : if (dep_bind)
12685 : {
12686 84 : gsi_replace (gsi_p, dep_bind, true);
12687 84 : gimple_bind_add_seq (dep_bind, dep_ilist);
12688 84 : gimple_bind_add_stmt (dep_bind, stmt);
12689 84 : gimple_bind_add_seq (dep_bind, dep_olist);
12690 84 : pop_gimplify_context (dep_bind);
12691 : }
12692 84 : return;
12693 : }
12694 :
12695 22486 : if (ctx->srecord_type)
12696 504 : create_task_copyfn (as_a <gomp_task *> (stmt), ctx);
12697 :
12698 22486 : gimple_seq tskred_ilist = NULL;
12699 22486 : gimple_seq tskred_olist = NULL;
12700 22486 : if ((is_task_ctx (ctx)
12701 3778 : && gimple_omp_task_taskloop_p (ctx->stmt)
12702 1330 : && omp_find_clause (gimple_omp_task_clauses (ctx->stmt),
12703 : OMP_CLAUSE_REDUCTION))
12704 25767 : || (is_parallel_ctx (ctx)
12705 16174 : && omp_find_clause (gimple_omp_parallel_clauses (stmt),
12706 : OMP_CLAUSE__REDUCTEMP_)))
12707 : {
12708 557 : if (dep_bind == NULL)
12709 : {
12710 557 : push_gimplify_context ();
12711 557 : dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12712 : }
12713 617 : lower_omp_task_reductions (ctx, is_task_ctx (ctx) ? OMP_TASKLOOP
12714 : : OMP_PARALLEL,
12715 557 : gimple_omp_taskreg_clauses (ctx->stmt),
12716 : &tskred_ilist, &tskred_olist);
12717 : }
12718 :
12719 22486 : push_gimplify_context ();
12720 :
12721 22486 : gimple_seq par_olist = NULL;
12722 22486 : gimple_seq par_ilist = NULL;
12723 22486 : gimple_seq par_rlist = NULL;
12724 22486 : lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx, NULL);
12725 22482 : lower_omp (&par_body, ctx);
12726 22482 : if (gimple_code (stmt) != GIMPLE_OMP_TASK)
12727 18704 : lower_reduction_clauses (clauses, &par_rlist, NULL, ctx);
12728 :
12729 : /* Declare all the variables created by mapping and the variables
12730 : declared in the scope of the parallel body. */
12731 22482 : record_vars_into (ctx->block_vars, child_fn);
12732 22482 : maybe_remove_omp_member_access_dummy_vars (par_bind);
12733 22482 : record_vars_into (gimple_bind_vars (par_bind), child_fn);
12734 :
12735 22482 : if (ctx->record_type)
12736 : {
12737 18018 : ctx->sender_decl
12738 35532 : = create_tmp_var (ctx->srecord_type ? ctx->srecord_type
12739 : : ctx->record_type, ".omp_data_o");
12740 18018 : DECL_NAMELESS (ctx->sender_decl) = 1;
12741 18018 : TREE_ADDRESSABLE (ctx->sender_decl) = 1;
12742 18018 : gimple_omp_taskreg_set_data_arg (stmt, ctx->sender_decl);
12743 : }
12744 :
12745 22482 : gimple_seq olist = NULL;
12746 22482 : gimple_seq ilist = NULL;
12747 22482 : lower_send_clauses (clauses, &ilist, &olist, ctx);
12748 22482 : lower_send_shared_vars (&ilist, &olist, ctx);
12749 :
12750 22482 : if (ctx->record_type)
12751 : {
12752 18018 : tree clobber = build_clobber (TREE_TYPE (ctx->sender_decl));
12753 18018 : gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
12754 : clobber));
12755 : }
12756 :
12757 : /* Once all the expansions are done, sequence all the different
12758 : fragments inside gimple_omp_body. */
12759 :
12760 22482 : gimple_seq new_body = NULL;
12761 :
12762 22482 : if (ctx->record_type)
12763 : {
12764 18018 : t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
12765 : /* fixup_child_record_type might have changed receiver_decl's type. */
12766 18018 : t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
12767 18018 : gimple_seq_add_stmt (&new_body,
12768 18018 : gimple_build_assign (ctx->receiver_decl, t));
12769 : }
12770 :
12771 22482 : gimple_seq_add_seq (&new_body, par_ilist);
12772 22482 : gimple_seq_add_seq (&new_body, par_body);
12773 22482 : gimple_seq_add_seq (&new_body, par_rlist);
12774 22482 : if (ctx->cancellable)
12775 135 : gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
12776 22482 : gimple_seq_add_seq (&new_body, par_olist);
12777 22482 : new_body = maybe_catch_exception (new_body);
12778 22482 : if (gimple_code (stmt) == GIMPLE_OMP_TASK)
12779 3778 : gimple_seq_add_stmt (&new_body,
12780 3778 : gimple_build_omp_continue (integer_zero_node,
12781 : integer_zero_node));
12782 22482 : gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
12783 22482 : gimple_omp_set_body (stmt, new_body);
12784 :
12785 22482 : if (dep_bind && gimple_bind_block (par_bind) == NULL_TREE)
12786 497 : bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12787 : else
12788 21985 : bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind));
12789 43302 : gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
12790 22482 : gimple_bind_add_seq (bind, ilist);
12791 22482 : gimple_bind_add_stmt (bind, stmt);
12792 22482 : gimple_bind_add_seq (bind, olist);
12793 :
12794 22482 : pop_gimplify_context (NULL);
12795 :
12796 22482 : if (dep_bind)
12797 : {
12798 1662 : gimple_bind_add_seq (dep_bind, dep_ilist);
12799 1662 : gimple_bind_add_seq (dep_bind, tskred_ilist);
12800 1662 : gimple_bind_add_stmt (dep_bind, bind);
12801 1662 : gimple_bind_add_seq (dep_bind, tskred_olist);
12802 1662 : gimple_bind_add_seq (dep_bind, dep_olist);
12803 1662 : pop_gimplify_context (dep_bind);
12804 : }
12805 : }
12806 :
12807 : /* Set EXPR as the hostaddr expression that should result from the clause C
12808 : in the target statement STMT. Returns the tree that should be
12809 : passed as the hostaddr (a pointer to the array containing the expanded
12810 : hostaddrs and sizes of the clause). */
12811 :
12812 : static tree
12813 37312 : lower_omp_map_iterator_expr (tree expr, tree c, gomp_target *stmt)
12814 : {
12815 37312 : if (!OMP_CLAUSE_HAS_ITERATORS (c))
12816 : return expr;
12817 :
12818 215 : tree iterator = OMP_CLAUSE_ITERATORS (c);
12819 215 : tree index = OMP_ITERATOR_INDEX (iterator);
12820 215 : tree elems = OMP_ITERATOR_ELEMS (iterator);
12821 215 : gimple_seq *loop_body_p = enter_omp_iterator_loop_context (c, stmt);
12822 :
12823 : /* IN LOOP BODY: */
12824 : /* elems[idx] = <expr>; */
12825 215 : tree lhs = build4 (ARRAY_REF, ptr_type_node, elems, index,
12826 : NULL_TREE, NULL_TREE);
12827 215 : tree mod_expr = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
12828 : void_type_node, lhs, expr);
12829 215 : gimplify_and_add (mod_expr, loop_body_p);
12830 215 : exit_omp_iterator_loop_context (c);
12831 :
12832 215 : return build_fold_addr_expr_with_type (elems, ptr_type_node);
12833 : }
12834 :
12835 : /* Set SIZE as the size expression that should result from the clause C
12836 : in the target statement STMT. Returns the tree that should be
12837 : passed as the clause size (a size_int with the value SIZE_MAX, indicating
12838 : that the clause uses an iterator). */
12839 :
12840 : static tree
12841 80146 : lower_omp_map_iterator_size (tree size, tree c, gomp_target *stmt)
12842 : {
12843 80146 : if (!OMP_CLAUSE_HAS_ITERATORS (c))
12844 : return size;
12845 :
12846 219 : tree iterator = OMP_CLAUSE_ITERATORS (c);
12847 219 : tree index = OMP_ITERATOR_INDEX (iterator);
12848 219 : tree elems = OMP_ITERATOR_ELEMS (iterator);
12849 219 : gimple_seq *loop_body_p = enter_omp_iterator_loop_context (c, stmt);
12850 :
12851 : /* IN LOOP BODY: */
12852 : /* elems[idx+1] = <size>; */
12853 219 : tree lhs = build4 (ARRAY_REF, ptr_type_node, elems,
12854 : size_binop (PLUS_EXPR, index, size_int (1)),
12855 : NULL_TREE, NULL_TREE);
12856 219 : tree mod_expr = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
12857 : void_type_node, lhs, size);
12858 219 : gimplify_and_add (mod_expr, loop_body_p);
12859 219 : exit_omp_iterator_loop_context (c);
12860 :
12861 219 : return size_int (SIZE_MAX);
12862 : }
12863 :
12864 : /* Lower the GIMPLE_OMP_TARGET in the current statement
12865 : in GSI_P. CTX holds context information for the directive. */
12866 :
12867 : static void
12868 37170 : lower_omp_target (gimple_stmt_iterator *gsi_p, omp_context *ctx)
12869 : {
12870 37170 : tree clauses;
12871 37170 : tree child_fn, t, c;
12872 37170 : gomp_target *stmt = as_a <gomp_target *> (gsi_stmt (*gsi_p));
12873 37170 : gbind *tgt_bind, *bind, *dep_bind = NULL;
12874 37170 : gimple_seq tgt_body, olist, ilist, fplist, new_body;
12875 37170 : location_t loc = gimple_location (stmt);
12876 37170 : bool offloaded, data_region;
12877 37170 : unsigned int map_cnt = 0;
12878 37170 : tree in_reduction_clauses = NULL_TREE;
12879 :
12880 37170 : tree deep_map_cnt = NULL_TREE;
12881 37170 : tree deep_map_data = NULL_TREE;
12882 37170 : tree deep_map_offset_data = NULL_TREE;
12883 37170 : tree deep_map_offset = NULL_TREE;
12884 :
12885 37170 : offloaded = is_gimple_omp_offloaded (stmt);
12886 37170 : switch (gimple_omp_target_kind (stmt))
12887 : {
12888 11815 : case GF_OMP_TARGET_KIND_REGION:
12889 11815 : tree *p, *q;
12890 11815 : q = &in_reduction_clauses;
12891 79451 : for (p = gimple_omp_target_clauses_ptr (stmt); *p; )
12892 67636 : if (OMP_CLAUSE_CODE (*p) == OMP_CLAUSE_IN_REDUCTION)
12893 : {
12894 422 : *q = *p;
12895 422 : q = &OMP_CLAUSE_CHAIN (*q);
12896 422 : *p = OMP_CLAUSE_CHAIN (*p);
12897 : }
12898 : else
12899 67214 : p = &OMP_CLAUSE_CHAIN (*p);
12900 11815 : *q = NULL_TREE;
12901 11815 : *p = in_reduction_clauses;
12902 : /* FALLTHRU */
12903 : case GF_OMP_TARGET_KIND_UPDATE:
12904 : case GF_OMP_TARGET_KIND_ENTER_DATA:
12905 : case GF_OMP_TARGET_KIND_EXIT_DATA:
12906 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
12907 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
12908 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
12909 : case GF_OMP_TARGET_KIND_OACC_UPDATE:
12910 : case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
12911 : case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
12912 : case GF_OMP_TARGET_KIND_OACC_DECLARE:
12913 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
12914 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
12915 : data_region = false;
12916 : break;
12917 : case GF_OMP_TARGET_KIND_DATA:
12918 : case GF_OMP_TARGET_KIND_OACC_DATA:
12919 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
12920 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
12921 : data_region = true;
12922 : break;
12923 0 : default:
12924 0 : gcc_unreachable ();
12925 : }
12926 :
12927 : /* Ensure that requires map is written via output_offload_tables, even if only
12928 : 'target (enter/exit) data' is used in the translation unit. */
12929 37170 : if (ENABLE_OFFLOADING && (omp_requires_mask & OMP_REQUIRES_TARGET_USED))
12930 : g->have_offload = true;
12931 :
12932 37170 : clauses = gimple_omp_target_clauses (stmt);
12933 :
12934 37170 : gimple_seq dep_ilist = NULL;
12935 37170 : gimple_seq dep_olist = NULL;
12936 37170 : bool has_depend = omp_find_clause (clauses, OMP_CLAUSE_DEPEND) != NULL_TREE;
12937 37170 : if (has_depend || in_reduction_clauses)
12938 : {
12939 392 : push_gimplify_context ();
12940 392 : dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
12941 392 : if (has_depend)
12942 329 : lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt),
12943 : &dep_ilist, &dep_olist);
12944 392 : if (in_reduction_clauses)
12945 314 : lower_rec_input_clauses (in_reduction_clauses, &dep_ilist, &dep_olist,
12946 : ctx, NULL);
12947 : }
12948 :
12949 37170 : tgt_bind = NULL;
12950 37170 : tgt_body = NULL;
12951 37170 : if (offloaded)
12952 : {
12953 21210 : tgt_bind = gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt));
12954 21210 : tgt_body = gimple_bind_body (tgt_bind);
12955 : }
12956 15960 : else if (data_region)
12957 4021 : tgt_body = gimple_omp_body (stmt);
12958 37170 : child_fn = ctx->cb.dst_fn;
12959 :
12960 37170 : push_gimplify_context ();
12961 37170 : fplist = NULL;
12962 :
12963 37170 : ilist = NULL;
12964 37170 : olist = NULL;
12965 :
12966 37170 : gimple_seq alloc_dlist = NULL;
12967 37170 : hash_map<tree, tree> alloc_map;
12968 74340 : hash_map<tree, gimple_seq> alloc_seq_map;
12969 :
12970 : /* The value expression for USES_ALLOCATORS needs to be setup before the
12971 : allocator is used via lower_private_allocate. */
12972 176484 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
12973 139314 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USES_ALLOCATORS)
12974 : {
12975 86 : tree allocator = OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c);
12976 86 : tree new_allocator = lookup_decl (allocator, ctx);
12977 86 : tree x = build_receiver_ref (allocator, false, ctx);
12978 86 : SET_DECL_VALUE_EXPR (new_allocator, x);
12979 86 : DECL_HAS_VALUE_EXPR_P (new_allocator) = 1;
12980 86 : map_cnt++;
12981 : }
12982 :
12983 176484 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
12984 139314 : switch (OMP_CLAUSE_CODE (c))
12985 : {
12986 : tree var, x, new_var, allocator, allocate_ptr, size;
12987 : gimple_seq alloc_seq;
12988 : bool by_ref;
12989 :
12990 : default:
12991 92890 : break;
12992 77876 : case OMP_CLAUSE_MAP:
12993 : #if CHECKING_P
12994 : /* First check what we're prepared to handle in the following. */
12995 77876 : switch (OMP_CLAUSE_MAP_KIND (c))
12996 : {
12997 : case GOMP_MAP_ALLOC:
12998 : case GOMP_MAP_TO:
12999 : case GOMP_MAP_FROM:
13000 : case GOMP_MAP_TOFROM:
13001 : case GOMP_MAP_POINTER:
13002 : case GOMP_MAP_TO_PSET:
13003 : case GOMP_MAP_DELETE:
13004 : case GOMP_MAP_RELEASE:
13005 : case GOMP_MAP_ALWAYS_TO:
13006 : case GOMP_MAP_ALWAYS_FROM:
13007 : case GOMP_MAP_ALWAYS_TOFROM:
13008 : case GOMP_MAP_FORCE_PRESENT:
13009 : case GOMP_MAP_ALWAYS_PRESENT_FROM:
13010 : case GOMP_MAP_ALWAYS_PRESENT_TO:
13011 : case GOMP_MAP_ALWAYS_PRESENT_TOFROM:
13012 :
13013 : case GOMP_MAP_FIRSTPRIVATE_POINTER:
13014 : case GOMP_MAP_FIRSTPRIVATE_REFERENCE:
13015 : case GOMP_MAP_STRUCT:
13016 : case GOMP_MAP_STRUCT_UNORD:
13017 : case GOMP_MAP_ALWAYS_POINTER:
13018 : case GOMP_MAP_ATTACH:
13019 : case GOMP_MAP_DETACH:
13020 : case GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION:
13021 : case GOMP_MAP_POINTER_TO_ZERO_LENGTH_ARRAY_SECTION:
13022 : break;
13023 2587 : case GOMP_MAP_IF_PRESENT:
13024 2587 : case GOMP_MAP_FORCE_ALLOC:
13025 2587 : case GOMP_MAP_FORCE_TO:
13026 2587 : case GOMP_MAP_FORCE_FROM:
13027 2587 : case GOMP_MAP_FORCE_TOFROM:
13028 2587 : case GOMP_MAP_FORCE_DEVICEPTR:
13029 2587 : case GOMP_MAP_DEVICE_RESIDENT:
13030 2587 : case GOMP_MAP_LINK:
13031 2587 : case GOMP_MAP_FORCE_DETACH:
13032 2587 : gcc_assert (is_gimple_omp_oacc (stmt));
13033 : break;
13034 0 : default:
13035 0 : gcc_unreachable ();
13036 : }
13037 : #endif
13038 : /* FALLTHRU */
13039 89446 : case OMP_CLAUSE_TO:
13040 89446 : case OMP_CLAUSE_FROM:
13041 89446 : oacc_firstprivate:
13042 89446 : var = OMP_CLAUSE_DECL (c);
13043 89446 : {
13044 89446 : tree extra = lang_hooks.decls.omp_deep_mapping_cnt (stmt, c, &ilist);
13045 89446 : if (extra != NULL_TREE && deep_map_cnt != NULL_TREE)
13046 59 : deep_map_cnt = fold_build2_loc (OMP_CLAUSE_LOCATION (c), PLUS_EXPR,
13047 : size_type_node, deep_map_cnt,
13048 : extra);
13049 89387 : else if (extra != NULL_TREE)
13050 : deep_map_cnt = extra;
13051 : }
13052 :
13053 89302 : if (deep_map_cnt
13054 89668 : && OMP_CLAUSE_HAS_ITERATORS (c))
13055 0 : sorry ("iterators used together with deep mapping are not "
13056 : "supported yet");
13057 :
13058 89446 : if (!DECL_P (var))
13059 : {
13060 36887 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
13061 36887 : || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
13062 35281 : && (OMP_CLAUSE_MAP_KIND (c)
13063 : != GOMP_MAP_FIRSTPRIVATE_POINTER)))
13064 36887 : map_cnt++;
13065 46424 : continue;
13066 : }
13067 :
13068 52559 : if (DECL_SIZE (var)
13069 52559 : && !poly_int_tree_p (DECL_SIZE (var)))
13070 : {
13071 717 : tree var2 = DECL_VALUE_EXPR (var);
13072 717 : gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
13073 717 : var2 = TREE_OPERAND (var2, 0);
13074 717 : gcc_assert (DECL_P (var2));
13075 : var = var2;
13076 : }
13077 :
13078 52559 : if (offloaded
13079 36693 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13080 85356 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
13081 29557 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE))
13082 : {
13083 3669 : if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
13084 : {
13085 415 : if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx))
13086 415 : && varpool_node::get_create (var)->offloadable)
13087 2 : continue;
13088 :
13089 413 : tree type = build_pointer_type (TREE_TYPE (var));
13090 413 : tree new_var = lookup_decl (var, ctx);
13091 413 : x = create_tmp_var_raw (type, get_name (new_var));
13092 413 : gimple_add_tmp_var (x);
13093 413 : x = build_simple_mem_ref (x);
13094 413 : SET_DECL_VALUE_EXPR (new_var, x);
13095 413 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13096 : }
13097 3667 : continue;
13098 3667 : }
13099 :
13100 48890 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13101 38926 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
13102 38719 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
13103 49151 : && is_omp_target (stmt))
13104 : {
13105 237 : gcc_assert (maybe_lookup_field (c, ctx));
13106 237 : map_cnt++;
13107 237 : continue;
13108 : }
13109 :
13110 48653 : if (!maybe_lookup_field (c, ctx))
13111 5631 : continue;
13112 :
13113 : /* Don't remap compute constructs' reduction variables, because the
13114 : intermediate result must be local to each gang. */
13115 70872 : if (offloaded && !(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13116 23954 : && is_gimple_omp_oacc (ctx->stmt)
13117 13572 : && OMP_CLAUSE_MAP_IN_REDUCTION (c)))
13118 : {
13119 27070 : x = build_receiver_ref (c, true, ctx);
13120 27070 : tree new_var = lookup_decl (var, ctx);
13121 :
13122 27070 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13123 23174 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
13124 3014 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
13125 30084 : && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
13126 425 : x = build_simple_mem_ref (x);
13127 27070 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
13128 : {
13129 3896 : gcc_assert (is_gimple_omp_oacc (ctx->stmt));
13130 3896 : if (omp_privatize_by_reference (new_var)
13131 3896 : && (TREE_CODE (TREE_TYPE (new_var)) != POINTER_TYPE
13132 44 : || DECL_BY_REFERENCE (var)))
13133 : {
13134 : /* Create a local object to hold the instance
13135 : value. */
13136 455 : tree type = TREE_TYPE (TREE_TYPE (new_var));
13137 455 : const char *id = IDENTIFIER_POINTER (DECL_NAME (new_var));
13138 455 : tree inst = create_tmp_var (type, id);
13139 455 : gimplify_assign (inst, fold_indirect_ref (x), &fplist);
13140 455 : x = build_fold_addr_expr (inst);
13141 : }
13142 3896 : gimplify_assign (new_var, x, &fplist);
13143 : }
13144 23174 : else if (DECL_P (new_var))
13145 : {
13146 23174 : SET_DECL_VALUE_EXPR (new_var, x);
13147 23174 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13148 : }
13149 : else
13150 0 : gcc_unreachable ();
13151 : }
13152 43022 : map_cnt++;
13153 43022 : break;
13154 :
13155 15306 : case OMP_CLAUSE_FIRSTPRIVATE:
13156 15306 : omp_firstprivate_recv:
13157 15306 : gcc_checking_assert (offloaded);
13158 15306 : if (is_gimple_omp_oacc (ctx->stmt))
13159 : {
13160 : /* No 'firstprivate' clauses on OpenACC 'kernels'. */
13161 3896 : gcc_checking_assert (!is_oacc_kernels (ctx));
13162 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
13163 3896 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
13164 :
13165 3896 : goto oacc_firstprivate;
13166 : }
13167 11410 : map_cnt++;
13168 11410 : var = OMP_CLAUSE_DECL (c);
13169 11410 : new_var = lookup_decl (var, ctx);
13170 11410 : allocator = NULL_TREE;
13171 11410 : allocate_ptr = NULL_TREE;
13172 11410 : size = TREE_TYPE (var);
13173 11410 : by_ref = omp_privatize_by_reference (var);
13174 11410 : if (by_ref)
13175 828 : size = TREE_TYPE (size);
13176 11410 : size = TYPE_SIZE_UNIT (size);
13177 11410 : if (is_variable_sized (var, by_ref))
13178 35 : size = lookup_decl (size, ctx);
13179 11410 : alloc_seq = NULL;
13180 11410 : if (lower_private_allocate (var, new_var, allocator, allocate_ptr,
13181 : &alloc_seq, ctx, by_ref, size))
13182 : {
13183 65 : alloc_map.put (new_var, allocate_ptr);
13184 65 : alloc_seq_map.put (new_var, alloc_seq);
13185 : }
13186 11410 : if (!by_ref && !is_gimple_reg_type (TREE_TYPE (var)))
13187 : {
13188 143 : if (is_variable_sized (var))
13189 : {
13190 6 : tree pvar = DECL_VALUE_EXPR (var);
13191 6 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
13192 6 : pvar = TREE_OPERAND (pvar, 0);
13193 6 : gcc_assert (DECL_P (pvar));
13194 6 : tree new_pvar = lookup_decl (pvar, ctx);
13195 6 : x = build_fold_indirect_ref (new_pvar);
13196 6 : TREE_THIS_NOTRAP (x) = 1;
13197 : }
13198 137 : else if (allocate_ptr)
13199 19 : x = build_fold_indirect_ref (allocate_ptr);
13200 : else
13201 118 : x = build_receiver_ref (var, true, ctx);
13202 143 : SET_DECL_VALUE_EXPR (new_var, x);
13203 143 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13204 : }
13205 : /* Fortran array descriptors: firstprivate of data + attach. */
13206 11410 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
13207 11391 : && lang_hooks.decls.omp_array_data (var, true)
13208 11441 : && lang_hooks.decls.omp_array_data_privatize (var))
13209 23 : map_cnt += 2;
13210 :
13211 11558 : do_dtor:
13212 11558 : if (allocator)
13213 : {
13214 122 : if (!is_gimple_val (allocator))
13215 : {
13216 0 : tree avar = create_tmp_var (TREE_TYPE (allocator));
13217 0 : gimplify_assign (avar, allocator, &alloc_dlist);
13218 0 : allocator = avar;
13219 : }
13220 122 : if (!is_gimple_val (allocate_ptr))
13221 : {
13222 0 : tree apvar = create_tmp_var (TREE_TYPE (allocate_ptr));
13223 0 : gimplify_assign (apvar, allocate_ptr, &alloc_dlist);
13224 0 : allocate_ptr = apvar;
13225 : }
13226 122 : tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
13227 122 : gimple *g = gimple_build_call (f, 2, allocate_ptr, allocator);
13228 122 : gimple_seq_add_stmt (&alloc_dlist, g);
13229 : }
13230 : break;
13231 :
13232 228 : case OMP_CLAUSE_PRIVATE:
13233 228 : gcc_checking_assert (offloaded);
13234 228 : if (is_gimple_omp_oacc (ctx->stmt))
13235 : {
13236 : /* No 'private' clauses on OpenACC 'kernels'. */
13237 80 : gcc_checking_assert (!is_oacc_kernels (ctx));
13238 : /* Likewise, on OpenACC 'kernels' decomposed parts. */
13239 80 : gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
13240 :
13241 : break;
13242 : }
13243 148 : var = OMP_CLAUSE_DECL (c);
13244 148 : new_var = lookup_decl (var, ctx);
13245 148 : allocator = NULL_TREE;
13246 148 : allocate_ptr = NULL_TREE;
13247 148 : alloc_seq = NULL;
13248 148 : size = TREE_TYPE (var);
13249 148 : by_ref = omp_privatize_by_reference (var);
13250 148 : if (by_ref)
13251 22 : size = TREE_TYPE (size);
13252 148 : size = TYPE_SIZE_UNIT (size);
13253 148 : if (is_variable_sized (var, by_ref))
13254 21 : size = lookup_decl (size, ctx);
13255 148 : lower_private_allocate (var, new_var, allocator, allocate_ptr,
13256 : &alloc_seq, ctx, by_ref, size);
13257 148 : if (allocate_ptr)
13258 : {
13259 57 : alloc_map.put (new_var, allocate_ptr);
13260 57 : alloc_seq_map.put (new_var, alloc_seq);
13261 : }
13262 148 : if (!allocate_ptr && is_variable_sized (var))
13263 : {
13264 1 : tree new_var = lookup_decl (var, ctx);
13265 1 : tree pvar = DECL_VALUE_EXPR (var);
13266 1 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
13267 1 : pvar = TREE_OPERAND (pvar, 0);
13268 1 : gcc_assert (DECL_P (pvar));
13269 1 : tree new_pvar = lookup_decl (pvar, ctx);
13270 1 : x = build_fold_indirect_ref (new_pvar);
13271 1 : TREE_THIS_NOTRAP (x) = 1;
13272 1 : SET_DECL_VALUE_EXPR (new_var, x);
13273 1 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13274 : }
13275 148 : goto do_dtor;
13276 :
13277 2430 : case OMP_CLAUSE_USE_DEVICE_PTR:
13278 2430 : case OMP_CLAUSE_USE_DEVICE_ADDR:
13279 2430 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
13280 2430 : case OMP_CLAUSE_IS_DEVICE_PTR:
13281 2430 : var = OMP_CLAUSE_DECL (c);
13282 2430 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
13283 : {
13284 252 : while (TREE_CODE (var) == INDIRECT_REF
13285 252 : || TREE_CODE (var) == ARRAY_REF)
13286 17 : var = TREE_OPERAND (var, 0);
13287 235 : if (lang_hooks.decls.omp_array_data (var, true))
13288 19 : goto omp_firstprivate_recv;
13289 : }
13290 2411 : map_cnt++;
13291 2411 : if (is_variable_sized (var))
13292 : {
13293 12 : tree new_var = lookup_decl (var, ctx);
13294 12 : tree pvar = DECL_VALUE_EXPR (var);
13295 12 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
13296 12 : pvar = TREE_OPERAND (pvar, 0);
13297 12 : gcc_assert (DECL_P (pvar));
13298 12 : tree new_pvar = lookup_decl (pvar, ctx);
13299 12 : x = build_fold_indirect_ref (new_pvar);
13300 12 : TREE_THIS_NOTRAP (x) = 1;
13301 12 : SET_DECL_VALUE_EXPR (new_var, x);
13302 12 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13303 : }
13304 2399 : else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
13305 542 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
13306 2071 : && !omp_privatize_by_reference (var)
13307 703 : && !omp_is_allocatable_or_ptr (var)
13308 564 : && !lang_hooks.decls.omp_array_data (var, true))
13309 4433 : || TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
13310 : {
13311 389 : tree new_var = lookup_decl (var, ctx);
13312 389 : tree type = build_pointer_type (TREE_TYPE (var));
13313 389 : x = create_tmp_var_raw (type, get_name (new_var));
13314 389 : gimple_add_tmp_var (x);
13315 389 : x = build_simple_mem_ref (x);
13316 389 : SET_DECL_VALUE_EXPR (new_var, x);
13317 389 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13318 : }
13319 : else
13320 : {
13321 2010 : tree new_var = lookup_decl (var, ctx);
13322 2010 : x = create_tmp_var_raw (TREE_TYPE (new_var), get_name (new_var));
13323 2010 : gimple_add_tmp_var (x);
13324 2010 : SET_DECL_VALUE_EXPR (new_var, x);
13325 2010 : DECL_HAS_VALUE_EXPR_P (new_var) = 1;
13326 : }
13327 : break;
13328 : }
13329 :
13330 37170 : if (offloaded)
13331 : {
13332 21210 : target_nesting_level++;
13333 21210 : lower_omp (&tgt_body, ctx);
13334 21210 : target_nesting_level--;
13335 : }
13336 15960 : else if (data_region)
13337 4021 : lower_omp (&tgt_body, ctx);
13338 :
13339 37170 : if (offloaded)
13340 : {
13341 : /* Declare all the variables created by mapping and the variables
13342 : declared in the scope of the target body. */
13343 21210 : record_vars_into (ctx->block_vars, child_fn);
13344 21210 : maybe_remove_omp_member_access_dummy_vars (tgt_bind);
13345 21210 : record_vars_into (gimple_bind_vars (tgt_bind), child_fn);
13346 : }
13347 :
13348 37170 : if (ctx->record_type)
13349 : {
13350 32279 : if (deep_map_cnt && TREE_CODE (deep_map_cnt) == INTEGER_CST)
13351 : /* map_cnt = map_cnt + tree_to_hwi (deep_map_cnt); */
13352 : /* deep_map_cnt = NULL_TREE; */
13353 0 : gcc_unreachable ();
13354 144 : else if (deep_map_cnt)
13355 : {
13356 144 : gcc_assert (flexible_array_type_p (ctx->record_type));
13357 144 : tree n = create_tmp_var_raw (size_type_node, "nn_map");
13358 144 : gimple_add_tmp_var (n);
13359 144 : gimplify_assign (n, deep_map_cnt, &ilist);
13360 144 : deep_map_cnt = n;
13361 : }
13362 144 : ctx->sender_decl
13363 32279 : = create_tmp_var (deep_map_cnt ? build_pointer_type (ctx->record_type)
13364 : : ctx->record_type, ".omp_data_arr");
13365 32279 : DECL_NAMELESS (ctx->sender_decl) = 1;
13366 32279 : TREE_ADDRESSABLE (ctx->sender_decl) = 1;
13367 64414 : t = make_tree_vec (deep_map_cnt ? 4 : 3);
13368 32279 : TREE_VEC_ELT (t, 0) = ctx->sender_decl;
13369 32279 : TREE_VEC_ELT (t, 1)
13370 64558 : = create_tmp_var (deep_map_cnt
13371 144 : ? build_pointer_type (size_type_node)
13372 32135 : : build_array_type_nelts (size_type_node, map_cnt),
13373 : ".omp_data_sizes");
13374 32279 : DECL_NAMELESS (TREE_VEC_ELT (t, 1)) = 1;
13375 32279 : TREE_ADDRESSABLE (TREE_VEC_ELT (t, 1)) = 1;
13376 32279 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 1;
13377 32279 : tree tkind_type = short_unsigned_type_node;
13378 32279 : int talign_shift = 8;
13379 32279 : TREE_VEC_ELT (t, 2)
13380 64558 : = create_tmp_var (deep_map_cnt
13381 144 : ? build_pointer_type (tkind_type)
13382 32135 : : build_array_type_nelts (tkind_type, map_cnt),
13383 : ".omp_data_kinds");
13384 32279 : DECL_NAMELESS (TREE_VEC_ELT (t, 2)) = 1;
13385 32279 : TREE_ADDRESSABLE (TREE_VEC_ELT (t, 2)) = 1;
13386 32279 : TREE_STATIC (TREE_VEC_ELT (t, 2)) = 1;
13387 32279 : gimple_omp_target_set_data_arg (stmt, t);
13388 :
13389 32279 : if (deep_map_cnt)
13390 : {
13391 144 : tree tmp, size;
13392 144 : size = create_tmp_var (size_type_node, NULL);
13393 144 : DECL_NAMELESS (size) = 1;
13394 144 : gimplify_assign (size,
13395 : fold_build2_loc (UNKNOWN_LOCATION, PLUS_EXPR,
13396 : size_type_node, deep_map_cnt,
13397 : build_int_cst (size_type_node,
13398 144 : map_cnt)), &ilist);
13399 144 : TREE_VEC_ELT (t, 3) = size;
13400 :
13401 144 : tree call = builtin_decl_explicit (BUILT_IN_MALLOC);
13402 144 : size = fold_build2_loc (UNKNOWN_LOCATION, MULT_EXPR,
13403 : size_type_node, deep_map_cnt,
13404 144 : TYPE_SIZE_UNIT (ptr_type_node));
13405 144 : size = fold_build2_loc (UNKNOWN_LOCATION, PLUS_EXPR,
13406 : size_type_node, size,
13407 144 : TYPE_SIZE_UNIT (ctx->record_type));
13408 144 : tmp = build_call_expr_loc (input_location, call, 1, size);
13409 144 : gimplify_assign (ctx->sender_decl, tmp, &ilist);
13410 :
13411 144 : size = fold_build2_loc (UNKNOWN_LOCATION, MULT_EXPR,
13412 144 : size_type_node, TREE_VEC_ELT (t, 3),
13413 144 : TYPE_SIZE_UNIT (size_type_node));
13414 144 : tmp = build_call_expr_loc (input_location, call, 1, size);
13415 144 : gimplify_assign (TREE_VEC_ELT (t, 1), tmp, &ilist);
13416 :
13417 144 : size = fold_build2_loc (UNKNOWN_LOCATION, MULT_EXPR,
13418 144 : size_type_node, TREE_VEC_ELT (t, 3),
13419 144 : TYPE_SIZE_UNIT (tkind_type));
13420 144 : tmp = build_call_expr_loc (input_location, call, 1, size);
13421 144 : gimplify_assign (TREE_VEC_ELT (t, 2), tmp, &ilist);
13422 144 : tree field = TYPE_FIELDS (TREE_TYPE (TREE_TYPE (ctx->sender_decl)));
13423 595 : for ( ; DECL_CHAIN (field) != NULL_TREE; field = DECL_CHAIN (field))
13424 : ;
13425 144 : gcc_assert (TREE_CODE (TREE_TYPE (field)));
13426 144 : tmp = build_fold_indirect_ref (ctx->sender_decl);
13427 144 : deep_map_data = omp_build_component_ref (tmp, field);
13428 144 : deep_map_offset_data = create_tmp_var_raw (size_type_node,
13429 : "map_offset_data");
13430 144 : deep_map_offset = create_tmp_var_raw (size_type_node, "map_offset");
13431 144 : gimple_add_tmp_var (deep_map_offset_data);
13432 144 : gimple_add_tmp_var (deep_map_offset);
13433 144 : gimplify_assign (deep_map_offset_data, build_int_cst (size_type_node,
13434 : 0), &ilist);
13435 144 : gimplify_assign (deep_map_offset, build_int_cst (size_type_node,
13436 144 : map_cnt), &ilist);
13437 : }
13438 :
13439 32279 : vec<constructor_elt, va_gc> *vsize;
13440 32279 : vec<constructor_elt, va_gc> *vkind;
13441 32279 : vec_alloc (vsize, map_cnt);
13442 32279 : vec_alloc (vkind, map_cnt);
13443 32279 : unsigned int map_idx = 0;
13444 :
13445 162202 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
13446 129923 : switch (OMP_CLAUSE_CODE (c))
13447 : {
13448 : tree ovar, nc, s, purpose, var, x, type;
13449 : unsigned int talign;
13450 :
13451 : default:
13452 125959 : break;
13453 :
13454 87947 : case OMP_CLAUSE_MAP:
13455 87947 : case OMP_CLAUSE_TO:
13456 87947 : case OMP_CLAUSE_FROM:
13457 87947 : oacc_firstprivate_map:
13458 87947 : nc = c;
13459 87947 : ovar = OMP_CLAUSE_DECL (c);
13460 87947 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13461 87947 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
13462 72969 : || (OMP_CLAUSE_MAP_KIND (c)
13463 : == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
13464 : break;
13465 84110 : if (deep_map_cnt)
13466 : {
13467 428 : unsigned HOST_WIDE_INT tkind2;
13468 428 : switch (OMP_CLAUSE_CODE (c))
13469 : {
13470 424 : case OMP_CLAUSE_MAP:
13471 424 : tkind2 = OMP_CLAUSE_MAP_KIND (c);
13472 424 : if (OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (c)
13473 424 : && (((tkind2 & GOMP_MAP_FLAG_SPECIAL_BITS)
13474 : & ~GOMP_MAP_IMPLICIT)
13475 : == 0))
13476 : {
13477 : /* If this is an implicit map, and the GOMP_MAP_IMPLICIT
13478 : bits are not interfered by other special bit
13479 : encodings, then turn the GOMP_IMPLICIT_BIT flag on
13480 : for the runtime to see. */
13481 120 : tkind2 |= GOMP_MAP_IMPLICIT;
13482 : }
13483 : break;
13484 : case OMP_CLAUSE_FIRSTPRIVATE: tkind2 = GOMP_MAP_TO; break;
13485 : case OMP_CLAUSE_TO: tkind2 = GOMP_MAP_TO; break;
13486 2 : case OMP_CLAUSE_FROM: tkind2 = GOMP_MAP_FROM; break;
13487 0 : default: gcc_unreachable ();
13488 : }
13489 428 : lang_hooks.decls.omp_deep_mapping (stmt, c, tkind2,
13490 : deep_map_data,
13491 428 : TREE_VEC_ELT (t, 1),
13492 428 : TREE_VEC_ELT (t, 2),
13493 : deep_map_offset_data,
13494 : deep_map_offset, &ilist);
13495 : }
13496 84110 : if (!DECL_P (ovar))
13497 : {
13498 36887 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13499 36887 : && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
13500 : {
13501 0 : nc = OMP_CLAUSE_CHAIN (c);
13502 0 : gcc_checking_assert (OMP_CLAUSE_DECL (nc)
13503 : == get_base_address (ovar));
13504 0 : ovar = OMP_CLAUSE_DECL (nc);
13505 : }
13506 : else
13507 : {
13508 36887 : tree x = build_sender_ref (c, ctx);
13509 36887 : tree v = ovar;
13510 36887 : if (in_reduction_clauses
13511 163 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13512 37050 : && OMP_CLAUSE_MAP_IN_REDUCTION (c))
13513 : {
13514 104 : v = unshare_expr (v);
13515 104 : tree *p = &v;
13516 104 : while (handled_component_p (*p)
13517 : || TREE_CODE (*p) == INDIRECT_REF
13518 : || TREE_CODE (*p) == ADDR_EXPR
13519 : || TREE_CODE (*p) == MEM_REF
13520 256 : || TREE_CODE (*p) == NON_LVALUE_EXPR)
13521 152 : p = &TREE_OPERAND (*p, 0);
13522 104 : tree d = *p;
13523 104 : if (is_variable_sized (d))
13524 : {
13525 16 : gcc_assert (DECL_HAS_VALUE_EXPR_P (d));
13526 16 : d = DECL_VALUE_EXPR (d);
13527 16 : gcc_assert (TREE_CODE (d) == INDIRECT_REF);
13528 16 : d = TREE_OPERAND (d, 0);
13529 16 : gcc_assert (DECL_P (d));
13530 : }
13531 104 : splay_tree_key key
13532 104 : = (splay_tree_key) &DECL_CONTEXT (d);
13533 104 : tree nd = (tree) splay_tree_lookup (ctx->field_map,
13534 104 : key)->value;
13535 104 : if (d == *p)
13536 88 : *p = nd;
13537 : else
13538 16 : *p = build_fold_indirect_ref (nd);
13539 : }
13540 36887 : v = build_fold_addr_expr_with_type (v, ptr_type_node);
13541 36887 : v = lower_omp_map_iterator_expr (v, c, stmt);
13542 36887 : gimplify_assign (x, v, &ilist);
13543 36887 : nc = NULL_TREE;
13544 : }
13545 : }
13546 : else
13547 : {
13548 47223 : if (DECL_SIZE (ovar)
13549 47223 : && !poly_int_tree_p (DECL_SIZE (ovar)))
13550 : {
13551 7 : tree ovar2 = DECL_VALUE_EXPR (ovar);
13552 7 : gcc_assert (TREE_CODE (ovar2) == INDIRECT_REF);
13553 7 : ovar2 = TREE_OPERAND (ovar2, 0);
13554 7 : gcc_assert (DECL_P (ovar2));
13555 : ovar = ovar2;
13556 : }
13557 47223 : if (!maybe_lookup_field (c, ctx)
13558 51187 : && !(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13559 3964 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
13560 3964 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)))
13561 3964 : continue;
13562 : }
13563 :
13564 80146 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar));
13565 80146 : if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign)
13566 6409 : talign = DECL_ALIGN_UNIT (ovar);
13567 :
13568 80146 : var = NULL_TREE;
13569 80146 : if (nc)
13570 : {
13571 43259 : if (in_reduction_clauses
13572 897 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13573 44156 : && OMP_CLAUSE_MAP_IN_REDUCTION (c))
13574 : {
13575 268 : tree d = ovar;
13576 268 : if (is_variable_sized (d))
13577 : {
13578 0 : gcc_assert (DECL_HAS_VALUE_EXPR_P (d));
13579 0 : d = DECL_VALUE_EXPR (d);
13580 0 : gcc_assert (TREE_CODE (d) == INDIRECT_REF);
13581 0 : d = TREE_OPERAND (d, 0);
13582 0 : gcc_assert (DECL_P (d));
13583 : }
13584 268 : splay_tree_key key
13585 268 : = (splay_tree_key) &DECL_CONTEXT (d);
13586 268 : tree nd = (tree) splay_tree_lookup (ctx->field_map,
13587 268 : key)->value;
13588 268 : if (d == ovar)
13589 : var = nd;
13590 : else
13591 0 : var = build_fold_indirect_ref (nd);
13592 : }
13593 : else
13594 42991 : var = lookup_decl_in_outer_ctx (ovar, ctx);
13595 : }
13596 43259 : if (nc
13597 43259 : && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13598 33295 : && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
13599 33088 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
13600 261 : && is_omp_target (stmt))
13601 : {
13602 237 : x = build_sender_ref (c, ctx);
13603 237 : gimplify_assign (x, build_fold_addr_expr (var), &ilist);
13604 : }
13605 79909 : else if (nc)
13606 : {
13607 43022 : x = build_sender_ref (nc, ctx);
13608 :
13609 43022 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
13610 33058 : && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
13611 5184 : && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
13612 48206 : && TREE_CODE (TREE_TYPE (ovar)) == ARRAY_TYPE)
13613 : {
13614 425 : gcc_assert (offloaded);
13615 425 : tree avar = build_fold_addr_expr (var);
13616 425 : if (!OMP_CLAUSE_ITERATORS (c))
13617 : {
13618 425 : tree tmp = create_tmp_var (TREE_TYPE (TREE_TYPE (x)));
13619 425 : mark_addressable (tmp);
13620 425 : gimplify_assign (tmp, avar, &ilist);
13621 425 : avar = tmp;
13622 : }
13623 425 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (x)));
13624 425 : avar = build_fold_addr_expr (avar);
13625 425 : avar = lower_omp_map_iterator_expr (avar, c, stmt);
13626 425 : gimplify_assign (x, avar, &ilist);
13627 : }
13628 42597 : else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
13629 : {
13630 3896 : gcc_assert (is_gimple_omp_oacc (ctx->stmt));
13631 3896 : if (!omp_privatize_by_reference (var))
13632 : {
13633 3441 : if (is_gimple_reg (var)
13634 3441 : && OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
13635 2578 : suppress_warning (var);
13636 3441 : var = build_fold_addr_expr (var);
13637 : }
13638 : else
13639 455 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
13640 3896 : gimplify_assign (x, var, &ilist);
13641 : }
13642 38701 : else if (is_gimple_reg (var))
13643 : {
13644 5285 : gcc_assert (offloaded);
13645 5285 : tree avar = create_tmp_var (TREE_TYPE (var));
13646 5285 : mark_addressable (avar);
13647 5285 : enum gomp_map_kind map_kind = OMP_CLAUSE_MAP_KIND (c);
13648 5285 : if (GOMP_MAP_COPY_TO_P (map_kind)
13649 : || map_kind == GOMP_MAP_POINTER
13650 475 : || map_kind == GOMP_MAP_TO_PSET
13651 2 : || map_kind == GOMP_MAP_FORCE_DEVICEPTR)
13652 : {
13653 : /* If we need to initialize a temporary
13654 : with VAR because it is not addressable, and
13655 : the variable hasn't been initialized yet, then
13656 : we'll get a warning for the store to avar.
13657 : Don't warn in that case, the mapping might
13658 : be implicit. */
13659 5283 : suppress_warning (var, OPT_Wuninitialized);
13660 5283 : gimplify_assign (avar, var, &ilist);
13661 : }
13662 5285 : avar = build_fold_addr_expr (avar);
13663 5285 : gimplify_assign (x, avar, &ilist);
13664 5285 : if ((GOMP_MAP_COPY_FROM_P (map_kind)
13665 1486 : || map_kind == GOMP_MAP_FORCE_DEVICEPTR)
13666 9084 : && !TYPE_READONLY (TREE_TYPE (var)))
13667 : {
13668 3799 : x = unshare_expr (x);
13669 3799 : x = build_simple_mem_ref (x);
13670 3799 : gimplify_assign (var, x, &olist);
13671 : }
13672 : }
13673 : else
13674 : {
13675 : /* While MAP is handled explicitly by the FE,
13676 : for 'target update', only the identified is passed. */
13677 33416 : if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FROM
13678 28292 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TO)
13679 34360 : && (omp_is_allocatable_or_ptr (var)
13680 120 : && omp_check_optional_argument (var, false)))
13681 0 : var = build_fold_indirect_ref (var);
13682 33416 : else if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FROM
13683 28292 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_TO)
13684 34360 : || (!omp_is_allocatable_or_ptr (var)
13685 5948 : && !omp_check_optional_argument (var, false)))
13686 33296 : var = build_fold_addr_expr (var);
13687 33416 : gimplify_assign (x, var, &ilist);
13688 : }
13689 : }
13690 80146 : s = NULL_TREE;
13691 80146 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
13692 : {
13693 3896 : gcc_checking_assert (is_gimple_omp_oacc (ctx->stmt));
13694 3896 : s = TREE_TYPE (ovar);
13695 3896 : if (TREE_CODE (s) == REFERENCE_TYPE
13696 3896 : || omp_check_optional_argument (ovar, false))
13697 449 : s = TREE_TYPE (s);
13698 3896 : s = TYPE_SIZE_UNIT (s);
13699 : }
13700 : else
13701 76250 : s = OMP_CLAUSE_SIZE (c);
13702 80146 : if (s == NULL_TREE)
13703 153 : s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
13704 80146 : s = fold_convert (size_type_node, s);
13705 80146 : s = lower_omp_map_iterator_size (s, c, stmt);
13706 80146 : purpose = size_int (map_idx++);
13707 80146 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
13708 80146 : if (TREE_CODE (s) != INTEGER_CST)
13709 16154 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
13710 :
13711 80146 : unsigned HOST_WIDE_INT tkind, tkind_zero;
13712 80146 : switch (OMP_CLAUSE_CODE (c))
13713 : {
13714 68576 : case OMP_CLAUSE_MAP:
13715 68576 : tkind = OMP_CLAUSE_MAP_KIND (c);
13716 68576 : tkind_zero = tkind;
13717 68576 : if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c))
13718 5281 : switch (tkind)
13719 : {
13720 : case GOMP_MAP_ALLOC:
13721 : case GOMP_MAP_IF_PRESENT:
13722 : case GOMP_MAP_TO:
13723 : case GOMP_MAP_FROM:
13724 : case GOMP_MAP_TOFROM:
13725 : case GOMP_MAP_ALWAYS_TO:
13726 : case GOMP_MAP_ALWAYS_FROM:
13727 : case GOMP_MAP_ALWAYS_TOFROM:
13728 : case GOMP_MAP_ALWAYS_PRESENT_TO:
13729 : case GOMP_MAP_ALWAYS_PRESENT_FROM:
13730 : case GOMP_MAP_ALWAYS_PRESENT_TOFROM:
13731 : case GOMP_MAP_RELEASE:
13732 : case GOMP_MAP_FORCE_TO:
13733 : case GOMP_MAP_FORCE_FROM:
13734 : case GOMP_MAP_FORCE_TOFROM:
13735 : case GOMP_MAP_FORCE_PRESENT:
13736 68576 : tkind_zero = GOMP_MAP_ZERO_LEN_ARRAY_SECTION;
13737 : break;
13738 21 : case GOMP_MAP_DELETE:
13739 21 : tkind_zero = GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION;
13740 : default:
13741 : break;
13742 : }
13743 68576 : if (tkind_zero != tkind)
13744 : {
13745 5281 : if (integer_zerop (s))
13746 : tkind = tkind_zero;
13747 4551 : else if (integer_nonzerop (s))
13748 68576 : tkind_zero = tkind;
13749 : }
13750 68576 : if (tkind_zero == tkind
13751 67115 : && OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (c)
13752 68576 : && (((tkind & GOMP_MAP_FLAG_SPECIAL_BITS)
13753 5234 : & ~GOMP_MAP_IMPLICIT)
13754 : == 0))
13755 : {
13756 : /* If this is an implicit map, and the GOMP_MAP_IMPLICIT
13757 : bits are not interfered by other special bit encodings,
13758 : then turn the GOMP_IMPLICIT_BIT flag on for the runtime
13759 : to see. */
13760 5191 : tkind |= GOMP_MAP_IMPLICIT;
13761 5191 : tkind_zero = tkind;
13762 : }
13763 : break;
13764 3896 : case OMP_CLAUSE_FIRSTPRIVATE:
13765 3896 : gcc_checking_assert (is_gimple_omp_oacc (ctx->stmt));
13766 : tkind = GOMP_MAP_TO;
13767 : tkind_zero = tkind;
13768 : break;
13769 1651 : case OMP_CLAUSE_TO:
13770 1651 : tkind
13771 1651 : = (OMP_CLAUSE_MOTION_PRESENT (c)
13772 1651 : ? GOMP_MAP_ALWAYS_PRESENT_TO : GOMP_MAP_TO);
13773 : tkind_zero = tkind;
13774 : break;
13775 6023 : case OMP_CLAUSE_FROM:
13776 6023 : tkind
13777 6023 : = (OMP_CLAUSE_MOTION_PRESENT (c)
13778 6023 : ? GOMP_MAP_ALWAYS_PRESENT_FROM : GOMP_MAP_FROM);
13779 : tkind_zero = tkind;
13780 : break;
13781 0 : default:
13782 0 : gcc_unreachable ();
13783 : }
13784 68576 : gcc_checking_assert (tkind
13785 : < (HOST_WIDE_INT_C (1U) << talign_shift));
13786 80146 : gcc_checking_assert (tkind_zero
13787 : < (HOST_WIDE_INT_C (1U) << talign_shift));
13788 80146 : talign = ceil_log2 (talign);
13789 80146 : tkind |= talign << talign_shift;
13790 80146 : tkind_zero |= talign << talign_shift;
13791 80146 : gcc_checking_assert (tkind
13792 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
13793 80146 : gcc_checking_assert (tkind_zero
13794 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
13795 80146 : if (tkind == tkind_zero)
13796 78685 : x = build_int_cstu (tkind_type, tkind);
13797 : else
13798 : {
13799 1461 : TREE_STATIC (TREE_VEC_ELT (t, 2)) = 0;
13800 1461 : x = build3 (COND_EXPR, tkind_type,
13801 : fold_build2 (EQ_EXPR, boolean_type_node,
13802 : unshare_expr (s), size_zero_node),
13803 1461 : build_int_cstu (tkind_type, tkind_zero),
13804 1461 : build_int_cstu (tkind_type, tkind));
13805 : }
13806 80146 : CONSTRUCTOR_APPEND_ELT (vkind, purpose, x);
13807 80146 : if (nc && nc != c)
13808 0 : c = nc;
13809 : break;
13810 :
13811 15306 : case OMP_CLAUSE_FIRSTPRIVATE:
13812 15306 : omp_has_device_addr_descr:
13813 15306 : if (is_gimple_omp_oacc (ctx->stmt))
13814 3896 : goto oacc_firstprivate_map;
13815 11410 : ovar = OMP_CLAUSE_DECL (c);
13816 11410 : if (omp_privatize_by_reference (ovar))
13817 828 : talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
13818 : else
13819 10582 : talign = DECL_ALIGN_UNIT (ovar);
13820 11410 : var = lookup_decl_in_outer_ctx (ovar, ctx);
13821 11410 : x = build_sender_ref (ovar, ctx);
13822 11410 : tkind = GOMP_MAP_FIRSTPRIVATE;
13823 11410 : type = TREE_TYPE (ovar);
13824 11410 : if (omp_privatize_by_reference (ovar))
13825 828 : type = TREE_TYPE (type);
13826 11410 : if ((INTEGRAL_TYPE_P (type)
13827 11004 : && TYPE_PRECISION (type) <= POINTER_SIZE)
13828 11420 : || TREE_CODE (type) == POINTER_TYPE)
13829 : {
13830 11149 : tkind = GOMP_MAP_FIRSTPRIVATE_INT;
13831 11149 : tree t = var;
13832 11149 : if (omp_privatize_by_reference (var))
13833 773 : t = build_simple_mem_ref (var);
13834 10376 : else if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
13835 9310 : suppress_warning (var);
13836 11149 : if (TREE_CODE (type) != POINTER_TYPE)
13837 10994 : t = fold_convert (pointer_sized_int_node, t);
13838 11149 : t = fold_convert (TREE_TYPE (x), t);
13839 11149 : gimplify_assign (x, t, &ilist);
13840 : }
13841 261 : else if (omp_privatize_by_reference (var))
13842 55 : gimplify_assign (x, var, &ilist);
13843 206 : else if (is_gimple_reg (var))
13844 : {
13845 51 : tree avar = create_tmp_var (TREE_TYPE (var));
13846 51 : mark_addressable (avar);
13847 51 : if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
13848 44 : suppress_warning (var);
13849 51 : gimplify_assign (avar, var, &ilist);
13850 51 : avar = build_fold_addr_expr (avar);
13851 51 : gimplify_assign (x, avar, &ilist);
13852 : }
13853 : else
13854 : {
13855 155 : var = build_fold_addr_expr (var);
13856 155 : gimplify_assign (x, var, &ilist);
13857 : }
13858 261 : if (tkind == GOMP_MAP_FIRSTPRIVATE_INT)
13859 11149 : s = size_int (0);
13860 261 : else if (omp_privatize_by_reference (ovar))
13861 55 : s = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
13862 : else
13863 206 : s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
13864 11410 : s = fold_convert (size_type_node, s);
13865 11410 : purpose = size_int (map_idx++);
13866 11410 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
13867 11410 : if (TREE_CODE (s) != INTEGER_CST)
13868 35 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
13869 :
13870 11410 : gcc_checking_assert (tkind
13871 : < (HOST_WIDE_INT_C (1U) << talign_shift));
13872 11410 : talign = ceil_log2 (talign);
13873 11410 : tkind |= talign << talign_shift;
13874 11410 : gcc_checking_assert (tkind
13875 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
13876 11410 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
13877 : build_int_cstu (tkind_type, tkind));
13878 : /* Fortran array descriptors: firstprivate of data + attach. */
13879 11410 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
13880 11391 : && lang_hooks.decls.omp_array_data (ovar, true)
13881 11441 : && lang_hooks.decls.omp_array_data_privatize (ovar))
13882 : {
13883 23 : tree not_null_lb, null_lb, after_lb;
13884 23 : tree var1, var2, size1, size2;
13885 23 : tree present = omp_check_optional_argument (ovar, true);
13886 23 : if (present)
13887 : {
13888 1 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
13889 1 : not_null_lb = create_artificial_label (clause_loc);
13890 1 : null_lb = create_artificial_label (clause_loc);
13891 1 : after_lb = create_artificial_label (clause_loc);
13892 1 : gimple_seq seq = NULL;
13893 1 : present = force_gimple_operand (present, &seq, true,
13894 : NULL_TREE);
13895 1 : gimple_seq_add_seq (&ilist, seq);
13896 1 : gimple_seq_add_stmt (&ilist,
13897 1 : gimple_build_cond_from_tree (present,
13898 : not_null_lb, null_lb));
13899 1 : gimple_seq_add_stmt (&ilist,
13900 1 : gimple_build_label (not_null_lb));
13901 : }
13902 23 : var1 = lang_hooks.decls.omp_array_data (var, false);
13903 23 : size1 = lang_hooks.decls.omp_array_size (var, &ilist);
13904 23 : var2 = build_fold_addr_expr (x);
13905 23 : if (!POINTER_TYPE_P (TREE_TYPE (var)))
13906 0 : var = build_fold_addr_expr (var);
13907 23 : size2 = fold_build2 (POINTER_DIFF_EXPR, ssizetype,
13908 : build_fold_addr_expr (var1), var);
13909 23 : size2 = fold_convert (sizetype, size2);
13910 23 : if (present)
13911 : {
13912 1 : tree tmp = create_tmp_var (TREE_TYPE (var1));
13913 1 : gimplify_assign (tmp, var1, &ilist);
13914 1 : var1 = tmp;
13915 1 : tmp = create_tmp_var (TREE_TYPE (var2));
13916 1 : gimplify_assign (tmp, var2, &ilist);
13917 1 : var2 = tmp;
13918 1 : tmp = create_tmp_var (TREE_TYPE (size1));
13919 1 : gimplify_assign (tmp, size1, &ilist);
13920 1 : size1 = tmp;
13921 1 : tmp = create_tmp_var (TREE_TYPE (size2));
13922 1 : gimplify_assign (tmp, size2, &ilist);
13923 1 : size2 = tmp;
13924 1 : gimple_seq_add_stmt (&ilist, gimple_build_goto (after_lb));
13925 1 : gimple_seq_add_stmt (&ilist, gimple_build_label (null_lb));
13926 1 : gimplify_assign (var1, null_pointer_node, &ilist);
13927 1 : gimplify_assign (var2, null_pointer_node, &ilist);
13928 1 : gimplify_assign (size1, size_zero_node, &ilist);
13929 1 : gimplify_assign (size2, size_zero_node, &ilist);
13930 1 : gimple_seq_add_stmt (&ilist, gimple_build_label (after_lb));
13931 : }
13932 23 : x = build_sender_ref ((splay_tree_key) &DECL_NAME (ovar), ctx);
13933 23 : gimplify_assign (x, var1, &ilist);
13934 23 : tkind = GOMP_MAP_FIRSTPRIVATE;
13935 23 : talign = DECL_ALIGN_UNIT (ovar);
13936 23 : talign = ceil_log2 (talign);
13937 23 : tkind |= talign << talign_shift;
13938 23 : gcc_checking_assert (tkind
13939 : <= tree_to_uhwi (
13940 : TYPE_MAX_VALUE (tkind_type)));
13941 23 : purpose = size_int (map_idx++);
13942 23 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, size1);
13943 23 : if (TREE_CODE (size1) != INTEGER_CST)
13944 23 : TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
13945 23 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
13946 : build_int_cstu (tkind_type, tkind));
13947 23 : x = build_sender_ref ((splay_tree_key) &DECL_UID (ovar), ctx);
13948 23 : gimplify_assign (x, var2, &ilist);
13949 23 : tkind = GOMP_MAP_ATTACH;
13950 23 : purpose = size_int (map_idx++);
13951 23 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, size2);
13952 23 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
13953 : build_int_cstu (tkind_type, tkind));
13954 : }
13955 : break;
13956 :
13957 2430 : case OMP_CLAUSE_USE_DEVICE_PTR:
13958 2430 : case OMP_CLAUSE_USE_DEVICE_ADDR:
13959 2430 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
13960 2430 : case OMP_CLAUSE_IS_DEVICE_PTR:
13961 2430 : ovar = OMP_CLAUSE_DECL (c);
13962 2430 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
13963 : {
13964 235 : if (lang_hooks.decls.omp_array_data (ovar, true))
13965 19 : goto omp_has_device_addr_descr;
13966 233 : while (TREE_CODE (ovar) == INDIRECT_REF
13967 233 : || TREE_CODE (ovar) == ARRAY_REF)
13968 17 : ovar = TREE_OPERAND (ovar, 0);
13969 : }
13970 2411 : var = lookup_decl_in_outer_ctx (ovar, ctx);
13971 :
13972 2411 : if (lang_hooks.decls.omp_array_data (ovar, true))
13973 : {
13974 601 : tkind = ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
13975 601 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
13976 601 : ? GOMP_MAP_USE_DEVICE_PTR : GOMP_MAP_FIRSTPRIVATE_INT);
13977 601 : x = build_sender_ref ((splay_tree_key) &DECL_NAME (ovar), ctx);
13978 : }
13979 1810 : else if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
13980 1810 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
13981 : {
13982 1453 : tkind = GOMP_MAP_USE_DEVICE_PTR;
13983 1453 : x = build_sender_ref ((splay_tree_key) &DECL_UID (ovar), ctx);
13984 : }
13985 : else
13986 : {
13987 357 : tkind = GOMP_MAP_FIRSTPRIVATE_INT;
13988 357 : x = build_sender_ref (ovar, ctx);
13989 : }
13990 :
13991 2411 : if (is_gimple_omp_oacc (ctx->stmt))
13992 : {
13993 147 : gcc_assert (tkind == GOMP_MAP_USE_DEVICE_PTR);
13994 :
13995 147 : if (OMP_CLAUSE_USE_DEVICE_PTR_IF_PRESENT (c))
13996 57 : tkind = GOMP_MAP_USE_DEVICE_PTR_IF_PRESENT;
13997 : }
13998 :
13999 2411 : type = TREE_TYPE (ovar);
14000 2411 : if (lang_hooks.decls.omp_array_data (ovar, true))
14001 601 : var = lang_hooks.decls.omp_array_data (var, false);
14002 1810 : else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
14003 542 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
14004 1484 : && !omp_privatize_by_reference (ovar)
14005 512 : && !omp_is_allocatable_or_ptr (ovar))
14006 3247 : || TREE_CODE (type) == ARRAY_TYPE)
14007 401 : var = build_fold_addr_expr (var);
14008 : else
14009 : {
14010 1409 : if (omp_privatize_by_reference (ovar)
14011 370 : || omp_check_optional_argument (ovar, false)
14012 1779 : || omp_is_allocatable_or_ptr (ovar))
14013 : {
14014 1212 : type = TREE_TYPE (type);
14015 1212 : if (POINTER_TYPE_P (type)
14016 1212 : && TREE_CODE (type) != ARRAY_TYPE
14017 1571 : && ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_USE_DEVICE_ADDR
14018 36 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
14019 28 : && !omp_is_allocatable_or_ptr (ovar))
14020 337 : || (omp_privatize_by_reference (ovar)
14021 337 : && omp_is_allocatable_or_ptr (ovar))))
14022 357 : var = build_simple_mem_ref (var);
14023 1212 : var = fold_convert (TREE_TYPE (x), var);
14024 : }
14025 : }
14026 2411 : tree present;
14027 2411 : present = omp_check_optional_argument (ovar, true);
14028 2411 : if (present)
14029 : {
14030 878 : tree null_label = create_artificial_label (UNKNOWN_LOCATION);
14031 878 : tree notnull_label = create_artificial_label (UNKNOWN_LOCATION);
14032 878 : tree opt_arg_label = create_artificial_label (UNKNOWN_LOCATION);
14033 878 : tree new_x = unshare_expr (x);
14034 878 : gimplify_expr (&present, &ilist, NULL, is_gimple_val,
14035 : fb_rvalue);
14036 878 : gcond *cond = gimple_build_cond_from_tree (present,
14037 : notnull_label,
14038 : null_label);
14039 878 : gimple_seq_add_stmt (&ilist, cond);
14040 878 : gimple_seq_add_stmt (&ilist, gimple_build_label (null_label));
14041 878 : gimplify_assign (new_x, null_pointer_node, &ilist);
14042 878 : gimple_seq_add_stmt (&ilist, gimple_build_goto (opt_arg_label));
14043 878 : gimple_seq_add_stmt (&ilist,
14044 878 : gimple_build_label (notnull_label));
14045 878 : gimplify_assign (x, var, &ilist);
14046 878 : gimple_seq_add_stmt (&ilist,
14047 878 : gimple_build_label (opt_arg_label));
14048 : }
14049 : else
14050 1533 : gimplify_assign (x, var, &ilist);
14051 2411 : s = size_int (0);
14052 2411 : purpose = size_int (map_idx++);
14053 2411 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
14054 2411 : gcc_checking_assert (tkind
14055 : < (HOST_WIDE_INT_C (1U) << talign_shift));
14056 2411 : gcc_checking_assert (tkind
14057 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
14058 2411 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
14059 : build_int_cstu (tkind_type, tkind));
14060 2411 : break;
14061 :
14062 86 : case OMP_CLAUSE_USES_ALLOCATORS:
14063 86 : tree allocator = OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c);
14064 86 : tree memspace = OMP_CLAUSE_USES_ALLOCATORS_MEMSPACE (c);
14065 86 : tree traits = OMP_CLAUSE_USES_ALLOCATORS_TRAITS (c);
14066 :
14067 86 : tree ntraits, traits_var;
14068 86 : if (traits == NULL_TREE)
14069 : {
14070 29 : ntraits = integer_zero_node;
14071 29 : traits_var = null_pointer_node;
14072 : }
14073 : else
14074 : {
14075 57 : location_t loc = OMP_CLAUSE_LOCATION (c);
14076 57 : ntraits = array_type_nelts_top (TREE_TYPE (traits));
14077 57 : traits_var = build_fold_addr_expr_loc (loc, traits);
14078 : }
14079 :
14080 86 : if (memspace == NULL_TREE)
14081 61 : memspace = build_int_cst (pointer_sized_int_node, 0);
14082 : else
14083 25 : memspace = fold_convert (pointer_sized_int_node, memspace);
14084 :
14085 86 : tree arr_type
14086 86 : = build_array_type_nelts (pointer_sized_int_node, 3);
14087 86 : tree uses_allocators_descr
14088 86 : = create_tmp_var (arr_type, "uses_allocator_descr");
14089 86 : tree ua_descr[3];
14090 344 : for (int i = 0; i < 3; i++)
14091 258 : ua_descr[i] = build4 (ARRAY_REF, pointer_sized_int_node,
14092 : uses_allocators_descr,
14093 258 : build_int_cst (size_type_node, i),
14094 : NULL_TREE, NULL_TREE);
14095 86 : gimplify_assign (ua_descr[0],
14096 : fold_convert (pointer_sized_int_node, memspace),
14097 : &ilist);
14098 86 : gimplify_assign (ua_descr[1],
14099 : fold_convert (pointer_sized_int_node, ntraits),
14100 : &ilist);
14101 86 : gimplify_assign (ua_descr[2],
14102 : fold_convert (pointer_sized_int_node, traits_var),
14103 : &ilist);
14104 :
14105 86 : x = build_sender_ref (allocator, ctx);
14106 86 : tree ptr = build_fold_addr_expr (uses_allocators_descr);
14107 86 : gimplify_assign (x, fold_convert (TREE_TYPE (x), ptr), &ilist);
14108 :
14109 86 : s = size_int (0);
14110 86 : purpose = size_int (map_idx++);
14111 86 : CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
14112 86 : tkind = GOMP_MAP_USES_ALLOCATORS;
14113 86 : gcc_checking_assert (tkind
14114 : < (HOST_WIDE_INT_C (1U) << talign_shift));
14115 86 : talign = TYPE_ALIGN_UNIT (pointer_sized_int_node);
14116 86 : talign = ceil_log2 (talign);
14117 86 : tkind |= talign << talign_shift;
14118 86 : gcc_checking_assert (tkind
14119 : <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
14120 86 : CONSTRUCTOR_APPEND_ELT (vkind, purpose,
14121 : build_int_cstu (tkind_type, tkind));
14122 86 : break;
14123 : }
14124 :
14125 32279 : gcc_assert (map_idx == map_cnt);
14126 :
14127 32279 : if (!deep_map_cnt)
14128 : {
14129 32135 : DECL_INITIAL (TREE_VEC_ELT (t, 1))
14130 32135 : = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), vsize);
14131 32135 : DECL_INITIAL (TREE_VEC_ELT (t, 2))
14132 64270 : = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 2)), vkind);
14133 : }
14134 96837 : for (int i = 1; i <= 2; i++)
14135 64558 : if (deep_map_cnt || !TREE_STATIC (TREE_VEC_ELT (t, i)))
14136 : {
14137 9657 : tree tmp = TREE_VEC_ELT (t, i);
14138 9657 : if (deep_map_cnt)
14139 : {
14140 288 : const char *prefix = (i == 1 ? ".omp_data_sizes0"
14141 : : ".omp_data_kinds0");
14142 144 : tree type = (i == 1) ? size_type_node : tkind_type;
14143 288 : type = build_array_type_nelts (type, map_cnt);
14144 288 : tree var = create_tmp_var (type, prefix);
14145 288 : DECL_NAMELESS (var) = 1;
14146 288 : TREE_ADDRESSABLE (var) = 1;
14147 288 : TREE_STATIC (var) = TREE_STATIC (tmp);
14148 288 : DECL_INITIAL (var) = build_constructor (type, i == 1
14149 : ? vsize : vkind);
14150 288 : tmp = var;
14151 288 : TREE_STATIC (TREE_VEC_ELT (t, i)) = 0;
14152 : }
14153 :
14154 9657 : gimple_seq initlist = NULL;
14155 9657 : force_gimple_operand (build1 (DECL_EXPR, void_type_node, tmp),
14156 : &initlist, true, NULL_TREE);
14157 9657 : gimple_seq_add_seq (&ilist, initlist);
14158 :
14159 9657 : if (deep_map_cnt)
14160 : {
14161 288 : tree tmp2;
14162 288 : tree call = builtin_decl_explicit (BUILT_IN_MEMCPY);
14163 288 : tmp2 = TYPE_SIZE_UNIT (TREE_TYPE (tmp));
14164 576 : call = build_call_expr_loc (input_location, call, 3,
14165 288 : TREE_VEC_ELT (t, i),
14166 : build_fold_addr_expr (tmp), tmp2);
14167 288 : gimplify_and_add (call, &ilist);
14168 : }
14169 :
14170 9657 : if (!TREE_STATIC (tmp))
14171 : {
14172 9439 : tree clobber = build_clobber (TREE_TYPE (tmp));
14173 9439 : gimple_seq_add_stmt (&olist,
14174 9439 : gimple_build_assign (tmp, clobber));
14175 : }
14176 9657 : if (deep_map_cnt)
14177 : {
14178 288 : tmp = TREE_VEC_ELT (t, i);
14179 288 : tree call = builtin_decl_explicit (BUILT_IN_FREE);
14180 288 : call = build_call_expr_loc (input_location, call, 1, tmp);
14181 288 : gimplify_and_add (call, &olist);
14182 288 : tree clobber = build_clobber (TREE_TYPE (tmp));
14183 288 : gimple_seq_add_stmt (&olist,
14184 288 : gimple_build_assign (tmp, clobber));
14185 : }
14186 : }
14187 54901 : else if (omp_maybe_offloaded_ctx (ctx->outer))
14188 : {
14189 3296 : tree id = get_identifier ("omp declare target");
14190 3296 : tree decl = TREE_VEC_ELT (t, i);
14191 3296 : DECL_ATTRIBUTES (decl)
14192 3296 : = tree_cons (id, NULL_TREE, DECL_ATTRIBUTES (decl));
14193 3296 : varpool_node *node = varpool_node::get (decl);
14194 3296 : if (node)
14195 : {
14196 0 : node->offloadable = 1;
14197 0 : if (ENABLE_OFFLOADING)
14198 : {
14199 : g->have_offload = true;
14200 : vec_safe_push (offload_vars, t);
14201 : }
14202 : }
14203 : }
14204 :
14205 32279 : if (deep_map_cnt)
14206 : {
14207 144 : tree call = builtin_decl_explicit (BUILT_IN_FREE);
14208 144 : call = build_call_expr_loc (input_location, call, 1,
14209 144 : TREE_VEC_ELT (t, 0));
14210 144 : gimplify_and_add (call, &olist);
14211 :
14212 144 : gimplify_expr (&TREE_VEC_ELT (t, 1), &ilist, NULL, is_gimple_val,
14213 : fb_rvalue);
14214 : }
14215 :
14216 32279 : tree clobber = build_clobber (TREE_TYPE (ctx->sender_decl));
14217 32279 : gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
14218 : clobber));
14219 : }
14220 :
14221 : /* Once all the expansions are done, sequence all the different
14222 : fragments inside gimple_omp_body. */
14223 :
14224 37170 : new_body = NULL;
14225 :
14226 37170 : if (offloaded
14227 21210 : && ctx->record_type)
14228 : {
14229 16556 : t = ctx->sender_decl;
14230 16556 : if (!deep_map_cnt)
14231 16449 : t = build_fold_addr_expr_loc (loc, t);
14232 : /* fixup_child_record_type might have changed receiver_decl's type. */
14233 16556 : t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
14234 16556 : if (!AGGREGATE_TYPE_P (TREE_TYPE (ctx->sender_decl)))
14235 107 : gimplify_assign (ctx->receiver_decl, t, &new_body);
14236 : else
14237 16449 : gimple_seq_add_stmt (&new_body,
14238 16449 : gimple_build_assign (ctx->receiver_decl, t));
14239 : }
14240 37170 : gimple_seq_add_seq (&new_body, fplist);
14241 :
14242 37170 : if (offloaded || data_region)
14243 : {
14244 141173 : tree prev = NULL_TREE;
14245 : bool by_ref;
14246 141173 : for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
14247 115942 : switch (OMP_CLAUSE_CODE (c))
14248 : {
14249 : tree var, x, new_var, *allocate_ptr;
14250 : default:
14251 : break;
14252 15306 : case OMP_CLAUSE_FIRSTPRIVATE:
14253 15306 : omp_firstprivatize_data_region:
14254 15306 : if (is_gimple_omp_oacc (ctx->stmt))
14255 : break;
14256 11410 : var = OMP_CLAUSE_DECL (c);
14257 11410 : new_var = lookup_decl (var, ctx);
14258 11410 : allocate_ptr = alloc_map.get (new_var);
14259 11410 : by_ref = omp_privatize_by_reference (var);
14260 11410 : if (allocate_ptr)
14261 : {
14262 65 : if (is_variable_sized (var, by_ref))
14263 : /* Handle this in the next pass when the size is
14264 : available. */
14265 : break;
14266 :
14267 50 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14268 50 : gcc_assert (allocate_seq);
14269 50 : gimple_seq_add_seq (&new_body, *allocate_seq);
14270 :
14271 50 : if (by_ref)
14272 : {
14273 4 : x = fold_convert (TREE_TYPE (new_var), *allocate_ptr);
14274 4 : gimplify_assign (new_var, x, &new_body);
14275 4 : new_var = build_fold_indirect_ref (new_var);
14276 : }
14277 : else
14278 46 : new_var = build_simple_mem_ref (*allocate_ptr);
14279 : }
14280 11395 : if (by_ref || is_gimple_reg_type (TREE_TYPE (var)))
14281 : {
14282 11254 : tree type;
14283 11254 : type = TREE_TYPE (var);
14284 11254 : if (by_ref)
14285 815 : type = TREE_TYPE (type);
14286 11254 : if ((INTEGRAL_TYPE_P (type)
14287 11004 : && TYPE_PRECISION (type) <= POINTER_SIZE)
14288 11264 : || TREE_CODE (type) == POINTER_TYPE)
14289 : {
14290 11149 : x = build_receiver_ref (var, false, ctx);
14291 11149 : if (TREE_CODE (type) != POINTER_TYPE)
14292 10994 : x = fold_convert (pointer_sized_int_node, x);
14293 11149 : x = fold_convert (type, x);
14294 11149 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14295 : fb_rvalue);
14296 11149 : if (by_ref && !allocate_ptr)
14297 : {
14298 772 : tree v = create_tmp_var_raw (type, get_name (var));
14299 772 : gimple_add_tmp_var (v);
14300 772 : TREE_ADDRESSABLE (v) = 1;
14301 772 : gimple_seq_add_stmt (&new_body,
14302 772 : gimple_build_assign (v, x));
14303 772 : x = build_fold_addr_expr (v);
14304 : }
14305 11149 : gimplify_assign (new_var, x, &new_body);
14306 : }
14307 : else
14308 : {
14309 105 : x = build_receiver_ref (var, allocate_ptr || !by_ref, ctx);
14310 105 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14311 : fb_rvalue);
14312 105 : gimplify_assign (new_var, x, &new_body);
14313 : }
14314 : }
14315 141 : else if (is_variable_sized (var))
14316 : {
14317 4 : tree pvar = DECL_VALUE_EXPR (var);
14318 4 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14319 4 : pvar = TREE_OPERAND (pvar, 0);
14320 4 : gcc_assert (DECL_P (pvar));
14321 4 : tree new_var = lookup_decl (pvar, ctx);
14322 4 : x = build_receiver_ref (var, false, ctx);
14323 4 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14324 4 : gimple_seq_add_stmt (&new_body,
14325 4 : gimple_build_assign (new_var, x));
14326 : }
14327 137 : else if (allocate_ptr)
14328 : {
14329 19 : x = build_receiver_ref (var, true, ctx);
14330 19 : new_var = unshare_expr (new_var);
14331 19 : x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
14332 19 : gimplify_and_add (x, &new_body);
14333 : }
14334 : break;
14335 228 : case OMP_CLAUSE_PRIVATE:
14336 228 : if (is_gimple_omp_oacc (ctx->stmt))
14337 : break;
14338 148 : var = OMP_CLAUSE_DECL (c);
14339 148 : new_var = lookup_decl (var, ctx);
14340 148 : allocate_ptr = alloc_map.get (new_var);
14341 148 : by_ref = omp_privatize_by_reference (var);
14342 148 : if (allocate_ptr)
14343 : {
14344 57 : if (is_variable_sized (var, by_ref))
14345 : /* Handle this in the next pass when the size is
14346 : available. */
14347 : break;
14348 :
14349 45 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14350 45 : gcc_assert (allocate_seq);
14351 45 : gimple_seq_add_seq (&new_body, *allocate_seq);
14352 :
14353 45 : if (!by_ref)
14354 44 : new_var = build_simple_mem_ref (*allocate_ptr);
14355 : }
14356 135 : if (by_ref)
14357 : {
14358 12 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14359 12 : if (!allocate_ptr)
14360 : {
14361 11 : x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
14362 11 : if (TREE_CONSTANT (x))
14363 : {
14364 3 : x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
14365 : get_name (var));
14366 3 : gimple_add_tmp_var (x);
14367 3 : TREE_ADDRESSABLE (x) = 1;
14368 3 : x = build_fold_addr_expr_loc (clause_loc, x);
14369 : }
14370 : else
14371 : break;
14372 :
14373 3 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
14374 3 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14375 : fb_rvalue);
14376 : }
14377 : else
14378 1 : x = *allocate_ptr;
14379 :
14380 4 : gimple_seq_add_stmt (&new_body,
14381 4 : gimple_build_assign (new_var, x));
14382 : }
14383 : break;
14384 2430 : case OMP_CLAUSE_USE_DEVICE_PTR:
14385 2430 : case OMP_CLAUSE_USE_DEVICE_ADDR:
14386 2430 : case OMP_CLAUSE_HAS_DEVICE_ADDR:
14387 2430 : case OMP_CLAUSE_IS_DEVICE_PTR:
14388 2430 : gimple_seq assign_body;
14389 2430 : bool is_array_data;
14390 2430 : bool do_optional_check;
14391 2430 : assign_body = NULL;
14392 2430 : do_optional_check = false;
14393 2430 : var = OMP_CLAUSE_DECL (c);
14394 2430 : is_array_data = lang_hooks.decls.omp_array_data (var, true) != NULL;
14395 2430 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR && is_array_data)
14396 19 : goto omp_firstprivatize_data_region;
14397 :
14398 2411 : if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
14399 2411 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
14400 4108 : x = build_sender_ref (is_array_data
14401 601 : ? (splay_tree_key) &DECL_NAME (var)
14402 1453 : : (splay_tree_key) &DECL_UID (var), ctx);
14403 : else
14404 : {
14405 357 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
14406 : {
14407 233 : while (TREE_CODE (var) == INDIRECT_REF
14408 233 : || TREE_CODE (var) == ARRAY_REF)
14409 17 : var = TREE_OPERAND (var, 0);
14410 : }
14411 357 : x = build_receiver_ref (var, false, ctx);
14412 : }
14413 :
14414 2411 : if (is_array_data)
14415 : {
14416 601 : bool is_ref = omp_privatize_by_reference (var);
14417 601 : do_optional_check = true;
14418 : /* First, we copy the descriptor data from the host; then
14419 : we update its data to point to the target address. */
14420 601 : new_var = lookup_decl (var, ctx);
14421 601 : new_var = DECL_VALUE_EXPR (new_var);
14422 601 : tree v = new_var;
14423 601 : tree v2 = var;
14424 601 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_PTR
14425 601 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR)
14426 601 : v2 = maybe_lookup_decl_in_outer_ctx (var, ctx);
14427 :
14428 601 : if (is_ref)
14429 : {
14430 396 : v2 = build_fold_indirect_ref (v2);
14431 396 : v = create_tmp_var_raw (TREE_TYPE (v2), get_name (var));
14432 396 : gimple_add_tmp_var (v);
14433 396 : TREE_ADDRESSABLE (v) = 1;
14434 396 : gimplify_assign (v, v2, &assign_body);
14435 396 : tree rhs = build_fold_addr_expr (v);
14436 396 : gimple_seq_add_stmt (&assign_body,
14437 396 : gimple_build_assign (new_var, rhs));
14438 : }
14439 : else
14440 205 : gimplify_assign (new_var, v2, &assign_body);
14441 :
14442 601 : v2 = lang_hooks.decls.omp_array_data (unshare_expr (v), false);
14443 601 : gcc_assert (v2);
14444 601 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14445 1202 : gimple_seq_add_stmt (&assign_body,
14446 601 : gimple_build_assign (v2, x));
14447 : }
14448 1810 : else if (is_variable_sized (var))
14449 : {
14450 12 : tree pvar = DECL_VALUE_EXPR (var);
14451 12 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14452 12 : pvar = TREE_OPERAND (pvar, 0);
14453 12 : gcc_assert (DECL_P (pvar));
14454 12 : new_var = lookup_decl (pvar, ctx);
14455 12 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14456 24 : gimple_seq_add_stmt (&assign_body,
14457 12 : gimple_build_assign (new_var, x));
14458 : }
14459 1798 : else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
14460 536 : || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
14461 1476 : && !omp_privatize_by_reference (var)
14462 504 : && !omp_is_allocatable_or_ptr (var))
14463 3231 : || TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
14464 : {
14465 389 : new_var = lookup_decl (var, ctx);
14466 389 : new_var = DECL_VALUE_EXPR (new_var);
14467 389 : gcc_assert (TREE_CODE (new_var) == MEM_REF);
14468 389 : new_var = TREE_OPERAND (new_var, 0);
14469 389 : gcc_assert (DECL_P (new_var));
14470 389 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14471 778 : gimple_seq_add_stmt (&assign_body,
14472 389 : gimple_build_assign (new_var, x));
14473 : }
14474 : else
14475 : {
14476 1409 : tree type = TREE_TYPE (var);
14477 1409 : new_var = lookup_decl (var, ctx);
14478 1409 : if (omp_privatize_by_reference (var))
14479 : {
14480 1039 : type = TREE_TYPE (type);
14481 1039 : if (POINTER_TYPE_P (type)
14482 1039 : && TREE_CODE (type) != ARRAY_TYPE
14483 1398 : && ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_USE_DEVICE_ADDR
14484 36 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
14485 331 : || (omp_privatize_by_reference (var)
14486 331 : && omp_is_allocatable_or_ptr (var))))
14487 : {
14488 357 : tree v = create_tmp_var_raw (type, get_name (var));
14489 357 : gimple_add_tmp_var (v);
14490 357 : TREE_ADDRESSABLE (v) = 1;
14491 357 : x = fold_convert (type, x);
14492 357 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val,
14493 : fb_rvalue);
14494 357 : gimple_seq_add_stmt (&assign_body,
14495 357 : gimple_build_assign (v, x));
14496 357 : x = build_fold_addr_expr (v);
14497 357 : do_optional_check = true;
14498 : }
14499 : }
14500 1409 : new_var = DECL_VALUE_EXPR (new_var);
14501 1409 : x = fold_convert (TREE_TYPE (new_var), x);
14502 1409 : gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
14503 1409 : gimple_seq_add_stmt (&assign_body,
14504 1409 : gimple_build_assign (new_var, x));
14505 : }
14506 2411 : tree present;
14507 1002 : present = ((do_optional_check
14508 958 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
14509 952 : && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR)
14510 2342 : ? omp_check_optional_argument (OMP_CLAUSE_DECL (c), true)
14511 : : NULL_TREE);
14512 2411 : if (present)
14513 : {
14514 480 : tree null_label = create_artificial_label (UNKNOWN_LOCATION);
14515 480 : tree notnull_label = create_artificial_label (UNKNOWN_LOCATION);
14516 480 : tree opt_arg_label = create_artificial_label (UNKNOWN_LOCATION);
14517 480 : glabel *null_glabel = gimple_build_label (null_label);
14518 480 : glabel *notnull_glabel = gimple_build_label (notnull_label);
14519 480 : ggoto *opt_arg_ggoto = gimple_build_goto (opt_arg_label);
14520 480 : gimplify_expr (&x, &new_body, NULL, is_gimple_val,
14521 : fb_rvalue);
14522 480 : gimplify_expr (&present, &new_body, NULL, is_gimple_val,
14523 : fb_rvalue);
14524 480 : gcond *cond = gimple_build_cond_from_tree (present,
14525 : notnull_label,
14526 : null_label);
14527 480 : gimple_seq_add_stmt (&new_body, cond);
14528 480 : gimple_seq_add_stmt (&new_body, null_glabel);
14529 480 : gimplify_assign (new_var, null_pointer_node, &new_body);
14530 480 : gimple_seq_add_stmt (&new_body, opt_arg_ggoto);
14531 480 : gimple_seq_add_stmt (&new_body, notnull_glabel);
14532 480 : gimple_seq_add_seq (&new_body, assign_body);
14533 480 : gimple_seq_add_stmt (&new_body,
14534 480 : gimple_build_label (opt_arg_label));
14535 : }
14536 : else
14537 1931 : gimple_seq_add_seq (&new_body, assign_body);
14538 : break;
14539 : }
14540 : /* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass,
14541 : so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
14542 : are already handled. Similarly OMP_CLAUSE_PRIVATE for VLAs
14543 : or references to VLAs. */
14544 141173 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
14545 115942 : switch (OMP_CLAUSE_CODE (c))
14546 : {
14547 : tree var, new_var, *allocate_ptr;
14548 : default:
14549 100675 : break;
14550 63539 : case OMP_CLAUSE_MAP:
14551 63539 : if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
14552 63539 : || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
14553 : {
14554 3669 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14555 3669 : poly_int64 offset = 0;
14556 3669 : gcc_assert (prev);
14557 3669 : var = OMP_CLAUSE_DECL (c);
14558 3669 : if (DECL_P (var)
14559 3669 : && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
14560 1125 : && is_global_var (maybe_lookup_decl_in_outer_ctx (var,
14561 : ctx))
14562 3767 : && varpool_node::get_create (var)->offloadable)
14563 : break;
14564 3667 : if (TREE_CODE (var) == INDIRECT_REF
14565 3667 : && TREE_CODE (TREE_OPERAND (var, 0)) == COMPONENT_REF)
14566 0 : var = TREE_OPERAND (var, 0);
14567 3667 : if (TREE_CODE (var) == COMPONENT_REF)
14568 : {
14569 0 : var = get_addr_base_and_unit_offset (var, &offset);
14570 0 : gcc_assert (var != NULL_TREE && DECL_P (var));
14571 : }
14572 3667 : else if (DECL_SIZE (var)
14573 3667 : && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST)
14574 : {
14575 710 : tree var2 = DECL_VALUE_EXPR (var);
14576 710 : gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
14577 710 : var2 = TREE_OPERAND (var2, 0);
14578 710 : gcc_assert (DECL_P (var2));
14579 : var = var2;
14580 : }
14581 3667 : tree new_var = lookup_decl (var, ctx), x;
14582 3667 : tree type = TREE_TYPE (new_var);
14583 3667 : bool is_ref;
14584 3667 : if (TREE_CODE (OMP_CLAUSE_DECL (c)) == INDIRECT_REF
14585 3667 : && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c), 0))
14586 : == COMPONENT_REF))
14587 : {
14588 0 : type = TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c), 0));
14589 0 : is_ref = true;
14590 0 : new_var = build2 (MEM_REF, type,
14591 : build_fold_addr_expr (new_var),
14592 : build_int_cst (build_pointer_type (type),
14593 : offset));
14594 : }
14595 3667 : else if (TREE_CODE (OMP_CLAUSE_DECL (c)) == COMPONENT_REF)
14596 : {
14597 0 : type = TREE_TYPE (OMP_CLAUSE_DECL (c));
14598 0 : is_ref = TREE_CODE (type) == REFERENCE_TYPE;
14599 0 : new_var = build2 (MEM_REF, type,
14600 : build_fold_addr_expr (new_var),
14601 : build_int_cst (build_pointer_type (type),
14602 : offset));
14603 : }
14604 : else
14605 3667 : is_ref = omp_privatize_by_reference (var);
14606 3667 : if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
14607 : is_ref = false;
14608 6705 : bool ref_to_array = false;
14609 6705 : bool ref_to_ptr = false;
14610 3238 : if (is_ref)
14611 : {
14612 200 : type = TREE_TYPE (type);
14613 200 : if (TREE_CODE (type) == ARRAY_TYPE)
14614 : {
14615 194 : type = build_pointer_type (type);
14616 194 : ref_to_array = true;
14617 : }
14618 : }
14619 3467 : else if (TREE_CODE (type) == ARRAY_TYPE)
14620 : {
14621 413 : tree decl2 = DECL_VALUE_EXPR (new_var);
14622 413 : gcc_assert (TREE_CODE (decl2) == MEM_REF);
14623 413 : decl2 = TREE_OPERAND (decl2, 0);
14624 413 : gcc_assert (DECL_P (decl2));
14625 413 : new_var = decl2;
14626 413 : type = TREE_TYPE (new_var);
14627 : }
14628 3054 : else if (TREE_CODE (type) == REFERENCE_TYPE
14629 3054 : && TREE_CODE (TREE_TYPE (type)) == POINTER_TYPE)
14630 : {
14631 156 : type = TREE_TYPE (type);
14632 156 : ref_to_ptr = true;
14633 : }
14634 3667 : x = build_receiver_ref (prev, false, ctx);
14635 3667 : x = fold_convert_loc (clause_loc, type, x);
14636 3667 : if (!integer_zerop (OMP_CLAUSE_SIZE (c)))
14637 : {
14638 762 : tree bias = OMP_CLAUSE_SIZE (c);
14639 762 : if (DECL_P (bias))
14640 364 : bias = lookup_decl (bias, ctx);
14641 762 : bias = fold_convert_loc (clause_loc, sizetype, bias);
14642 762 : bias = fold_build1_loc (clause_loc, NEGATE_EXPR, sizetype,
14643 : bias);
14644 762 : x = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
14645 762 : TREE_TYPE (x), x, bias);
14646 : }
14647 3667 : if (ref_to_array)
14648 194 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
14649 3667 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14650 3667 : if ((is_ref && !ref_to_array)
14651 3661 : || ref_to_ptr)
14652 : {
14653 162 : tree t = create_tmp_var_raw (type, get_name (var));
14654 162 : gimple_add_tmp_var (t);
14655 162 : TREE_ADDRESSABLE (t) = 1;
14656 162 : gimple_seq_add_stmt (&new_body,
14657 162 : gimple_build_assign (t, x));
14658 162 : x = build_fold_addr_expr_loc (clause_loc, t);
14659 : }
14660 3667 : gimple_seq_add_stmt (&new_body,
14661 3667 : gimple_build_assign (new_var, x));
14662 3667 : prev = NULL_TREE;
14663 : }
14664 59870 : else if (OMP_CLAUSE_CHAIN (c)
14665 43260 : && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c))
14666 : == OMP_CLAUSE_MAP
14667 97696 : && (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c))
14668 : == GOMP_MAP_FIRSTPRIVATE_POINTER
14669 34586 : || (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c))
14670 : == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
14671 : prev = c;
14672 : break;
14673 228 : case OMP_CLAUSE_PRIVATE:
14674 228 : var = OMP_CLAUSE_DECL (c);
14675 228 : by_ref = omp_privatize_by_reference (var);
14676 228 : new_var = lookup_decl (var, ctx);
14677 228 : allocate_ptr = alloc_map.get (new_var);
14678 228 : if (is_variable_sized (var, by_ref)
14679 228 : || (!allocate_ptr && by_ref && !is_gimple_omp_oacc (ctx->stmt)))
14680 : {
14681 24 : if (allocate_ptr)
14682 : {
14683 12 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14684 12 : gcc_assert (allocate_seq);
14685 12 : gimple_seq_add_seq (&new_body, *allocate_seq);
14686 : }
14687 24 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14688 24 : tree pvar = var;
14689 24 : if (!by_ref)
14690 : {
14691 3 : pvar = DECL_VALUE_EXPR (var);
14692 3 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14693 3 : pvar = TREE_OPERAND (pvar, 0);
14694 : }
14695 24 : gcc_assert (DECL_P (pvar));
14696 24 : tree new_pvar = lookup_decl (pvar, ctx);
14697 24 : tree x;
14698 24 : if (!allocate_ptr)
14699 : {
14700 12 : tree atmp = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
14701 12 : tree ty = TREE_TYPE (new_var);
14702 12 : if (by_ref)
14703 11 : ty = TREE_TYPE (ty);
14704 12 : x = TYPE_SIZE_UNIT (ty);
14705 12 : if (TREE_CONSTANT (x))
14706 : break;
14707 9 : tree al = size_int (TYPE_ALIGN (ty));
14708 9 : x = build_call_expr_loc (clause_loc, atmp, 2, x, al);
14709 9 : x = fold_convert_loc (clause_loc, TREE_TYPE (new_pvar), x);
14710 : }
14711 : else
14712 12 : x = *allocate_ptr;
14713 21 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14714 21 : gimple_seq_add_stmt (&new_body,
14715 21 : gimple_build_assign (new_pvar, x));
14716 : }
14717 : break;
14718 15287 : case OMP_CLAUSE_FIRSTPRIVATE:
14719 15287 : var = OMP_CLAUSE_DECL (c);
14720 15287 : by_ref = omp_privatize_by_reference (var);
14721 15287 : if (is_variable_sized (var, by_ref))
14722 : {
14723 35 : tree new_var = lookup_decl (var, ctx);
14724 35 : tree *allocate_ptr = alloc_map.get (new_var);
14725 35 : if (!allocate_ptr)
14726 : break;
14727 15 : gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
14728 15 : gcc_assert (allocate_seq);
14729 15 : gimple_seq_add_seq (&new_body, *allocate_seq);
14730 :
14731 15 : location_t clause_loc = OMP_CLAUSE_LOCATION (c);
14732 15 : tree pvar = var;
14733 15 : if (!by_ref)
14734 : {
14735 2 : pvar = DECL_VALUE_EXPR (var);
14736 2 : gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
14737 2 : pvar = TREE_OPERAND (pvar, 0);
14738 : }
14739 15 : gcc_assert (DECL_P (pvar));
14740 15 : tree new_pvar = lookup_decl (pvar, ctx);
14741 15 : tree x = fold_convert_loc (clause_loc, TREE_TYPE (new_pvar),
14742 15 : *allocate_ptr);
14743 15 : gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
14744 15 : gimple_seq_add_stmt (&new_body,
14745 15 : gimple_build_assign (new_pvar, x));
14746 :
14747 15 : x = build_receiver_ref (var, true, ctx);
14748 15 : new_var = unshare_expr (new_var);
14749 15 : if (by_ref)
14750 13 : new_var = build_fold_indirect_ref (new_var);
14751 15 : x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
14752 15 : gimplify_and_add (x, &new_body);
14753 : }
14754 : }
14755 :
14756 25231 : gimple_seq fork_seq = NULL;
14757 25231 : gimple_seq join_seq = NULL;
14758 :
14759 25231 : if (offloaded && is_gimple_omp_oacc (ctx->stmt))
14760 : {
14761 : /* If there are reductions on the offloaded region itself, treat
14762 : them as a dummy GANG loop. */
14763 9395 : tree level = build_int_cst (integer_type_node, GOMP_DIM_GANG);
14764 :
14765 9395 : gcall *private_marker = lower_oacc_private_marker (ctx);
14766 :
14767 9395 : if (private_marker)
14768 30 : gimple_call_set_arg (private_marker, 2, level);
14769 :
14770 9395 : lower_oacc_reductions (gimple_location (ctx->stmt), clauses, level,
14771 : false, NULL, private_marker, NULL, &fork_seq,
14772 : &join_seq, ctx);
14773 : }
14774 :
14775 25231 : gimple_seq_add_seq (&new_body, fork_seq);
14776 25231 : gimple_seq_add_seq (&new_body, tgt_body);
14777 25231 : gimple_seq_add_seq (&new_body, join_seq);
14778 25231 : gimple_seq_add_seq (&new_body, alloc_dlist);
14779 :
14780 25231 : if (offloaded)
14781 : {
14782 21210 : new_body = maybe_catch_exception (new_body);
14783 21210 : gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
14784 : }
14785 25231 : gimple_omp_set_body (stmt, new_body);
14786 : }
14787 :
14788 37170 : gsi_insert_seq_before (gsi_p, gimple_omp_target_iterator_loops (stmt),
14789 : GSI_SAME_STMT);
14790 37170 : gimple_omp_target_set_iterator_loops (stmt, NULL);
14791 58380 : bind = gimple_build_bind (NULL, NULL,
14792 21210 : tgt_bind ? gimple_bind_block (tgt_bind)
14793 : : NULL_TREE);
14794 73948 : gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
14795 37170 : gimple_bind_add_seq (bind, ilist);
14796 37170 : gimple_bind_add_stmt (bind, stmt);
14797 37170 : gimple_bind_add_seq (bind, olist);
14798 :
14799 37170 : pop_gimplify_context (NULL);
14800 :
14801 37170 : if (dep_bind)
14802 : {
14803 392 : gimple_bind_add_seq (dep_bind, dep_ilist);
14804 392 : gimple_bind_add_stmt (dep_bind, bind);
14805 392 : gimple_bind_add_seq (dep_bind, dep_olist);
14806 392 : pop_gimplify_context (dep_bind);
14807 : }
14808 37170 : }
14809 :
14810 : /* Generate code to implement the action-clauses (destroy, init, use) of an
14811 : OpenMP interop construct. */
14812 :
14813 : static void
14814 492 : lower_omp_interop_action_clauses (gimple_seq *seq, vec<tree> &objs,
14815 : vec<tree> *interop_types = NULL,
14816 : vec<tree> *prefer_types = NULL)
14817 : {
14818 492 : if (objs.length () == 0)
14819 229 : return;
14820 :
14821 263 : enum omp_clause_code action = OMP_CLAUSE_CODE (objs[0]);
14822 263 : if (action == OMP_CLAUSE_INIT)
14823 402 : gcc_checking_assert (objs.length () == interop_types->length ()
14824 : && objs.length () == prefer_types->length ());
14825 : else
14826 129 : gcc_assert (prefer_types == NULL && interop_types == NULL);
14827 :
14828 263 : tree ret_objs = NULL_TREE, ret_interop_types = NULL_TREE,
14829 263 : ret_prefer_types = NULL_TREE;
14830 :
14831 : /* Build an array of interop objects. */
14832 :
14833 263 : tree type_obj_pref = build_array_type_nelts (ptr_type_node, objs.length ());
14834 263 : ret_objs = create_tmp_var (type_obj_pref, "interopobjs");
14835 :
14836 263 : bool have_pref_type = false;
14837 263 : if (action == OMP_CLAUSE_INIT)
14838 : {
14839 592 : for (tree pref_type : prefer_types)
14840 260 : if (pref_type != NULL_TREE)
14841 : {
14842 : have_pref_type = true;
14843 : break;
14844 : }
14845 134 : tree type_tgtsync
14846 134 : = build_array_type_nelts (integer_type_node, objs.length ());
14847 134 : ret_interop_types = create_tmp_var (type_tgtsync, "tgt_tgtsync");
14848 134 : if (have_pref_type)
14849 70 : ret_prefer_types = create_tmp_var (type_obj_pref, "pref_type");
14850 : else
14851 : {
14852 64 : ret_prefer_types = null_pointer_node;
14853 64 : prefer_types->truncate (0);
14854 : }
14855 : }
14856 :
14857 856 : for (size_t i = 0; !objs.is_empty (); i++)
14858 : {
14859 593 : tree offset = build_int_cst (integer_type_node, i);
14860 593 : tree init = build4 (ARRAY_REF, ptr_type_node, ret_objs, offset, NULL_TREE,
14861 : NULL_TREE);
14862 593 : tree obj = OMP_CLAUSE_DECL (objs.pop ());
14863 593 : if (TREE_CODE (TREE_TYPE (obj)) == REFERENCE_TYPE)
14864 6 : obj = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (obj)), obj);
14865 593 : if (action != OMP_CLAUSE_USE
14866 593 : && TREE_CODE (TREE_TYPE (obj)) != POINTER_TYPE)
14867 : /* For modifying actions, we need a pointer. */
14868 483 : obj = build_fold_addr_expr (obj);
14869 110 : else if (action == OMP_CLAUSE_USE
14870 110 : && TREE_CODE (TREE_TYPE (obj)) == POINTER_TYPE)
14871 : /* For use action, we need the value. */
14872 2 : obj = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (obj)), obj);
14873 593 : init = build2 (MODIFY_EXPR, ptr_type_node, init,
14874 : fold_convert (ptr_type_node, obj));
14875 593 : gimplify_and_add (init, seq);
14876 :
14877 593 : if (action == OMP_CLAUSE_INIT)
14878 : {
14879 384 : init = build4 (ARRAY_REF, integer_type_node, ret_interop_types,
14880 : offset, NULL_TREE, NULL_TREE);
14881 384 : init = build2 (MODIFY_EXPR, integer_type_node, init,
14882 384 : interop_types->pop ());
14883 384 : gimplify_and_add (init, seq);
14884 :
14885 384 : if (have_pref_type)
14886 : {
14887 194 : tree prefer_type = prefer_types->pop ();
14888 194 : tree pref = (prefer_type == NULL_TREE
14889 194 : ? null_pointer_node
14890 194 : : build_fold_addr_expr (prefer_type));
14891 194 : init = build4 (ARRAY_REF, ptr_type_node, ret_prefer_types, offset,
14892 : NULL_TREE, NULL_TREE);
14893 194 : init = build2 (MODIFY_EXPR, ptr_type_node, init, pref);
14894 194 : gimplify_and_add (init, seq);
14895 : }
14896 : }
14897 : }
14898 263 : if (action == OMP_CLAUSE_INIT)
14899 : {
14900 134 : if (have_pref_type)
14901 70 : ret_prefer_types = build_fold_addr_expr (ret_prefer_types);
14902 134 : ret_interop_types = build_fold_addr_expr (ret_interop_types);
14903 : }
14904 263 : ret_objs = build_fold_addr_expr (ret_objs);
14905 :
14906 660 : gcc_assert (objs.is_empty ()
14907 : && (!interop_types || interop_types->is_empty ())
14908 : && (!prefer_types || prefer_types->is_empty ()));
14909 :
14910 263 : objs.safe_push (ret_objs);
14911 263 : if (action == OMP_CLAUSE_INIT)
14912 : {
14913 134 : interop_types->safe_push (ret_interop_types);
14914 134 : prefer_types->safe_push (ret_prefer_types);
14915 : }
14916 : }
14917 :
14918 : /* Lower code for an OpenMP interop directive. */
14919 :
14920 : static void
14921 164 : lower_omp_interop (gimple_stmt_iterator *gsi_p, omp_context *ctx)
14922 : {
14923 164 : push_gimplify_context ();
14924 :
14925 164 : tree block = make_node (BLOCK);
14926 164 : gbind *bind = gimple_build_bind (NULL, NULL, block);
14927 164 : gimple_seq bind_body = NULL;
14928 :
14929 : /* Emit call to GOMP_interop:
14930 : void
14931 : GOMP_interop (int device_num, int n_init, omp_interop_t **init,
14932 : const void *target_targetsync, const void *prefer_type,
14933 : int n_use, omp_interop_t *use, int n_destroy,
14934 : omp_interop_t **destroy, unsigned int flags,
14935 : void **depend) */
14936 :
14937 164 : tree flags = NULL_TREE;
14938 164 : tree depend = null_pointer_node;
14939 164 : tree device_num = NULL_TREE;
14940 :
14941 164 : auto_vec<tree> init_objs, use_objs, destroy_objs, prefer_type,
14942 164 : target_targetsync;
14943 164 : gimple_seq dep_ilist = NULL, dep_olist = NULL;
14944 164 : tree clauses = gimple_omp_interop_clauses (gsi_stmt (*gsi_p));
14945 868 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
14946 : {
14947 704 : switch (OMP_CLAUSE_CODE (c))
14948 : {
14949 384 : case OMP_CLAUSE_INIT:
14950 384 : {
14951 384 : init_objs.safe_push (c);
14952 384 : int target_targetsync_bits = 0;
14953 384 : if (OMP_CLAUSE_INIT_TARGET (c))
14954 229 : target_targetsync_bits |= GOMP_INTEROP_TARGET;
14955 384 : if (OMP_CLAUSE_INIT_TARGETSYNC (c))
14956 204 : target_targetsync_bits |= GOMP_INTEROP_TARGETSYNC;
14957 384 : tree t = build_int_cst (integer_type_node, target_targetsync_bits);
14958 384 : target_targetsync.safe_push (t);
14959 384 : prefer_type.safe_push (OMP_CLAUSE_INIT_PREFER_TYPE (c));
14960 : }
14961 384 : break;
14962 106 : case OMP_CLAUSE_USE:
14963 106 : use_objs.safe_push (c);
14964 106 : break;
14965 103 : case OMP_CLAUSE_DESTROY:
14966 103 : destroy_objs.safe_push (c);
14967 103 : break;
14968 50 : case OMP_CLAUSE_NOWAIT:
14969 50 : flags = build_int_cst (integer_type_node, GOMP_INTEROP_FLAG_NOWAIT);
14970 50 : break;
14971 35 : case OMP_CLAUSE_DEPEND:
14972 35 : {
14973 35 : tree *cp = gimple_omp_interop_clauses_ptr (gsi_stmt (*gsi_p));
14974 35 : lower_depend_clauses (cp, &dep_ilist, &dep_olist);
14975 35 : depend = OMP_CLAUSE_DECL (*cp);
14976 : }
14977 35 : break;
14978 26 : case OMP_CLAUSE_DEVICE:
14979 26 : device_num = OMP_CLAUSE_DEVICE_ID (c);
14980 26 : break;
14981 0 : default:
14982 0 : gcc_unreachable ();
14983 : }
14984 : }
14985 :
14986 164 : if (flags == NULL_TREE)
14987 114 : flags = build_int_cst (integer_type_node, 0);
14988 :
14989 164 : if (device_num == NULL_TREE)
14990 138 : device_num = build_int_cst (integer_type_node, GOMP_DEVICE_DEFAULT_OMP_61);
14991 :
14992 298 : tree n_init = build_int_cst (integer_type_node, init_objs.length ());
14993 234 : tree n_use = build_int_cst (integer_type_node, use_objs.length ());
14994 223 : tree n_destroy = build_int_cst (integer_type_node, destroy_objs.length ());
14995 :
14996 164 : lower_omp_interop_action_clauses (&bind_body, init_objs, &target_targetsync,
14997 : &prefer_type);
14998 164 : lower_omp_interop_action_clauses (&bind_body, use_objs);
14999 164 : lower_omp_interop_action_clauses (&bind_body, destroy_objs);
15000 :
15001 164 : gimple_seq_add_seq (&bind_body, dep_ilist);
15002 164 : tree fn = builtin_decl_explicit (BUILT_IN_GOMP_INTEROP);
15003 298 : tree init_arg = init_objs.length () ? init_objs.pop () : null_pointer_node;
15004 164 : tree target_targetsync_arg = target_targetsync.length ()
15005 134 : ? target_targetsync.pop ()
15006 164 : : null_pointer_node;
15007 164 : tree prefer_type_arg
15008 298 : = prefer_type.length () ? prefer_type.pop () : null_pointer_node;
15009 234 : tree use_arg = use_objs.length () ? use_objs.pop () : null_pointer_node;
15010 164 : tree destroy_arg
15011 223 : = destroy_objs.length () ? destroy_objs.pop () : null_pointer_node;
15012 164 : gcall *call
15013 164 : = gimple_build_call (fn, 11, device_num, n_init, init_arg,
15014 : target_targetsync_arg, prefer_type_arg, n_use, use_arg,
15015 : n_destroy, destroy_arg, flags, depend);
15016 164 : gimple_seq_add_stmt (&bind_body, call);
15017 164 : gimple_seq_add_seq (&bind_body, dep_olist);
15018 :
15019 164 : gsi_replace (gsi_p, bind, true);
15020 164 : gimple_bind_set_body (bind, bind_body);
15021 164 : pop_gimplify_context (bind);
15022 164 : gimple_bind_append_vars (bind, ctx->block_vars);
15023 164 : BLOCK_VARS (block) = ctx->block_vars;
15024 164 : }
15025 :
15026 : /* Expand code for an OpenMP teams directive. */
15027 :
15028 : static void
15029 5924 : lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx)
15030 : {
15031 5924 : gomp_teams *teams_stmt = as_a <gomp_teams *> (gsi_stmt (*gsi_p));
15032 5924 : push_gimplify_context ();
15033 :
15034 5924 : tree block = make_node (BLOCK);
15035 5924 : gbind *bind = gimple_build_bind (NULL, NULL, block);
15036 5924 : gsi_replace (gsi_p, bind, true);
15037 5924 : gimple_seq bind_body = NULL;
15038 5924 : gimple_seq dlist = NULL;
15039 5924 : gimple_seq olist = NULL;
15040 :
15041 5924 : tree num_teams = omp_find_clause (gimple_omp_teams_clauses (teams_stmt),
15042 5924 : OMP_CLAUSE_NUM_TEAMS);
15043 5924 : tree num_teams_lower = NULL_TREE;
15044 5924 : if (num_teams == NULL_TREE)
15045 5300 : num_teams = build_int_cst (unsigned_type_node, 0);
15046 : else
15047 : {
15048 624 : num_teams_lower = OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (num_teams);
15049 624 : if (num_teams_lower)
15050 : {
15051 136 : num_teams_lower = fold_convert (unsigned_type_node, num_teams_lower);
15052 136 : gimplify_expr (&num_teams_lower, &bind_body, NULL, is_gimple_val,
15053 : fb_rvalue);
15054 : }
15055 624 : num_teams = OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (num_teams);
15056 : // FIXME: Handle dim(x), cf. omp-expand.cc's get_target_arguments. */
15057 624 : if (TREE_CODE (num_teams) == TREE_LIST && TREE_CHAIN (num_teams))
15058 : {
15059 14 : num_teams_lower = TREE_VALUE (num_teams);
15060 14 : num_teams = TREE_VALUE (TREE_CHAIN (num_teams));
15061 : }
15062 610 : else if (TREE_CODE (num_teams) == TREE_LIST)
15063 80 : num_teams = TREE_VALUE (num_teams);
15064 624 : num_teams = fold_convert (unsigned_type_node, num_teams);
15065 624 : gimplify_expr (&num_teams, &bind_body, NULL, is_gimple_val, fb_rvalue);
15066 : }
15067 5924 : if (num_teams_lower == NULL_TREE)
15068 5774 : num_teams_lower = num_teams;
15069 5924 : tree thread_limit = omp_find_clause (gimple_omp_teams_clauses (teams_stmt),
15070 5924 : OMP_CLAUSE_THREAD_LIMIT);
15071 5924 : if (thread_limit == NULL_TREE)
15072 5446 : thread_limit = build_int_cst (unsigned_type_node, 0);
15073 : else
15074 : {
15075 478 : thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit);
15076 : // FIXME: Handle dim(x), cf. omp-expand.cc's get_target_arguments. */
15077 478 : if (TREE_CODE (thread_limit) == TREE_LIST)
15078 66 : thread_limit = TREE_VALUE (thread_limit);
15079 478 : thread_limit = fold_convert (unsigned_type_node, thread_limit);
15080 478 : gimplify_expr (&thread_limit, &bind_body, NULL, is_gimple_val,
15081 : fb_rvalue);
15082 : }
15083 5924 : location_t loc = gimple_location (teams_stmt);
15084 5924 : tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS4);
15085 5924 : tree rettype = TREE_TYPE (TREE_TYPE (decl));
15086 5924 : tree first = create_tmp_var (rettype);
15087 5924 : gimple_seq_add_stmt (&bind_body,
15088 5924 : gimple_build_assign (first, build_one_cst (rettype)));
15089 5924 : tree llabel = create_artificial_label (loc);
15090 5924 : gimple_seq_add_stmt (&bind_body, gimple_build_label (llabel));
15091 5924 : gimple *call
15092 5924 : = gimple_build_call (decl, 4, num_teams_lower, num_teams, thread_limit,
15093 : first);
15094 5924 : gimple_set_location (call, loc);
15095 5924 : tree temp = create_tmp_var (rettype);
15096 5924 : gimple_call_set_lhs (call, temp);
15097 5924 : gimple_seq_add_stmt (&bind_body, call);
15098 :
15099 5924 : tree tlabel = create_artificial_label (loc);
15100 5924 : tree flabel = create_artificial_label (loc);
15101 5924 : gimple *cond = gimple_build_cond (NE_EXPR, temp, build_zero_cst (rettype),
15102 : tlabel, flabel);
15103 5924 : gimple_seq_add_stmt (&bind_body, cond);
15104 5924 : gimple_seq_add_stmt (&bind_body, gimple_build_label (tlabel));
15105 5924 : gimple_seq_add_stmt (&bind_body,
15106 5924 : gimple_build_assign (first, build_zero_cst (rettype)));
15107 :
15108 5924 : lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt),
15109 : &bind_body, &dlist, ctx, NULL);
15110 5924 : lower_omp (gimple_omp_body_ptr (teams_stmt), ctx);
15111 5924 : lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt), &olist,
15112 : NULL, ctx);
15113 5924 : gimple_seq_add_stmt (&bind_body, teams_stmt);
15114 :
15115 5924 : gimple_seq_add_seq (&bind_body, gimple_omp_body (teams_stmt));
15116 5924 : gimple_omp_set_body (teams_stmt, NULL);
15117 5924 : gimple_seq_add_seq (&bind_body, olist);
15118 5924 : gimple_seq_add_seq (&bind_body, dlist);
15119 5924 : gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true));
15120 5924 : gimple_seq_add_stmt (&bind_body, gimple_build_goto (llabel));
15121 5924 : gimple_seq_add_stmt (&bind_body, gimple_build_label (flabel));
15122 5924 : gimple_bind_set_body (bind, bind_body);
15123 :
15124 5924 : pop_gimplify_context (bind);
15125 :
15126 5924 : gimple_bind_append_vars (bind, ctx->block_vars);
15127 5924 : BLOCK_VARS (block) = ctx->block_vars;
15128 5924 : if (BLOCK_VARS (block))
15129 689 : TREE_USED (block) = 1;
15130 5924 : }
15131 :
15132 : /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
15133 : regimplified. If DATA is non-NULL, lower_omp_1 is outside
15134 : of OMP context, but with make_addressable_vars set. */
15135 :
15136 : static tree
15137 3213477 : lower_omp_regimplify_p (tree *tp, int *walk_subtrees,
15138 : void *data)
15139 : {
15140 3213477 : tree t = *tp;
15141 :
15142 : /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
15143 3213477 : if ((VAR_P (t) || TREE_CODE (t) == PARM_DECL || TREE_CODE (t) == RESULT_DECL)
15144 1495904 : && data == NULL
15145 1435454 : && DECL_HAS_VALUE_EXPR_P (t))
15146 : return t;
15147 :
15148 3107587 : if (make_addressable_vars
15149 423128 : && DECL_P (t)
15150 3312910 : && bitmap_bit_p (make_addressable_vars, DECL_UID (t)))
15151 : return t;
15152 :
15153 : /* If a global variable has been privatized, TREE_CONSTANT on
15154 : ADDR_EXPR might be wrong. */
15155 3104998 : if (data == NULL && TREE_CODE (t) == ADDR_EXPR)
15156 109115 : recompute_tree_invariant_for_addr_expr (t);
15157 :
15158 3104998 : *walk_subtrees = !IS_TYPE_OR_DECL_P (t);
15159 3104998 : return NULL_TREE;
15160 : }
15161 :
15162 : /* Data to be communicated between lower_omp_regimplify_operands and
15163 : lower_omp_regimplify_operands_p. */
15164 :
15165 : struct lower_omp_regimplify_operands_data
15166 : {
15167 : omp_context *ctx;
15168 : vec<tree> *decls;
15169 : };
15170 :
15171 : /* Helper function for lower_omp_regimplify_operands. Find
15172 : omp_member_access_dummy_var vars and adjust temporarily their
15173 : DECL_VALUE_EXPRs if needed. */
15174 :
15175 : static tree
15176 427063 : lower_omp_regimplify_operands_p (tree *tp, int *walk_subtrees,
15177 : void *data)
15178 : {
15179 427063 : tree t = omp_member_access_dummy_var (*tp);
15180 427063 : if (t)
15181 : {
15182 9 : struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
15183 9 : lower_omp_regimplify_operands_data *ldata
15184 : = (lower_omp_regimplify_operands_data *) wi->info;
15185 9 : tree o = maybe_lookup_decl (t, ldata->ctx);
15186 9 : if (o == NULL_TREE)
15187 5 : o = maybe_lookup_decl_in_outer_ctx (t, ldata->ctx);
15188 9 : if (o != t)
15189 : {
15190 9 : ldata->decls->safe_push (DECL_VALUE_EXPR (*tp));
15191 9 : ldata->decls->safe_push (*tp);
15192 9 : tree v = unshare_and_remap (DECL_VALUE_EXPR (*tp), t, o);
15193 9 : SET_DECL_VALUE_EXPR (*tp, v);
15194 : }
15195 : }
15196 427063 : *walk_subtrees = !IS_TYPE_OR_DECL_P (*tp);
15197 427063 : return NULL_TREE;
15198 : }
15199 :
15200 : /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
15201 : of omp_member_access_dummy_var vars during regimplification. */
15202 :
15203 : static void
15204 108324 : lower_omp_regimplify_operands (omp_context *ctx, gimple *stmt,
15205 : gimple_stmt_iterator *gsi_p)
15206 : {
15207 108324 : auto_vec<tree, 10> decls;
15208 108324 : if (ctx)
15209 : {
15210 105802 : struct walk_stmt_info wi;
15211 105802 : memset (&wi, '\0', sizeof (wi));
15212 105802 : struct lower_omp_regimplify_operands_data data;
15213 105802 : data.ctx = ctx;
15214 105802 : data.decls = &decls;
15215 105802 : wi.info = &data;
15216 105802 : walk_gimple_op (stmt, lower_omp_regimplify_operands_p, &wi);
15217 : }
15218 108324 : gimple_regimplify_operands (stmt, gsi_p);
15219 324981 : while (!decls.is_empty ())
15220 : {
15221 9 : tree t = decls.pop ();
15222 9 : tree v = decls.pop ();
15223 9 : SET_DECL_VALUE_EXPR (t, v);
15224 : }
15225 108324 : }
15226 :
15227 : static void
15228 2508935 : lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx)
15229 : {
15230 2508935 : gimple *stmt = gsi_stmt (*gsi_p);
15231 2508935 : struct walk_stmt_info wi;
15232 2508935 : gcall *call_stmt;
15233 :
15234 2508935 : if (gimple_has_location (stmt))
15235 2060726 : input_location = gimple_location (stmt);
15236 :
15237 2508935 : if (make_addressable_vars)
15238 154214 : memset (&wi, '\0', sizeof (wi));
15239 :
15240 : /* If we have issued syntax errors, avoid doing any heavy lifting.
15241 : Just replace the OMP directives with a NOP to avoid
15242 : confusing RTL expansion. */
15243 2508935 : if (seen_error () && is_gimple_omp (stmt))
15244 : {
15245 11363 : gsi_replace (gsi_p, gimple_build_nop (), true);
15246 11363 : return;
15247 : }
15248 :
15249 2497572 : switch (gimple_code (stmt))
15250 : {
15251 156229 : case GIMPLE_COND:
15252 156229 : {
15253 156229 : gcond *cond_stmt = as_a <gcond *> (stmt);
15254 88795 : if ((ctx || make_addressable_vars)
15255 159837 : && (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
15256 : lower_omp_regimplify_p,
15257 : ctx ? NULL : &wi, NULL)
15258 136529 : || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
15259 : lower_omp_regimplify_p,
15260 : ctx ? NULL : &wi, NULL)))
15261 1247 : lower_omp_regimplify_operands (ctx, cond_stmt, gsi_p);
15262 : }
15263 : break;
15264 20 : case GIMPLE_CATCH:
15265 20 : lower_omp (gimple_catch_handler_ptr (as_a <gcatch *> (stmt)), ctx);
15266 20 : break;
15267 0 : case GIMPLE_EH_FILTER:
15268 0 : lower_omp (gimple_eh_filter_failure_ptr (stmt), ctx);
15269 0 : break;
15270 25948 : case GIMPLE_TRY:
15271 25948 : lower_omp (gimple_try_eval_ptr (stmt), ctx);
15272 25943 : lower_omp (gimple_try_cleanup_ptr (stmt), ctx);
15273 25943 : break;
15274 0 : case GIMPLE_ASSUME:
15275 0 : lower_omp (gimple_assume_body_ptr (stmt), ctx);
15276 0 : break;
15277 5 : case GIMPLE_TRANSACTION:
15278 5 : lower_omp (gimple_transaction_body_ptr (as_a <gtransaction *> (stmt)),
15279 : ctx);
15280 5 : break;
15281 168492 : case GIMPLE_BIND:
15282 168492 : if (ctx && is_gimple_omp_oacc (ctx->stmt))
15283 : {
15284 17785 : tree vars = gimple_bind_vars (as_a <gbind *> (stmt));
15285 17785 : oacc_privatization_scan_decl_chain (ctx, vars);
15286 : }
15287 168492 : lower_omp (gimple_bind_body_ptr (as_a <gbind *> (stmt)), ctx);
15288 168485 : maybe_remove_omp_member_access_dummy_vars (as_a <gbind *> (stmt));
15289 168485 : break;
15290 20036 : case GIMPLE_OMP_PARALLEL:
15291 20036 : case GIMPLE_OMP_TASK:
15292 20036 : ctx = maybe_lookup_ctx (stmt);
15293 20036 : gcc_assert (ctx);
15294 20036 : if (ctx->cancellable)
15295 135 : ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
15296 20036 : lower_omp_taskreg (gsi_p, ctx);
15297 20036 : break;
15298 46966 : case GIMPLE_OMP_FOR:
15299 46966 : ctx = maybe_lookup_ctx (stmt);
15300 46966 : gcc_assert (ctx);
15301 46966 : if (ctx->cancellable)
15302 84 : ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
15303 46966 : lower_omp_for (gsi_p, ctx);
15304 46966 : break;
15305 386 : case GIMPLE_OMP_SECTIONS:
15306 386 : ctx = maybe_lookup_ctx (stmt);
15307 386 : gcc_assert (ctx);
15308 386 : if (ctx->cancellable)
15309 19 : ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
15310 386 : lower_omp_sections (gsi_p, ctx);
15311 386 : break;
15312 161 : case GIMPLE_OMP_SCOPE:
15313 161 : ctx = maybe_lookup_ctx (stmt);
15314 161 : gcc_assert (ctx);
15315 161 : lower_omp_scope (gsi_p, ctx);
15316 161 : break;
15317 659 : case GIMPLE_OMP_DISPATCH:
15318 659 : ctx = maybe_lookup_ctx (stmt);
15319 659 : gcc_assert (ctx);
15320 659 : lower_omp_dispatch (gsi_p, ctx);
15321 659 : break;
15322 164 : case GIMPLE_OMP_INTEROP:
15323 164 : ctx = maybe_lookup_ctx (stmt);
15324 164 : gcc_assert (ctx);
15325 164 : lower_omp_interop (gsi_p, ctx);
15326 164 : break;
15327 1127 : case GIMPLE_OMP_SINGLE:
15328 1127 : ctx = maybe_lookup_ctx (stmt);
15329 1127 : gcc_assert (ctx);
15330 1127 : lower_omp_single (gsi_p, ctx);
15331 1127 : break;
15332 658 : case GIMPLE_OMP_STRUCTURED_BLOCK:
15333 : /* We have already done error checking at this point, so these nodes
15334 : can be completely removed and replaced with their body. */
15335 658 : ctx = maybe_lookup_ctx (stmt);
15336 658 : gcc_assert (ctx);
15337 658 : lower_omp (gimple_omp_body_ptr (stmt), ctx);
15338 658 : gsi_replace_with_seq (gsi_p, gimple_omp_body (stmt), true);
15339 658 : break;
15340 1118 : case GIMPLE_OMP_MASTER:
15341 1118 : case GIMPLE_OMP_MASKED:
15342 1118 : ctx = maybe_lookup_ctx (stmt);
15343 1118 : gcc_assert (ctx);
15344 1118 : lower_omp_master (gsi_p, ctx);
15345 1118 : break;
15346 536 : case GIMPLE_OMP_TASKGROUP:
15347 536 : ctx = maybe_lookup_ctx (stmt);
15348 536 : gcc_assert (ctx);
15349 536 : lower_omp_taskgroup (gsi_p, ctx);
15350 536 : break;
15351 1124 : case GIMPLE_OMP_ORDERED:
15352 1124 : ctx = maybe_lookup_ctx (stmt);
15353 1124 : gcc_assert (ctx);
15354 1124 : lower_omp_ordered (gsi_p, ctx);
15355 1124 : break;
15356 1276 : case GIMPLE_OMP_SCAN:
15357 1276 : ctx = maybe_lookup_ctx (stmt);
15358 1276 : gcc_assert (ctx);
15359 1276 : lower_omp_scan (gsi_p, ctx);
15360 1276 : break;
15361 311 : case GIMPLE_OMP_CRITICAL:
15362 311 : ctx = maybe_lookup_ctx (stmt);
15363 311 : gcc_assert (ctx);
15364 311 : lower_omp_critical (gsi_p, ctx);
15365 311 : break;
15366 2516 : case GIMPLE_OMP_ATOMIC_LOAD:
15367 85 : if ((ctx || make_addressable_vars)
15368 2516 : && walk_tree (gimple_omp_atomic_load_rhs_ptr (
15369 : as_a <gomp_atomic_load *> (stmt)),
15370 : lower_omp_regimplify_p, ctx ? NULL : &wi, NULL))
15371 1425 : lower_omp_regimplify_operands (ctx, stmt, gsi_p);
15372 : break;
15373 37170 : case GIMPLE_OMP_TARGET:
15374 37170 : ctx = maybe_lookup_ctx (stmt);
15375 37170 : gcc_assert (ctx);
15376 37170 : lower_omp_target (gsi_p, ctx);
15377 37170 : break;
15378 8458 : case GIMPLE_OMP_TEAMS:
15379 8458 : ctx = maybe_lookup_ctx (stmt);
15380 8458 : gcc_assert (ctx);
15381 8458 : if (gimple_omp_teams_host (as_a <gomp_teams *> (stmt)))
15382 2534 : lower_omp_taskreg (gsi_p, ctx);
15383 : else
15384 5924 : lower_omp_teams (gsi_p, ctx);
15385 : break;
15386 135012 : case GIMPLE_CALL:
15387 135012 : tree fndecl;
15388 135012 : call_stmt = as_a <gcall *> (stmt);
15389 135012 : fndecl = gimple_call_fndecl (call_stmt);
15390 135012 : if (fndecl
15391 135012 : && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL))
15392 38723 : switch (DECL_FUNCTION_CODE (fndecl))
15393 : {
15394 727 : case BUILT_IN_GOMP_BARRIER:
15395 727 : if (ctx == NULL)
15396 : break;
15397 : /* FALLTHRU */
15398 1037 : case BUILT_IN_GOMP_CANCEL:
15399 1037 : case BUILT_IN_GOMP_CANCELLATION_POINT:
15400 1037 : omp_context *cctx;
15401 1037 : cctx = ctx;
15402 1037 : if (gimple_code (cctx->stmt) == GIMPLE_OMP_SECTION)
15403 43 : cctx = cctx->outer;
15404 1037 : gcc_assert (gimple_call_lhs (call_stmt) == NULL_TREE);
15405 1037 : if (!cctx->cancellable)
15406 : {
15407 679 : if (DECL_FUNCTION_CODE (fndecl)
15408 : == BUILT_IN_GOMP_CANCELLATION_POINT)
15409 : {
15410 0 : stmt = gimple_build_nop ();
15411 0 : gsi_replace (gsi_p, stmt, false);
15412 : }
15413 : break;
15414 : }
15415 358 : if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_GOMP_BARRIER)
15416 : {
15417 14 : fndecl = builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL);
15418 14 : gimple_call_set_fndecl (call_stmt, fndecl);
15419 14 : gimple_call_set_fntype (call_stmt, TREE_TYPE (fndecl));
15420 : }
15421 358 : tree lhs;
15422 358 : lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl)));
15423 358 : gimple_call_set_lhs (call_stmt, lhs);
15424 358 : tree fallthru_label;
15425 358 : fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
15426 358 : gimple *g;
15427 358 : g = gimple_build_label (fallthru_label);
15428 358 : gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
15429 358 : g = gimple_build_cond (NE_EXPR, lhs,
15430 358 : fold_convert (TREE_TYPE (lhs),
15431 : boolean_false_node),
15432 : cctx->cancel_label, fallthru_label);
15433 358 : gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
15434 358 : break;
15435 : default:
15436 : break;
15437 : }
15438 134977 : goto regimplify;
15439 :
15440 : case GIMPLE_ASSIGN:
15441 1455192 : for (omp_context *up = ctx; up; up = up->outer)
15442 : {
15443 685264 : if (gimple_code (up->stmt) == GIMPLE_OMP_ORDERED
15444 : || gimple_code (up->stmt) == GIMPLE_OMP_CRITICAL
15445 : || gimple_code (up->stmt) == GIMPLE_OMP_TASKGROUP
15446 : || gimple_code (up->stmt) == GIMPLE_OMP_SCOPE
15447 : || gimple_code (up->stmt) == GIMPLE_OMP_SECTION
15448 : || gimple_code (up->stmt) == GIMPLE_OMP_SCAN
15449 : || (gimple_code (up->stmt) == GIMPLE_OMP_TARGET
15450 229998 : && (gimple_omp_target_kind (up->stmt)
15451 : == GF_OMP_TARGET_KIND_DATA)))
15452 90715 : continue;
15453 594549 : else if (!up->lastprivate_conditional_map)
15454 : break;
15455 4204 : tree lhs = get_base_address (gimple_assign_lhs (stmt));
15456 4204 : if (TREE_CODE (lhs) == MEM_REF
15457 26 : && DECL_P (TREE_OPERAND (lhs, 0))
15458 4227 : && TREE_CODE (TREE_TYPE (TREE_OPERAND (lhs,
15459 : 0))) == REFERENCE_TYPE)
15460 19 : lhs = TREE_OPERAND (lhs, 0);
15461 4204 : if (DECL_P (lhs))
15462 3161 : if (tree *v = up->lastprivate_conditional_map->get (lhs))
15463 : {
15464 382 : tree clauses;
15465 382 : if (up->combined_into_simd_safelen1)
15466 : {
15467 54 : up = up->outer;
15468 54 : if (gimple_code (up->stmt) == GIMPLE_OMP_SCAN)
15469 3 : up = up->outer;
15470 : }
15471 382 : if (gimple_code (up->stmt) == GIMPLE_OMP_FOR)
15472 298 : clauses = gimple_omp_for_clauses (up->stmt);
15473 : else
15474 84 : clauses = gimple_omp_sections_clauses (up->stmt);
15475 382 : tree c = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_);
15476 382 : if (!OMP_CLAUSE__CONDTEMP__ITER (c))
15477 348 : c = omp_find_clause (OMP_CLAUSE_CHAIN (c),
15478 : OMP_CLAUSE__CONDTEMP_);
15479 382 : gcc_assert (OMP_CLAUSE__CONDTEMP__ITER (c));
15480 382 : gimple *g = gimple_build_assign (*v, OMP_CLAUSE_DECL (c));
15481 382 : gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
15482 : }
15483 : }
15484 : /* FALLTHRU */
15485 :
15486 2024212 : default:
15487 2024212 : regimplify:
15488 1066597 : if ((ctx || make_addressable_vars)
15489 2100063 : && walk_gimple_op (stmt, lower_omp_regimplify_p,
15490 : ctx ? NULL : &wi))
15491 : {
15492 : /* Just remove clobbers, this should happen only if we have
15493 : "privatized" local addressable variables in SIMD regions,
15494 : the clobber isn't needed in that case and gimplifying address
15495 : of the ARRAY_REF into a pointer and creating MEM_REF based
15496 : clobber would create worse code than we get with the clobber
15497 : dropped. */
15498 105807 : if (gimple_clobber_p (stmt))
15499 : {
15500 155 : gsi_replace (gsi_p, gimple_build_nop (), true);
15501 155 : break;
15502 : }
15503 105652 : lower_omp_regimplify_operands (ctx, stmt, gsi_p);
15504 : }
15505 : break;
15506 : }
15507 : }
15508 :
15509 : static void
15510 411539 : lower_omp (gimple_seq *body, omp_context *ctx)
15511 : {
15512 411539 : location_t saved_location = input_location;
15513 411539 : gimple_stmt_iterator gsi;
15514 3283722 : for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
15515 2508935 : lower_omp_1 (&gsi, ctx);
15516 : /* During gimplification, we haven't folded statements inside offloading
15517 : or taskreg regions (gimplify.cc:maybe_fold_stmt); do that now. */
15518 411523 : if (target_nesting_level || taskreg_nesting_level)
15519 656970 : for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
15520 443691 : fold_stmt (&gsi);
15521 411523 : input_location = saved_location;
15522 411523 : }
15523 :
15524 : /* Main entry point. */
15525 :
15526 : static unsigned int
15527 2973071 : execute_lower_omp (void)
15528 : {
15529 2973071 : gimple_seq body;
15530 2973071 : int i;
15531 2973071 : omp_context *ctx;
15532 :
15533 : /* This pass always runs, to provide PROP_gimple_lomp.
15534 : But often, there is nothing to do. */
15535 2973071 : if (flag_openacc == 0 && flag_openmp == 0
15536 2918136 : && flag_openmp_simd == 0)
15537 : return 0;
15538 :
15539 57192 : all_contexts = splay_tree_new (splay_tree_compare_pointers, 0,
15540 : delete_omp_context);
15541 :
15542 57192 : body = gimple_body (current_function_decl);
15543 :
15544 57192 : scan_omp (&body, NULL);
15545 57192 : gcc_assert (taskreg_nesting_level == 0);
15546 83327 : FOR_EACH_VEC_ELT (taskreg_contexts, i, ctx)
15547 26135 : finish_taskreg_scan (ctx);
15548 57192 : taskreg_contexts.release ();
15549 :
15550 57192 : if (all_contexts->root)
15551 : {
15552 25857 : if (make_addressable_vars)
15553 622 : push_gimplify_context ();
15554 25857 : lower_omp (&body, NULL);
15555 25853 : if (make_addressable_vars)
15556 622 : pop_gimplify_context (NULL);
15557 : }
15558 :
15559 57188 : if (all_contexts)
15560 : {
15561 57188 : splay_tree_delete (all_contexts);
15562 57188 : all_contexts = NULL;
15563 : }
15564 57188 : BITMAP_FREE (make_addressable_vars);
15565 57188 : BITMAP_FREE (global_nonaddressable_vars);
15566 :
15567 : /* If current function is a method, remove artificial dummy VAR_DECL created
15568 : for non-static data member privatization, they aren't needed for
15569 : debuginfo nor anything else, have been already replaced everywhere in the
15570 : IL and cause problems with LTO. */
15571 57188 : if (DECL_ARGUMENTS (current_function_decl)
15572 32615 : && DECL_ARTIFICIAL (DECL_ARGUMENTS (current_function_decl))
15573 66740 : && (TREE_CODE (TREE_TYPE (DECL_ARGUMENTS (current_function_decl)))
15574 : == POINTER_TYPE))
15575 8990 : remove_member_access_dummy_vars (DECL_INITIAL (current_function_decl));
15576 :
15577 58164 : for (auto task_stmt : task_cpyfns)
15578 504 : finalize_task_copyfn (task_stmt);
15579 57188 : task_cpyfns.release ();
15580 57188 : return 0;
15581 : }
15582 :
15583 : namespace {
15584 :
15585 : const pass_data pass_data_lower_omp =
15586 : {
15587 : GIMPLE_PASS, /* type */
15588 : "omplower", /* name */
15589 : OPTGROUP_OMP, /* optinfo_flags */
15590 : TV_NONE, /* tv_id */
15591 : PROP_gimple_any, /* properties_required */
15592 : PROP_gimple_lomp | PROP_gimple_lomp_dev, /* properties_provided */
15593 : 0, /* properties_destroyed */
15594 : 0, /* todo_flags_start */
15595 : 0, /* todo_flags_finish */
15596 : };
15597 :
15598 : class pass_lower_omp : public gimple_opt_pass
15599 : {
15600 : public:
15601 293828 : pass_lower_omp (gcc::context *ctxt)
15602 587656 : : gimple_opt_pass (pass_data_lower_omp, ctxt)
15603 : {}
15604 :
15605 : /* opt_pass methods: */
15606 2973071 : unsigned int execute (function *) final override
15607 : {
15608 2973071 : return execute_lower_omp ();
15609 : }
15610 :
15611 : }; // class pass_lower_omp
15612 :
15613 : } // anon namespace
15614 :
15615 : gimple_opt_pass *
15616 293828 : make_pass_lower_omp (gcc::context *ctxt)
15617 : {
15618 293828 : return new pass_lower_omp (ctxt);
15619 : }
15620 :
15621 : /* The following is a utility to diagnose structured block violations.
15622 : It is not part of the "omplower" pass, as that's invoked too late. It
15623 : should be invoked by the respective front ends after gimplification. */
15624 :
15625 : static splay_tree all_labels;
15626 :
15627 : /* Check for mismatched contexts and generate an error if needed. Return
15628 : true if an error is detected. */
15629 :
15630 : static bool
15631 562345 : diagnose_sb_0 (gimple_stmt_iterator *gsi_p,
15632 : gimple *branch_ctx, gimple *label_ctx)
15633 : {
15634 562345 : gcc_checking_assert (!branch_ctx || is_gimple_omp (branch_ctx));
15635 562345 : gcc_checking_assert (!label_ctx || is_gimple_omp (label_ctx));
15636 :
15637 562345 : if (label_ctx == branch_ctx)
15638 : return false;
15639 :
15640 180 : const char* kind = NULL;
15641 :
15642 180 : if (flag_openacc)
15643 : {
15644 7 : if ((branch_ctx && is_gimple_omp_oacc (branch_ctx))
15645 16 : || (label_ctx && is_gimple_omp_oacc (label_ctx)))
15646 : {
15647 : gcc_checking_assert (kind == NULL);
15648 : kind = "OpenACC";
15649 : }
15650 : }
15651 : if (kind == NULL)
15652 : {
15653 164 : gcc_checking_assert (flag_openmp || flag_openmp_simd);
15654 : kind = "OpenMP";
15655 : }
15656 :
15657 : /* Previously we kept track of the label's entire context in diagnose_sb_[12]
15658 : so we could traverse it and issue a correct "exit" or "enter" error
15659 : message upon a structured block violation.
15660 :
15661 : We built the context by building a list with tree_cons'ing, but there is
15662 : no easy counterpart in gimple tuples. It seems like far too much work
15663 : for issuing exit/enter error messages. If someone really misses the
15664 : distinct error message... patches welcome. */
15665 :
15666 : #if 0
15667 : /* Try to avoid confusing the user by producing and error message
15668 : with correct "exit" or "enter" verbiage. We prefer "exit"
15669 : unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
15670 : if (branch_ctx == NULL)
15671 : exit_p = false;
15672 : else
15673 : {
15674 : while (label_ctx)
15675 : {
15676 : if (TREE_VALUE (label_ctx) == branch_ctx)
15677 : {
15678 : exit_p = false;
15679 : break;
15680 : }
15681 : label_ctx = TREE_CHAIN (label_ctx);
15682 : }
15683 : }
15684 :
15685 : if (exit_p)
15686 : error ("invalid exit from %s structured block", kind);
15687 : else
15688 : error ("invalid entry to %s structured block", kind);
15689 : #endif
15690 :
15691 : /* If it's obvious we have an invalid entry, be specific about the error. */
15692 180 : if (branch_ctx == NULL)
15693 53 : error ("invalid entry to %s structured block", kind);
15694 : else
15695 : {
15696 : /* Otherwise, be vague and lazy, but efficient. */
15697 127 : error ("invalid branch to/from %s structured block", kind);
15698 : }
15699 :
15700 180 : gsi_replace (gsi_p, gimple_build_nop (), false);
15701 180 : return true;
15702 : }
15703 :
15704 : /* Pass 1: Create a minimal tree of structured blocks, and record
15705 : where each label is found. */
15706 :
15707 : static tree
15708 3355310 : diagnose_sb_1 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
15709 : struct walk_stmt_info *wi)
15710 : {
15711 3355310 : gimple *context = (gimple *) wi->info;
15712 3355310 : gimple *inner_context;
15713 3355310 : gimple *stmt = gsi_stmt (*gsi_p);
15714 :
15715 3355310 : *handled_ops_p = true;
15716 :
15717 3355310 : switch (gimple_code (stmt))
15718 : {
15719 320580 : WALK_SUBSTMTS;
15720 :
15721 83794 : case GIMPLE_OMP_PARALLEL:
15722 83794 : case GIMPLE_OMP_TASK:
15723 83794 : case GIMPLE_OMP_SCOPE:
15724 83794 : case GIMPLE_OMP_SECTIONS:
15725 83794 : case GIMPLE_OMP_SINGLE:
15726 83794 : case GIMPLE_OMP_SECTION:
15727 83794 : case GIMPLE_OMP_STRUCTURED_BLOCK:
15728 83794 : case GIMPLE_OMP_MASTER:
15729 83794 : case GIMPLE_OMP_MASKED:
15730 83794 : case GIMPLE_OMP_ORDERED:
15731 83794 : case GIMPLE_OMP_SCAN:
15732 83794 : case GIMPLE_OMP_CRITICAL:
15733 83794 : case GIMPLE_OMP_TARGET:
15734 83794 : case GIMPLE_OMP_TEAMS:
15735 83794 : case GIMPLE_OMP_TASKGROUP:
15736 : /* The minimal context here is just the current OMP construct. */
15737 83794 : inner_context = stmt;
15738 83794 : wi->info = inner_context;
15739 83794 : walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
15740 83794 : wi->info = context;
15741 83794 : break;
15742 :
15743 52836 : case GIMPLE_OMP_FOR:
15744 52836 : inner_context = stmt;
15745 52836 : wi->info = inner_context;
15746 : /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
15747 : walk them. */
15748 52836 : walk_gimple_seq (gimple_omp_for_pre_body (stmt),
15749 : diagnose_sb_1, NULL, wi);
15750 52836 : walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
15751 52836 : wi->info = context;
15752 52836 : break;
15753 :
15754 559445 : case GIMPLE_LABEL:
15755 1118890 : splay_tree_insert (all_labels,
15756 559445 : (splay_tree_key) gimple_label_label (
15757 559445 : as_a <glabel *> (stmt)),
15758 : (splay_tree_value) context);
15759 559445 : break;
15760 :
15761 : default:
15762 : break;
15763 : }
15764 :
15765 3355310 : return NULL_TREE;
15766 : }
15767 :
15768 : /* Pass 2: Check each branch and see if its context differs from that of
15769 : the destination label's context. */
15770 :
15771 : static tree
15772 3355310 : diagnose_sb_2 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
15773 : struct walk_stmt_info *wi)
15774 : {
15775 3355310 : gimple *context = (gimple *) wi->info;
15776 3355310 : splay_tree_node n;
15777 3355310 : gimple *stmt = gsi_stmt (*gsi_p);
15778 :
15779 3355310 : *handled_ops_p = true;
15780 :
15781 3355310 : switch (gimple_code (stmt))
15782 : {
15783 320580 : WALK_SUBSTMTS;
15784 :
15785 83794 : case GIMPLE_OMP_PARALLEL:
15786 83794 : case GIMPLE_OMP_TASK:
15787 83794 : case GIMPLE_OMP_SCOPE:
15788 83794 : case GIMPLE_OMP_SECTIONS:
15789 83794 : case GIMPLE_OMP_SINGLE:
15790 83794 : case GIMPLE_OMP_SECTION:
15791 83794 : case GIMPLE_OMP_STRUCTURED_BLOCK:
15792 83794 : case GIMPLE_OMP_MASTER:
15793 83794 : case GIMPLE_OMP_MASKED:
15794 83794 : case GIMPLE_OMP_ORDERED:
15795 83794 : case GIMPLE_OMP_SCAN:
15796 83794 : case GIMPLE_OMP_CRITICAL:
15797 83794 : case GIMPLE_OMP_TARGET:
15798 83794 : case GIMPLE_OMP_TEAMS:
15799 83794 : case GIMPLE_OMP_TASKGROUP:
15800 83794 : wi->info = stmt;
15801 83794 : walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
15802 83794 : wi->info = context;
15803 83794 : break;
15804 :
15805 52836 : case GIMPLE_OMP_FOR:
15806 52836 : wi->info = stmt;
15807 : /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
15808 : walk them. */
15809 52836 : walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt),
15810 : diagnose_sb_2, NULL, wi);
15811 52836 : walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
15812 52836 : wi->info = context;
15813 52836 : break;
15814 :
15815 207120 : case GIMPLE_COND:
15816 207120 : {
15817 207120 : gcond *cond_stmt = as_a <gcond *> (stmt);
15818 207120 : tree lab = gimple_cond_true_label (cond_stmt);
15819 207120 : if (lab)
15820 : {
15821 207120 : n = splay_tree_lookup (all_labels,
15822 : (splay_tree_key) lab);
15823 414240 : diagnose_sb_0 (gsi_p, context,
15824 207120 : n ? (gimple *) n->value : NULL);
15825 : }
15826 207120 : lab = gimple_cond_false_label (cond_stmt);
15827 207120 : if (lab)
15828 : {
15829 207120 : n = splay_tree_lookup (all_labels,
15830 : (splay_tree_key) lab);
15831 414240 : diagnose_sb_0 (gsi_p, context,
15832 207120 : n ? (gimple *) n->value : NULL);
15833 : }
15834 : }
15835 : break;
15836 :
15837 112173 : case GIMPLE_GOTO:
15838 112173 : {
15839 112173 : tree lab = gimple_goto_dest (stmt);
15840 112173 : if (TREE_CODE (lab) != LABEL_DECL)
15841 : break;
15842 :
15843 112173 : n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
15844 112173 : diagnose_sb_0 (gsi_p, context, n ? (gimple *) n->value : NULL);
15845 : }
15846 112173 : break;
15847 :
15848 424 : case GIMPLE_SWITCH:
15849 424 : {
15850 424 : gswitch *switch_stmt = as_a <gswitch *> (stmt);
15851 424 : unsigned int i;
15852 1724 : for (i = 0; i < gimple_switch_num_labels (switch_stmt); ++i)
15853 : {
15854 1316 : tree lab = CASE_LABEL (gimple_switch_label (switch_stmt, i));
15855 1316 : n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
15856 1316 : if (n && diagnose_sb_0 (gsi_p, context, (gimple *) n->value))
15857 : break;
15858 : }
15859 : }
15860 : break;
15861 :
15862 742 : case GIMPLE_ASM:
15863 742 : {
15864 742 : gasm *asm_stmt = as_a <gasm *> (stmt);
15865 757 : for (unsigned i = 0; i < gimple_asm_nlabels (asm_stmt); ++i)
15866 : {
15867 21 : tree lab = TREE_VALUE (gimple_asm_label_op (asm_stmt, i));
15868 21 : n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
15869 21 : if (n && diagnose_sb_0 (gsi_p, context, (gimple *) n->value))
15870 : break;
15871 : }
15872 : }
15873 : break;
15874 :
15875 34595 : case GIMPLE_RETURN:
15876 34595 : diagnose_sb_0 (gsi_p, context, NULL);
15877 34595 : break;
15878 :
15879 : default:
15880 : break;
15881 : }
15882 :
15883 3355310 : return NULL_TREE;
15884 : }
15885 :
15886 : static unsigned int
15887 57201 : diagnose_omp_structured_block_errors (void)
15888 : {
15889 57201 : struct walk_stmt_info wi;
15890 57201 : gimple_seq body = gimple_body (current_function_decl);
15891 :
15892 57201 : all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0);
15893 :
15894 57201 : memset (&wi, 0, sizeof (wi));
15895 57201 : walk_gimple_seq (body, diagnose_sb_1, NULL, &wi);
15896 :
15897 57201 : memset (&wi, 0, sizeof (wi));
15898 57201 : wi.want_locations = true;
15899 57201 : walk_gimple_seq_mod (&body, diagnose_sb_2, NULL, &wi);
15900 :
15901 57201 : gimple_set_body (current_function_decl, body);
15902 :
15903 57201 : splay_tree_delete (all_labels);
15904 57201 : all_labels = NULL;
15905 :
15906 57201 : return 0;
15907 : }
15908 :
15909 : namespace {
15910 :
15911 : const pass_data pass_data_diagnose_omp_blocks =
15912 : {
15913 : GIMPLE_PASS, /* type */
15914 : "*diagnose_omp_blocks", /* name */
15915 : OPTGROUP_OMP, /* optinfo_flags */
15916 : TV_NONE, /* tv_id */
15917 : PROP_gimple_any, /* properties_required */
15918 : 0, /* properties_provided */
15919 : 0, /* properties_destroyed */
15920 : 0, /* todo_flags_start */
15921 : 0, /* todo_flags_finish */
15922 : };
15923 :
15924 : class pass_diagnose_omp_blocks : public gimple_opt_pass
15925 : {
15926 : public:
15927 293828 : pass_diagnose_omp_blocks (gcc::context *ctxt)
15928 587656 : : gimple_opt_pass (pass_data_diagnose_omp_blocks, ctxt)
15929 : {}
15930 :
15931 : /* opt_pass methods: */
15932 2973085 : bool gate (function *) final override
15933 : {
15934 2973085 : return flag_openacc || flag_openmp || flag_openmp_simd;
15935 : }
15936 57201 : unsigned int execute (function *) final override
15937 : {
15938 57201 : return diagnose_omp_structured_block_errors ();
15939 : }
15940 :
15941 : }; // class pass_diagnose_omp_blocks
15942 :
15943 : } // anon namespace
15944 :
15945 : gimple_opt_pass *
15946 293828 : make_pass_diagnose_omp_blocks (gcc::context *ctxt)
15947 : {
15948 293828 : return new pass_diagnose_omp_blocks (ctxt);
15949 : }
15950 :
15951 :
15952 : #include "gt-omp-low.h"
|
