LCOV - code coverage report
Current view: top level - gcc - omp-low.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 95.1 % 9350 8888
Test Date: 2026-11-14 16:39:50 Functions: 98.5 % 131 129
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Lowering pass for OMP directives.  Converts OMP directives into explicit
       2              :    calls to the runtime library (libgomp), data marshalling to implement data
       3              :    sharing and copying clauses, offloading to accelerators, and more.
       4              : 
       5              :    Contributed by Diego Novillo <dnovillo@redhat.com>
       6              : 
       7              :    Copyright (C) 2005-2026 Free Software Foundation, Inc.
       8              : 
       9              : This file is part of GCC.
      10              : 
      11              : GCC is free software; you can redistribute it and/or modify it under
      12              : the terms of the GNU General Public License as published by the Free
      13              : Software Foundation; either version 3, or (at your option) any later
      14              : version.
      15              : 
      16              : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      17              : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      18              : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      19              : for more details.
      20              : 
      21              : You should have received a copy of the GNU General Public License
      22              : along with GCC; see the file COPYING3.  If not see
      23              : <http://www.gnu.org/licenses/>.  */
      24              : 
      25              : #include "config.h"
      26              : #include "system.h"
      27              : #include "coretypes.h"
      28              : #include "backend.h"
      29              : #include "target.h"
      30              : #include "tree.h"
      31              : #include "gimple.h"
      32              : #include "tree-pass.h"
      33              : #include "ssa.h"
      34              : #include "cgraph.h"
      35              : #include "pretty-print.h"
      36              : #include "diagnostic-core.h"
      37              : #include "fold-const.h"
      38              : #include "stor-layout.h"
      39              : #include "internal-fn.h"
      40              : #include "gimple-iterator.h"
      41              : #include "gimple-fold.h"
      42              : #include "gimplify.h"
      43              : #include "gimplify-me.h"
      44              : #include "gimple-walk.h"
      45              : #include "tree-iterator.h"
      46              : #include "tree-inline.h"
      47              : #include "langhooks.h"
      48              : #include "tree-dfa.h"
      49              : #include "tree-ssa.h"
      50              : #include "splay-tree.h"
      51              : #include "omp-general.h"
      52              : #include "omp-low.h"
      53              : #include "gimple-low.h"
      54              : #include "alloc-pool.h"
      55              : #include "symbol-summary.h"
      56              : #include "tree-nested.h"
      57              : #include "context.h"
      58              : #include "gomp-constants.h"
      59              : #include "gimple-pretty-print.h"
      60              : #include "stringpool.h"
      61              : #include "attribs.h"
      62              : #include "omp-offload.h"
      63              : 
      64              : /* Lowering of OMP parallel and workshare constructs proceeds in two
      65              :    phases.  The first phase scans the function looking for OMP statements
      66              :    and then for variables that must be replaced to satisfy data sharing
      67              :    clauses.  The second phase expands code for the constructs, as well as
      68              :    re-gimplifying things when variables have been replaced with complex
      69              :    expressions.
      70              : 
      71              :    Final code generation is done by pass_expand_omp.  The flowgraph is
      72              :    scanned for regions which are then moved to a new
      73              :    function, to be invoked by the thread library, or offloaded.  */
      74              : 
      75              : /* Context structure.  Used to store information about each parallel
      76              :    directive in the code.  */
      77              : 
      78              : struct omp_context
      79              : {
      80              :   /* This field must be at the beginning, as we do "inheritance": Some
      81              :      callback functions for tree-inline.cc (e.g., omp_copy_decl)
      82              :      receive a copy_body_data pointer that is up-casted to an
      83              :      omp_context pointer.  */
      84              :   copy_body_data cb;
      85              : 
      86              :   /* The tree of contexts corresponding to the encountered constructs.  */
      87              :   struct omp_context *outer;
      88              :   gimple *stmt;
      89              : 
      90              :   /* Map variables to fields in a structure that allows communication
      91              :      between sending and receiving threads.  */
      92              :   splay_tree field_map;
      93              :   tree record_type;
      94              :   tree sender_decl;
      95              :   tree receiver_decl;
      96              : 
      97              :   /* These are used just by task contexts, if task firstprivate fn is
      98              :      needed.  srecord_type is used to communicate from the thread
      99              :      that encountered the task construct to task firstprivate fn,
     100              :      record_type is allocated by GOMP_task, initialized by task firstprivate
     101              :      fn and passed to the task body fn.  */
     102              :   splay_tree sfield_map;
     103              :   tree srecord_type;
     104              : 
     105              :   /* A chain of variables to add to the top-level block surrounding the
     106              :      construct.  In the case of a parallel, this is in the child function.  */
     107              :   tree block_vars;
     108              : 
     109              :   /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
     110              :      barriers should jump to during omplower pass.  */
     111              :   tree cancel_label;
     112              : 
     113              :   /* The sibling GIMPLE_OMP_FOR simd with _simt_ clause or NULL
     114              :      otherwise.  */
     115              :   gimple *simt_stmt;
     116              : 
     117              :   /* For task reductions registered in this context, a vector containing
     118              :      the length of the private copies block (if constant, otherwise NULL)
     119              :      and then offsets (if constant, otherwise NULL) for each entry.  */
     120              :   vec<tree> task_reductions;
     121              : 
     122              :   /* A hash map from the reduction clauses to the registered array
     123              :      elts.  */
     124              :   hash_map<tree, unsigned> *task_reduction_map;
     125              : 
     126              :   /* And a hash map from the lastprivate(conditional:) variables to their
     127              :      corresponding tracking loop iteration variables.  */
     128              :   hash_map<tree, tree> *lastprivate_conditional_map;
     129              : 
     130              :   /* And a hash map from the allocate variables to their corresponding
     131              :      allocators.  */
     132              :   hash_map<tree, tree> *allocate_map;
     133              : 
     134              :   /* A tree_list of the reduction clauses in this context. This is
     135              :     only used for checking the consistency of OpenACC reduction
     136              :     clauses in scan_omp_for and is not guaranteed to contain a valid
     137              :     value outside of this function. */
     138              :   tree local_reduction_clauses;
     139              : 
     140              :   /* A tree_list of the reduction clauses in outer contexts. This is
     141              :     only used for checking the consistency of OpenACC reduction
     142              :     clauses in scan_omp_for and is not guaranteed to contain a valid
     143              :     value outside of this function. */
     144              :   tree outer_reduction_clauses;
     145              : 
     146              :   /* Nesting depth of this context.  Used to beautify error messages re
     147              :      invalid gotos.  The outermost ctx is depth 1, with depth 0 being
     148              :      reserved for the main body of the function.  */
     149              :   int depth;
     150              : 
     151              :   /* True if this parallel directive is nested within another.  */
     152              :   bool is_nested;
     153              : 
     154              :   /* True if this construct can be cancelled.  */
     155              :   bool cancellable;
     156              : 
     157              :   /* True if lower_omp_1 should look up lastprivate conditional in parent
     158              :      context.  */
     159              :   bool combined_into_simd_safelen1;
     160              : 
     161              :   /* True if there is nested scan context with inclusive clause.  */
     162              :   bool scan_inclusive;
     163              : 
     164              :   /* True if there is nested scan context with exclusive clause.  */
     165              :   bool scan_exclusive;
     166              : 
     167              :   /* True in the second simd loop of for simd with inscan reductions.  */
     168              :   bool for_simd_scan_phase;
     169              : 
     170              :   /* True if there is order(concurrent) clause on the construct.  */
     171              :   bool order_concurrent;
     172              : 
     173              :   /* True if there is bind clause on the construct (i.e. a loop construct).  */
     174              :   bool loop_p;
     175              : 
     176              :   /* Only used for omp target contexts.  True if a teams construct is
     177              :      strictly nested in it.  */
     178              :   bool teams_nested_p;
     179              : 
     180              :   /* Only used for omp target contexts.  True if an OpenMP construct other
     181              :      than teams is strictly nested in it.  */
     182              :   bool nonteams_nested_p;
     183              : 
     184              :   /* Candidates for adjusting OpenACC privatization level.  */
     185              :   vec<tree> oacc_privatization_candidates;
     186              : };
     187              : 
     188              : static splay_tree all_contexts;
     189              : static int taskreg_nesting_level;
     190              : static int target_nesting_level;
     191              : static bitmap make_addressable_vars;
     192              : static bitmap global_nonaddressable_vars;
     193              : static vec<omp_context *> taskreg_contexts;
     194              : static vec<gomp_task *> task_cpyfns;
     195              : 
     196              : static void scan_omp (gimple_seq *, omp_context *);
     197              : static tree scan_omp_1_op (tree *, int *, void *);
     198              : static bool omp_maybe_offloaded_ctx (omp_context *ctx);
     199              : 
     200              : #define WALK_SUBSTMTS  \
     201              :     case GIMPLE_BIND: \
     202              :     case GIMPLE_TRY: \
     203              :     case GIMPLE_CATCH: \
     204              :     case GIMPLE_EH_FILTER: \
     205              :     case GIMPLE_ASSUME: \
     206              :     case GIMPLE_TRANSACTION: \
     207              :       /* The sub-statements for these should be walked.  */ \
     208              :       *handled_ops_p = false; \
     209              :       break;
     210              : 
     211              : /* Return whether CTX represents an OpenACC 'parallel' or 'serial' construct.
     212              :    (This doesn't include OpenACC 'kernels' decomposed parts.)  */
     213              : 
     214              : static bool
     215        33860 : is_oacc_parallel_or_serial (omp_context *ctx)
     216              : {
     217        33860 :   enum gimple_code outer_type = gimple_code (ctx->stmt);
     218        33860 :   return ((outer_type == GIMPLE_OMP_TARGET)
     219        33860 :           && ((gimple_omp_target_kind (ctx->stmt)
     220              :                == GF_OMP_TARGET_KIND_OACC_PARALLEL)
     221        15518 :               || (gimple_omp_target_kind (ctx->stmt)
     222        33860 :                   == GF_OMP_TARGET_KIND_OACC_SERIAL)));
     223              : }
     224              : 
     225              : /* Return whether CTX represents an OpenACC 'kernels' construct.
     226              :    (This doesn't include OpenACC 'kernels' decomposed parts.)  */
     227              : 
     228              : static bool
     229        49900 : is_oacc_kernels (omp_context *ctx)
     230              : {
     231        49900 :   enum gimple_code outer_type = gimple_code (ctx->stmt);
     232        49900 :   return ((outer_type == GIMPLE_OMP_TARGET)
     233        49900 :           && (gimple_omp_target_kind (ctx->stmt)
     234        49900 :               == GF_OMP_TARGET_KIND_OACC_KERNELS));
     235              : }
     236              : 
     237              : /* Return whether CTX represents an OpenACC 'kernels' decomposed part.  */
     238              : 
     239              : static bool
     240        21645 : is_oacc_kernels_decomposed_part (omp_context *ctx)
     241              : {
     242        21645 :   enum gimple_code outer_type = gimple_code (ctx->stmt);
     243        21645 :   return ((outer_type == GIMPLE_OMP_TARGET)
     244        21645 :           && ((gimple_omp_target_kind (ctx->stmt)
     245              :                == GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED)
     246        14552 :               || (gimple_omp_target_kind (ctx->stmt)
     247              :                   == GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE)
     248        14552 :               || (gimple_omp_target_kind (ctx->stmt)
     249        21645 :                   == GF_OMP_TARGET_KIND_OACC_DATA_KERNELS)));
     250              : }
     251              : 
     252              : /* Return true if STMT corresponds to an OpenMP target region.  */
     253              : static bool
     254        43357 : is_omp_target (gimple *stmt)
     255              : {
     256        43357 :   if (gimple_code (stmt) == GIMPLE_OMP_TARGET)
     257              :     {
     258         6523 :       int kind = gimple_omp_target_kind (stmt);
     259         6523 :       return (kind == GF_OMP_TARGET_KIND_REGION
     260         6523 :               || kind == GF_OMP_TARGET_KIND_DATA
     261         6523 :               || kind == GF_OMP_TARGET_KIND_ENTER_DATA
     262         6523 :               || kind == GF_OMP_TARGET_KIND_EXIT_DATA);
     263              :     }
     264              :   return false;
     265              : }
     266              : 
     267              : /* If DECL is the artificial dummy VAR_DECL created for non-static
     268              :    data member privatization, return the underlying "this" parameter,
     269              :    otherwise return NULL.  */
     270              : 
     271              : tree
     272       544380 : omp_member_access_dummy_var (tree decl)
     273              : {
     274       544380 :   if (!VAR_P (decl)
     275       334465 :       || !DECL_ARTIFICIAL (decl)
     276       164217 :       || !DECL_IGNORED_P (decl)
     277       158138 :       || !DECL_HAS_VALUE_EXPR_P (decl)
     278       557152 :       || !lang_hooks.decls.omp_disregard_value_expr (decl, false))
     279              :     return NULL_TREE;
     280              : 
     281         2260 :   tree v = DECL_VALUE_EXPR (decl);
     282         2260 :   if (TREE_CODE (v) != COMPONENT_REF)
     283              :     return NULL_TREE;
     284              : 
     285        16164 :   while (1)
     286         9197 :     switch (TREE_CODE (v))
     287              :       {
     288         6967 :       case COMPONENT_REF:
     289         6967 :       case MEM_REF:
     290         6967 :       case INDIRECT_REF:
     291         6967 :       CASE_CONVERT:
     292         6967 :       case POINTER_PLUS_EXPR:
     293         6967 :         v = TREE_OPERAND (v, 0);
     294         6967 :         continue;
     295         2211 :       case PARM_DECL:
     296         2211 :         if (DECL_CONTEXT (v) == current_function_decl
     297         2211 :             && DECL_ARTIFICIAL (v)
     298         4422 :             && TREE_CODE (TREE_TYPE (v)) == POINTER_TYPE)
     299         2211 :           return v;
     300              :         return NULL_TREE;
     301              :       default:
     302              :         return NULL_TREE;
     303              :       }
     304              : }
     305              : 
     306              : /* Helper for unshare_and_remap, called through walk_tree.  */
     307              : 
     308              : static tree
     309         1579 : unshare_and_remap_1 (tree *tp, int *walk_subtrees, void *data)
     310              : {
     311         1579 :   tree *pair = (tree *) data;
     312         1579 :   if (*tp == pair[0])
     313              :     {
     314          233 :       *tp = unshare_expr (pair[1]);
     315          233 :       *walk_subtrees = 0;
     316              :     }
     317         1346 :   else if (IS_TYPE_OR_DECL_P (*tp))
     318          244 :     *walk_subtrees = 0;
     319         1579 :   return NULL_TREE;
     320              : }
     321              : 
     322              : /* Return unshare_expr (X) with all occurrences of FROM
     323              :    replaced with TO.  */
     324              : 
     325              : static tree
     326          150 : unshare_and_remap (tree x, tree from, tree to)
     327              : {
     328          150 :   tree pair[2] = { from, to };
     329          150 :   x = unshare_expr (x);
     330          150 :   walk_tree (&x, unshare_and_remap_1, pair, NULL);
     331          150 :   return x;
     332              : }
     333              : 
     334              : /* Convenience function for calling scan_omp_1_op on tree operands.  */
     335              : 
     336              : static inline tree
     337       340392 : scan_omp_op (tree *tp, omp_context *ctx)
     338              : {
     339       340392 :   struct walk_stmt_info wi;
     340              : 
     341       340392 :   memset (&wi, 0, sizeof (wi));
     342       340392 :   wi.info = ctx;
     343       340392 :   wi.want_locations = true;
     344              : 
     345       340392 :   return walk_tree (tp, scan_omp_1_op, &wi, NULL);
     346              : }
     347              : 
     348              : static void lower_omp (gimple_seq *, omp_context *);
     349              : static tree lookup_decl_in_outer_ctx (tree, omp_context *);
     350              : static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *);
     351              : 
     352              : /* Return true if CTX is for an omp parallel.  */
     353              : 
     354              : static inline bool
     355       779973 : is_parallel_ctx (omp_context *ctx)
     356              : {
     357        61645 :   return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL;
     358              : }
     359              : 
     360              : 
     361              : /* Return true if CTX is for an omp task.  */
     362              : 
     363              : static inline bool
     364       238529 : is_task_ctx (omp_context *ctx)
     365              : {
     366        39390 :   return gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
     367              : }
     368              : 
     369              : 
     370              : /* Return true if CTX is for an omp taskloop.  */
     371              : 
     372              : static inline bool
     373         8193 : is_taskloop_ctx (omp_context *ctx)
     374              : {
     375         8193 :   return gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
     376         8193 :          && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_TASKLOOP;
     377              : }
     378              : 
     379              : 
     380              : /* Return true if CTX is for a host omp teams.  */
     381              : 
     382              : static inline bool
     383       523363 : is_host_teams_ctx (omp_context *ctx)
     384              : {
     385       523363 :   return gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
     386       523363 :          && gimple_omp_teams_host (as_a <gomp_teams *> (ctx->stmt));
     387              : }
     388              : 
     389              : /* Return true if CTX is for an omp parallel or omp task or host omp teams
     390              :    (the last one is strictly not a task region in OpenMP speak, but we
     391              :    need to treat it similarly).  */
     392              : 
     393              : static inline bool
     394       666635 : is_taskreg_ctx (omp_context *ctx)
     395              : {
     396       666635 :   return is_parallel_ctx (ctx) || is_task_ctx (ctx) || is_host_teams_ctx (ctx);
     397              : }
     398              : 
     399              : /* Return true if EXPR is variable sized.  If IS_REF is true, then
     400              :    EXPR is assumed to be a reference and the object it refers
     401              :    to is checked instead.  */
     402              : 
     403              : static inline bool
     404       910988 : is_variable_sized (const_tree expr, bool is_ref = false)
     405              : {
     406       910988 :   if (is_ref)
     407         2146 :     expr = TREE_TYPE (expr);
     408       910988 :   return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr)));
     409              : }
     410              : 
     411              : /* Lookup variables.  The "maybe" form
     412              :    allows for the variable form to not have been entered, otherwise we
     413              :    assert that the variable must have been entered.  */
     414              : 
     415              : static inline tree
     416       720060 : lookup_decl (tree var, omp_context *ctx)
     417              : {
     418      1440013 :   tree *n = ctx->cb.decl_map->get (var);
     419       720060 :   return *n;
     420              : }
     421              : 
     422              : static inline tree
     423       721448 : maybe_lookup_decl (const_tree var, omp_context *ctx)
     424              : {
     425       721448 :   tree *n = ctx->cb.decl_map->get (const_cast<tree> (var));
     426       721448 :   return n ? *n : NULL_TREE;
     427              : }
     428              : 
     429              : static inline tree
     430       128080 : lookup_field (tree var, omp_context *ctx)
     431              : {
     432       128080 :   splay_tree_node n;
     433       256160 :   n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
     434       128080 :   return (tree) n->value;
     435              : }
     436              : 
     437              : static inline tree
     438       170303 : lookup_sfield (splay_tree_key key, omp_context *ctx)
     439              : {
     440       170303 :   splay_tree_node n;
     441       170303 :   n = splay_tree_lookup (ctx->sfield_map
     442              :                          ? ctx->sfield_map : ctx->field_map, key);
     443       170303 :   return (tree) n->value;
     444              : }
     445              : 
     446              : static inline tree
     447          967 : lookup_sfield (tree var, omp_context *ctx)
     448              : {
     449          967 :   return lookup_sfield ((splay_tree_key) var, ctx);
     450              : }
     451              : 
     452              : static inline tree
     453       222415 : maybe_lookup_field (splay_tree_key key, omp_context *ctx)
     454              : {
     455       222415 :   splay_tree_node n;
     456       222415 :   n = splay_tree_lookup (ctx->field_map, key);
     457       222415 :   return n ? (tree) n->value : NULL_TREE;
     458              : }
     459              : 
     460              : static inline tree
     461       222415 : maybe_lookup_field (tree var, omp_context *ctx)
     462              : {
     463         2555 :   return maybe_lookup_field ((splay_tree_key) var, ctx);
     464              : }
     465              : 
     466              : /* Return true if DECL should be copied by pointer.  SHARED_CTX is
     467              :    the parallel context if DECL is to be shared.  */
     468              : 
     469              : static bool
     470       234069 : use_pointer_for_field (tree decl, omp_context *shared_ctx)
     471              : {
     472       456378 :   if (AGGREGATE_TYPE_P (TREE_TYPE (decl))
     473       217051 :       || TYPE_ATOMIC (TREE_TYPE (decl))
     474       451120 :       || POLY_INT_CST_P (DECL_SIZE (decl)))
     475              :     return true;
     476              : 
     477              :   /* We can only use copy-in/copy-out semantics for shared variables
     478              :      when we know the value is not accessible from an outer scope.  */
     479       217021 :   if (shared_ctx)
     480              :     {
     481        29703 :       gcc_assert (!is_gimple_omp_oacc (shared_ctx->stmt));
     482              : 
     483              :       /* ??? Trivially accessible from anywhere.  But why would we even
     484              :          be passing an address in this case?  Should we simply assert
     485              :          this to be false, or should we have a cleanup pass that removes
     486              :          these from the list of mappings?  */
     487        29703 :       if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, shared_ctx)))
     488              :         return true;
     489              : 
     490              :       /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
     491              :          without analyzing the expression whether or not its location
     492              :          is accessible to anyone else.  In the case of nested parallel
     493              :          regions it certainly may be.  */
     494        29703 :       if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl))
     495              :         return true;
     496              : 
     497              :       /* Do not use copy-in/copy-out for variables that have their
     498              :          address taken.  */
     499        28789 :       if (is_global_var (decl))
     500              :         {
     501              :           /* For file scope vars, track whether we've seen them as
     502              :              non-addressable initially and in that case, keep the same
     503              :              answer for the duration of the pass, even when they are made
     504              :              addressable later on e.g. through reduction expansion.  Global
     505              :              variables which weren't addressable before the pass will not
     506              :              have their privatized copies address taken.  See PR91216.  */
     507         1235 :           if (!TREE_ADDRESSABLE (decl))
     508              :             {
     509          480 :               if (!global_nonaddressable_vars)
     510           75 :                 global_nonaddressable_vars = BITMAP_ALLOC (NULL);
     511          480 :               bitmap_set_bit (global_nonaddressable_vars, DECL_UID (decl));
     512              :             }
     513          755 :           else if (!global_nonaddressable_vars
     514          990 :                    || !bitmap_bit_p (global_nonaddressable_vars,
     515          235 :                                      DECL_UID (decl)))
     516              :             return true;
     517              :         }
     518        27554 :       else if (TREE_ADDRESSABLE (decl))
     519              :         return true;
     520              : 
     521              :       /* lower_send_shared_vars only uses copy-in, but not copy-out
     522              :          for these.  */
     523        18568 :       if (TREE_READONLY (decl)
     524        18568 :           || ((TREE_CODE (decl) == RESULT_DECL
     525        11909 :                || TREE_CODE (decl) == PARM_DECL)
     526          673 :               && DECL_BY_REFERENCE (decl)))
     527              :         return false;
     528              : 
     529              :       /* Disallow copy-in/out in nested parallel if
     530              :          decl is shared in outer parallel, otherwise
     531              :          each thread could store the shared variable
     532              :          in its own copy-in location, making the
     533              :          variable no longer really shared.  */
     534        11677 :       if (shared_ctx->is_nested)
     535              :         {
     536         2476 :           omp_context *up;
     537              : 
     538         5985 :           for (up = shared_ctx->outer; up; up = up->outer)
     539         5711 :             if ((is_taskreg_ctx (up)
     540         3646 :                  || (gimple_code (up->stmt) == GIMPLE_OMP_TARGET
     541          411 :                      && is_gimple_omp_offloaded (up->stmt)))
     542         6122 :                 && maybe_lookup_decl (decl, up))
     543              :               break;
     544              : 
     545         2476 :           if (up)
     546              :             {
     547         2202 :               tree c;
     548              : 
     549         2202 :               if (gimple_code (up->stmt) == GIMPLE_OMP_TARGET)
     550              :                 {
     551          309 :                   for (c = gimple_omp_target_clauses (up->stmt);
     552         2031 :                        c; c = OMP_CLAUSE_CHAIN (c))
     553         1932 :                     if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
     554         1932 :                         && OMP_CLAUSE_DECL (c) == decl)
     555              :                       break;
     556              :                 }
     557              :               else
     558         1893 :                 for (c = gimple_omp_taskreg_clauses (up->stmt);
     559         6925 :                      c; c = OMP_CLAUSE_CHAIN (c))
     560         6468 :                   if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
     561         6468 :                       && OMP_CLAUSE_DECL (c) == decl)
     562              :                     break;
     563              : 
     564         2202 :               if (c)
     565         1646 :                 goto maybe_mark_addressable_and_ret;
     566              :             }
     567              :         }
     568              : 
     569              :       /* For tasks avoid using copy-in/out.  As tasks can be
     570              :          deferred or executed in different thread, when GOMP_task
     571              :          returns, the task hasn't necessarily terminated.  */
     572        10031 :       if (is_task_ctx (shared_ctx))
     573              :         {
     574         2201 :           tree outer;
     575          555 :         maybe_mark_addressable_and_ret:
     576         2201 :           outer = maybe_lookup_decl_in_outer_ctx (decl, shared_ctx);
     577         2201 :           if (is_gimple_reg (outer) && !omp_member_access_dummy_var (outer))
     578              :             {
     579              :               /* Taking address of OUTER in lower_send_shared_vars
     580              :                  might need regimplification of everything that uses the
     581              :                  variable.  */
     582          986 :               if (!make_addressable_vars)
     583          493 :                 make_addressable_vars = BITMAP_ALLOC (NULL);
     584          986 :               bitmap_set_bit (make_addressable_vars, DECL_UID (outer));
     585          986 :               TREE_ADDRESSABLE (outer) = 1;
     586              :             }
     587              :           return true;
     588              :         }
     589              :     }
     590              : 
     591              :   return false;
     592              : }
     593              : 
     594              : /* Construct a new automatic decl similar to VAR.  */
     595              : 
     596              : static tree
     597       231043 : omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx)
     598              : {
     599       231043 :   tree copy = copy_var_decl (var, name, type);
     600              : 
     601       231043 :   DECL_CONTEXT (copy) = current_function_decl;
     602              : 
     603       231043 :   if (ctx)
     604              :     {
     605       230762 :       DECL_CHAIN (copy) = ctx->block_vars;
     606       230762 :       ctx->block_vars = copy;
     607              :     }
     608              :   else
     609          281 :     record_vars (copy);
     610              : 
     611              :   /* If VAR is listed in make_addressable_vars, it wasn't
     612              :      originally addressable, but was only later made so.
     613              :      We don't need to take address of privatizations
     614              :      from that var.  */
     615       231043 :   if (TREE_ADDRESSABLE (var)
     616       231043 :       && ((make_addressable_vars
     617         3470 :            && bitmap_bit_p (make_addressable_vars, DECL_UID (var)))
     618        40028 :           || (global_nonaddressable_vars
     619          515 :               && bitmap_bit_p (global_nonaddressable_vars, DECL_UID (var)))))
     620         2079 :     TREE_ADDRESSABLE (copy) = 0;
     621              : 
     622       231043 :   return copy;
     623              : }
     624              : 
     625              : static tree
     626       231043 : omp_copy_decl_1 (tree var, omp_context *ctx)
     627              : {
     628       231043 :   return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx);
     629              : }
     630              : 
     631              : /* Build tree nodes to access the field for VAR on the receiver side.  */
     632              : 
     633              : static tree
     634       123620 : build_receiver_ref (tree var, bool by_ref, omp_context *ctx)
     635              : {
     636       123620 :   tree x, field = lookup_field (var, ctx);
     637              : 
     638              :   /* If the receiver record type was remapped in the child function,
     639              :      remap the field into the new record type.  */
     640       123620 :   x = maybe_lookup_field (field, ctx);
     641       123620 :   if (x != NULL)
     642        11295 :     field = x;
     643              : 
     644       123620 :   x = build_simple_mem_ref (ctx->receiver_decl);
     645       123620 :   TREE_THIS_NOTRAP (x) = 1;
     646       123620 :   x = omp_build_component_ref (x, field);
     647       123620 :   if (by_ref)
     648              :     {
     649        36338 :       x = build_simple_mem_ref (x);
     650        36338 :       TREE_THIS_NOTRAP (x) = 1;
     651              :     }
     652              : 
     653       123620 :   return x;
     654              : }
     655              : 
     656              : /* Build tree nodes to access VAR in the scope outer to CTX.  In the case
     657              :    of a parallel, this is a component reference; for workshare constructs
     658              :    this is some variable.  */
     659              : 
     660              : static tree
     661       114907 : build_outer_var_ref (tree var, omp_context *ctx,
     662              :                      enum omp_clause_code code = OMP_CLAUSE_ERROR)
     663              : {
     664       114907 :   tree x;
     665       114907 :   omp_context *outer = ctx->outer;
     666       115296 :   for (; outer; outer = outer->outer)
     667              :     {
     668        82750 :       if (gimple_code (outer->stmt) == GIMPLE_OMP_TASKGROUP)
     669          373 :         continue;
     670        82393 :       if (gimple_code (outer->stmt) == GIMPLE_OMP_SCOPE
     671        82377 :           && !maybe_lookup_decl (var, outer))
     672           16 :         continue;
     673              :       break;
     674              :     }
     675              : 
     676       114907 :   if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx)))
     677              :     x = var;
     678       106762 :   else if (is_variable_sized (var))
     679              :     {
     680           51 :       x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0);
     681           51 :       x = build_outer_var_ref (x, ctx, code);
     682           51 :       x = build_simple_mem_ref (x);
     683              :     }
     684       106711 :   else if (is_taskreg_ctx (ctx))
     685              :     {
     686        64067 :       bool by_ref = use_pointer_for_field (var, NULL);
     687        64067 :       x = build_receiver_ref (var, by_ref, ctx);
     688              :     }
     689        42644 :   else if ((gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
     690        41271 :             && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
     691        14150 :            || ctx->loop_p
     692        13377 :            || code == OMP_CLAUSE_ALLOCATE
     693        55765 :            || (code == OMP_CLAUSE_PRIVATE
     694           40 :                && (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
     695              :                    || gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS
     696              :                    || gimple_code (ctx->stmt) == GIMPLE_OMP_SINGLE)))
     697              :     {
     698              :       /* #pragma omp simd isn't a worksharing construct, and can reference
     699              :          even private vars in its linear etc. clauses.
     700              :          Similarly for OMP_CLAUSE_PRIVATE with outer ref, that can refer
     701              :          to private vars in all worksharing constructs.  */
     702        29543 :       x = NULL_TREE;
     703        29543 :       if (outer && is_taskreg_ctx (outer))
     704         1002 :         x = lookup_decl (var, outer);
     705        28541 :       else if (outer)
     706        22343 :         x = maybe_lookup_decl_in_outer_ctx (var, ctx);
     707        23345 :       if (x == NULL_TREE)
     708              :         x = var;
     709              :     }
     710        13101 :   else if (code == OMP_CLAUSE_LASTPRIVATE && is_taskloop_ctx (ctx))
     711              :     {
     712          689 :       gcc_assert (outer);
     713          689 :       splay_tree_node n
     714         1378 :         = splay_tree_lookup (outer->field_map,
     715          689 :                              (splay_tree_key) &DECL_UID (var));
     716          689 :       if (n == NULL)
     717              :         {
     718          610 :           if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, outer)))
     719              :             x = var;
     720              :           else
     721          384 :             x = lookup_decl (var, outer);
     722              :         }
     723              :       else
     724              :         {
     725           79 :           tree field = (tree) n->value;
     726              :           /* If the receiver record type was remapped in the child function,
     727              :              remap the field into the new record type.  */
     728           79 :           x = maybe_lookup_field (field, outer);
     729           79 :           if (x != NULL)
     730            0 :             field = x;
     731              : 
     732           79 :           x = build_simple_mem_ref (outer->receiver_decl);
     733           79 :           x = omp_build_component_ref (x, field);
     734           79 :           if (use_pointer_for_field (var, outer))
     735           59 :             x = build_simple_mem_ref (x);
     736              :         }
     737              :     }
     738        12412 :   else if (outer)
     739        12220 :     x = lookup_decl (var, outer);
     740          192 :   else if (omp_privatize_by_reference (var))
     741              :     /* This can happen with orphaned constructs.  If var is reference, it is
     742              :        possible it is shared and as such valid.  */
     743              :     x = var;
     744           94 :   else if (omp_member_access_dummy_var (var))
     745              :     x = var;
     746              :   else
     747            0 :     gcc_unreachable ();
     748              : 
     749       114907 :   if (x == var)
     750              :     {
     751        16338 :       tree t = omp_member_access_dummy_var (var);
     752        16338 :       if (t)
     753              :         {
     754          220 :           x = DECL_VALUE_EXPR (var);
     755          220 :           tree o = maybe_lookup_decl_in_outer_ctx (t, ctx);
     756          220 :           if (o != t)
     757           54 :             x = unshare_and_remap (x, t, o);
     758              :           else
     759          166 :             x = unshare_expr (x);
     760              :         }
     761              :     }
     762              : 
     763       114907 :   if (omp_privatize_by_reference (var))
     764         3146 :     x = build_simple_mem_ref (x);
     765              : 
     766       114907 :   return x;
     767              : }
     768              : 
     769              : /* Build tree nodes to access the field for VAR on the sender side.  */
     770              : 
     771              : static tree
     772       169257 : build_sender_ref (splay_tree_key key, omp_context *ctx)
     773              : {
     774       169257 :   tree field = lookup_sfield (key, ctx);
     775       169257 :   tree tmp = ctx->sender_decl;
     776       169257 :   if (POINTER_TYPE_P (TREE_TYPE (tmp)))
     777          451 :     tmp = build_fold_indirect_ref (tmp);
     778       169257 :   return omp_build_component_ref (tmp, field);
     779              : }
     780              : 
     781              : static tree
     782       165103 : build_sender_ref (tree var, omp_context *ctx)
     783              : {
     784            0 :   return build_sender_ref ((splay_tree_key) var, ctx);
     785              : }
     786              : 
     787              : /* Add a new field for VAR inside the structure CTX.  If BY_REF is true,
     788              :    use a pointer to the VAR rather than VAR itself.
     789              :    MASK is a bit mask of other options.  Bits are interpreted as:
     790              :       1: Install VAR in ctx->field_map.
     791              :       2: Install VAR in ctx->sfield_map.
     792              :       4: VAR is an array, convert it to a pointer.
     793              :       8: Use DECL_UID (VAR) instead of VAR as key.
     794              :      16: Use DECL_NAME (VAR) instead of VAR as key.
     795              :      32: Don't dereference omp_is_reference types.
     796              :    KEY_EXPR allows specifying something other than VAR as the lookup key.
     797              :    If specified, it also overrides the 8 and 16 MASK bits.  */
     798              : 
     799              : static void
     800       132132 : install_var_field (tree var, bool by_ref, int mask, omp_context *ctx,
     801              :                    tree key_expr = NULL_TREE)
     802              : {
     803       132132 :   tree field, type, sfield = NULL_TREE;
     804       132132 :   splay_tree_key key = (splay_tree_key) var;
     805              : 
     806       132132 :   if (key_expr)
     807              :     /* Allow caller to explicitly set the expression used as the key.  */
     808        44824 :     key = (splay_tree_key) key_expr;
     809              :   else
     810              :     {
     811        87308 :       if ((mask & 16) != 0)
     812              :         {
     813          624 :           key = (splay_tree_key) &DECL_NAME (var);
     814          624 :           gcc_checking_assert (key != (splay_tree_key) var);
     815              :         }
     816        87308 :       if ((mask & 8) != 0)
     817              :         {
     818         1619 :           key = (splay_tree_key) &DECL_UID (var);
     819         1619 :           gcc_checking_assert (key != (splay_tree_key) var);
     820              :         }
     821              :     }
     822       132132 :   gcc_assert ((mask & 1) == 0
     823              :               || !splay_tree_lookup (ctx->field_map, key));
     824       132132 :   gcc_assert ((mask & 2) == 0 || !ctx->sfield_map
     825              :               || !splay_tree_lookup (ctx->sfield_map, key));
     826       132132 :   gcc_assert ((mask & 3) == 3
     827              :               || !is_gimple_omp_oacc (ctx->stmt));
     828              : 
     829       132132 :   type = TREE_TYPE (var);
     830       132132 :   if ((mask & 16) != 0)
     831          624 :     type = lang_hooks.decls.omp_array_data (var, true);
     832              : 
     833              :   /* Prevent redeclaring the var in the split-off function with a restrict
     834              :      pointer type.  Note that we only clear type itself, restrict qualifiers in
     835              :      the pointed-to type will be ignored by points-to analysis.  */
     836       132132 :   if (POINTER_TYPE_P (type)
     837       132132 :       && TYPE_RESTRICT (type))
     838         7022 :     type = build_qualified_type (type,
     839         7022 :                                  TYPE_QUALS (type)
     840              :                                  .without (TYPE_QUAL_RESTRICT));
     841              : 
     842       132132 :   if (mask & 4)
     843              :     {
     844          425 :       gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
     845          425 :       type = build_pointer_type (build_pointer_type (type));
     846              :     }
     847       131707 :   else if (by_ref)
     848        64492 :     type = build_pointer_type (type);
     849        67215 :   else if ((mask & (32 | 3)) == 1
     850        67215 :            && omp_privatize_by_reference (var))
     851          162 :     type = TREE_TYPE (type);
     852              : 
     853       132132 :   field = build_decl (DECL_SOURCE_LOCATION (var),
     854       132132 :                       FIELD_DECL, DECL_NAME (var), type);
     855              : 
     856              :   /* Remember what variable this field was created for.  This does have a
     857              :      side effect of making dwarf2out ignore this member, so for helpful
     858              :      debugging we clear it later in delete_omp_context.  */
     859       132132 :   DECL_ABSTRACT_ORIGIN (field) = var;
     860       132132 :   if ((mask & 16) == 0 && type == TREE_TYPE (var))
     861              :     {
     862        62701 :       SET_DECL_ALIGN (field, DECL_ALIGN (var));
     863        62701 :       DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
     864        62701 :       TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
     865              :     }
     866              :   else
     867        69431 :     SET_DECL_ALIGN (field, TYPE_ALIGN (type));
     868              : 
     869       132132 :   if ((mask & 3) == 3)
     870              :     {
     871       131068 :       insert_field_into_struct (ctx->record_type, field);
     872       131068 :       if (ctx->srecord_type)
     873              :         {
     874          682 :           sfield = build_decl (DECL_SOURCE_LOCATION (var),
     875          682 :                                FIELD_DECL, DECL_NAME (var), type);
     876          682 :           DECL_ABSTRACT_ORIGIN (sfield) = var;
     877          682 :           SET_DECL_ALIGN (sfield, DECL_ALIGN (field));
     878          682 :           DECL_USER_ALIGN (sfield) = DECL_USER_ALIGN (field);
     879          682 :           TREE_THIS_VOLATILE (sfield) = TREE_THIS_VOLATILE (field);
     880          682 :           insert_field_into_struct (ctx->srecord_type, sfield);
     881              :         }
     882              :     }
     883              :   else
     884              :     {
     885         1064 :       if (ctx->srecord_type == NULL_TREE)
     886              :         {
     887          550 :           tree t;
     888              : 
     889          550 :           ctx->srecord_type = lang_hooks.types.make_type (RECORD_TYPE);
     890          550 :           ctx->sfield_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
     891         2033 :           for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t))
     892              :             {
     893         1483 :               sfield = build_decl (DECL_SOURCE_LOCATION (t),
     894         1483 :                                    FIELD_DECL, DECL_NAME (t), TREE_TYPE (t));
     895         1483 :               DECL_ABSTRACT_ORIGIN (sfield) = DECL_ABSTRACT_ORIGIN (t);
     896         1483 :               insert_field_into_struct (ctx->srecord_type, sfield);
     897         2966 :               splay_tree_insert (ctx->sfield_map,
     898         1483 :                                  (splay_tree_key) DECL_ABSTRACT_ORIGIN (t),
     899              :                                  (splay_tree_value) sfield);
     900              :             }
     901              :         }
     902         1064 :       sfield = field;
     903         1064 :       insert_field_into_struct ((mask & 1) ? ctx->record_type
     904              :                                 : ctx->srecord_type, field);
     905              :     }
     906              : 
     907       132132 :   if (mask & 1)
     908       131739 :     splay_tree_insert (ctx->field_map, key, (splay_tree_value) field);
     909       132132 :   if ((mask & 2) && ctx->sfield_map)
     910         1075 :     splay_tree_insert (ctx->sfield_map, key, (splay_tree_value) sfield);
     911       132132 : }
     912              : 
     913              : static tree
     914       230612 : install_var_local (tree var, omp_context *ctx)
     915              : {
     916       230612 :   tree new_var = omp_copy_decl_1 (var, ctx);
     917       230612 :   insert_decl_map (&ctx->cb, var, new_var);
     918       230612 :   return new_var;
     919              : }
     920              : 
     921              : /* Adjust the replacement for DECL in CTX for the new context.  This means
     922              :    copying the DECL_VALUE_EXPR, and fixing up the type.  */
     923              : 
     924              : static void
     925       205587 : fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug)
     926              : {
     927       205587 :   tree new_decl, size;
     928              : 
     929       205587 :   new_decl = lookup_decl (decl, ctx);
     930              : 
     931       205587 :   TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), &ctx->cb);
     932              : 
     933       410030 :   if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug)
     934       205776 :       && DECL_HAS_VALUE_EXPR_P (decl))
     935              :     {
     936         1333 :       tree ve = DECL_VALUE_EXPR (decl);
     937         1333 :       walk_tree (&ve, copy_tree_body_r, &ctx->cb, NULL);
     938         1333 :       SET_DECL_VALUE_EXPR (new_decl, ve);
     939         1333 :       DECL_HAS_VALUE_EXPR_P (new_decl) = 1;
     940              :     }
     941              : 
     942       205587 :   if (!TREE_CONSTANT (DECL_SIZE (new_decl)))
     943              :     {
     944         1144 :       size = remap_decl (DECL_SIZE (decl), &ctx->cb);
     945         1144 :       if (size == error_mark_node)
     946          321 :         size = TYPE_SIZE (TREE_TYPE (new_decl));
     947         1144 :       DECL_SIZE (new_decl) = size;
     948              : 
     949         1144 :       size = remap_decl (DECL_SIZE_UNIT (decl), &ctx->cb);
     950         1144 :       if (size == error_mark_node)
     951          308 :         size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl));
     952         1144 :       DECL_SIZE_UNIT (new_decl) = size;
     953              :     }
     954       205587 : }
     955              : 
     956              : /* The callback for remap_decl.  Search all containing contexts for a
     957              :    mapping of the variable; this avoids having to duplicate the splay
     958              :    tree ahead of time.  We know a mapping doesn't already exist in the
     959              :    given context.  Create new mappings to implement default semantics.  */
     960              : 
     961              : static tree
     962       466309 : omp_copy_decl (tree var, copy_body_data *cb)
     963              : {
     964       466309 :   omp_context *ctx = (omp_context *) cb;
     965       466309 :   tree new_var;
     966              : 
     967       466309 :   if (TREE_CODE (var) == LABEL_DECL)
     968              :     {
     969       189902 :       if (FORCED_LABEL (var) || DECL_NONLOCAL (var))
     970              :         return var;
     971       189896 :       new_var = create_artificial_label (DECL_SOURCE_LOCATION (var));
     972       189896 :       DECL_CONTEXT (new_var) = current_function_decl;
     973       189896 :       insert_decl_map (&ctx->cb, var, new_var);
     974       189896 :       return new_var;
     975              :     }
     976              : 
     977       358347 :   while (!is_taskreg_ctx (ctx))
     978              :     {
     979       339023 :       ctx = ctx->outer;
     980       339023 :       if (ctx == NULL)
     981              :         return var;
     982       280248 :       new_var = maybe_lookup_decl (var, ctx);
     983       280248 :       if (new_var)
     984              :         return new_var;
     985              :     }
     986              : 
     987        19324 :   if (is_global_var (var) || decl_function_context (var) != ctx->cb.src_fn)
     988              :     return var;
     989              : 
     990         5876 :   return error_mark_node;
     991              : }
     992              : 
     993              : /* Create a new context, with OUTER_CTX being the surrounding context.  */
     994              : 
     995              : static omp_context *
     996       138952 : new_omp_context (gimple *stmt, omp_context *outer_ctx)
     997              : {
     998       138952 :   omp_context *ctx = XCNEW (omp_context);
     999              : 
    1000       138952 :   splay_tree_insert (all_contexts, (splay_tree_key) stmt,
    1001              :                      (splay_tree_value) ctx);
    1002       138952 :   ctx->stmt = stmt;
    1003              : 
    1004       138952 :   if (outer_ctx)
    1005              :     {
    1006        74905 :       ctx->outer = outer_ctx;
    1007        74905 :       ctx->cb = outer_ctx->cb;
    1008        74905 :       ctx->cb.block = NULL;
    1009        74905 :       ctx->depth = outer_ctx->depth + 1;
    1010              :     }
    1011              :   else
    1012              :     {
    1013        64047 :       ctx->cb.src_fn = current_function_decl;
    1014        64047 :       ctx->cb.dst_fn = current_function_decl;
    1015        64047 :       ctx->cb.src_node = cgraph_node::get (current_function_decl);
    1016        64047 :       gcc_checking_assert (ctx->cb.src_node);
    1017        64047 :       ctx->cb.dst_node = ctx->cb.src_node;
    1018        64047 :       ctx->cb.src_cfun = cfun;
    1019        64047 :       ctx->cb.copy_decl = omp_copy_decl;
    1020        64047 :       ctx->cb.eh_lp_nr = 0;
    1021        64047 :       ctx->cb.transform_call_graph_edges = CB_CGE_MOVE;
    1022        64047 :       ctx->cb.adjust_array_error_bounds = true;
    1023        64047 :       ctx->cb.dont_remap_vla_if_no_change = true;
    1024        64047 :       ctx->depth = 1;
    1025              :     }
    1026              : 
    1027       138952 :   ctx->cb.decl_map = new hash_map<tree, tree>;
    1028              : 
    1029       138952 :   return ctx;
    1030              : }
    1031              : 
    1032              : static gimple_seq maybe_catch_exception (gimple_seq);
    1033              : 
    1034              : /* Finalize task copyfn.  */
    1035              : 
    1036              : static void
    1037          513 : finalize_task_copyfn (gomp_task *task_stmt)
    1038              : {
    1039          513 :   struct function *child_cfun;
    1040          513 :   tree child_fn;
    1041          513 :   gimple_seq seq = NULL, new_seq;
    1042          513 :   gbind *bind;
    1043              : 
    1044          513 :   child_fn = gimple_omp_task_copy_fn (task_stmt);
    1045          513 :   if (child_fn == NULL_TREE)
    1046            0 :     return;
    1047              : 
    1048          513 :   child_cfun = DECL_STRUCT_FUNCTION (child_fn);
    1049          513 :   DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
    1050              : 
    1051          513 :   push_cfun (child_cfun);
    1052          513 :   bind = gimplify_body (child_fn, false);
    1053          513 :   gimple_seq_add_stmt (&seq, bind);
    1054          513 :   new_seq = maybe_catch_exception (seq);
    1055          513 :   if (new_seq != seq)
    1056              :     {
    1057          341 :       bind = gimple_build_bind (NULL, new_seq, NULL);
    1058          341 :       seq = NULL;
    1059          341 :       gimple_seq_add_stmt (&seq, bind);
    1060              :     }
    1061          513 :   gimple_set_body (child_fn, seq);
    1062          513 :   pop_cfun ();
    1063              : 
    1064              :   /* Inform the callgraph about the new function.  */
    1065          513 :   cgraph_node *node = cgraph_node::get_create (child_fn);
    1066          513 :   node->parallelized_function = 1;
    1067          513 :   cgraph_node::add_new_function (child_fn, false);
    1068              : }
    1069              : 
    1070              : /* Destroy a omp_context data structures.  Called through the splay tree
    1071              :    value delete callback.  */
    1072              : 
    1073              : static void
    1074       138946 : delete_omp_context (splay_tree_value value)
    1075              : {
    1076       138946 :   omp_context *ctx = (omp_context *) value;
    1077              : 
    1078       277892 :   delete ctx->cb.decl_map;
    1079              : 
    1080       138946 :   if (ctx->field_map)
    1081        69934 :     splay_tree_delete (ctx->field_map);
    1082       138946 :   if (ctx->sfield_map)
    1083          550 :     splay_tree_delete (ctx->sfield_map);
    1084              : 
    1085              :   /* We hijacked DECL_ABSTRACT_ORIGIN earlier.  We need to clear it before
    1086              :      it produces corrupt debug information.  */
    1087       138946 :   if (ctx->record_type)
    1088              :     {
    1089        53880 :       tree t;
    1090       223958 :       for (t = TYPE_FIELDS (ctx->record_type); t ; t = DECL_CHAIN (t))
    1091       170078 :         DECL_ABSTRACT_ORIGIN (t) = NULL;
    1092              :     }
    1093       138946 :   if (ctx->srecord_type)
    1094              :     {
    1095          550 :       tree t;
    1096         3108 :       for (t = TYPE_FIELDS (ctx->srecord_type); t ; t = DECL_CHAIN (t))
    1097         2558 :         DECL_ABSTRACT_ORIGIN (t) = NULL;
    1098              :     }
    1099              : 
    1100       138946 :   if (ctx->task_reduction_map)
    1101              :     {
    1102         1176 :       ctx->task_reductions.release ();
    1103         2352 :       delete ctx->task_reduction_map;
    1104              :     }
    1105              : 
    1106       139219 :   delete ctx->lastprivate_conditional_map;
    1107       140454 :   delete ctx->allocate_map;
    1108              : 
    1109       138946 :   XDELETE (ctx);
    1110       138946 : }
    1111              : 
    1112              : /* Fix up RECEIVER_DECL with a type that has been remapped to the child
    1113              :    context.  */
    1114              : 
    1115              : static void
    1116        37176 : fixup_child_record_type (omp_context *ctx)
    1117              : {
    1118        37176 :   tree f, type = ctx->record_type;
    1119              : 
    1120        37176 :   if (!ctx->receiver_decl)
    1121              :     return;
    1122              :   /* ??? It isn't sufficient to just call remap_type here, because
    1123              :      variably_modified_type_p doesn't work the way we expect for
    1124              :      record types.  Testing each field for whether it needs remapping
    1125              :      and creating a new record by hand works, however.  */
    1126       162843 :   for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
    1127       126958 :     if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
    1128              :       break;
    1129        37176 :   if (f)
    1130              :     {
    1131         1291 :       tree name, new_fields = NULL;
    1132              : 
    1133         1291 :       type = lang_hooks.types.make_type (RECORD_TYPE);
    1134         1291 :       name = DECL_NAME (TYPE_NAME (ctx->record_type));
    1135         1291 :       name = build_decl (DECL_SOURCE_LOCATION (ctx->receiver_decl),
    1136              :                          TYPE_DECL, name, type);
    1137         1291 :       TYPE_NAME (type) = name;
    1138              : 
    1139        13576 :       for (f = TYPE_FIELDS (ctx->record_type); f ; f = DECL_CHAIN (f))
    1140              :         {
    1141        12285 :           tree new_f = copy_node (f);
    1142        12285 :           DECL_CONTEXT (new_f) = type;
    1143        12285 :           TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &ctx->cb);
    1144        12285 :           DECL_CHAIN (new_f) = new_fields;
    1145        12285 :           walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &ctx->cb, NULL);
    1146        12285 :           walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r,
    1147              :                      &ctx->cb, NULL);
    1148        12285 :           walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
    1149              :                      &ctx->cb, NULL);
    1150        12285 :           new_fields = new_f;
    1151              : 
    1152              :           /* Arrange to be able to look up the receiver field
    1153              :              given the sender field.  */
    1154        12285 :           splay_tree_insert (ctx->field_map, (splay_tree_key) f,
    1155              :                              (splay_tree_value) new_f);
    1156              :         }
    1157         1291 :       TYPE_FIELDS (type) = nreverse (new_fields);
    1158         1291 :       layout_type (type);
    1159              :     }
    1160              : 
    1161              :   /* In a target region we never modify any of the pointers in *.omp_data_i,
    1162              :      so attempt to help the optimizers.  */
    1163        37176 :   if (is_gimple_omp_offloaded (ctx->stmt))
    1164        17720 :     type = build_qualified_type (type, TYPE_QUAL_CONST);
    1165              : 
    1166        37176 :   TREE_TYPE (ctx->receiver_decl)
    1167       111528 :     = build_qualified_type (flexible_array_type_p (type)
    1168          107 :                             ? build_pointer_type (type)
    1169        37069 :                             : build_reference_type (type), TYPE_QUAL_RESTRICT);
    1170              : }
    1171              : 
    1172              : /* Instantiate decls as necessary in CTX to satisfy the data sharing
    1173              :    specified by CLAUSES.  */
    1174              : 
    1175              : static void
    1176       131041 : scan_sharing_clauses (tree clauses, omp_context *ctx)
    1177              : {
    1178       131041 :   tree c, decl;
    1179       131041 :   bool scan_array_reductions = false;
    1180       131041 :   bool flex_array_ptr = false;
    1181              : 
    1182       553751 :   for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
    1183       422710 :     if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_ALLOCATE
    1184       422710 :         && (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) == NULL_TREE
    1185              :             /* omp_default_mem_alloc is 1 */
    1186         1297 :             || !integer_onep (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c))
    1187          572 :             || OMP_CLAUSE_ALLOCATE_ALIGN (c) != NULL_TREE))
    1188              :       {
    1189              :         /* The allocate clauses that appear on a target construct or on
    1190              :            constructs in a target region must specify an allocator expression
    1191              :            unless a requires directive with the dynamic_allocators clause
    1192              :            is present in the same compilation unit.  */
    1193         1985 :         if (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) == NULL_TREE
    1194         1216 :             && ((omp_requires_mask & OMP_REQUIRES_DYNAMIC_ALLOCATORS) == 0)
    1195         3189 :             && omp_maybe_offloaded_ctx (ctx))
    1196           14 :           error_at (OMP_CLAUSE_LOCATION (c), "%<allocate%> clause must"
    1197              :                     " specify an allocator here");
    1198         1985 :         if ((omp_requires_mask & OMP_REQUIRES_DYNAMIC_ALLOCATORS) == 0
    1199         1973 :             && OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) != NULL_TREE
    1200          769 :             && DECL_P (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c))
    1201         2494 :             && !DECL_ARTIFICIAL (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c)))
    1202              :           {
    1203          279 :             tree alloc2 = OMP_CLAUSE_ALLOCATE_ALLOCATOR (c);
    1204          279 :             if (TREE_CODE (alloc2) == MEM_REF
    1205          279 :                 || TREE_CODE (alloc2) == INDIRECT_REF)
    1206            0 :               alloc2 = TREE_OPERAND (alloc2, 0);
    1207          279 :             omp_context *ctx2 = ctx;
    1208          740 :             for (; ctx2; ctx2 = ctx2->outer)
    1209          480 :               if (is_gimple_omp_offloaded (ctx2->stmt))
    1210              :                 break;
    1211          279 :             if (ctx2 != NULL)
    1212              :               {
    1213           19 :                 tree c2 = gimple_omp_target_clauses (ctx2->stmt);
    1214          108 :                 for (; c2; c2 = OMP_CLAUSE_CHAIN (c2))
    1215          106 :                   if (OMP_CLAUSE_CODE (c2) == OMP_CLAUSE_USES_ALLOCATORS
    1216          123 :                       && operand_equal_p (
    1217           17 :                            alloc2, OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c2)))
    1218              :                     break;
    1219           19 :                 if (c2 == NULL_TREE)
    1220            4 :                   error_at (EXPR_LOC_OR_LOC (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c),
    1221              :                                              OMP_CLAUSE_LOCATION (c)),
    1222              :                             "allocator %qE in %<allocate%> clause inside a "
    1223              :                             "target region must be specified in an "
    1224              :                             "%<uses_allocators%> clause on the %<target%> "
    1225              :                             "directive", alloc2);
    1226              :               }
    1227              :           }
    1228         1985 :         if (ctx->allocate_map == NULL)
    1229         1508 :           ctx->allocate_map = new hash_map<tree, tree>;
    1230         1985 :         tree val = integer_zero_node;
    1231         1985 :         if (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c))
    1232          769 :           val = OMP_CLAUSE_ALLOCATE_ALLOCATOR (c);
    1233         1985 :         if (OMP_CLAUSE_ALLOCATE_ALIGN (c))
    1234          295 :           val = build_tree_list (val, OMP_CLAUSE_ALLOCATE_ALIGN (c));
    1235         1985 :         ctx->allocate_map->put (OMP_CLAUSE_DECL (c), val);
    1236              :       }
    1237              : 
    1238       553751 :   for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
    1239              :     {
    1240       422710 :       bool by_ref;
    1241              : 
    1242       422710 :       switch (OMP_CLAUSE_CODE (c))
    1243              :         {
    1244        73503 :         case OMP_CLAUSE_PRIVATE:
    1245        73503 :           decl = OMP_CLAUSE_DECL (c);
    1246        73503 :           if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
    1247          165 :             goto do_private;
    1248        73338 :           else if (!is_variable_sized (decl))
    1249        72974 :             install_var_local (decl, ctx);
    1250              :           break;
    1251              : 
    1252        46994 :         case OMP_CLAUSE_SHARED:
    1253        46994 :           decl = OMP_CLAUSE_DECL (c);
    1254        46994 :           if (ctx->allocate_map && ctx->allocate_map->get (decl))
    1255           37 :             ctx->allocate_map->remove (decl);
    1256              :           /* Ignore shared directives in teams construct inside of
    1257              :              target construct.  */
    1258        46994 :           if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
    1259        46994 :               && !is_host_teams_ctx (ctx))
    1260              :             {
    1261              :               /* Global variables don't need to be copied,
    1262              :                  the receiver side will use them directly.  */
    1263        11864 :               tree odecl = maybe_lookup_decl_in_outer_ctx (decl, ctx);
    1264        11864 :               if (is_global_var (odecl))
    1265              :                 break;
    1266         8135 :               insert_decl_map (&ctx->cb, decl, odecl);
    1267         8135 :               break;
    1268              :             }
    1269        35130 :           gcc_assert (is_taskreg_ctx (ctx));
    1270        35130 :           gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl))
    1271              :                       || !is_variable_sized (decl));
    1272              :           /* Global variables don't need to be copied,
    1273              :              the receiver side will use them directly.  */
    1274        35130 :           if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
    1275              :             break;
    1276        29052 :           if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
    1277              :             {
    1278           86 :               use_pointer_for_field (decl, ctx);
    1279           86 :               break;
    1280              :             }
    1281        28966 :           by_ref = use_pointer_for_field (decl, NULL);
    1282        55433 :           if ((! TREE_READONLY (decl) && !OMP_CLAUSE_SHARED_READONLY (c))
    1283        19367 :               || TREE_ADDRESSABLE (decl)
    1284        19010 :               || by_ref
    1285        47976 :               || omp_privatize_by_reference (decl))
    1286              :             {
    1287        12101 :               by_ref = use_pointer_for_field (decl, ctx);
    1288        12101 :               install_var_field (decl, by_ref, 3, ctx);
    1289        12101 :               install_var_local (decl, ctx);
    1290        12101 :               break;
    1291              :             }
    1292              :           /* We don't need to copy const scalar vars back.  */
    1293        16865 :           OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE);
    1294        16865 :           goto do_private;
    1295              : 
    1296        14692 :         case OMP_CLAUSE_REDUCTION:
    1297              :           /* Collect 'reduction' clauses on OpenACC compute construct.  */
    1298        14692 :           if (is_gimple_omp_oacc (ctx->stmt)
    1299        14692 :               && is_gimple_omp_offloaded (ctx->stmt))
    1300              :             {
    1301              :               /* No 'reduction' clauses on OpenACC 'kernels'.  */
    1302          790 :               gcc_checking_assert (!is_oacc_kernels (ctx));
    1303              :               /* Likewise, on OpenACC 'kernels' decomposed parts.  */
    1304          790 :               gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
    1305              : 
    1306          790 :               ctx->local_reduction_clauses
    1307          790 :                 = tree_cons (NULL, c, ctx->local_reduction_clauses);
    1308              :             }
    1309              :           /* FALLTHRU */
    1310              : 
    1311        16791 :         case OMP_CLAUSE_IN_REDUCTION:
    1312        16791 :           decl = OMP_CLAUSE_DECL (c);
    1313        16791 :           if (ctx->allocate_map
    1314        16791 :               && ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
    1315          473 :                    && (OMP_CLAUSE_REDUCTION_INSCAN (c)
    1316          473 :                        || OMP_CLAUSE_REDUCTION_TASK (c)))
    1317          481 :                   || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
    1318          473 :                   || is_task_ctx (ctx)))
    1319              :             {
    1320              :               /* For now.  */
    1321           17 :               if (ctx->allocate_map->get (decl))
    1322           17 :                 ctx->allocate_map->remove (decl);
    1323              :             }
    1324        16791 :           if (TREE_CODE (decl) == MEM_REF)
    1325              :             {
    1326         2441 :               tree t = TREE_OPERAND (decl, 0);
    1327         2441 :               if (TREE_CODE (t) == POINTER_PLUS_EXPR)
    1328          436 :                 t = TREE_OPERAND (t, 0);
    1329         2441 :               if (INDIRECT_REF_P (t)
    1330         2329 :                   || TREE_CODE (t) == ADDR_EXPR)
    1331         1414 :                 t = TREE_OPERAND (t, 0);
    1332         2441 :               if (is_omp_target (ctx->stmt))
    1333              :                 {
    1334           96 :                   if (is_variable_sized (t))
    1335              :                     {
    1336           32 :                       gcc_assert (DECL_HAS_VALUE_EXPR_P (t));
    1337           32 :                       t = DECL_VALUE_EXPR (t);
    1338           32 :                       gcc_assert (INDIRECT_REF_P (t));
    1339           32 :                       t = TREE_OPERAND (t, 0);
    1340           32 :                       gcc_assert (DECL_P (t));
    1341              :                     }
    1342           96 :                   tree at = t;
    1343           96 :                   if (ctx->outer)
    1344           96 :                     scan_omp_op (&at, ctx->outer);
    1345           96 :                   tree nt = omp_copy_decl_1 (at, ctx->outer);
    1346          192 :                   splay_tree_insert (ctx->field_map,
    1347           96 :                                      (splay_tree_key) &DECL_CONTEXT (t),
    1348              :                                      (splay_tree_value) nt);
    1349           96 :                   if (at != t)
    1350           96 :                     splay_tree_insert (ctx->field_map,
    1351           48 :                                        (splay_tree_key) &DECL_CONTEXT (at),
    1352              :                                        (splay_tree_value) nt);
    1353           96 :                   break;
    1354              :                 }
    1355         2345 :               install_var_local (t, ctx);
    1356         2345 :               if (is_taskreg_ctx (ctx)
    1357         1897 :                   && (!is_global_var (maybe_lookup_decl_in_outer_ctx (t, ctx))
    1358          387 :                       || (is_task_ctx (ctx)
    1359          310 :                           && (TREE_CODE (TREE_TYPE (t)) == POINTER_TYPE
    1360          234 :                               || (TREE_CODE (TREE_TYPE (t)) == REFERENCE_TYPE
    1361            0 :                                   && (TREE_CODE (TREE_TYPE (TREE_TYPE (t)))
    1362              :                                       == POINTER_TYPE)))))
    1363         1586 :                   && !is_variable_sized (t)
    1364         3872 :                   && (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
    1365          692 :                       || (!OMP_CLAUSE_REDUCTION_TASK (c)
    1366          653 :                           && !is_task_ctx (ctx))))
    1367              :                 {
    1368         1410 :                   by_ref = use_pointer_for_field (t, NULL);
    1369         1410 :                   if (is_task_ctx (ctx)
    1370          835 :                       && TREE_CODE (TREE_TYPE (t)) == REFERENCE_TYPE
    1371         1570 :                       && TREE_CODE (TREE_TYPE (TREE_TYPE (t))) == POINTER_TYPE)
    1372              :                     {
    1373           32 :                       install_var_field (t, false, 1, ctx);
    1374           32 :                       install_var_field (t, by_ref, 2, ctx);
    1375              :                     }
    1376              :                   else
    1377         1378 :                     install_var_field (t, by_ref, 3, ctx);
    1378              :                 }
    1379              :               break;
    1380              :             }
    1381        14350 :           if (is_omp_target (ctx->stmt))
    1382              :             {
    1383          335 :               tree at = decl;
    1384          335 :               if (ctx->outer)
    1385           54 :                 scan_omp_op (&at, ctx->outer);
    1386          335 :               tree nt = omp_copy_decl_1 (at, ctx->outer);
    1387          670 :               splay_tree_insert (ctx->field_map,
    1388          335 :                                  (splay_tree_key) &DECL_CONTEXT (decl),
    1389              :                                  (splay_tree_value) nt);
    1390          335 :               if (at != decl)
    1391           24 :                 splay_tree_insert (ctx->field_map,
    1392           12 :                                    (splay_tree_key) &DECL_CONTEXT (at),
    1393              :                                    (splay_tree_value) nt);
    1394          335 :               break;
    1395              :             }
    1396        14015 :           if (is_task_ctx (ctx)
    1397        26935 :               || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
    1398        12920 :                   && OMP_CLAUSE_REDUCTION_TASK (c)
    1399          390 :                   && is_parallel_ctx (ctx)))
    1400              :             {
    1401              :               /* Global variables don't need to be copied,
    1402              :                  the receiver side will use them directly.  */
    1403         1183 :               if (!is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
    1404              :                 {
    1405          487 :                   by_ref = use_pointer_for_field (decl, ctx);
    1406          487 :                   if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION)
    1407          325 :                     install_var_field (decl, by_ref, 3, ctx);
    1408              :                 }
    1409         1183 :               install_var_local (decl, ctx);
    1410         1183 :               break;
    1411              :             }
    1412        12832 :           if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
    1413        12832 :               && OMP_CLAUSE_REDUCTION_TASK (c))
    1414              :             {
    1415          302 :               install_var_local (decl, ctx);
    1416          302 :               break;
    1417              :             }
    1418        12530 :           goto do_private;
    1419              : 
    1420        22774 :         case OMP_CLAUSE_LASTPRIVATE:
    1421              :           /* Let the corresponding firstprivate clause create
    1422              :              the variable.  */
    1423        22774 :           if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
    1424              :             break;
    1425              :           /* FALLTHRU */
    1426              : 
    1427        59809 :         case OMP_CLAUSE_FIRSTPRIVATE:
    1428        59809 :         case OMP_CLAUSE_LINEAR:
    1429        59809 :           decl = OMP_CLAUSE_DECL (c);
    1430        89758 :         do_private:
    1431        89758 :           if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
    1432              :                || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR
    1433              :                || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
    1434        46133 :               && is_gimple_omp_offloaded (ctx->stmt))
    1435              :             {
    1436        16458 :               if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
    1437        16458 :                   || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR
    1438          240 :                       && lang_hooks.decls.omp_array_data (decl, true)))
    1439              :                 {
    1440              :                   /* OpenACC firstprivate clauses are later processed with same
    1441              :                      code path as map clauses in lower_omp_target, so follow
    1442              :                      the same convention of using the whole clause expression
    1443              :                      as splay-tree key.  */
    1444        16088 :                   tree k = (is_oacc_parallel_or_serial (ctx) ? c : NULL_TREE);
    1445        16088 :                   by_ref = !omp_privatize_by_reference (decl);
    1446        16088 :                   install_var_field (decl, by_ref, 3, ctx, k);
    1447              :                 }
    1448          370 :               else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
    1449              :                 {
    1450          221 :                   if (INDIRECT_REF_P (decl))
    1451            0 :                     decl = TREE_OPERAND (decl, 0);
    1452          221 :                   install_var_field (decl, true, 3, ctx);
    1453              :                 }
    1454          149 :               else if (TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
    1455            8 :                   install_var_field (decl, true, 3, ctx);
    1456              :               else
    1457          141 :                 install_var_field (decl, false, 3, ctx);
    1458              :             }
    1459        89758 :           if (is_variable_sized (decl))
    1460              :             {
    1461           68 :               if (is_task_ctx (ctx))
    1462              :                 {
    1463           14 :                   if (ctx->allocate_map
    1464           14 :                       && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
    1465              :                     {
    1466              :                       /* For now.  */
    1467            0 :                       if (ctx->allocate_map->get (decl))
    1468            0 :                         ctx->allocate_map->remove (decl);
    1469              :                     }
    1470           14 :                   install_var_field (decl, false, 1, ctx);
    1471              :                 }
    1472              :               break;
    1473              :             }
    1474        89690 :           else if (is_taskreg_ctx (ctx))
    1475              :             {
    1476        38545 :               bool global
    1477        38545 :                 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx));
    1478        38545 :               by_ref = use_pointer_for_field (decl, NULL);
    1479              : 
    1480        38545 :               if (is_task_ctx (ctx)
    1481        38545 :                   && (global || by_ref || omp_privatize_by_reference (decl)))
    1482              :                 {
    1483          625 :                   if (ctx->allocate_map
    1484          625 :                       && ctx->allocate_map->get (decl))
    1485           59 :                     install_var_field (decl, by_ref, 32 | 1, ctx);
    1486              :                   else
    1487          566 :                     install_var_field (decl, false, 1, ctx);
    1488          625 :                   if (!global)
    1489          361 :                     install_var_field (decl, by_ref, 2, ctx);
    1490              :                 }
    1491        37920 :               else if (!global)
    1492        36647 :                 install_var_field (decl, by_ref, 3, ctx);
    1493              :             }
    1494        89690 :           install_var_local (decl, ctx);
    1495              :           /* For descr arrays on target: firstprivatize data + attach ptr.  */
    1496        89690 :           if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
    1497        45689 :               && is_gimple_omp_offloaded (ctx->stmt)
    1498        16059 :               && !is_gimple_omp_oacc (ctx->stmt)
    1499        11811 :               && lang_hooks.decls.omp_array_data (decl, true)
    1500        89721 :               && lang_hooks.decls.omp_array_data_privatize (decl))
    1501              :             {
    1502           23 :               install_var_field (decl, false, 16 | 3, ctx);
    1503           23 :               install_var_field (decl, true, 8 | 3, ctx);
    1504              :             }
    1505              :           break;
    1506              : 
    1507         2118 :         case OMP_CLAUSE_USE_DEVICE_PTR:
    1508         2118 :         case OMP_CLAUSE_USE_DEVICE_ADDR:
    1509         2118 :           decl = OMP_CLAUSE_DECL (c);
    1510              : 
    1511              :           /* Fortran array descriptors.  */
    1512         2118 :           if (lang_hooks.decls.omp_array_data (decl, true))
    1513          601 :             install_var_field (decl, false, 19, ctx);
    1514         1517 :           else if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
    1515         1312 :                     && !omp_privatize_by_reference (decl)
    1516          513 :                     && !omp_is_allocatable_or_ptr (decl))
    1517         2454 :                    || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
    1518          407 :             install_var_field (decl, true, 11, ctx);
    1519              :           else
    1520         1110 :             install_var_field (decl, false, 11, ctx);
    1521         2118 :           if (DECL_SIZE (decl)
    1522         2118 :               && !poly_int_tree_p (DECL_SIZE (decl)))
    1523              :             {
    1524            6 :               tree decl2 = DECL_VALUE_EXPR (decl);
    1525            6 :               gcc_assert (INDIRECT_REF_P (decl2));
    1526            6 :               decl2 = TREE_OPERAND (decl2, 0);
    1527            6 :               gcc_assert (DECL_P (decl2));
    1528            6 :               install_var_local (decl2, ctx);
    1529              :             }
    1530         2118 :           install_var_local (decl, ctx);
    1531         2118 :           break;
    1532              : 
    1533          240 :         case OMP_CLAUSE_HAS_DEVICE_ADDR:
    1534          240 :           decl = OMP_CLAUSE_DECL (c);
    1535          240 :           while (INDIRECT_REF_P (decl)
    1536          257 :                  || TREE_CODE (decl) == ARRAY_REF)
    1537           17 :             decl = TREE_OPERAND (decl, 0);
    1538          240 :           goto do_private;
    1539              : 
    1540          149 :         case OMP_CLAUSE_IS_DEVICE_PTR:
    1541          149 :           decl = OMP_CLAUSE_DECL (c);
    1542          149 :           goto do_private;
    1543              : 
    1544        20211 :         case OMP_CLAUSE__LOOPTEMP_:
    1545        20211 :         case OMP_CLAUSE__REDUCTEMP_:
    1546        20211 :           gcc_assert (is_taskreg_ctx (ctx));
    1547        20211 :           decl = OMP_CLAUSE_DECL (c);
    1548        20211 :           install_var_field (decl, false, 3, ctx);
    1549        20211 :           install_var_local (decl, ctx);
    1550        20211 :           break;
    1551              : 
    1552          899 :         case OMP_CLAUSE_COPYPRIVATE:
    1553          899 :         case OMP_CLAUSE_COPYIN:
    1554          899 :           decl = OMP_CLAUSE_DECL (c);
    1555          899 :           by_ref = use_pointer_for_field (decl, NULL);
    1556          899 :           install_var_field (decl, by_ref, 3, ctx);
    1557          899 :           break;
    1558              : 
    1559        55715 :         case OMP_CLAUSE_FINAL:
    1560        55715 :         case OMP_CLAUSE_IF:
    1561        55715 :         case OMP_CLAUSE_SELF:
    1562        55715 :         case OMP_CLAUSE_NUM_THREADS:
    1563        55715 :         case OMP_CLAUSE_NUM_TEAMS:
    1564        55715 :         case OMP_CLAUSE_THREAD_LIMIT:
    1565        55715 :         case OMP_CLAUSE_DEVICE:
    1566        55715 :         case OMP_CLAUSE_SCHEDULE:
    1567        55715 :         case OMP_CLAUSE_DIST_SCHEDULE:
    1568        55715 :         case OMP_CLAUSE_DEPEND:
    1569        55715 :         case OMP_CLAUSE_PRIORITY:
    1570        55715 :         case OMP_CLAUSE_GRAINSIZE:
    1571        55715 :         case OMP_CLAUSE_NUM_TASKS:
    1572        55715 :         case OMP_CLAUSE_NUM_GANGS:
    1573        55715 :         case OMP_CLAUSE_NUM_WORKERS:
    1574        55715 :         case OMP_CLAUSE_VECTOR_LENGTH:
    1575        55715 :         case OMP_CLAUSE_DETACH:
    1576        55715 :         case OMP_CLAUSE_FILTER:
    1577        55715 :           if (ctx->outer)
    1578        17667 :             scan_omp_op (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer);
    1579              :           break;
    1580              : 
    1581        88546 :         case OMP_CLAUSE_TO:
    1582        88546 :         case OMP_CLAUSE_FROM:
    1583        88546 :         case OMP_CLAUSE_MAP:
    1584        88546 :           if (ctx->outer)
    1585        15339 :             scan_omp_op (&OMP_CLAUSE_SIZE (c), ctx->outer);
    1586        88546 :           decl = OMP_CLAUSE_DECL (c);
    1587              :           /* If requested, make 'decl' addressable.  */
    1588        88546 :           if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
    1589        88546 :               && OMP_CLAUSE_MAP_DECL_MAKE_ADDRESSABLE (c))
    1590              :             {
    1591          433 :               gcc_checking_assert (DECL_P (decl));
    1592              : 
    1593          433 :               bool decl_addressable = TREE_ADDRESSABLE (decl);
    1594          433 :               if (!decl_addressable)
    1595              :                 {
    1596          219 :                   if (!make_addressable_vars)
    1597          131 :                     make_addressable_vars = BITMAP_ALLOC (NULL);
    1598          219 :                   bitmap_set_bit (make_addressable_vars, DECL_UID (decl));
    1599          219 :                   TREE_ADDRESSABLE (decl) = 1;
    1600              :                 }
    1601              : 
    1602          433 :               if (dump_enabled_p ())
    1603              :                 {
    1604          421 :                   location_t loc = OMP_CLAUSE_LOCATION (c);
    1605          421 :                   const dump_user_location_t d_u_loc
    1606          421 :                     = dump_user_location_t::from_location_t (loc);
    1607              :                   /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */
    1608              : #if __GNUC__ >= 10
    1609          421 : # pragma GCC diagnostic push
    1610          421 : # pragma GCC diagnostic ignored "-Wformat"
    1611              : #endif
    1612          421 :                   if (!decl_addressable)
    1613          207 :                     dump_printf_loc (MSG_NOTE, d_u_loc,
    1614              :                                      "variable %<%T%>"
    1615              :                                      " made addressable\n",
    1616              :                                      decl);
    1617              :                   else
    1618          214 :                     dump_printf_loc (MSG_NOTE, d_u_loc,
    1619              :                                      "variable %<%T%>"
    1620              :                                      " already made addressable\n",
    1621              :                                      decl);
    1622              : #if __GNUC__ >= 10
    1623          421 : # pragma GCC diagnostic pop
    1624              : #endif
    1625              :                 }
    1626              : 
    1627              :               /* Done.  */
    1628          433 :               OMP_CLAUSE_MAP_DECL_MAKE_ADDRESSABLE (c) = 0;
    1629              :             }
    1630              :           /* Global variables with "omp declare target" attribute
    1631              :              don't need to be copied, the receiver side will use them
    1632              :              directly.  However, global variables with "omp declare target link"
    1633              :              attribute need to be copied.  Or when ALWAYS modifier is used.  */
    1634        88546 :           if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
    1635        80730 :               && DECL_P (decl)
    1636        44429 :               && ((OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER
    1637        40604 :                    && (OMP_CLAUSE_MAP_KIND (c)
    1638              :                        != GOMP_MAP_FIRSTPRIVATE_REFERENCE)
    1639        40175 :                    && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ATTACH
    1640        39937 :                    && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_DETACH)
    1641         4557 :                   || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
    1642        41077 :               && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_TO
    1643        40940 :               && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_FROM
    1644        40907 :               && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_TOFROM
    1645        40415 :               && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_TO
    1646        40385 :               && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_FROM
    1647        40363 :               && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_PRESENT_TOFROM
    1648        40349 :               && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_TO_PSET
    1649        36358 :               && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
    1650         7582 :               && varpool_node::get_create (decl)->offloadable
    1651        93834 :               && !lookup_attribute ("omp declare target link",
    1652         5288 :                                     DECL_ATTRIBUTES (decl)))
    1653              :             break;
    1654        83299 :           if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
    1655        83299 :               && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER)
    1656              :             {
    1657              :               /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
    1658              :                  not offloaded; there is nothing to map for those.  */
    1659        10478 :               if (!is_gimple_omp_offloaded (ctx->stmt)
    1660         4979 :                   && !POINTER_TYPE_P (TREE_TYPE (decl))
    1661        10742 :                   && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
    1662              :                 break;
    1663              :             }
    1664        83035 :           if (!flex_array_ptr)
    1665        82807 :             flex_array_ptr = lang_hooks.decls.omp_deep_mapping_p (ctx->stmt, c);
    1666        83035 :           if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
    1667        75219 :               && DECL_P (decl)
    1668        38918 :               && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
    1669        38680 :                   || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
    1670        83338 :               && is_omp_target (ctx->stmt))
    1671              :             {
    1672              :               /* If this is an offloaded region, an attach operation should
    1673              :                  only exist when the pointer variable is mapped in a prior
    1674              :                  clause.  An exception is if we have a reference (to pointer):
    1675              :                  in that case we should have mapped "*decl" in a previous
    1676              :                  mapping instead of "decl".  Skip the assertion in that case.
    1677              :                  If we had an error, we may not have attempted to sort clauses
    1678              :                  properly, so avoid the test.  */
    1679          265 :               if (TREE_CODE (TREE_TYPE (decl)) != REFERENCE_TYPE
    1680          233 :                   && is_gimple_omp_offloaded (ctx->stmt)
    1681          335 :                   && !seen_error ())
    1682           62 :                 gcc_assert
    1683              :                   (maybe_lookup_decl (decl, ctx)
    1684              :                    || (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
    1685              :                        && lookup_attribute ("omp declare target",
    1686              :                                             DECL_ATTRIBUTES (decl))));
    1687              : 
    1688              :               /* By itself, attach/detach is generated as part of pointer
    1689              :                  variable mapping and should not create new variables in the
    1690              :                  offloaded region, however sender refs for it must be created
    1691              :                  for its address to be passed to the runtime.  */
    1692          265 :               tree field
    1693          265 :                 = build_decl (OMP_CLAUSE_LOCATION (c),
    1694              :                               FIELD_DECL, NULL_TREE, ptr_type_node);
    1695          265 :               SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
    1696          265 :               insert_field_into_struct (ctx->record_type, field);
    1697              :               /* To not clash with a map of the pointer variable itself,
    1698              :                  attach/detach maps have their field looked up by the *clause*
    1699              :                  tree expression, not the decl.  */
    1700          265 :               gcc_assert (!splay_tree_lookup (ctx->field_map,
    1701              :                                               (splay_tree_key) c));
    1702          265 :               splay_tree_insert (ctx->field_map, (splay_tree_key) c,
    1703              :                                  (splay_tree_value) field);
    1704          265 :               break;
    1705              :             }
    1706        82770 :           if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
    1707        82770 :               && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
    1708        71131 :                   || (OMP_CLAUSE_MAP_KIND (c)
    1709              :                       == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
    1710              :             {
    1711         4252 :               if (TREE_CODE (decl) == COMPONENT_REF
    1712         4252 :                   || (INDIRECT_REF_P (decl)
    1713            0 :                       && TREE_CODE (TREE_OPERAND (decl, 0)) == COMPONENT_REF
    1714            0 :                       && (((TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0)))
    1715              :                             == REFERENCE_TYPE)
    1716            0 :                            || (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0)))
    1717              :                                == POINTER_TYPE)))))
    1718              :                 break;
    1719         4252 :               if (DECL_SIZE (decl)
    1720         4252 :                   && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
    1721              :                 {
    1722          712 :                   tree decl2 = DECL_VALUE_EXPR (decl);
    1723          712 :                   gcc_assert (INDIRECT_REF_P (decl2));
    1724          712 :                   decl2 = TREE_OPERAND (decl2, 0);
    1725          712 :                   gcc_assert (DECL_P (decl2));
    1726          712 :                   install_var_local (decl2, ctx);
    1727              :                 }
    1728         4252 :               install_var_local (decl, ctx);
    1729         4252 :               break;
    1730              :             }
    1731        78518 :           if (DECL_P (decl))
    1732              :             {
    1733        40576 :               if (DECL_SIZE (decl)
    1734        40576 :                   && !poly_int_tree_p (DECL_SIZE (decl)))
    1735              :                 {
    1736            0 :                   tree decl2 = DECL_VALUE_EXPR (decl);
    1737            0 :                   gcc_assert (INDIRECT_REF_P (decl2));
    1738            0 :                   decl2 = TREE_OPERAND (decl2, 0);
    1739            0 :                   gcc_assert (DECL_P (decl2));
    1740            0 :                   install_var_field (decl2, true, 3, ctx, c);
    1741            0 :                   install_var_local (decl2, ctx);
    1742            0 :                   install_var_local (decl, ctx);
    1743              :                 }
    1744              :               else
    1745              :                 {
    1746        40576 :                   if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
    1747        34401 :                       && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
    1748         5193 :                       && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
    1749        45769 :                       && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
    1750          425 :                     install_var_field (decl, true, 7, ctx, c);
    1751              :                   else
    1752        40151 :                     install_var_field (decl, true, 3, ctx, c);
    1753        40576 :                   if (is_gimple_omp_offloaded (ctx->stmt)
    1754        40576 :                       && !(is_gimple_omp_oacc (ctx->stmt)
    1755        13857 :                            && OMP_CLAUSE_MAP_IN_REDUCTION (c)))
    1756        23829 :                     install_var_local (decl, ctx);
    1757              :                 }
    1758              :             }
    1759              :           else
    1760              :             {
    1761        37942 :               tree base = get_base_address (decl);
    1762        37942 :               tree nc = OMP_CLAUSE_CHAIN (c);
    1763        37942 :               if (DECL_P (base)
    1764         9358 :                   && nc != NULL_TREE
    1765         6737 :                   && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP
    1766         5573 :                   && OMP_CLAUSE_DECL (nc) == base
    1767         1184 :                   && OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_POINTER
    1768        38610 :                   && integer_zerop (OMP_CLAUSE_SIZE (nc)))
    1769              :                 {
    1770            0 :                   OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) = 1;
    1771            0 :                   OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc) = 1;
    1772              :                 }
    1773              :               else
    1774              :                 {
    1775        37942 :                   if (ctx->outer)
    1776              :                     {
    1777         7152 :                       scan_omp_op (&OMP_CLAUSE_DECL (c), ctx->outer);
    1778         7152 :                       decl = OMP_CLAUSE_DECL (c);
    1779              :                     }
    1780        37942 :                   gcc_assert (!splay_tree_lookup (ctx->field_map,
    1781              :                                                   (splay_tree_key) decl));
    1782        37942 :                   tree field
    1783        37942 :                     = build_decl (OMP_CLAUSE_LOCATION (c),
    1784              :                                   FIELD_DECL, NULL_TREE, ptr_type_node);
    1785        37942 :                   SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
    1786        37942 :                   insert_field_into_struct (ctx->record_type, field);
    1787        37942 :                   splay_tree_insert (ctx->field_map, (splay_tree_key) c,
    1788              :                                      (splay_tree_value) field);
    1789              :                 }
    1790              :             }
    1791              :           break;
    1792              : 
    1793         3988 :         case OMP_CLAUSE_ORDER:
    1794         3988 :           ctx->order_concurrent = true;
    1795         3988 :           break;
    1796              : 
    1797         1922 :         case OMP_CLAUSE_BIND:
    1798         1922 :           ctx->loop_p = true;
    1799         1922 :           break;
    1800              : 
    1801              :         case OMP_CLAUSE_NOWAIT:
    1802              :         case OMP_CLAUSE_ORDERED:
    1803              :         case OMP_CLAUSE_COLLAPSE:
    1804              :         case OMP_CLAUSE_UNTIED:
    1805              :         case OMP_CLAUSE_MERGEABLE:
    1806              :         case OMP_CLAUSE_PROC_BIND:
    1807              :         case OMP_CLAUSE_SAFELEN:
    1808              :         case OMP_CLAUSE_SIMDLEN:
    1809              :         case OMP_CLAUSE_THREADS:
    1810              :         case OMP_CLAUSE_SIMD:
    1811              :         case OMP_CLAUSE_NOGROUP:
    1812              :         case OMP_CLAUSE_DEFAULTMAP:
    1813              :         case OMP_CLAUSE_ASYNC:
    1814              :         case OMP_CLAUSE_WAIT:
    1815              :         case OMP_CLAUSE_GANG:
    1816              :         case OMP_CLAUSE_WORKER:
    1817              :         case OMP_CLAUSE_VECTOR:
    1818              :         case OMP_CLAUSE_INDEPENDENT:
    1819              :         case OMP_CLAUSE_AUTO:
    1820              :         case OMP_CLAUSE_SEQ:
    1821              :         case OMP_CLAUSE_TILE:
    1822              :         case OMP_CLAUSE__SIMT_:
    1823              :         case OMP_CLAUSE_DEFAULT:
    1824              :         case OMP_CLAUSE_NONTEMPORAL:
    1825              :         case OMP_CLAUSE_IF_PRESENT:
    1826              :         case OMP_CLAUSE_FINALIZE:
    1827              :         case OMP_CLAUSE_TASK_REDUCTION:
    1828              :         case OMP_CLAUSE_ALLOCATE:
    1829              :         case OMP_CLAUSE_DEVICE_TYPE:
    1830              :           break;
    1831              : 
    1832          112 :         case OMP_CLAUSE_USES_ALLOCATORS:
    1833          112 :           decl = OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c);
    1834          112 :           gcc_assert (DECL_P (decl));
    1835          112 :           gcc_assert (is_gimple_omp_offloaded (ctx->stmt));
    1836          112 :           install_var_field (decl, false, 3, ctx);
    1837          112 :           install_var_local (decl, ctx);
    1838          112 :           break;
    1839              : 
    1840          162 :         case OMP_CLAUSE_ALIGNED:
    1841          162 :           decl = OMP_CLAUSE_DECL (c);
    1842          162 :           if (is_global_var (decl)
    1843          162 :               && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
    1844           86 :             install_var_local (decl, ctx);
    1845              :           break;
    1846              : 
    1847          361 :         case OMP_CLAUSE__CONDTEMP_:
    1848          361 :           decl = OMP_CLAUSE_DECL (c);
    1849          361 :           if (is_parallel_ctx (ctx))
    1850              :             {
    1851          118 :               install_var_field (decl, false, 3, ctx);
    1852          118 :               install_var_local (decl, ctx);
    1853              :             }
    1854          243 :           else if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
    1855          243 :                    && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
    1856          368 :                    && !OMP_CLAUSE__CONDTEMP__ITER (c))
    1857          125 :             install_var_local (decl, ctx);
    1858              :           break;
    1859              : 
    1860              :         case OMP_CLAUSE_INIT:
    1861              :         case OMP_CLAUSE_USE:
    1862              :         case OMP_CLAUSE_DESTROY:
    1863              :         case OMP_CLAUSE_MESSAGE:
    1864              :           break;
    1865              : 
    1866            0 :         case OMP_CLAUSE__CACHE_:
    1867            0 :         case OMP_CLAUSE_NOHOST:
    1868            0 :         default:
    1869            0 :           gcc_unreachable ();
    1870              :         }
    1871              :     }
    1872              : 
    1873       553751 :   for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
    1874              :     {
    1875       422710 :       switch (OMP_CLAUSE_CODE (c))
    1876              :         {
    1877        22774 :         case OMP_CLAUSE_LASTPRIVATE:
    1878              :           /* Let the corresponding firstprivate clause create
    1879              :              the variable.  */
    1880        22774 :           if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
    1881         7245 :             scan_array_reductions = true;
    1882        22774 :           if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
    1883              :             break;
    1884              :           /* FALLTHRU */
    1885              : 
    1886       150566 :         case OMP_CLAUSE_FIRSTPRIVATE:
    1887       150566 :         case OMP_CLAUSE_PRIVATE:
    1888       150566 :         case OMP_CLAUSE_LINEAR:
    1889       150566 :         case OMP_CLAUSE_HAS_DEVICE_ADDR:
    1890       150566 :         case OMP_CLAUSE_IS_DEVICE_PTR:
    1891       150566 :           decl = OMP_CLAUSE_DECL (c);
    1892       150566 :           if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
    1893              :             {
    1894          257 :               while (INDIRECT_REF_P (decl)
    1895          257 :                      || TREE_CODE (decl) == ARRAY_REF)
    1896           17 :                 decl = TREE_OPERAND (decl, 0);
    1897              :             }
    1898              : 
    1899       150566 :           if (is_variable_sized (decl))
    1900              :             {
    1901          432 :               if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
    1902              :                    || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR
    1903              :                    || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
    1904           61 :                   && is_gimple_omp_offloaded (ctx->stmt))
    1905              :                 {
    1906           16 :                   tree decl2 = DECL_VALUE_EXPR (decl);
    1907           16 :                   gcc_assert (INDIRECT_REF_P (decl2));
    1908           16 :                   decl2 = TREE_OPERAND (decl2, 0);
    1909           16 :                   gcc_assert (DECL_P (decl2));
    1910           16 :                   install_var_local (decl2, ctx);
    1911           16 :                   fixup_remapped_decl (decl2, ctx, false);
    1912              :                 }
    1913          432 :               install_var_local (decl, ctx);
    1914              :             }
    1915       301132 :           fixup_remapped_decl (decl, ctx,
    1916       150566 :                                OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE
    1917        73503 :                                && OMP_CLAUSE_PRIVATE_DEBUG (c));
    1918       150566 :           if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
    1919       150566 :               && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
    1920              :             scan_array_reductions = true;
    1921              :           break;
    1922              : 
    1923        16791 :         case OMP_CLAUSE_REDUCTION:
    1924        16791 :         case OMP_CLAUSE_IN_REDUCTION:
    1925        16791 :           decl = OMP_CLAUSE_DECL (c);
    1926        16791 :           if (TREE_CODE (decl) != MEM_REF && !is_omp_target (ctx->stmt))
    1927              :             {
    1928        14015 :               if (is_variable_sized (decl))
    1929            0 :                 install_var_local (decl, ctx);
    1930        14015 :               fixup_remapped_decl (decl, ctx, false);
    1931              :             }
    1932        16791 :           if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
    1933       422710 :             scan_array_reductions = true;
    1934              :           break;
    1935              : 
    1936          530 :         case OMP_CLAUSE_TASK_REDUCTION:
    1937          530 :           if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
    1938       422710 :             scan_array_reductions = true;
    1939              :           break;
    1940              : 
    1941        30129 :         case OMP_CLAUSE_SHARED:
    1942              :           /* Ignore shared directives in teams construct inside of
    1943              :              target construct.  */
    1944        30129 :           if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
    1945        30129 :               && !is_host_teams_ctx (ctx))
    1946              :             break;
    1947        18265 :           decl = OMP_CLAUSE_DECL (c);
    1948        18265 :           if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
    1949              :             break;
    1950        12187 :           if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
    1951              :             {
    1952           86 :               if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl,
    1953              :                                                                  ctx->outer)))
    1954              :                 break;
    1955           79 :               bool by_ref = use_pointer_for_field (decl, ctx);
    1956           79 :               install_var_field (decl, by_ref, 11, ctx);
    1957           79 :               break;
    1958              :             }
    1959        12101 :           fixup_remapped_decl (decl, ctx, false);
    1960        12101 :           break;
    1961              : 
    1962        80730 :         case OMP_CLAUSE_MAP:
    1963        80730 :           if (!is_gimple_omp_offloaded (ctx->stmt))
    1964              :             break;
    1965        53898 :           decl = OMP_CLAUSE_DECL (c);
    1966        53898 :           if (DECL_P (decl)
    1967        33905 :               && ((OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER
    1968        30248 :                    && (OMP_CLAUSE_MAP_KIND (c)
    1969              :                        != GOMP_MAP_FIRSTPRIVATE_REFERENCE))
    1970         4086 :                   || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
    1971        31024 :               && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
    1972        61303 :               && varpool_node::get_create (decl)->offloadable)
    1973              :             break;
    1974        48194 :           if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
    1975        45982 :                || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
    1976         2212 :               && is_omp_target (ctx->stmt)
    1977        50197 :               && !is_gimple_omp_offloaded (ctx->stmt))
    1978              :             break;
    1979        48194 :           if (DECL_P (decl))
    1980              :             {
    1981        28201 :               if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
    1982        25160 :                    || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER)
    1983         6696 :                   && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
    1984        29828 :                   && !COMPLETE_TYPE_P (TREE_TYPE (decl)))
    1985              :                 {
    1986           24 :                   tree new_decl = lookup_decl (decl, ctx);
    1987           24 :                   TREE_TYPE (new_decl)
    1988           48 :                     = remap_type (TREE_TYPE (decl), &ctx->cb);
    1989              :                 }
    1990        28177 :               else if (DECL_SIZE (decl)
    1991        28177 :                        && !poly_int_tree_p (DECL_SIZE (decl)))
    1992              :                 {
    1993          712 :                   tree decl2 = DECL_VALUE_EXPR (decl);
    1994          712 :                   gcc_assert (INDIRECT_REF_P (decl2));
    1995          712 :                   decl2 = TREE_OPERAND (decl2, 0);
    1996          712 :                   gcc_assert (DECL_P (decl2));
    1997          712 :                   fixup_remapped_decl (decl2, ctx, false);
    1998          712 :                   fixup_remapped_decl (decl, ctx, true);
    1999              :                 }
    2000              :               else
    2001        27465 :                 fixup_remapped_decl (decl, ctx, false);
    2002              :             }
    2003              :           break;
    2004              : 
    2005              :         case OMP_CLAUSE_COPYPRIVATE:
    2006              :         case OMP_CLAUSE_COPYIN:
    2007              :         case OMP_CLAUSE_DEFAULT:
    2008              :         case OMP_CLAUSE_IF:
    2009              :         case OMP_CLAUSE_SELF:
    2010              :         case OMP_CLAUSE_NUM_THREADS:
    2011              :         case OMP_CLAUSE_NUM_TEAMS:
    2012              :         case OMP_CLAUSE_THREAD_LIMIT:
    2013              :         case OMP_CLAUSE_DEVICE:
    2014              :         case OMP_CLAUSE_SCHEDULE:
    2015              :         case OMP_CLAUSE_DIST_SCHEDULE:
    2016              :         case OMP_CLAUSE_NOWAIT:
    2017              :         case OMP_CLAUSE_ORDERED:
    2018              :         case OMP_CLAUSE_COLLAPSE:
    2019              :         case OMP_CLAUSE_UNTIED:
    2020              :         case OMP_CLAUSE_FINAL:
    2021              :         case OMP_CLAUSE_MERGEABLE:
    2022              :         case OMP_CLAUSE_PROC_BIND:
    2023              :         case OMP_CLAUSE_SAFELEN:
    2024              :         case OMP_CLAUSE_SIMDLEN:
    2025              :         case OMP_CLAUSE_ALIGNED:
    2026              :         case OMP_CLAUSE_DEPEND:
    2027              :         case OMP_CLAUSE_DETACH:
    2028              :         case OMP_CLAUSE_ALLOCATE:
    2029              :         case OMP_CLAUSE__LOOPTEMP_:
    2030              :         case OMP_CLAUSE__REDUCTEMP_:
    2031              :         case OMP_CLAUSE_TO:
    2032              :         case OMP_CLAUSE_FROM:
    2033              :         case OMP_CLAUSE_PRIORITY:
    2034              :         case OMP_CLAUSE_GRAINSIZE:
    2035              :         case OMP_CLAUSE_NUM_TASKS:
    2036              :         case OMP_CLAUSE_THREADS:
    2037              :         case OMP_CLAUSE_SIMD:
    2038              :         case OMP_CLAUSE_NOGROUP:
    2039              :         case OMP_CLAUSE_DEFAULTMAP:
    2040              :         case OMP_CLAUSE_ORDER:
    2041              :         case OMP_CLAUSE_BIND:
    2042              :         case OMP_CLAUSE_USE_DEVICE_PTR:
    2043              :         case OMP_CLAUSE_USE_DEVICE_ADDR:
    2044              :         case OMP_CLAUSE_NONTEMPORAL:
    2045              :         case OMP_CLAUSE_ASYNC:
    2046              :         case OMP_CLAUSE_WAIT:
    2047              :         case OMP_CLAUSE_NUM_GANGS:
    2048              :         case OMP_CLAUSE_NUM_WORKERS:
    2049              :         case OMP_CLAUSE_VECTOR_LENGTH:
    2050              :         case OMP_CLAUSE_GANG:
    2051              :         case OMP_CLAUSE_WORKER:
    2052              :         case OMP_CLAUSE_VECTOR:
    2053              :         case OMP_CLAUSE_INDEPENDENT:
    2054              :         case OMP_CLAUSE_AUTO:
    2055              :         case OMP_CLAUSE_SEQ:
    2056              :         case OMP_CLAUSE_TILE:
    2057              :         case OMP_CLAUSE__SIMT_:
    2058              :         case OMP_CLAUSE_IF_PRESENT:
    2059              :         case OMP_CLAUSE_FINALIZE:
    2060              :         case OMP_CLAUSE_FILTER:
    2061              :         case OMP_CLAUSE__CONDTEMP_:
    2062              :         case OMP_CLAUSE_INIT:
    2063              :         case OMP_CLAUSE_USE:
    2064              :         case OMP_CLAUSE_DESTROY:
    2065              :         case OMP_CLAUSE_DEVICE_TYPE:
    2066              :         case OMP_CLAUSE_USES_ALLOCATORS:
    2067              :         case OMP_CLAUSE_MESSAGE:
    2068              :           break;
    2069              : 
    2070            0 :         case OMP_CLAUSE__CACHE_:
    2071            0 :         case OMP_CLAUSE_NOHOST:
    2072            0 :         default:
    2073            0 :           gcc_unreachable ();
    2074              :         }
    2075              :     }
    2076              : 
    2077       131041 :   gcc_checking_assert (!scan_array_reductions
    2078              :                        || !is_gimple_omp_oacc (ctx->stmt));
    2079              :   if (scan_array_reductions)
    2080              :     {
    2081        34699 :       for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
    2082        29488 :         if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
    2083        27085 :              || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
    2084        26437 :              || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
    2085        30216 :             && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
    2086              :           {
    2087         2229 :             omp_context *rctx = ctx;
    2088         2229 :             if (is_omp_target (ctx->stmt))
    2089           60 :               rctx = ctx->outer;
    2090         2229 :             scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), rctx);
    2091         2229 :             scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), rctx);
    2092              :           }
    2093        27259 :         else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
    2094        27259 :                  && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
    2095         7245 :           scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
    2096        20014 :         else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
    2097        20014 :                  && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
    2098          656 :           scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
    2099              :     }
    2100       131041 :   if (flex_array_ptr)
    2101              :     {
    2102          144 :       tree field = build_range_type (size_type_node,
    2103              :                                      build_int_cstu (size_type_node, 0),
    2104              :                                      NULL_TREE);
    2105          144 :       field = build_array_type (ptr_type_node, field);
    2106          144 :       field = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE, field);
    2107          144 :       SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
    2108          144 :       DECL_CONTEXT (field) = ctx->record_type;
    2109          144 :       DECL_CHAIN (field) = TYPE_FIELDS (ctx->record_type);
    2110          144 :       TYPE_FIELDS (ctx->record_type) = field;
    2111              :     }
    2112       131041 : }
    2113              : 
    2114              : /* Create a new name for omp child function.  Returns an identifier. */
    2115              : 
    2116              : static tree
    2117        52123 : create_omp_child_function_name (bool task_copy)
    2118              : {
    2119       103696 :   return clone_function_name_numbered (current_function_decl,
    2120        52123 :                                        task_copy ? "_omp_cpyfn" : "_omp_fn");
    2121              : }
    2122              : 
    2123              : /* Return true if CTX may belong to offloaded code: either if current function
    2124              :    is offloaded, or any enclosing context corresponds to a target region.  */
    2125              : 
    2126              : static bool
    2127       124211 : omp_maybe_offloaded_ctx (omp_context *ctx)
    2128              : {
    2129       124211 :   if (cgraph_node::get (current_function_decl)->offloadable)
    2130              :     return true;
    2131       205784 :   for (; ctx; ctx = ctx->outer)
    2132       121527 :     if (is_gimple_omp_offloaded (ctx->stmt))
    2133              :       return true;
    2134              :   return false;
    2135              : }
    2136              : 
    2137              : /* Build a decl for the omp child function.  It'll not contain a body
    2138              :    yet, just the bare decl. If HOST_ONLY, do not create a device version.  */
    2139              : 
    2140              : static void
    2141        52123 : create_omp_child_function (omp_context *ctx, bool task_copy, bool host_only)
    2142              : {
    2143        52123 :   tree decl, type, name, t;
    2144              : 
    2145        52123 :   name = create_omp_child_function_name (task_copy);
    2146        52123 :   if (task_copy)
    2147          550 :     type = build_function_type_list (void_type_node, ptr_type_node,
    2148              :                                      ptr_type_node, NULL_TREE);
    2149              :   else
    2150        51573 :     type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE);
    2151              : 
    2152        52123 :   decl = build_decl (gimple_location (ctx->stmt), FUNCTION_DECL, name, type);
    2153              : 
    2154        52123 :   gcc_checking_assert (!is_gimple_omp_oacc (ctx->stmt)
    2155              :                        || !task_copy);
    2156        40737 :   if (!task_copy)
    2157        51573 :     ctx->cb.dst_fn = decl;
    2158              :   else
    2159          550 :     gimple_omp_task_set_copy_fn (ctx->stmt, decl);
    2160              : 
    2161        52123 :   TREE_STATIC (decl) = 1;
    2162        52123 :   TREE_USED (decl) = 1;
    2163        52123 :   DECL_ARTIFICIAL (decl) = 1;
    2164        52123 :   DECL_IGNORED_P (decl) = 0;
    2165        52123 :   TREE_PUBLIC (decl) = 0;
    2166        52123 :   DECL_UNINLINABLE (decl) = 1;
    2167        52123 :   DECL_EXTERNAL (decl) = 0;
    2168        52123 :   DECL_CONTEXT (decl) = NULL_TREE;
    2169        52123 :   DECL_INITIAL (decl) = make_node (BLOCK);
    2170        52123 :   BLOCK_SUPERCONTEXT (DECL_INITIAL (decl)) = decl;
    2171        52123 :   DECL_ATTRIBUTES (decl) = DECL_ATTRIBUTES (current_function_decl);
    2172              :   /* Remove omp declare simd attribute from the new attributes.  */
    2173        52123 :   if (tree a = lookup_attribute ("omp declare simd", DECL_ATTRIBUTES (decl)))
    2174              :     {
    2175            9 :       while (tree a2 = lookup_attribute ("omp declare simd", TREE_CHAIN (a)))
    2176              :         a = a2;
    2177            9 :       a = TREE_CHAIN (a);
    2178           22 :       for (tree *p = &DECL_ATTRIBUTES (decl); *p != a;)
    2179           13 :         if (is_attribute_p ("omp declare simd", get_attribute_name (*p)))
    2180            9 :           *p = TREE_CHAIN (*p);
    2181              :         else
    2182              :           {
    2183            4 :             tree chain = TREE_CHAIN (*p);
    2184            4 :             *p = copy_node (*p);
    2185            4 :             p = &TREE_CHAIN (*p);
    2186            4 :             *p = chain;
    2187              :           }
    2188              :     }
    2189       104246 :   DECL_FUNCTION_SPECIFIC_OPTIMIZATION (decl)
    2190        52123 :     = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (current_function_decl);
    2191       104246 :   DECL_FUNCTION_SPECIFIC_TARGET (decl)
    2192        52123 :     = DECL_FUNCTION_SPECIFIC_TARGET (current_function_decl);
    2193       104246 :   DECL_FUNCTION_VERSIONED (decl)
    2194        52123 :     = DECL_FUNCTION_VERSIONED (current_function_decl);
    2195              : 
    2196        52123 :   if (omp_maybe_offloaded_ctx (ctx) && !host_only)
    2197              :     {
    2198        31199 :       cgraph_node::get_create (decl)->offloadable = 1;
    2199        31199 :       if (ENABLE_OFFLOADING)
    2200              :         g->have_offload = true;
    2201              :     }
    2202              : 
    2203        52123 :   if (cgraph_node::get_create (decl)->offloadable)
    2204              :     {
    2205        31199 :       const char *target_attr = (is_gimple_omp_offloaded (ctx->stmt)
    2206        31199 :                                  ? "omp target entrypoint"
    2207         6242 :                                  : "omp declare target");
    2208        31199 :       if (lookup_attribute ("omp declare target",
    2209        31199 :                             DECL_ATTRIBUTES (current_function_decl)))
    2210              :         {
    2211         3032 :           if (is_gimple_omp_offloaded (ctx->stmt))
    2212         1620 :             DECL_ATTRIBUTES (decl)
    2213         1620 :               = remove_attribute ("omp declare target",
    2214         1620 :                                   copy_list (DECL_ATTRIBUTES (decl)));
    2215              :           else
    2216              :             target_attr = NULL;
    2217              :         }
    2218         1620 :       if (target_attr
    2219        29787 :           && is_gimple_omp_offloaded (ctx->stmt)
    2220        24957 :           && lookup_attribute ("noclone", DECL_ATTRIBUTES (decl)) == NULL_TREE)
    2221        19450 :         DECL_ATTRIBUTES (decl) = tree_cons (get_identifier ("noclone"),
    2222        19450 :                                            NULL_TREE, DECL_ATTRIBUTES (decl));
    2223        31199 :       if (target_attr)
    2224        29787 :         DECL_ATTRIBUTES (decl)
    2225        59574 :           = tree_cons (get_identifier (target_attr),
    2226        29787 :                        NULL_TREE, DECL_ATTRIBUTES (decl));
    2227              :     }
    2228              : 
    2229        52123 :   t = build_decl (DECL_SOURCE_LOCATION (decl),
    2230              :                   RESULT_DECL, NULL_TREE, void_type_node);
    2231        52123 :   DECL_ARTIFICIAL (t) = 1;
    2232        52123 :   DECL_IGNORED_P (t) = 1;
    2233        52123 :   DECL_CONTEXT (t) = decl;
    2234        52123 :   DECL_RESULT (decl) = t;
    2235              : 
    2236        52123 :   tree data_name = get_identifier (".omp_data_i");
    2237        52123 :   t = build_decl (DECL_SOURCE_LOCATION (decl), PARM_DECL, data_name,
    2238              :                   ptr_type_node);
    2239        52123 :   DECL_ARTIFICIAL (t) = 1;
    2240        52123 :   DECL_NAMELESS (t) = 1;
    2241        52123 :   DECL_ARG_TYPE (t) = ptr_type_node;
    2242        52123 :   DECL_CONTEXT (t) = current_function_decl;
    2243        52123 :   TREE_USED (t) = 1;
    2244        52123 :   TREE_READONLY (t) = 1;
    2245        52123 :   DECL_ARGUMENTS (decl) = t;
    2246        52123 :   if (!task_copy)
    2247        51573 :     ctx->receiver_decl = t;
    2248              :   else
    2249              :     {
    2250          550 :       t = build_decl (DECL_SOURCE_LOCATION (decl),
    2251              :                       PARM_DECL, get_identifier (".omp_data_o"),
    2252              :                       ptr_type_node);
    2253          550 :       DECL_ARTIFICIAL (t) = 1;
    2254          550 :       DECL_NAMELESS (t) = 1;
    2255          550 :       DECL_ARG_TYPE (t) = ptr_type_node;
    2256          550 :       DECL_CONTEXT (t) = current_function_decl;
    2257          550 :       TREE_USED (t) = 1;
    2258          550 :       TREE_ADDRESSABLE (t) = 1;
    2259          550 :       DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
    2260          550 :       DECL_ARGUMENTS (decl) = t;
    2261              :     }
    2262              : 
    2263              :   /* Allocate memory for the function structure.  The call to
    2264              :      allocate_struct_function clobbers CFUN, so we need to restore
    2265              :      it afterward.  */
    2266        52123 :   push_struct_function (decl);
    2267        52123 :   cfun->function_end_locus = gimple_location (ctx->stmt);
    2268        52123 :   init_tree_ssa (cfun);
    2269        52123 :   pop_cfun ();
    2270        52123 : }
    2271              : 
    2272              : /* Callback for walk_gimple_seq.  Check if combined parallel
    2273              :    contains gimple_omp_for_combined_into_p OMP_FOR.  */
    2274              : 
    2275              : tree
    2276        35863 : omp_find_combined_for (gimple_stmt_iterator *gsi_p,
    2277              :                        bool *handled_ops_p,
    2278              :                        struct walk_stmt_info *wi)
    2279              : {
    2280        35863 :   gimple *stmt = gsi_stmt (*gsi_p);
    2281              : 
    2282        35863 :   *handled_ops_p = true;
    2283        35863 :   switch (gimple_code (stmt))
    2284              :     {
    2285        20118 :     WALK_SUBSTMTS;
    2286              : 
    2287        13574 :     case GIMPLE_OMP_FOR:
    2288        13574 :       if (gimple_omp_for_combined_into_p (stmt)
    2289        13574 :           && gimple_omp_for_kind (stmt)
    2290         8009 :              == *(const enum gf_mask *) (wi->info))
    2291              :         {
    2292         8009 :           wi->info = stmt;
    2293         8009 :           return integer_zero_node;
    2294              :         }
    2295              :       break;
    2296              :     default:
    2297              :       break;
    2298              :     }
    2299              :   return NULL;
    2300              : }
    2301              : 
    2302              : /* Add _LOOPTEMP_/_REDUCTEMP_ clauses on OpenMP parallel or task.  */
    2303              : 
    2304              : static void
    2305        14159 : add_taskreg_looptemp_clauses (enum gf_mask msk, gimple *stmt,
    2306              :                               omp_context *outer_ctx)
    2307              : {
    2308        14159 :   struct walk_stmt_info wi;
    2309              : 
    2310        14159 :   memset (&wi, 0, sizeof (wi));
    2311        14159 :   wi.val_only = true;
    2312        14159 :   wi.info = (void *) &msk;
    2313        14159 :   walk_gimple_seq (gimple_omp_body (stmt), omp_find_combined_for, NULL, &wi);
    2314        14159 :   if (wi.info != (void *) &msk)
    2315              :     {
    2316         8009 :       gomp_for *for_stmt = as_a <gomp_for *> ((gimple *) wi.info);
    2317         8009 :       struct omp_for_data fd;
    2318         8009 :       omp_extract_for_data (for_stmt, &fd, NULL);
    2319              :       /* We need two temporaries with fd.loop.v type (istart/iend)
    2320              :          and then (fd.collapse - 1) temporaries with the same
    2321              :          type for count2 ... countN-1 vars if not constant.  */
    2322         8009 :       size_t count = 2, i;
    2323         8009 :       tree type = fd.iter_type;
    2324         8009 :       if (fd.collapse > 1
    2325         2648 :           && TREE_CODE (fd.loop.n2) != INTEGER_CST)
    2326              :         {
    2327         1466 :           count += fd.collapse - 1;
    2328              :           /* If there are lastprivate clauses on the inner
    2329              :              GIMPLE_OMP_FOR, add one more temporaries for the total number
    2330              :              of iterations (product of count1 ... countN-1).  */
    2331         1466 :           if (omp_find_clause (gimple_omp_for_clauses (for_stmt),
    2332              :                                OMP_CLAUSE_LASTPRIVATE)
    2333         1466 :               || (msk == GF_OMP_FOR_KIND_FOR
    2334         1335 :                   && omp_find_clause (gimple_omp_parallel_clauses (stmt),
    2335              :                                       OMP_CLAUSE_LASTPRIVATE)))
    2336              :             {
    2337          726 :               tree temp = create_tmp_var (type);
    2338          726 :               tree c = build_omp_clause (UNKNOWN_LOCATION,
    2339              :                                          OMP_CLAUSE__LOOPTEMP_);
    2340          726 :               insert_decl_map (&outer_ctx->cb, temp, temp);
    2341          726 :               OMP_CLAUSE_DECL (c) = temp;
    2342          726 :               OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
    2343          726 :               gimple_omp_taskreg_set_clauses (stmt, c);
    2344              :             }
    2345         1466 :           if (fd.non_rect
    2346           24 :               && fd.last_nonrect == fd.first_nonrect + 1)
    2347           12 :             if (tree v = gimple_omp_for_index (for_stmt, fd.last_nonrect))
    2348           12 :               if (!TYPE_UNSIGNED (TREE_TYPE (v)))
    2349              :                 {
    2350           12 :                   v = gimple_omp_for_index (for_stmt, fd.first_nonrect);
    2351           12 :                   tree type2 = TREE_TYPE (v);
    2352           12 :                   count++;
    2353           48 :                   for (i = 0; i < 3; i++)
    2354              :                     {
    2355           36 :                       tree temp = create_tmp_var (type2);
    2356           36 :                       tree c = build_omp_clause (UNKNOWN_LOCATION,
    2357              :                                                  OMP_CLAUSE__LOOPTEMP_);
    2358           36 :                       insert_decl_map (&outer_ctx->cb, temp, temp);
    2359           36 :                       OMP_CLAUSE_DECL (c) = temp;
    2360           36 :                       OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
    2361           36 :                       gimple_omp_taskreg_set_clauses (stmt, c);
    2362              :                     }
    2363              :                 }
    2364              :         }
    2365        26872 :       for (i = 0; i < count; i++)
    2366              :         {
    2367        18863 :           tree temp = create_tmp_var (type);
    2368        18863 :           tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
    2369        18863 :           insert_decl_map (&outer_ctx->cb, temp, temp);
    2370        18863 :           OMP_CLAUSE_DECL (c) = temp;
    2371        18863 :           OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
    2372        18863 :           gimple_omp_taskreg_set_clauses (stmt, c);
    2373              :         }
    2374              :     }
    2375        14159 :   if (msk == GF_OMP_FOR_KIND_TASKLOOP
    2376        15726 :       && omp_find_clause (gimple_omp_task_clauses (stmt),
    2377              :                           OMP_CLAUSE_REDUCTION))
    2378              :     {
    2379          517 :       tree type = build_pointer_type (pointer_sized_int_node);
    2380          517 :       tree temp = create_tmp_var (type);
    2381          517 :       tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
    2382          517 :       insert_decl_map (&outer_ctx->cb, temp, temp);
    2383          517 :       OMP_CLAUSE_DECL (c) = temp;
    2384          517 :       OMP_CLAUSE_CHAIN (c) = gimple_omp_task_clauses (stmt);
    2385          517 :       gimple_omp_task_set_clauses (stmt, c);
    2386              :     }
    2387        14159 : }
    2388              : 
    2389              : /* Scan an OpenMP parallel directive.  */
    2390              : 
    2391              : static void
    2392        18393 : scan_omp_parallel (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
    2393              : {
    2394        18393 :   omp_context *ctx;
    2395        18393 :   tree name;
    2396        18393 :   gomp_parallel *stmt = as_a <gomp_parallel *> (gsi_stmt (*gsi));
    2397              : 
    2398              :   /* Ignore parallel directives with empty bodies, unless there
    2399              :      are copyin clauses.  */
    2400        18393 :   if (optimize > 0
    2401        12590 :       && empty_body_p (gimple_omp_body (stmt))
    2402        18440 :       && omp_find_clause (gimple_omp_parallel_clauses (stmt),
    2403              :                           OMP_CLAUSE_COPYIN) == NULL)
    2404              :     {
    2405           41 :       gsi_replace (gsi, gimple_build_nop (), false);
    2406           41 :       return;
    2407              :     }
    2408              : 
    2409        18352 :   if (gimple_omp_parallel_combined_p (stmt))
    2410        12592 :     add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR, stmt, outer_ctx);
    2411        18352 :   for (tree c = omp_find_clause (gimple_omp_parallel_clauses (stmt),
    2412              :                                  OMP_CLAUSE_REDUCTION);
    2413        21221 :        c; c = omp_find_clause (OMP_CLAUSE_CHAIN (c), OMP_CLAUSE_REDUCTION))
    2414         3986 :     if (OMP_CLAUSE_REDUCTION_TASK (c))
    2415              :       {
    2416           69 :         tree type = build_pointer_type (pointer_sized_int_node);
    2417           69 :         tree temp = create_tmp_var (type);
    2418           69 :         tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
    2419           69 :         if (outer_ctx)
    2420           18 :           insert_decl_map (&outer_ctx->cb, temp, temp);
    2421           69 :         OMP_CLAUSE_DECL (c) = temp;
    2422           69 :         OMP_CLAUSE_CHAIN (c) = gimple_omp_parallel_clauses (stmt);
    2423           69 :         gimple_omp_parallel_set_clauses (stmt, c);
    2424           69 :         break;
    2425              :       }
    2426         3917 :     else if (OMP_CLAUSE_CHAIN (c) == NULL_TREE)
    2427              :       break;
    2428              : 
    2429        18352 :   ctx = new_omp_context (stmt, outer_ctx);
    2430        18352 :   taskreg_contexts.safe_push (ctx);
    2431        18352 :   if (taskreg_nesting_level > 1)
    2432         6287 :     ctx->is_nested = true;
    2433        18352 :   ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
    2434        18352 :   ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
    2435        18352 :   name = create_tmp_var_name (".omp_data_s");
    2436        18352 :   name = build_decl (gimple_location (stmt),
    2437              :                      TYPE_DECL, name, ctx->record_type);
    2438        18352 :   DECL_ARTIFICIAL (name) = 1;
    2439        18352 :   DECL_NAMELESS (name) = 1;
    2440        18352 :   TYPE_NAME (ctx->record_type) = name;
    2441        18352 :   TYPE_ARTIFICIAL (ctx->record_type) = 1;
    2442        18352 :   create_omp_child_function (ctx, false, false);
    2443        18352 :   gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn);
    2444              : 
    2445        18352 :   scan_sharing_clauses (gimple_omp_parallel_clauses (stmt), ctx);
    2446        18352 :   scan_omp (gimple_omp_body_ptr (stmt), ctx);
    2447              : 
    2448        18352 :   if (TYPE_FIELDS (ctx->record_type) == NULL)
    2449         3617 :     ctx->record_type = ctx->receiver_decl = NULL;
    2450              : }
    2451              : 
    2452              : /* Scan an OpenMP task directive.  */
    2453              : 
    2454              : static void
    2455         5651 : scan_omp_task (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
    2456              : {
    2457         5651 :   omp_context *ctx;
    2458         5651 :   tree name, t;
    2459         5651 :   gomp_task *stmt = as_a <gomp_task *> (gsi_stmt (*gsi));
    2460              : 
    2461              :   /* Ignore task directives with empty bodies, unless they have depend
    2462              :      clause.  */
    2463         5651 :   if (optimize > 0
    2464         2575 :       && gimple_omp_body (stmt)
    2465         2532 :       && empty_body_p (gimple_omp_body (stmt))
    2466         5707 :       && !omp_find_clause (gimple_omp_task_clauses (stmt), OMP_CLAUSE_DEPEND))
    2467              :     {
    2468           21 :       gsi_replace (gsi, gimple_build_nop (), false);
    2469          135 :       return;
    2470              :     }
    2471              : 
    2472         5630 :   if (gimple_omp_task_taskloop_p (stmt))
    2473         1567 :     add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP, stmt, outer_ctx);
    2474              : 
    2475         5630 :   ctx = new_omp_context (stmt, outer_ctx);
    2476              : 
    2477         5630 :   if (gimple_omp_task_taskwait_p (stmt))
    2478              :     {
    2479           93 :       scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
    2480           93 :       return;
    2481              :     }
    2482              : 
    2483         5537 :   taskreg_contexts.safe_push (ctx);
    2484         5537 :   if (taskreg_nesting_level > 1)
    2485         2342 :     ctx->is_nested = true;
    2486         5537 :   ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
    2487         5537 :   ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
    2488         5537 :   name = create_tmp_var_name (".omp_data_s");
    2489         5537 :   name = build_decl (gimple_location (stmt),
    2490              :                      TYPE_DECL, name, ctx->record_type);
    2491         5537 :   DECL_ARTIFICIAL (name) = 1;
    2492         5537 :   DECL_NAMELESS (name) = 1;
    2493         5537 :   TYPE_NAME (ctx->record_type) = name;
    2494         5537 :   TYPE_ARTIFICIAL (ctx->record_type) = 1;
    2495         5537 :   create_omp_child_function (ctx, false, false);
    2496         5537 :   gimple_omp_task_set_child_fn (stmt, ctx->cb.dst_fn);
    2497              : 
    2498         5537 :   scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
    2499              : 
    2500         5537 :   if (ctx->srecord_type)
    2501              :     {
    2502          550 :       name = create_tmp_var_name (".omp_data_a");
    2503          550 :       name = build_decl (gimple_location (stmt),
    2504              :                          TYPE_DECL, name, ctx->srecord_type);
    2505          550 :       DECL_ARTIFICIAL (name) = 1;
    2506          550 :       DECL_NAMELESS (name) = 1;
    2507          550 :       TYPE_NAME (ctx->srecord_type) = name;
    2508          550 :       TYPE_ARTIFICIAL (ctx->srecord_type) = 1;
    2509          550 :       create_omp_child_function (ctx, true, false);
    2510              :     }
    2511              : 
    2512         5537 :   scan_omp (gimple_omp_body_ptr (stmt), ctx);
    2513              : 
    2514         5537 :   if (TYPE_FIELDS (ctx->record_type) == NULL)
    2515              :     {
    2516         2082 :       ctx->record_type = ctx->receiver_decl = NULL;
    2517         2082 :       t = build_int_cst (long_integer_type_node, 0);
    2518         2082 :       gimple_omp_task_set_arg_size (stmt, t);
    2519         2082 :       t = build_int_cst (long_integer_type_node, 1);
    2520         2082 :       gimple_omp_task_set_arg_align (stmt, t);
    2521              :     }
    2522              : }
    2523              : 
    2524              : /* Helper function for finish_taskreg_scan, called through walk_tree.
    2525              :    If maybe_lookup_decl_in_outer_context returns non-NULL for some
    2526              :    tree, replace it in the expression.  */
    2527              : 
    2528              : static tree
    2529          276 : finish_taskreg_remap (tree *tp, int *walk_subtrees, void *data)
    2530              : {
    2531          276 :   if (VAR_P (*tp))
    2532              :     {
    2533           96 :       omp_context *ctx = (omp_context *) data;
    2534           96 :       tree t = maybe_lookup_decl_in_outer_ctx (*tp, ctx);
    2535           96 :       if (t != *tp)
    2536              :         {
    2537            9 :           if (DECL_HAS_VALUE_EXPR_P (t))
    2538            0 :             t = unshare_expr (DECL_VALUE_EXPR (t));
    2539            9 :           *tp = t;
    2540              :         }
    2541           96 :       *walk_subtrees = 0;
    2542              :     }
    2543          180 :   else if (IS_TYPE_OR_DECL_P (*tp))
    2544            0 :     *walk_subtrees = 0;
    2545          276 :   return NULL_TREE;
    2546              : }
    2547              : 
    2548              : /* If any decls have been made addressable during scan_omp,
    2549              :    adjust their fields if needed, and layout record types
    2550              :    of parallel/task constructs.  */
    2551              : 
    2552              : static void
    2553        26610 : finish_taskreg_scan (omp_context *ctx)
    2554              : {
    2555        26610 :   if (ctx->record_type == NULL_TREE)
    2556              :     return;
    2557              : 
    2558              :   /* If any make_addressable_vars were needed, verify all
    2559              :      OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK,TEAMS}
    2560              :      statements if use_pointer_for_field hasn't changed
    2561              :      because of that.  If it did, update field types now.  */
    2562        19456 :   if (make_addressable_vars)
    2563              :     {
    2564         2216 :       tree c;
    2565              : 
    2566        12369 :       for (c = gimple_omp_taskreg_clauses (ctx->stmt);
    2567        12369 :            c; c = OMP_CLAUSE_CHAIN (c))
    2568        10153 :         if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
    2569        10153 :             && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
    2570              :           {
    2571         2617 :             tree decl = OMP_CLAUSE_DECL (c);
    2572              : 
    2573              :             /* Global variables don't need to be copied,
    2574              :                the receiver side will use them directly.  */
    2575         2617 :             if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
    2576          282 :               continue;
    2577         2335 :             if (!bitmap_bit_p (make_addressable_vars, DECL_UID (decl))
    2578         2335 :                 || !use_pointer_for_field (decl, ctx))
    2579         1784 :               continue;
    2580          551 :             tree field = lookup_field (decl, ctx);
    2581          551 :             if (TREE_CODE (TREE_TYPE (field)) == POINTER_TYPE
    2582          551 :                 && TREE_TYPE (TREE_TYPE (field)) == TREE_TYPE (decl))
    2583          497 :               continue;
    2584           54 :             TREE_TYPE (field) = build_pointer_type (TREE_TYPE (decl));
    2585           54 :             TREE_THIS_VOLATILE (field) = 0;
    2586           54 :             DECL_USER_ALIGN (field) = 0;
    2587           54 :             SET_DECL_ALIGN (field, TYPE_ALIGN (TREE_TYPE (field)));
    2588           54 :             if (TYPE_ALIGN (ctx->record_type) < DECL_ALIGN (field))
    2589           16 :               SET_TYPE_ALIGN (ctx->record_type, DECL_ALIGN (field));
    2590           54 :             if (ctx->srecord_type)
    2591              :               {
    2592            0 :                 tree sfield = lookup_sfield (decl, ctx);
    2593            0 :                 TREE_TYPE (sfield) = TREE_TYPE (field);
    2594            0 :                 TREE_THIS_VOLATILE (sfield) = 0;
    2595            0 :                 DECL_USER_ALIGN (sfield) = 0;
    2596            0 :                 SET_DECL_ALIGN (sfield, DECL_ALIGN (field));
    2597            0 :                 if (TYPE_ALIGN (ctx->srecord_type) < DECL_ALIGN (sfield))
    2598            0 :                   SET_TYPE_ALIGN (ctx->srecord_type, DECL_ALIGN (sfield));
    2599              :               }
    2600              :           }
    2601              :     }
    2602              : 
    2603        19456 :   if (gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL)
    2604              :     {
    2605        14735 :       tree clauses = gimple_omp_parallel_clauses (ctx->stmt);
    2606        14735 :       tree c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
    2607        14735 :       if (c)
    2608              :         {
    2609              :           /* Move the _reductemp_ clause first.  GOMP_parallel_reductions
    2610              :              expects to find it at the start of data.  */
    2611           69 :           tree f = lookup_field (OMP_CLAUSE_DECL (c), ctx);
    2612           69 :           tree *p = &TYPE_FIELDS (ctx->record_type);
    2613          155 :           while (*p)
    2614          155 :             if (*p == f)
    2615              :               {
    2616           69 :                 *p = DECL_CHAIN (*p);
    2617           69 :                 break;
    2618              :               }
    2619              :             else
    2620           86 :               p = &DECL_CHAIN (*p);
    2621           69 :           DECL_CHAIN (f) = TYPE_FIELDS (ctx->record_type);
    2622           69 :           TYPE_FIELDS (ctx->record_type) = f;
    2623              :         }
    2624        14735 :       layout_type (ctx->record_type);
    2625        14735 :       fixup_child_record_type (ctx);
    2626              :     }
    2627         4721 :   else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
    2628              :     {
    2629         1266 :       layout_type (ctx->record_type);
    2630         1266 :       fixup_child_record_type (ctx);
    2631              :     }
    2632              :   else
    2633              :     {
    2634         3455 :       location_t loc = gimple_location (ctx->stmt);
    2635         3455 :       tree *p, vla_fields = NULL_TREE, *q = &vla_fields;
    2636         3455 :       tree detach_clause
    2637         3455 :         = omp_find_clause (gimple_omp_task_clauses (ctx->stmt),
    2638              :                            OMP_CLAUSE_DETACH);
    2639              :       /* Move VLA fields to the end.  */
    2640         3455 :       p = &TYPE_FIELDS (ctx->record_type);
    2641        15410 :       while (*p)
    2642        11955 :         if (!TYPE_SIZE_UNIT (TREE_TYPE (*p))
    2643        11955 :             || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p))))
    2644              :           {
    2645           96 :             *q = *p;
    2646           96 :             *p = TREE_CHAIN (*p);
    2647           96 :             TREE_CHAIN (*q) = NULL_TREE;
    2648           96 :             q = &TREE_CHAIN (*q);
    2649              :           }
    2650              :         else
    2651        11859 :           p = &DECL_CHAIN (*p);
    2652         3455 :       *p = vla_fields;
    2653         3455 :       if (gimple_omp_task_taskloop_p (ctx->stmt))
    2654              :         {
    2655              :           /* Move fields corresponding to first and second _looptemp_
    2656              :              clause first.  There are filled by GOMP_taskloop
    2657              :              and thus need to be in specific positions.  */
    2658         1567 :           tree clauses = gimple_omp_task_clauses (ctx->stmt);
    2659         1567 :           tree c1 = omp_find_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
    2660         1567 :           tree c2 = omp_find_clause (OMP_CLAUSE_CHAIN (c1),
    2661              :                                      OMP_CLAUSE__LOOPTEMP_);
    2662         1567 :           tree c3 = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
    2663         1567 :           tree f1 = lookup_field (OMP_CLAUSE_DECL (c1), ctx);
    2664         1567 :           tree f2 = lookup_field (OMP_CLAUSE_DECL (c2), ctx);
    2665         2084 :           tree f3 = c3 ? lookup_field (OMP_CLAUSE_DECL (c3), ctx) : NULL_TREE;
    2666         1567 :           p = &TYPE_FIELDS (ctx->record_type);
    2667         8873 :           while (*p)
    2668         7306 :             if (*p == f1 || *p == f2 || *p == f3)
    2669         3651 :               *p = DECL_CHAIN (*p);
    2670              :             else
    2671         3655 :               p = &DECL_CHAIN (*p);
    2672         1567 :           DECL_CHAIN (f1) = f2;
    2673         1567 :           if (c3)
    2674              :             {
    2675          517 :               DECL_CHAIN (f2) = f3;
    2676          517 :               DECL_CHAIN (f3) = TYPE_FIELDS (ctx->record_type);
    2677              :             }
    2678              :           else
    2679         1050 :             DECL_CHAIN (f2) = TYPE_FIELDS (ctx->record_type);
    2680         1567 :           TYPE_FIELDS (ctx->record_type) = f1;
    2681         1567 :           if (ctx->srecord_type)
    2682              :             {
    2683          389 :               f1 = lookup_sfield (OMP_CLAUSE_DECL (c1), ctx);
    2684          389 :               f2 = lookup_sfield (OMP_CLAUSE_DECL (c2), ctx);
    2685          389 :               if (c3)
    2686          186 :                 f3 = lookup_sfield (OMP_CLAUSE_DECL (c3), ctx);
    2687          389 :               p = &TYPE_FIELDS (ctx->srecord_type);
    2688         2099 :               while (*p)
    2689         1710 :                 if (*p == f1 || *p == f2 || *p == f3)
    2690          964 :                   *p = DECL_CHAIN (*p);
    2691              :                 else
    2692          746 :                   p = &DECL_CHAIN (*p);
    2693          389 :               DECL_CHAIN (f1) = f2;
    2694          389 :               DECL_CHAIN (f2) = TYPE_FIELDS (ctx->srecord_type);
    2695          389 :               if (c3)
    2696              :                 {
    2697          186 :                   DECL_CHAIN (f2) = f3;
    2698          186 :                   DECL_CHAIN (f3) = TYPE_FIELDS (ctx->srecord_type);
    2699              :                 }
    2700              :               else
    2701          203 :                 DECL_CHAIN (f2) = TYPE_FIELDS (ctx->srecord_type);
    2702          389 :               TYPE_FIELDS (ctx->srecord_type) = f1;
    2703              :             }
    2704              :         }
    2705         3455 :       if (detach_clause)
    2706              :         {
    2707          189 :           tree c, field;
    2708              : 
    2709              :           /* Look for a firstprivate clause with the detach event handle.  */
    2710          540 :           for (c = gimple_omp_taskreg_clauses (ctx->stmt);
    2711          540 :                c; c = OMP_CLAUSE_CHAIN (c))
    2712              :             {
    2713          540 :               if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FIRSTPRIVATE)
    2714          347 :                 continue;
    2715          193 :               if (maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_DECL (c), ctx)
    2716          193 :                   == OMP_CLAUSE_DECL (detach_clause))
    2717              :                 break;
    2718              :             }
    2719              : 
    2720          189 :           gcc_assert (c);
    2721          189 :           field = lookup_field (OMP_CLAUSE_DECL (c), ctx);
    2722              : 
    2723              :           /* Move field corresponding to the detach clause first.
    2724              :              This is filled by GOMP_task and needs to be in a
    2725              :              specific position.  */
    2726          189 :           p = &TYPE_FIELDS (ctx->record_type);
    2727          518 :           while (*p)
    2728          329 :             if (*p == field)
    2729          189 :               *p = DECL_CHAIN (*p);
    2730              :             else
    2731          140 :               p = &DECL_CHAIN (*p);
    2732          189 :           DECL_CHAIN (field) = TYPE_FIELDS (ctx->record_type);
    2733          189 :           TYPE_FIELDS (ctx->record_type) = field;
    2734          189 :           if (ctx->srecord_type)
    2735              :             {
    2736            3 :               field = lookup_sfield (OMP_CLAUSE_DECL (c), ctx);
    2737            3 :               p = &TYPE_FIELDS (ctx->srecord_type);
    2738            9 :               while (*p)
    2739            6 :                 if (*p == field)
    2740            3 :                   *p = DECL_CHAIN (*p);
    2741              :                 else
    2742            3 :                   p = &DECL_CHAIN (*p);
    2743            3 :               DECL_CHAIN (field) = TYPE_FIELDS (ctx->srecord_type);
    2744            3 :               TYPE_FIELDS (ctx->srecord_type) = field;
    2745              :             }
    2746              :         }
    2747         3455 :       layout_type (ctx->record_type);
    2748         3455 :       fixup_child_record_type (ctx);
    2749         3455 :       if (ctx->srecord_type)
    2750          550 :         layout_type (ctx->srecord_type);
    2751         3455 :       tree t = fold_convert_loc (loc, long_integer_type_node,
    2752         3455 :                                  TYPE_SIZE_UNIT (ctx->record_type));
    2753         3455 :       if (TREE_CODE (t) != INTEGER_CST)
    2754              :         {
    2755           21 :           t = unshare_expr (t);
    2756           21 :           walk_tree (&t, finish_taskreg_remap, ctx, NULL);
    2757              :         }
    2758         3455 :       gimple_omp_task_set_arg_size (ctx->stmt, t);
    2759         3455 :       t = build_int_cst (long_integer_type_node,
    2760         3455 :                          TYPE_ALIGN_UNIT (ctx->record_type));
    2761         3455 :       gimple_omp_task_set_arg_align (ctx->stmt, t);
    2762              :     }
    2763              : }
    2764              : 
    2765              : /* Find the enclosing offload context.  */
    2766              : 
    2767              : static omp_context *
    2768         9665 : enclosing_target_ctx (omp_context *ctx)
    2769              : {
    2770        43359 :   for (; ctx; ctx = ctx->outer)
    2771        41745 :     if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET)
    2772              :       break;
    2773              : 
    2774        22111 :   return ctx;
    2775              : }
    2776              : 
    2777              : /* Return whether CTX's parent compute construct is an OpenACC 'kernels'
    2778              :    construct.
    2779              :    (This doesn't include OpenACC 'kernels' decomposed parts.)  */
    2780              : 
    2781              : static bool
    2782        11325 : ctx_in_oacc_kernels_region (omp_context *ctx)
    2783              : {
    2784        39137 :   for (;ctx != NULL; ctx = ctx->outer)
    2785              :     {
    2786        29472 :       gimple *stmt = ctx->stmt;
    2787        29472 :       if (gimple_code (stmt) == GIMPLE_OMP_TARGET
    2788        29472 :           && gimple_omp_target_kind (stmt) == GF_OMP_TARGET_KIND_OACC_KERNELS)
    2789              :         return true;
    2790              :     }
    2791              : 
    2792              :   return false;
    2793              : }
    2794              : 
    2795              : /* Check the parallelism clauses inside a OpenACC 'kernels' region.
    2796              :    (This doesn't include OpenACC 'kernels' decomposed parts.)
    2797              :    Until kernels handling moves to use the same loop indirection
    2798              :    scheme as parallel, we need to do this checking early.  */
    2799              : 
    2800              : static unsigned
    2801         7151 : check_oacc_kernel_gwv (gomp_for *stmt, omp_context *ctx)
    2802              : {
    2803         7151 :   bool checking = true;
    2804         7151 :   unsigned outer_mask = 0;
    2805         7151 :   unsigned this_mask = 0;
    2806         7151 :   bool has_seq = false, has_auto = false;
    2807              : 
    2808         7151 :   if (ctx->outer)
    2809         4819 :     outer_mask = check_oacc_kernel_gwv (NULL,  ctx->outer);
    2810         7151 :   if (!stmt)
    2811              :     {
    2812         4819 :       checking = false;
    2813         4819 :       if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR)
    2814              :         return outer_mask;
    2815         1730 :       stmt = as_a <gomp_for *> (ctx->stmt);
    2816              :     }
    2817              : 
    2818        11330 :   for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
    2819              :     {
    2820         7268 :       switch (OMP_CLAUSE_CODE (c))
    2821              :         {
    2822          645 :         case OMP_CLAUSE_GANG:
    2823          645 :           this_mask |= GOMP_DIM_MASK (GOMP_DIM_GANG);
    2824          645 :           break;
    2825          439 :         case OMP_CLAUSE_WORKER:
    2826          439 :           this_mask |= GOMP_DIM_MASK (GOMP_DIM_WORKER);
    2827          439 :           break;
    2828          342 :         case OMP_CLAUSE_VECTOR:
    2829          342 :           this_mask |= GOMP_DIM_MASK (GOMP_DIM_VECTOR);
    2830          342 :           break;
    2831           63 :         case OMP_CLAUSE_SEQ:
    2832           63 :           has_seq = true;
    2833           63 :           break;
    2834           94 :         case OMP_CLAUSE_AUTO:
    2835           94 :           has_auto = true;
    2836           94 :           break;
    2837              :         default:
    2838              :           break;
    2839              :         }
    2840              :     }
    2841              : 
    2842         4062 :   if (checking)
    2843              :     {
    2844         2332 :       if (has_seq && (this_mask || has_auto))
    2845           36 :         error_at (gimple_location (stmt), "%<seq%> overrides other"
    2846              :                   " OpenACC loop specifiers");
    2847         2296 :       else if (has_auto && this_mask)
    2848           30 :         error_at (gimple_location (stmt), "%<auto%> conflicts with other"
    2849              :                   " OpenACC loop specifiers");
    2850              : 
    2851         2332 :       if (this_mask & outer_mask)
    2852           15 :         error_at (gimple_location (stmt), "inner loop uses same"
    2853              :                   " OpenACC parallelism as containing loop");
    2854              :     }
    2855              : 
    2856         4062 :   return outer_mask | this_mask;
    2857              : }
    2858              : 
    2859              : /* Scan a GIMPLE_OMP_FOR.  */
    2860              : 
    2861              : static omp_context *
    2862        52983 : scan_omp_for (gomp_for *stmt, omp_context *outer_ctx)
    2863              : {
    2864        52983 :   omp_context *ctx;
    2865        52983 :   size_t i;
    2866        52983 :   tree clauses = gimple_omp_for_clauses (stmt);
    2867              : 
    2868        52983 :   ctx = new_omp_context (stmt, outer_ctx);
    2869              : 
    2870        52983 :   if (is_gimple_omp_oacc (stmt))
    2871              :     {
    2872        12446 :       omp_context *tgt = enclosing_target_ctx (outer_ctx);
    2873              : 
    2874        12446 :       if (!(tgt && is_oacc_kernels (tgt)))
    2875        32251 :         for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
    2876              :           {
    2877        22137 :             tree c_op0;
    2878        22137 :             switch (OMP_CLAUSE_CODE (c))
    2879              :               {
    2880         1952 :               case OMP_CLAUSE_GANG:
    2881         1952 :                 c_op0 = OMP_CLAUSE_GANG_EXPR (c);
    2882         1952 :                 break;
    2883              : 
    2884         1426 :               case OMP_CLAUSE_WORKER:
    2885         1426 :                 c_op0 = OMP_CLAUSE_WORKER_EXPR (c);
    2886         1426 :                 break;
    2887              : 
    2888         1627 :               case OMP_CLAUSE_VECTOR:
    2889         1627 :                 c_op0 = OMP_CLAUSE_VECTOR_EXPR (c);
    2890         1627 :                 break;
    2891              : 
    2892        17132 :               default:
    2893        17132 :                 continue;
    2894              :               }
    2895              : 
    2896         5005 :             if (c_op0)
    2897              :               {
    2898              :                 /* By construction, this is impossible for OpenACC 'kernels'
    2899              :                    decomposed parts.  */
    2900          210 :                 gcc_assert (!(tgt && is_oacc_kernels_decomposed_part (tgt)));
    2901              : 
    2902          210 :                 error_at (OMP_CLAUSE_LOCATION (c),
    2903              :                           "argument not permitted on %qs clause",
    2904          210 :                           omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
    2905          210 :                 if (tgt)
    2906          180 :                   inform (gimple_location (tgt->stmt),
    2907              :                           "enclosing parent compute construct");
    2908           30 :                 else if (oacc_get_fn_attrib (current_function_decl))
    2909           30 :                   inform (DECL_SOURCE_LOCATION (current_function_decl),
    2910              :                           "enclosing routine");
    2911              :                 else
    2912            0 :                   gcc_unreachable ();
    2913              :               }
    2914              :           }
    2915              : 
    2916        12446 :       if (tgt && is_oacc_kernels (tgt))
    2917         2332 :         check_oacc_kernel_gwv (stmt, ctx);
    2918              : 
    2919              :       /* Collect all variables named in reductions on this loop.  Ensure
    2920              :          that, if this loop has a reduction on some variable v, and there is
    2921              :          a reduction on v somewhere in an outer context, then there is a
    2922              :          reduction on v on all intervening loops as well.  */
    2923        12446 :       tree local_reduction_clauses = NULL;
    2924        39374 :       for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
    2925              :         {
    2926        26928 :           if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
    2927         4712 :             local_reduction_clauses
    2928         4712 :               = tree_cons (NULL, c, local_reduction_clauses);
    2929              :         }
    2930        12446 :       if (ctx->outer_reduction_clauses == NULL && ctx->outer != NULL)
    2931        11920 :         ctx->outer_reduction_clauses
    2932        11920 :           = chainon (unshare_expr (ctx->outer->local_reduction_clauses),
    2933              :                      ctx->outer->outer_reduction_clauses);
    2934        12446 :       tree outer_reduction_clauses = ctx->outer_reduction_clauses;
    2935        12446 :       tree local_iter = local_reduction_clauses;
    2936        17158 :       for (; local_iter; local_iter = TREE_CHAIN (local_iter))
    2937              :         {
    2938         4712 :           tree local_clause = TREE_VALUE (local_iter);
    2939         4712 :           tree local_var = OMP_CLAUSE_DECL (local_clause);
    2940         4712 :           tree_code local_op = OMP_CLAUSE_REDUCTION_CODE (local_clause);
    2941         4712 :           bool have_outer_reduction = false;
    2942         4712 :           tree ctx_iter = outer_reduction_clauses;
    2943         5826 :           for (; ctx_iter; ctx_iter = TREE_CHAIN (ctx_iter))
    2944              :             {
    2945         3325 :               tree outer_clause = TREE_VALUE (ctx_iter);
    2946         3325 :               tree outer_var = OMP_CLAUSE_DECL (outer_clause);
    2947         3325 :               tree_code outer_op = OMP_CLAUSE_REDUCTION_CODE (outer_clause);
    2948         3325 :               if (outer_var == local_var && outer_op != local_op)
    2949              :                 {
    2950          535 :                   if (warning_at (OMP_CLAUSE_LOCATION (local_clause),
    2951          535 :                                   OPT_Wopenmp, "conflicting reduction "
    2952              :                                                "operations for %qE",
    2953              :                                   local_var))
    2954          535 :                     inform (OMP_CLAUSE_LOCATION (outer_clause),
    2955              :                             "location of the previous reduction for %qE",
    2956              :                             outer_var);
    2957              :                 }
    2958         3325 :               if (outer_var == local_var)
    2959              :                 {
    2960              :                   have_outer_reduction = true;
    2961              :                   break;
    2962              :                 }
    2963              :             }
    2964         4712 :           if (have_outer_reduction)
    2965              :             {
    2966              :               /* There is a reduction on outer_var both on this loop and on
    2967              :                  some enclosing loop.  Walk up the context tree until such a
    2968              :                  loop with a reduction on outer_var is found, and complain
    2969              :                  about all intervening loops that do not have such a
    2970              :                  reduction.  */
    2971         2211 :               struct omp_context *curr_loop = ctx->outer;
    2972         2211 :               bool found = false;
    2973         2826 :               while (curr_loop != NULL)
    2974              :                 {
    2975         2826 :                   tree curr_iter = curr_loop->local_reduction_clauses;
    2976         3460 :                   for (; curr_iter; curr_iter = TREE_CHAIN (curr_iter))
    2977              :                     {
    2978         2845 :                       tree curr_clause = TREE_VALUE (curr_iter);
    2979         2845 :                       tree curr_var = OMP_CLAUSE_DECL (curr_clause);
    2980         2845 :                       if (curr_var == local_var)
    2981              :                         {
    2982              :                           found = true;
    2983              :                           break;
    2984              :                         }
    2985              :                     }
    2986         2826 :                   if (!found)
    2987          615 :                     warning_at (gimple_location (curr_loop->stmt), OPT_Wopenmp,
    2988              :                                 "nested loop in reduction needs "
    2989              :                                 "reduction clause for %qE",
    2990              :                                 local_var);
    2991              :                   else
    2992              :                     break;
    2993          615 :                   curr_loop = curr_loop->outer;
    2994              :                 }
    2995              :             }
    2996              :         }
    2997        12446 :       ctx->local_reduction_clauses = local_reduction_clauses;
    2998        12446 :       ctx->outer_reduction_clauses
    2999        12446 :         = chainon (unshare_expr (ctx->local_reduction_clauses),
    3000              :                    ctx->outer_reduction_clauses);
    3001              : 
    3002        12446 :       if (tgt && is_oacc_kernels (tgt))
    3003              :         {
    3004              :           /* Strip out reductions, as they are not handled yet.  */
    3005              :           tree *prev_ptr = &clauses;
    3006              : 
    3007         7123 :           while (tree probe = *prev_ptr)
    3008              :             {
    3009         4791 :               tree *next_ptr = &OMP_CLAUSE_CHAIN (probe);
    3010              : 
    3011         4791 :               if (OMP_CLAUSE_CODE (probe) == OMP_CLAUSE_REDUCTION)
    3012          718 :                 *prev_ptr = *next_ptr;
    3013              :               else
    3014              :                 prev_ptr = next_ptr;
    3015              :             }
    3016              : 
    3017         2332 :           gimple_omp_for_set_clauses (stmt, clauses);
    3018              :         }
    3019              :     }
    3020              : 
    3021        52983 :   scan_sharing_clauses (clauses, ctx);
    3022              : 
    3023        52983 :   scan_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
    3024       180987 :   for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
    3025              :     {
    3026        75021 :       scan_omp_op (gimple_omp_for_index_ptr (stmt, i), ctx);
    3027        75021 :       scan_omp_op (gimple_omp_for_initial_ptr (stmt, i), ctx);
    3028        75021 :       scan_omp_op (gimple_omp_for_final_ptr (stmt, i), ctx);
    3029        75021 :       scan_omp_op (gimple_omp_for_incr_ptr (stmt, i), ctx);
    3030              :     }
    3031        52983 :   scan_omp (gimple_omp_body_ptr (stmt), ctx);
    3032        52983 :   return ctx;
    3033              : }
    3034              : 
    3035              : /* Duplicate #pragma omp simd, one for SIMT, another one for SIMD.  */
    3036              : 
    3037              : static void
    3038            0 : scan_omp_simd (gimple_stmt_iterator *gsi, gomp_for *stmt,
    3039              :                omp_context *outer_ctx)
    3040              : {
    3041            0 :   gbind *bind = gimple_build_bind (NULL, NULL, NULL);
    3042            0 :   gsi_replace (gsi, bind, false);
    3043            0 :   gimple_seq seq = NULL;
    3044            0 :   gimple *g = gimple_build_call_internal (IFN_GOMP_USE_SIMT, 0);
    3045            0 :   tree cond = create_tmp_var_raw (integer_type_node);
    3046            0 :   DECL_CONTEXT (cond) = current_function_decl;
    3047            0 :   DECL_SEEN_IN_BIND_EXPR_P (cond) = 1;
    3048            0 :   gimple_bind_set_vars (bind, cond);
    3049            0 :   gimple_call_set_lhs (g, cond);
    3050            0 :   gimple_seq_add_stmt (&seq, g);
    3051            0 :   tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
    3052            0 :   tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
    3053            0 :   tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
    3054            0 :   g = gimple_build_cond (NE_EXPR, cond, integer_zero_node, lab1, lab2);
    3055            0 :   gimple_seq_add_stmt (&seq, g);
    3056            0 :   g = gimple_build_label (lab1);
    3057            0 :   gimple_seq_add_stmt (&seq, g);
    3058            0 :   gimple_seq new_seq = copy_gimple_seq_and_replace_locals (stmt);
    3059            0 :   gomp_for *new_stmt = as_a <gomp_for *> (new_seq);
    3060            0 :   tree clause = build_omp_clause (gimple_location (stmt), OMP_CLAUSE__SIMT_);
    3061            0 :   OMP_CLAUSE_CHAIN (clause) = gimple_omp_for_clauses (new_stmt);
    3062            0 :   gimple_omp_for_set_clauses (new_stmt, clause);
    3063            0 :   gimple_seq_add_stmt (&seq, new_stmt);
    3064            0 :   g = gimple_build_goto (lab3);
    3065            0 :   gimple_seq_add_stmt (&seq, g);
    3066            0 :   g = gimple_build_label (lab2);
    3067            0 :   gimple_seq_add_stmt (&seq, g);
    3068            0 :   gimple_seq_add_stmt (&seq, stmt);
    3069            0 :   g = gimple_build_label (lab3);
    3070            0 :   gimple_seq_add_stmt (&seq, g);
    3071            0 :   gimple_bind_set_body (bind, seq);
    3072            0 :   update_stmt (bind);
    3073            0 :   scan_omp_for (new_stmt, outer_ctx);
    3074            0 :   scan_omp_for (stmt, outer_ctx)->simt_stmt = new_stmt;
    3075            0 : }
    3076              : 
    3077              : static tree omp_find_scan (gimple_stmt_iterator *, bool *,
    3078              :                            struct walk_stmt_info *);
    3079              : static omp_context *maybe_lookup_ctx (gimple *);
    3080              : 
    3081              : /* Duplicate #pragma omp simd, one for the scan input phase loop and one
    3082              :    for scan phase loop.  */
    3083              : 
    3084              : static void
    3085           91 : scan_omp_simd_scan (gimple_stmt_iterator *gsi, gomp_for *stmt,
    3086              :                     omp_context *outer_ctx)
    3087              : {
    3088              :   /* The only change between inclusive and exclusive scan will be
    3089              :      within the first simd loop, so just use inclusive in the
    3090              :      worksharing loop.  */
    3091           91 :   outer_ctx->scan_inclusive = true;
    3092           91 :   tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_INCLUSIVE);
    3093           91 :   OMP_CLAUSE_DECL (c) = integer_zero_node;
    3094              : 
    3095           91 :   gomp_scan *input_stmt = gimple_build_omp_scan (NULL, NULL_TREE);
    3096           91 :   gomp_scan *scan_stmt = gimple_build_omp_scan (NULL, c);
    3097           91 :   gsi_replace (gsi, input_stmt, false);
    3098           91 :   gimple_seq input_body = NULL;
    3099           91 :   gimple_seq_add_stmt (&input_body, stmt);
    3100           91 :   gsi_insert_after (gsi, scan_stmt, GSI_NEW_STMT);
    3101              : 
    3102           91 :   gimple_stmt_iterator input1_gsi = gsi_none ();
    3103           91 :   struct walk_stmt_info wi;
    3104           91 :   memset (&wi, 0, sizeof (wi));
    3105           91 :   wi.val_only = true;
    3106           91 :   wi.info = (void *) &input1_gsi;
    3107           91 :   walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), omp_find_scan, NULL, &wi);
    3108           91 :   gcc_assert (!gsi_end_p (input1_gsi));
    3109              : 
    3110           91 :   gimple *input_stmt1 = gsi_stmt (input1_gsi);
    3111           91 :   gsi_next (&input1_gsi);
    3112           91 :   gimple *scan_stmt1 = gsi_stmt (input1_gsi);
    3113           91 :   gcc_assert (scan_stmt1 && gimple_code (scan_stmt1) == GIMPLE_OMP_SCAN);
    3114           91 :   c = gimple_omp_scan_clauses (as_a <gomp_scan *> (scan_stmt1));
    3115          182 :   if (c && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_EXCLUSIVE)
    3116              :     std::swap (input_stmt1, scan_stmt1);
    3117              : 
    3118           91 :   gimple_seq input_body1 = gimple_omp_body (input_stmt1);
    3119           91 :   gimple_omp_set_body (input_stmt1, NULL);
    3120              : 
    3121           91 :   gimple_seq scan_body = copy_gimple_seq_and_replace_locals (stmt);
    3122           91 :   gomp_for *new_stmt = as_a <gomp_for *> (scan_body);
    3123              : 
    3124           91 :   gimple_omp_set_body (input_stmt1, input_body1);
    3125           91 :   gimple_omp_set_body (scan_stmt1, NULL);
    3126              : 
    3127           91 :   gimple_stmt_iterator input2_gsi = gsi_none ();
    3128           91 :   memset (&wi, 0, sizeof (wi));
    3129           91 :   wi.val_only = true;
    3130           91 :   wi.info = (void *) &input2_gsi;
    3131           91 :   walk_gimple_seq_mod (gimple_omp_body_ptr (new_stmt), omp_find_scan,
    3132              :                        NULL, &wi);
    3133           91 :   gcc_assert (!gsi_end_p (input2_gsi));
    3134              : 
    3135           91 :   gimple *input_stmt2 = gsi_stmt (input2_gsi);
    3136           91 :   gsi_next (&input2_gsi);
    3137           91 :   gimple *scan_stmt2 = gsi_stmt (input2_gsi);
    3138           91 :   gcc_assert (scan_stmt2 && gimple_code (scan_stmt2) == GIMPLE_OMP_SCAN);
    3139          182 :   if (c && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_EXCLUSIVE)
    3140              :     std::swap (input_stmt2, scan_stmt2);
    3141              : 
    3142           91 :   gimple_omp_set_body (input_stmt2, NULL);
    3143              : 
    3144           91 :   gimple_omp_set_body (input_stmt, input_body);
    3145           91 :   gimple_omp_set_body (scan_stmt, scan_body);
    3146              : 
    3147           91 :   omp_context *ctx = new_omp_context (input_stmt, outer_ctx);
    3148           91 :   scan_omp (gimple_omp_body_ptr (input_stmt), ctx);
    3149              : 
    3150           91 :   ctx = new_omp_context (scan_stmt, outer_ctx);
    3151           91 :   scan_omp (gimple_omp_body_ptr (scan_stmt), ctx);
    3152              : 
    3153           91 :   maybe_lookup_ctx (new_stmt)->for_simd_scan_phase = true;
    3154           91 : }
    3155              : 
    3156              : /* Scan an OpenMP sections directive.  */
    3157              : 
    3158              : static void
    3159          589 : scan_omp_sections (gomp_sections *stmt, omp_context *outer_ctx)
    3160              : {
    3161          589 :   omp_context *ctx;
    3162              : 
    3163          589 :   ctx = new_omp_context (stmt, outer_ctx);
    3164          589 :   scan_sharing_clauses (gimple_omp_sections_clauses (stmt), ctx);
    3165          589 :   scan_omp (gimple_omp_body_ptr (stmt), ctx);
    3166          589 : }
    3167              : 
    3168              : /* Scan an OpenMP single directive.  */
    3169              : 
    3170              : static void
    3171         1252 : scan_omp_single (gomp_single *stmt, omp_context *outer_ctx)
    3172              : {
    3173         1252 :   omp_context *ctx;
    3174         1252 :   tree name;
    3175              : 
    3176         1252 :   ctx = new_omp_context (stmt, outer_ctx);
    3177         1252 :   ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
    3178         1252 :   ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
    3179         1252 :   name = create_tmp_var_name (".omp_copy_s");
    3180         1252 :   name = build_decl (gimple_location (stmt),
    3181              :                      TYPE_DECL, name, ctx->record_type);
    3182         1252 :   TYPE_NAME (ctx->record_type) = name;
    3183              : 
    3184         1252 :   scan_sharing_clauses (gimple_omp_single_clauses (stmt), ctx);
    3185         1252 :   scan_omp (gimple_omp_body_ptr (stmt), ctx);
    3186              : 
    3187         1252 :   if (TYPE_FIELDS (ctx->record_type) == NULL)
    3188         1102 :     ctx->record_type = NULL;
    3189              :   else
    3190          150 :     layout_type (ctx->record_type);
    3191         1252 : }
    3192              : 
    3193              : /* Scan a GIMPLE_OMP_TARGET.  */
    3194              : 
    3195              : static void
    3196        42078 : scan_omp_target (gomp_target *stmt, omp_context *outer_ctx)
    3197              : {
    3198        42078 :   omp_context *ctx;
    3199        42078 :   tree name;
    3200        42078 :   bool offloaded = is_gimple_omp_offloaded (stmt);
    3201        42078 :   tree clauses = gimple_omp_target_clauses (stmt);
    3202              : 
    3203        42078 :   ctx = new_omp_context (stmt, outer_ctx);
    3204        42078 :   ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
    3205        42078 :   ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
    3206        42078 :   name = create_tmp_var_name (".omp_data_t");
    3207        42078 :   name = build_decl (gimple_location (stmt),
    3208              :                      TYPE_DECL, name, ctx->record_type);
    3209        42078 :   DECL_ARTIFICIAL (name) = 1;
    3210        42078 :   DECL_NAMELESS (name) = 1;
    3211        42078 :   TYPE_NAME (ctx->record_type) = name;
    3212        42078 :   TYPE_ARTIFICIAL (ctx->record_type) = 1;
    3213              : 
    3214        42078 :   if (offloaded)
    3215              :     {
    3216        24963 :       tree c = omp_find_clause (clauses, OMP_CLAUSE_DEVICE_TYPE);
    3217        24963 :       bool host_only
    3218        24963 :         = c && OMP_CLAUSE_DEVICE_TYPE_KIND (c) == OMP_CLAUSE_DEVICE_TYPE_HOST;
    3219        24963 :       create_omp_child_function (ctx, false, host_only);
    3220        24963 :       gimple_omp_target_set_child_fn (stmt, ctx->cb.dst_fn);
    3221              :     }
    3222              : 
    3223        42078 :   scan_sharing_clauses (clauses, ctx);
    3224        42078 :   scan_omp (gimple_omp_body_ptr (stmt), ctx);
    3225              : 
    3226        42078 :   if (TYPE_FIELDS (ctx->record_type) == NULL)
    3227         7798 :     ctx->record_type = ctx->receiver_decl = NULL;
    3228              :   else
    3229              :     {
    3230        34280 :       TYPE_FIELDS (ctx->record_type)
    3231        34280 :         = nreverse (TYPE_FIELDS (ctx->record_type));
    3232        34280 :       if (flag_checking)
    3233              :         {
    3234        34280 :           unsigned int align = DECL_ALIGN (TYPE_FIELDS (ctx->record_type));
    3235        34280 :           for (tree field = TYPE_FIELDS (ctx->record_type);
    3236       131941 :                field;
    3237        97661 :                field = DECL_CHAIN (field))
    3238        97661 :             gcc_assert (DECL_ALIGN (field) == align);
    3239              :         }
    3240        34280 :       layout_type (ctx->record_type);
    3241        34280 :       if (offloaded)
    3242        17720 :         fixup_child_record_type (ctx);
    3243              :     }
    3244              : 
    3245        42078 :   if (ctx->teams_nested_p && ctx->nonteams_nested_p)
    3246              :     {
    3247            4 :       error_at (gimple_location (stmt),
    3248              :                 "%<target%> construct with nested %<teams%> construct "
    3249              :                 "contains directives outside of the %<teams%> construct");
    3250            4 :       gimple_omp_set_body (stmt, gimple_build_bind (NULL, NULL, NULL));
    3251              :     }
    3252        42078 : }
    3253              : 
    3254              : /* Scan an OpenMP teams directive.  */
    3255              : 
    3256              : static void
    3257         8878 : scan_omp_teams (gomp_teams *stmt, omp_context *outer_ctx)
    3258              : {
    3259         8878 :   omp_context *ctx = new_omp_context (stmt, outer_ctx);
    3260              : 
    3261         8878 :   if (!gimple_omp_teams_host (stmt))
    3262              :     {
    3263         6157 :       scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
    3264         6157 :       scan_omp (gimple_omp_body_ptr (stmt), ctx);
    3265         6157 :       return;
    3266              :     }
    3267         2721 :   taskreg_contexts.safe_push (ctx);
    3268         2721 :   gcc_assert (taskreg_nesting_level == 1);
    3269         2721 :   ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
    3270         2721 :   ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
    3271         2721 :   tree name = create_tmp_var_name (".omp_data_s");
    3272         2721 :   name = build_decl (gimple_location (stmt),
    3273              :                      TYPE_DECL, name, ctx->record_type);
    3274         2721 :   DECL_ARTIFICIAL (name) = 1;
    3275         2721 :   DECL_NAMELESS (name) = 1;
    3276         2721 :   TYPE_NAME (ctx->record_type) = name;
    3277         2721 :   TYPE_ARTIFICIAL (ctx->record_type) = 1;
    3278         2721 :   create_omp_child_function (ctx, false, false);
    3279         2721 :   gimple_omp_teams_set_child_fn (stmt, ctx->cb.dst_fn);
    3280              : 
    3281         2721 :   scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
    3282         2721 :   scan_omp (gimple_omp_body_ptr (stmt), ctx);
    3283              : 
    3284         2721 :   if (TYPE_FIELDS (ctx->record_type) == NULL)
    3285         1455 :     ctx->record_type = ctx->receiver_decl = NULL;
    3286              : }
    3287              : 
    3288              : /* Check nesting restrictions.  */
    3289              : static bool
    3290       158419 : check_omp_nesting_restrictions (gimple *stmt, omp_context *ctx)
    3291              : {
    3292       158419 :   tree c;
    3293              : 
    3294              :   /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
    3295              :      inside an OpenACC CTX.  */
    3296       158419 :   if (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
    3297       158419 :       || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE)
    3298              :     /* ..., except for the atomic codes that OpenACC shares with OpenMP.  */
    3299              :     ;
    3300       284632 :   else if (!(is_gimple_omp (stmt)
    3301       140187 :              && is_gimple_omp_oacc (stmt)))
    3302              :     {
    3303       114181 :       if (oacc_get_fn_attrib (cfun->decl) != NULL)
    3304              :         {
    3305           48 :           error_at (gimple_location (stmt),
    3306              :                     "non-OpenACC construct inside of OpenACC routine");
    3307           48 :           return false;
    3308              :         }
    3309              :       else
    3310       238484 :         for (omp_context *octx = ctx; octx != NULL; octx = octx->outer)
    3311       248878 :           if (is_gimple_omp (octx->stmt)
    3312       124527 :               && is_gimple_omp_oacc (octx->stmt))
    3313              :             {
    3314          176 :               error_at (gimple_location (stmt),
    3315              :                         "non-OpenACC construct inside of OpenACC region");
    3316          176 :               return false;
    3317              :             }
    3318              :     }
    3319              : 
    3320       158195 :   if (ctx != NULL)
    3321              :     {
    3322        84713 :       if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
    3323        84713 :           && gimple_omp_target_kind (ctx->stmt) == GF_OMP_TARGET_KIND_REGION)
    3324              :         {
    3325         8381 :           c = omp_find_clause (gimple_omp_target_clauses (ctx->stmt),
    3326              :                                OMP_CLAUSE_DEVICE);
    3327         8381 :           if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
    3328              :             {
    3329            9 :               error_at (gimple_location (stmt),
    3330              :                         "OpenMP constructs are not allowed in target region "
    3331              :                         "with %<ancestor%>");
    3332            9 :               return false;
    3333              :             }
    3334              : 
    3335         8372 :           if (gimple_code (stmt) == GIMPLE_OMP_TEAMS && !ctx->teams_nested_p)
    3336         6153 :             ctx->teams_nested_p = true;
    3337              :           else
    3338         2219 :             ctx->nonteams_nested_p = true;
    3339              :         }
    3340        84704 :       if (gimple_code (ctx->stmt) == GIMPLE_OMP_SCAN
    3341          188 :           && ctx->outer
    3342        84892 :           && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR)
    3343              :         ctx = ctx->outer;
    3344        84704 :       if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
    3345        28449 :           && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
    3346        86547 :           && !ctx->loop_p)
    3347              :         {
    3348         1623 :           c = NULL_TREE;
    3349         1623 :           if (ctx->order_concurrent
    3350         1623 :               && (gimple_code (stmt) == GIMPLE_OMP_ORDERED
    3351          280 :                   || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
    3352          190 :                   || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE))
    3353              :             {
    3354          210 :               error_at (gimple_location (stmt),
    3355              :                         "OpenMP constructs other than %<parallel%>, %<loop%>"
    3356              :                         " or %<simd%> may not be nested inside a region with"
    3357              :                         " the %<order(concurrent)%> clause");
    3358          210 :               return false;
    3359              :             }
    3360         1413 :           if (gimple_code (stmt) == GIMPLE_OMP_ORDERED)
    3361              :             {
    3362          133 :               c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
    3363          133 :               if (omp_find_clause (c, OMP_CLAUSE_SIMD))
    3364              :                 {
    3365          133 :                   if (omp_find_clause (c, OMP_CLAUSE_THREADS)
    3366          133 :                       && (ctx->outer == NULL
    3367           29 :                           || !gimple_omp_for_combined_into_p (ctx->stmt)
    3368           25 :                           || gimple_code (ctx->outer->stmt) != GIMPLE_OMP_FOR
    3369           25 :                           || (gimple_omp_for_kind (ctx->outer->stmt)
    3370              :                               != GF_OMP_FOR_KIND_FOR)
    3371           25 :                           || !gimple_omp_for_combined_p (ctx->outer->stmt)))
    3372              :                     {
    3373            8 :                       error_at (gimple_location (stmt),
    3374              :                                 "%<ordered simd threads%> must be closely "
    3375              :                                 "nested inside of %<%s simd%> region",
    3376            8 :                                 lang_GNU_Fortran () ? "do" : "for");
    3377            8 :                       return false;
    3378              :                     }
    3379              :                   return true;
    3380              :                 }
    3381              :             }
    3382              :           else if (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
    3383              :                    || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE
    3384              :                    || gimple_code (stmt) == GIMPLE_OMP_SCAN
    3385              :                    || gimple_code (stmt) == GIMPLE_OMP_STRUCTURED_BLOCK)
    3386              :             return true;
    3387              :           else if (gimple_code (stmt) == GIMPLE_OMP_FOR
    3388              :                    && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
    3389              :             return true;
    3390           60 :           error_at (gimple_location (stmt),
    3391              :                     "OpenMP constructs other than "
    3392              :                     "%<ordered simd%>, %<simd%>, %<loop%> or %<atomic%> may "
    3393              :                     "not be nested inside %<simd%> region");
    3394           60 :           return false;
    3395              :         }
    3396        83081 :       else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
    3397              :         {
    3398         5938 :           if ((gimple_code (stmt) != GIMPLE_OMP_FOR
    3399         5614 :                || (gimple_omp_for_kind (stmt) != GF_OMP_FOR_KIND_DISTRIBUTE
    3400           25 :                    && omp_find_clause (gimple_omp_for_clauses (stmt),
    3401              :                                        OMP_CLAUSE_BIND) == NULL_TREE))
    3402         5954 :               && gimple_code (stmt) != GIMPLE_OMP_PARALLEL)
    3403              :             {
    3404          132 :               error_at (gimple_location (stmt),
    3405              :                         "only %<distribute%>, %<parallel%> or %<loop%> "
    3406              :                         "regions are allowed to be strictly nested inside "
    3407              :                         "%<teams%> region");
    3408          132 :               return false;
    3409              :             }
    3410              :         }
    3411        77143 :       else if (ctx->order_concurrent
    3412         1953 :                && gimple_code (stmt) != GIMPLE_OMP_PARALLEL
    3413         1470 :                && (gimple_code (stmt) != GIMPLE_OMP_FOR
    3414         1196 :                    || gimple_omp_for_kind (stmt) != GF_OMP_FOR_KIND_SIMD)
    3415          289 :                && gimple_code (stmt) != GIMPLE_OMP_SCAN
    3416        77432 :                && gimple_code (stmt) != GIMPLE_OMP_STRUCTURED_BLOCK)
    3417              :         {
    3418          285 :           if (ctx->loop_p)
    3419          135 :             error_at (gimple_location (stmt),
    3420              :                       "OpenMP constructs other than %<parallel%>, %<loop%> or "
    3421              :                       "%<simd%> may not be nested inside a %<loop%> region");
    3422              :           else
    3423          150 :             error_at (gimple_location (stmt),
    3424              :                       "OpenMP constructs other than %<parallel%>, %<loop%> or "
    3425              :                       "%<simd%> may not be nested inside a region with "
    3426              :                       "the %<order(concurrent)%> clause");
    3427              :           return false;
    3428              :         }
    3429              :     }
    3430       156146 :   switch (gimple_code (stmt))
    3431              :     {
    3432        53108 :     case GIMPLE_OMP_FOR:
    3433        53108 :       if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_SIMD)
    3434              :         return true;
    3435        42216 :       if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
    3436              :         {
    3437         8306 :           if (ctx != NULL && gimple_code (ctx->stmt) != GIMPLE_OMP_TEAMS)
    3438              :             {
    3439            0 :               error_at (gimple_location (stmt),
    3440              :                         "%<distribute%> region must be strictly nested "
    3441              :                         "inside %<teams%> construct");
    3442            0 :               return false;
    3443              :             }
    3444              :           return true;
    3445              :         }
    3446              :       /* We split taskloop into task and nested taskloop in it.  */
    3447        33910 :       if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_TASKLOOP)
    3448              :         return true;
    3449              :       /* For now, hope this will change and loop bind(parallel) will not
    3450              :          be allowed in lots of contexts.  */
    3451        30776 :       if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR
    3452        49041 :           && omp_find_clause (gimple_omp_for_clauses (stmt), OMP_CLAUSE_BIND))
    3453              :         return true;
    3454        30142 :       if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
    3455              :         {
    3456        12511 :           bool ok = false;
    3457              : 
    3458        12511 :           if (ctx)
    3459        11964 :             switch (gimple_code (ctx->stmt))
    3460              :               {
    3461         4049 :               case GIMPLE_OMP_FOR:
    3462         4049 :                 ok = (gimple_omp_for_kind (ctx->stmt)
    3463              :                       == GF_OMP_FOR_KIND_OACC_LOOP);
    3464         4049 :                 break;
    3465              : 
    3466         7887 :               case GIMPLE_OMP_TARGET:
    3467         7887 :                 switch (gimple_omp_target_kind (ctx->stmt))
    3468              :                   {
    3469              :                   case GF_OMP_TARGET_KIND_OACC_PARALLEL:
    3470              :                   case GF_OMP_TARGET_KIND_OACC_KERNELS:
    3471              :                   case GF_OMP_TARGET_KIND_OACC_SERIAL:
    3472              :                   case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
    3473              :                   case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
    3474              :                     ok = true;
    3475              :                     break;
    3476              : 
    3477              :                   default:
    3478              :                     break;
    3479              :                   }
    3480              : 
    3481              :               default:
    3482              :                 break;
    3483              :               }
    3484          547 :           else if (oacc_get_fn_attrib (current_function_decl))
    3485              :             ok = true;
    3486         4049 :           if (!ok)
    3487              :             {
    3488           65 :               error_at (gimple_location (stmt),
    3489              :                         "OpenACC loop directive must be associated with"
    3490              :                         " an OpenACC compute region");
    3491           65 :               return false;
    3492              :             }
    3493              :         }
    3494              :       /* FALLTHRU */
    3495        34275 :     case GIMPLE_CALL:
    3496        34275 :       if (is_gimple_call (stmt)
    3497        34275 :           && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
    3498              :               == BUILT_IN_GOMP_CANCEL
    3499         3081 :               || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
    3500              :                  == BUILT_IN_GOMP_CANCELLATION_POINT))
    3501              :         {
    3502         1925 :           const char *bad = NULL;
    3503         1925 :           const char *kind = NULL;
    3504         1925 :           const char *construct
    3505         1925 :             = (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
    3506              :                == BUILT_IN_GOMP_CANCEL)
    3507         1925 :               ? "cancel"
    3508          808 :               : "cancellation point";
    3509         1925 :           if (ctx == NULL)
    3510              :             {
    3511           41 :               error_at (gimple_location (stmt), "orphaned %qs construct",
    3512              :                         construct);
    3513           41 :               return false;
    3514              :             }
    3515         1884 :           switch (tree_fits_shwi_p (gimple_call_arg (stmt, 0))
    3516         1884 :                   ? tree_to_shwi (gimple_call_arg (stmt, 0))
    3517              :                   : 0)
    3518              :             {
    3519          546 :             case 1:
    3520          546 :               if (gimple_code (ctx->stmt) != GIMPLE_OMP_PARALLEL)
    3521              :                 bad = "parallel";
    3522          226 :               else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
    3523              :                        == BUILT_IN_GOMP_CANCEL
    3524          226 :                        && !integer_zerop (gimple_call_arg (stmt, 1)))
    3525          177 :                 ctx->cancellable = true;
    3526              :               kind = "parallel";
    3527              :               break;
    3528          427 :             case 2:
    3529          427 :               if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
    3530          427 :                   || gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR)
    3531              :                 bad = "for";
    3532          117 :               else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
    3533              :                        == BUILT_IN_GOMP_CANCEL
    3534          117 :                        && !integer_zerop (gimple_call_arg (stmt, 1)))
    3535              :                 {
    3536          106 :                   ctx->cancellable = true;
    3537          106 :                   if (omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
    3538              :                                        OMP_CLAUSE_NOWAIT))
    3539            5 :                     warning_at (gimple_location (stmt), OPT_Wopenmp,
    3540              :                                 "%<cancel for%> inside "
    3541              :                                 "%<nowait%> for construct");
    3542          106 :                   if (omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
    3543              :                                        OMP_CLAUSE_ORDERED))
    3544            5 :                     warning_at (gimple_location (stmt), OPT_Wopenmp,
    3545              :                                 "%<cancel for%> inside "
    3546              :                                 "%<ordered%> for construct");
    3547              :                 }
    3548              :               kind = "for";
    3549              :               break;
    3550          383 :             case 4:
    3551          383 :               if (gimple_code (ctx->stmt) != GIMPLE_OMP_SECTIONS
    3552          383 :                   && gimple_code (ctx->stmt) != GIMPLE_OMP_SECTION)
    3553              :                 bad = "sections";
    3554           93 :               else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
    3555              :                        == BUILT_IN_GOMP_CANCEL
    3556           93 :                        && !integer_zerop (gimple_call_arg (stmt, 1)))
    3557              :                 {
    3558           73 :                   if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
    3559              :                     {
    3560            0 :                       ctx->cancellable = true;
    3561            0 :                       if (omp_find_clause (gimple_omp_sections_clauses
    3562            0 :                                                                 (ctx->stmt),
    3563              :                                            OMP_CLAUSE_NOWAIT))
    3564            0 :                         warning_at (gimple_location (stmt), OPT_Wopenmp,
    3565              :                                     "%<cancel sections%> inside "
    3566              :                                     "%<nowait%> sections construct");
    3567              :                     }
    3568              :                   else
    3569              :                     {
    3570           73 :                       gcc_assert (ctx->outer
    3571              :                                   && gimple_code (ctx->outer->stmt)
    3572              :                                      == GIMPLE_OMP_SECTIONS);
    3573           73 :                       ctx->outer->cancellable = true;
    3574           73 :                       if (omp_find_clause (gimple_omp_sections_clauses
    3575           73 :                                                         (ctx->outer->stmt),
    3576              :                                            OMP_CLAUSE_NOWAIT))
    3577           10 :                         warning_at (gimple_location (stmt), OPT_Wopenmp,
    3578              :                                     "%<cancel sections%> inside "
    3579              :                                     "%<nowait%> sections construct");
    3580              :                     }
    3581              :                 }
    3582              :               kind = "sections";
    3583              :               break;
    3584          528 :             case 8:
    3585          528 :               if (!is_task_ctx (ctx)
    3586          528 :                   && (!is_taskloop_ctx (ctx)
    3587           54 :                       || ctx->outer == NULL
    3588           54 :                       || !is_task_ctx (ctx->outer)))
    3589              :                 bad = "task";
    3590              :               else
    3591              :                 {
    3592          248 :                   for (omp_context *octx = ctx->outer;
    3593          439 :                        octx; octx = octx->outer)
    3594              :                     {
    3595          425 :                       switch (gimple_code (octx->stmt))
    3596              :                         {
    3597              :                         case GIMPLE_OMP_TASKGROUP:
    3598              :                           break;
    3599           40 :                         case GIMPLE_OMP_TARGET:
    3600           40 :                           if (gimple_omp_target_kind (octx->stmt)
    3601              :                               != GF_OMP_TARGET_KIND_REGION)
    3602           20 :                             continue;
    3603              :                           /* FALLTHRU */
    3604           99 :                         case GIMPLE_OMP_PARALLEL:
    3605           99 :                         case GIMPLE_OMP_TEAMS:
    3606           99 :                           error_at (gimple_location (stmt),
    3607              :                                     "%<%s taskgroup%> construct not closely "
    3608              :                                     "nested inside of %<taskgroup%> region",
    3609              :                                     construct);
    3610           99 :                           return false;
    3611          104 :                         case GIMPLE_OMP_TASK:
    3612          104 :                           if (gimple_omp_task_taskloop_p (octx->stmt)
    3613           94 :                               && octx->outer
    3614          198 :                               && is_taskloop_ctx (octx->outer))
    3615              :                             {
    3616           94 :                               tree clauses
    3617           94 :                                 = gimple_omp_for_clauses (octx->outer->stmt);
    3618           94 :                               if (!omp_find_clause (clauses, OMP_CLAUSE_NOGROUP))
    3619              :                                 break;
    3620              :                             }
    3621           60 :                           continue;
    3622          111 :                         default:
    3623          111 :                           continue;
    3624          171 :                         }
    3625              :                       break;
    3626              :                     }
    3627          149 :                   ctx->cancellable = true;
    3628              :                 }
    3629              :               kind = "taskgroup";
    3630              :               break;
    3631            0 :             default:
    3632            0 :               error_at (gimple_location (stmt), "invalid arguments");
    3633            0 :               return false;
    3634              :             }
    3635          341 :           if (bad)
    3636              :             {
    3637         1200 :               error_at (gimple_location (stmt),
    3638              :                         "%<%s %s%> construct not closely nested inside of %qs",
    3639              :                         construct, kind, bad);
    3640         1200 :               return false;
    3641              :             }
    3642              :         }
    3643              :       /* FALLTHRU */
    3644              :     case GIMPLE_OMP_SECTIONS:
    3645              :     case GIMPLE_OMP_SINGLE:
    3646        55767 :       for (; ctx != NULL; ctx = ctx->outer)
    3647        37636 :         switch (gimple_code (ctx->stmt))
    3648              :           {
    3649         6873 :           case GIMPLE_OMP_FOR:
    3650         6873 :             if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
    3651         6873 :                 && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
    3652              :               break;
    3653              :             /* FALLTHRU */
    3654          778 :           case GIMPLE_OMP_SECTIONS:
    3655          778 :           case GIMPLE_OMP_SINGLE:
    3656          778 :           case GIMPLE_OMP_ORDERED:
    3657          778 :           case GIMPLE_OMP_MASTER:
    3658          778 :           case GIMPLE_OMP_MASKED:
    3659          778 :           case GIMPLE_OMP_TASK:
    3660          778 :           case GIMPLE_OMP_CRITICAL:
    3661          778 :             if (is_gimple_call (stmt))
    3662              :               {
    3663          707 :                 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
    3664              :                     != BUILT_IN_GOMP_BARRIER)
    3665              :                   return true;
    3666           23 :                 error_at (gimple_location (stmt),
    3667              :                           "barrier region may not be closely nested inside "
    3668              :                           "of work-sharing, %<loop%>, %<critical%>, "
    3669              :                           "%<ordered%>, %<master%>, %<masked%>, explicit "
    3670              :                           "%<task%> or %<taskloop%> region");
    3671           23 :                 return false;
    3672              :               }
    3673           71 :             error_at (gimple_location (stmt),
    3674              :                       "work-sharing region may not be closely nested inside "
    3675              :                       "of work-sharing, %<loop%>, %<critical%>, %<ordered%>, "
    3676              :                       "%<master%>, %<masked%>, explicit %<task%> or "
    3677              :                       "%<taskloop%> region");
    3678           71 :             return false;
    3679              :           case GIMPLE_OMP_PARALLEL:
    3680              :           case GIMPLE_OMP_TEAMS:
    3681              :             return true;
    3682        13437 :           case GIMPLE_OMP_TARGET:
    3683        13437 :             if (gimple_omp_target_kind (ctx->stmt)
    3684              :                 == GF_OMP_TARGET_KIND_REGION)
    3685              :               return true;
    3686              :             break;
    3687              :           default:
    3688              :             break;
    3689              :           }
    3690              :       break;
    3691              :     case GIMPLE_OMP_MASTER:
    3692              :     case GIMPLE_OMP_MASKED:
    3693         1526 :       for (; ctx != NULL; ctx = ctx->outer)
    3694         1017 :         switch (gimple_code (ctx->stmt))
    3695              :           {
    3696           14 :           case GIMPLE_OMP_FOR:
    3697           14 :             if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
    3698           14 :                 && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
    3699              :               break;
    3700              :             /* FALLTHRU */
    3701           30 :           case GIMPLE_OMP_SECTIONS:
    3702           30 :           case GIMPLE_OMP_SINGLE:
    3703           30 :           case GIMPLE_OMP_TASK:
    3704           60 :             error_at (gimple_location (stmt),
    3705              :                       "%qs region may not be closely nested inside "
    3706              :                       "of work-sharing, %<loop%>, explicit %<task%> or "
    3707              :                       "%<taskloop%> region",
    3708           30 :                       gimple_code (stmt) == GIMPLE_OMP_MASTER
    3709              :                       ? "master" : "masked");
    3710           30 :             return false;
    3711              :           case GIMPLE_OMP_PARALLEL:
    3712              :           case GIMPLE_OMP_TEAMS:
    3713              :             return true;
    3714           10 :           case GIMPLE_OMP_TARGET:
    3715           10 :             if (gimple_omp_target_kind (ctx->stmt)
    3716              :                 == GF_OMP_TARGET_KIND_REGION)
    3717              :               return true;
    3718              :             break;
    3719              :           default:
    3720              :             break;
    3721              :           }
    3722              :       break;
    3723              :     case GIMPLE_OMP_SCOPE:
    3724          302 :       for (; ctx != NULL; ctx = ctx->outer)
    3725          183 :         switch (gimple_code (ctx->stmt))
    3726              :           {
    3727            7 :           case GIMPLE_OMP_FOR:
    3728            7 :             if (gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR
    3729            7 :                 && gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_TASKLOOP)
    3730              :               break;
    3731              :             /* FALLTHRU */
    3732           25 :           case GIMPLE_OMP_SECTIONS:
    3733           25 :           case GIMPLE_OMP_SINGLE:
    3734           25 :           case GIMPLE_OMP_TASK:
    3735           25 :           case GIMPLE_OMP_CRITICAL:
    3736           25 :           case GIMPLE_OMP_ORDERED:
    3737           25 :           case GIMPLE_OMP_MASTER:
    3738           25 :           case GIMPLE_OMP_MASKED:
    3739           25 :             error_at (gimple_location (stmt),
    3740              :                       "%<scope%> region may not be closely nested inside "
    3741              :                       "of work-sharing, %<loop%>, explicit %<task%>, "
    3742              :                       "%<taskloop%>, %<critical%>, %<ordered%>, %<master%>, "
    3743              :                       "or %<masked%> region");
    3744           25 :             return false;
    3745              :           case GIMPLE_OMP_PARALLEL:
    3746              :           case GIMPLE_OMP_TEAMS:
    3747              :             return true;
    3748            5 :           case GIMPLE_OMP_TARGET:
    3749            5 :             if (gimple_omp_target_kind (ctx->stmt)
    3750              :                 == GF_OMP_TARGET_KIND_REGION)
    3751              :               return true;
    3752              :             break;
    3753              :           default:
    3754              :             break;
    3755              :           }
    3756              :       break;
    3757         5655 :     case GIMPLE_OMP_TASK:
    3758        23275 :       for (c = gimple_omp_task_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
    3759        17624 :         if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS)
    3760              :           {
    3761            4 :             enum omp_clause_doacross_kind kind = OMP_CLAUSE_DOACROSS_KIND (c);
    3762            4 :             error_at (OMP_CLAUSE_LOCATION (c),
    3763              :                       "%<%s(%s)%> is only allowed in %<omp ordered%>",
    3764            4 :                       OMP_CLAUSE_DOACROSS_DEPEND (c) ? "depend" : "doacross",
    3765              :                       kind == OMP_CLAUSE_DOACROSS_SOURCE ? "source" : "sink");
    3766            4 :             return false;
    3767              :           }
    3768              :       break;
    3769         1635 :     case GIMPLE_OMP_ORDERED:
    3770         2881 :       for (c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
    3771         2881 :            c; c = OMP_CLAUSE_CHAIN (c))
    3772              :         {
    3773         1262 :           if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DOACROSS)
    3774              :             {
    3775          173 :               if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
    3776              :                 {
    3777            4 :                   error_at (OMP_CLAUSE_LOCATION (c),
    3778              :                             "invalid depend kind in omp %<ordered%> %<depend%>");
    3779            4 :                   return false;
    3780              :                 }
    3781          169 :               gcc_assert (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_THREADS
    3782              :                           || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SIMD);
    3783          169 :               continue;
    3784              :             }
    3785              : 
    3786         1089 :           tree oclause;
    3787              :           /* Look for containing ordered(N) loop.  */
    3788         1089 :           if (ctx == NULL
    3789         1077 :               || gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
    3790         1089 :               || (oclause
    3791         1077 :                   = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
    3792              :                                      OMP_CLAUSE_ORDERED)) == NULL_TREE)
    3793              :             {
    3794           12 :               error_at (OMP_CLAUSE_LOCATION (c),
    3795              :                         "%<ordered%> construct with %<depend%> clause "
    3796              :                         "must be closely nested inside an %<ordered%> loop");
    3797           12 :               return false;
    3798              :             }
    3799              :         }
    3800         1619 :       c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
    3801         1619 :       if (omp_find_clause (c, OMP_CLAUSE_SIMD))
    3802              :         {
    3803              :           /* ordered simd must be closely nested inside of simd region,
    3804              :              and simd region must not encounter constructs other than
    3805              :              ordered simd, therefore ordered simd may be either orphaned,
    3806              :              or ctx->stmt must be simd.  The latter case is handled already
    3807              :              earlier.  */
    3808           36 :           if (ctx != NULL)
    3809              :             {
    3810           10 :               error_at (gimple_location (stmt),
    3811              :                         "%<ordered%> %<simd%> must be closely nested inside "
    3812              :                         "%<simd%> region");
    3813           10 :               return false;
    3814              :             }
    3815              :         }
    3816         1609 :       for (; ctx != NULL; ctx = ctx->outer)
    3817         1474 :         switch (gimple_code (ctx->stmt))
    3818              :           {
    3819           38 :           case GIMPLE_OMP_CRITICAL:
    3820           38 :           case GIMPLE_OMP_TASK:
    3821           38 :           case GIMPLE_OMP_ORDERED:
    3822           38 :           ordered_in_taskloop:
    3823           38 :             error_at (gimple_location (stmt),
    3824              :                       "%<ordered%> region may not be closely nested inside "
    3825              :                       "of %<critical%>, %<ordered%>, explicit %<task%> or "
    3826              :                       "%<taskloop%> region");
    3827           38 :             return false;
    3828         1416 :           case GIMPLE_OMP_FOR:
    3829         1416 :             if (gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_TASKLOOP)
    3830           10 :               goto ordered_in_taskloop;
    3831         1406 :             tree o;
    3832         1406 :             o = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
    3833              :                                  OMP_CLAUSE_ORDERED);
    3834         1406 :             if (o == NULL)
    3835              :               {
    3836           22 :                 error_at (gimple_location (stmt),
    3837              :                           "%<ordered%> region must be closely nested inside "
    3838              :                           "a loop region with an %<ordered%> clause");
    3839           22 :                 return false;
    3840              :               }
    3841         1384 :             if (!gimple_omp_ordered_standalone_p (stmt))
    3842              :               {
    3843          375 :                 if (OMP_CLAUSE_ORDERED_DOACROSS (o))
    3844              :                   {
    3845           10 :                     error_at (gimple_location (stmt),
    3846              :                               "%<ordered%> construct without %<doacross%> or "
    3847              :                               "%<depend%> clauses must not have the same "
    3848              :                               "binding region as %<ordered%> construct with "
    3849              :                               "those clauses");
    3850           10 :                     return false;
    3851              :                   }
    3852          365 :                 else if (OMP_CLAUSE_ORDERED_EXPR (o))
    3853              :                   {
    3854           23 :                     tree co
    3855           23 :                       = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
    3856              :                                          OMP_CLAUSE_COLLAPSE);
    3857           23 :                     HOST_WIDE_INT
    3858           23 :                       o_n = tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (o));
    3859           23 :                     HOST_WIDE_INT c_n = 1;
    3860           23 :                     if (co)
    3861            5 :                       c_n = tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (co));
    3862           23 :                     if (o_n != c_n)
    3863              :                       {
    3864           10 :                         error_at (gimple_location (stmt),
    3865              :                                   "%<ordered%> construct without %<doacross%> "
    3866              :                                   "or %<depend%> clauses binds to loop where "
    3867              :                                   "%<collapse%> argument %wd is different from "
    3868              :                                   "%<ordered%> argument %wd", c_n, o_n);
    3869           10 :                         return false;
    3870              :                       }
    3871              :                   }
    3872              :               }
    3873              :             return true;
    3874           10 :           case GIMPLE_OMP_TARGET:
    3875           10 :             if (gimple_omp_target_kind (ctx->stmt)
    3876              :                 != GF_OMP_TARGET_KIND_REGION)
    3877              :               break;
    3878              :             /* FALLTHRU */
    3879           30 :           case GIMPLE_OMP_PARALLEL:
    3880           30 :           case GIMPLE_OMP_TEAMS:
    3881           30 :             error_at (gimple_location (stmt),
    3882              :                       "%<ordered%> region must be closely nested inside "
    3883              :                       "a loop region with an %<ordered%> clause");
    3884           30 :             return false;
    3885              :           default:
    3886              :             break;
    3887              :           }
    3888              :       break;
    3889          468 :     case GIMPLE_OMP_CRITICAL:
    3890          468 :       {
    3891          468 :         tree this_stmt_name
    3892          468 :           = gimple_omp_critical_name (as_a <gomp_critical *> (stmt));
    3893          905 :         for (; ctx != NULL; ctx = ctx->outer)
    3894          443 :           if (gomp_critical *other_crit
    3895          462 :                 = dyn_cast <gomp_critical *> (ctx->stmt))
    3896           25 :             if (this_stmt_name == gimple_omp_critical_name (other_crit))
    3897              :               {
    3898            6 :                 error_at (gimple_location (stmt),
    3899              :                           "%<critical%> region may not be nested inside "
    3900              :                            "a %<critical%> region with the same name");
    3901            6 :                 return false;
    3902              :               }
    3903              :       }
    3904              :       break;
    3905         8922 :     case GIMPLE_OMP_TEAMS:
    3906         8922 :       if (ctx == NULL)
    3907              :         break;
    3908         6201 :       else if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET
    3909         6201 :                || (gimple_omp_target_kind (ctx->stmt)
    3910              :                    != GF_OMP_TARGET_KIND_REGION))
    3911              :         {
    3912              :           /* Teams construct can appear either strictly nested inside of
    3913              :              target construct with no intervening stmts, or can be encountered
    3914              :              only by initial task (so must not appear inside any OpenMP
    3915              :              construct.  */
    3916           44 :           error_at (gimple_location (stmt),
    3917              :                     "%<teams%> construct must be closely nested inside of "
    3918              :                     "%<target%> construct or not nested in any OpenMP "
    3919              :                     "construct");
    3920           44 :           return false;
    3921              :         }
    3922              :       break;
    3923        42461 :     case GIMPLE_OMP_TARGET:
    3924       190310 :       for (c = gimple_omp_target_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
    3925       147853 :         if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS)
    3926              :           {
    3927            4 :             enum omp_clause_doacross_kind kind = OMP_CLAUSE_DOACROSS_KIND (c);
    3928            4 :             error_at (OMP_CLAUSE_LOCATION (c),
    3929              :                       "%<depend(%s)%> is only allowed in %<omp ordered%>",
    3930              :                       kind == OMP_CLAUSE_DOACROSS_SOURCE ? "source" : "sink");
    3931            4 :             return false;
    3932              :           }
    3933        42457 :       if (is_gimple_omp_offloaded (stmt)
    3934        42457 :           && oacc_get_fn_attrib (cfun->decl) != NULL)
    3935              :         {
    3936            4 :           error_at (gimple_location (stmt),
    3937              :                     "OpenACC region inside of OpenACC routine, nested "
    3938              :                     "parallelism not supported yet");
    3939            4 :           return false;
    3940              :         }
    3941        49869 :       for (; ctx != NULL; ctx = ctx->outer)
    3942              :         {
    3943         7822 :           if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET)
    3944              :             {
    3945          768 :               if (is_gimple_omp (stmt)
    3946          768 :                   && is_gimple_omp_oacc (stmt)
    3947          242 :                   && is_gimple_omp (ctx->stmt))
    3948              :                 {
    3949          242 :                   error_at (gimple_location (stmt),
    3950              :                             "OpenACC construct inside of non-OpenACC region");
    3951          242 :                   return false;
    3952              :                 }
    3953          526 :               continue;
    3954              :             }
    3955              : 
    3956         7054 :           const char *stmt_name, *ctx_stmt_name;
    3957         7054 :           switch (gimple_omp_target_kind (stmt))
    3958              :             {
    3959              :             case GF_OMP_TARGET_KIND_REGION: stmt_name = "target"; break;
    3960          399 :             case GF_OMP_TARGET_KIND_DATA: stmt_name = "target data"; break;
    3961         1688 :             case GF_OMP_TARGET_KIND_UPDATE: stmt_name = "target update"; break;
    3962           12 :             case GF_OMP_TARGET_KIND_ENTER_DATA:
    3963           12 :               stmt_name = "target enter data"; break;
    3964           18 :             case GF_OMP_TARGET_KIND_EXIT_DATA:
    3965           18 :               stmt_name = "target exit data"; break;
    3966         1442 :             case GF_OMP_TARGET_KIND_OACC_PARALLEL: stmt_name = "parallel"; break;
    3967         1621 :             case GF_OMP_TARGET_KIND_OACC_KERNELS: stmt_name = "kernels"; break;
    3968          183 :             case GF_OMP_TARGET_KIND_OACC_SERIAL: stmt_name = "serial"; break;
    3969          386 :             case GF_OMP_TARGET_KIND_OACC_DATA: stmt_name = "data"; break;
    3970          256 :             case GF_OMP_TARGET_KIND_OACC_UPDATE: stmt_name = "update"; break;
    3971           94 :             case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
    3972           94 :               stmt_name = "enter data"; break;
    3973           86 :             case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
    3974           86 :               stmt_name = "exit data"; break;
    3975           64 :             case GF_OMP_TARGET_KIND_OACC_DECLARE: stmt_name = "declare"; break;
    3976          128 :             case GF_OMP_TARGET_KIND_OACC_HOST_DATA: stmt_name = "host_data";
    3977          128 :               break;
    3978              :             case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
    3979              :             case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
    3980              :             case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
    3981              :               /* OpenACC 'kernels' decomposed parts.  */
    3982         1621 :               stmt_name = "kernels"; break;
    3983            0 :             default: gcc_unreachable ();
    3984              :             }
    3985         7054 :           switch (gimple_omp_target_kind (ctx->stmt))
    3986              :             {
    3987              :             case GF_OMP_TARGET_KIND_REGION: ctx_stmt_name = "target"; break;
    3988         2714 :             case GF_OMP_TARGET_KIND_DATA: ctx_stmt_name = "target data"; break;
    3989           35 :             case GF_OMP_TARGET_KIND_OACC_PARALLEL:
    3990           35 :               ctx_stmt_name = "parallel"; break;
    3991              :             case GF_OMP_TARGET_KIND_OACC_KERNELS:
    3992         1037 :               ctx_stmt_name = "kernels"; break;
    3993           35 :             case GF_OMP_TARGET_KIND_OACC_SERIAL:
    3994           35 :               ctx_stmt_name = "serial"; break;
    3995         3117 :             case GF_OMP_TARGET_KIND_OACC_DATA: ctx_stmt_name = "data"; break;
    3996            8 :             case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
    3997            8 :               ctx_stmt_name = "host_data"; break;
    3998              :             case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
    3999              :             case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
    4000              :             case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
    4001              :               /* OpenACC 'kernels' decomposed parts.  */
    4002         1037 :               ctx_stmt_name = "kernels"; break;
    4003            0 :             default: gcc_unreachable ();
    4004              :             }
    4005              : 
    4006              :           /* OpenACC/OpenMP mismatch?  */
    4007        14108 :           if (is_gimple_omp_oacc (stmt)
    4008         7054 :               != is_gimple_omp_oacc (ctx->stmt))
    4009              :             {
    4010           56 :               error_at (gimple_location (stmt),
    4011              :                         "%s %qs construct inside of %s %qs region",
    4012              :                         (is_gimple_omp_oacc (stmt)
    4013              :                          ? "OpenACC" : "OpenMP"), stmt_name,
    4014              :                         (is_gimple_omp_oacc (ctx->stmt)
    4015              :                          ? "OpenACC" : "OpenMP"), ctx_stmt_name);
    4016           28 :               return false;
    4017              :             }
    4018         7026 :           if (is_gimple_omp_offloaded (ctx->stmt))
    4019              :             {
    4020              :               /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX.  */
    4021          185 :               if (is_gimple_omp_oacc (ctx->stmt))
    4022              :                 {
    4023          105 :                   error_at (gimple_location (stmt),
    4024              :                             "%qs construct inside of %qs region",
    4025              :                             stmt_name, ctx_stmt_name);
    4026          105 :                   return false;
    4027              :                 }
    4028              :               else
    4029              :                 {
    4030           80 :                   if ((gimple_omp_target_kind (ctx->stmt)
    4031              :                        == GF_OMP_TARGET_KIND_REGION)
    4032           80 :                       && (gimple_omp_target_kind (stmt)
    4033              :                           == GF_OMP_TARGET_KIND_REGION))
    4034              :                     {
    4035           58 :                       c = omp_find_clause (gimple_omp_target_clauses (stmt),
    4036              :                                            OMP_CLAUSE_DEVICE);
    4037           58 :                       if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
    4038              :                         break;
    4039              :                     }
    4040           49 :                   warning_at (gimple_location (stmt), OPT_Wopenmp,
    4041              :                               "%qs construct inside of %qs region",
    4042              :                               stmt_name, ctx_stmt_name);
    4043              :                 }
    4044              :             }
    4045              :         }
    4046              :       break;
    4047              :     default:
    4048              :       break;
    4049              :     }
    4050              :   return true;
    4051              : }
    4052              : 
    4053              : 
    4054              : /* Helper function scan_omp.
    4055              : 
    4056              :    Callback for walk_tree or operators in walk_gimple_stmt used to
    4057              :    scan for OMP directives in TP.  */
    4058              : 
    4059              : static tree
    4060      9930159 : scan_omp_1_op (tree *tp, int *walk_subtrees, void *data)
    4061              : {
    4062      9930159 :   struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
    4063      9930159 :   omp_context *ctx = (omp_context *) wi->info;
    4064      9930159 :   tree t = *tp;
    4065      9930159 :   tree tmp;
    4066              : 
    4067      9930159 :   switch (TREE_CODE (t))
    4068              :     {
    4069      4203310 :     case VAR_DECL:
    4070      4203310 :     case PARM_DECL:
    4071      4203310 :     case LABEL_DECL:
    4072      4203310 :     case RESULT_DECL:
    4073      4203310 :       if (ctx)
    4074              :         {
    4075      2139957 :           tmp = NULL_TREE;
    4076      2139957 :           if (TREE_CODE (t) == VAR_DECL
    4077      3833479 :               && (tmp = lookup_attribute ("omp allocate var",
    4078      1693522 :                                           DECL_ATTRIBUTES (t))) != NULL_TREE)
    4079          105 :             t = TREE_VALUE (TREE_VALUE (tmp));
    4080      2139957 :           tree repl = remap_decl (t, &ctx->cb);
    4081      2139957 :           gcc_checking_assert (TREE_CODE (repl) != ERROR_MARK);
    4082      2139957 :           if (tmp != NULL_TREE  && t != repl)
    4083           51 :             *tp = build_fold_addr_expr (repl);
    4084      2139906 :           else if (tmp == NULL_TREE)
    4085      2139852 :             *tp = repl;
    4086              :         }
    4087              :       break;
    4088              : 
    4089       197459 :     case INDIRECT_REF:
    4090       197459 :     case MEM_REF:
    4091       197459 :       if (ctx
    4092        78466 :           && TREE_CODE (TREE_OPERAND (t, 0)) == VAR_DECL
    4093       256827 :           && ((tmp = lookup_attribute ("omp allocate var",
    4094        59368 :                                        DECL_ATTRIBUTES (TREE_OPERAND (t, 0))))
    4095              :                != NULL_TREE))
    4096              :         {
    4097          261 :           tmp = TREE_VALUE (TREE_VALUE (tmp));
    4098          261 :           tree repl = remap_decl (tmp, &ctx->cb);
    4099          261 :           gcc_checking_assert (TREE_CODE (repl) != ERROR_MARK);
    4100          261 :           if (tmp != repl)
    4101          245 :             *tp = repl;
    4102              :           break;
    4103              :         }
    4104      5726588 :       gcc_fallthrough ();
    4105              : 
    4106      5726588 :     default:
    4107      5726588 :       if (ctx && TYPE_P (t))
    4108            5 :         *tp = remap_type (t, &ctx->cb);
    4109      5726583 :       else if (!DECL_P (t))
    4110              :         {
    4111      4750902 :           *walk_subtrees = 1;
    4112      4750902 :           if (ctx)
    4113              :             {
    4114      1436695 :               tree tem = remap_type (TREE_TYPE (t), &ctx->cb);
    4115      1436695 :               if (tem != TREE_TYPE (t))
    4116              :                 {
    4117         9946 :                   if (TREE_CODE (t) == INTEGER_CST)
    4118         4202 :                     *tp = wide_int_to_tree (tem, wi::to_wide (t));
    4119              :                   else
    4120         5744 :                     TREE_TYPE (t) = tem;
    4121              :                 }
    4122              :             }
    4123              :         }
    4124              :       break;
    4125              :     }
    4126              : 
    4127      9930159 :   return NULL_TREE;
    4128              : }
    4129              : 
    4130              : /* Return true if FNDECL is a setjmp or a longjmp.  */
    4131              : 
    4132              : static bool
    4133         3086 : setjmp_or_longjmp_p (const_tree fndecl)
    4134              : {
    4135         3086 :   if (fndecl_built_in_p (fndecl, BUILT_IN_SETJMP, BUILT_IN_LONGJMP))
    4136              :     return true;
    4137              : 
    4138         3086 :   tree declname = DECL_NAME (fndecl);
    4139         3086 :   if (!declname
    4140         3086 :       || (DECL_CONTEXT (fndecl) != NULL_TREE
    4141         2391 :           && TREE_CODE (DECL_CONTEXT (fndecl)) != TRANSLATION_UNIT_DECL)
    4142         5913 :       || !TREE_PUBLIC (fndecl))
    4143              :     return false;
    4144              : 
    4145         2665 :   const char *name = IDENTIFIER_POINTER (declname);
    4146         2665 :   return !strcmp (name, "setjmp") || !strcmp (name, "longjmp");
    4147              : }
    4148              : 
    4149              : /* Helper function for scan_omp.
    4150              : 
    4151              :    Callback for walk_gimple_stmt used to scan for OMP directives in
    4152              :    the current statement in GSI.  */
    4153              : 
    4154              : static tree
    4155      3458031 : scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
    4156              :                  struct walk_stmt_info *wi)
    4157              : {
    4158      3458031 :   gimple *stmt = gsi_stmt (*gsi);
    4159      3458031 :   omp_context *ctx = (omp_context *) wi->info;
    4160              : 
    4161      3458031 :   if (gimple_has_location (stmt))
    4162      2819100 :     input_location = gimple_location (stmt);
    4163              : 
    4164              :   /* Check the nesting restrictions.  */
    4165      3458031 :   bool remove = false;
    4166      3458031 :   if (is_gimple_omp (stmt))
    4167       154161 :     remove = !check_omp_nesting_restrictions (stmt, ctx);
    4168      3303870 :   else if (is_gimple_call (stmt))
    4169              :     {
    4170       239866 :       tree fndecl = gimple_call_fndecl (stmt);
    4171       239866 :       if (fndecl)
    4172              :         {
    4173       222752 :           if (ctx
    4174        68869 :               && gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
    4175        14776 :               && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
    4176         3086 :               && setjmp_or_longjmp_p (fndecl)
    4177       222760 :               && !ctx->loop_p)
    4178              :             {
    4179            4 :               remove = true;
    4180            4 :               error_at (gimple_location (stmt),
    4181              :                         "setjmp/longjmp inside %<simd%> construct");
    4182              :             }
    4183       222748 :           else if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
    4184        66498 :             switch (DECL_FUNCTION_CODE (fndecl))
    4185              :               {
    4186         4258 :               case BUILT_IN_GOMP_BARRIER:
    4187         4258 :               case BUILT_IN_GOMP_CANCEL:
    4188         4258 :               case BUILT_IN_GOMP_CANCELLATION_POINT:
    4189         4258 :               case BUILT_IN_GOMP_TASKYIELD:
    4190         4258 :               case BUILT_IN_GOMP_TASKWAIT:
    4191         4258 :               case BUILT_IN_GOMP_TASKGROUP_START:
    4192         4258 :               case BUILT_IN_GOMP_TASKGROUP_END:
    4193         4258 :                 remove = !check_omp_nesting_restrictions (stmt, ctx);
    4194         4258 :                 break;
    4195              :               default:
    4196              :                 break;
    4197              :               }
    4198       156250 :           else if (ctx)
    4199              :             {
    4200        46682 :               omp_context *octx = ctx;
    4201        46682 :               if (gimple_code (ctx->stmt) == GIMPLE_OMP_SCAN && ctx->outer)
    4202        46682 :                 octx = ctx->outer;
    4203        46682 :               if (octx->order_concurrent && omp_runtime_api_call (fndecl))
    4204              :                 {
    4205          240 :                   remove = true;
    4206          240 :                   error_at (gimple_location (stmt),
    4207              :                             "OpenMP runtime API call %qD in a region with "
    4208              :                             "%<order(concurrent)%> clause", fndecl);
    4209              :                 }
    4210        46682 :               if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
    4211         3343 :                   && omp_runtime_api_call (fndecl)
    4212          342 :                   && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
    4213              :                        != strlen ("omp_get_num_teams"))
    4214          188 :                       || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
    4215              :                                  "omp_get_num_teams") != 0)
    4216          169 :                   && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
    4217              :                        != strlen ("omp_get_num_teams_dim"))
    4218            6 :                       || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
    4219              :                                  "omp_get_num_teams_dim") != 0)
    4220          163 :                   && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
    4221              :                        != strlen ("omp_get_team_num"))
    4222          124 :                       || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
    4223              :                                  "omp_get_team_num") != 0)
    4224        46721 :                   && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
    4225              :                        != strlen ("omp_get_team_num_dim"))
    4226           12 :                       || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
    4227              :                                  "omp_get_team_num_dim") != 0))
    4228              :                 {
    4229           27 :                   remove = true;
    4230           27 :                   error_at (gimple_location (stmt),
    4231              :                             "OpenMP runtime API call %qD strictly nested in a "
    4232              :                             "%<teams%> region", fndecl);
    4233              :                 }
    4234        46682 :               if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
    4235        13440 :                   && (gimple_omp_target_kind (ctx->stmt)
    4236              :                       == GF_OMP_TARGET_KIND_REGION)
    4237        51735 :                   && omp_runtime_api_call (fndecl))
    4238              :                 {
    4239          396 :                   tree tgt_clauses = gimple_omp_target_clauses (ctx->stmt);
    4240          396 :                   tree c = omp_find_clause (tgt_clauses, OMP_CLAUSE_DEVICE);
    4241          396 :                   if (c && OMP_CLAUSE_DEVICE_ANCESTOR (c))
    4242            5 :                     error_at (gimple_location (stmt),
    4243              :                               "OpenMP runtime API call %qD in a region with "
    4244              :                               "%<device(ancestor)%> clause", fndecl);
    4245              :                 }
    4246              :             }
    4247              :         }
    4248              :     }
    4249       205105 :   if (remove)
    4250              :     {
    4251         3326 :       stmt = gimple_build_nop ();
    4252         3326 :       gsi_replace (gsi, stmt, false);
    4253              :     }
    4254              : 
    4255      3458031 :   *handled_ops_p = true;
    4256              : 
    4257      3458031 :   switch (gimple_code (stmt))
    4258              :     {
    4259        18393 :     case GIMPLE_OMP_PARALLEL:
    4260        18393 :       taskreg_nesting_level++;
    4261        18393 :       scan_omp_parallel (gsi, ctx);
    4262        18393 :       taskreg_nesting_level--;
    4263        18393 :       break;
    4264              : 
    4265         5651 :     case GIMPLE_OMP_TASK:
    4266         5651 :       taskreg_nesting_level++;
    4267         5651 :       scan_omp_task (gsi, ctx);
    4268         5651 :       taskreg_nesting_level--;
    4269         5651 :       break;
    4270              : 
    4271        53074 :     case GIMPLE_OMP_FOR:
    4272        53074 :       if ((gimple_omp_for_kind (as_a <gomp_for *> (stmt))
    4273              :            == GF_OMP_FOR_KIND_SIMD)
    4274        10946 :           && gimple_omp_for_combined_into_p (stmt)
    4275        60809 :           && gimple_code (ctx->stmt) != GIMPLE_OMP_SCAN)
    4276              :         {
    4277         7553 :           tree clauses = gimple_omp_for_clauses (as_a <gomp_for *> (stmt));
    4278         7553 :           tree c = omp_find_clause (clauses, OMP_CLAUSE_REDUCTION);
    4279         7553 :           if (c && OMP_CLAUSE_REDUCTION_INSCAN (c) && !seen_error ())
    4280              :             {
    4281           91 :               scan_omp_simd_scan (gsi, as_a <gomp_for *> (stmt), ctx);
    4282           91 :               break;
    4283              :             }
    4284              :         }
    4285        52983 :       if ((gimple_omp_for_kind (as_a <gomp_for *> (stmt))
    4286              :            == GF_OMP_FOR_KIND_SIMD)
    4287        10855 :           && omp_maybe_offloaded_ctx (ctx)
    4288         3656 :           && omp_max_simt_vf ()
    4289        52983 :           && gimple_omp_for_collapse (stmt) == 1)
    4290            0 :         scan_omp_simd (gsi, as_a <gomp_for *> (stmt), ctx);
    4291              :       else
    4292        52983 :         scan_omp_for (as_a <gomp_for *> (stmt), ctx);
    4293              :       break;
    4294              : 
    4295          212 :     case GIMPLE_OMP_SCOPE:
    4296          212 :       ctx = new_omp_context (stmt, ctx);
    4297          212 :       scan_sharing_clauses (gimple_omp_scope_clauses (stmt), ctx);
    4298          212 :       scan_omp (gimple_omp_body_ptr (stmt), ctx);
    4299          212 :       break;
    4300              : 
    4301          852 :     case GIMPLE_OMP_DISPATCH:
    4302          852 :       ctx = new_omp_context (stmt, ctx);
    4303          852 :       scan_omp (gimple_omp_body_ptr (stmt), ctx);
    4304          852 :       break;
    4305              : 
    4306          612 :     case GIMPLE_OMP_INTEROP:
    4307          612 :       ctx = new_omp_context (stmt, ctx);
    4308          612 :       break;
    4309              : 
    4310          589 :     case GIMPLE_OMP_SECTIONS:
    4311          589 :       scan_omp_sections (as_a <gomp_sections *> (stmt), ctx);
    4312          589 :       break;
    4313              : 
    4314         1252 :     case GIMPLE_OMP_SINGLE:
    4315         1252 :       scan_omp_single (as_a <gomp_single *> (stmt), ctx);
    4316         1252 :       break;
    4317              : 
    4318         1324 :     case GIMPLE_OMP_SCAN:
    4319         1324 :       if (tree clauses = gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt)))
    4320              :         {
    4321          626 :           if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_INCLUSIVE)
    4322          324 :             ctx->scan_inclusive = true;
    4323          302 :           else if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_EXCLUSIVE)
    4324          302 :             ctx->scan_exclusive = true;
    4325              :         }
    4326              :       /* FALLTHRU */
    4327         6265 :     case GIMPLE_OMP_SECTION:
    4328         6265 :     case GIMPLE_OMP_STRUCTURED_BLOCK:
    4329         6265 :     case GIMPLE_OMP_MASTER:
    4330         6265 :     case GIMPLE_OMP_ORDERED:
    4331         6265 :     case GIMPLE_OMP_CRITICAL:
    4332         6265 :       ctx = new_omp_context (stmt, ctx);
    4333         6265 :       scan_omp (gimple_omp_body_ptr (stmt), ctx);
    4334         6265 :       break;
    4335              : 
    4336          460 :     case GIMPLE_OMP_MASKED:
    4337          460 :       ctx = new_omp_context (stmt, ctx);
    4338          460 :       scan_sharing_clauses (gimple_omp_masked_clauses (stmt), ctx);
    4339          460 :       scan_omp (gimple_omp_body_ptr (stmt), ctx);
    4340          460 :       break;
    4341              : 
    4342          607 :     case GIMPLE_OMP_TASKGROUP:
    4343          607 :       ctx = new_omp_context (stmt, ctx);
    4344          607 :       scan_sharing_clauses (gimple_omp_taskgroup_clauses (stmt), ctx);
    4345          607 :       scan_omp (gimple_omp_body_ptr (stmt), ctx);
    4346          607 :       break;
    4347              : 
    4348        42078 :     case GIMPLE_OMP_TARGET:
    4349        42078 :       if (is_gimple_omp_offloaded (stmt))
    4350              :         {
    4351        24963 :           taskreg_nesting_level++;
    4352        24963 :           scan_omp_target (as_a <gomp_target *> (stmt), ctx);
    4353        24963 :           taskreg_nesting_level--;
    4354              :         }
    4355              :       else
    4356        17115 :         scan_omp_target (as_a <gomp_target *> (stmt), ctx);
    4357              :       break;
    4358              : 
    4359         8878 :     case GIMPLE_OMP_TEAMS:
    4360         8878 :       if (gimple_omp_teams_host (as_a <gomp_teams *> (stmt)))
    4361              :         {
    4362         2721 :           taskreg_nesting_level++;
    4363         2721 :           scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
    4364         2721 :           taskreg_nesting_level--;
    4365              :         }
    4366              :       else
    4367         6157 :         scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
    4368              :       break;
    4369              : 
    4370       288094 :     case GIMPLE_BIND:
    4371       288094 :       {
    4372       288094 :         tree var;
    4373              : 
    4374       288094 :         *handled_ops_p = false;
    4375       288094 :         if (ctx)
    4376       154097 :           for (var = gimple_bind_vars (as_a <gbind *> (stmt));
    4377       582779 :                var ;
    4378       428682 :                var = DECL_CHAIN (var))
    4379       428682 :             insert_decl_map (&ctx->cb, var, var);
    4380              :       }
    4381              :       break;
    4382      3031014 :     default:
    4383      3031014 :       *handled_ops_p = false;
    4384      3031014 :       break;
    4385              :     }
    4386              : 
    4387      3458031 :   return NULL_TREE;
    4388              : }
    4389              : 
    4390              : 
    4391              : /* Scan all the statements starting at the current statement.  CTX
    4392              :    contains context information about the OMP directives and
    4393              :    clauses found during the scan.  */
    4394              : 
    4395              : static void
    4396       263142 : scan_omp (gimple_seq *body_p, omp_context *ctx)
    4397              : {
    4398       263142 :   location_t saved_location;
    4399       263142 :   struct walk_stmt_info wi;
    4400              : 
    4401       263142 :   memset (&wi, 0, sizeof (wi));
    4402       263142 :   wi.info = ctx;
    4403       263142 :   wi.want_locations = true;
    4404              : 
    4405       263142 :   saved_location = input_location;
    4406       263142 :   walk_gimple_seq_mod (body_p, scan_omp_1_stmt, scan_omp_1_op, &wi);
    4407       263142 :   input_location = saved_location;
    4408       263142 : }
    4409              : 
    4410              : /* Re-gimplification and code generation routines.  */
    4411              : 
    4412              : /* Remove omp_member_access_dummy_var variables from gimple_bind_vars
    4413              :    of BIND if in a method.  */
    4414              : 
    4415              : static void
    4416       260605 : maybe_remove_omp_member_access_dummy_vars (gbind *bind)
    4417              : {
    4418       260605 :   if (DECL_ARGUMENTS (current_function_decl)
    4419        98671 :       && DECL_ARTIFICIAL (DECL_ARGUMENTS (current_function_decl))
    4420       271359 :       && (TREE_CODE (TREE_TYPE (DECL_ARGUMENTS (current_function_decl)))
    4421              :           == POINTER_TYPE))
    4422              :     {
    4423         3520 :       tree vars = gimple_bind_vars (bind);
    4424        19484 :       for (tree *pvar = &vars; *pvar; )
    4425        15964 :         if (omp_member_access_dummy_var (*pvar))
    4426          719 :           *pvar = DECL_CHAIN (*pvar);
    4427              :         else
    4428        15245 :           pvar = &DECL_CHAIN (*pvar);
    4429         3520 :       gimple_bind_set_vars (bind, vars);
    4430              :     }
    4431       260605 : }
    4432              : 
    4433              : /* Remove omp_member_access_dummy_var variables from BLOCK_VARS of
    4434              :    block and its subblocks.  */
    4435              : 
    4436              : static void
    4437        14494 : remove_member_access_dummy_vars (tree block)
    4438              : {
    4439        25745 :   for (tree *pvar = &BLOCK_VARS (block); *pvar; )
    4440        11251 :     if (omp_member_access_dummy_var (*pvar))
    4441          108 :       *pvar = DECL_CHAIN (*pvar);
    4442              :     else
    4443        11143 :       pvar = &DECL_CHAIN (*pvar);
    4444              : 
    4445        19150 :   for (block = BLOCK_SUBBLOCKS (block); block; block = BLOCK_CHAIN (block))
    4446         4656 :     remove_member_access_dummy_vars (block);
    4447        14494 : }
    4448              : 
    4449              : /* If a context was created for STMT when it was scanned, return it.  */
    4450              : 
    4451              : static omp_context *
    4452       122167 : maybe_lookup_ctx (gimple *stmt)
    4453              : {
    4454       122167 :   splay_tree_node n;
    4455       122167 :   n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt);
    4456       122167 :   return n ? (omp_context *) n->value : NULL;
    4457              : }
    4458              : 
    4459              : 
    4460              : /* Find the mapping for DECL in CTX or the immediately enclosing
    4461              :    context that has a mapping for DECL.
    4462              : 
    4463              :    If CTX is a nested parallel directive, we may have to use the decl
    4464              :    mappings created in CTX's parent context.  Suppose that we have the
    4465              :    following parallel nesting (variable UIDs showed for clarity):
    4466              : 
    4467              :         iD.1562 = 0;
    4468              :         #omp parallel shared(iD.1562)           -> outer parallel
    4469              :           iD.1562 = iD.1562 + 1;
    4470              : 
    4471              :           #omp parallel shared (iD.1562)        -> inner parallel
    4472              :              iD.1562 = iD.1562 - 1;
    4473              : 
    4474              :    Each parallel structure will create a distinct .omp_data_s structure
    4475              :    for copying iD.1562 in/out of the directive:
    4476              : 
    4477              :         outer parallel          .omp_data_s.1.i -> iD.1562
    4478              :         inner parallel          .omp_data_s.2.i -> iD.1562
    4479              : 
    4480              :    A shared variable mapping will produce a copy-out operation before
    4481              :    the parallel directive and a copy-in operation after it.  So, in
    4482              :    this case we would have:
    4483              : 
    4484              :         iD.1562 = 0;
    4485              :         .omp_data_o.1.i = iD.1562;
    4486              :         #omp parallel shared(iD.1562)           -> outer parallel
    4487              :           .omp_data_i.1 = &.omp_data_o.1
    4488              :           .omp_data_i.1->i = .omp_data_i.1->i + 1;
    4489              : 
    4490              :           .omp_data_o.2.i = iD.1562;            -> **
    4491              :           #omp parallel shared(iD.1562)         -> inner parallel
    4492              :             .omp_data_i.2 = &.omp_data_o.2
    4493              :             .omp_data_i.2->i = .omp_data_i.2->i - 1;
    4494              : 
    4495              : 
    4496              :     ** This is a problem.  The symbol iD.1562 cannot be referenced
    4497              :        inside the body of the outer parallel region.  But since we are
    4498              :        emitting this copy operation while expanding the inner parallel
    4499              :        directive, we need to access the CTX structure of the outer
    4500              :        parallel directive to get the correct mapping:
    4501              : 
    4502              :           .omp_data_o.2.i = .omp_data_i.1->i
    4503              : 
    4504              :     Since there may be other workshare or parallel directives enclosing
    4505              :     the parallel directive, it may be necessary to walk up the context
    4506              :     parent chain.  This is not a problem in general because nested
    4507              :     parallelism happens only rarely.  */
    4508              : 
    4509              : static tree
    4510       126935 : lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
    4511              : {
    4512       126935 :   tree t;
    4513       126935 :   omp_context *up;
    4514              : 
    4515       183920 :   for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
    4516        56985 :     t = maybe_lookup_decl (decl, up);
    4517              : 
    4518       126935 :   gcc_assert (!ctx->is_nested || t || is_global_var (decl));
    4519              : 
    4520        98436 :   return t ? t : decl;
    4521              : }
    4522              : 
    4523              : 
    4524              : /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
    4525              :    in outer contexts.  */
    4526              : 
    4527              : static tree
    4528       358704 : maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
    4529              : {
    4530       358704 :   tree t = NULL;
    4531       358704 :   omp_context *up;
    4532              : 
    4533       630595 :   for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
    4534       271891 :     t = maybe_lookup_decl (decl, up);
    4535              : 
    4536       358704 :   return t ? t : decl;
    4537              : }
    4538              : 
    4539              : 
    4540              : /* Construct the initialization value for reduction operation OP.  */
    4541              : 
    4542              : tree
    4543        13271 : omp_reduction_init_op (location_t loc, enum tree_code op, tree type)
    4544              : {
    4545        13271 :   switch (op)
    4546              :     {
    4547        10937 :     case PLUS_EXPR:
    4548        10937 :     case MINUS_EXPR:
    4549        10937 :     case BIT_IOR_EXPR:
    4550        10937 :     case BIT_XOR_EXPR:
    4551        10937 :     case TRUTH_OR_EXPR:
    4552        10937 :     case TRUTH_ORIF_EXPR:
    4553        10937 :     case TRUTH_XOR_EXPR:
    4554        10937 :     case NE_EXPR:
    4555        10937 :       return build_zero_cst (type);
    4556              : 
    4557         1430 :     case MULT_EXPR:
    4558         1430 :     case TRUTH_AND_EXPR:
    4559         1430 :     case TRUTH_ANDIF_EXPR:
    4560         1430 :     case EQ_EXPR:
    4561         1430 :       return fold_convert_loc (loc, type, integer_one_node);
    4562              : 
    4563          208 :     case BIT_AND_EXPR:
    4564          208 :       return fold_convert_loc (loc, type, integer_minus_one_node);
    4565              : 
    4566          388 :     case MAX_EXPR:
    4567          388 :       if (SCALAR_FLOAT_TYPE_P (type))
    4568              :         {
    4569          158 :           REAL_VALUE_TYPE min;
    4570          158 :           if (HONOR_INFINITIES (type))
    4571          158 :             real_arithmetic (&min, NEGATE_EXPR, &dconstinf, NULL);
    4572              :           else
    4573            0 :             real_maxval (&min, 1, TYPE_MODE (type));
    4574          158 :           return build_real (type, min);
    4575              :         }
    4576          230 :       else if (POINTER_TYPE_P (type))
    4577              :         {
    4578            2 :           wide_int min
    4579            2 :             = wi::min_value (TYPE_PRECISION (type), TYPE_SIGN (type));
    4580            2 :           return wide_int_to_tree (type, min);
    4581            2 :         }
    4582              :       else
    4583              :         {
    4584          228 :           gcc_assert (INTEGRAL_TYPE_P (type));
    4585          228 :           return TYPE_MIN_VALUE (type);
    4586              :         }
    4587              : 
    4588          308 :     case MIN_EXPR:
    4589          308 :       if (SCALAR_FLOAT_TYPE_P (type))
    4590              :         {
    4591          108 :           REAL_VALUE_TYPE max;
    4592          108 :           if (HONOR_INFINITIES (type))
    4593          108 :             max = dconstinf;
    4594              :           else
    4595            0 :             real_maxval (&max, 0, TYPE_MODE (type));
    4596          108 :           return build_real (type, max);
    4597              :         }
    4598          200 :       else if (POINTER_TYPE_P (type))
    4599              :         {
    4600            2 :           wide_int max
    4601            2 :             = wi::max_value (TYPE_PRECISION (type), TYPE_SIGN (type));
    4602            2 :           return wide_int_to_tree (type, max);
    4603            2 :         }
    4604              :       else
    4605              :         {
    4606          198 :           gcc_assert (INTEGRAL_TYPE_P (type));
    4607          198 :           return TYPE_MAX_VALUE (type);
    4608              :         }
    4609              : 
    4610            0 :     default:
    4611            0 :       gcc_unreachable ();
    4612              :     }
    4613              : }
    4614              : 
    4615              : /* Construct the initialization value for reduction CLAUSE.  */
    4616              : 
    4617              : tree
    4618        10651 : omp_reduction_init (tree clause, tree type)
    4619              : {
    4620        10651 :   return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause),
    4621        10651 :                                 OMP_CLAUSE_REDUCTION_CODE (clause), type);
    4622              : }
    4623              : 
    4624              : /* Return alignment to be assumed for var in CLAUSE, which should be
    4625              :    OMP_CLAUSE_ALIGNED.  */
    4626              : 
    4627              : static tree
    4628          137 : omp_clause_aligned_alignment (tree clause)
    4629              : {
    4630          137 :   if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause))
    4631          120 :     return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause);
    4632              : 
    4633              :   /* Otherwise return implementation defined alignment.  */
    4634           17 :   unsigned int al = 1;
    4635           17 :   opt_scalar_mode mode_iter;
    4636           17 :   auto_vector_modes modes;
    4637           17 :   targetm.vectorize.autovectorize_vector_modes (&modes, true);
    4638           17 :   static enum mode_class classes[]
    4639              :     = { MODE_INT, MODE_VECTOR_INT, MODE_FLOAT, MODE_VECTOR_FLOAT };
    4640           68 :   for (int i = 0; i < 4; i += 2)
    4641              :     /* The for loop above dictates that we only walk through scalar classes.  */
    4642          255 :     FOR_EACH_MODE_IN_CLASS (mode_iter, classes[i])
    4643              :       {
    4644          221 :         scalar_mode mode = mode_iter.require ();
    4645          221 :         machine_mode vmode = targetm.vectorize.preferred_simd_mode (mode);
    4646          221 :         if (GET_MODE_CLASS (vmode) != classes[i + 1])
    4647          102 :           continue;
    4648              :         machine_mode alt_vmode;
    4649          476 :         for (unsigned int j = 0; j < modes.length (); ++j)
    4650          527 :           if (related_vector_mode (modes[j], mode).exists (&alt_vmode)
    4651          578 :               && known_ge (GET_MODE_SIZE (alt_vmode), GET_MODE_SIZE (vmode)))
    4652          119 :             vmode = alt_vmode;
    4653              : 
    4654          119 :         tree type = lang_hooks.types.type_for_mode (mode, 1);
    4655          119 :         if (type == NULL_TREE || TYPE_MODE (type) != mode)
    4656           17 :           continue;
    4657          102 :         type = build_vector_type_for_mode (type, vmode);
    4658          102 :         if (TYPE_MODE (type) != vmode)
    4659            0 :           continue;
    4660          102 :         if (TYPE_ALIGN_UNIT (type) > al)
    4661           17 :           al = TYPE_ALIGN_UNIT (type);
    4662              :       }
    4663           17 :   return build_int_cst (integer_type_node, al);
    4664           17 : }
    4665              : 
    4666              : 
    4667              : /* This structure is part of the interface between lower_rec_simd_input_clauses
    4668              :    and lower_rec_input_clauses.  */
    4669              : 
    4670              : class omplow_simd_context {
    4671              : public:
    4672        77766 :   omplow_simd_context () { memset (this, 0, sizeof (*this)); }
    4673              :   tree idx;
    4674              :   tree lane;
    4675              :   tree lastlane;
    4676              :   vec<tree, va_heap> simt_eargs;
    4677              :   gimple_seq simt_dlist;
    4678              :   poly_uint64 max_vf;
    4679              :   bool is_simt;
    4680              : };
    4681              : 
    4682              : /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
    4683              :    privatization.  */
    4684              : 
    4685              : static bool
    4686        10247 : lower_rec_simd_input_clauses (tree new_var, omp_context *ctx,
    4687              :                               omplow_simd_context *sctx, tree &ivar,
    4688              :                               tree &lvar, tree *rvar = NULL,
    4689              :                               tree *rvar2 = NULL)
    4690              : {
    4691        10247 :   if (known_eq (sctx->max_vf, 0U))
    4692              :     {
    4693         4809 :       sctx->max_vf = (sctx->is_simt ? omp_max_simt_vf ()
    4694         4809 :                       : omp_max_vf (omp_maybe_offloaded_ctx (ctx)));
    4695         4809 :       if (maybe_gt (sctx->max_vf, 1U))
    4696              :         {
    4697         3455 :           tree c = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
    4698              :                                     OMP_CLAUSE_SAFELEN);
    4699         3455 :           if (c)
    4700              :             {
    4701           88 :               poly_uint64 safe_len;
    4702           88 :               if (!poly_int_tree_p (OMP_CLAUSE_SAFELEN_EXPR (c), &safe_len)
    4703           88 :                   || maybe_lt (safe_len, 1U))
    4704            0 :                 sctx->max_vf = 1;
    4705              :               else
    4706           88 :                 sctx->max_vf = lower_bound (sctx->max_vf, safe_len);
    4707              :             }
    4708              :         }
    4709         4809 :       if (sctx->is_simt && !known_eq (sctx->max_vf, 1U))
    4710              :         {
    4711            0 :           for (tree c = gimple_omp_for_clauses (ctx->stmt); c;
    4712            0 :                c = OMP_CLAUSE_CHAIN (c))
    4713              :             {
    4714            0 :               if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
    4715            0 :                 continue;
    4716              : 
    4717            0 :               if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
    4718              :                 {
    4719              :                   /* UDR reductions are not supported yet for SIMT, disable
    4720              :                      SIMT.  */
    4721            0 :                   sctx->max_vf = 1;
    4722            0 :                   break;
    4723              :                 }
    4724              : 
    4725            0 :               if (truth_value_p (OMP_CLAUSE_REDUCTION_CODE (c))
    4726            0 :                   && !INTEGRAL_TYPE_P (TREE_TYPE (new_var)))
    4727              :                 {
    4728              :                   /* Doing boolean operations on non-integral types is
    4729              :                      for conformance only, it's not worth supporting this
    4730              :                      for SIMT.  */
    4731            0 :                   sctx->max_vf = 1;
    4732            0 :                   break;
    4733              :               }
    4734              :             }
    4735              :         }
    4736         4809 :       if (maybe_gt (sctx->max_vf, 1U))
    4737              :         {
    4738         3442 :           sctx->idx = create_tmp_var (unsigned_type_node);
    4739         3442 :           sctx->lane = create_tmp_var (unsigned_type_node);
    4740              :         }
    4741              :     }
    4742        10247 :   if (known_eq (sctx->max_vf, 1U))
    4743              :     return false;
    4744              : 
    4745         7183 :   if (sctx->is_simt)
    4746              :     {
    4747            0 :       if (is_gimple_reg (new_var))
    4748              :         {
    4749            0 :           ivar = lvar = new_var;
    4750            0 :           return true;
    4751              :         }
    4752            0 :       tree type = TREE_TYPE (new_var), ptype = build_pointer_type (type);
    4753            0 :       ivar = lvar = create_tmp_var (type);
    4754            0 :       TREE_ADDRESSABLE (ivar) = 1;
    4755            0 :       DECL_ATTRIBUTES (ivar) = tree_cons (get_identifier ("omp simt private"),
    4756            0 :                                           NULL, DECL_ATTRIBUTES (ivar));
    4757            0 :       sctx->simt_eargs.safe_push (build1 (ADDR_EXPR, ptype, ivar));
    4758            0 :       tree clobber = build_clobber (type);
    4759            0 :       gimple *g = gimple_build_assign (ivar, clobber);
    4760            0 :       gimple_seq_add_stmt (&sctx->simt_dlist, g);
    4761              :     }
    4762              :   else
    4763              :     {
    4764         7183 :       tree atype = build_array_type_nelts (TREE_TYPE (new_var), sctx->max_vf);
    4765         7183 :       tree avar = create_tmp_var_raw (atype);
    4766         7183 :       if (TREE_ADDRESSABLE (new_var))
    4767          920 :         TREE_ADDRESSABLE (avar) = 1;
    4768         7183 :       DECL_ATTRIBUTES (avar)
    4769         7183 :         = tree_cons (get_identifier ("omp simd array"), NULL,
    4770         7183 :                      DECL_ATTRIBUTES (avar));
    4771         7183 :       gimple_add_tmp_var (avar);
    4772         7183 :       tree iavar = avar;
    4773         7183 :       if (rvar && !ctx->for_simd_scan_phase)
    4774              :         {
    4775              :           /* For inscan reductions, create another array temporary,
    4776              :              which will hold the reduced value.  */
    4777          232 :           iavar = create_tmp_var_raw (atype);
    4778          232 :           if (TREE_ADDRESSABLE (new_var))
    4779           33 :             TREE_ADDRESSABLE (iavar) = 1;
    4780          232 :           DECL_ATTRIBUTES (iavar)
    4781          232 :             = tree_cons (get_identifier ("omp simd array"), NULL,
    4782              :                          tree_cons (get_identifier ("omp simd inscan"), NULL,
    4783          232 :                                     DECL_ATTRIBUTES (iavar)));
    4784          232 :           gimple_add_tmp_var (iavar);
    4785          232 :           ctx->cb.decl_map->put (avar, iavar);
    4786          232 :           if (sctx->lastlane == NULL_TREE)
    4787          184 :             sctx->lastlane = create_tmp_var (unsigned_type_node);
    4788          232 :           *rvar = build4 (ARRAY_REF, TREE_TYPE (new_var), iavar,
    4789              :                           sctx->lastlane, NULL_TREE, NULL_TREE);
    4790          232 :           TREE_THIS_NOTRAP (*rvar) = 1;
    4791              : 
    4792          232 :           if (ctx->scan_exclusive)
    4793              :             {
    4794              :               /* And for exclusive scan yet another one, which will
    4795              :                  hold the value during the scan phase.  */
    4796          114 :               tree savar = create_tmp_var_raw (atype);
    4797          114 :               if (TREE_ADDRESSABLE (new_var))
    4798           17 :                 TREE_ADDRESSABLE (savar) = 1;
    4799          114 :               DECL_ATTRIBUTES (savar)
    4800          114 :                 = tree_cons (get_identifier ("omp simd array"), NULL,
    4801              :                              tree_cons (get_identifier ("omp simd inscan "
    4802              :                                                         "exclusive"), NULL,
    4803          114 :                                         DECL_ATTRIBUTES (savar)));
    4804          114 :               gimple_add_tmp_var (savar);
    4805          114 :               ctx->cb.decl_map->put (iavar, savar);
    4806          114 :               *rvar2 = build4 (ARRAY_REF, TREE_TYPE (new_var), savar,
    4807              :                                sctx->idx, NULL_TREE, NULL_TREE);
    4808          114 :               TREE_THIS_NOTRAP (*rvar2) = 1;
    4809              :             }
    4810              :         }
    4811         7183 :       ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), iavar, sctx->idx,
    4812              :                      NULL_TREE, NULL_TREE);
    4813         7183 :       lvar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, sctx->lane,
    4814              :                      NULL_TREE, NULL_TREE);
    4815         7183 :       TREE_THIS_NOTRAP (ivar) = 1;
    4816         7183 :       TREE_THIS_NOTRAP (lvar) = 1;
    4817              :     }
    4818         7183 :   if (DECL_P (new_var))
    4819              :     {
    4820         7029 :       SET_DECL_VALUE_EXPR (new_var, lvar);
    4821         7029 :       DECL_HAS_VALUE_EXPR_P (new_var) = 1;
    4822              :     }
    4823              :   return true;
    4824              : }
    4825              : 
    4826              : /* Helper function of lower_rec_input_clauses.  For a reference
    4827              :    in simd reduction, add an underlying variable it will reference.  */
    4828              : 
    4829              : static void
    4830           64 : handle_simd_reference (location_t loc, tree new_vard, gimple_seq *ilist)
    4831              : {
    4832           64 :   tree z = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard)));
    4833           64 :   if (TREE_CONSTANT (z))
    4834              :     {
    4835           64 :       z = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard)),
    4836              :                               get_name (new_vard));
    4837           64 :       gimple_add_tmp_var (z);
    4838           64 :       TREE_ADDRESSABLE (z) = 1;
    4839           64 :       z = build_fold_addr_expr_loc (loc, z);
    4840           64 :       gimplify_assign (new_vard, z, ilist);
    4841              :     }
    4842           64 : }
    4843              : 
    4844              : /* Helper function for lower_rec_input_clauses.  Emit into ilist sequence
    4845              :    code to emit (type) (tskred_temp[idx]).  */
    4846              : 
    4847              : static tree
    4848         1133 : task_reduction_read (gimple_seq *ilist, tree tskred_temp, tree type,
    4849              :                      unsigned idx)
    4850              : {
    4851         1133 :   unsigned HOST_WIDE_INT sz
    4852         1133 :     = tree_to_uhwi (TYPE_SIZE_UNIT (pointer_sized_int_node));
    4853         1133 :   tree r = build2 (MEM_REF, pointer_sized_int_node,
    4854         1133 :                    tskred_temp, build_int_cst (TREE_TYPE (tskred_temp),
    4855         1133 :                                                idx * sz));
    4856         1133 :   tree v = create_tmp_var (pointer_sized_int_node);
    4857         1133 :   gimple *g = gimple_build_assign (v, r);
    4858         1133 :   gimple_seq_add_stmt (ilist, g);
    4859         1133 :   if (!useless_type_conversion_p (type, pointer_sized_int_node))
    4860              :     {
    4861          885 :       v = create_tmp_var (type);
    4862          885 :       g = gimple_build_assign (v, NOP_EXPR, gimple_assign_lhs (g));
    4863          885 :       gimple_seq_add_stmt (ilist, g);
    4864              :     }
    4865         1133 :   return v;
    4866              : }
    4867              : 
    4868              : /* Lower early initialization of privatized variable NEW_VAR
    4869              :    if it needs an allocator (has allocate clause).  */
    4870              : 
    4871              : static bool
    4872       145317 : lower_private_allocate (tree var, tree new_var, tree &allocator,
    4873              :                         tree &allocate_ptr, gimple_seq *ilist,
    4874              :                         omp_context *ctx, bool is_ref, tree size)
    4875              : {
    4876       145317 :   if (allocator)
    4877              :     return false;
    4878       145214 :   gcc_assert (allocate_ptr == NULL_TREE);
    4879       145214 :   if (ctx->allocate_map
    4880         3240 :       && (DECL_P (new_var) || (TYPE_P (new_var) && size)))
    4881         3237 :     if (tree *allocatorp = ctx->allocate_map->get (var))
    4882         1257 :       allocator = *allocatorp;
    4883       145214 :   if (allocator == NULL_TREE)
    4884              :     return false;
    4885         1257 :   if (!is_ref && omp_privatize_by_reference (var))
    4886              :     {
    4887            0 :       allocator = NULL_TREE;
    4888            0 :       return false;
    4889              :     }
    4890              : 
    4891         1257 :   if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET && DECL_P (allocator))
    4892              :     {
    4893            4 :       allocator = lookup_decl (allocator, ctx);
    4894            4 :       gcc_checking_assert (allocator != NULL_TREE);
    4895              :     }
    4896              : 
    4897         1257 :   unsigned HOST_WIDE_INT ialign = 0;
    4898         1257 :   if (TREE_CODE (allocator) == TREE_LIST)
    4899              :     {
    4900          248 :       ialign = tree_to_uhwi (TREE_VALUE (allocator));
    4901          248 :       allocator = TREE_PURPOSE (allocator);
    4902              :     }
    4903         1257 :   if (TREE_CODE (allocator) != INTEGER_CST)
    4904          443 :     allocator = build_outer_var_ref (allocator, ctx, OMP_CLAUSE_ALLOCATE);
    4905         1257 :   allocator = fold_convert (pointer_sized_int_node, allocator);
    4906         1257 :   if (TREE_CODE (allocator) != INTEGER_CST)
    4907              :     {
    4908          443 :       tree var = create_tmp_var (TREE_TYPE (allocator));
    4909          443 :       gimplify_assign (var, allocator, ilist);
    4910          443 :       allocator = var;
    4911              :     }
    4912              : 
    4913         1257 :   tree ptr_type, align, sz = size;
    4914         1257 :   if (TYPE_P (new_var))
    4915              :     {
    4916          300 :       ptr_type = build_pointer_type (new_var);
    4917          300 :       ialign = MAX (ialign, TYPE_ALIGN_UNIT (new_var));
    4918              :     }
    4919          957 :   else if (is_ref)
    4920              :     {
    4921          123 :       ptr_type = build_pointer_type (TREE_TYPE (TREE_TYPE (new_var)));
    4922          123 :       ialign = MAX (ialign, TYPE_ALIGN_UNIT (TREE_TYPE (ptr_type)));
    4923              :     }
    4924              :   else
    4925              :     {
    4926          834 :       ptr_type = build_pointer_type (TREE_TYPE (new_var));
    4927          834 :       ialign = MAX (ialign, DECL_ALIGN_UNIT (new_var));
    4928          834 :       if (sz == NULL_TREE)
    4929          729 :         sz = fold_convert (size_type_node, DECL_SIZE_UNIT (new_var));
    4930              :     }
    4931         1257 :   align = build_int_cst (size_type_node, ialign);
    4932         1257 :   if (TREE_CODE (sz) != INTEGER_CST)
    4933              :     {
    4934           54 :       tree szvar = create_tmp_var (size_type_node);
    4935           54 :       gimplify_assign (szvar, sz, ilist);
    4936           54 :       sz = szvar;
    4937              :     }
    4938         1257 :   allocate_ptr = create_tmp_var (ptr_type);
    4939         1257 :   tree a = builtin_decl_explicit (BUILT_IN_GOMP_ALLOC);
    4940         1257 :   gimple *g = gimple_build_call (a, 3, align, sz, allocator);
    4941         1257 :   gimple_call_set_lhs (g, allocate_ptr);
    4942         1257 :   gimple_seq_add_stmt (ilist, g);
    4943         1257 :   if (!is_ref)
    4944              :     {
    4945          834 :       tree x = build_simple_mem_ref (allocate_ptr);
    4946          834 :       TREE_THIS_NOTRAP (x) = 1;
    4947          834 :       SET_DECL_VALUE_EXPR (new_var, x);
    4948          834 :       DECL_HAS_VALUE_EXPR_P (new_var) = 1;
    4949              :     }
    4950              :   return true;
    4951              : }
    4952              : 
    4953              : /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
    4954              :    from the receiver (aka child) side and initializers for REFERENCE_TYPE
    4955              :    private variables.  Initialization statements go in ILIST, while calls
    4956              :    to destructors go in DLIST.  */
    4957              : 
    4958              : static void
    4959        77766 : lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist,
    4960              :                          omp_context *ctx, struct omp_for_data *fd)
    4961              : {
    4962        77766 :   tree c, copyin_seq, x, ptr;
    4963        77766 :   bool copyin_by_ref = false;
    4964        77766 :   bool lastprivate_firstprivate = false;
    4965        77766 :   bool reduction_omp_orig_ref = false;
    4966        77766 :   int pass;
    4967        77766 :   bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
    4968        77766 :                   && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD);
    4969        77766 :   omplow_simd_context sctx = omplow_simd_context ();
    4970        77766 :   tree simt_lane = NULL_TREE, simtrec = NULL_TREE;
    4971        77766 :   tree ivar = NULL_TREE, lvar = NULL_TREE, uid = NULL_TREE;
    4972        77766 :   gimple_seq llist[4] = { };
    4973        77766 :   tree nonconst_simd_if = NULL_TREE;
    4974              : 
    4975        77766 :   copyin_seq = NULL;
    4976        77766 :   sctx.is_simt = is_simd && omp_find_clause (clauses, OMP_CLAUSE__SIMT_);
    4977              : 
    4978              :   /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
    4979              :      with data sharing clauses referencing variable sized vars.  That
    4980              :      is unnecessarily hard to support and very unlikely to result in
    4981              :      vectorized code anyway.  */
    4982         9390 :   if (is_simd)
    4983        61587 :     for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
    4984        52197 :       switch (OMP_CLAUSE_CODE (c))
    4985              :         {
    4986         7613 :         case OMP_CLAUSE_LINEAR:
    4987         7613 :           if (OMP_CLAUSE_LINEAR_ARRAY (c))
    4988          216 :             sctx.max_vf = 1;
    4989              :           /* FALLTHRU */
    4990        25196 :         case OMP_CLAUSE_PRIVATE:
    4991        25196 :         case OMP_CLAUSE_FIRSTPRIVATE:
    4992        25196 :         case OMP_CLAUSE_LASTPRIVATE:
    4993        25196 :           if (is_variable_sized (OMP_CLAUSE_DECL (c)))
    4994            0 :             sctx.max_vf = 1;
    4995        25196 :           else if (omp_privatize_by_reference (OMP_CLAUSE_DECL (c)))
    4996              :             {
    4997          279 :               tree rtype = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
    4998          279 :               if (!TREE_CONSTANT (TYPE_SIZE_UNIT (rtype)))
    4999           96 :                 sctx.max_vf = 1;
    5000              :             }
    5001              :           break;
    5002         2595 :         case OMP_CLAUSE_REDUCTION:
    5003         2595 :         case OMP_CLAUSE_IN_REDUCTION:
    5004         2595 :           if (TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF
    5005         2595 :               || is_variable_sized (OMP_CLAUSE_DECL (c)))
    5006          180 :             sctx.max_vf = 1;
    5007         2415 :           else if (omp_privatize_by_reference (OMP_CLAUSE_DECL (c)))
    5008              :             {
    5009          144 :               tree rtype = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
    5010          144 :               if (!TREE_CONSTANT (TYPE_SIZE_UNIT (rtype)))
    5011            0 :                 sctx.max_vf = 1;
    5012              :             }
    5013              :           break;
    5014          759 :         case OMP_CLAUSE_IF:
    5015          759 :           if (integer_zerop (OMP_CLAUSE_IF_EXPR (c)))
    5016          124 :             sctx.max_vf = 1;
    5017          635 :           else if (TREE_CODE (OMP_CLAUSE_IF_EXPR (c)) != INTEGER_CST)
    5018          625 :             nonconst_simd_if = OMP_CLAUSE_IF_EXPR (c);
    5019              :           break;
    5020          623 :         case OMP_CLAUSE_SIMDLEN:
    5021          623 :           if (integer_onep (OMP_CLAUSE_SIMDLEN_EXPR (c)))
    5022          110 :             sctx.max_vf = 1;
    5023              :           break;
    5024          184 :         case OMP_CLAUSE__CONDTEMP_:
    5025              :           /* FIXME: lastprivate(conditional:) not handled for SIMT yet.  */
    5026          184 :           if (sctx.is_simt)
    5027            0 :             sctx.max_vf = 1;
    5028              :           break;
    5029        22840 :         default:
    5030        22840 :           continue;
    5031        22840 :         }
    5032              : 
    5033              :   /* Add a placeholder for simduid.  */
    5034        77766 :   if (sctx.is_simt && maybe_ne (sctx.max_vf, 1U))
    5035            0 :     sctx.simt_eargs.safe_push (NULL_TREE);
    5036              : 
    5037        77766 :   unsigned task_reduction_cnt = 0;
    5038        77766 :   unsigned task_reduction_cntorig = 0;
    5039        77766 :   unsigned task_reduction_cnt_full = 0;
    5040        77766 :   unsigned task_reduction_cntorig_full = 0;
    5041        77766 :   unsigned task_reduction_other_cnt = 0;
    5042        77766 :   tree tskred_atype = NULL_TREE, tskred_avar = NULL_TREE;
    5043        77766 :   tree tskred_base = NULL_TREE, tskred_temp = NULL_TREE;
    5044              :   /* Do all the fixed sized types in the first pass, and the variable sized
    5045              :      types in the second pass.  This makes sure that the scalar arguments to
    5046              :      the variable sized types are processed before we use them in the
    5047              :      variable sized operations.  For task reductions we use 4 passes, in the
    5048              :      first two we ignore them, in the third one gather arguments for
    5049              :      GOMP_task_reduction_remap call and in the last pass actually handle
    5050              :      the task reductions.  */
    5051       472778 :   for (pass = 0; pass < ((task_reduction_cnt || task_reduction_other_cnt)
    5052       236389 :                          ? 4 : 2); ++pass)
    5053              :     {
    5054       158627 :       if (pass == 2 && task_reduction_cnt)
    5055              :         {
    5056          878 :           tskred_atype
    5057          878 :             = build_array_type_nelts (ptr_type_node, task_reduction_cnt
    5058          878 :                                                      + task_reduction_cntorig);
    5059          878 :           tskred_avar = create_tmp_var_raw (tskred_atype);
    5060          878 :           gimple_add_tmp_var (tskred_avar);
    5061          878 :           TREE_ADDRESSABLE (tskred_avar) = 1;
    5062          878 :           task_reduction_cnt_full = task_reduction_cnt;
    5063          878 :           task_reduction_cntorig_full = task_reduction_cntorig;
    5064              :         }
    5065       157749 :       else if (pass == 3 && task_reduction_cnt)
    5066              :         {
    5067          878 :           x = builtin_decl_explicit (BUILT_IN_GOMP_TASK_REDUCTION_REMAP);
    5068          878 :           gimple *g
    5069          878 :             = gimple_build_call (x, 3, size_int (task_reduction_cnt),
    5070          878 :                                  size_int (task_reduction_cntorig),
    5071              :                                  build_fold_addr_expr (tskred_avar));
    5072          878 :           gimple_seq_add_stmt (ilist, g);
    5073              :         }
    5074       158627 :       if (pass == 3 && task_reduction_other_cnt)
    5075              :         {
    5076              :           /* For reduction clauses, build
    5077              :              tskred_base = (void *) tskred_temp[2]
    5078              :                            + omp_get_thread_num () * tskred_temp[1]
    5079              :              or if tskred_temp[1] is known to be constant, that constant
    5080              :              directly.  This is the start of the private reduction copy block
    5081              :              for the current thread.  */
    5082          849 :           tree v = create_tmp_var (integer_type_node);
    5083          849 :           x = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
    5084          849 :           gimple *g = gimple_build_call (x, 0);
    5085          849 :           gimple_call_set_lhs (g, v);
    5086          849 :           gimple_seq_add_stmt (ilist, g);
    5087          849 :           c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
    5088          849 :           tskred_temp = OMP_CLAUSE_DECL (c);
    5089          849 :           if (is_taskreg_ctx (ctx))
    5090          559 :             tskred_temp = lookup_decl (tskred_temp, ctx);
    5091          849 :           tree v2 = create_tmp_var (sizetype);
    5092          849 :           g = gimple_build_assign (v2, NOP_EXPR, v);
    5093          849 :           gimple_seq_add_stmt (ilist, g);
    5094          849 :           if (ctx->task_reductions[0])
    5095          832 :             v = fold_convert (sizetype, ctx->task_reductions[0]);
    5096              :           else
    5097           17 :             v = task_reduction_read (ilist, tskred_temp, sizetype, 1);
    5098          849 :           tree v3 = create_tmp_var (sizetype);
    5099          849 :           g = gimple_build_assign (v3, MULT_EXPR, v2, v);
    5100          849 :           gimple_seq_add_stmt (ilist, g);
    5101          849 :           v = task_reduction_read (ilist, tskred_temp, ptr_type_node, 2);
    5102          849 :           tskred_base = create_tmp_var (ptr_type_node);
    5103          849 :           g = gimple_build_assign (tskred_base, POINTER_PLUS_EXPR, v, v3);
    5104          849 :           gimple_seq_add_stmt (ilist, g);
    5105              :         }
    5106       158627 :       task_reduction_cnt = 0;
    5107       158627 :       task_reduction_cntorig = 0;
    5108       158627 :       task_reduction_other_cnt = 0;
    5109       765452 :       for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
    5110              :         {
    5111       606829 :           enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c);
    5112       606829 :           tree var, new_var;
    5113       606829 :           bool by_ref;
    5114       606829 :           location_t clause_loc = OMP_CLAUSE_LOCATION (c);
    5115       606829 :           bool task_reduction_p = false;
    5116       606829 :           bool task_reduction_needs_orig_p = false;
    5117       606829 :           tree cond = NULL_TREE;
    5118       606829 :           tree allocator, allocate_ptr;
    5119              : 
    5120       606829 :           switch (c_kind)
    5121              :             {
    5122       136504 :             case OMP_CLAUSE_PRIVATE:
    5123       136504 :               if (OMP_CLAUSE_PRIVATE_DEBUG (c))
    5124       436719 :                 continue;
    5125              :               break;
    5126        60228 :             case OMP_CLAUSE_SHARED:
    5127              :               /* Ignore shared directives in teams construct inside
    5128              :                  of target construct.  */
    5129        60228 :               if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
    5130        60228 :                   && !is_host_teams_ctx (ctx))
    5131        23462 :                 continue;
    5132        36766 :               if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL)
    5133              :                 {
    5134        12080 :                   gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c)
    5135              :                               || is_global_var (OMP_CLAUSE_DECL (c)));
    5136        12080 :                   continue;
    5137              :                 }
    5138              :             case OMP_CLAUSE_FIRSTPRIVATE:
    5139              :             case OMP_CLAUSE_COPYIN:
    5140              :               break;
    5141        15680 :             case OMP_CLAUSE_LINEAR:
    5142        15680 :               if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c)
    5143        15680 :                   && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
    5144              :                 lastprivate_firstprivate = true;
    5145              :               break;
    5146        37676 :             case OMP_CLAUSE_REDUCTION:
    5147        37676 :             case OMP_CLAUSE_IN_REDUCTION:
    5148        37676 :               if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
    5149        29328 :                   || is_task_ctx (ctx)
    5150        64664 :                   || OMP_CLAUSE_REDUCTION_TASK (c))
    5151              :                 {
    5152        12948 :                   task_reduction_p = true;
    5153        12948 :                   if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
    5154              :                     {
    5155         4600 :                       task_reduction_other_cnt++;
    5156         4600 :                       if (pass == 2)
    5157         1150 :                         continue;
    5158              :                     }
    5159              :                   else
    5160         8348 :                     task_reduction_cnt++;
    5161        11798 :                   if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
    5162              :                     {
    5163          756 :                       var = OMP_CLAUSE_DECL (c);
    5164              :                       /* If var is a global variable that isn't privatized
    5165              :                          in outer contexts, we don't need to look up the
    5166              :                          original address, it is always the address of the
    5167              :                          global variable itself.  */
    5168          756 :                       if (!DECL_P (var)
    5169          332 :                           || omp_privatize_by_reference (var)
    5170         1013 :                           || !is_global_var
    5171          257 :                                 (maybe_lookup_decl_in_outer_ctx (var, ctx)))
    5172              :                         {
    5173          640 :                           task_reduction_needs_orig_p = true;
    5174          640 :                           if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
    5175          532 :                             task_reduction_cntorig++;
    5176              :                         }
    5177              :                     }
    5178              :                 }
    5179        24728 :               else if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
    5180       359292 :                 reduction_omp_orig_ref = true;
    5181              :               break;
    5182         3396 :             case OMP_CLAUSE__REDUCTEMP_:
    5183         3396 :               if (!is_taskreg_ctx (ctx))
    5184         1160 :                 continue;
    5185              :               /* FALLTHRU */
    5186       111856 :             case OMP_CLAUSE__LOOPTEMP_:
    5187              :               /* Handle _looptemp_/_reductemp_ clauses only on
    5188              :                  parallel/task.  */
    5189       111856 :               if (fd)
    5190        69442 :                 continue;
    5191              :               break;
    5192        43724 :             case OMP_CLAUSE_LASTPRIVATE:
    5193        43724 :               if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
    5194              :                 {
    5195         2066 :                   lastprivate_firstprivate = true;
    5196         2066 :                   if (pass != 0 || is_taskloop_ctx (ctx))
    5197         1347 :                     continue;
    5198              :                 }
    5199              :               /* Even without corresponding firstprivate, if
    5200              :                  decl is Fortran allocatable, it needs outer var
    5201              :                  reference.  */
    5202        41658 :               else if (pass == 0
    5203        62451 :                        && lang_hooks.decls.omp_private_outer_ref
    5204        20793 :                                                         (OMP_CLAUSE_DECL (c)))
    5205              :                 lastprivate_firstprivate = true;
    5206              :               break;
    5207          274 :             case OMP_CLAUSE_ALIGNED:
    5208          274 :               if (pass != 1)
    5209          137 :                 continue;
    5210          137 :               var = OMP_CLAUSE_DECL (c);
    5211          137 :               if (TREE_CODE (TREE_TYPE (var)) == POINTER_TYPE
    5212          137 :                   && !is_global_var (var))
    5213              :                 {
    5214           73 :                   new_var = maybe_lookup_decl (var, ctx);
    5215           73 :                   if (new_var == NULL_TREE)
    5216            6 :                     new_var = maybe_lookup_decl_in_outer_ctx (var, ctx);
    5217           73 :                   x = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
    5218           73 :                   tree alarg = omp_clause_aligned_alignment (c);
    5219           73 :                   alarg = fold_convert_loc (clause_loc, size_type_node, alarg);
    5220           73 :                   x = build_call_expr_loc (clause_loc, x, 2, new_var, alarg);
    5221           73 :                   x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
    5222           73 :                   x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
    5223           73 :                   gimplify_and_add (x, ilist);
    5224              :                 }
    5225           64 :               else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
    5226           64 :                        && is_global_var (var))
    5227              :                 {
    5228           64 :                   tree ptype = build_pointer_type (TREE_TYPE (var)), t, t2;
    5229           64 :                   new_var = lookup_decl (var, ctx);
    5230           64 :                   t = maybe_lookup_decl_in_outer_ctx (var, ctx);
    5231           64 :                   t = build_fold_addr_expr_loc (clause_loc, t);
    5232           64 :                   t2 = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
    5233           64 :                   tree alarg = omp_clause_aligned_alignment (c);
    5234           64 :                   alarg = fold_convert_loc (clause_loc, size_type_node, alarg);
    5235           64 :                   t = build_call_expr_loc (clause_loc, t2, 2, t, alarg);
    5236           64 :                   t = fold_convert_loc (clause_loc, ptype, t);
    5237           64 :                   x = create_tmp_var (ptype);
    5238           64 :                   t = build2 (MODIFY_EXPR, ptype, x, t);
    5239           64 :                   gimplify_and_add (t, ilist);
    5240           64 :                   t = build_simple_mem_ref_loc (clause_loc, x);
    5241           64 :                   SET_DECL_VALUE_EXPR (new_var, t);
    5242           64 :                   DECL_HAS_VALUE_EXPR_P (new_var) = 1;
    5243              :                 }
    5244          137 :               continue;
    5245         1514 :             case OMP_CLAUSE__CONDTEMP_:
    5246         1514 :               if (is_parallel_ctx (ctx)
    5247         1514 :                   || (is_simd && !OMP_CLAUSE__CONDTEMP__ITER (c)))
    5248              :                 break;
    5249         1094 :               continue;
    5250       136604 :             default:
    5251       136604 :               continue;
    5252       137835 :             }
    5253              : 
    5254       359292 :           if (task_reduction_p != (pass >= 2))
    5255        15708 :             continue;
    5256              : 
    5257       343584 :           allocator = NULL_TREE;
    5258       343584 :           allocate_ptr = NULL_TREE;
    5259       343584 :           new_var = var = OMP_CLAUSE_DECL (c);
    5260       343584 :           if ((c_kind == OMP_CLAUSE_REDUCTION
    5261       343584 :                || c_kind == OMP_CLAUSE_IN_REDUCTION)
    5262        29998 :               && TREE_CODE (var) == MEM_REF)
    5263              :             {
    5264         4506 :               var = TREE_OPERAND (var, 0);
    5265         4506 :               if (TREE_CODE (var) == POINTER_PLUS_EXPR)
    5266          773 :                 var = TREE_OPERAND (var, 0);
    5267         4506 :               if (TREE_CODE (var) == INDIRECT_REF
    5268         4282 :                   || TREE_CODE (var) == ADDR_EXPR)
    5269         2578 :                 var = TREE_OPERAND (var, 0);
    5270         4506 :               if (is_variable_sized (var))
    5271              :                 {
    5272          179 :                   gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
    5273          179 :                   var = DECL_VALUE_EXPR (var);
    5274          179 :                   gcc_assert (TREE_CODE (var) == INDIRECT_REF);
    5275          179 :                   var = TREE_OPERAND (var, 0);
    5276          179 :                   gcc_assert (DECL_P (var));
    5277              :                 }
    5278         4506 :               new_var = var;
    5279              :             }
    5280        29998 :           if (c_kind == OMP_CLAUSE_IN_REDUCTION && is_omp_target (ctx->stmt))
    5281              :             {
    5282          844 :               splay_tree_key key = (splay_tree_key) &DECL_CONTEXT (var);
    5283          844 :               new_var = (tree) splay_tree_lookup (ctx->field_map, key)->value;
    5284              :             }
    5285       342740 :           else if (c_kind != OMP_CLAUSE_COPYIN)
    5286       341712 :             new_var = lookup_decl (var, ctx);
    5287              : 
    5288       343584 :           if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN)
    5289              :             {
    5290        25038 :               if (pass != 0)
    5291        12518 :                 continue;
    5292              :             }
    5293              :           /* C/C++ array section reductions.  */
    5294       318546 :           else if ((c_kind == OMP_CLAUSE_REDUCTION
    5295              :                     || c_kind == OMP_CLAUSE_IN_REDUCTION)
    5296       318546 :                    && var != OMP_CLAUSE_DECL (c))
    5297              :             {
    5298         4506 :               if (pass == 0)
    5299          948 :                 continue;
    5300              : 
    5301         3558 :               tree bias = TREE_OPERAND (OMP_CLAUSE_DECL (c), 1);
    5302         3558 :               tree orig_var = TREE_OPERAND (OMP_CLAUSE_DECL (c), 0);
    5303              : 
    5304         3558 :               if (TREE_CODE (orig_var) == POINTER_PLUS_EXPR)
    5305              :                 {
    5306          723 :                   tree b = TREE_OPERAND (orig_var, 1);
    5307          723 :                   if (is_omp_target (ctx->stmt))
    5308              :                     b = NULL_TREE;
    5309              :                   else
    5310          723 :                     b = maybe_lookup_decl (b, ctx);
    5311          723 :                   if (b == NULL)
    5312              :                     {
    5313           17 :                       b = TREE_OPERAND (orig_var, 1);
    5314           17 :                       b = maybe_lookup_decl_in_outer_ctx (b, ctx);
    5315              :                     }
    5316          723 :                   if (integer_zerop (bias))
    5317              :                     bias = b;
    5318              :                   else
    5319              :                     {
    5320            0 :                       bias = fold_convert_loc (clause_loc,
    5321            0 :                                                TREE_TYPE (b), bias);
    5322            0 :                       bias = fold_build2_loc (clause_loc, PLUS_EXPR,
    5323            0 :                                               TREE_TYPE (b), b, bias);
    5324              :                     }
    5325          723 :                   orig_var = TREE_OPERAND (orig_var, 0);
    5326              :                 }
    5327         3558 :               if (pass == 2)
    5328              :                 {
    5329         1149 :                   tree out = maybe_lookup_decl_in_outer_ctx (var, ctx);
    5330         1149 :                   if (is_global_var (out)
    5331          241 :                       && TREE_CODE (TREE_TYPE (out)) != POINTER_TYPE
    5332         1336 :                       && (TREE_CODE (TREE_TYPE (out)) != REFERENCE_TYPE
    5333            0 :                           || (TREE_CODE (TREE_TYPE (TREE_TYPE (out)))
    5334              :                               != POINTER_TYPE)))
    5335              :                     x = var;
    5336          962 :                   else if (is_omp_target (ctx->stmt))
    5337              :                     x = out;
    5338              :                   else
    5339              :                     {
    5340          866 :                       bool by_ref = use_pointer_for_field (var, NULL);
    5341          866 :                       x = build_receiver_ref (var, by_ref, ctx);
    5342          866 :                       if (TREE_CODE (TREE_TYPE (var)) == REFERENCE_TYPE
    5343          866 :                           && (TREE_CODE (TREE_TYPE (TREE_TYPE (var)))
    5344              :                               == POINTER_TYPE))
    5345           32 :                         x = build_fold_addr_expr (x);
    5346              :                     }
    5347         1149 :                   if (TREE_CODE (orig_var) == INDIRECT_REF)
    5348           40 :                     x = build_simple_mem_ref (x);
    5349         1109 :                   else if (TREE_CODE (orig_var) == ADDR_EXPR)
    5350              :                     {
    5351          646 :                       if (var == TREE_OPERAND (orig_var, 0))
    5352          583 :                         x = build_fold_addr_expr (x);
    5353              :                     }
    5354         1149 :                   bias = fold_convert (sizetype, bias);
    5355         1149 :                   x = fold_convert (ptr_type_node, x);
    5356         1149 :                   x = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
    5357         1149 :                                        TREE_TYPE (x), x, bias);
    5358         1149 :                   unsigned cnt = task_reduction_cnt - 1;
    5359         1149 :                   if (!task_reduction_needs_orig_p)
    5360         1061 :                     cnt += (task_reduction_cntorig_full
    5361         1061 :                             - task_reduction_cntorig);
    5362              :                   else
    5363           88 :                     cnt = task_reduction_cntorig - 1;
    5364         1149 :                   tree r = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
    5365         1149 :                                    size_int (cnt), NULL_TREE, NULL_TREE);
    5366         1149 :                   gimplify_assign (r, x, ilist);
    5367         1149 :                   continue;
    5368         1149 :                 }
    5369              : 
    5370         2409 :               if (TREE_CODE (orig_var) == INDIRECT_REF
    5371         2297 :                   || TREE_CODE (orig_var) == ADDR_EXPR)
    5372         1398 :                 orig_var = TREE_OPERAND (orig_var, 0);
    5373         2409 :               tree d = OMP_CLAUSE_DECL (c);
    5374         2409 :               tree type = TREE_TYPE (d);
    5375         2409 :               gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
    5376         2409 :               tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
    5377         2409 :               tree sz = v;
    5378         2409 :               const char *name = get_name (orig_var);
    5379         2409 :               if (pass != 3 && !TREE_CONSTANT (v))
    5380              :                 {
    5381           93 :                   tree t;
    5382           93 :                   if (is_omp_target (ctx->stmt))
    5383              :                     t = NULL_TREE;
    5384              :                   else
    5385           93 :                     t = maybe_lookup_decl (v, ctx);
    5386           93 :                   if (t)
    5387           88 :                     v = t;
    5388              :                   else
    5389            5 :                     v = maybe_lookup_decl_in_outer_ctx (v, ctx);
    5390           93 :                   gimplify_expr (&v, ilist, NULL, is_gimple_val, fb_rvalue);
    5391          186 :                   t = fold_build2_loc (clause_loc, PLUS_EXPR,
    5392           93 :                                        TREE_TYPE (v), v,
    5393           93 :                                        build_int_cst (TREE_TYPE (v), 1));
    5394           93 :                   sz = fold_build2_loc (clause_loc, MULT_EXPR,
    5395           93 :                                         TREE_TYPE (v), t,
    5396           93 :                                         TYPE_SIZE_UNIT (TREE_TYPE (type)));
    5397              :                 }
    5398         2409 :               if (pass == 3)
    5399              :                 {
    5400         1461 :                   tree xv = create_tmp_var (ptr_type_node);
    5401         1461 :                   if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
    5402              :                     {
    5403         1149 :                       unsigned cnt = task_reduction_cnt - 1;
    5404         1149 :                       if (!task_reduction_needs_orig_p)
    5405         1061 :                         cnt += (task_reduction_cntorig_full
    5406         1061 :                                 - task_reduction_cntorig);
    5407              :                       else
    5408           88 :                         cnt = task_reduction_cntorig - 1;
    5409         1149 :                       x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
    5410         1149 :                                   size_int (cnt), NULL_TREE, NULL_TREE);
    5411              : 
    5412         1149 :                       gimple *g = gimple_build_assign (xv, x);
    5413         1149 :                       gimple_seq_add_stmt (ilist, g);
    5414              :                     }
    5415              :                   else
    5416              :                     {
    5417          312 :                       unsigned int idx = *ctx->task_reduction_map->get (c);
    5418          312 :                       tree off;
    5419          312 :                       if (ctx->task_reductions[1 + idx])
    5420           81 :                         off = fold_convert (sizetype,
    5421              :                                             ctx->task_reductions[1 + idx]);
    5422              :                       else
    5423          231 :                         off = task_reduction_read (ilist, tskred_temp, sizetype,
    5424          231 :                                                    7 + 3 * idx + 1);
    5425          312 :                       gimple *g = gimple_build_assign (xv, POINTER_PLUS_EXPR,
    5426              :                                                        tskred_base, off);
    5427          312 :                       gimple_seq_add_stmt (ilist, g);
    5428              :                     }
    5429         1461 :                   x = fold_convert (build_pointer_type (boolean_type_node),
    5430              :                                     xv);
    5431         1461 :                   if (TREE_CONSTANT (v))
    5432         1065 :                     x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (x), x,
    5433              :                                      TYPE_SIZE_UNIT (type));
    5434              :                   else
    5435              :                     {
    5436          396 :                       tree t;
    5437          396 :                       if (is_omp_target (ctx->stmt))
    5438              :                         t = NULL_TREE;
    5439              :                       else
    5440          336 :                         t = maybe_lookup_decl (v, ctx);
    5441          336 :                       if (t)
    5442          316 :                         v = t;
    5443              :                       else
    5444           80 :                         v = maybe_lookup_decl_in_outer_ctx (v, ctx);
    5445          396 :                       gimplify_expr (&v, ilist, NULL, is_gimple_val,
    5446              :                                      fb_rvalue);
    5447          792 :                       t = fold_build2_loc (clause_loc, PLUS_EXPR,
    5448          396 :                                            TREE_TYPE (v), v,
    5449          396 :                                            build_int_cst (TREE_TYPE (v), 1));
    5450          396 :                       t = fold_build2_loc (clause_loc, MULT_EXPR,
    5451          396 :                                            TREE_TYPE (v), t,
    5452          396 :                                            TYPE_SIZE_UNIT (TREE_TYPE (type)));
    5453          396 :                       x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (x), x, t);
    5454              :                     }
    5455         1461 :                   cond = create_tmp_var (TREE_TYPE (x));
    5456         1461 :                   gimplify_assign (cond, x, ilist);
    5457         1461 :                   x = xv;
    5458              :                 }
    5459          948 :               else if (lower_private_allocate (var, type, allocator,
    5460              :                                                allocate_ptr, ilist, ctx,
    5461              :                                                true,
    5462          948 :                                                TREE_CONSTANT (v)
    5463          855 :                                                ? TYPE_SIZE_UNIT (type)
    5464              :                                                : sz))
    5465          300 :                 x = allocate_ptr;
    5466          648 :               else if (TREE_CONSTANT (v))
    5467              :                 {
    5468          560 :                   x = create_tmp_var_raw (type, name);
    5469          560 :                   gimple_add_tmp_var (x);
    5470          560 :                   TREE_ADDRESSABLE (x) = 1;
    5471          560 :                   x = build_fold_addr_expr_loc (clause_loc, x);
    5472              :                 }
    5473              :               else
    5474              :                 {
    5475           88 :                   tree atmp
    5476           88 :                     = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
    5477           88 :                   tree al = size_int (TYPE_ALIGN (TREE_TYPE (type)));
    5478           88 :                   x = build_call_expr_loc (clause_loc, atmp, 2, sz, al);
    5479              :                 }
    5480              : 
    5481         2409 :               tree ptype = build_pointer_type (TREE_TYPE (type));
    5482         2409 :               x = fold_convert_loc (clause_loc, ptype, x);
    5483         2409 :               tree y = create_tmp_var (ptype, name);
    5484         2409 :               gimplify_assign (y, x, ilist);
    5485         2409 :               x = y;
    5486         2409 :               tree yb = y;
    5487              : 
    5488         2409 :               if (!integer_zerop (bias))
    5489              :                 {
    5490         1542 :                   bias = fold_convert_loc (clause_loc, pointer_sized_int_node,
    5491              :                                            bias);
    5492         1542 :                   yb = fold_convert_loc (clause_loc, pointer_sized_int_node,
    5493              :                                          x);
    5494         1542 :                   yb = fold_build2_loc (clause_loc, MINUS_EXPR,
    5495              :                                         pointer_sized_int_node, yb, bias);
    5496         1542 :                   x = fold_convert_loc (clause_loc, TREE_TYPE (x), yb);
    5497         1542 :                   yb = create_tmp_var (ptype, name);
    5498         1542 :                   gimplify_assign (yb, x, ilist);
    5499         1542 :                   x = yb;
    5500              :                 }
    5501              : 
    5502         2409 :               d = TREE_OPERAND (d, 0);
    5503         2409 :               if (TREE_CODE (d) == POINTER_PLUS_EXPR)
    5504          420 :                 d = TREE_OPERAND (d, 0);
    5505         2409 :               if (TREE_CODE (d) == ADDR_EXPR)
    5506              :                 {
    5507         1286 :                   if (orig_var != var)
    5508              :                     {
    5509           91 :                       gcc_assert (is_variable_sized (orig_var));
    5510           91 :                       x = fold_convert_loc (clause_loc, TREE_TYPE (new_var),
    5511              :                                             x);
    5512           91 :                       gimplify_assign (new_var, x, ilist);
    5513           91 :                       tree new_orig_var = lookup_decl (orig_var, ctx);
    5514           91 :                       tree t = build_fold_indirect_ref (new_var);
    5515           91 :                       DECL_IGNORED_P (new_var) = 0;
    5516           91 :                       TREE_THIS_NOTRAP (t) = 1;
    5517           91 :                       SET_DECL_VALUE_EXPR (new_orig_var, t);
    5518           91 :                       DECL_HAS_VALUE_EXPR_P (new_orig_var) = 1;
    5519              :                     }
    5520              :                   else
    5521              :                     {
    5522         1195 :                       x = build2 (MEM_REF, TREE_TYPE (new_var), x,
    5523              :                                   build_int_cst (ptype, 0));
    5524         1195 :                       SET_DECL_VALUE_EXPR (new_var, x);
    5525         1195 :                       DECL_HAS_VALUE_EXPR_P (new_var) = 1;
    5526              :                     }
    5527              :                 }
    5528              :               else
    5529              :                 {
    5530         1123 :                   gcc_assert (orig_var == var);
    5531         1123 :                   if (TREE_CODE (d) == INDIRECT_REF)
    5532              :                     {
    5533          112 :                       x = create_tmp_var (ptype, name);
    5534          112 :                       TREE_ADDRESSABLE (x) = 1;
    5535          112 :                       gimplify_assign (x, yb, ilist);
    5536          112 :                       x = build_fold_addr_expr_loc (clause_loc, x);
    5537              :                     }
    5538         1123 :                   x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
    5539         1123 :                   gimplify_assign (new_var, x, ilist);
    5540              :                 }
    5541              :               /* GOMP_taskgroup_reduction_register memsets the whole
    5542              :                  array to zero.  If the initializer is zero, we don't
    5543              :                  need to initialize it again, just mark it as ever
    5544              :                  used unconditionally, i.e. cond = true.  */
    5545         2409 :               if (cond
    5546         1461 :                   && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) == NULL_TREE
    5547         3342 :                   && initializer_zerop (omp_reduction_init (c,
    5548          933 :                                                             TREE_TYPE (type))))
    5549              :                 {
    5550          624 :                   gimple *g = gimple_build_assign (build_simple_mem_ref (cond),
    5551              :                                                    boolean_true_node);
    5552          624 :                   gimple_seq_add_stmt (ilist, g);
    5553          624 :                   continue;
    5554          624 :                 }
    5555         1785 :               tree end = create_artificial_label (UNKNOWN_LOCATION);
    5556         1785 :               if (cond)
    5557              :                 {
    5558          837 :                   gimple *g;
    5559          837 :                   if (!is_parallel_ctx (ctx))
    5560              :                     {
    5561          786 :                       tree condv = create_tmp_var (boolean_type_node);
    5562          786 :                       g = gimple_build_assign (condv,
    5563              :                                                build_simple_mem_ref (cond));
    5564          786 :                       gimple_seq_add_stmt (ilist, g);
    5565          786 :                       tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
    5566          786 :                       g = gimple_build_cond (NE_EXPR, condv,
    5567              :                                              boolean_false_node, end, lab1);
    5568          786 :                       gimple_seq_add_stmt (ilist, g);
    5569          786 :                       gimple_seq_add_stmt (ilist, gimple_build_label (lab1));
    5570              :                     }
    5571          837 :                   g = gimple_build_assign (build_simple_mem_ref (cond),
    5572              :                                            boolean_true_node);
    5573          837 :                   gimple_seq_add_stmt (ilist, g);
    5574              :                 }
    5575              : 
    5576         1785 :               tree y1 = create_tmp_var (ptype);
    5577         1785 :               gimplify_assign (y1, y, ilist);
    5578         1785 :               tree i2 = NULL_TREE, y2 = NULL_TREE;
    5579         1785 :               tree body2 = NULL_TREE, end2 = NULL_TREE;
    5580         1785 :               tree y3 = NULL_TREE, y4 = NULL_TREE;
    5581         1785 :               if (task_reduction_needs_orig_p)
    5582              :                 {
    5583          112 :                   y3 = create_tmp_var (ptype);
    5584          112 :                   tree ref;
    5585          112 :                   if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
    5586           88 :                     ref = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
    5587           88 :                                   size_int (task_reduction_cnt_full
    5588              :                                             + task_reduction_cntorig - 1),
    5589              :                                   NULL_TREE, NULL_TREE);
    5590              :                   else
    5591              :                     {
    5592           24 :                       unsigned int idx = *ctx->task_reduction_map->get (c);
    5593           24 :                       ref = task_reduction_read (ilist, tskred_temp, ptype,
    5594           24 :                                                  7 + 3 * idx);
    5595              :                     }
    5596          112 :                   gimplify_assign (y3, ref, ilist);
    5597              :                 }
    5598         1673 :               else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) || is_simd)
    5599              :                 {
    5600          668 :                   if (pass != 3)
    5601              :                     {
    5602          252 :                       y2 = create_tmp_var (ptype);
    5603          252 :                       gimplify_assign (y2, y, ilist);
    5604              :                     }
    5605          668 :                   if (is_simd || OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
    5606              :                     {
    5607          188 :                       tree ref = build_outer_var_ref (var, ctx);
    5608              :                       /* For ref build_outer_var_ref already performs this.  */
    5609          188 :                       if (TREE_CODE (d) == INDIRECT_REF)
    5610            0 :                         gcc_assert (omp_privatize_by_reference (var));
    5611          188 :                       else if (TREE_CODE (d) == ADDR_EXPR)
    5612           96 :                         ref = build_fold_addr_expr (ref);
    5613           92 :                       else if (omp_privatize_by_reference (var))
    5614            8 :                         ref = build_fold_addr_expr (ref);
    5615          188 :                       ref = fold_convert_loc (clause_loc, ptype, ref);
    5616          188 :                       if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)
    5617          188 :                           && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
    5618              :                         {
    5619            8 :                           y3 = create_tmp_var (ptype);
    5620            8 :                           gimplify_assign (y3, unshare_expr (ref), ilist);
    5621              :                         }
    5622          188 :                       if (is_simd)
    5623              :                         {
    5624          180 :                           y4 = create_tmp_var (ptype);
    5625          180 :                           gimplify_assign (y4, ref, dlist);
    5626              :                         }
    5627              :                     }
    5628              :                 }
    5629         1785 :               tree i = create_tmp_var (TREE_TYPE (v));
    5630         1785 :               gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), ilist);
    5631         1785 :               tree body = create_artificial_label (UNKNOWN_LOCATION);
    5632         1785 :               gimple_seq_add_stmt (ilist, gimple_build_label (body));
    5633         1785 :               if (y2)
    5634              :                 {
    5635          252 :                   i2 = create_tmp_var (TREE_TYPE (v));
    5636          252 :                   gimplify_assign (i2, build_int_cst (TREE_TYPE (v), 0), dlist);
    5637          252 :                   body2 = create_artificial_label (UNKNOWN_LOCATION);
    5638          252 :                   end2 = create_artificial_label (UNKNOWN_LOCATION);
    5639          252 :                   gimple_seq_add_stmt (dlist, gimple_build_label (body2));
    5640              :                 }
    5641         1785 :               if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
    5642              :                 {
    5643          600 :                   tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
    5644          600 :                   tree decl_placeholder
    5645          600 :                     = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
    5646          600 :                   SET_DECL_VALUE_EXPR (decl_placeholder,
    5647              :                                        build_simple_mem_ref (y1));
    5648          600 :                   DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
    5649          600 :                   SET_DECL_VALUE_EXPR (placeholder,
    5650              :                                        y3 ? build_simple_mem_ref (y3)
    5651              :                                        : error_mark_node);
    5652          600 :                   DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
    5653          600 :                   x = lang_hooks.decls.omp_clause_default_ctor
    5654          720 :                                 (c, build_simple_mem_ref (y1),
    5655          120 :                                  y3 ? build_simple_mem_ref (y3) : NULL_TREE);
    5656          600 :                   if (x)
    5657          152 :                     gimplify_and_add (x, ilist);
    5658          600 :                   if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
    5659              :                     {
    5660          576 :                       gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
    5661          576 :                       lower_omp (&tseq, ctx);
    5662          576 :                       gimple_seq_add_seq (ilist, tseq);
    5663              :                     }
    5664          600 :                   OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
    5665          600 :                   if (is_simd)
    5666              :                     {
    5667            0 :                       SET_DECL_VALUE_EXPR (decl_placeholder,
    5668              :                                            build_simple_mem_ref (y2));
    5669            0 :                       SET_DECL_VALUE_EXPR (placeholder,
    5670              :                                            build_simple_mem_ref (y4));
    5671            0 :                       gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
    5672            0 :                       lower_omp (&tseq, ctx);
    5673            0 :                       gimple_seq_add_seq (dlist, tseq);
    5674            0 :                       OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
    5675              :                     }
    5676          600 :                   DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
    5677          600 :                   DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 0;
    5678          600 :                   if (y2)
    5679              :                     {
    5680           72 :                       x = lang_hooks.decls.omp_clause_dtor
    5681           72 :                                                 (c, build_simple_mem_ref (y2));
    5682           72 :                       if (x)
    5683           40 :                         gimplify_and_add (x, dlist);
    5684              :                     }
    5685              :                 }
    5686              :               else
    5687              :                 {
    5688         1185 :                   x = omp_reduction_init (c, TREE_TYPE (type));
    5689         1185 :                   enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
    5690              : 
    5691              :                   /* reduction(-:var) sums up the partial results, so it
    5692              :                      acts identically to reduction(+:var).  */
    5693         1185 :                   if (code == MINUS_EXPR)
    5694            0 :                     code = PLUS_EXPR;
    5695              : 
    5696         1185 :                   gimplify_assign (build_simple_mem_ref (y1), x, ilist);
    5697         1185 :                   if (is_simd)
    5698              :                     {
    5699          180 :                       x = build2 (code, TREE_TYPE (type),
    5700              :                                   build_simple_mem_ref (y4),
    5701              :                                   build_simple_mem_ref (y2));
    5702          180 :                       gimplify_assign (build_simple_mem_ref (y4), x, dlist);
    5703              :                     }
    5704              :                 }
    5705         1785 :               gimple *g
    5706         1785 :                 = gimple_build_assign (y1, POINTER_PLUS_EXPR, y1,
    5707         1785 :                                        TYPE_SIZE_UNIT (TREE_TYPE (type)));
    5708         1785 :               gimple_seq_add_stmt (ilist, g);
    5709         1785 :               if (y3)
    5710              :                 {
    5711          120 :                   g = gimple_build_assign (y3, POINTER_PLUS_EXPR, y3,
    5712          120 :                                            TYPE_SIZE_UNIT (TREE_TYPE (type)));
    5713          120 :                   gimple_seq_add_stmt (ilist, g);
    5714              :                 }
    5715         1785 :               g = gimple_build_assign (i, PLUS_EXPR, i,
    5716         1785 :                                        build_int_cst (TREE_TYPE (i), 1));
    5717         1785 :               gimple_seq_add_stmt (ilist, g);
    5718         1785 :               g = gimple_build_cond (LE_EXPR, i, v, body, end);
    5719         1785 :               gimple_seq_add_stmt (ilist, g);
    5720         1785 :               gimple_seq_add_stmt (ilist, gimple_build_label (end));
    5721         1785 :               if (y2)
    5722              :                 {
    5723          252 :                   g = gimple_build_assign (y2, POINTER_PLUS_EXPR, y2,
    5724          252 :                                            TYPE_SIZE_UNIT (TREE_TYPE (type)));
    5725          252 :                   gimple_seq_add_stmt (dlist, g);
    5726          252 :                   if (y4)
    5727              :                     {
    5728          180 :                       g = gimple_build_assign
    5729          180 :                                         (y4, POINTER_PLUS_EXPR, y4,
    5730          180 :                                          TYPE_SIZE_UNIT (TREE_TYPE (type)));
    5731          180 :                       gimple_seq_add_stmt (dlist, g);
    5732              :                     }
    5733          252 :                   g = gimple_build_assign (i2, PLUS_EXPR, i2,
    5734          252 :                                            build_int_cst (TREE_TYPE (i2), 1));
    5735          252 :                   gimple_seq_add_stmt (dlist, g);
    5736          252 :                   g = gimple_build_cond (LE_EXPR, i2, v, body2, end2);
    5737          252 :                   gimple_seq_add_stmt (dlist, g);
    5738          252 :                   gimple_seq_add_stmt (dlist, gimple_build_label (end2));
    5739              :                 }
    5740         1785 :               if (allocator)
    5741              :                 {
    5742          300 :                   tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
    5743          300 :                   g = gimple_build_call (f, 2, allocate_ptr, allocator);
    5744          300 :                   gimple_seq_add_stmt (dlist, g);
    5745              :                 }
    5746         1785 :               continue;
    5747         1785 :             }
    5748       314040 :           else if (pass == 2)
    5749              :             {
    5750          938 :               tree out = maybe_lookup_decl_in_outer_ctx (var, ctx);
    5751          938 :               if (is_global_var (out))
    5752              :                 x = var;
    5753          382 :               else if (is_omp_target (ctx->stmt))
    5754              :                 x = out;
    5755              :               else
    5756              :                 {
    5757          322 :                   bool by_ref = use_pointer_for_field (var, ctx);
    5758          322 :                   x = build_receiver_ref (var, by_ref, ctx);
    5759              :                 }
    5760          938 :               if (!omp_privatize_by_reference (var))
    5761          836 :                 x = build_fold_addr_expr (x);
    5762          938 :               x = fold_convert (ptr_type_node, x);
    5763          938 :               unsigned cnt = task_reduction_cnt - 1;
    5764          938 :               if (!task_reduction_needs_orig_p)
    5765          893 :                 cnt += task_reduction_cntorig_full - task_reduction_cntorig;
    5766              :               else
    5767           45 :                 cnt = task_reduction_cntorig - 1;
    5768          938 :               tree r = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
    5769          938 :                                size_int (cnt), NULL_TREE, NULL_TREE);
    5770          938 :               gimplify_assign (r, x, ilist);
    5771          938 :               continue;
    5772          938 :             }
    5773       313102 :           else if (pass == 3)
    5774              :             {
    5775         1776 :               tree type = TREE_TYPE (new_var);
    5776         1776 :               if (!omp_privatize_by_reference (var))
    5777         1654 :                 type = build_pointer_type (type);
    5778         1776 :               if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
    5779              :                 {
    5780          938 :                   unsigned cnt = task_reduction_cnt - 1;
    5781          938 :                   if (!task_reduction_needs_orig_p)
    5782          893 :                     cnt += (task_reduction_cntorig_full
    5783          893 :                             - task_reduction_cntorig);
    5784              :                   else
    5785           45 :                     cnt = task_reduction_cntorig - 1;
    5786          938 :                   x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
    5787          938 :                               size_int (cnt), NULL_TREE, NULL_TREE);
    5788              :                 }
    5789              :               else
    5790              :                 {
    5791          838 :                   unsigned int idx = *ctx->task_reduction_map->get (c);
    5792          838 :                   tree off;
    5793          838 :                   if (ctx->task_reductions[1 + idx])
    5794          838 :                     off = fold_convert (sizetype,
    5795              :                                         ctx->task_reductions[1 + idx]);
    5796              :                   else
    5797            0 :                     off = task_reduction_read (ilist, tskred_temp, sizetype,
    5798            0 :                                                7 + 3 * idx + 1);
    5799          838 :                   x = fold_build2 (POINTER_PLUS_EXPR, ptr_type_node,
    5800              :                                    tskred_base, off);
    5801              :                 }
    5802         1776 :               x = fold_convert (type, x);
    5803         1776 :               tree t;
    5804         1776 :               if (omp_privatize_by_reference (var))
    5805              :                 {
    5806          122 :                   gimplify_assign (new_var, x, ilist);
    5807          122 :                   t = new_var;
    5808          122 :                   new_var = build_simple_mem_ref (new_var);
    5809              :                 }
    5810              :               else
    5811              :                 {
    5812         1654 :                   t = create_tmp_var (type);
    5813         1654 :                   gimplify_assign (t, x, ilist);
    5814         1654 :                   SET_DECL_VALUE_EXPR (new_var, build_simple_mem_ref (t));
    5815         1654 :                   DECL_HAS_VALUE_EXPR_P (new_var) = 1;
    5816              :                 }
    5817         1776 :               t = fold_convert (build_pointer_type (boolean_type_node), t);
    5818         1776 :               t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t,
    5819              :                                TYPE_SIZE_UNIT (TREE_TYPE (type)));
    5820         1776 :               cond = create_tmp_var (TREE_TYPE (t));
    5821         1776 :               gimplify_assign (cond, t, ilist);
    5822              :             }
    5823       311326 :           else if (is_variable_sized (var))
    5824              :             {
    5825              :               /* For variable sized types, we need to allocate the
    5826              :                  actual storage here.  Call alloca and store the
    5827              :                  result in the pointer decl that we created elsewhere.  */
    5828          270 :               if (pass == 0)
    5829          138 :                 continue;
    5830              : 
    5831          132 :               if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx))
    5832              :                 {
    5833          118 :                   tree tmp;
    5834              : 
    5835          118 :                   ptr = DECL_VALUE_EXPR (new_var);
    5836          118 :                   gcc_assert (TREE_CODE (ptr) == INDIRECT_REF);
    5837          118 :                   ptr = TREE_OPERAND (ptr, 0);
    5838          118 :                   gcc_assert (DECL_P (ptr));
    5839          118 :                   x = TYPE_SIZE_UNIT (TREE_TYPE (new_var));
    5840              : 
    5841          118 :                   if (lower_private_allocate (var, new_var, allocator,
    5842              :                                               allocate_ptr, ilist, ctx,
    5843              :                                               false, x))
    5844            8 :                     tmp = allocate_ptr;
    5845              :                   else
    5846              :                     {
    5847              :                       /* void *tmp = __builtin_alloca */
    5848          110 :                       tree atmp
    5849          110 :                         = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
    5850          110 :                       gcall *stmt
    5851          110 :                         = gimple_build_call (atmp, 2, x,
    5852          110 :                                              size_int (DECL_ALIGN (var)));
    5853          110 :                       cfun->calls_alloca = 1;
    5854          110 :                       tmp = create_tmp_var_raw (ptr_type_node);
    5855          110 :                       gimple_add_tmp_var (tmp);
    5856          110 :                       gimple_call_set_lhs (stmt, tmp);
    5857              : 
    5858          110 :                       gimple_seq_add_stmt (ilist, stmt);
    5859              :                     }
    5860              : 
    5861          118 :                   x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp);
    5862          118 :                   gimplify_assign (ptr, x, ilist);
    5863              :                 }
    5864              :             }
    5865       311056 :           else if (omp_privatize_by_reference (var)
    5866       311056 :                    && (c_kind != OMP_CLAUSE_FIRSTPRIVATE
    5867         2016 :                        || !OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c)))
    5868              :             {
    5869              :               /* For references that are being privatized for Fortran,
    5870              :                  allocate new backing storage for the new pointer
    5871              :                  variable.  This allows us to avoid changing all the
    5872              :                  code that expects a pointer to something that expects
    5873              :                  a direct variable.  */
    5874         4913 :               if (pass == 0)
    5875         2559 :                 continue;
    5876              : 
    5877         2354 :               x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
    5878         2354 :               if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx))
    5879              :                 {
    5880          146 :                   x = build_receiver_ref (var, false, ctx);
    5881          146 :                   if (ctx->allocate_map)
    5882           16 :                     if (tree *allocatep = ctx->allocate_map->get (var))
    5883              :                       {
    5884           16 :                         allocator = *allocatep;
    5885           16 :                         if (TREE_CODE (allocator) == TREE_LIST)
    5886            0 :                           allocator = TREE_PURPOSE (allocator);
    5887           16 :                         if (TREE_CODE (allocator) != INTEGER_CST)
    5888            8 :                           allocator = build_outer_var_ref (allocator, ctx);
    5889           16 :                         allocator = fold_convert (pointer_sized_int_node,
    5890              :                                                   allocator);
    5891           16 :                         allocate_ptr = unshare_expr (x);
    5892              :                       }
    5893          146 :                   if (allocator == NULL_TREE)
    5894          130 :                     x = build_fold_addr_expr_loc (clause_loc, x);
    5895              :                 }
    5896         2208 :               else if (lower_private_allocate (var, new_var, allocator,
    5897              :                                                allocate_ptr,
    5898              :                                                ilist, ctx, true, x))
    5899           95 :                 x = allocate_ptr;
    5900         2113 :               else if (TREE_CONSTANT (x))
    5901              :                 {
    5902              :                   /* For reduction in SIMD loop, defer adding the
    5903              :                      initialization of the reference, because if we decide
    5904              :                      to use SIMD array for it, the initialization could cause
    5905              :                      expansion ICE.  Ditto for other privatization clauses.  */
    5906         1404 :                   if (is_simd)
    5907              :                     x = NULL_TREE;
    5908              :                   else
    5909              :                     {
    5910         1077 :                       x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
    5911              :                                               get_name (var));
    5912         1077 :                       gimple_add_tmp_var (x);
    5913         1077 :                       TREE_ADDRESSABLE (x) = 1;
    5914         1077 :                       x = build_fold_addr_expr_loc (clause_loc, x);
    5915              :                     }
    5916              :                 }
    5917              :               else
    5918              :                 {
    5919          709 :                   tree atmp
    5920          709 :                     = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
    5921          709 :                   tree rtype = TREE_TYPE (TREE_TYPE (new_var));
    5922          709 :                   tree al = size_int (TYPE_ALIGN (rtype));
    5923          709 :                   x = build_call_expr_loc (clause_loc, atmp, 2, x, al);
    5924              :                 }
    5925              : 
    5926         2027 :               if (x)
    5927              :                 {
    5928         2027 :                   x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
    5929         2027 :                   gimplify_assign (new_var, x, ilist);
    5930              :                 }
    5931              : 
    5932         2354 :               new_var = build_simple_mem_ref_loc (clause_loc, new_var);
    5933              :             }
    5934       306143 :           else if ((c_kind == OMP_CLAUSE_REDUCTION
    5935              :                     || c_kind == OMP_CLAUSE_IN_REDUCTION)
    5936       306143 :                    && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
    5937              :             {
    5938         1982 :               if (pass == 0)
    5939          991 :                 continue;
    5940              :             }
    5941       304161 :           else if (pass != 0)
    5942       151824 :             continue;
    5943              : 
    5944       170110 :           switch (OMP_CLAUSE_CODE (c))
    5945              :             {
    5946        12006 :             case OMP_CLAUSE_SHARED:
    5947              :               /* Ignore shared directives in teams construct inside
    5948              :                  target construct.  */
    5949        12006 :               if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
    5950        12006 :                   && !is_host_teams_ctx (ctx))
    5951            0 :                 continue;
    5952              :               /* Shared global vars are just accessed directly.  */
    5953        12006 :               if (is_global_var (new_var))
    5954              :                 break;
    5955              :               /* For taskloop firstprivate/lastprivate, represented
    5956              :                  as firstprivate and shared clause on the task, new_var
    5957              :                  is the firstprivate var.  */
    5958        11839 :               if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
    5959              :                 break;
    5960              :               /* Set up the DECL_VALUE_EXPR for shared variables now.  This
    5961              :                  needs to be delayed until after fixup_child_record_type so
    5962              :                  that we get the correct type during the dereference.  */
    5963        11527 :               by_ref = use_pointer_for_field (var, ctx);
    5964        11527 :               x = build_receiver_ref (var, by_ref, ctx);
    5965        11527 :               SET_DECL_VALUE_EXPR (new_var, x);
    5966        11527 :               DECL_HAS_VALUE_EXPR_P (new_var) = 1;
    5967              : 
    5968              :               /* ??? If VAR is not passed by reference, and the variable
    5969              :                  hasn't been initialized yet, then we'll get a warning for
    5970              :                  the store into the omp_data_s structure.  Ideally, we'd be
    5971              :                  able to notice this and not store anything at all, but
    5972              :                  we're generating code too early.  Suppress the warning.  */
    5973        11527 :               if (!by_ref)
    5974         5243 :                 suppress_warning (var, OPT_Wuninitialized);
    5975              :               break;
    5976              : 
    5977          210 :             case OMP_CLAUSE__CONDTEMP_:
    5978          210 :               if (is_parallel_ctx (ctx))
    5979              :                 {
    5980          103 :                   x = build_receiver_ref (var, false, ctx);
    5981          103 :                   SET_DECL_VALUE_EXPR (new_var, x);
    5982          103 :                   DECL_HAS_VALUE_EXPR_P (new_var) = 1;
    5983              :                 }
    5984          107 :               else if (is_simd && !OMP_CLAUSE__CONDTEMP__ITER (c))
    5985              :                 {
    5986          107 :                   x = build_zero_cst (TREE_TYPE (var));
    5987          107 :                   goto do_private;
    5988              :                 }
    5989              :               break;
    5990              : 
    5991        21301 :             case OMP_CLAUSE_LASTPRIVATE:
    5992        21301 :               if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
    5993              :                 break;
    5994              :               /* FALLTHRU */
    5995              : 
    5996        87639 :             case OMP_CLAUSE_PRIVATE:
    5997        87639 :               if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_PRIVATE)
    5998        20793 :                 x = build_outer_var_ref (var, ctx);
    5999        66846 :               else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
    6000              :                 {
    6001          163 :                   if (is_task_ctx (ctx))
    6002           12 :                     x = build_receiver_ref (var, false, ctx);
    6003              :                   else
    6004          151 :                     x = build_outer_var_ref (var, ctx, OMP_CLAUSE_PRIVATE);
    6005              :                 }
    6006              :               else
    6007              :                 x = NULL;
    6008        94317 :             do_private:
    6009        94317 :               tree nx;
    6010        94317 :               bool copy_ctor;
    6011        94317 :               copy_ctor = false;
    6012        94317 :               lower_private_allocate (var, new_var, allocator, allocate_ptr,
    6013              :                                       ilist, ctx, false, NULL_TREE);
    6014        94317 :               nx = unshare_expr (new_var);
    6015        94317 :               if (is_simd
    6016        24240 :                   && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
    6017       101703 :                   && OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c))
    6018           22 :                 copy_ctor = true;
    6019        94317 :               if (copy_ctor)
    6020           22 :                 nx = lang_hooks.decls.omp_clause_copy_ctor (c, nx, x);
    6021              :               else
    6022        94295 :                 nx = lang_hooks.decls.omp_clause_default_ctor (c, nx, x);
    6023        94317 :               if (is_simd)
    6024              :                 {
    6025        24240 :                   tree y = lang_hooks.decls.omp_clause_dtor (c, new_var);
    6026        23556 :                   if ((TREE_ADDRESSABLE (new_var) || nx || y
    6027        23540 :                        || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
    6028         7010 :                            && (gimple_omp_for_collapse (ctx->stmt) != 1
    6029         1375 :                                || (gimple_omp_for_index (ctx->stmt, 0)
    6030              :                                    != new_var)))
    6031        16832 :                        || OMP_CLAUSE_CODE (c) == OMP_CLAUSE__CONDTEMP_
    6032        16725 :                        || omp_privatize_by_reference (var))
    6033        31105 :                       && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
    6034              :                                                        ivar, lvar))
    6035              :                     {
    6036         5983 :                       if (omp_privatize_by_reference (var))
    6037              :                         {
    6038           52 :                           gcc_assert (TREE_CODE (new_var) == MEM_REF);
    6039           52 :                           tree new_vard = TREE_OPERAND (new_var, 0);
    6040           52 :                           gcc_assert (DECL_P (new_vard));
    6041           52 :                           SET_DECL_VALUE_EXPR (new_vard,
    6042              :                                                build_fold_addr_expr (lvar));
    6043           52 :                           DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
    6044              :                         }
    6045              : 
    6046         5983 :                       if (nx)
    6047              :                         {
    6048           30 :                           tree iv = unshare_expr (ivar);
    6049           30 :                           if (copy_ctor)
    6050           22 :                             x = lang_hooks.decls.omp_clause_copy_ctor (c, iv,
    6051              :                                                                        x);
    6052              :                           else
    6053            8 :                             x = lang_hooks.decls.omp_clause_default_ctor (c,
    6054              :                                                                           iv,
    6055              :                                                                           x);
    6056              :                         }
    6057         5953 :                       else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__CONDTEMP_)
    6058              :                         {
    6059           37 :                           x = build2 (MODIFY_EXPR, TREE_TYPE (ivar),
    6060              :                                       unshare_expr (ivar), x);
    6061           37 :                           nx = x;
    6062              :                         }
    6063         5983 :                       if (nx && x)
    6064           67 :                         gimplify_and_add (x, &llist[0]);
    6065         5983 :                       if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
    6066         5983 :                           && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
    6067              :                         {
    6068           37 :                           tree v = new_var;
    6069           37 :                           if (!DECL_P (v))
    6070              :                             {
    6071            4 :                               gcc_assert (TREE_CODE (v) == MEM_REF);
    6072            4 :                               v = TREE_OPERAND (v, 0);
    6073            4 :                               gcc_assert (DECL_P (v));
    6074              :                             }
    6075           37 :                           v = *ctx->lastprivate_conditional_map->get (v);
    6076           37 :                           tree t = create_tmp_var (TREE_TYPE (v));
    6077           37 :                           tree z = build_zero_cst (TREE_TYPE (v));
    6078           37 :                           tree orig_v
    6079           37 :                             = build_outer_var_ref (var, ctx,
    6080              :                                                    OMP_CLAUSE_LASTPRIVATE);
    6081           37 :                           gimple_seq_add_stmt (dlist,
    6082           37 :                                                gimple_build_assign (t, z));
    6083           37 :                           gcc_assert (DECL_HAS_VALUE_EXPR_P (v));
    6084           37 :                           tree civar = DECL_VALUE_EXPR (v);
    6085           37 :                           gcc_assert (TREE_CODE (civar) == ARRAY_REF);
    6086           37 :                           civar = unshare_expr (civar);
    6087           37 :                           TREE_OPERAND (civar, 1) = sctx.idx;
    6088           37 :                           x = build2 (MODIFY_EXPR, TREE_TYPE (t), t,
    6089              :                                       unshare_expr (civar));
    6090           74 :                           x = build2 (COMPOUND_EXPR, TREE_TYPE (orig_v), x,
    6091           37 :                                       build2 (MODIFY_EXPR, TREE_TYPE (orig_v),
    6092              :                                               orig_v, unshare_expr (ivar)));
    6093           37 :                           tree cond = build2 (LT_EXPR, boolean_type_node, t,
    6094              :                                               civar);
    6095           37 :                           x = build3 (COND_EXPR, void_type_node, cond, x,
    6096              :                                       void_node);
    6097           37 :                           gimple_seq tseq = NULL;
    6098           37 :                           gimplify_and_add (x, &tseq);
    6099           37 :                           if (ctx->outer)
    6100           27 :                             lower_omp (&tseq, ctx->outer);
    6101           37 :                           gimple_seq_add_seq (&llist[1], tseq);
    6102              :                         }
    6103         5983 :                       if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
    6104         5983 :                           && ctx->for_simd_scan_phase)
    6105              :                         {
    6106            9 :                           x = unshare_expr (ivar);
    6107            9 :                           tree orig_v
    6108            9 :                             = build_outer_var_ref (var, ctx,
    6109              :                                                    OMP_CLAUSE_LASTPRIVATE);
    6110            9 :                           x = lang_hooks.decls.omp_clause_assign_op (c, x,
    6111              :                                                                      orig_v);
    6112            9 :                           gimplify_and_add (x, &llist[0]);
    6113              :                         }
    6114         5983 :                       if (y)
    6115              :                         {
    6116           30 :                           y = lang_hooks.decls.omp_clause_dtor (c, ivar);
    6117           30 :                           if (y)
    6118           30 :                             gimplify_and_add (y, &llist[1]);
    6119              :                         }
    6120              :                       break;
    6121              :                     }
    6122        18257 :                   if (omp_privatize_by_reference (var))
    6123              :                     {
    6124           28 :                       gcc_assert (TREE_CODE (new_var) == MEM_REF);
    6125           28 :                       tree new_vard = TREE_OPERAND (new_var, 0);
    6126           28 :                       gcc_assert (DECL_P (new_vard));
    6127           28 :                       tree type = TREE_TYPE (TREE_TYPE (new_vard));
    6128           28 :                       x = TYPE_SIZE_UNIT (type);
    6129           28 :                       if (TREE_CONSTANT (x))
    6130              :                         {
    6131           28 :                           x = create_tmp_var_raw (type, get_name (var));
    6132           28 :                           gimple_add_tmp_var (x);
    6133           28 :                           TREE_ADDRESSABLE (x) = 1;
    6134           28 :                           x = build_fold_addr_expr_loc (clause_loc, x);
    6135           28 :                           x = fold_convert_loc (clause_loc,
    6136           28 :                                                 TREE_TYPE (new_vard), x);
    6137           28 :                           gimplify_assign (new_vard, x, ilist);
    6138              :                         }
    6139              :                     }
    6140              :                 }
    6141        88334 :               if (nx)
    6142          488 :                 gimplify_and_add (nx, ilist);
    6143        88334 :               if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
    6144        15170 :                   && is_simd
    6145        90097 :                   && ctx->for_simd_scan_phase)
    6146              :                 {
    6147            5 :                   tree orig_v = build_outer_var_ref (var, ctx,
    6148              :                                                      OMP_CLAUSE_LASTPRIVATE);
    6149            5 :                   x = lang_hooks.decls.omp_clause_assign_op (c, new_var,
    6150              :                                                              orig_v);
    6151            5 :                   gimplify_and_add (x, ilist);
    6152              :                 }
    6153              :               /* FALLTHRU */
    6154              : 
    6155       119300 :             do_dtor:
    6156       119300 :               x = lang_hooks.decls.omp_clause_dtor (c, new_var);
    6157       119300 :               if (x)
    6158         1134 :                 gimplify_and_add (x, dlist);
    6159       119300 :               if (allocator)
    6160              :                 {
    6161          891 :                   if (!is_gimple_val (allocator))
    6162              :                     {
    6163           22 :                       tree avar = create_tmp_var (TREE_TYPE (allocator));
    6164           22 :                       gimplify_assign (avar, allocator, dlist);
    6165           22 :                       allocator = avar;
    6166              :                     }
    6167          891 :                   if (!is_gimple_val (allocate_ptr))
    6168              :                     {
    6169           59 :                       tree apvar = create_tmp_var (TREE_TYPE (allocate_ptr));
    6170           59 :                       gimplify_assign (apvar, allocate_ptr, dlist);
    6171           59 :                       allocate_ptr = apvar;
    6172              :                     }
    6173          891 :                   tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
    6174          891 :                   gimple *g
    6175          891 :                     = gimple_build_call (f, 2, allocate_ptr, allocator);
    6176          891 :                   gimple_seq_add_stmt (dlist, g);
    6177              :                 }
    6178              :               break;
    6179              : 
    6180         7840 :             case OMP_CLAUSE_LINEAR:
    6181         7840 :               if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c))
    6182         1269 :                 goto do_firstprivate;
    6183         6571 :               if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
    6184              :                 x = NULL;
    6185              :               else
    6186         3605 :                 x = build_outer_var_ref (var, ctx);
    6187         6571 :               goto do_private;
    6188              : 
    6189        28794 :             case OMP_CLAUSE_FIRSTPRIVATE:
    6190        28794 :               if (is_task_ctx (ctx))
    6191              :                 {
    6192         4645 :                   if ((omp_privatize_by_reference (var)
    6193          146 :                        && !OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c))
    6194         4645 :                       || is_variable_sized (var))
    6195          160 :                     goto do_dtor;
    6196         4485 :                   else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var,
    6197              :                                                                           ctx))
    6198         4485 :                            || use_pointer_for_field (var, NULL))
    6199              :                     {
    6200          430 :                       x = build_receiver_ref (var, false, ctx);
    6201          430 :                       if (ctx->allocate_map)
    6202           43 :                         if (tree *allocatep = ctx->allocate_map->get (var))
    6203              :                           {
    6204           43 :                             allocator = *allocatep;
    6205           43 :                             if (TREE_CODE (allocator) == TREE_LIST)
    6206            4 :                               allocator = TREE_PURPOSE (allocator);
    6207           43 :                             if (TREE_CODE (allocator) != INTEGER_CST)
    6208           14 :                               allocator = build_outer_var_ref (allocator, ctx);
    6209           43 :                             allocator = fold_convert (pointer_sized_int_node,
    6210              :                                                       allocator);
    6211           43 :                             allocate_ptr = unshare_expr (x);
    6212           43 :                             x = build_simple_mem_ref (x);
    6213           43 :                             TREE_THIS_NOTRAP (x) = 1;
    6214              :                           }
    6215          430 :                       SET_DECL_VALUE_EXPR (new_var, x);
    6216          430 :                       DECL_HAS_VALUE_EXPR_P (new_var) = 1;
    6217          430 :                       goto do_dtor;
    6218              :                     }
    6219              :                 }
    6220        28204 :               if (OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c)
    6221        28204 :                   && omp_privatize_by_reference (var))
    6222              :                 {
    6223            0 :                   x = build_outer_var_ref (var, ctx);
    6224            0 :                   gcc_assert (TREE_CODE (x) == MEM_REF
    6225              :                               && integer_zerop (TREE_OPERAND (x, 1)));
    6226            0 :                   x = TREE_OPERAND (x, 0);
    6227            0 :                   x = lang_hooks.decls.omp_clause_copy_ctor
    6228            0 :                                                 (c, unshare_expr (new_var), x);
    6229            0 :                   gimplify_and_add (x, ilist);
    6230            0 :                   goto do_dtor;
    6231              :                 }
    6232        29473 :             do_firstprivate:
    6233        29473 :               lower_private_allocate (var, new_var, allocator, allocate_ptr,
    6234              :                                       ilist, ctx, false, NULL_TREE);
    6235        29473 :               x = build_outer_var_ref (var, ctx);
    6236        29473 :               if (is_simd)
    6237              :                 {
    6238         1063 :                   if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
    6239         1063 :                       && gimple_omp_for_combined_into_p (ctx->stmt))
    6240              :                     {
    6241          668 :                       tree t = OMP_CLAUSE_LINEAR_STEP (c);
    6242          668 :                       if (DECL_P (t))
    6243          114 :                         t = build_outer_var_ref (t, ctx);
    6244          668 :                       tree stept = TREE_TYPE (t);
    6245          668 :                       tree ct = omp_find_clause (clauses,
    6246              :                                                  OMP_CLAUSE__LOOPTEMP_);
    6247          668 :                       gcc_assert (ct);
    6248          668 :                       tree l = OMP_CLAUSE_DECL (ct);
    6249          668 :                       tree n1 = fd->loop.n1;
    6250          668 :                       tree step = fd->loop.step;
    6251          668 :                       tree itype = TREE_TYPE (l);
    6252          668 :                       if (POINTER_TYPE_P (itype))
    6253            0 :                         itype = signed_type_for (itype);
    6254          668 :                       l = fold_build2 (MINUS_EXPR, itype, l, n1);
    6255          668 :                       if (TYPE_UNSIGNED (itype)
    6256          668 :                           && fd->loop.cond_code == GT_EXPR)
    6257            0 :                         l = fold_build2 (TRUNC_DIV_EXPR, itype,
    6258              :                                          fold_build1 (NEGATE_EXPR, itype, l),
    6259              :                                          fold_build1 (NEGATE_EXPR,
    6260              :                                                       itype, step));
    6261              :                       else
    6262          668 :                         l = fold_build2 (TRUNC_DIV_EXPR, itype, l, step);
    6263          668 :                       t = fold_build2 (MULT_EXPR, stept,
    6264              :                                        fold_convert (stept, l), t);
    6265              : 
    6266          668 :                       if (OMP_CLAUSE_LINEAR_ARRAY (c))
    6267              :                         {
    6268          108 :                           if (omp_privatize_by_reference (var))
    6269              :                             {
    6270           72 :                               gcc_assert (TREE_CODE (new_var) == MEM_REF);
    6271           72 :                               tree new_vard = TREE_OPERAND (new_var, 0);
    6272           72 :                               gcc_assert (DECL_P (new_vard));
    6273           72 :                               tree type = TREE_TYPE (TREE_TYPE (new_vard));
    6274           72 :                               nx = TYPE_SIZE_UNIT (type);
    6275           72 :                               if (TREE_CONSTANT (nx))
    6276              :                                 {
    6277           24 :                                   nx = create_tmp_var_raw (type,
    6278              :                                                            get_name (var));
    6279           24 :                                   gimple_add_tmp_var (nx);
    6280           24 :                                   TREE_ADDRESSABLE (nx) = 1;
    6281           24 :                                   nx = build_fold_addr_expr_loc (clause_loc,
    6282              :                                                                  nx);
    6283           24 :                                   nx = fold_convert_loc (clause_loc,
    6284           24 :                                                          TREE_TYPE (new_vard),
    6285              :                                                          nx);
    6286           24 :                                   gimplify_assign (new_vard, nx, ilist);
    6287              :                                 }
    6288              :                             }
    6289              : 
    6290          108 :                           x = lang_hooks.decls.omp_clause_linear_ctor
    6291          108 :                                                         (c, new_var, x, t);
    6292          108 :                           gimplify_and_add (x, ilist);
    6293          108 :                           goto do_dtor;
    6294              :                         }
    6295              : 
    6296          560 :                       if (POINTER_TYPE_P (TREE_TYPE (x)))
    6297           11 :                         x = fold_build_pointer_plus (x, t);
    6298              :                       else
    6299          549 :                         x = fold_build2 (PLUS_EXPR, TREE_TYPE (x), x,
    6300              :                                          fold_convert (TREE_TYPE (x), t));
    6301              :                     }
    6302              : 
    6303          955 :                   if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LINEAR
    6304          934 :                        || TREE_ADDRESSABLE (new_var)
    6305          799 :                        || omp_privatize_by_reference (var))
    6306         1217 :                       && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
    6307              :                                                        ivar, lvar))
    6308              :                     {
    6309          152 :                       if (omp_privatize_by_reference (var))
    6310              :                         {
    6311           22 :                           gcc_assert (TREE_CODE (new_var) == MEM_REF);
    6312           22 :                           tree new_vard = TREE_OPERAND (new_var, 0);
    6313           22 :                           gcc_assert (DECL_P (new_vard));
    6314           22 :                           SET_DECL_VALUE_EXPR (new_vard,
    6315              :                                                build_fold_addr_expr (lvar));
    6316           22 :                           DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
    6317              :                         }
    6318          152 :                       if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR)
    6319              :                         {
    6320          131 :                           tree iv = create_tmp_var (TREE_TYPE (new_var));
    6321          131 :                           x = lang_hooks.decls.omp_clause_copy_ctor (c, iv, x);
    6322          131 :                           gimplify_and_add (x, ilist);
    6323          131 :                           gimple_stmt_iterator gsi
    6324          131 :                             = gsi_start (*gimple_omp_body_ptr (ctx->stmt));
    6325          131 :                           gassign *g
    6326          131 :                             = gimple_build_assign (unshare_expr (lvar), iv);
    6327          131 :                           gsi_insert_before_without_update (&gsi, g,
    6328              :                                                             GSI_SAME_STMT);
    6329          131 :                           tree t = OMP_CLAUSE_LINEAR_STEP (c);
    6330          131 :                           enum tree_code code = PLUS_EXPR;
    6331          131 :                           if (POINTER_TYPE_P (TREE_TYPE (new_var)))
    6332              :                             code = POINTER_PLUS_EXPR;
    6333          131 :                           g = gimple_build_assign (iv, code, iv, t);
    6334          131 :                           gsi_insert_before_without_update (&gsi, g,
    6335              :                                                             GSI_SAME_STMT);
    6336          131 :                           break;
    6337              :                         }
    6338           21 :                       x = lang_hooks.decls.omp_clause_copy_ctor
    6339           21 :                                                 (c, unshare_expr (ivar), x);
    6340           21 :                       gimplify_and_add (x, &llist[0]);
    6341           21 :                       x = lang_hooks.decls.omp_clause_dtor (c, ivar);
    6342           21 :                       if (x)
    6343           21 :                         gimplify_and_add (x, &llist[1]);
    6344              :                       break;
    6345              :                     }
    6346          803 :                   if (omp_privatize_by_reference (var))
    6347              :                     {
    6348          105 :                       gcc_assert (TREE_CODE (new_var) == MEM_REF);
    6349          105 :                       tree new_vard = TREE_OPERAND (new_var, 0);
    6350          105 :                       gcc_assert (DECL_P (new_vard));
    6351          105 :                       tree type = TREE_TYPE (TREE_TYPE (new_vard));
    6352          105 :                       nx = TYPE_SIZE_UNIT (type);
    6353          105 :                       if (TREE_CONSTANT (nx))
    6354              :                         {
    6355           57 :                           nx = create_tmp_var_raw (type, get_name (var));
    6356           57 :                           gimple_add_tmp_var (nx);
    6357           57 :                           TREE_ADDRESSABLE (nx) = 1;
    6358           57 :                           nx = build_fold_addr_expr_loc (clause_loc, nx);
    6359           57 :                           nx = fold_convert_loc (clause_loc,
    6360           57 :                                                  TREE_TYPE (new_vard), nx);
    6361           57 :                           gimplify_assign (new_vard, nx, ilist);
    6362              :                         }
    6363              :                     }
    6364              :                 }
    6365        29213 :               x = lang_hooks.decls.omp_clause_copy_ctor
    6366        29213 :                                                 (c, unshare_expr (new_var), x);
    6367        29209 :               gimplify_and_add (x, ilist);
    6368        29209 :               goto do_dtor;
    6369              : 
    6370        19434 :             case OMP_CLAUSE__LOOPTEMP_:
    6371        19434 :             case OMP_CLAUSE__REDUCTEMP_:
    6372        19434 :               gcc_assert (is_taskreg_ctx (ctx));
    6373        19434 :               x = build_outer_var_ref (var, ctx);
    6374        19434 :               x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
    6375        19434 :               gimplify_and_add (x, ilist);
    6376        19434 :               break;
    6377              : 
    6378          514 :             case OMP_CLAUSE_COPYIN:
    6379          514 :               by_ref = use_pointer_for_field (var, NULL);
    6380          514 :               x = build_receiver_ref (var, by_ref, ctx);
    6381          514 :               x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x);
    6382          514 :               append_to_statement_list (x, &copyin_seq);
    6383          514 :               copyin_by_ref |= by_ref;
    6384          514 :               break;
    6385              : 
    6386        13165 :             case OMP_CLAUSE_REDUCTION:
    6387        13165 :             case OMP_CLAUSE_IN_REDUCTION:
    6388              :               /* OpenACC reductions are initialized using the
    6389              :                  GOACC_REDUCTION internal function.  */
    6390        13165 :               if (is_gimple_omp_oacc (ctx->stmt))
    6391              :                 break;
    6392         9183 :               if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
    6393              :                 {
    6394         1491 :                   tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
    6395         1491 :                   gimple *tseq;
    6396         1491 :                   tree ptype = TREE_TYPE (placeholder);
    6397         1491 :                   if (cond)
    6398              :                     {
    6399          327 :                       x = error_mark_node;
    6400          327 :                       if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)
    6401          327 :                           && !task_reduction_needs_orig_p)
    6402           32 :                         x = var;
    6403          295 :                       else if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
    6404              :                         {
    6405           57 :                           tree pptype = build_pointer_type (ptype);
    6406           57 :                           if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
    6407           45 :                             x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
    6408           45 :                                         size_int (task_reduction_cnt_full
    6409              :                                                   + task_reduction_cntorig - 1),
    6410              :                                         NULL_TREE, NULL_TREE);
    6411              :                           else
    6412              :                             {
    6413           12 :                               unsigned int idx
    6414           12 :                                 = *ctx->task_reduction_map->get (c);
    6415           12 :                               x = task_reduction_read (ilist, tskred_temp,
    6416           12 :                                                        pptype, 7 + 3 * idx);
    6417              :                             }
    6418           57 :                           x = fold_convert (pptype, x);
    6419           57 :                           x = build_simple_mem_ref (x);
    6420              :                         }
    6421              :                     }
    6422              :                   else
    6423              :                     {
    6424         1164 :                       lower_private_allocate (var, new_var, allocator,
    6425              :                                               allocate_ptr, ilist, ctx, false,
    6426              :                                               NULL_TREE);
    6427         1164 :                       x = build_outer_var_ref (var, ctx);
    6428              : 
    6429         1164 :                       if (omp_privatize_by_reference (var)
    6430         1164 :                           && !useless_type_conversion_p (ptype, TREE_TYPE (x)))
    6431           49 :                         x = build_fold_addr_expr_loc (clause_loc, x);
    6432              :                     }
    6433         1491 :                   SET_DECL_VALUE_EXPR (placeholder, x);
    6434         1491 :                   DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
    6435         1491 :                   tree new_vard = new_var;
    6436         1491 :                   if (omp_privatize_by_reference (var))
    6437              :                     {
    6438          219 :                       gcc_assert (TREE_CODE (new_var) == MEM_REF);
    6439          219 :                       new_vard = TREE_OPERAND (new_var, 0);
    6440          219 :                       gcc_assert (DECL_P (new_vard));
    6441              :                     }
    6442         1491 :                   tree rvar = NULL_TREE, *rvarp = NULL, rvar2 = NULL_TREE;
    6443         1491 :                   if (is_simd
    6444          312 :                       && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
    6445         1803 :                       && OMP_CLAUSE_REDUCTION_INSCAN (c))
    6446              :                     rvarp = &rvar;
    6447         1491 :                   if (is_simd
    6448         1491 :                       && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
    6449              :                                                        ivar, lvar, rvarp,
    6450              :                                                        &rvar2))
    6451              :                     {
    6452          175 :                       if (new_vard == new_var)
    6453              :                         {
    6454          129 :                           gcc_assert (DECL_VALUE_EXPR (new_var) == lvar);
    6455          129 :                           SET_DECL_VALUE_EXPR (new_var, ivar);
    6456              :                         }
    6457              :                       else
    6458              :                         {
    6459           46 :                           SET_DECL_VALUE_EXPR (new_vard,
    6460              :                                                build_fold_addr_expr (ivar));
    6461           46 :                           DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
    6462              :                         }
    6463          175 :                       x = lang_hooks.decls.omp_clause_default_ctor
    6464          175 :                                 (c, unshare_expr (ivar),
    6465              :                                  build_outer_var_ref (var, ctx));
    6466          175 :                       if (rvarp && ctx->for_simd_scan_phase)
    6467              :                         {
    6468           16 :                           if (x)
    6469            8 :                             gimplify_and_add (x, &llist[0]);
    6470           16 :                           x = lang_hooks.decls.omp_clause_dtor (c, ivar);
    6471           16 :                           if (x)
    6472            8 :                             gimplify_and_add (x, &llist[1]);
    6473          502 :                           break;
    6474              :                         }
    6475           78 :                       else if (rvarp)
    6476              :                         {
    6477           78 :                           if (x)
    6478              :                             {
    6479           38 :                               gimplify_and_add (x, &llist[0]);
    6480              : 
    6481           38 :                               tree ivar2 = unshare_expr (lvar);
    6482           38 :                               TREE_OPERAND (ivar2, 1) = sctx.idx;
    6483           38 :                               x = lang_hooks.decls.omp_clause_default_ctor
    6484           38 :                                     (c, ivar2, build_outer_var_ref (var, ctx));
    6485           38 :                               gimplify_and_add (x, &llist[0]);
    6486              : 
    6487           38 :                               if (rvar2)
    6488              :                                 {
    6489           16 :                                   x = lang_hooks.decls.omp_clause_default_ctor
    6490           16 :                                         (c, unshare_expr (rvar2),
    6491              :                                          build_outer_var_ref (var, ctx));
    6492           16 :                                   gimplify_and_add (x, &llist[0]);
    6493              :                                 }
    6494              : 
    6495              :                               /* For types that need construction, add another
    6496              :                                  private var which will be default constructed
    6497              :                                  and optionally initialized with
    6498              :                                  OMP_CLAUSE_REDUCTION_GIMPLE_INIT, as in the
    6499              :                                  loop we want to assign this value instead of
    6500              :                                  constructing and destructing it in each
    6501              :                                  iteration.  */
    6502           38 :                               tree nv = create_tmp_var_raw (TREE_TYPE (ivar));
    6503           38 :                               gimple_add_tmp_var (nv);
    6504           60 :                               ctx->cb.decl_map->put (TREE_OPERAND (rvar2
    6505              :                                                                    ? rvar2
    6506              :                                                                    : ivar, 0),
    6507              :                                                      nv);
    6508           38 :                               x = lang_hooks.decls.omp_clause_default_ctor
    6509           38 :                                     (c, nv, build_outer_var_ref (var, ctx));
    6510           38 :                               gimplify_and_add (x, ilist);
    6511              : 
    6512           38 :                               if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
    6513              :                                 {
    6514           19 :                                   tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
    6515           19 :                                   x = DECL_VALUE_EXPR (new_vard);
    6516           19 :                                   tree vexpr = nv;
    6517           19 :                                   if (new_vard != new_var)
    6518           11 :                                     vexpr = build_fold_addr_expr (nv);
    6519           19 :                                   SET_DECL_VALUE_EXPR (new_vard, vexpr);
    6520           19 :                                   lower_omp (&tseq, ctx);
    6521           19 :                                   SET_DECL_VALUE_EXPR (new_vard, x);
    6522           19 :                                   gimple_seq_add_seq (ilist, tseq);
    6523           19 :                                   OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
    6524              :                                 }
    6525              : 
    6526           38 :                               x = lang_hooks.decls.omp_clause_dtor (c, nv);
    6527           38 :                               if (x)
    6528           38 :                                 gimplify_and_add (x, dlist);
    6529              :                             }
    6530              : 
    6531           78 :                           tree ref = build_outer_var_ref (var, ctx);
    6532           78 :                           x = unshare_expr (ivar);
    6533           78 :                           x = lang_hooks.decls.omp_clause_assign_op (c, x,
    6534              :                                                                      ref);
    6535           78 :                           gimplify_and_add (x, &llist[0]);
    6536              : 
    6537           78 :                           ref = build_outer_var_ref (var, ctx);
    6538           78 :                           x = lang_hooks.decls.omp_clause_assign_op (c, ref,
    6539              :                                                                      rvar);
    6540           78 :                           gimplify_and_add (x, &llist[3]);
    6541              : 
    6542           78 :                           DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
    6543           78 :                           if (new_vard == new_var)
    6544           48 :                             SET_DECL_VALUE_EXPR (new_var, lvar);
    6545              :                           else
    6546           30 :                             SET_DECL_VALUE_EXPR (new_vard,
    6547              :                                                  build_fold_addr_expr (lvar));
    6548              : 
    6549           78 :                           x = lang_hooks.decls.omp_clause_dtor (c, ivar);
    6550           78 :                           if (x)
    6551           38 :                             gimplify_and_add (x, &llist[1]);
    6552              : 
    6553           78 :                           tree ivar2 = unshare_expr (lvar);
    6554           78 :                           TREE_OPERAND (ivar2, 1) = sctx.idx;
    6555           78 :                           x = lang_hooks.decls.omp_clause_dtor (c, ivar2);
    6556           78 :                           if (x)
    6557           38 :                             gimplify_and_add (x, &llist[1]);
    6558              : 
    6559           78 :                           if (rvar2)
    6560              :                             {
    6561           36 :                               x = lang_hooks.decls.omp_clause_dtor (c, rvar2);
    6562           36 :                               if (x)
    6563           16 :                                 gimplify_and_add (x, &llist[1]);
    6564              :                             }
    6565              :                           break;
    6566              :                         }
    6567           81 :                       if (x)
    6568           34 :                         gimplify_and_add (x, &llist[0]);
    6569           81 :                       if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
    6570              :                         {
    6571           77 :                           tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
    6572           77 :                           lower_omp (&tseq, ctx);
    6573           77 :                           gimple_seq_add_seq (&llist[0], tseq);
    6574              :                         }
    6575           81 :                       OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
    6576           81 :                       tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
    6577           81 :                       lower_omp (&tseq, ctx);
    6578           81 :                       gimple_seq_add_seq (&llist[1], tseq);
    6579           81 :                       OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
    6580           81 :                       DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
    6581           81 :                       if (new_vard == new_var)
    6582           71 :                         SET_DECL_VALUE_EXPR (new_var, lvar);
    6583              :                       else
    6584           10 :                         SET_DECL_VALUE_EXPR (new_vard,
    6585              :                                              build_fold_addr_expr (lvar));
    6586           81 :                       x = lang_hooks.decls.omp_clause_dtor (c, ivar);
    6587           81 :                       if (x)
    6588           38 :                         gimplify_and_add (x, &llist[1]);
    6589              :                       break;
    6590              :                     }
    6591              :                   /* If this is a reference to constant size reduction var
    6592              :                      with placeholder, we haven't emitted the initializer
    6593              :                      for it because it is undesirable if SIMD arrays are used.
    6594              :                      But if they aren't used, we need to emit the deferred
    6595              :                      initialization now.  */
    6596         1316 :                   else if (omp_privatize_by_reference (var) && is_simd)
    6597           46 :                     handle_simd_reference (clause_loc, new_vard, ilist);
    6598              : 
    6599         1316 :                   tree lab2 = NULL_TREE;
    6600         1316 :                   if (cond)
    6601              :                     {
    6602          327 :                       gimple *g;
    6603          327 :                       if (!is_parallel_ctx (ctx))
    6604              :                         {
    6605          302 :                           tree condv = create_tmp_var (boolean_type_node);
    6606          302 :                           tree m = build_simple_mem_ref (cond);
    6607          302 :                           g = gimple_build_assign (condv, m);
    6608          302 :                           gimple_seq_add_stmt (ilist, g);
    6609          302 :                           tree lab1
    6610          302 :                             = create_artificial_label (UNKNOWN_LOCATION);
    6611          302 :                           lab2 = create_artificial_label (UNKNOWN_LOCATION);
    6612          302 :                           g = gimple_build_cond (NE_EXPR, condv,
    6613              :                                                  boolean_false_node,
    6614              :                                                  lab2, lab1);
    6615          302 :                           gimple_seq_add_stmt (ilist, g);
    6616          302 :                           gimple_seq_add_stmt (ilist,
    6617          302 :                                                gimple_build_label (lab1));
    6618              :                         }
    6619          327 :                       g = gimple_build_assign (build_simple_mem_ref (cond),
    6620              :                                                boolean_true_node);
    6621          327 :                       gimple_seq_add_stmt (ilist, g);
    6622              :                     }
    6623          327 :                   x = lang_hooks.decls.omp_clause_default_ctor
    6624         1316 :                                 (c, unshare_expr (new_var),
    6625              :                                  cond ? NULL_TREE
    6626          989 :                                  : build_outer_var_ref (var, ctx));
    6627         1316 :                   if (x)
    6628          302 :                     gimplify_and_add (x, ilist);
    6629              : 
    6630         1316 :                   if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
    6631         1316 :                       && OMP_CLAUSE_REDUCTION_INSCAN (c))
    6632              :                     {
    6633          158 :                       if (ctx->for_simd_scan_phase)
    6634          989 :                         goto do_dtor;
    6635          142 :                       if (x || (!is_simd
    6636           32 :                                 && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)))
    6637              :                         {
    6638           70 :                           tree nv = create_tmp_var_raw (TREE_TYPE (new_var));
    6639           70 :                           gimple_add_tmp_var (nv);
    6640           70 :                           ctx->cb.decl_map->put (new_vard, nv);
    6641           70 :                           x = lang_hooks.decls.omp_clause_default_ctor
    6642           70 :                                 (c, nv, build_outer_var_ref (var, ctx));
    6643           70 :                           if (x)
    6644           70 :                             gimplify_and_add (x, ilist);
    6645           70 :                           if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
    6646              :                             {
    6647           35 :                               tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
    6648           35 :                               tree vexpr = nv;
    6649           35 :                               if (new_vard != new_var)
    6650           19 :                                 vexpr = build_fold_addr_expr (nv);
    6651           35 :                               SET_DECL_VALUE_EXPR (new_vard, vexpr);
    6652           35 :                               DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
    6653           35 :                               lower_omp (&tseq, ctx);
    6654           35 :                               SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
    6655           35 :                               DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
    6656           35 :                               gimple_seq_add_seq (ilist, tseq);
    6657              :                             }
    6658           70 :                           OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
    6659           70 :                           if (is_simd && ctx->scan_exclusive)
    6660              :                             {
    6661           16 :                               tree nv2
    6662           16 :                                 = create_tmp_var_raw (TREE_TYPE (new_var));
    6663           16 :                               gimple_add_tmp_var (nv2);
    6664           16 :                               ctx->cb.decl_map->put (nv, nv2);
    6665           16 :                               x = lang_hooks.decls.omp_clause_default_ctor
    6666           16 :                                     (c, nv2, build_outer_var_ref (var, ctx));
    6667           16 :                               gimplify_and_add (x, ilist);
    6668           16 :                               x = lang_hooks.decls.omp_clause_dtor (c, nv2);
    6669           16 :                               if (x)
    6670           16 :                                 gimplify_and_add (x, dlist);
    6671              :                             }
    6672           70 :                           x = lang_hooks.decls.omp_clause_dtor (c, nv);
    6673           70 :                           if (x)
    6674           70 :                             gimplify_and_add (x, dlist);
    6675              :                         }
    6676           72 :                       else if (is_simd
    6677           40 :                                && ctx->scan_exclusive
    6678           92 :                                && TREE_ADDRESSABLE (TREE_TYPE (new_var)))
    6679              :                         {
    6680            0 :                           tree nv2 = create_tmp_var_raw (TREE_TYPE (new_var));
    6681            0 :                           gimple_add_tmp_var (nv2);
    6682            0 :                           ctx->cb.decl_map->put (new_vard, nv2);
    6683            0 :                           x = lang_hooks.decls.omp_clause_dtor (c, nv2);
    6684            0 :                           if (x)
    6685            0 :                             gimplify_and_add (x, dlist);
    6686              :                         }
    6687          142 :                       DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
    6688          142 :                       goto do_dtor;
    6689              :                     }
    6690              : 
    6691         1158 :                   if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
    6692              :                     {
    6693         1110 :                       tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
    6694         1110 :                       if (c_kind == OMP_CLAUSE_IN_REDUCTION
    6695         1110 :                           && is_omp_target (ctx->stmt))
    6696              :                         {
    6697           12 :                           tree d = maybe_lookup_decl_in_outer_ctx (var, ctx);
    6698           12 :                           tree oldv = NULL_TREE;
    6699           12 :                           gcc_assert (d);
    6700           12 :                           if (DECL_HAS_VALUE_EXPR_P (d))
    6701            6 :                             oldv = DECL_VALUE_EXPR (d);
    6702           12 :                           SET_DECL_VALUE_EXPR (d, new_vard);
    6703           12 :                           DECL_HAS_VALUE_EXPR_P (d) = 1;
    6704           12 :                           lower_omp (&tseq, ctx);
    6705           12 :                           if (oldv)
    6706            6 :                             SET_DECL_VALUE_EXPR (d, oldv);
    6707              :                           else
    6708              :                             {
    6709            6 :                               SET_DECL_VALUE_EXPR (d, NULL_TREE);
    6710            6 :                               DECL_HAS_VALUE_EXPR_P (d) = 0;
    6711              :                             }
    6712              :                         }
    6713              :                       else
    6714         1098 :                         lower_omp (&tseq, ctx);
    6715         1110 :                       gimple_seq_add_seq (ilist, tseq);
    6716              :                     }
    6717         1158 :                   OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
    6718         1158 :                   if (is_simd)
    6719              :                     {
    6720           43 :                       tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
    6721           43 :                       lower_omp (&tseq, ctx);
    6722           43 :                       gimple_seq_add_seq (dlist, tseq);
    6723           43 :                       OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
    6724              :                     }
    6725         1158 :                   DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
    6726         1158 :                   if (cond)
    6727              :                     {
    6728          327 :                       if (lab2)
    6729          302 :                         gimple_seq_add_stmt (ilist, gimple_build_label (lab2));
    6730              :                       break;
    6731              :                     }
    6732          831 :                   goto do_dtor;
    6733              :                 }
    6734              :               else
    6735              :                 {
    6736         7692 :                   x = omp_reduction_init (c, TREE_TYPE (new_var));
    6737         7692 :                   gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE);
    6738         7692 :                   enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
    6739              : 
    6740         7692 :                   if (cond)
    6741              :                     {
    6742         1449 :                       gimple *g;
    6743         1449 :                       tree lab2 = NULL_TREE;
    6744              :                       /* GOMP_taskgroup_reduction_register memsets the whole
    6745              :                          array to zero.  If the initializer is zero, we don't
    6746              :                          need to initialize it again, just mark it as ever
    6747              :                          used unconditionally, i.e. cond = true.  */
    6748         1449 :                       if (initializer_zerop (x))
    6749              :                         {
    6750         1268 :                           g = gimple_build_assign (build_simple_mem_ref (cond),
    6751              :                                                    boolean_true_node);
    6752         1268 :                           gimple_seq_add_stmt (ilist, g);
    6753         3271 :                           break;
    6754              :                         }
    6755              : 
    6756              :                       /* Otherwise, emit
    6757              :                          if (!cond) { cond = true; new_var = x; }  */
    6758          181 :                       if (!is_parallel_ctx (ctx))
    6759              :                         {
    6760          177 :                           tree condv = create_tmp_var (boolean_type_node);
    6761          177 :                           tree m = build_simple_mem_ref (cond);
    6762          177 :                           g = gimple_build_assign (condv, m);
    6763          177 :                           gimple_seq_add_stmt (ilist, g);
    6764          177 :                           tree lab1
    6765          177 :                             = create_artificial_label (UNKNOWN_LOCATION);
    6766          177 :                           lab2 = create_artificial_label (UNKNOWN_LOCATION);
    6767          177 :                           g = gimple_build_cond (NE_EXPR, condv,
    6768              :                                                  boolean_false_node,
    6769              :                                                  lab2, lab1);
    6770          177 :                           gimple_seq_add_stmt (ilist, g);
    6771          177 :                           gimple_seq_add_stmt (ilist,
    6772          177 :                                                gimple_build_label (lab1));
    6773              :                         }
    6774          181 :                       g = gimple_build_assign (build_simple_mem_ref (cond),
    6775              :                                                boolean_true_node);
    6776          181 :                       gimple_seq_add_stmt (ilist, g);
    6777          181 :                       gimplify_assign (new_var, x, ilist);
    6778          181 :                       if (lab2)
    6779          177 :                         gimple_seq_add_stmt (ilist, gimple_build_label (lab2));
    6780              :                       break;
    6781              :                     }
    6782              : 
    6783              :                   /* reduction(-:var) sums up the partial results, so it
    6784              :                      acts identically to reduction(+:var).  */
    6785         6243 :                   if (code == MINUS_EXPR)
    6786           33 :                     code = PLUS_EXPR;
    6787              : 
    6788         6243 :                   bool is_truth_op
    6789         6243 :                     = (code == TRUTH_ANDIF_EXPR || code == TRUTH_ORIF_EXPR);
    6790         6243 :                   tree new_vard = new_var;
    6791         6243 :                   if (is_simd && omp_privatize_by_reference (var))
    6792              :                     {
    6793           52 :                       gcc_assert (TREE_CODE (new_var) == MEM_REF);
    6794           52 :                       new_vard = TREE_OPERAND (new_var, 0);
    6795           52 :                       gcc_assert (DECL_P (new_vard));
    6796              :                     }
    6797         6243 :                   tree rvar = NULL_TREE, *rvarp = NULL, rvar2 = NULL_TREE;
    6798         6243 :                   if (is_simd
    6799         2103 :                       && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
    6800         8346 :                       && OMP_CLAUSE_REDUCTION_INSCAN (c))
    6801              :                     rvarp = &rvar;
    6802         6243 :                   if (is_simd
    6803         6243 :                       && lower_rec_simd_input_clauses (new_var, ctx, &sctx,
    6804              :                                                        ivar, lvar, rvarp,
    6805              :                                                        &rvar2))
    6806              :                     {
    6807          873 :                       if (new_vard != new_var)
    6808              :                         {
    6809           34 :                           SET_DECL_VALUE_EXPR (new_vard,
    6810              :                                                build_fold_addr_expr (lvar));
    6811           34 :                           DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
    6812              :                         }
    6813              : 
    6814          873 :                       tree ref = build_outer_var_ref (var, ctx);
    6815              : 
    6816          873 :                       if (rvarp)
    6817              :                         {
    6818          193 :                           if (ctx->for_simd_scan_phase)
    6819              :                             break;
    6820          154 :                           gimplify_assign (ivar, ref, &llist[0]);
    6821          154 :                           ref = build_outer_var_ref (var, ctx);
    6822          154 :                           gimplify_assign (ref, rvar, &llist[3]);
    6823          154 :                           break;
    6824              :                         }
    6825              : 
    6826          680 :                       gimplify_assign (unshare_expr (ivar), x, &llist[0]);
    6827              : 
    6828          680 :                       if (sctx.is_simt)
    6829              :                         {
    6830            0 :                           if (!simt_lane)
    6831            0 :                             simt_lane = create_tmp_var (unsigned_type_node);
    6832            0 :                           x = build_call_expr_internal_loc
    6833            0 :                             (UNKNOWN_LOCATION, IFN_GOMP_SIMT_XCHG_BFLY,
    6834            0 :                              TREE_TYPE (ivar), 2, ivar, simt_lane);
    6835              :                           /* Make sure x is evaluated unconditionally.  */
    6836            0 :                           tree bfly_var = create_tmp_var (TREE_TYPE (ivar));
    6837            0 :                           gimplify_assign (bfly_var, x, &llist[2]);
    6838            0 :                           x = build2 (code, TREE_TYPE (ivar), ivar, bfly_var);
    6839            0 :                           gimplify_assign (ivar, x, &llist[2]);
    6840              :                         }
    6841          680 :                       tree ivar2 = ivar;
    6842          680 :                       tree ref2 = ref;
    6843          680 :                       if (is_truth_op)
    6844              :                         {
    6845           99 :                           tree zero = build_zero_cst (TREE_TYPE (ivar));
    6846           99 :                           ivar2 = fold_build2_loc (clause_loc, NE_EXPR,
    6847              :                                                    boolean_type_node, ivar,
    6848              :                                                    zero);
    6849           99 :                           ref2 = fold_build2_loc (clause_loc, NE_EXPR,
    6850              :                                                   boolean_type_node, ref,
    6851              :                                                   zero);
    6852              :                         }
    6853          680 :                       x = build2 (code, TREE_TYPE (ref), ref2, ivar2);
    6854          680 :                       if (is_truth_op)
    6855           99 :                         x = fold_convert (TREE_TYPE (ref), x);
    6856          680 :                       ref = build_outer_var_ref (var, ctx);
    6857          680 :                       gimplify_assign (ref, x, &llist[1]);
    6858              : 
    6859              :                     }
    6860              :                   else
    6861              :                     {
    6862         5370 :                       lower_private_allocate (var, new_var, allocator,
    6863              :                                               allocate_ptr, ilist, ctx,
    6864              :                                               false, NULL_TREE);
    6865         5370 :                       if (omp_privatize_by_reference (var) && is_simd)
    6866           18 :                         handle_simd_reference (clause_loc, new_vard, ilist);
    6867         5370 :                       if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
    6868         5370 :                           && OMP_CLAUSE_REDUCTION_INSCAN (c))
    6869              :                         break;
    6870         5009 :                       gimplify_assign (new_var, x, ilist);
    6871         5009 :                       if (is_simd)
    6872              :                         {
    6873         1022 :                           tree ref = build_outer_var_ref (var, ctx);
    6874         1022 :                           tree new_var2 = new_var;
    6875         1022 :                           tree ref2 = ref;
    6876         1022 :                           if (is_truth_op)
    6877              :                             {
    6878           24 :                               tree zero = build_zero_cst (TREE_TYPE (new_var));
    6879           24 :                               new_var2
    6880           24 :                                 = fold_build2_loc (clause_loc, NE_EXPR,
    6881              :                                                    boolean_type_node, new_var,
    6882              :                                                    zero);
    6883           24 :                               ref2 = fold_build2_loc (clause_loc, NE_EXPR,
    6884              :                                                       boolean_type_node, ref,
    6885              :                                                       zero);
    6886              :                             }
    6887         1022 :                           x = build2 (code, TREE_TYPE (ref2), ref2, new_var2);
    6888         1022 :                           if (is_truth_op)
    6889           24 :                             x = fold_convert (TREE_TYPE (new_var), x);
    6890         1022 :                           ref = build_outer_var_ref (var, ctx);
    6891         1022 :                           gimplify_assign (ref, x, dlist);
    6892              :                         }
    6893         5009 :                       if (allocator)
    6894           70 :                         goto do_dtor;
    6895              :                     }
    6896              :                 }
    6897         5619 :               break;
    6898              : 
    6899            0 :             default:
    6900            0 :               gcc_unreachable ();
    6901              :             }
    6902              :         }
    6903              :     }
    6904        77762 :   if (tskred_avar)
    6905              :     {
    6906          878 :       tree clobber = build_clobber (TREE_TYPE (tskred_avar));
    6907          878 :       gimple_seq_add_stmt (ilist, gimple_build_assign (tskred_avar, clobber));
    6908              :     }
    6909              : 
    6910        77762 :   if (known_eq (sctx.max_vf, 1U))
    6911              :     {
    6912         1789 :       sctx.is_simt = false;
    6913         1789 :       if (ctx->lastprivate_conditional_map)
    6914              :         {
    6915           52 :           if (gimple_omp_for_combined_into_p (ctx->stmt))
    6916              :             {
    6917              :               /* Signal to lower_omp_1 that it should use parent context.  */
    6918           43 :               ctx->combined_into_simd_safelen1 = true;
    6919          370 :               for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
    6920          327 :                 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
    6921          327 :                     && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
    6922              :                   {
    6923           57 :                     tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
    6924           57 :                     omp_context *outer = ctx->outer;
    6925           57 :                     if (gimple_code (outer->stmt) == GIMPLE_OMP_SCAN)
    6926            6 :                       outer = outer->outer;
    6927           57 :                     tree *v = ctx->lastprivate_conditional_map->get (o);
    6928           57 :                     tree po = lookup_decl (OMP_CLAUSE_DECL (c), outer);
    6929           57 :                     tree *pv = outer->lastprivate_conditional_map->get (po);
    6930           57 :                     *v = *pv;
    6931              :                   }
    6932              :             }
    6933              :           else
    6934              :             {
    6935              :               /* When not vectorized, treat lastprivate(conditional:) like
    6936              :                  normal lastprivate, as there will be just one simd lane
    6937              :                  writing the privatized variable.  */
    6938            9 :               delete ctx->lastprivate_conditional_map;
    6939            9 :               ctx->lastprivate_conditional_map = NULL;
    6940              :             }
    6941              :         }
    6942              :     }
    6943              : 
    6944        77762 :   if (nonconst_simd_if)
    6945              :     {
    6946          625 :       if (sctx.lane == NULL_TREE)
    6947              :         {
    6948          565 :           sctx.idx = create_tmp_var (unsigned_type_node);
    6949          565 :           sctx.lane = create_tmp_var (unsigned_type_node);
    6950              :         }
    6951              :       /* FIXME: For now.  */
    6952          625 :       sctx.is_simt = false;
    6953              :     }
    6954              : 
    6955        77762 :   if (sctx.lane || sctx.is_simt)
    6956              :     {
    6957         4007 :       uid = create_tmp_var (ptr_type_node, "simduid");
    6958              :       /* Don't want uninit warnings on simduid, it is always uninitialized,
    6959              :          but we use it not for the value, but for the DECL_UID only.  */
    6960         4007 :       suppress_warning (uid, OPT_Wuninitialized);
    6961         4007 :       c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SIMDUID_);
    6962         4007 :       OMP_CLAUSE__SIMDUID__DECL (c) = uid;
    6963         4007 :       OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
    6964         4007 :       gimple_omp_for_set_clauses (ctx->stmt, c);
    6965              :     }
    6966              :   /* Emit calls denoting privatized variables and initializing a pointer to
    6967              :      structure that holds private variables as fields after ompdevlow pass.  */
    6968        77762 :   if (sctx.is_simt)
    6969              :     {
    6970            0 :       sctx.simt_eargs[0] = uid;
    6971            0 :       gimple *g
    6972            0 :         = gimple_build_call_internal_vec (IFN_GOMP_SIMT_ENTER, sctx.simt_eargs);
    6973            0 :       gimple_call_set_lhs (g, uid);
    6974            0 :       gimple_seq_add_stmt (ilist, g);
    6975            0 :       sctx.simt_eargs.release ();
    6976              : 
    6977            0 :       simtrec = create_tmp_var (ptr_type_node, ".omp_simt");
    6978            0 :       g = gimple_build_call_internal (IFN_GOMP_SIMT_ENTER_ALLOC, 1, uid);
    6979            0 :       gimple_call_set_lhs (g, simtrec);
    6980            0 :       gimple_seq_add_stmt (ilist, g);
    6981              :     }
    6982        77762 :   if (sctx.lane)
    6983              :     {
    6984         7389 :       gimple *g = gimple_build_call_internal (IFN_GOMP_SIMD_LANE,
    6985              :                                               2 + (nonconst_simd_if != NULL),
    6986              :                                               uid, integer_zero_node,
    6987              :                                               nonconst_simd_if);
    6988         4007 :       gimple_call_set_lhs (g, sctx.lane);
    6989         4007 :       gimple_stmt_iterator gsi = gsi_start (*gimple_omp_body_ptr (ctx->stmt));
    6990         4007 :       gsi_insert_before_without_update (&gsi, g, GSI_SAME_STMT);
    6991         4007 :       g = gimple_build_assign (sctx.lane, INTEGER_CST,
    6992              :                                build_int_cst (unsigned_type_node, 0));
    6993         4007 :       gimple_seq_add_stmt (ilist, g);
    6994         4007 :       if (sctx.lastlane)
    6995              :         {
    6996          184 :           g = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
    6997              :                                           2, uid, sctx.lane);
    6998          184 :           gimple_call_set_lhs (g, sctx.lastlane);
    6999          184 :           gimple_seq_add_stmt (dlist, g);
    7000          184 :           gimple_seq_add_seq (dlist, llist[3]);
    7001              :         }
    7002              :       /* Emit reductions across SIMT lanes in log_2(simt_vf) steps.  */
    7003         4007 :       if (llist[2])
    7004              :         {
    7005            0 :           tree simt_vf = create_tmp_var (unsigned_type_node);
    7006            0 :           g = gimple_build_call_internal (IFN_GOMP_SIMT_VF, 0);
    7007            0 :           gimple_call_set_lhs (g, simt_vf);
    7008            0 :           gimple_seq_add_stmt (dlist, g);
    7009              : 
    7010            0 :           tree t = build_int_cst (unsigned_type_node, 1);
    7011            0 :           g = gimple_build_assign (simt_lane, INTEGER_CST, t);
    7012            0 :           gimple_seq_add_stmt (dlist, g);
    7013              : 
    7014            0 :           t = build_int_cst (unsigned_type_node, 0);
    7015            0 :           g = gimple_build_assign (sctx.idx, INTEGER_CST, t);
    7016            0 :           gimple_seq_add_stmt (dlist, g);
    7017              : 
    7018            0 :           tree body = create_artificial_label (UNKNOWN_LOCATION);
    7019            0 :           tree header = create_artificial_label (UNKNOWN_LOCATION);
    7020            0 :           tree end = create_artificial_label (UNKNOWN_LOCATION);
    7021            0 :           gimple_seq_add_stmt (dlist, gimple_build_goto (header));
    7022            0 :           gimple_seq_add_stmt (dlist, gimple_build_label (body));
    7023              : 
    7024            0 :           gimple_seq_add_seq (dlist, llist[2]);
    7025              : 
    7026            0 :           g = gimple_build_assign (simt_lane, LSHIFT_EXPR, simt_lane, integer_one_node);
    7027            0 :           gimple_seq_add_stmt (dlist, g);
    7028              : 
    7029            0 :           gimple_seq_add_stmt (dlist, gimple_build_label (header));
    7030            0 :           g = gimple_build_cond (LT_EXPR, simt_lane, simt_vf, body, end);
    7031            0 :           gimple_seq_add_stmt (dlist, g);
    7032              : 
    7033            0 :           gimple_seq_add_stmt (dlist, gimple_build_label (end));
    7034              :         }
    7035        12021 :       for (int i = 0; i < 2; i++)
    7036         8014 :         if (llist[i])
    7037              :           {
    7038         1730 :             tree vf = create_tmp_var (unsigned_type_node);
    7039         1730 :             g = gimple_build_call_internal (IFN_GOMP_SIMD_VF, 1, uid);
    7040         1730 :             gimple_call_set_lhs (g, vf);
    7041         1730 :             gimple_seq *seq = i == 0 ? ilist : dlist;
    7042         1730 :             gimple_seq_add_stmt (seq, g);
    7043         1730 :             tree t = build_int_cst (unsigned_type_node, 0);
    7044         1730 :             g = gimple_build_assign (sctx.idx, INTEGER_CST, t);
    7045         1730 :             gimple_seq_add_stmt (seq, g);
    7046         1730 :             tree body = create_artificial_label (UNKNOWN_LOCATION);
    7047         1730 :             tree header = create_artificial_label (UNKNOWN_LOCATION);
    7048         1730 :             tree end = create_artificial_label (UNKNOWN_LOCATION);
    7049         1730 :             gimple_seq_add_stmt (seq, gimple_build_goto (header));
    7050         1730 :             gimple_seq_add_stmt (seq, gimple_build_label (body));
    7051         1730 :             gimple_seq_add_seq (seq, llist[i]);
    7052         1730 :             t = build_int_cst (unsigned_type_node, 1);
    7053         1730 :             g = gimple_build_assign (sctx.idx, PLUS_EXPR, sctx.idx, t);
    7054         1730 :             gimple_seq_add_stmt (seq, g);
    7055         1730 :             gimple_seq_add_stmt (seq, gimple_build_label (header));
    7056         1730 :             g = gimple_build_cond (LT_EXPR, sctx.idx, vf, body, end);
    7057         1730 :             gimple_seq_add_stmt (seq, g);
    7058         1730 :             gimple_seq_add_stmt (seq, gimple_build_label (end));
    7059              :           }
    7060              :     }
    7061        77762 :   if (sctx.is_simt)
    7062              :     {
    7063            0 :       gimple_seq_add_seq (dlist, sctx.simt_dlist);
    7064            0 :       gimple *g
    7065            0 :         = gimple_build_call_internal (IFN_GOMP_SIMT_EXIT, 1, simtrec);
    7066            0 :       gimple_seq_add_stmt (dlist, g);
    7067              :     }
    7068              : 
    7069              :   /* The copyin sequence is not to be executed by the main thread, since
    7070              :      that would result in self-copies.  Perhaps not visible to scalars,
    7071              :      but it certainly is to C++ operator=.  */
    7072        77762 :   if (copyin_seq)
    7073              :     {
    7074          440 :       x = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM),
    7075              :                            0);
    7076          440 :       x = build2 (NE_EXPR, boolean_type_node, x,
    7077          440 :                   build_int_cst (TREE_TYPE (x), 0));
    7078          440 :       x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL);
    7079          440 :       gimplify_and_add (x, ilist);
    7080              :     }
    7081              : 
    7082              :   /* If any copyin variable is passed by reference, we must ensure the
    7083              :      master thread doesn't modify it before it is copied over in all
    7084              :      threads.  Similarly for variables in both firstprivate and
    7085              :      lastprivate clauses we need to ensure the lastprivate copying
    7086              :      happens after firstprivate copying in all threads.  And similarly
    7087              :      for UDRs if initializer expression refers to omp_orig.  */
    7088        77762 :   if (copyin_by_ref || lastprivate_firstprivate
    7089        75822 :       || (reduction_omp_orig_ref
    7090          101 :           && !ctx->scan_inclusive
    7091          101 :           && !ctx->scan_exclusive))
    7092              :     {
    7093              :       /* Don't add any barrier for #pragma omp simd or
    7094              :          #pragma omp distribute.  */
    7095         2041 :       if (!is_task_ctx (ctx)
    7096         2041 :           && (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
    7097         1486 :               || gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_FOR))
    7098              :         {
    7099          904 :           int barrier_kind;
    7100          904 :           switch (gimple_code (ctx->stmt))
    7101              :             {
    7102              :             case GIMPLE_OMP_SECTIONS:
    7103              :             case GIMPLE_OMP_SCOPE:
    7104              :             case GIMPLE_OMP_FOR:
    7105              :               barrier_kind = GOMP_BARRIER_IMPLICIT_WORKSHARE;
    7106              :               break;
    7107          540 :             case GIMPLE_OMP_PARALLEL:
    7108          540 :             case GIMPLE_OMP_TEAMS:
    7109          540 :               barrier_kind = GOMP_BARRIER_IMPLICIT_PARALLEL;
    7110          540 :               break;
    7111            0 :             default:
    7112            0 :               gcc_unreachable ();
    7113              :             }
    7114          904 :           gimple_seq_add_stmt (ilist,
    7115              :                                omp_build_barrier (NULL_TREE, barrier_kind));
    7116              :         }
    7117              :     }
    7118              : 
    7119              :   /* If max_vf is non-zero, then we can use only a vectorization factor
    7120              :      up to the max_vf we chose.  So stick it into the safelen clause.  */
    7121        77762 :   if (maybe_ne (sctx.max_vf, 0U))
    7122              :     {
    7123         5231 :       tree c = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
    7124              :                                 OMP_CLAUSE_SAFELEN);
    7125         5231 :       poly_uint64 safe_len;
    7126         5231 :       if (c == NULL_TREE
    7127         5231 :           || (poly_int_tree_p (OMP_CLAUSE_SAFELEN_EXPR (c), &safe_len)
    7128          618 :               && maybe_gt (safe_len, sctx.max_vf)))
    7129              :         {
    7130         5148 :           c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_SAFELEN);
    7131         5148 :           OMP_CLAUSE_SAFELEN_EXPR (c) = build_int_cst (integer_type_node,
    7132         5148 :                                                        sctx.max_vf);
    7133         5148 :           OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
    7134         5148 :           gimple_omp_for_set_clauses (ctx->stmt, c);
    7135              :         }
    7136              :     }
    7137        77762 : }
    7138              : 
    7139              : /* Create temporary variables for lastprivate(conditional:) implementation
    7140              :    in context CTX with CLAUSES.  */
    7141              : 
    7142              : static void
    7143        47488 : lower_lastprivate_conditional_clauses (tree *clauses, omp_context *ctx)
    7144              : {
    7145        47488 :   tree iter_type = NULL_TREE;
    7146        47488 :   tree cond_ptr = NULL_TREE;
    7147        47488 :   tree iter_var = NULL_TREE;
    7148        47488 :   bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
    7149        47488 :                   && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD);
    7150        47488 :   tree next = *clauses;
    7151       227790 :   for (tree c = *clauses; c; c = OMP_CLAUSE_CHAIN (c))
    7152       180302 :     if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
    7153       180302 :         && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
    7154              :       {
    7155          386 :         if (is_simd)
    7156              :           {
    7157          107 :             tree cc = omp_find_clause (next, OMP_CLAUSE__CONDTEMP_);
    7158          107 :             gcc_assert (cc);
    7159          107 :             if (iter_type == NULL_TREE)
    7160              :               {
    7161           77 :                 iter_type = TREE_TYPE (OMP_CLAUSE_DECL (cc));
    7162           77 :                 iter_var = create_tmp_var_raw (iter_type);
    7163           77 :                 DECL_CONTEXT (iter_var) = current_function_decl;
    7164           77 :                 DECL_SEEN_IN_BIND_EXPR_P (iter_var) = 1;
    7165           77 :                 DECL_CHAIN (iter_var) = ctx->block_vars;
    7166           77 :                 ctx->block_vars = iter_var;
    7167           77 :                 tree c3
    7168           77 :                   = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__CONDTEMP_);
    7169           77 :                 OMP_CLAUSE__CONDTEMP__ITER (c3) = 1;
    7170           77 :                 OMP_CLAUSE_DECL (c3) = iter_var;
    7171           77 :                 OMP_CLAUSE_CHAIN (c3) = *clauses;
    7172           77 :                 *clauses = c3;
    7173           77 :                 ctx->lastprivate_conditional_map = new hash_map<tree, tree>;
    7174              :               }
    7175          107 :             next = OMP_CLAUSE_CHAIN (cc);
    7176          107 :             tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
    7177          107 :             tree v = lookup_decl (OMP_CLAUSE_DECL (cc), ctx);
    7178          107 :             ctx->lastprivate_conditional_map->put (o, v);
    7179          107 :             continue;
    7180          107 :           }
    7181          279 :         if (iter_type == NULL)
    7182              :           {
    7183          205 :             if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR)
    7184              :               {
    7185          193 :                 struct omp_for_data fd;
    7186          193 :                 omp_extract_for_data (as_a <gomp_for *> (ctx->stmt), &fd,
    7187              :                                       NULL);
    7188          193 :                 iter_type = unsigned_type_for (fd.iter_type);
    7189              :               }
    7190           12 :             else if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
    7191           12 :               iter_type = unsigned_type_node;
    7192          205 :             tree c2 = omp_find_clause (*clauses, OMP_CLAUSE__CONDTEMP_);
    7193          205 :             if (c2)
    7194              :               {
    7195          103 :                 cond_ptr
    7196          103 :                   = lookup_decl_in_outer_ctx (OMP_CLAUSE_DECL (c2), ctx);
    7197          103 :                 OMP_CLAUSE_DECL (c2) = cond_ptr;
    7198              :               }
    7199              :             else
    7200              :               {
    7201          102 :                 cond_ptr = create_tmp_var_raw (build_pointer_type (iter_type));
    7202          102 :                 DECL_CONTEXT (cond_ptr) = current_function_decl;
    7203          102 :                 DECL_SEEN_IN_BIND_EXPR_P (cond_ptr) = 1;
    7204          102 :                 DECL_CHAIN (cond_ptr) = ctx->block_vars;
    7205          102 :                 ctx->block_vars = cond_ptr;
    7206          102 :                 c2 = build_omp_clause (UNKNOWN_LOCATION,
    7207              :                                        OMP_CLAUSE__CONDTEMP_);
    7208          102 :                 OMP_CLAUSE_DECL (c2) = cond_ptr;
    7209          102 :                 OMP_CLAUSE_CHAIN (c2) = *clauses;
    7210          102 :                 *clauses = c2;
    7211              :               }
    7212          205 :             iter_var = create_tmp_var_raw (iter_type);
    7213          205 :             DECL_CONTEXT (iter_var) = current_function_decl;
    7214          205 :             DECL_SEEN_IN_BIND_EXPR_P (iter_var) = 1;
    7215          205 :             DECL_CHAIN (iter_var) = ctx->block_vars;
    7216          205 :             ctx->block_vars = iter_var;
    7217          205 :             tree c3
    7218          205 :               = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__CONDTEMP_);
    7219          205 :             OMP_CLAUSE__CONDTEMP__ITER (c3) = 1;
    7220          205 :             OMP_CLAUSE_DECL (c3) = iter_var;
    7221          205 :             OMP_CLAUSE_CHAIN (c3) = OMP_CLAUSE_CHAIN (c2);
    7222          205 :             OMP_CLAUSE_CHAIN (c2) = c3;
    7223          205 :             ctx->lastprivate_conditional_map = new hash_map<tree, tree>;
    7224              :           }
    7225          279 :         tree v = create_tmp_var_raw (iter_type);
    7226          279 :         DECL_CONTEXT (v) = current_function_decl;
    7227          279 :         DECL_SEEN_IN_BIND_EXPR_P (v) = 1;
    7228          279 :         DECL_CHAIN (v) = ctx->block_vars;
    7229          279 :         ctx->block_vars = v;
    7230          279 :         tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
    7231          279 :         ctx->lastprivate_conditional_map->put (o, v);
    7232              :       }
    7233        47488 : }
    7234              : 
    7235              : 
    7236              : /* Generate code to implement the LASTPRIVATE clauses.  This is used for
    7237              :    both parallel and workshare constructs.  PREDICATE may be NULL if it's
    7238              :    always true.  BODY_P is the sequence to insert early initialization
    7239              :    if needed, STMT_LIST is where the non-conditional lastprivate handling
    7240              :    goes into and CSTMT_LIST is a sequence that needs to be run in a critical
    7241              :    section.  */
    7242              : 
    7243              : static void
    7244        47488 : lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *body_p,
    7245              :                            gimple_seq *stmt_list, gimple_seq *cstmt_list,
    7246              :                            omp_context *ctx)
    7247              : {
    7248        47488 :   tree x, c, label = NULL, orig_clauses = clauses;
    7249        47488 :   bool par_clauses = false;
    7250        47488 :   tree simduid = NULL, lastlane = NULL, simtcond = NULL, simtlast = NULL;
    7251        47488 :   unsigned HOST_WIDE_INT conditional_off = 0;
    7252        47488 :   gimple_seq post_stmt_list = NULL;
    7253              : 
    7254              :   /* Early exit if there are no lastprivate or linear clauses.  */
    7255       197051 :   for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
    7256       162059 :     if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LASTPRIVATE
    7257       162059 :         || (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LINEAR
    7258         6797 :             && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses)))
    7259              :       break;
    7260        47488 :   if (clauses == NULL)
    7261              :     {
    7262              :       /* If this was a workshare clause, see if it had been combined
    7263              :          with its parallel.  In that case, look for the clauses on the
    7264              :          parallel statement itself.  */
    7265        34992 :       if (is_parallel_ctx (ctx))
    7266        30989 :         return;
    7267              : 
    7268        34992 :       ctx = ctx->outer;
    7269        34992 :       if (ctx == NULL || !is_parallel_ctx (ctx))
    7270              :         return;
    7271              : 
    7272        12312 :       clauses = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
    7273              :                                  OMP_CLAUSE_LASTPRIVATE);
    7274        12312 :       if (clauses == NULL)
    7275              :         return;
    7276              :       par_clauses = true;
    7277              :     }
    7278              : 
    7279        16499 :   bool maybe_simt = false;
    7280        16499 :   if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
    7281        16499 :       && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
    7282              :     {
    7283         7431 :       maybe_simt = omp_find_clause (orig_clauses, OMP_CLAUSE__SIMT_);
    7284         7431 :       simduid = omp_find_clause (orig_clauses, OMP_CLAUSE__SIMDUID_);
    7285         7431 :       if (simduid)
    7286         3552 :         simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
    7287              :     }
    7288              : 
    7289        16499 :   if (predicate)
    7290              :     {
    7291        16376 :       gcond *stmt;
    7292        16376 :       tree label_true, arm1, arm2;
    7293        16376 :       enum tree_code pred_code = TREE_CODE (predicate);
    7294              : 
    7295        16376 :       label = create_artificial_label (UNKNOWN_LOCATION);
    7296        16376 :       label_true = create_artificial_label (UNKNOWN_LOCATION);
    7297        16376 :       if (TREE_CODE_CLASS (pred_code) == tcc_comparison)
    7298              :         {
    7299        16376 :           arm1 = TREE_OPERAND (predicate, 0);
    7300        16376 :           arm2 = TREE_OPERAND (predicate, 1);
    7301        16376 :           gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
    7302        16376 :           gimplify_expr (&arm2, stmt_list, NULL, is_gimple_val, fb_rvalue);
    7303              :         }
    7304              :       else
    7305              :         {
    7306            0 :           arm1 = predicate;
    7307            0 :           gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
    7308            0 :           arm2 = boolean_false_node;
    7309            0 :           pred_code = NE_EXPR;
    7310              :         }
    7311        16376 :       if (maybe_simt)
    7312              :         {
    7313            0 :           c = build2 (pred_code, boolean_type_node, arm1, arm2);
    7314            0 :           c = fold_convert (integer_type_node, c);
    7315            0 :           simtcond = create_tmp_var (integer_type_node);
    7316            0 :           gimplify_assign (simtcond, c, stmt_list);
    7317            0 :           gcall *g = gimple_build_call_internal (IFN_GOMP_SIMT_VOTE_ANY,
    7318              :                                                  1, simtcond);
    7319            0 :           c = create_tmp_var (integer_type_node);
    7320            0 :           gimple_call_set_lhs (g, c);
    7321            0 :           gimple_seq_add_stmt (stmt_list, g);
    7322            0 :           stmt = gimple_build_cond (NE_EXPR, c, integer_zero_node,
    7323              :                                     label_true, label);
    7324              :         }
    7325              :       else
    7326        16376 :         stmt = gimple_build_cond (pred_code, arm1, arm2, label_true, label);
    7327        16376 :       gimple_seq_add_stmt (stmt_list, stmt);
    7328        16376 :       gimple_seq_add_stmt (stmt_list, gimple_build_label (label_true));
    7329              :     }
    7330              : 
    7331        16499 :   tree cond_ptr = NULL_TREE;
    7332        61608 :   for (c = clauses; c ;)
    7333              :     {
    7334        56376 :       tree var, new_var;
    7335        56376 :       location_t clause_loc = OMP_CLAUSE_LOCATION (c);
    7336        56376 :       gimple_seq *this_stmt_list = stmt_list;
    7337        56376 :       tree lab2 = NULL_TREE;
    7338              : 
    7339        56376 :       if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
    7340        21824 :           && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c)
    7341          386 :           && ctx->lastprivate_conditional_map
    7342        56749 :           && !ctx->combined_into_simd_safelen1)
    7343              :         {
    7344          316 :           gcc_assert (body_p);
    7345          316 :           if (simduid)
    7346           37 :             goto next;
    7347          279 :           if (cond_ptr == NULL_TREE)
    7348              :             {
    7349          205 :               cond_ptr = omp_find_clause (orig_clauses, OMP_CLAUSE__CONDTEMP_);
    7350          205 :               cond_ptr = OMP_CLAUSE_DECL (cond_ptr);
    7351              :             }
    7352          279 :           tree type = TREE_TYPE (TREE_TYPE (cond_ptr));
    7353          279 :           tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
    7354          279 :           tree v = *ctx->lastprivate_conditional_map->get (o);
    7355          279 :           gimplify_assign (v, build_zero_cst (type), body_p);
    7356          279 :           this_stmt_list = cstmt_list;
    7357          279 :           tree mem;
    7358          279 :           if (POINTER_TYPE_P (TREE_TYPE (cond_ptr)))
    7359              :             {
    7360          136 :               mem = build2 (MEM_REF, type, cond_ptr,
    7361          136 :                             build_int_cst (TREE_TYPE (cond_ptr),
    7362          136 :                                            conditional_off));
    7363          136 :               conditional_off += tree_to_uhwi (TYPE_SIZE_UNIT (type));
    7364              :             }
    7365              :           else
    7366          143 :             mem = build4 (ARRAY_REF, type, cond_ptr,
    7367          143 :                           size_int (conditional_off++), NULL_TREE, NULL_TREE);
    7368          279 :           tree mem2 = copy_node (mem);
    7369          279 :           gimple_seq seq = NULL;
    7370          279 :           mem = force_gimple_operand (mem, &seq, true, NULL_TREE);
    7371          279 :           gimple_seq_add_seq (this_stmt_list, seq);
    7372          279 :           tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
    7373          279 :           lab2 = create_artificial_label (UNKNOWN_LOCATION);
    7374          279 :           gimple *g = gimple_build_cond (GT_EXPR, v, mem, lab1, lab2);
    7375          279 :           gimple_seq_add_stmt (this_stmt_list, g);
    7376          279 :           gimple_seq_add_stmt (this_stmt_list, gimple_build_label (lab1));
    7377          279 :           gimplify_assign (mem2, v, this_stmt_list);
    7378              :         }
    7379        56060 :       else if (predicate
    7380        55814 :                && ctx->combined_into_simd_safelen1
    7381          116 :                && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
    7382           57 :                && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c)
    7383        56117 :                && ctx->lastprivate_conditional_map)
    7384              :         this_stmt_list = &post_stmt_list;
    7385              : 
    7386        56339 :       if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
    7387        56339 :           || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
    7388         4874 :               && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c)))
    7389              :         {
    7390        26661 :           var = OMP_CLAUSE_DECL (c);
    7391        26661 :           if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
    7392        21787 :               && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)
    7393        27692 :               && is_taskloop_ctx (ctx))
    7394              :             {
    7395          312 :               gcc_checking_assert (ctx->outer && is_task_ctx (ctx->outer));
    7396          312 :               new_var = lookup_decl (var, ctx->outer);
    7397              :             }
    7398              :           else
    7399              :             {
    7400        26349 :               new_var = lookup_decl (var, ctx);
    7401              :               /* Avoid uninitialized warnings for lastprivate and
    7402              :                  for linear iterators.  */
    7403        26349 :               if (predicate
    7404        52505 :                   && (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
    7405         4874 :                       || OMP_CLAUSE_LINEAR_NO_COPYIN (c)))
    7406        24887 :                 suppress_warning (new_var, OPT_Wuninitialized);
    7407              :             }
    7408              : 
    7409        26661 :           if (!maybe_simt && simduid && DECL_HAS_VALUE_EXPR_P (new_var))
    7410              :             {
    7411         5864 :               tree val = DECL_VALUE_EXPR (new_var);
    7412         5864 :               if (TREE_CODE (val) == ARRAY_REF
    7413         5816 :                   && VAR_P (TREE_OPERAND (val, 0))
    7414        11680 :                   && lookup_attribute ("omp simd array",
    7415         5816 :                                        DECL_ATTRIBUTES (TREE_OPERAND (val,
    7416              :                                                                       0))))
    7417              :                 {
    7418         5816 :                   if (lastlane == NULL)
    7419              :                     {
    7420         2759 :                       lastlane = create_tmp_var (unsigned_type_node);
    7421         2759 :                       gcall *g
    7422         2759 :                         = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
    7423              :                                                       2, simduid,
    7424         2759 :                                                       TREE_OPERAND (val, 1));
    7425         2759 :                       gimple_call_set_lhs (g, lastlane);
    7426         2759 :                       gimple_seq_add_stmt (this_stmt_list, g);
    7427              :                     }
    7428         5816 :                   new_var = build4 (ARRAY_REF, TREE_TYPE (val),
    7429         5816 :                                     TREE_OPERAND (val, 0), lastlane,
    7430              :                                     NULL_TREE, NULL_TREE);
    7431         5816 :                   TREE_THIS_NOTRAP (new_var) = 1;
    7432              :                 }
    7433              :             }
    7434        20797 :           else if (maybe_simt)
    7435              :             {
    7436            0 :               tree val = (DECL_HAS_VALUE_EXPR_P (new_var)
    7437            0 :                           ? DECL_VALUE_EXPR (new_var)
    7438              :                           : new_var);
    7439            0 :               if (simtlast == NULL)
    7440              :                 {
    7441            0 :                   simtlast = create_tmp_var (unsigned_type_node);
    7442            0 :                   gcall *g = gimple_build_call_internal
    7443            0 :                     (IFN_GOMP_SIMT_LAST_LANE, 1, simtcond);
    7444            0 :                   gimple_call_set_lhs (g, simtlast);
    7445            0 :                   gimple_seq_add_stmt (this_stmt_list, g);
    7446              :                 }
    7447            0 :               x = build_call_expr_internal_loc
    7448            0 :                 (UNKNOWN_LOCATION, IFN_GOMP_SIMT_XCHG_IDX,
    7449            0 :                  TREE_TYPE (val), 2, val, simtlast);
    7450            0 :               new_var = unshare_expr (new_var);
    7451            0 :               gimplify_assign (new_var, x, this_stmt_list);
    7452            0 :               new_var = unshare_expr (new_var);
    7453              :             }
    7454              : 
    7455        26661 :           if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
    7456        26661 :               && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
    7457              :             {
    7458         7159 :               lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
    7459         7159 :               gimple_seq_add_seq (this_stmt_list,
    7460         7159 :                                   OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c));
    7461         7159 :               OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL;
    7462              :             }
    7463        19502 :           else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
    7464        19502 :                    && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
    7465              :             {
    7466          426 :               lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
    7467          426 :               gimple_seq_add_seq (this_stmt_list,
    7468          426 :                                   OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c));
    7469          426 :               OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c) = NULL;
    7470              :             }
    7471              : 
    7472        26661 :           x = NULL_TREE;
    7473        26661 :           if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
    7474        21787 :               && OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c)
    7475        26721 :               && is_taskloop_ctx (ctx))
    7476              :             {
    7477           76 :               tree ovar = maybe_lookup_decl_in_outer_ctx (var,
    7478           38 :                                                           ctx->outer->outer);
    7479           38 :               if (is_global_var (ovar))
    7480            7 :                 x = ovar;
    7481              :             }
    7482            7 :           if (!x)
    7483        26654 :             x = build_outer_var_ref (var, ctx, OMP_CLAUSE_LASTPRIVATE);
    7484        26661 :           if (omp_privatize_by_reference (var))
    7485          657 :             new_var = build_simple_mem_ref_loc (clause_loc, new_var);
    7486        26661 :           x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var);
    7487        26661 :           gimplify_and_add (x, this_stmt_list);
    7488              : 
    7489        26661 :           if (lab2)
    7490          279 :             gimple_seq_add_stmt (this_stmt_list, gimple_build_label (lab2));
    7491              :         }
    7492              : 
    7493        56376 :      next:
    7494        56376 :       c = OMP_CLAUSE_CHAIN (c);
    7495        56376 :       if (c == NULL && !par_clauses)
    7496              :         {
    7497              :           /* If this was a workshare clause, see if it had been combined
    7498              :              with its parallel.  In that case, continue looking for the
    7499              :              clauses also on the parallel statement itself.  */
    7500        12496 :           if (is_parallel_ctx (ctx))
    7501              :             break;
    7502              : 
    7503        12496 :           ctx = ctx->outer;
    7504        12496 :           if (ctx == NULL || !is_parallel_ctx (ctx))
    7505              :             break;
    7506              : 
    7507         1229 :           c = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
    7508              :                                OMP_CLAUSE_LASTPRIVATE);
    7509         1229 :           par_clauses = true;
    7510              :         }
    7511              :     }
    7512              : 
    7513        16499 :   if (label)
    7514        16376 :     gimple_seq_add_stmt (stmt_list, gimple_build_label (label));
    7515        16499 :   gimple_seq_add_seq (stmt_list, post_stmt_list);
    7516              : }
    7517              : 
    7518              : /* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL
    7519              :    (which might be a placeholder).  INNER is true if this is an inner
    7520              :    axis of a multi-axis loop.  FORK and JOIN are (optional) fork and
    7521              :    join markers.  Generate the before-loop forking sequence in
    7522              :    FORK_SEQ and the after-loop joining sequence to JOIN_SEQ.  The
    7523              :    general form of these sequences is
    7524              : 
    7525              :      GOACC_REDUCTION_SETUP
    7526              :      GOACC_FORK
    7527              :      GOACC_REDUCTION_INIT
    7528              :      ...
    7529              :      GOACC_REDUCTION_FINI
    7530              :      GOACC_JOIN
    7531              :      GOACC_REDUCTION_TEARDOWN.  */
    7532              : 
    7533              : static void
    7534        25946 : lower_oacc_reductions (location_t loc, tree clauses, tree level, bool inner,
    7535              :                        gcall *fork, gcall *private_marker, gcall *join,
    7536              :                        gimple_seq *fork_seq, gimple_seq *join_seq,
    7537              :                        omp_context *ctx)
    7538              : {
    7539        25946 :   gimple_seq before_fork = NULL;
    7540        25946 :   gimple_seq after_fork = NULL;
    7541        25946 :   gimple_seq before_join = NULL;
    7542        25946 :   gimple_seq after_join = NULL;
    7543        25946 :   tree init_code = NULL_TREE, fini_code = NULL_TREE,
    7544        25946 :     setup_code = NULL_TREE, teardown_code = NULL_TREE;
    7545        25946 :   unsigned offset = 0;
    7546              : 
    7547        96516 :   for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
    7548        70570 :     if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
    7549              :       {
    7550              :         /* No 'reduction' clauses on OpenACC 'kernels'.  */
    7551         7751 :         gcc_checking_assert (!is_oacc_kernels (ctx));
    7552              :         /* Likewise, on OpenACC 'kernels' decomposed parts.  */
    7553         7751 :         gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
    7554              : 
    7555         7751 :         tree orig = OMP_CLAUSE_DECL (c);
    7556         7751 :         tree orig_clause;
    7557         7751 :         tree var = maybe_lookup_decl (orig, ctx);
    7558         7751 :         tree ref_to_res = NULL_TREE;
    7559         7751 :         tree incoming, outgoing, v1, v2, v3;
    7560         7751 :         bool is_private = false;
    7561              : 
    7562         7751 :         enum tree_code rcode = OMP_CLAUSE_REDUCTION_CODE (c);
    7563         7751 :         if (rcode == MINUS_EXPR)
    7564              :           rcode = PLUS_EXPR;
    7565         6609 :         else if (rcode == TRUTH_ANDIF_EXPR)
    7566              :           rcode = BIT_AND_EXPR;
    7567         6431 :         else if (rcode == TRUTH_ORIF_EXPR)
    7568          433 :           rcode = BIT_IOR_EXPR;
    7569         7751 :         tree op = build_int_cst (unsigned_type_node, rcode);
    7570              : 
    7571         7751 :         if (!var)
    7572            4 :           var = orig;
    7573              : 
    7574         7751 :         incoming = outgoing = var;
    7575              : 
    7576         7751 :         if (!inner)
    7577              :           {
    7578              :             /* See if an outer construct also reduces this variable.  */
    7579              :             omp_context *outer = ctx;
    7580              : 
    7581         7385 :             while (omp_context *probe = outer->outer)
    7582              :               {
    7583         4708 :                 enum gimple_code type = gimple_code (probe->stmt);
    7584         4708 :                 tree cls;
    7585              : 
    7586         4708 :                 switch (type)
    7587              :                   {
    7588         2220 :                   case GIMPLE_OMP_FOR:
    7589         2220 :                     cls = gimple_omp_for_clauses (probe->stmt);
    7590         2220 :                     break;
    7591              : 
    7592         2488 :                   case GIMPLE_OMP_TARGET:
    7593              :                     /* No 'reduction' clauses inside OpenACC 'kernels'
    7594              :                        regions.  */
    7595         2488 :                     gcc_checking_assert (!is_oacc_kernels (probe));
    7596              : 
    7597         2488 :                     if (!is_gimple_omp_offloaded (probe->stmt))
    7598           26 :                       goto do_lookup;
    7599              : 
    7600         2462 :                     cls = gimple_omp_target_clauses (probe->stmt);
    7601         2462 :                     break;
    7602              : 
    7603            0 :                   default:
    7604            0 :                     goto do_lookup;
    7605              :                   }
    7606              : 
    7607         4682 :                 outer = probe;
    7608        14249 :                 for (; cls;  cls = OMP_CLAUSE_CHAIN (cls))
    7609        11632 :                   if (OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_REDUCTION
    7610        11632 :                       && orig == OMP_CLAUSE_DECL (cls))
    7611              :                     {
    7612         1772 :                       incoming = outgoing = lookup_decl (orig, probe);
    7613         1772 :                       goto has_outer_reduction;
    7614              :                     }
    7615         9860 :                   else if ((OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_FIRSTPRIVATE
    7616         8971 :                             || OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_PRIVATE)
    7617        11164 :                            && orig == OMP_CLAUSE_DECL (cls))
    7618              :                     {
    7619          293 :                       is_private = true;
    7620          293 :                       goto do_lookup;
    7621              :                     }
    7622              :               }
    7623              : 
    7624         2677 :           do_lookup:
    7625              :             /* This is the outermost construct with this reduction,
    7626              :                see if there's a mapping for it.  */
    7627         2996 :             orig_clause = NULL_TREE;
    7628         2996 :             if (gimple_code (outer->stmt) == GIMPLE_OMP_TARGET)
    7629         7557 :               for (tree cls = gimple_omp_target_clauses (outer->stmt);
    7630         7557 :                    cls; cls = OMP_CLAUSE_CHAIN (cls))
    7631         7250 :                 if (OMP_CLAUSE_CODE (cls) == OMP_CLAUSE_MAP
    7632         4823 :                     && orig == OMP_CLAUSE_DECL (cls)
    7633         9805 :                     && maybe_lookup_field (cls, outer))
    7634              :                   {
    7635              :                     orig_clause = cls;
    7636              :                     break;
    7637              :                   }
    7638         2996 :             if (orig_clause != NULL_TREE && !is_private)
    7639              :               {
    7640         2553 :                 ref_to_res = build_receiver_ref (orig_clause, false, outer);
    7641         2553 :                 if (omp_privatize_by_reference (orig))
    7642           79 :                   ref_to_res = build_simple_mem_ref (ref_to_res);
    7643              : 
    7644         2553 :                 tree type = TREE_TYPE (var);
    7645         2553 :                 if (POINTER_TYPE_P (type))
    7646           79 :                   type = TREE_TYPE (type);
    7647              : 
    7648         2553 :                 outgoing = var;
    7649         2553 :                 incoming = omp_reduction_init_op (loc, rcode, type);
    7650              :               }
    7651              :             else
    7652              :               {
    7653              :                 /* Try to look at enclosing contexts for reduction var,
    7654              :                    use original if no mapping found.  */
    7655          443 :                 tree t = NULL_TREE;
    7656          443 :                 omp_context *c = ctx->outer;
    7657          895 :                 while (c && !t)
    7658              :                   {
    7659          452 :                     t = maybe_lookup_decl (orig, c);
    7660          452 :                     c = c->outer;
    7661              :                   }
    7662          443 :                 incoming = outgoing = (t ? t : orig);
    7663              :               }
    7664              : 
    7665         2553 :           has_outer_reduction:;
    7666              :           }
    7667              : 
    7668         4325 :         if (!ref_to_res)
    7669         5198 :           ref_to_res = integer_zero_node;
    7670              : 
    7671         7751 :         if (omp_privatize_by_reference (orig))
    7672              :           {
    7673          159 :             tree type = TREE_TYPE (var);
    7674          159 :             const char *id = IDENTIFIER_POINTER (DECL_NAME (var));
    7675              : 
    7676          159 :             if (!inner)
    7677              :               {
    7678          110 :                 tree x = create_tmp_var (TREE_TYPE (type), id);
    7679          110 :                 gimplify_assign (var, build_fold_addr_expr (x), fork_seq);
    7680              :               }
    7681              : 
    7682          159 :             v1 = create_tmp_var (type, id);
    7683          159 :             v2 = create_tmp_var (type, id);
    7684          159 :             v3 = create_tmp_var (type, id);
    7685              : 
    7686          159 :             gimplify_assign (v1, var, fork_seq);
    7687          159 :             gimplify_assign (v2, var, fork_seq);
    7688          159 :             gimplify_assign (v3, var, fork_seq);
    7689              : 
    7690          159 :             var = build_simple_mem_ref (var);
    7691          159 :             v1 = build_simple_mem_ref (v1);
    7692          159 :             v2 = build_simple_mem_ref (v2);
    7693          159 :             v3 = build_simple_mem_ref (v3);
    7694          159 :             outgoing = build_simple_mem_ref (outgoing);
    7695              : 
    7696          159 :             if (!TREE_CONSTANT (incoming))
    7697           80 :               incoming = build_simple_mem_ref (incoming);
    7698              :           }
    7699              :         else
    7700              :           /* Note that 'var' might be a mem ref.  */
    7701              :           v1 = v2 = v3 = var;
    7702              : 
    7703              :         /* Determine position in reduction buffer, which may be used
    7704              :            by target.  The parser has ensured that this is not a
    7705              :            variable-sized type.  */
    7706         7751 :         fixed_size_mode mode
    7707         7751 :           = as_a <fixed_size_mode> (TYPE_MODE (TREE_TYPE (var)));
    7708         7751 :         unsigned align = GET_MODE_ALIGNMENT (mode) /  BITS_PER_UNIT;
    7709         7751 :         offset = (offset + align - 1) & ~(align - 1);
    7710         7751 :         tree off = build_int_cst (sizetype, offset);
    7711         7751 :         offset += GET_MODE_SIZE (mode);
    7712              : 
    7713         7751 :         if (!init_code)
    7714              :           {
    7715         6969 :             init_code = build_int_cst (integer_type_node,
    7716              :                                        IFN_GOACC_REDUCTION_INIT);
    7717         6969 :             fini_code = build_int_cst (integer_type_node,
    7718              :                                        IFN_GOACC_REDUCTION_FINI);
    7719         6969 :             setup_code = build_int_cst (integer_type_node,
    7720              :                                         IFN_GOACC_REDUCTION_SETUP);
    7721         6969 :             teardown_code = build_int_cst (integer_type_node,
    7722              :                                            IFN_GOACC_REDUCTION_TEARDOWN);
    7723              :           }
    7724              : 
    7725         7751 :         tree setup_call
    7726        15502 :           = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
    7727         7751 :                                           TREE_TYPE (var), 6, setup_code,
    7728              :                                           unshare_expr (ref_to_res),
    7729              :                                           unshare_expr (incoming),
    7730              :                                           level, op, off);
    7731         7751 :         tree init_call
    7732        15502 :           = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
    7733         7751 :                                           TREE_TYPE (var), 6, init_code,
    7734              :                                           unshare_expr (ref_to_res),
    7735              :                                           unshare_expr (v1), level, op, off);
    7736         7751 :         tree fini_call
    7737        15502 :           = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
    7738         7751 :                                           TREE_TYPE (var), 6, fini_code,
    7739              :                                           unshare_expr (ref_to_res),
    7740              :                                           unshare_expr (v2), level, op, off);
    7741         7751 :         tree teardown_call
    7742        15502 :           = build_call_expr_internal_loc (loc, IFN_GOACC_REDUCTION,
    7743         7751 :                                           TREE_TYPE (var), 6, teardown_code,
    7744              :                                           ref_to_res, unshare_expr (v3),
    7745              :                                           level, op, off);
    7746              : 
    7747         7751 :         gimplify_assign (unshare_expr (v1), setup_call, &before_fork);
    7748         7751 :         gimplify_assign (unshare_expr (v2), init_call, &after_fork);
    7749         7751 :         gimplify_assign (unshare_expr (v3), fini_call, &before_join);
    7750         7751 :         gimplify_assign (unshare_expr (outgoing), teardown_call, &after_join);
    7751              :       }
    7752              : 
    7753              :   /* Now stitch things together.  */
    7754        25946 :   gimple_seq_add_seq (fork_seq, before_fork);
    7755        25946 :   if (private_marker)
    7756          259 :     gimple_seq_add_stmt (fork_seq, private_marker);
    7757        25946 :   if (fork)
    7758        16551 :     gimple_seq_add_stmt (fork_seq, fork);
    7759        25946 :   gimple_seq_add_seq (fork_seq, after_fork);
    7760              : 
    7761        25946 :   gimple_seq_add_seq (join_seq, before_join);
    7762        25946 :   if (join)
    7763        16551 :     gimple_seq_add_stmt (join_seq, join);
    7764        25946 :   gimple_seq_add_seq (join_seq, after_join);
    7765        25946 : }
    7766              : 
    7767              : /* Generate code to implement the REDUCTION clauses, append it
    7768              :    to STMT_SEQP.  CLIST if non-NULL is a pointer to a sequence
    7769              :    that should be emitted also inside of the critical section,
    7770              :    in that case clear *CLIST afterwards, otherwise leave it as is
    7771              :    and let the caller emit it itself.  */
    7772              : 
    7773              : static void
    7774        72466 : lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp,
    7775              :                          gimple_seq *clist, omp_context *ctx)
    7776              : {
    7777        72466 :   gimple_seq sub_seq = NULL;
    7778        72466 :   gimple *stmt;
    7779        72466 :   tree x, c;
    7780        72466 :   int count = 0;
    7781              : 
    7782              :   /* OpenACC loop reductions are handled elsewhere.  */
    7783        72466 :   if (is_gimple_omp_oacc (ctx->stmt))
    7784        71332 :     return;
    7785              : 
    7786              :   /* SIMD reductions are handled in lower_rec_input_clauses.  */
    7787        61141 :   if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
    7788        61141 :       && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
    7789              :     return;
    7790              : 
    7791              :   /* inscan reductions are handled elsewhere.  */
    7792        51751 :   if (ctx->scan_inclusive || ctx->scan_exclusive)
    7793              :     return;
    7794              : 
    7795              :   /* First see if there is exactly one reduction clause.  Use OMP_ATOMIC
    7796              :      update in that case, otherwise use a lock.  */
    7797       243639 :   for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c))
    7798       193010 :     if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
    7799       193010 :         && !OMP_CLAUSE_REDUCTION_TASK (c))
    7800              :       {
    7801         4727 :         if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)
    7802         4727 :             || TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF)
    7803              :           {
    7804              :             /* Never use OMP_ATOMIC for array reductions or UDRs.  */
    7805              :             count = -1;
    7806              :             break;
    7807              :           }
    7808         3790 :         count++;
    7809              :       }
    7810              : 
    7811        51566 :   if (count == 0)
    7812              :     return;
    7813              : 
    7814        15410 :   for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
    7815              :     {
    7816        14276 :       tree var, ref, new_var, orig_var;
    7817        14276 :       enum tree_code code;
    7818        14276 :       location_t clause_loc = OMP_CLAUSE_LOCATION (c);
    7819              : 
    7820        23013 :       if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
    7821        14276 :           || OMP_CLAUSE_REDUCTION_TASK (c))
    7822         8737 :         continue;
    7823              : 
    7824         5539 :       enum omp_clause_code ccode = OMP_CLAUSE_REDUCTION;
    7825         5539 :       orig_var = var = OMP_CLAUSE_DECL (c);
    7826         5539 :       if (TREE_CODE (var) == MEM_REF)
    7827              :         {
    7828          764 :           var = TREE_OPERAND (var, 0);
    7829          764 :           if (TREE_CODE (var) == POINTER_PLUS_EXPR)
    7830           50 :             var = TREE_OPERAND (var, 0);
    7831          764 :           if (TREE_CODE (var) == ADDR_EXPR)
    7832          322 :             var = TREE_OPERAND (var, 0);
    7833              :           else
    7834              :             {
    7835              :               /* If this is a pointer or referenced based array
    7836              :                  section, the var could be private in the outer
    7837              :                  context e.g. on orphaned loop construct.  Pretend this
    7838              :                  is private variable's outer reference.  */
    7839          442 :               ccode = OMP_CLAUSE_PRIVATE;
    7840          442 :               if (INDIRECT_REF_P (var))
    7841           72 :                 var = TREE_OPERAND (var, 0);
    7842              :             }
    7843          764 :           orig_var = var;
    7844          764 :           if (is_variable_sized (var))
    7845              :             {
    7846           25 :               gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
    7847           25 :               var = DECL_VALUE_EXPR (var);
    7848           25 :               gcc_assert (INDIRECT_REF_P (var));
    7849           25 :               var = TREE_OPERAND (var, 0);
    7850           25 :               gcc_assert (DECL_P (var));
    7851              :             }
    7852              :         }
    7853         5539 :       new_var = lookup_decl (var, ctx);
    7854         5539 :       if (var == OMP_CLAUSE_DECL (c)
    7855         5539 :           && omp_privatize_by_reference (var))
    7856          161 :         new_var = build_simple_mem_ref_loc (clause_loc, new_var);
    7857         5539 :       ref = build_outer_var_ref (var, ctx, ccode);
    7858         5539 :       code = OMP_CLAUSE_REDUCTION_CODE (c);
    7859              : 
    7860              :       /* reduction(-:var) sums up the partial results, so it acts
    7861              :          identically to reduction(+:var).  */
    7862         5539 :       if (code == MINUS_EXPR)
    7863           57 :         code = PLUS_EXPR;
    7864              : 
    7865         5539 :       bool is_truth_op = (code == TRUTH_ANDIF_EXPR || code == TRUTH_ORIF_EXPR);
    7866         5539 :       if (count == 1)
    7867              :         {
    7868         3341 :           tree addr = build_fold_addr_expr_loc (clause_loc, ref);
    7869              : 
    7870         3341 :           addr = save_expr (addr);
    7871         3341 :           ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr);
    7872         3341 :           tree new_var2 = new_var;
    7873         3341 :           tree ref2 = ref;
    7874         3341 :           if (is_truth_op)
    7875              :             {
    7876          784 :               tree zero = build_zero_cst (TREE_TYPE (new_var));
    7877          784 :               new_var2 = fold_build2_loc (clause_loc, NE_EXPR,
    7878              :                                           boolean_type_node, new_var, zero);
    7879          784 :               ref2 = fold_build2_loc (clause_loc, NE_EXPR, boolean_type_node,
    7880              :                                       ref, zero);
    7881              :             }
    7882         3341 :           x = fold_build2_loc (clause_loc, code, TREE_TYPE (new_var2), ref2,
    7883              :                                new_var2);
    7884         3341 :           if (is_truth_op)
    7885          784 :             x = fold_convert (TREE_TYPE (new_var), x);
    7886         3341 :           x = build2 (OMP_ATOMIC, void_type_node, addr, x);
    7887         3341 :           OMP_ATOMIC_MEMORY_ORDER (x) = OMP_MEMORY_ORDER_RELAXED;
    7888         3341 :           gimplify_and_add (x, stmt_seqp);
    7889         3341 :           return;
    7890              :         }
    7891         2198 :       else if (TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF)
    7892              :         {
    7893          764 :           tree d = OMP_CLAUSE_DECL (c);
    7894          764 :           tree type = TREE_TYPE (d);
    7895          764 :           tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
    7896          764 :           tree i = create_tmp_var (TREE_TYPE (v));
    7897          764 :           tree ptype = build_pointer_type (TREE_TYPE (type));
    7898          764 :           tree bias = TREE_OPERAND (d, 1);
    7899          764 :           d = TREE_OPERAND (d, 0);
    7900          764 :           if (TREE_CODE (d) == POINTER_PLUS_EXPR)
    7901              :             {
    7902           50 :               tree b = TREE_OPERAND (d, 1);
    7903           50 :               b = maybe_lookup_decl (b, ctx);
    7904           50 :               if (b == NULL)
    7905              :                 {
    7906            2 :                   b = TREE_OPERAND (d, 1);
    7907            2 :                   b = maybe_lookup_decl_in_outer_ctx (b, ctx);
    7908              :                 }
    7909           50 :               if (integer_zerop (bias))
    7910              :                 bias = b;
    7911              :               else
    7912              :                 {
    7913            0 :                   bias = fold_convert_loc (clause_loc, TREE_TYPE (b), bias);
    7914            0 :                   bias = fold_build2_loc (clause_loc, PLUS_EXPR,
    7915            0 :                                           TREE_TYPE (b), b, bias);
    7916              :                 }
    7917           50 :               d = TREE_OPERAND (d, 0);
    7918              :             }
    7919              :           /* For ref build_outer_var_ref already performs this, so
    7920              :              only new_var needs a dereference.  */
    7921          764 :           if (INDIRECT_REF_P (d))
    7922              :             {
    7923           72 :               new_var = build_simple_mem_ref_loc (clause_loc, new_var);
    7924           72 :               gcc_assert (omp_privatize_by_reference (var)
    7925              :                           && var == orig_var);
    7926              :             }
    7927          692 :           else if (TREE_CODE (d) == ADDR_EXPR)
    7928              :             {
    7929          322 :               if (orig_var == var)
    7930              :                 {
    7931          297 :                   new_var = build_fold_addr_expr (new_var);
    7932          297 :                   ref = build_fold_addr_expr (ref);
    7933              :                 }
    7934              :             }
    7935              :           else
    7936              :             {
    7937          370 :               gcc_assert (orig_var == var);
    7938          370 :               if (omp_privatize_by_reference (var))
    7939          106 :                 ref = build_fold_addr_expr (ref);
    7940              :             }
    7941          764 :           if (DECL_P (v))
    7942              :             {
    7943           93 :               tree t = maybe_lookup_decl (v, ctx);
    7944           93 :               if (t)
    7945              :                 v = t;
    7946              :               else
    7947            5 :                 v = maybe_lookup_decl_in_outer_ctx (v, ctx);
    7948           93 :               gimplify_expr (&v, stmt_seqp, NULL, is_gimple_val, fb_rvalue);
    7949              :             }
    7950          764 :           if (!integer_zerop (bias))
    7951              :             {
    7952          432 :               bias = fold_convert_loc (clause_loc, sizetype, bias);
    7953          864 :               new_var = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
    7954          432 :                                          TREE_TYPE (new_var), new_var,
    7955              :                                          unshare_expr (bias));
    7956          432 :               ref = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
    7957          432 :                                          TREE_TYPE (ref), ref, bias);
    7958              :             }
    7959          764 :           new_var = fold_convert_loc (clause_loc, ptype, new_var);
    7960          764 :           ref = fold_convert_loc (clause_loc, ptype, ref);
    7961          764 :           tree m = create_tmp_var (ptype);
    7962          764 :           gimplify_assign (m, new_var, stmt_seqp);
    7963          764 :           new_var = m;
    7964          764 :           m = create_tmp_var (ptype);
    7965          764 :           gimplify_assign (m, ref, stmt_seqp);
    7966          764 :           ref = m;
    7967          764 :           gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), stmt_seqp);
    7968          764 :           tree body = create_artificial_label (UNKNOWN_LOCATION);
    7969          764 :           tree end = create_artificial_label (UNKNOWN_LOCATION);
    7970          764 :           gimple_seq_add_stmt (&sub_seq, gimple_build_label (body));
    7971          764 :           tree priv = build_simple_mem_ref_loc (clause_loc, new_var);
    7972          764 :           tree out = build_simple_mem_ref_loc (clause_loc, ref);
    7973          764 :           if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
    7974              :             {
    7975           72 :               tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
    7976           72 :               tree decl_placeholder
    7977           72 :                 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
    7978           72 :               SET_DECL_VALUE_EXPR (placeholder, out);
    7979           72 :               DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
    7980           72 :               SET_DECL_VALUE_EXPR (decl_placeholder, priv);
    7981           72 :               DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
    7982           72 :               lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
    7983           72 :               gimple_seq_add_seq (&sub_seq,
    7984           72 :                                   OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
    7985           72 :               OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
    7986           72 :               OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
    7987           72 :               OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c) = NULL;
    7988              :             }
    7989              :           else
    7990              :             {
    7991          692 :               tree out2 = out;
    7992          692 :               tree priv2 = priv;
    7993          692 :               if (is_truth_op)
    7994              :                 {
    7995            0 :                   tree zero = build_zero_cst (TREE_TYPE (out));
    7996            0 :                   out2 = fold_build2_loc (clause_loc, NE_EXPR,
    7997              :                                           boolean_type_node, out, zero);
    7998            0 :                   priv2 = fold_build2_loc (clause_loc, NE_EXPR,
    7999              :                                            boolean_type_node, priv, zero);
    8000              :                 }
    8001          692 :               x = build2 (code, TREE_TYPE (out2), out2, priv2);
    8002          692 :               if (is_truth_op)
    8003            0 :                 x = fold_convert (TREE_TYPE (out), x);
    8004          692 :               out = unshare_expr (out);
    8005          692 :               gimplify_assign (out, x, &sub_seq);
    8006              :             }
    8007          764 :           gimple *g = gimple_build_assign (new_var, POINTER_PLUS_EXPR, new_var,
    8008          764 :                                            TYPE_SIZE_UNIT (TREE_TYPE (type)));
    8009          764 :           gimple_seq_add_stmt (&sub_seq, g);
    8010          764 :           g = gimple_build_assign (ref, POINTER_PLUS_EXPR, ref,
    8011          764 :                                    TYPE_SIZE_UNIT (TREE_TYPE (type)));
    8012          764 :           gimple_seq_add_stmt (&sub_seq, g);
    8013          764 :           g = gimple_build_assign (i, PLUS_EXPR, i,
    8014          764 :                                    build_int_cst (TREE_TYPE (i), 1));
    8015          764 :           gimple_seq_add_stmt (&sub_seq, g);
    8016          764 :           g = gimple_build_cond (LE_EXPR, i, v, body, end);
    8017          764 :           gimple_seq_add_stmt (&sub_seq, g);
    8018          764 :           gimple_seq_add_stmt (&sub_seq, gimple_build_label (end));
    8019              :         }
    8020         1434 :       else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
    8021              :         {
    8022          788 :           tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
    8023              : 
    8024          788 :           if (omp_privatize_by_reference (var)
    8025          845 :               && !useless_type_conversion_p (TREE_TYPE (placeholder),
    8026           57 :                                              TREE_TYPE (ref)))
    8027           49 :             ref = build_fold_addr_expr_loc (clause_loc, ref);
    8028          788 :           SET_DECL_VALUE_EXPR (placeholder, ref);
    8029          788 :           DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
    8030          788 :           lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
    8031          788 :           gimple_seq_add_seq (&sub_seq, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
    8032          788 :           OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
    8033          788 :           OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
    8034              :         }
    8035              :       else
    8036              :         {
    8037          646 :           tree new_var2 = new_var;
    8038          646 :           tree ref2 = ref;
    8039          646 :           if (is_truth_op)
    8040              :             {
    8041           78 :               tree zero = build_zero_cst (TREE_TYPE (new_var));
    8042           78 :               new_var2 = fold_build2_loc (clause_loc, NE_EXPR,
    8043              :                                           boolean_type_node, new_var, zero);
    8044           78 :               ref2 = fold_build2_loc (clause_loc, NE_EXPR, boolean_type_node,
    8045              :                                       ref, zero);
    8046              :             }
    8047          646 :           x = build2 (code, TREE_TYPE (ref), ref2, new_var2);
    8048          646 :           if (is_truth_op)
    8049           78 :             x = fold_convert (TREE_TYPE (new_var), x);
    8050          646 :           ref = build_outer_var_ref (var, ctx);
    8051          646 :           gimplify_assign (ref, x, &sub_seq);
    8052              :         }
    8053              :     }
    8054              : 
    8055         1134 :   stmt
    8056         1134 :     = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_START),
    8057              :                          0);
    8058         1134 :   gimple_seq_add_stmt (stmt_seqp, stmt);
    8059              : 
    8060         1134 :   gimple_seq_add_seq (stmt_seqp, sub_seq);
    8061              : 
    8062         1134 :   if (clist)
    8063              :     {
    8064          173 :       gimple_seq_add_seq (stmt_seqp, *clist);
    8065          173 :       *clist = NULL;
    8066              :     }
    8067              : 
    8068         1134 :   stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_END),
    8069              :                             0);
    8070         1134 :   gimple_seq_add_stmt (stmt_seqp, stmt);
    8071              : }
    8072              : 
    8073              : 
    8074              : /* Generate code to implement the COPYPRIVATE clauses.  */
    8075              : 
    8076              : static void
    8077          118 : lower_copyprivate_clauses (tree clauses, gimple_seq *slist, gimple_seq *rlist,
    8078              :                             omp_context *ctx)
    8079              : {
    8080          118 :   tree c;
    8081              : 
    8082          469 :   for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
    8083              :     {
    8084          351 :       tree var, new_var, ref, x;
    8085          351 :       bool by_ref;
    8086          351 :       location_t clause_loc = OMP_CLAUSE_LOCATION (c);
    8087              : 
    8088          351 :       if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE)
    8089            6 :         continue;
    8090              : 
    8091          345 :       var = OMP_CLAUSE_DECL (c);
    8092          345 :       by_ref = use_pointer_for_field (var, NULL);
    8093              : 
    8094          345 :       ref = build_sender_ref (var, ctx);
    8095          345 :       x = new_var = lookup_decl_in_outer_ctx (var, ctx);
    8096          345 :       if (by_ref)
    8097              :         {
    8098           74 :           x = build_fold_addr_expr_loc (clause_loc, new_var);
    8099           74 :           x = fold_convert_loc (clause_loc, TREE_TYPE (ref), x);
    8100              :         }
    8101          345 :       gimplify_assign (ref, x, slist);
    8102              : 
    8103          345 :       ref = build_receiver_ref (var, false, ctx);
    8104          345 :       if (by_ref)
    8105              :         {
    8106           74 :           ref = fold_convert_loc (clause_loc,
    8107           74 :                                   build_pointer_type (TREE_TYPE (new_var)),
    8108              :                                   ref);
    8109           74 :           ref = build_fold_indirect_ref_loc (clause_loc, ref);
    8110              :         }
    8111          345 :       if (omp_privatize_by_reference (var))
    8112              :         {
    8113          171 :           ref = fold_convert_loc (clause_loc, TREE_TYPE (new_var), ref);
    8114          171 :           ref = build_simple_mem_ref_loc (clause_loc, ref);
    8115          171 :           new_var = build_simple_mem_ref_loc (clause_loc, new_var);
    8116              :         }
    8117          345 :       x = lang_hooks.decls.omp_clause_assign_op (c, new_var, ref);
    8118          345 :       gimplify_and_add (x, rlist);
    8119              :     }
    8120          118 : }
    8121              : 
    8122              : 
    8123              : /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
    8124              :    and REDUCTION from the sender (aka parent) side.  */
    8125              : 
    8126              : static void
    8127        22721 : lower_send_clauses (tree clauses, gimple_seq *ilist, gimple_seq *olist,
    8128              :                     omp_context *ctx)
    8129              : {
    8130        22721 :   tree c, t;
    8131        22721 :   int ignored_looptemp = 0;
    8132        22721 :   bool is_taskloop = false;
    8133              : 
    8134              :   /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
    8135              :      by GOMP_taskloop.  */
    8136        22721 :   if (is_task_ctx (ctx) && gimple_omp_task_taskloop_p (ctx->stmt))
    8137              :     {
    8138              :       ignored_looptemp = 2;
    8139              :       is_taskloop = true;
    8140              :     }
    8141              : 
    8142       118024 :   for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
    8143              :     {
    8144        95303 :       tree val, ref, x, var;
    8145        95303 :       bool by_ref, do_in = false, do_out = false;
    8146        95303 :       location_t clause_loc = OMP_CLAUSE_LOCATION (c);
    8147              : 
    8148        95303 :       switch (OMP_CLAUSE_CODE (c))
    8149              :         {
    8150         4721 :         case OMP_CLAUSE_PRIVATE:
    8151         4721 :           if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
    8152              :             break;
    8153         4566 :           continue;
    8154              :         case OMP_CLAUSE_FIRSTPRIVATE:
    8155              :         case OMP_CLAUSE_COPYIN:
    8156              :         case OMP_CLAUSE_LASTPRIVATE:
    8157              :         case OMP_CLAUSE_IN_REDUCTION:
    8158              :         case OMP_CLAUSE__REDUCTEMP_:
    8159              :           break;
    8160         4910 :         case OMP_CLAUSE_REDUCTION:
    8161         4910 :           if (is_task_ctx (ctx) || OMP_CLAUSE_REDUCTION_TASK (c))
    8162          753 :             continue;
    8163              :           break;
    8164        17670 :         case OMP_CLAUSE_SHARED:
    8165        17670 :           if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
    8166              :             break;
    8167        17358 :           continue;
    8168        18875 :         case OMP_CLAUSE__LOOPTEMP_:
    8169        18875 :           if (ignored_looptemp)
    8170              :             {
    8171         2672 :               ignored_looptemp--;
    8172         2672 :               continue;
    8173              :             }
    8174              :           break;
    8175        12132 :         default:
    8176        12132 :           continue;
    8177              :         }
    8178              : 
    8179        57822 :       val = OMP_CLAUSE_DECL (c);
    8180        57822 :       if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
    8181        53665 :            || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION)
    8182        59487 :           && TREE_CODE (val) == MEM_REF)
    8183              :         {
    8184         1653 :           val = TREE_OPERAND (val, 0);
    8185         1653 :           if (TREE_CODE (val) == POINTER_PLUS_EXPR)
    8186          351 :             val = TREE_OPERAND (val, 0);
    8187         1653 :           if (INDIRECT_REF_P (val)
    8188         1551 :               || TREE_CODE (val) == ADDR_EXPR)
    8189          942 :             val = TREE_OPERAND (val, 0);
    8190         1653 :           if (is_variable_sized (val))
    8191           56 :             continue;
    8192              :         }
    8193              : 
    8194              :       /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
    8195              :          outer taskloop region.  */
    8196        57766 :       omp_context *ctx_for_o = ctx;
    8197        57766 :       if (is_taskloop
    8198         3917 :           && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
    8199        58078 :           && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
    8200          312 :         ctx_for_o = ctx->outer;
    8201              : 
    8202        57766 :       var = lookup_decl_in_outer_ctx (val, ctx_for_o);
    8203              : 
    8204        57766 :       if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN
    8205        57252 :           && is_global_var (var)
    8206        60079 :           && (val == OMP_CLAUSE_DECL (c)
    8207          273 :               || !is_task_ctx (ctx)
    8208          241 :               || (TREE_CODE (TREE_TYPE (val)) != POINTER_TYPE
    8209          187 :                   && (TREE_CODE (TREE_TYPE (val)) != REFERENCE_TYPE
    8210            0 :                       || (TREE_CODE (TREE_TYPE (TREE_TYPE (val)))
    8211              :                           != POINTER_TYPE)))))
    8212         2259 :         continue;
    8213              : 
    8214        55507 :       t = omp_member_access_dummy_var (var);
    8215        55507 :       if (t)
    8216              :         {
    8217          249 :           var = DECL_VALUE_EXPR (var);
    8218          249 :           tree o = maybe_lookup_decl_in_outer_ctx (t, ctx_for_o);
    8219          249 :           if (o != t)
    8220           69 :             var = unshare_and_remap (var, t, o);
    8221              :           else
    8222          180 :             var = unshare_expr (var);
    8223              :         }
    8224              : 
    8225        55507 :       if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED)
    8226              :         {
    8227              :           /* Handle taskloop firstprivate/lastprivate, where the
    8228              :              lastprivate on GIMPLE_OMP_TASK is represented as
    8229              :              OMP_CLAUSE_SHARED_FIRSTPRIVATE.  */
    8230           79 :           tree f = lookup_sfield ((splay_tree_key) &DECL_UID (val), ctx);
    8231           79 :           x = omp_build_component_ref (ctx->sender_decl, f);
    8232           79 :           if (use_pointer_for_field (val, ctx))
    8233           59 :             var = build_fold_addr_expr (var);
    8234           79 :           gimplify_assign (x, var, ilist);
    8235           79 :           DECL_ABSTRACT_ORIGIN (f) = NULL;
    8236           79 :           continue;
    8237           79 :         }
    8238              : 
    8239        55486 :       if (((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
    8240        52016 :             && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IN_REDUCTION)
    8241         4569 :            || val == OMP_CLAUSE_DECL (c))
    8242       109478 :           && is_variable_sized (val))
    8243           58 :         continue;
    8244        55370 :       by_ref = use_pointer_for_field (val, NULL);
    8245              : 
    8246        55370 :       switch (OMP_CLAUSE_CODE (c))
    8247              :         {
    8248        26904 :         case OMP_CLAUSE_FIRSTPRIVATE:
    8249        26904 :           if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c)
    8250         7277 :               && !by_ref
    8251        34127 :               && is_task_ctx (ctx))
    8252         2331 :             suppress_warning (var);
    8253              :           do_in = true;
    8254              :           break;
    8255              : 
    8256              :         case OMP_CLAUSE_PRIVATE:
    8257              :         case OMP_CLAUSE_COPYIN:
    8258              :         case OMP_CLAUSE__LOOPTEMP_:
    8259              :         case OMP_CLAUSE__REDUCTEMP_:
    8260              :           do_in = true;
    8261              :           break;
    8262              : 
    8263         6467 :         case OMP_CLAUSE_LASTPRIVATE:
    8264         6467 :           if (by_ref || omp_privatize_by_reference (val))
    8265              :             {
    8266          384 :               if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
    8267          239 :                 continue;
    8268              :               do_in = true;
    8269              :             }
    8270              :           else
    8271              :             {
    8272         6083 :               do_out = true;
    8273         6083 :               if (lang_hooks.decls.omp_private_outer_ref (val))
    8274              :                 do_in = true;
    8275              :             }
    8276              :           break;
    8277              : 
    8278         4569 :         case OMP_CLAUSE_REDUCTION:
    8279         4569 :         case OMP_CLAUSE_IN_REDUCTION:
    8280         4569 :           do_in = true;
    8281         4569 :           if (val == OMP_CLAUSE_DECL (c))
    8282              :             {
    8283         3191 :               if (is_task_ctx (ctx))
    8284          322 :                 by_ref = use_pointer_for_field (val, ctx);
    8285              :               else
    8286         2869 :                 do_out = !(by_ref || omp_privatize_by_reference (val));
    8287              :             }
    8288              :           else
    8289         1378 :             by_ref = TREE_CODE (TREE_TYPE (val)) == ARRAY_TYPE;
    8290              :           break;
    8291              : 
    8292            0 :         default:
    8293            0 :           gcc_unreachable ();
    8294              :         }
    8295              : 
    8296         4031 :       if (do_in)
    8297              :         {
    8298        49072 :           ref = build_sender_ref (val, ctx);
    8299        49072 :           x = by_ref ? build_fold_addr_expr_loc (clause_loc, var) : var;
    8300        49072 :           gimplify_assign (ref, x, ilist);
    8301        49072 :           if (is_task_ctx (ctx))
    8302         6267 :             DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref, 1)) = NULL;
    8303              :         }
    8304              : 
    8305        55131 :       if (do_out)
    8306              :         {
    8307         8400 :           ref = build_sender_ref (val, ctx);
    8308         8400 :           gimplify_assign (var, ref, olist);
    8309              :         }
    8310              :     }
    8311        22721 : }
    8312              : 
    8313              : /* Generate code to implement SHARED from the sender (aka parent)
    8314              :    side.  This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
    8315              :    list things that got automatically shared.  */
    8316              : 
    8317              : static void
    8318        22721 : lower_send_shared_vars (gimple_seq *ilist, gimple_seq *olist, omp_context *ctx)
    8319              : {
    8320        22721 :   tree var, ovar, nvar, t, f, x, record_type;
    8321              : 
    8322        22721 :   if (ctx->record_type == NULL)
    8323         4540 :     return;
    8324              : 
    8325        18181 :   record_type = ctx->srecord_type ? ctx->srecord_type : ctx->record_type;
    8326        87438 :   for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
    8327              :     {
    8328        69257 :       ovar = DECL_ABSTRACT_ORIGIN (f);
    8329        69257 :       if (!ovar || TREE_CODE (ovar) == FIELD_DECL)
    8330         6346 :         continue;
    8331              : 
    8332        62911 :       nvar = maybe_lookup_decl (ovar, ctx);
    8333       114091 :       if (!nvar
    8334        62498 :           || !DECL_HAS_VALUE_EXPR_P (nvar)
    8335        74863 :           || (ctx->allocate_map
    8336          317 :               && ctx->allocate_map->get (ovar)))
    8337        51180 :         continue;
    8338              : 
    8339              :       /* If CTX is a nested parallel directive.  Find the immediately
    8340              :          enclosing parallel or workshare construct that contains a
    8341              :          mapping for OVAR.  */
    8342        11731 :       var = lookup_decl_in_outer_ctx (ovar, ctx);
    8343              : 
    8344        11731 :       t = omp_member_access_dummy_var (var);
    8345        11731 :       if (t)
    8346              :         {
    8347           49 :           var = DECL_VALUE_EXPR (var);
    8348           49 :           tree o = maybe_lookup_decl_in_outer_ctx (t, ctx);
    8349           49 :           if (o != t)
    8350           18 :             var = unshare_and_remap (var, t, o);
    8351              :           else
    8352           31 :             var = unshare_expr (var);
    8353              :         }
    8354              : 
    8355        11731 :       if (use_pointer_for_field (ovar, ctx))
    8356              :         {
    8357         6490 :           x = build_sender_ref (ovar, ctx);
    8358         6490 :           if (TREE_CODE (TREE_TYPE (f)) == ARRAY_TYPE
    8359         6490 :               && TREE_TYPE (f) == TREE_TYPE (ovar))
    8360              :             {
    8361          103 :               gcc_assert (is_parallel_ctx (ctx)
    8362              :                           && DECL_ARTIFICIAL (ovar));
    8363              :               /* _condtemp_ clause.  */
    8364          103 :               var = build_constructor (TREE_TYPE (x), NULL);
    8365              :             }
    8366              :           else
    8367         6387 :             var = build_fold_addr_expr (var);
    8368         6490 :           gimplify_assign (x, var, ilist);
    8369              :         }
    8370              :       else
    8371              :         {
    8372         5241 :           x = build_sender_ref (ovar, ctx);
    8373         5241 :           gimplify_assign (x, var, ilist);
    8374              : 
    8375         5241 :           if (!TREE_READONLY (var)
    8376              :               /* We don't need to receive a new reference to a result
    8377              :                  or parm decl.  In fact we may not store to it as we will
    8378              :                  invalidate any pending RSO and generate wrong gimple
    8379              :                  during inlining.  */
    8380         5241 :               && !((TREE_CODE (var) == RESULT_DECL
    8381         3348 :                     || TREE_CODE (var) == PARM_DECL)
    8382          182 :                    && DECL_BY_REFERENCE (var)))
    8383              :             {
    8384         3277 :               x = build_sender_ref (ovar, ctx);
    8385         3277 :               gimplify_assign (var, x, olist);
    8386              :             }
    8387              :         }
    8388              :     }
    8389              : }
    8390              : 
    8391              : /* Emit an OpenACC head marker call, encapulating the partitioning and
    8392              :    other information that must be processed by the target compiler.
    8393              :    Return the maximum number of dimensions the associated loop might
    8394              :    be partitioned over.  */
    8395              : 
    8396              : static unsigned
    8397         9665 : lower_oacc_head_mark (location_t loc, tree ddvar, tree clauses,
    8398              :                       gimple_seq *seq, omp_context *ctx)
    8399              : {
    8400         9665 :   unsigned levels = 0;
    8401         9665 :   unsigned tag = 0;
    8402         9665 :   tree gang_static = NULL_TREE;
    8403         9665 :   auto_vec<tree, 5> args;
    8404              : 
    8405        19330 :   args.quick_push (build_int_cst
    8406         9665 :                    (integer_type_node, IFN_UNIQUE_OACC_HEAD_MARK));
    8407         9665 :   args.quick_push (ddvar);
    8408        40519 :   for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
    8409              :     {
    8410        21189 :       switch (OMP_CLAUSE_CODE (c))
    8411              :         {
    8412         1856 :         case OMP_CLAUSE_GANG:
    8413         1856 :           tag |= OLF_DIM_GANG;
    8414         1856 :           gang_static = OMP_CLAUSE_GANG_STATIC_EXPR (c);
    8415              :           /* static:* is represented by -1, and we can ignore it, as
    8416              :              scheduling is always static.  */
    8417         1856 :           if (gang_static && integer_minus_onep (gang_static))
    8418           70 :             gang_static = NULL_TREE;
    8419         1856 :           levels++;
    8420         1856 :           break;
    8421              : 
    8422         1330 :         case OMP_CLAUSE_WORKER:
    8423         1330 :           tag |= OLF_DIM_WORKER;
    8424         1330 :           levels++;
    8425         1330 :           break;
    8426              : 
    8427         1523 :         case OMP_CLAUSE_VECTOR:
    8428         1523 :           tag |= OLF_DIM_VECTOR;
    8429         1523 :           levels++;
    8430         1523 :           break;
    8431              : 
    8432          314 :         case OMP_CLAUSE_SEQ:
    8433          314 :           tag |= OLF_SEQ;
    8434          314 :           break;
    8435              : 
    8436          409 :         case OMP_CLAUSE_AUTO:
    8437          409 :           tag |= OLF_AUTO;
    8438          409 :           break;
    8439              : 
    8440          260 :         case OMP_CLAUSE_INDEPENDENT:
    8441          260 :           tag |= OLF_INDEPENDENT;
    8442          260 :           break;
    8443              : 
    8444          136 :         case OMP_CLAUSE_TILE:
    8445          136 :           tag |= OLF_TILE;
    8446          136 :           break;
    8447              : 
    8448         3986 :         case OMP_CLAUSE_REDUCTION:
    8449         3986 :           tag |= OLF_REDUCTION;
    8450         3986 :           break;
    8451              : 
    8452        11375 :         default:
    8453        11375 :           continue;
    8454              :         }
    8455              :     }
    8456              : 
    8457         9665 :   if (gang_static)
    8458              :     {
    8459          146 :       if (DECL_P (gang_static))
    8460           16 :         gang_static = build_outer_var_ref (gang_static, ctx);
    8461          146 :       tag |= OLF_GANG_STATIC;
    8462              :     }
    8463              : 
    8464         9665 :   omp_context *tgt = enclosing_target_ctx (ctx);
    8465         9665 :   if (!tgt || is_oacc_parallel_or_serial (tgt))
    8466              :     ;
    8467           62 :   else if (is_oacc_kernels (tgt))
    8468              :     /* Not using this loops handling inside OpenACC 'kernels' regions.  */
    8469            0 :     gcc_unreachable ();
    8470           62 :   else if (is_oacc_kernels_decomposed_part (tgt))
    8471              :     ;
    8472              :   else
    8473            0 :     gcc_unreachable ();
    8474              : 
    8475              :   /* In a parallel region, loops are implicitly INDEPENDENT.  */
    8476         9665 :   if (!tgt || is_oacc_parallel_or_serial (tgt))
    8477         9603 :     tag |= OLF_INDEPENDENT;
    8478              : 
    8479              :   /* Loops inside OpenACC 'kernels' decomposed parts' regions are expected to
    8480              :      have an explicit 'seq' or 'independent' clause, and no 'auto' clause.  */
    8481         9665 :   if (tgt && is_oacc_kernels_decomposed_part (tgt))
    8482              :     {
    8483           62 :       gcc_assert (tag & (OLF_SEQ | OLF_INDEPENDENT));
    8484           62 :       gcc_assert (!(tag & OLF_AUTO));
    8485              :     }
    8486              : 
    8487         9665 :   if (tag & OLF_TILE)
    8488              :     /* Tiling could use all 3 levels.  */
    8489              :     levels = 3;
    8490              :   else
    8491              :     {
    8492              :       /* A loop lacking SEQ, GANG, WORKER and/or VECTOR could be AUTO.
    8493              :          Ensure at least one level, or 2 for possible auto
    8494              :          partitioning */
    8495         9529 :       bool maybe_auto = !(tag & (((GOMP_DIM_MASK (GOMP_DIM_MAX) - 1)
    8496              :                                   << OLF_DIM_BASE) | OLF_SEQ));
    8497              : 
    8498         9529 :       if (levels < 1u + maybe_auto)
    8499              :         levels = 1u + maybe_auto;
    8500              :     }
    8501              : 
    8502         9665 :   args.quick_push (build_int_cst (integer_type_node, levels));
    8503         9665 :   args.quick_push (build_int_cst (integer_type_node, tag));
    8504         9665 :   if (gang_static)
    8505          146 :     args.quick_push (gang_static);
    8506              : 
    8507         9665 :   gcall *call = gimple_build_call_internal_vec (IFN_UNIQUE, args);
    8508         9665 :   gimple_set_location (call, loc);
    8509         9665 :   gimple_set_lhs (call, ddvar);
    8510         9665 :   gimple_seq_add_stmt (seq, call);
    8511              : 
    8512         9665 :   return levels;
    8513         9665 : }
    8514              : 
    8515              : /* Emit an OpenACC lopp head or tail marker to SEQ.  LEVEL is the
    8516              :    partitioning level of the enclosed region.  */
    8517              : 
    8518              : static void
    8519        42767 : lower_oacc_loop_marker (location_t loc, tree ddvar, bool head,
    8520              :                         tree tofollow, gimple_seq *seq)
    8521              : {
    8522        42767 :   int marker_kind = (head ? IFN_UNIQUE_OACC_HEAD_MARK
    8523              :                      : IFN_UNIQUE_OACC_TAIL_MARK);
    8524        42767 :   tree marker = build_int_cst (integer_type_node, marker_kind);
    8525        42767 :   int nargs = 2 + (tofollow != NULL_TREE);
    8526        42767 :   gcall *call = gimple_build_call_internal (IFN_UNIQUE, nargs,
    8527              :                                             marker, ddvar, tofollow);
    8528        42767 :   gimple_set_location (call, loc);
    8529        42767 :   gimple_set_lhs (call, ddvar);
    8530        42767 :   gimple_seq_add_stmt (seq, call);
    8531        42767 : }
    8532              : 
    8533              : /* Generate the before and after OpenACC loop sequences.  CLAUSES are
    8534              :    the loop clauses, from which we extract reductions.  Initialize
    8535              :    HEAD and TAIL.  */
    8536              : 
    8537              : static void
    8538         9665 : lower_oacc_head_tail (location_t loc, tree clauses, gcall *private_marker,
    8539              :                       gimple_seq *head, gimple_seq *tail, omp_context *ctx)
    8540              : {
    8541         9665 :   bool inner = false;
    8542         9665 :   tree ddvar = create_tmp_var (integer_type_node, ".data_dep");
    8543         9665 :   gimple_seq_add_stmt (head, gimple_build_assign (ddvar, integer_zero_node));
    8544              : 
    8545         9665 :   unsigned count = lower_oacc_head_mark (loc, ddvar, clauses, head, ctx);
    8546              : 
    8547         9665 :   if (private_marker)
    8548              :     {
    8549          229 :       gimple_set_location (private_marker, loc);
    8550          229 :       gimple_call_set_lhs (private_marker, ddvar);
    8551          229 :       gimple_call_set_arg (private_marker, 1, ddvar);
    8552              :     }
    8553              : 
    8554         9665 :   tree fork_kind = build_int_cst (unsigned_type_node, IFN_UNIQUE_OACC_FORK);
    8555         9665 :   tree join_kind = build_int_cst (unsigned_type_node, IFN_UNIQUE_OACC_JOIN);
    8556              : 
    8557         9665 :   gcc_assert (count);
    8558        26216 :   for (unsigned done = 1; count; count--, done++)
    8559              :     {
    8560        16551 :       gimple_seq fork_seq = NULL;
    8561        16551 :       gimple_seq join_seq = NULL;
    8562              : 
    8563        16551 :       tree place = build_int_cst (integer_type_node, -1);
    8564        16551 :       gcall *fork = gimple_build_call_internal (IFN_UNIQUE, 3,
    8565              :                                                 fork_kind, ddvar, place);
    8566        16551 :       gimple_set_location (fork, loc);
    8567        16551 :       gimple_set_lhs (fork, ddvar);
    8568              : 
    8569        16551 :       gcall *join = gimple_build_call_internal (IFN_UNIQUE, 3,
    8570              :                                                 join_kind, ddvar, place);
    8571        16551 :       gimple_set_location (join, loc);
    8572        16551 :       gimple_set_lhs (join, ddvar);
    8573              : 
    8574              :       /* Mark the beginning of this level sequence.  */
    8575        16551 :       if (inner)
    8576         6886 :         lower_oacc_loop_marker (loc, ddvar, true,
    8577         6886 :                                 build_int_cst (integer_type_node, count),
    8578              :                                 &fork_seq);
    8579        16551 :       lower_oacc_loop_marker (loc, ddvar, false,
    8580        16551 :                               build_int_cst (integer_type_node, done),
    8581              :                               &join_seq);
    8582              : 
    8583        23437 :       lower_oacc_reductions (loc, clauses, place, inner,
    8584              :                              fork, (count == 1) ? private_marker : NULL,
    8585              :                              join, &fork_seq, &join_seq,  ctx);
    8586              : 
    8587              :       /* Append this level to head. */
    8588        16551 :       gimple_seq_add_seq (head, fork_seq);
    8589              :       /* Prepend it to tail.  */
    8590        16551 :       gimple_seq_add_seq (&join_seq, *tail);
    8591        16551 :       *tail = join_seq;
    8592              : 
    8593        16551 :       inner = true;
    8594              :     }
    8595              : 
    8596              :   /* Mark the end of the sequence.  */
    8597         9665 :   lower_oacc_loop_marker (loc, ddvar, true, NULL_TREE, head);
    8598         9665 :   lower_oacc_loop_marker (loc, ddvar, false, NULL_TREE, tail);
    8599         9665 : }
    8600              : 
    8601              : /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
    8602              :    catch handler and return it.  This prevents programs from violating the
    8603              :    structured block semantics with throws.  */
    8604              : 
    8605              : static gimple_seq
    8606        95852 : maybe_catch_exception (gimple_seq body)
    8607              : {
    8608        95852 :   gimple *g;
    8609        95852 :   tree decl;
    8610              : 
    8611        95852 :   if (!flag_exceptions)
    8612              :     return body;
    8613              : 
    8614        43023 :   if (lang_hooks.eh_protect_cleanup_actions != NULL)
    8615        42995 :     decl = lang_hooks.eh_protect_cleanup_actions ();
    8616              :   else
    8617           28 :     decl = builtin_decl_explicit (BUILT_IN_TRAP);
    8618              : 
    8619        43023 :   g = gimple_build_eh_must_not_throw (decl);
    8620        43023 :   g = gimple_build_try (body, gimple_seq_alloc_with_stmt (g),
    8621              :                         GIMPLE_TRY_CATCH);
    8622              : 
    8623        43023 :  return gimple_seq_alloc_with_stmt (g);
    8624              : }
    8625              : 
    8626              : 
    8627              : /* Routines to lower OMP directives into OMP-GIMPLE.  */
    8628              : 
    8629              : /* If ctx is a worksharing context inside of a cancellable parallel
    8630              :    region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
    8631              :    and conditional branch to parallel's cancel_label to handle
    8632              :    cancellation in the implicit barrier.  */
    8633              : 
    8634              : static void
    8635        48789 : maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple *omp_return,
    8636              :                                    gimple_seq *body)
    8637              : {
    8638        48789 :   gcc_assert (gimple_code (omp_return) == GIMPLE_OMP_RETURN);
    8639        48789 :   if (gimple_omp_return_nowait_p (omp_return))
    8640              :     return;
    8641        19679 :   for (omp_context *outer = ctx->outer; outer; outer = outer->outer)
    8642        16046 :     if (gimple_code (outer->stmt) == GIMPLE_OMP_PARALLEL
    8643        16046 :         && outer->cancellable)
    8644              :       {
    8645           65 :         tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_CANCEL);
    8646           65 :         tree c_bool_type = TREE_TYPE (TREE_TYPE (fndecl));
    8647           65 :         tree lhs = create_tmp_var (c_bool_type);
    8648           65 :         gimple_omp_return_set_lhs (omp_return, lhs);
    8649           65 :         tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
    8650           65 :         gimple *g = gimple_build_cond (NE_EXPR, lhs,
    8651              :                                        fold_convert (c_bool_type,
    8652              :                                                      boolean_false_node),
    8653              :                                        outer->cancel_label, fallthru_label);
    8654           65 :         gimple_seq_add_stmt (body, g);
    8655           65 :         gimple_seq_add_stmt (body, gimple_build_label (fallthru_label));
    8656              :       }
    8657        15981 :     else if (gimple_code (outer->stmt) != GIMPLE_OMP_TASKGROUP
    8658        15981 :              && gimple_code (outer->stmt) != GIMPLE_OMP_SCOPE)
    8659              :       return;
    8660              : }
    8661              : 
    8662              : /* Find the first task_reduction or reduction clause or return NULL
    8663              :    if there are none.  */
    8664              : 
    8665              : static inline tree
    8666        49034 : omp_task_reductions_find_first (tree clauses, enum tree_code code,
    8667              :                                 enum omp_clause_code ccode)
    8668              : {
    8669        64468 :   while (1)
    8670              :     {
    8671        56751 :       clauses = omp_find_clause (clauses, ccode);
    8672        56751 :       if (clauses == NULL_TREE)
    8673              :         return NULL_TREE;
    8674         9183 :       if (ccode != OMP_CLAUSE_REDUCTION
    8675         9183 :           || code == OMP_TASKLOOP
    8676         9183 :           || OMP_CLAUSE_REDUCTION_TASK (clauses))
    8677              :         return clauses;
    8678         7717 :       clauses = OMP_CLAUSE_CHAIN (clauses);
    8679              :     }
    8680              : }
    8681              : 
    8682              : static void lower_omp_task_reductions (omp_context *, enum tree_code, tree,
    8683              :                                        gimple_seq *, gimple_seq *);
    8684              : 
    8685              : /* Lower the OpenMP sections directive in the current statement in GSI_P.
    8686              :    CTX is the enclosing OMP context for the current statement.  */
    8687              : 
    8688              : static void
    8689          386 : lower_omp_sections (gimple_stmt_iterator *gsi_p, omp_context *ctx)
    8690              : {
    8691          386 :   tree block, control;
    8692          386 :   gimple_stmt_iterator tgsi;
    8693          386 :   gomp_sections *stmt;
    8694          386 :   gimple *t;
    8695          386 :   gbind *new_stmt, *bind;
    8696          386 :   gimple_seq ilist, dlist, olist, tred_dlist = NULL, clist = NULL, new_body;
    8697              : 
    8698          386 :   stmt = as_a <gomp_sections *> (gsi_stmt (*gsi_p));
    8699              : 
    8700          386 :   push_gimplify_context ();
    8701              : 
    8702          386 :   dlist = NULL;
    8703          386 :   ilist = NULL;
    8704              : 
    8705          386 :   tree rclauses
    8706          386 :     = omp_task_reductions_find_first (gimple_omp_sections_clauses (stmt),
    8707              :                                       OMP_SECTIONS, OMP_CLAUSE_REDUCTION);
    8708          386 :   tree rtmp = NULL_TREE;
    8709          386 :   if (rclauses)
    8710              :     {
    8711            8 :       tree type = build_pointer_type (pointer_sized_int_node);
    8712            8 :       tree temp = create_tmp_var (type);
    8713            8 :       tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
    8714            8 :       OMP_CLAUSE_DECL (c) = temp;
    8715            8 :       OMP_CLAUSE_CHAIN (c) = gimple_omp_sections_clauses (stmt);
    8716            8 :       gimple_omp_sections_set_clauses (stmt, c);
    8717            8 :       lower_omp_task_reductions (ctx, OMP_SECTIONS,
    8718              :                                  gimple_omp_sections_clauses (stmt),
    8719              :                                  &ilist, &tred_dlist);
    8720            8 :       rclauses = c;
    8721            8 :       rtmp = make_ssa_name (type);
    8722            8 :       gimple_seq_add_stmt (&ilist, gimple_build_assign (rtmp, temp));
    8723              :     }
    8724              : 
    8725          386 :   tree *clauses_ptr = gimple_omp_sections_clauses_ptr (stmt);
    8726          386 :   lower_lastprivate_conditional_clauses (clauses_ptr, ctx);
    8727              : 
    8728          386 :   lower_rec_input_clauses (gimple_omp_sections_clauses (stmt),
    8729              :                            &ilist, &dlist, ctx, NULL);
    8730              : 
    8731          386 :   control = create_tmp_var (unsigned_type_node, ".section");
    8732          386 :   gimple_omp_sections_set_control (stmt, control);
    8733              : 
    8734          386 :   new_body = gimple_omp_body (stmt);
    8735          386 :   gimple_omp_set_body (stmt, NULL);
    8736          386 :   tgsi = gsi_start (new_body);
    8737         1249 :   for (; !gsi_end_p (tgsi); gsi_next (&tgsi))
    8738              :     {
    8739          863 :       omp_context *sctx;
    8740          863 :       gimple *sec_start;
    8741              : 
    8742          863 :       sec_start = gsi_stmt (tgsi);
    8743          863 :       sctx = maybe_lookup_ctx (sec_start);
    8744          863 :       gcc_assert (sctx);
    8745              : 
    8746          863 :       lower_omp (gimple_omp_body_ptr (sec_start), sctx);
    8747          863 :       gsi_insert_seq_after (&tgsi, gimple_omp_body (sec_start),
    8748              :                             GSI_CONTINUE_LINKING);
    8749          863 :       gimple_omp_set_body (sec_start, NULL);
    8750              : 
    8751          863 :       if (gsi_one_before_end_p (tgsi))
    8752              :         {
    8753          386 :           gimple_seq l = NULL;
    8754          386 :           lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt), NULL,
    8755              :                                      &ilist, &l, &clist, ctx);
    8756          386 :           gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING);
    8757          386 :           gimple_omp_section_set_last (sec_start);
    8758              :         }
    8759              : 
    8760          863 :       gsi_insert_after (&tgsi, gimple_build_omp_return (false),
    8761              :                         GSI_CONTINUE_LINKING);
    8762              :     }
    8763              : 
    8764          386 :   block = make_node (BLOCK);
    8765          386 :   bind = gimple_build_bind (NULL, new_body, block);
    8766              : 
    8767          386 :   olist = NULL;
    8768          386 :   lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist,
    8769              :                            &clist, ctx);
    8770          386 :   if (clist)
    8771              :     {
    8772           10 :       tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_START);
    8773           10 :       gcall *g = gimple_build_call (fndecl, 0);
    8774           10 :       gimple_seq_add_stmt (&olist, g);
    8775           10 :       gimple_seq_add_seq (&olist, clist);
    8776           10 :       fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_END);
    8777           10 :       g = gimple_build_call (fndecl, 0);
    8778           10 :       gimple_seq_add_stmt (&olist, g);
    8779              :     }
    8780              : 
    8781          386 :   block = make_node (BLOCK);
    8782          386 :   new_stmt = gimple_build_bind (NULL, NULL, block);
    8783          386 :   gsi_replace (gsi_p, new_stmt, true);
    8784              : 
    8785          386 :   pop_gimplify_context (new_stmt);
    8786          386 :   gimple_bind_append_vars (new_stmt, ctx->block_vars);
    8787          386 :   BLOCK_VARS (block) = gimple_bind_vars (bind);
    8788          386 :   if (BLOCK_VARS (block))
    8789            0 :     TREE_USED (block) = 1;
    8790              : 
    8791          386 :   new_body = NULL;
    8792          386 :   gimple_seq_add_seq (&new_body, ilist);
    8793          386 :   gimple_seq_add_stmt (&new_body, stmt);
    8794          386 :   gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ());
    8795          386 :   gimple_seq_add_stmt (&new_body, bind);
    8796              : 
    8797          386 :   t = gimple_build_omp_continue (control, control);
    8798          386 :   gimple_seq_add_stmt (&new_body, t);
    8799              : 
    8800          386 :   gimple_seq_add_seq (&new_body, olist);
    8801          386 :   if (ctx->cancellable)
    8802           19 :     gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
    8803          386 :   gimple_seq_add_seq (&new_body, dlist);
    8804              : 
    8805          386 :   new_body = maybe_catch_exception (new_body);
    8806              : 
    8807          386 :   bool nowait = omp_find_clause (gimple_omp_sections_clauses (stmt),
    8808          386 :                                  OMP_CLAUSE_NOWAIT) != NULL_TREE;
    8809          386 :   t = gimple_build_omp_return (nowait);
    8810          386 :   gimple_seq_add_stmt (&new_body, t);
    8811          386 :   gimple_seq_add_seq (&new_body, tred_dlist);
    8812          386 :   maybe_add_implicit_barrier_cancel (ctx, t, &new_body);
    8813              : 
    8814          386 :   if (rclauses)
    8815            8 :     OMP_CLAUSE_DECL (rclauses) = rtmp;
    8816              : 
    8817          386 :   gimple_bind_set_body (new_stmt, new_body);
    8818          386 : }
    8819              : 
    8820              : 
    8821              : /* A subroutine of lower_omp_single.  Expand the simple form of
    8822              :    a GIMPLE_OMP_SINGLE, without a copyprivate clause:
    8823              : 
    8824              :         if (GOMP_single_start ())
    8825              :           BODY;
    8826              :         [ GOMP_barrier (); ]    -> unless 'nowait' is present.
    8827              : 
    8828              :   FIXME.  It may be better to delay expanding the logic of this until
    8829              :   pass_expand_omp.  The expanded logic may make the job more difficult
    8830              :   to a synchronization analysis pass.  */
    8831              : 
    8832              : static void
    8833         1022 : lower_omp_single_simple (gomp_single *single_stmt, gimple_seq *pre_p)
    8834              : {
    8835         1022 :   location_t loc = gimple_location (single_stmt);
    8836         1022 :   tree tlabel = create_artificial_label (loc);
    8837         1022 :   tree flabel = create_artificial_label (loc);
    8838         1022 :   gimple *call, *cond;
    8839         1022 :   tree lhs, decl;
    8840              : 
    8841         2040 :   decl = builtin_decl_explicit (flag_openmp_ompt
    8842              :                                   ? BUILT_IN_GOMP_SINGLE_START_WITH_END
    8843              :                                   : BUILT_IN_GOMP_SINGLE_START);
    8844         1022 :   lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)));
    8845         1022 :   call = gimple_build_call (decl, 0);
    8846         1022 :   gimple_call_set_lhs (call, lhs);
    8847         1022 :   gimple_seq_add_stmt (pre_p, call);
    8848              : 
    8849         1022 :   cond = gimple_build_cond (EQ_EXPR, lhs,
    8850         1022 :                             fold_convert_loc (loc, TREE_TYPE (lhs),
    8851              :                                               boolean_true_node),
    8852              :                             tlabel, flabel);
    8853         1022 :   gimple_seq_add_stmt (pre_p, cond);
    8854         1022 :   gimple_seq_add_stmt (pre_p, gimple_build_label (tlabel));
    8855         1022 :   gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
    8856         1022 :   gimple_seq_add_stmt (pre_p, gimple_build_label (flabel));
    8857         1022 : }
    8858              : 
    8859              : 
    8860              : /* A subroutine of lower_omp_single.  Expand the simple form of
    8861              :    a GIMPLE_OMP_SINGLE, with a copyprivate clause:
    8862              : 
    8863              :         #pragma omp single copyprivate (a, b, c)
    8864              : 
    8865              :    Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
    8866              : 
    8867              :       {
    8868              :         if ((copyout_p = GOMP_single_copy_start ()) == NULL)
    8869              :           {
    8870              :             BODY;
    8871              :             copyout.a = a;
    8872              :             copyout.b = b;
    8873              :             copyout.c = c;
    8874              :             GOMP_single_copy_end (&copyout);
    8875              :           }
    8876              :         else
    8877              :           {
    8878              :             a = copyout_p->a;
    8879              :             b = copyout_p->b;
    8880              :             c = copyout_p->c;
    8881              :           }
    8882              :         GOMP_barrier ();
    8883              :       }
    8884              : 
    8885              :   FIXME.  It may be better to delay expanding the logic of this until
    8886              :   pass_expand_omp.  The expanded logic may make the job more difficult
    8887              :   to a synchronization analysis pass.  */
    8888              : 
    8889              : static void
    8890          118 : lower_omp_single_copy (gomp_single *single_stmt, gimple_seq *pre_p,
    8891              :                        omp_context *ctx)
    8892              : {
    8893          118 :   tree ptr_type, t, l0, l1, l2, bfn_decl;
    8894          118 :   gimple_seq copyin_seq;
    8895          118 :   location_t loc = gimple_location (single_stmt);
    8896              : 
    8897          118 :   ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o");
    8898              : 
    8899          118 :   ptr_type = build_pointer_type (ctx->record_type);
    8900          118 :   ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i");
    8901              : 
    8902          118 :   l0 = create_artificial_label (loc);
    8903          118 :   l1 = create_artificial_label (loc);
    8904          118 :   l2 = create_artificial_label (loc);
    8905              : 
    8906          118 :   bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START);
    8907          118 :   t = build_call_expr_loc (loc, bfn_decl, 0);
    8908          118 :   t = fold_convert_loc (loc, ptr_type, t);
    8909          118 :   gimplify_assign (ctx->receiver_decl, t, pre_p);
    8910              : 
    8911          118 :   t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl,
    8912              :               build_int_cst (ptr_type, 0));
    8913          118 :   t = build3 (COND_EXPR, void_type_node, t,
    8914              :               build_and_jump (&l0), build_and_jump (&l1));
    8915          118 :   gimplify_and_add (t, pre_p);
    8916              : 
    8917          118 :   gimple_seq_add_stmt (pre_p, gimple_build_label (l0));
    8918              : 
    8919          118 :   gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
    8920              : 
    8921          118 :   copyin_seq = NULL;
    8922          118 :   lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt), pre_p,
    8923              :                               &copyin_seq, ctx);
    8924              : 
    8925          118 :   t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
    8926          118 :   bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END);
    8927          118 :   t = build_call_expr_loc (loc, bfn_decl, 1, t);
    8928          118 :   gimplify_and_add (t, pre_p);
    8929              : 
    8930          118 :   t = build_and_jump (&l2);
    8931          118 :   gimplify_and_add (t, pre_p);
    8932              : 
    8933          118 :   gimple_seq_add_stmt (pre_p, gimple_build_label (l1));
    8934              : 
    8935          118 :   gimple_seq_add_seq (pre_p, copyin_seq);
    8936              : 
    8937          118 :   gimple_seq_add_stmt (pre_p, gimple_build_label (l2));
    8938          118 : }
    8939              : 
    8940              : 
    8941              : /* Expand code for an OpenMP single directive.  */
    8942              : 
    8943              : static void
    8944         1140 : lower_omp_single (gimple_stmt_iterator *gsi_p, omp_context *ctx)
    8945              : {
    8946         1140 :   tree block;
    8947         1140 :   gomp_single *single_stmt = as_a <gomp_single *> (gsi_stmt (*gsi_p));
    8948         1140 :   gbind *bind;
    8949         1140 :   gimple_seq bind_body, bind_body_tail = NULL, dlist;
    8950              : 
    8951         1140 :   push_gimplify_context ();
    8952              : 
    8953         1140 :   block = make_node (BLOCK);
    8954         1140 :   bind = gimple_build_bind (NULL, NULL, block);
    8955         1140 :   gsi_replace (gsi_p, bind, true);
    8956         1140 :   bind_body = NULL;
    8957         1140 :   dlist = NULL;
    8958         1140 :   lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt),
    8959              :                            &bind_body, &dlist, ctx, NULL);
    8960         1140 :   lower_omp (gimple_omp_body_ptr (single_stmt), ctx);
    8961              : 
    8962         1140 :   gimple_seq_add_stmt (&bind_body, single_stmt);
    8963              : 
    8964         1140 :   if (ctx->record_type)
    8965          118 :     lower_omp_single_copy (single_stmt, &bind_body, ctx);
    8966              :   else
    8967         1022 :     lower_omp_single_simple (single_stmt, &bind_body);
    8968              : 
    8969         1140 :   gimple_omp_set_body (single_stmt, NULL);
    8970              : 
    8971         1140 :   gimple_seq_add_seq (&bind_body, dlist);
    8972              : 
    8973         1140 :   bind_body = maybe_catch_exception (bind_body);
    8974              : 
    8975         1140 :   bool nowait = omp_find_clause (gimple_omp_single_clauses (single_stmt),
    8976         1140 :                                  OMP_CLAUSE_NOWAIT) != NULL_TREE;
    8977         1140 :   gimple *g = gimple_build_omp_return (nowait);
    8978         1140 :   gimple_seq_add_stmt (&bind_body_tail, g);
    8979         1140 :   maybe_add_implicit_barrier_cancel (ctx, g, &bind_body_tail);
    8980              : 
    8981         1140 :   if (flag_openmp_ompt && !ctx->record_type)
    8982              :     {
    8983              :       /* Insert call to GOMP_single_end.  */
    8984            4 :       tree decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_END);
    8985            4 :       gimple *stmt = gimple_build_call (decl, 0);
    8986            4 :       gimple_seq_add_stmt (&bind_body_tail, stmt);
    8987              :     }
    8988              : 
    8989         1140 :   if (ctx->record_type)
    8990              :     {
    8991          118 :       gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
    8992          118 :       tree clobber = build_clobber (ctx->record_type);
    8993          118 :       gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
    8994              :                                                    clobber), GSI_SAME_STMT);
    8995              :     }
    8996         1140 :   gimple_seq_add_seq (&bind_body, bind_body_tail);
    8997         1140 :   gimple_bind_set_body (bind, bind_body);
    8998              : 
    8999         1140 :   pop_gimplify_context (bind);
    9000              : 
    9001         1140 :   gimple_bind_append_vars (bind, ctx->block_vars);
    9002         1140 :   BLOCK_VARS (block) = ctx->block_vars;
    9003         1140 :   if (BLOCK_VARS (block))
    9004           26 :     TREE_USED (block) = 1;
    9005         1140 : }
    9006              : 
    9007              : 
    9008              : /* Lower code for an OMP scope directive.  */
    9009              : 
    9010              : static void
    9011          161 : lower_omp_scope (gimple_stmt_iterator *gsi_p, omp_context *ctx)
    9012              : {
    9013          161 :   tree block;
    9014          161 :   gimple *scope_stmt = gsi_stmt (*gsi_p);
    9015          161 :   gbind *bind;
    9016          161 :   gimple_seq bind_body, bind_body_tail = NULL, dlist;
    9017          161 :   gimple_seq tred_dlist = NULL;
    9018              : 
    9019          161 :   push_gimplify_context ();
    9020              : 
    9021          161 :   block = make_node (BLOCK);
    9022          161 :   bind = gimple_build_bind (NULL, NULL, block);
    9023          161 :   gsi_replace (gsi_p, bind, true);
    9024          161 :   bind_body = NULL;
    9025          161 :   dlist = NULL;
    9026              : 
    9027          161 :   tree rclauses
    9028          161 :     = omp_task_reductions_find_first (gimple_omp_scope_clauses (scope_stmt),
    9029              :                                       OMP_SCOPE, OMP_CLAUSE_REDUCTION);
    9030          161 :   if (rclauses)
    9031              :     {
    9032           54 :       tree type = build_pointer_type (pointer_sized_int_node);
    9033           54 :       tree temp = create_tmp_var (type);
    9034           54 :       tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
    9035           54 :       OMP_CLAUSE_DECL (c) = temp;
    9036           54 :       OMP_CLAUSE_CHAIN (c) = gimple_omp_scope_clauses (scope_stmt);
    9037           54 :       gimple_omp_scope_set_clauses (scope_stmt, c);
    9038           54 :       lower_omp_task_reductions (ctx, OMP_SCOPE,
    9039              :                                  gimple_omp_scope_clauses (scope_stmt),
    9040              :                                  &bind_body, &tred_dlist);
    9041           54 :       rclauses = c;
    9042           54 :       tree fndecl = builtin_decl_explicit (
    9043           54 :         flag_openmp_ompt ? BUILT_IN_GOMP_SCOPE_START_WITH_END
    9044              :                          : BUILT_IN_GOMP_SCOPE_START);
    9045           54 :       gimple *stmt = gimple_build_call (fndecl, 1, temp);
    9046           54 :       gimple_seq_add_stmt (&bind_body, stmt);
    9047              :     }
    9048          107 :   else if (flag_openmp_ompt)
    9049              :     {
    9050            4 :       tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_SCOPE_START_WITH_END);
    9051            4 :       gimple *stmt = gimple_build_call (fndecl, 1, null_pointer_node);
    9052            4 :       gimple_seq_add_stmt (&bind_body, stmt);
    9053              :     }
    9054              : 
    9055          161 :   lower_rec_input_clauses (gimple_omp_scope_clauses (scope_stmt),
    9056              :                            &bind_body, &dlist, ctx, NULL);
    9057          161 :   lower_omp (gimple_omp_body_ptr (scope_stmt), ctx);
    9058              : 
    9059          161 :   gimple_seq_add_stmt (&bind_body, scope_stmt);
    9060              : 
    9061          161 :   gimple_seq_add_seq (&bind_body, gimple_omp_body (scope_stmt));
    9062              : 
    9063          161 :   gimple_omp_set_body (scope_stmt, NULL);
    9064              : 
    9065          161 :   gimple_seq clist = NULL;
    9066          161 :   lower_reduction_clauses (gimple_omp_scope_clauses (scope_stmt),
    9067              :                            &bind_body, &clist, ctx);
    9068          161 :   if (clist)
    9069              :     {
    9070            0 :       tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_START);
    9071            0 :       gcall *g = gimple_build_call (fndecl, 0);
    9072            0 :       gimple_seq_add_stmt (&bind_body, g);
    9073            0 :       gimple_seq_add_seq (&bind_body, clist);
    9074            0 :       fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_END);
    9075            0 :       g = gimple_build_call (fndecl, 0);
    9076            0 :       gimple_seq_add_stmt (&bind_body, g);
    9077              :     }
    9078              : 
    9079          161 :   gimple_seq_add_seq (&bind_body, dlist);
    9080              : 
    9081          161 :   bind_body = maybe_catch_exception (bind_body);
    9082              : 
    9083          161 :   if (flag_openmp_ompt)
    9084              :     {
    9085            8 :       tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_SCOPE_END);
    9086            8 :       gcall *g = gimple_build_call (fndecl, 0);
    9087            8 :       gimple_seq_add_stmt (&bind_body_tail, g);
    9088              :     }
    9089              : 
    9090          161 :   bool nowait = omp_find_clause (gimple_omp_scope_clauses (scope_stmt),
    9091          161 :                                  OMP_CLAUSE_NOWAIT) != NULL_TREE;
    9092          161 :   gimple *g = gimple_build_omp_return (nowait);
    9093          161 :   gimple_seq_add_stmt (&bind_body_tail, g);
    9094          161 :   gimple_seq_add_seq (&bind_body_tail, tred_dlist);
    9095          161 :   maybe_add_implicit_barrier_cancel (ctx, g, &bind_body_tail);
    9096          161 :   if (ctx->record_type)
    9097              :     {
    9098            0 :       gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
    9099            0 :       tree clobber = build_clobber (ctx->record_type);
    9100            0 :       gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
    9101              :                                                    clobber), GSI_SAME_STMT);
    9102              :     }
    9103          161 :   gimple_seq_add_seq (&bind_body, bind_body_tail);
    9104              : 
    9105          161 :   gimple_bind_set_body (bind, bind_body);
    9106              : 
    9107          161 :   pop_gimplify_context (bind);
    9108              : 
    9109          161 :   gimple_bind_append_vars (bind, ctx->block_vars);
    9110          161 :   BLOCK_VARS (block) = ctx->block_vars;
    9111          161 :   if (BLOCK_VARS (block))
    9112          121 :     TREE_USED (block) = 1;
    9113          161 : }
    9114              : 
    9115              : /* Lower code for an OMP dispatch directive.  */
    9116              : 
    9117              : static void
    9118          659 : lower_omp_dispatch (gimple_stmt_iterator *gsi_p, omp_context *ctx)
    9119              : {
    9120          659 :   tree block;
    9121          659 :   gimple *stmt = gsi_stmt (*gsi_p);
    9122          659 :   gbind *bind;
    9123              : 
    9124          659 :   push_gimplify_context ();
    9125              : 
    9126          659 :   block = make_node (BLOCK);
    9127          659 :   bind = gimple_build_bind (NULL, NULL, block);
    9128          659 :   gsi_replace (gsi_p, bind, true);
    9129              : 
    9130          659 :   lower_omp (gimple_omp_body_ptr (stmt), ctx);
    9131          659 :   gimple_bind_set_body (bind, maybe_catch_exception (gimple_omp_body (stmt)));
    9132              : 
    9133          659 :   pop_gimplify_context (bind);
    9134              : 
    9135          659 :   gimple_bind_append_vars (bind, ctx->block_vars);
    9136          659 :   BLOCK_VARS (block) = ctx->block_vars;
    9137          659 : }
    9138              : 
    9139              : /* Expand code for an OpenMP master or masked directive.  */
    9140              : 
    9141              : static void
    9142         1127 : lower_omp_master (gimple_stmt_iterator *gsi_p, omp_context *ctx)
    9143              : {
    9144         1127 :   tree block, lab = NULL, x, bfn_decl;
    9145         1127 :   gimple *stmt = gsi_stmt (*gsi_p);
    9146         1127 :   gbind *bind;
    9147         1127 :   location_t loc = gimple_location (stmt);
    9148         1127 :   gimple_seq tseq;
    9149         1127 :   tree filter = integer_zero_node;
    9150              : 
    9151         1127 :   push_gimplify_context ();
    9152              : 
    9153         1127 :   if (gimple_code (stmt) == GIMPLE_OMP_MASKED)
    9154              :     {
    9155          393 :       filter = omp_find_clause (gimple_omp_masked_clauses (stmt),
    9156              :                                 OMP_CLAUSE_FILTER);
    9157          393 :       if (filter)
    9158          252 :         filter = fold_convert (integer_type_node,
    9159              :                                OMP_CLAUSE_FILTER_EXPR (filter));
    9160              :       else
    9161          141 :         filter = integer_zero_node;
    9162              :     }
    9163         1127 :   block = make_node (BLOCK);
    9164         1127 :   bind = gimple_build_bind (NULL, NULL, block);
    9165         1127 :   gsi_replace (gsi_p, bind, true);
    9166         1127 :   gimple_bind_add_stmt (bind, stmt);
    9167              : 
    9168         1127 :   bfn_decl = builtin_decl_explicit (
    9169         1127 :     flag_openmp_ompt ? BUILT_IN_GOMP_HAS_MASKED_THREAD_NUM_WITH_END
    9170              :                      : BUILT_IN_GOMP_HAS_MASKED_THREAD_NUM);
    9171         1127 :   x = build_call_expr_loc (loc, bfn_decl, 1, filter);
    9172         1127 :   x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab));
    9173         1127 :   tseq = NULL;
    9174         1127 :   gimplify_and_add (x, &tseq);
    9175         1127 :   gimple_bind_add_seq (bind, tseq);
    9176              : 
    9177         1127 :   lower_omp (gimple_omp_body_ptr (stmt), ctx);
    9178         1127 :   gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
    9179         1127 :   gimple_bind_add_seq (bind, gimple_omp_body (stmt));
    9180         1127 :   gimple_omp_set_body (stmt, NULL);
    9181              : 
    9182         1127 :   if (flag_openmp_ompt)
    9183              :     {
    9184              :       /* Insert call to GOMP_masked_end at the end of the body.  */
    9185            4 :       tree decl = builtin_decl_explicit (BUILT_IN_GOMP_MASKED_END);
    9186            4 :       gcall *g = gimple_build_call (decl, 0);
    9187            4 :       gimple_bind_add_stmt (bind, g);
    9188              :     }
    9189              : 
    9190         1127 :   gimple_bind_add_stmt (bind, gimple_build_label (lab));
    9191              : 
    9192         1127 :   gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
    9193              : 
    9194         1127 :   pop_gimplify_context (bind);
    9195              : 
    9196         1127 :   gimple_bind_append_vars (bind, ctx->block_vars);
    9197         1127 :   BLOCK_VARS (block) = ctx->block_vars;
    9198         1127 : }
    9199              : 
    9200              : /* Helper function for lower_omp_task_reductions.  For a specific PASS
    9201              :    find out the current clause it should be processed, or return false
    9202              :    if all have been processed already.  */
    9203              : 
    9204              : static inline bool
    9205         8030 : omp_task_reduction_iterate (int pass, enum tree_code code,
    9206              :                             enum omp_clause_code ccode, tree *c, tree *decl,
    9207              :                             tree *type, tree *next)
    9208              : {
    9209        11412 :   for (; *c; *c = omp_find_clause (OMP_CLAUSE_CHAIN (*c), ccode))
    9210              :     {
    9211         6764 :       if (ccode == OMP_CLAUSE_REDUCTION
    9212         6708 :           && code != OMP_TASKLOOP
    9213         6708 :           && !OMP_CLAUSE_REDUCTION_TASK (*c))
    9214           56 :         continue;
    9215         6652 :       *decl = OMP_CLAUSE_DECL (*c);
    9216         6652 :       *type = TREE_TYPE (*decl);
    9217         6652 :       if (TREE_CODE (*decl) == MEM_REF)
    9218              :         {
    9219         1940 :           if (pass != 1)
    9220          970 :             continue;
    9221              :         }
    9222              :       else
    9223              :         {
    9224         4712 :           if (omp_privatize_by_reference (*decl))
    9225          168 :             *type = TREE_TYPE (*type);
    9226         4712 :           if (pass != (!TREE_CONSTANT (TYPE_SIZE_UNIT (*type))))
    9227         2356 :             continue;
    9228              :         }
    9229         3326 :       *next = omp_find_clause (OMP_CLAUSE_CHAIN (*c), ccode);
    9230         3326 :       return true;
    9231              :     }
    9232         4704 :   *decl = NULL_TREE;
    9233         4704 :   *type = NULL_TREE;
    9234         4704 :   *next = NULL_TREE;
    9235         4704 :   return false;
    9236              : }
    9237              : 
    9238              : /* Lower task_reduction and reduction clauses (the latter unless CODE is
    9239              :    OMP_TASKGROUP only with task modifier).  Register mapping of those in
    9240              :    START sequence and reducing them and unregister them in the END sequence.  */
    9241              : 
    9242              : static void
    9243         1385 : lower_omp_task_reductions (omp_context *ctx, enum tree_code code, tree clauses,
    9244              :                            gimple_seq *start, gimple_seq *end)
    9245              : {
    9246          849 :   enum omp_clause_code ccode
    9247              :     = (code == OMP_TASKGROUP
    9248         1385 :        ? OMP_CLAUSE_TASK_REDUCTION : OMP_CLAUSE_REDUCTION);
    9249         1385 :   tree cancellable = NULL_TREE;
    9250         1385 :   clauses = omp_task_reductions_find_first (clauses, code, ccode);
    9251         1385 :   if (clauses == NULL_TREE)
    9252          209 :     return;
    9253         1176 :   if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE)
    9254              :     {
    9255          325 :       for (omp_context *outer = ctx->outer; outer; outer = outer->outer)
    9256          239 :         if (gimple_code (outer->stmt) == GIMPLE_OMP_PARALLEL
    9257          239 :             && outer->cancellable)
    9258              :           {
    9259           14 :             cancellable = error_mark_node;
    9260           14 :             break;
    9261              :           }
    9262          225 :         else if (gimple_code (outer->stmt) != GIMPLE_OMP_TASKGROUP
    9263          225 :                  && gimple_code (outer->stmt) != GIMPLE_OMP_SCOPE)
    9264              :           break;
    9265              :     }
    9266         1176 :   tree record_type = lang_hooks.types.make_type (RECORD_TYPE);
    9267         1176 :   tree *last = &TYPE_FIELDS (record_type);
    9268         1176 :   unsigned cnt = 0;
    9269         1176 :   if (cancellable)
    9270              :     {
    9271           14 :       tree field = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE,
    9272              :                                ptr_type_node);
    9273           14 :       tree ifield = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE,
    9274              :                                 integer_type_node);
    9275           14 :       *last = field;
    9276           14 :       DECL_CHAIN (field) = ifield;
    9277           14 :       last = &DECL_CHAIN (ifield);
    9278           14 :       DECL_CONTEXT (field) = record_type;
    9279           14 :       if (TYPE_ALIGN (record_type) < DECL_ALIGN (field))
    9280            0 :         SET_TYPE_ALIGN (record_type, DECL_ALIGN (field));
    9281           14 :       DECL_CONTEXT (ifield) = record_type;
    9282           14 :       if (TYPE_ALIGN (record_type) < DECL_ALIGN (ifield))
    9283            0 :         SET_TYPE_ALIGN (record_type, DECL_ALIGN (ifield));
    9284              :     }
    9285         3528 :   for (int pass = 0; pass < 2; pass++)
    9286              :     {
    9287         2352 :       tree decl, type, next;
    9288         2352 :       for (tree c = clauses;
    9289         4015 :            omp_task_reduction_iterate (pass, code, ccode,
    9290         1663 :                                        &c, &decl, &type, &next); c = next)
    9291              :         {
    9292         1663 :           ++cnt;
    9293         1663 :           tree new_type = type;
    9294         1663 :           if (ctx->outer)
    9295         1166 :             new_type = remap_type (type, &ctx->outer->cb);
    9296         1663 :           tree field
    9297         1663 :             = build_decl (OMP_CLAUSE_LOCATION (c), FIELD_DECL,
    9298         1663 :                           DECL_P (decl) ? DECL_NAME (decl) : NULL_TREE,
    9299              :                           new_type);
    9300         1663 :           if (DECL_P (decl) && type == TREE_TYPE (decl))
    9301              :             {
    9302         1136 :               SET_DECL_ALIGN (field, DECL_ALIGN (decl));
    9303         1136 :               DECL_USER_ALIGN (field) = DECL_USER_ALIGN (decl);
    9304         1136 :               TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (decl);
    9305              :             }
    9306              :           else
    9307          527 :             SET_DECL_ALIGN (field, TYPE_ALIGN (type));
    9308         1663 :           DECL_CONTEXT (field) = record_type;
    9309         1663 :           if (TYPE_ALIGN (record_type) < DECL_ALIGN (field))
    9310         1200 :             SET_TYPE_ALIGN (record_type, DECL_ALIGN (field));
    9311         1663 :           *last = field;
    9312         1663 :           last = &DECL_CHAIN (field);
    9313         1663 :           tree bfield
    9314         1663 :             = build_decl (OMP_CLAUSE_LOCATION (c), FIELD_DECL, NULL_TREE,
    9315              :                           boolean_type_node);
    9316         1663 :           DECL_CONTEXT (bfield) = record_type;
    9317         1663 :           if (TYPE_ALIGN (record_type) < DECL_ALIGN (bfield))
    9318            0 :             SET_TYPE_ALIGN (record_type, DECL_ALIGN (bfield));
    9319         1663 :           *last = bfield;
    9320         1663 :           last = &DECL_CHAIN (bfield);
    9321              :         }
    9322              :     }
    9323         1176 :   *last = NULL_TREE;
    9324         1176 :   layout_type (record_type);
    9325              : 
    9326              :   /* Build up an array which registers with the runtime all the reductions
    9327              :      and deregisters them at the end.  Format documented in libgomp/task.c.  */
    9328         1176 :   tree atype = build_array_type_nelts (pointer_sized_int_node, 7 + cnt * 3);
    9329         1176 :   tree avar = create_tmp_var_raw (atype);
    9330         1176 :   gimple_add_tmp_var (avar);
    9331         1176 :   TREE_ADDRESSABLE (avar) = 1;
    9332         1176 :   tree r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_zero_node,
    9333              :                    NULL_TREE, NULL_TREE);
    9334         1176 :   tree t = build_int_cst (pointer_sized_int_node, cnt);
    9335         1176 :   gimple_seq_add_stmt (start, gimple_build_assign (r, t));
    9336         1176 :   gimple_seq seq = NULL;
    9337         1176 :   tree sz = fold_convert (pointer_sized_int_node,
    9338              :                           TYPE_SIZE_UNIT (record_type));
    9339         1176 :   int cachesz = 64;
    9340         1176 :   sz = fold_build2 (PLUS_EXPR, pointer_sized_int_node, sz,
    9341              :                     build_int_cst (pointer_sized_int_node, cachesz - 1));
    9342         1176 :   sz = fold_build2 (BIT_AND_EXPR, pointer_sized_int_node, sz,
    9343              :                     build_int_cst (pointer_sized_int_node, ~(cachesz - 1)));
    9344         1176 :   ctx->task_reductions.create (1 + cnt);
    9345         1176 :   ctx->task_reduction_map = new hash_map<tree, unsigned>;
    9346         2352 :   ctx->task_reductions.quick_push (TREE_CODE (sz) == INTEGER_CST
    9347         1216 :                                    ? sz : NULL_TREE);
    9348         1176 :   sz = force_gimple_operand (sz, &seq, true, NULL_TREE);
    9349         1176 :   gimple_seq_add_seq (start, seq);
    9350         1176 :   r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_one_node,
    9351              :               NULL_TREE, NULL_TREE);
    9352         1176 :   gimple_seq_add_stmt (start, gimple_build_assign (r, sz));
    9353         1176 :   r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (2),
    9354              :               NULL_TREE, NULL_TREE);
    9355         1176 :   t = build_int_cst (pointer_sized_int_node,
    9356         1176 :                      MAX (TYPE_ALIGN_UNIT (record_type), (unsigned) cachesz));
    9357         1176 :   gimple_seq_add_stmt (start, gimple_build_assign (r, t));
    9358         1176 :   r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (3),
    9359              :               NULL_TREE, NULL_TREE);
    9360         1176 :   t = build_int_cst (pointer_sized_int_node, -1);
    9361         1176 :   gimple_seq_add_stmt (start, gimple_build_assign (r, t));
    9362         1176 :   r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (4),
    9363              :               NULL_TREE, NULL_TREE);
    9364         1176 :   t = build_int_cst (pointer_sized_int_node, 0);
    9365         1176 :   gimple_seq_add_stmt (start, gimple_build_assign (r, t));
    9366              : 
    9367              :   /* In end, build a loop that iterates from 0 to < omp_get_num_threads ()
    9368              :      and for each task reduction checks a bool right after the private variable
    9369              :      within that thread's chunk; if the bool is clear, it hasn't been
    9370              :      initialized and thus isn't going to be reduced nor destructed, otherwise
    9371              :      reduce and destruct it.  */
    9372         1176 :   tree idx = create_tmp_var (size_type_node);
    9373         1176 :   gimple_seq_add_stmt (end, gimple_build_assign (idx, size_zero_node));
    9374         1176 :   tree num_thr_sz = create_tmp_var (size_type_node);
    9375         1176 :   tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
    9376         1176 :   tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
    9377         1176 :   tree lab3 = NULL_TREE, lab7 = NULL_TREE;
    9378         1176 :   gimple *g;
    9379         1176 :   if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE)
    9380              :     {
    9381              :       /* For worksharing constructs or scope, only perform it in the master
    9382              :          thread, with the exception of cancelled implicit barriers - then only
    9383              :          handle the current thread.  */
    9384          290 :       tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
    9385          290 :       t = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
    9386          290 :       tree thr_num = create_tmp_var (integer_type_node);
    9387          290 :       g = gimple_build_call (t, 0);
    9388          290 :       gimple_call_set_lhs (g, thr_num);
    9389          290 :       gimple_seq_add_stmt (end, g);
    9390          290 :       if (cancellable)
    9391              :         {
    9392           14 :           tree c;
    9393           14 :           tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
    9394           14 :           tree lab6 = create_artificial_label (UNKNOWN_LOCATION);
    9395           14 :           lab3 = create_artificial_label (UNKNOWN_LOCATION);
    9396           14 :           if (code == OMP_FOR)
    9397            4 :             c = gimple_omp_for_clauses (ctx->stmt);
    9398           10 :           else if (code == OMP_SECTIONS)
    9399            0 :             c = gimple_omp_sections_clauses (ctx->stmt);
    9400              :           else /* if (code == OMP_SCOPE) */
    9401           10 :             c = gimple_omp_scope_clauses (ctx->stmt);
    9402           14 :           c = OMP_CLAUSE_DECL (omp_find_clause (c, OMP_CLAUSE__REDUCTEMP_));
    9403           14 :           cancellable = c;
    9404           14 :           g = gimple_build_cond (NE_EXPR, c, build_zero_cst (TREE_TYPE (c)),
    9405              :                                  lab5, lab6);
    9406           14 :           gimple_seq_add_stmt (end, g);
    9407           14 :           gimple_seq_add_stmt (end, gimple_build_label (lab5));
    9408           14 :           g = gimple_build_assign (idx, NOP_EXPR, thr_num);
    9409           14 :           gimple_seq_add_stmt (end, g);
    9410           14 :           g = gimple_build_assign (num_thr_sz, PLUS_EXPR, idx,
    9411           14 :                                    build_one_cst (TREE_TYPE (idx)));
    9412           14 :           gimple_seq_add_stmt (end, g);
    9413           14 :           gimple_seq_add_stmt (end, gimple_build_goto (lab3));
    9414           14 :           gimple_seq_add_stmt (end, gimple_build_label (lab6));
    9415              :         }
    9416          290 :       g = gimple_build_cond (NE_EXPR, thr_num, integer_zero_node, lab2, lab4);
    9417          290 :       gimple_seq_add_stmt (end, g);
    9418          290 :       gimple_seq_add_stmt (end, gimple_build_label (lab4));
    9419              :     }
    9420         1176 :   if (code != OMP_PARALLEL)
    9421              :     {
    9422         1114 :       t = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
    9423         1114 :       tree num_thr = create_tmp_var (integer_type_node);
    9424         1114 :       g = gimple_build_call (t, 0);
    9425         1114 :       gimple_call_set_lhs (g, num_thr);
    9426         1114 :       gimple_seq_add_stmt (end, g);
    9427         1114 :       g = gimple_build_assign (num_thr_sz, NOP_EXPR, num_thr);
    9428         1114 :       gimple_seq_add_stmt (end, g);
    9429         1114 :       if (cancellable)
    9430           14 :         gimple_seq_add_stmt (end, gimple_build_label (lab3));
    9431              :     }
    9432              :   else
    9433              :     {
    9434           62 :       tree c = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
    9435              :                                 OMP_CLAUSE__REDUCTEMP_);
    9436           62 :       t = fold_convert (pointer_sized_int_node, OMP_CLAUSE_DECL (c));
    9437           62 :       t = fold_convert (size_type_node, t);
    9438           62 :       gimplify_assign (num_thr_sz, t, end);
    9439              :     }
    9440         1176 :   t = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (2),
    9441              :               NULL_TREE, NULL_TREE);
    9442         1176 :   tree data = create_tmp_var (pointer_sized_int_node);
    9443         1176 :   gimple_seq_add_stmt (end, gimple_build_assign (data, t));
    9444         1176 :   if (code == OMP_TASKLOOP)
    9445              :     {
    9446          497 :       lab7 = create_artificial_label (UNKNOWN_LOCATION);
    9447          497 :       g = gimple_build_cond (NE_EXPR, data,
    9448              :                              build_zero_cst (pointer_sized_int_node),
    9449              :                              lab1, lab7);
    9450          497 :       gimple_seq_add_stmt (end, g);
    9451              :     }
    9452         1176 :   gimple_seq_add_stmt (end, gimple_build_label (lab1));
    9453         1176 :   tree ptr;
    9454         1176 :   if (TREE_CODE (TYPE_SIZE_UNIT (record_type)) == INTEGER_CST)
    9455         1136 :     ptr = create_tmp_var (build_pointer_type (record_type));
    9456              :   else
    9457           40 :     ptr = create_tmp_var (ptr_type_node);
    9458         1176 :   gimple_seq_add_stmt (end, gimple_build_assign (ptr, NOP_EXPR, data));
    9459              : 
    9460         1176 :   tree field = TYPE_FIELDS (record_type);
    9461         1176 :   cnt = 0;
    9462         1176 :   if (cancellable)
    9463           14 :     field = DECL_CHAIN (DECL_CHAIN (field));
    9464         3528 :   for (int pass = 0; pass < 2; pass++)
    9465              :     {
    9466         2352 :       tree decl, type, next;
    9467         2352 :       for (tree c = clauses;
    9468         4015 :            omp_task_reduction_iterate (pass, code, ccode,
    9469         1663 :                                        &c, &decl, &type, &next); c = next)
    9470              :         {
    9471         1663 :           tree var = decl, ref;
    9472         1663 :           if (TREE_CODE (decl) == MEM_REF)
    9473              :             {
    9474          485 :               var = TREE_OPERAND (var, 0);
    9475          485 :               if (TREE_CODE (var) == POINTER_PLUS_EXPR)
    9476          108 :                 var = TREE_OPERAND (var, 0);
    9477          485 :               tree v = var;
    9478          485 :               if (TREE_CODE (var) == ADDR_EXPR)
    9479          310 :                 var = TREE_OPERAND (var, 0);
    9480          175 :               else if (INDIRECT_REF_P (var))
    9481            4 :                 var = TREE_OPERAND (var, 0);
    9482          485 :               tree orig_var = var;
    9483          485 :               if (is_variable_sized (var))
    9484              :                 {
    9485           31 :                   gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
    9486           31 :                   var = DECL_VALUE_EXPR (var);
    9487           31 :                   gcc_assert (INDIRECT_REF_P (var));
    9488           31 :                   var = TREE_OPERAND (var, 0);
    9489           31 :                   gcc_assert (DECL_P (var));
    9490              :                 }
    9491          485 :               t = ref = maybe_lookup_decl_in_outer_ctx (var, ctx);
    9492          485 :               if (orig_var != var)
    9493           31 :                 gcc_assert (TREE_CODE (v) == ADDR_EXPR);
    9494          454 :               else if (TREE_CODE (v) == ADDR_EXPR)
    9495          279 :                 t = build_fold_addr_expr (t);
    9496          175 :               else if (INDIRECT_REF_P (v))
    9497            4 :                 t = build_fold_indirect_ref (t);
    9498          485 :               if (TREE_CODE (TREE_OPERAND (decl, 0)) == POINTER_PLUS_EXPR)
    9499              :                 {
    9500          108 :                   tree b = TREE_OPERAND (TREE_OPERAND (decl, 0), 1);
    9501          108 :                   b = maybe_lookup_decl_in_outer_ctx (b, ctx);
    9502          108 :                   t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t, b);
    9503              :                 }
    9504          485 :               if (!integer_zerop (TREE_OPERAND (decl, 1)))
    9505          219 :                 t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t,
    9506              :                                  fold_convert (size_type_node,
    9507              :                                                TREE_OPERAND (decl, 1)));
    9508              :             }
    9509              :           else
    9510              :             {
    9511         1178 :               t = ref = maybe_lookup_decl_in_outer_ctx (var, ctx);
    9512         1178 :               if (!omp_privatize_by_reference (decl))
    9513         1136 :                 t = build_fold_addr_expr (t);
    9514              :             }
    9515         1663 :           t = fold_convert (pointer_sized_int_node, t);
    9516         1663 :           seq = NULL;
    9517         1663 :           t = force_gimple_operand (t, &seq, true, NULL_TREE);
    9518         1663 :           gimple_seq_add_seq (start, seq);
    9519         1663 :           r = build4 (ARRAY_REF, pointer_sized_int_node, avar,
    9520         1663 :                       size_int (7 + cnt * 3), NULL_TREE, NULL_TREE);
    9521         1663 :           gimple_seq_add_stmt (start, gimple_build_assign (r, t));
    9522         1663 :           t = unshare_expr (byte_position (field));
    9523         1663 :           t = fold_convert (pointer_sized_int_node, t);
    9524         1663 :           ctx->task_reduction_map->put (c, cnt);
    9525         3326 :           ctx->task_reductions.quick_push (TREE_CODE (t) == INTEGER_CST
    9526         2035 :                                            ? t : NULL_TREE);
    9527         1663 :           seq = NULL;
    9528         1663 :           t = force_gimple_operand (t, &seq, true, NULL_TREE);
    9529         1663 :           gimple_seq_add_seq (start, seq);
    9530         1663 :           r = build4 (ARRAY_REF, pointer_sized_int_node, avar,
    9531         1663 :                       size_int (7 + cnt * 3 + 1), NULL_TREE, NULL_TREE);
    9532         1663 :           gimple_seq_add_stmt (start, gimple_build_assign (r, t));
    9533              : 
    9534         1663 :           tree bfield = DECL_CHAIN (field);
    9535         1663 :           tree cond;
    9536         1663 :           if (code == OMP_PARALLEL
    9537         1663 :               || code == OMP_FOR
    9538              :               || code == OMP_SECTIONS
    9539              :               || code == OMP_SCOPE)
    9540              :             /* In parallel, worksharing or scope all threads unconditionally
    9541              :                initialize all their task reduction private variables.  */
    9542          565 :             cond = boolean_true_node;
    9543         1098 :           else if (TREE_TYPE (ptr) == ptr_type_node)
    9544              :             {
    9545          261 :               cond = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr,
    9546              :                              unshare_expr (byte_position (bfield)));
    9547          261 :               seq = NULL;
    9548          261 :               cond = force_gimple_operand (cond, &seq, true, NULL_TREE);
    9549          261 :               gimple_seq_add_seq (end, seq);
    9550          261 :               tree pbool = build_pointer_type (TREE_TYPE (bfield));
    9551          261 :               cond = build2 (MEM_REF, TREE_TYPE (bfield), cond,
    9552              :                              build_int_cst (pbool, 0));
    9553              :             }
    9554              :           else
    9555          837 :             cond = build3 (COMPONENT_REF, TREE_TYPE (bfield),
    9556              :                            build_simple_mem_ref (ptr), bfield, NULL_TREE);
    9557         1663 :           tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
    9558         1663 :           tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
    9559         1663 :           tree condv = create_tmp_var (boolean_type_node);
    9560         1663 :           gimple_seq_add_stmt (end, gimple_build_assign (condv, cond));
    9561         1663 :           g = gimple_build_cond (NE_EXPR, condv, boolean_false_node,
    9562              :                                  lab3, lab4);
    9563         1663 :           gimple_seq_add_stmt (end, g);
    9564         1663 :           gimple_seq_add_stmt (end, gimple_build_label (lab3));
    9565         1663 :           if (cancellable && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) == NULL_TREE)
    9566              :             {
    9567              :               /* If this reduction doesn't need destruction and parallel
    9568              :                  has been cancelled, there is nothing to do for this
    9569              :                  reduction, so jump around the merge operation.  */
    9570           18 :               tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
    9571           18 :               g = gimple_build_cond (NE_EXPR, cancellable,
    9572           18 :                                      build_zero_cst (TREE_TYPE (cancellable)),
    9573              :                                      lab4, lab5);
    9574           18 :               gimple_seq_add_stmt (end, g);
    9575           18 :               gimple_seq_add_stmt (end, gimple_build_label (lab5));
    9576              :             }
    9577              : 
    9578         1663 :           tree new_var;
    9579         1663 :           if (TREE_TYPE (ptr) == ptr_type_node)
    9580              :             {
    9581          428 :               new_var = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr,
    9582              :                                 unshare_expr (byte_position (field)));
    9583          428 :               seq = NULL;
    9584          428 :               new_var = force_gimple_operand (new_var, &seq, true, NULL_TREE);
    9585          428 :               gimple_seq_add_seq (end, seq);
    9586          428 :               tree pbool = build_pointer_type (TREE_TYPE (field));
    9587          428 :               new_var = build2 (MEM_REF, TREE_TYPE (field), new_var,
    9588              :                                 build_int_cst (pbool, 0));
    9589              :             }
    9590              :           else
    9591         1235 :             new_var = build3 (COMPONENT_REF, TREE_TYPE (field),
    9592              :                               build_simple_mem_ref (ptr), field, NULL_TREE);
    9593              : 
    9594         1663 :           enum tree_code rcode = OMP_CLAUSE_REDUCTION_CODE (c);
    9595         1663 :           if (TREE_CODE (decl) != MEM_REF
    9596         1663 :               && omp_privatize_by_reference (decl))
    9597           42 :             ref = build_simple_mem_ref (ref);
    9598              :           /* reduction(-:var) sums up the partial results, so it acts
    9599              :              identically to reduction(+:var).  */
    9600         1663 :           if (rcode == MINUS_EXPR)
    9601            0 :             rcode = PLUS_EXPR;
    9602         1663 :           if (TREE_CODE (decl) == MEM_REF)
    9603              :             {
    9604          485 :               tree type = TREE_TYPE (new_var);
    9605          485 :               tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
    9606          485 :               tree i = create_tmp_var (TREE_TYPE (v));
    9607          485 :               tree ptype = build_pointer_type (TREE_TYPE (type));
    9608          485 :               if (DECL_P (v))
    9609              :                 {
    9610          142 :                   v = maybe_lookup_decl_in_outer_ctx (v, ctx);
    9611          142 :                   tree vv = create_tmp_var (TREE_TYPE (v));
    9612          142 :                   gimplify_assign (vv, v, start);
    9613          142 :                   v = vv;
    9614              :                 }
    9615          485 :               ref = build4 (ARRAY_REF, pointer_sized_int_node, avar,
    9616          485 :                             size_int (7 + cnt * 3), NULL_TREE, NULL_TREE);
    9617          485 :               new_var = build_fold_addr_expr (new_var);
    9618          485 :               new_var = fold_convert (ptype, new_var);
    9619          485 :               ref = fold_convert (ptype, ref);
    9620          485 :               tree m = create_tmp_var (ptype);
    9621          485 :               gimplify_assign (m, new_var, end);
    9622          485 :               new_var = m;
    9623          485 :               m = create_tmp_var (ptype);
    9624          485 :               gimplify_assign (m, ref, end);
    9625          485 :               ref = m;
    9626          485 :               gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), end);
    9627          485 :               tree body = create_artificial_label (UNKNOWN_LOCATION);
    9628          485 :               tree endl = create_artificial_label (UNKNOWN_LOCATION);
    9629          485 :               gimple_seq_add_stmt (end, gimple_build_label (body));
    9630          485 :               tree priv = build_simple_mem_ref (new_var);
    9631          485 :               tree out = build_simple_mem_ref (ref);
    9632          485 :               if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
    9633              :                 {
    9634          161 :                   tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
    9635          161 :                   tree decl_placeholder
    9636          161 :                     = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
    9637          161 :                   tree lab6 = NULL_TREE;
    9638          161 :                   if (cancellable)
    9639              :                     {
    9640              :                       /* If this reduction needs destruction and parallel
    9641              :                          has been cancelled, jump around the merge operation
    9642              :                          to the destruction.  */
    9643            4 :                       tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
    9644            4 :                       lab6 = create_artificial_label (UNKNOWN_LOCATION);
    9645            4 :                       tree zero = build_zero_cst (TREE_TYPE (cancellable));
    9646            4 :                       g = gimple_build_cond (NE_EXPR, cancellable, zero,
    9647              :                                              lab6, lab5);
    9648            4 :                       gimple_seq_add_stmt (end, g);
    9649            4 :                       gimple_seq_add_stmt (end, gimple_build_label (lab5));
    9650              :                     }
    9651          161 :                   SET_DECL_VALUE_EXPR (placeholder, out);
    9652          161 :                   DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
    9653          161 :                   SET_DECL_VALUE_EXPR (decl_placeholder, priv);
    9654          161 :                   DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
    9655          161 :                   lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
    9656          322 :                   gimple_seq_add_seq (end,
    9657          161 :                                       OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
    9658          161 :                   OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
    9659          161 :                   if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
    9660              :                     {
    9661           56 :                       OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
    9662           56 :                       OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c) = NULL;
    9663              :                     }
    9664          161 :                   if (cancellable)
    9665            4 :                     gimple_seq_add_stmt (end, gimple_build_label (lab6));
    9666          161 :                   tree x = lang_hooks.decls.omp_clause_dtor (c, priv);
    9667          161 :                   if (x)
    9668              :                     {
    9669          137 :                       gimple_seq tseq = NULL;
    9670          137 :                       gimplify_stmt (&x, &tseq);
    9671          137 :                       gimple_seq_add_seq (end, tseq);
    9672              :                     }
    9673              :                 }
    9674              :               else
    9675              :                 {
    9676          324 :                   tree x = build2 (rcode, TREE_TYPE (out), out, priv);
    9677          324 :                   out = unshare_expr (out);
    9678          324 :                   gimplify_assign (out, x, end);
    9679              :                 }
    9680          485 :               gimple *g
    9681          485 :                 = gimple_build_assign (new_var, POINTER_PLUS_EXPR, new_var,
    9682          485 :                                        TYPE_SIZE_UNIT (TREE_TYPE (type)));
    9683          485 :               gimple_seq_add_stmt (end, g);
    9684          485 :               g = gimple_build_assign (ref, POINTER_PLUS_EXPR, ref,
    9685          485 :                                        TYPE_SIZE_UNIT (TREE_TYPE (type)));
    9686          485 :               gimple_seq_add_stmt (end, g);
    9687          485 :               g = gimple_build_assign (i, PLUS_EXPR, i,
    9688          485 :                                        build_int_cst (TREE_TYPE (i), 1));
    9689          485 :               gimple_seq_add_stmt (end, g);
    9690          485 :               g = gimple_build_cond (LE_EXPR, i, v, body, endl);
    9691          485 :               gimple_seq_add_stmt (end, g);
    9692          485 :               gimple_seq_add_stmt (end, gimple_build_label (endl));
    9693              :             }
    9694         1178 :           else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
    9695              :             {
    9696          135 :               tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
    9697          135 :               tree oldv = NULL_TREE;
    9698          135 :               tree lab6 = NULL_TREE;
    9699          135 :               if (cancellable)
    9700              :                 {
    9701              :                   /* If this reduction needs destruction and parallel
    9702              :                      has been cancelled, jump around the merge operation
    9703              :                      to the destruction.  */
    9704            4 :                   tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
    9705            4 :                   lab6 = create_artificial_label (UNKNOWN_LOCATION);
    9706            4 :                   tree zero = build_zero_cst (TREE_TYPE (cancellable));
    9707            4 :                   g = gimple_build_cond (NE_EXPR, cancellable, zero,
    9708              :                                          lab6, lab5);
    9709            4 :                   gimple_seq_add_stmt (end, g);
    9710            4 :                   gimple_seq_add_stmt (end, gimple_build_label (lab5));
    9711              :                 }
    9712          135 :               if (omp_privatize_by_reference (decl)
    9713          149 :                   && !useless_type_conversion_p (TREE_TYPE (placeholder),
    9714           14 :                                                  TREE_TYPE (ref)))
    9715            0 :                 ref = build_fold_addr_expr_loc (OMP_CLAUSE_LOCATION (c), ref);
    9716          135 :               ref = build_fold_addr_expr_loc (OMP_CLAUSE_LOCATION (c), ref);
    9717          135 :               tree refv = create_tmp_var (TREE_TYPE (ref));
    9718          135 :               gimplify_assign (refv, ref, end);
    9719          135 :               ref = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), refv);
    9720          135 :               SET_DECL_VALUE_EXPR (placeholder, ref);
    9721          135 :               DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
    9722          135 :               tree d = maybe_lookup_decl (decl, ctx);
    9723          135 :               gcc_assert (d);
    9724          135 :               if (DECL_HAS_VALUE_EXPR_P (d))
    9725            7 :                 oldv = DECL_VALUE_EXPR (d);
    9726          135 :               if (omp_privatize_by_reference (var))
    9727              :                 {
    9728           14 :                   tree v = fold_convert (TREE_TYPE (d),
    9729              :                                          build_fold_addr_expr (new_var));
    9730           14 :                   SET_DECL_VALUE_EXPR (d, v);
    9731              :                 }
    9732              :               else
    9733          121 :                 SET_DECL_VALUE_EXPR (d, new_var);
    9734          135 :               DECL_HAS_VALUE_EXPR_P (d) = 1;
    9735          135 :               lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
    9736          135 :               if (oldv)
    9737            7 :                 SET_DECL_VALUE_EXPR (d, oldv);
    9738              :               else
    9739              :                 {
    9740          128 :                   SET_DECL_VALUE_EXPR (d, NULL_TREE);
    9741          128 :                   DECL_HAS_VALUE_EXPR_P (d) = 0;
    9742              :                 }
    9743          135 :               gimple_seq_add_seq (end, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
    9744          135 :               OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
    9745          135 :               if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
    9746           24 :                 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
    9747          135 :               if (cancellable)
    9748            4 :                 gimple_seq_add_stmt (end, gimple_build_label (lab6));
    9749          135 :               tree x = lang_hooks.decls.omp_clause_dtor (c, new_var);
    9750          135 :               if (x)
    9751              :                 {
    9752           37 :                   gimple_seq tseq = NULL;
    9753           37 :                   gimplify_stmt (&x, &tseq);
    9754           37 :                   gimple_seq_add_seq (end, tseq);
    9755              :                 }
    9756              :             }
    9757              :           else
    9758              :             {
    9759         1043 :               tree x = build2 (rcode, TREE_TYPE (ref), ref, new_var);
    9760         1043 :               ref = unshare_expr (ref);
    9761         1043 :               gimplify_assign (ref, x, end);
    9762              :             }
    9763         1663 :           gimple_seq_add_stmt (end, gimple_build_label (lab4));
    9764         1663 :           ++cnt;
    9765         1663 :           field = DECL_CHAIN (bfield);
    9766              :         }
    9767              :     }
    9768              : 
    9769         1176 :   if (code == OMP_TASKGROUP)
    9770              :     {
    9771          327 :       t = builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_REDUCTION_REGISTER);
    9772          327 :       g = gimple_build_call (t, 1, build_fold_addr_expr (avar));
    9773          327 :       gimple_seq_add_stmt (start, g);
    9774              :     }
    9775              :   else
    9776              :     {
    9777          849 :       tree c;
    9778          849 :       if (code == OMP_FOR)
    9779          228 :         c = gimple_omp_for_clauses (ctx->stmt);
    9780          621 :       else if (code == OMP_SECTIONS)
    9781            8 :         c = gimple_omp_sections_clauses (ctx->stmt);
    9782          613 :       else if (code == OMP_SCOPE)
    9783           54 :         c = gimple_omp_scope_clauses (ctx->stmt);
    9784              :       else
    9785          559 :         c = gimple_omp_taskreg_clauses (ctx->stmt);
    9786          849 :       c = omp_find_clause (c, OMP_CLAUSE__REDUCTEMP_);
    9787          849 :       t = fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (c)),
    9788              :                         build_fold_addr_expr (avar));
    9789          849 :       gimplify_assign (OMP_CLAUSE_DECL (c), t, start);
    9790              :     }
    9791              : 
    9792         1176 :   gimple_seq_add_stmt (end, gimple_build_assign (data, PLUS_EXPR, data, sz));
    9793         1176 :   gimple_seq_add_stmt (end, gimple_build_assign (idx, PLUS_EXPR, idx,
    9794              :                                                  size_one_node));
    9795         1176 :   g = gimple_build_cond (NE_EXPR, idx, num_thr_sz, lab1, lab2);
    9796         1176 :   gimple_seq_add_stmt (end, g);
    9797         1176 :   gimple_seq_add_stmt (end, gimple_build_label (lab2));
    9798         1176 :   if (code == OMP_FOR || code == OMP_SECTIONS || code == OMP_SCOPE)
    9799              :     {
    9800          290 :       enum built_in_function bfn
    9801              :         = BUILT_IN_GOMP_WORKSHARE_TASK_REDUCTION_UNREGISTER;
    9802          290 :       t = builtin_decl_explicit (bfn);
    9803          290 :       tree c_bool_type = TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (t)));
    9804          290 :       tree arg;
    9805          290 :       if (cancellable)
    9806              :         {
    9807           14 :           arg = create_tmp_var (c_bool_type);
    9808           14 :           gimple_seq_add_stmt (end, gimple_build_assign (arg, NOP_EXPR,
    9809              :                                                          cancellable));
    9810              :         }
    9811              :       else
    9812          276 :         arg = build_int_cst (c_bool_type, 0);
    9813          290 :       g = gimple_build_call (t, 1, arg);
    9814          290 :     }
    9815              :   else
    9816              :     {
    9817          886 :       t = builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_REDUCTION_UNREGISTER);
    9818          886 :       g = gimple_build_call (t, 1, build_fold_addr_expr (avar));
    9819              :     }
    9820         1176 :   gimple_seq_add_stmt (end, g);
    9821         1176 :   if (lab7)
    9822          497 :     gimple_seq_add_stmt (end, gimple_build_label (lab7));
    9823         1176 :   t = build_constructor (atype, NULL);
    9824         1176 :   TREE_THIS_VOLATILE (t) = 1;
    9825         1176 :   gimple_seq_add_stmt (end, gimple_build_assign (avar, t));
    9826              : }
    9827              : 
    9828              : /* Expand code for an OpenMP taskgroup directive.  */
    9829              : 
    9830              : static void
    9831          536 : lower_omp_taskgroup (gimple_stmt_iterator *gsi_p, omp_context *ctx)
    9832              : {
    9833          536 :   gimple *stmt = gsi_stmt (*gsi_p);
    9834          536 :   gcall *x;
    9835          536 :   gbind *bind;
    9836          536 :   gimple_seq dseq = NULL;
    9837          536 :   tree block = make_node (BLOCK);
    9838              : 
    9839          536 :   bind = gimple_build_bind (NULL, NULL, block);
    9840          536 :   gsi_replace (gsi_p, bind, true);
    9841          536 :   gimple_bind_add_stmt (bind, stmt);
    9842              : 
    9843          536 :   push_gimplify_context ();
    9844              : 
    9845          536 :   x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START),
    9846              :                          0);
    9847          536 :   gimple_bind_add_stmt (bind, x);
    9848              : 
    9849          536 :   lower_omp_task_reductions (ctx, OMP_TASKGROUP,
    9850              :                              gimple_omp_taskgroup_clauses (stmt),
    9851              :                              gimple_bind_body_ptr (bind), &dseq);
    9852              : 
    9853          536 :   lower_omp (gimple_omp_body_ptr (stmt), ctx);
    9854          536 :   gimple_bind_add_seq (bind, gimple_omp_body (stmt));
    9855          536 :   gimple_omp_set_body (stmt, NULL);
    9856              : 
    9857          536 :   gimple_bind_add_seq (bind, dseq);
    9858              : 
    9859          536 :   pop_gimplify_context (bind);
    9860              : 
    9861          536 :   gimple_bind_append_vars (bind, ctx->block_vars);
    9862          536 :   BLOCK_VARS (block) = ctx->block_vars;
    9863          536 : }
    9864              : 
    9865              : 
    9866              : /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible.  */
    9867              : 
    9868              : static void
    9869            0 : lower_omp_ordered_clauses (gimple_stmt_iterator *gsi_p, gomp_ordered *ord_stmt,
    9870              :                            omp_context *ctx)
    9871              : {
    9872            0 :   struct omp_for_data fd;
    9873            0 :   if (!ctx->outer || gimple_code (ctx->outer->stmt) != GIMPLE_OMP_FOR)
    9874            0 :     return;
    9875              : 
    9876            0 :   unsigned int len = gimple_omp_for_collapse (ctx->outer->stmt);
    9877            0 :   struct omp_for_data_loop *loops = XALLOCAVEC (struct omp_for_data_loop, len);
    9878            0 :   omp_extract_for_data (as_a <gomp_for *> (ctx->outer->stmt), &fd, loops);
    9879            0 :   if (!fd.ordered)
    9880              :     return;
    9881              : 
    9882            0 :   tree *list_p = gimple_omp_ordered_clauses_ptr (ord_stmt);
    9883            0 :   tree c = gimple_omp_ordered_clauses (ord_stmt);
    9884            0 :   if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS
    9885            0 :       && OMP_CLAUSE_DOACROSS_KIND (c) == OMP_CLAUSE_DOACROSS_SINK)
    9886              :     {
    9887              :       /* Merge depend clauses from multiple adjacent
    9888              :          #pragma omp ordered depend(sink:...) constructs
    9889              :          into one #pragma omp ordered depend(sink:...), so that
    9890              :          we can optimize them together.  */
    9891            0 :       gimple_stmt_iterator gsi = *gsi_p;
    9892            0 :       gsi_next (&gsi);
    9893            0 :       while (!gsi_end_p (gsi))
    9894              :         {
    9895            0 :           gimple *stmt = gsi_stmt (gsi);
    9896            0 :           if (is_gimple_debug (stmt)
    9897            0 :               || gimple_code (stmt) == GIMPLE_NOP)
    9898              :             {
    9899            0 :               gsi_next (&gsi);
    9900            0 :               continue;
    9901              :             }
    9902            0 :           if (gimple_code (stmt) != GIMPLE_OMP_ORDERED)
    9903              :             break;
    9904            0 :           gomp_ordered *ord_stmt2 = as_a <gomp_ordered *> (stmt);
    9905            0 :           c = gimple_omp_ordered_clauses (ord_stmt2);
    9906            0 :           if (c == NULL_TREE
    9907            0 :               || OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DOACROSS
    9908            0 :               || OMP_CLAUSE_DOACROSS_KIND (c) != OMP_CLAUSE_DOACROSS_SINK)
    9909              :             break;
    9910            0 :           while (*list_p)
    9911            0 :             list_p = &OMP_CLAUSE_CHAIN (*list_p);
    9912            0 :           *list_p = c;
    9913            0 :           gsi_remove (&gsi, true);
    9914              :         }
    9915              :     }
    9916              : 
    9917              :   /* Canonicalize sink dependence clauses into one folded clause if
    9918              :      possible.
    9919              : 
    9920              :      The basic algorithm is to create a sink vector whose first
    9921              :      element is the GCD of all the first elements, and whose remaining
    9922              :      elements are the minimum of the subsequent columns.
    9923              : 
    9924              :      We ignore dependence vectors whose first element is zero because
    9925              :      such dependencies are known to be executed by the same thread.
    9926              : 
    9927              :      We take into account the direction of the loop, so a minimum
    9928              :      becomes a maximum if the loop is iterating forwards.  We also
    9929              :      ignore sink clauses where the loop direction is unknown, or where
    9930              :      the offsets are clearly invalid because they are not a multiple
    9931              :      of the loop increment.
    9932              : 
    9933              :      For example:
    9934              : 
    9935              :         #pragma omp for ordered(2)
    9936              :         for (i=0; i < N; ++i)
    9937              :           for (j=0; j < M; ++j)
    9938              :             {
    9939              :               #pragma omp ordered \
    9940              :                 depend(sink:i-8,j-2) \
    9941              :                 depend(sink:i,j-1) \    // Completely ignored because i+0.
    9942              :                 depend(sink:i-4,j-3) \
    9943              :                 depend(sink:i-6,j-4)
    9944              :               #pragma omp ordered depend(source)
    9945              :             }
    9946              : 
    9947              :      Folded clause is:
    9948              : 
    9949              :         depend(sink:-gcd(8,4,6),-min(2,3,4))
    9950              :           -or-
    9951              :         depend(sink:-2,-2)
    9952              :   */
    9953              : 
    9954              :   /* FIXME: Computing GCD's where the first element is zero is
    9955              :      non-trivial in the presence of collapsed loops.  Do this later.  */
    9956            0 :   if (fd.collapse > 1)
    9957              :     return;
    9958              : 
    9959            0 :   wide_int *folded_deps = XALLOCAVEC (wide_int, 2 * len - 1);
    9960              : 
    9961              :   /* wide_int is not a POD so it must be default-constructed.  */
    9962            0 :   for (unsigned i = 0; i != 2 * len - 1; ++i)
    9963            0 :     new (static_cast<void*>(folded_deps + i)) wide_int ();
    9964              : 
    9965              :   tree folded_dep = NULL_TREE;
    9966              :   /* TRUE if the first dimension's offset is negative.  */
    9967              :   bool neg_offset_p = false;
    9968              : 
    9969              :   list_p = gimple_omp_ordered_clauses_ptr (ord_stmt);
    9970              :   unsigned int i;
    9971            0 :   while ((c = *list_p) != NULL)
    9972              :     {
    9973            0 :       bool remove = false;
    9974              : 
    9975            0 :       gcc_assert (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS);
    9976            0 :       if (OMP_CLAUSE_DOACROSS_KIND (c) != OMP_CLAUSE_DOACROSS_SINK)
    9977            0 :         goto next_ordered_clause;
    9978              : 
    9979            0 :       tree vec;
    9980            0 :       for (vec = OMP_CLAUSE_DECL (c), i = 0;
    9981            0 :            vec && TREE_CODE (vec) == TREE_LIST;
    9982            0 :            vec = TREE_CHAIN (vec), ++i)
    9983              :         {
    9984            0 :           gcc_assert (i < len);
    9985              : 
    9986              :           /* omp_extract_for_data has canonicalized the condition.  */
    9987            0 :           gcc_assert (fd.loops[i].cond_code == LT_EXPR
    9988              :                       || fd.loops[i].cond_code == GT_EXPR);
    9989            0 :           bool forward = fd.loops[i].cond_code == LT_EXPR;
    9990            0 :           bool maybe_lexically_later = true;
    9991              : 
    9992              :           /* While the committee makes up its mind, bail if we have any
    9993              :              non-constant steps.  */
    9994            0 :           if (TREE_CODE (fd.loops[i].step) != INTEGER_CST)
    9995            0 :             goto lower_omp_ordered_ret;
    9996              : 
    9997            0 :           tree itype = TREE_TYPE (TREE_VALUE (vec));
    9998            0 :           if (POINTER_TYPE_P (itype))
    9999            0 :             itype = sizetype;
   10000            0 :           wide_int offset = wide_int::from (wi::to_wide (TREE_PURPOSE (vec)),
   10001            0 :                                             TYPE_PRECISION (itype),
   10002            0 :                                             TYPE_SIGN (itype));
   10003              : 
   10004              :           /* Ignore invalid offsets that are not multiples of the step.  */
   10005            0 :           if (!wi::multiple_of_p (wi::abs (offset),
   10006            0 :                                   wi::abs (wi::to_wide (fd.loops[i].step)),
   10007              :                                   UNSIGNED))
   10008              :             {
   10009            0 :               warning_at (OMP_CLAUSE_LOCATION (c), OPT_Wopenmp,
   10010              :                           "ignoring %<sink%> clause with offset that is not "
   10011              :                           "a multiple of the loop step");
   10012            0 :               remove = true;
   10013            0 :               goto next_ordered_clause;
   10014              :             }
   10015              : 
   10016              :           /* Calculate the first dimension.  The first dimension of
   10017              :              the folded dependency vector is the GCD of the first
   10018              :              elements, while ignoring any first elements whose offset
   10019              :              is 0.  */
   10020            0 :           if (i == 0)
   10021              :             {
   10022              :               /* Ignore dependence vectors whose first dimension is 0.  */
   10023            0 :               if (offset == 0)
   10024              :                 {
   10025            0 :                   remove = true;
   10026            0 :                   goto next_ordered_clause;
   10027              :                 }
   10028              :               else
   10029              :                 {
   10030            0 :                   if (!TYPE_UNSIGNED (itype) && (forward ^ wi::neg_p (offset)))
   10031              :                     {
   10032            0 :                       error_at (OMP_CLAUSE_LOCATION (c),
   10033              :                                 "first offset must be in opposite direction "
   10034              :                                 "of loop iterations");
   10035            0 :                       goto lower_omp_ordered_ret;
   10036              :                     }
   10037            0 :                   if (forward)
   10038            0 :                     offset = -offset;
   10039            0 :                   neg_offset_p = forward;
   10040              :                   /* Initialize the first time around.  */
   10041            0 :                   if (folded_dep == NULL_TREE)
   10042              :                     {
   10043            0 :                       folded_dep = c;
   10044            0 :                       folded_deps[0] = offset;
   10045              :                     }
   10046              :                   else
   10047            0 :                     folded_deps[0] = wi::gcd (folded_deps[0],
   10048            0 :                                               offset, UNSIGNED);
   10049              :                 }
   10050              :             }
   10051              :           /* Calculate minimum for the remaining dimensions.  */
   10052              :           else
   10053              :             {
   10054            0 :               folded_deps[len + i - 1] = offset;
   10055            0 :               if (folded_dep == c)
   10056            0 :                 folded_deps[i] = offset;
   10057            0 :               else if (maybe_lexically_later
   10058            0 :                        && !wi::eq_p (folded_deps[i], offset))
   10059              :                 {
   10060            0 :                   if (forward ^ wi::gts_p (folded_deps[i], offset))
   10061              :                     {
   10062              :                       unsigned int j;
   10063            0 :                       folded_dep = c;
   10064            0 :                       for (j = 1; j <= i; j++)
   10065            0 :                         folded_deps[j] = folded_deps[len + j - 1];
   10066              :                     }
   10067              :                   else
   10068            0 :                     maybe_lexically_later = false;
   10069              :                 }
   10070              :             }
   10071            0 :         }
   10072            0 :       gcc_assert (i == len);
   10073              : 
   10074              :       remove = true;
   10075              : 
   10076            0 :     next_ordered_clause:
   10077            0 :       if (remove)
   10078            0 :         *list_p = OMP_CLAUSE_CHAIN (c);
   10079              :       else
   10080            0 :         list_p = &OMP_CLAUSE_CHAIN (c);
   10081              :     }
   10082              : 
   10083            0 :   if (folded_dep)
   10084              :     {
   10085            0 :       if (neg_offset_p)
   10086            0 :         folded_deps[0] = -folded_deps[0];
   10087              : 
   10088            0 :       tree itype = TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep)));
   10089            0 :       if (POINTER_TYPE_P (itype))
   10090            0 :         itype = sizetype;
   10091              : 
   10092            0 :       TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep))
   10093            0 :         = wide_int_to_tree (itype, folded_deps[0]);
   10094            0 :       OMP_CLAUSE_CHAIN (folded_dep) = gimple_omp_ordered_clauses (ord_stmt);
   10095            0 :       *gimple_omp_ordered_clauses_ptr (ord_stmt) = folded_dep;
   10096              :     }
   10097              : 
   10098            0 :  lower_omp_ordered_ret:
   10099              : 
   10100              :   /* Ordered without clauses is #pragma omp threads, while we want
   10101              :      a nop instead if we remove all clauses.  */
   10102            0 :   if (gimple_omp_ordered_clauses (ord_stmt) == NULL_TREE)
   10103            0 :     gsi_replace (gsi_p, gimple_build_nop (), true);
   10104              : }
   10105              : 
   10106              : 
   10107              : /* Expand code for an OpenMP ordered directive.  */
   10108              : 
   10109              : static void
   10110         1128 : lower_omp_ordered (gimple_stmt_iterator *gsi_p, omp_context *ctx)
   10111              : {
   10112         1128 :   tree block;
   10113         1128 :   gimple *stmt = gsi_stmt (*gsi_p), *g;
   10114         1128 :   gomp_ordered *ord_stmt = as_a <gomp_ordered *> (stmt);
   10115         1128 :   gcall *x;
   10116         1128 :   gbind *bind;
   10117         1128 :   bool simd = omp_find_clause (gimple_omp_ordered_clauses (ord_stmt),
   10118         1128 :                                OMP_CLAUSE_SIMD);
   10119              :   /* FIXME: this should check presence of OMP_CLAUSE__SIMT_ on the enclosing
   10120              :      loop.  */
   10121         1128 :   bool maybe_simt
   10122         1128 :     = simd && omp_maybe_offloaded_ctx (ctx) && omp_max_simt_vf () > 1;
   10123         1128 :   bool threads = omp_find_clause (gimple_omp_ordered_clauses (ord_stmt),
   10124         1128 :                                   OMP_CLAUSE_THREADS);
   10125              : 
   10126         1128 :   if (gimple_omp_ordered_standalone_p (ord_stmt))
   10127              :     {
   10128              :       /* FIXME: This is needs to be moved to the expansion to verify various
   10129              :          conditions only testable on cfg with dominators computed, and also
   10130              :          all the depend clauses to be merged still might need to be available
   10131              :          for the runtime checks.  */
   10132              :       if (0)
   10133              :         lower_omp_ordered_clauses (gsi_p, ord_stmt, ctx);
   10134         1128 :       return;
   10135              :     }
   10136              : 
   10137          415 :   push_gimplify_context ();
   10138              : 
   10139          415 :   block = make_node (BLOCK);
   10140          415 :   bind = gimple_build_bind (NULL, NULL, block);
   10141          415 :   gsi_replace (gsi_p, bind, true);
   10142          415 :   gimple_bind_add_stmt (bind, stmt);
   10143              : 
   10144          415 :   if (simd)
   10145              :     {
   10146           81 :       x = gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START, 1,
   10147           81 :                                       build_int_cst (integer_type_node, threads));
   10148           81 :       cfun->has_simduid_loops = true;
   10149              :     }
   10150              :   else
   10151          334 :     x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START),
   10152              :                            0);
   10153          415 :   gimple_bind_add_stmt (bind, x);
   10154              : 
   10155          415 :   tree counter = NULL_TREE, test = NULL_TREE, body = NULL_TREE;
   10156          415 :   if (maybe_simt)
   10157              :     {
   10158            0 :       counter = create_tmp_var (integer_type_node);
   10159            0 :       g = gimple_build_call_internal (IFN_GOMP_SIMT_LANE, 0);
   10160            0 :       gimple_call_set_lhs (g, counter);
   10161            0 :       gimple_bind_add_stmt (bind, g);
   10162              : 
   10163            0 :       body = create_artificial_label (UNKNOWN_LOCATION);
   10164            0 :       test = create_artificial_label (UNKNOWN_LOCATION);
   10165            0 :       gimple_bind_add_stmt (bind, gimple_build_label (body));
   10166              : 
   10167            0 :       tree simt_pred = create_tmp_var (integer_type_node);
   10168            0 :       g = gimple_build_call_internal (IFN_GOMP_SIMT_ORDERED_PRED, 1, counter);
   10169            0 :       gimple_call_set_lhs (g, simt_pred);
   10170            0 :       gimple_bind_add_stmt (bind, g);
   10171              : 
   10172            0 :       tree t = create_artificial_label (UNKNOWN_LOCATION);
   10173            0 :       g = gimple_build_cond (EQ_EXPR, simt_pred, integer_zero_node, t, test);
   10174            0 :       gimple_bind_add_stmt (bind, g);
   10175              : 
   10176            0 :       gimple_bind_add_stmt (bind, gimple_build_label (t));
   10177              :     }
   10178          415 :   lower_omp (gimple_omp_body_ptr (stmt), ctx);
   10179          415 :   gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
   10180          415 :   gimple_bind_add_seq (bind, gimple_omp_body (stmt));
   10181          415 :   gimple_omp_set_body (stmt, NULL);
   10182              : 
   10183          415 :   if (maybe_simt)
   10184              :     {
   10185            0 :       gimple_bind_add_stmt (bind, gimple_build_label (test));
   10186            0 :       g = gimple_build_assign (counter, MINUS_EXPR, counter, integer_one_node);
   10187            0 :       gimple_bind_add_stmt (bind, g);
   10188              : 
   10189            0 :       tree c = build2 (GE_EXPR, boolean_type_node, counter, integer_zero_node);
   10190            0 :       tree nonneg = create_tmp_var (integer_type_node);
   10191            0 :       gimple_seq tseq = NULL;
   10192            0 :       gimplify_assign (nonneg, fold_convert (integer_type_node, c), &tseq);
   10193            0 :       gimple_bind_add_seq (bind, tseq);
   10194              : 
   10195            0 :       g = gimple_build_call_internal (IFN_GOMP_SIMT_VOTE_ANY, 1, nonneg);
   10196            0 :       gimple_call_set_lhs (g, nonneg);
   10197            0 :       gimple_bind_add_stmt (bind, g);
   10198              : 
   10199            0 :       tree end = create_artificial_label (UNKNOWN_LOCATION);
   10200            0 :       g = gimple_build_cond (NE_EXPR, nonneg, integer_zero_node, body, end);
   10201            0 :       gimple_bind_add_stmt (bind, g);
   10202              : 
   10203            0 :       gimple_bind_add_stmt (bind, gimple_build_label (end));
   10204              :     }
   10205          415 :   if (simd)
   10206           81 :     x = gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END, 1,
   10207           81 :                                     build_int_cst (integer_type_node, threads));
   10208              :   else
   10209          334 :     x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END),
   10210              :                            0);
   10211          415 :   gimple_bind_add_stmt (bind, x);
   10212              : 
   10213          415 :   gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
   10214              : 
   10215          415 :   pop_gimplify_context (bind);
   10216              : 
   10217          415 :   gimple_bind_append_vars (bind, ctx->block_vars);
   10218          415 :   BLOCK_VARS (block) = gimple_bind_vars (bind);
   10219              : }
   10220              : 
   10221              : 
   10222              : /* Expand code for an OpenMP scan directive and the structured block
   10223              :    before the scan directive.  */
   10224              : 
   10225              : static void
   10226         1592 : lower_omp_scan (gimple_stmt_iterator *gsi_p, omp_context *ctx)
   10227              : {
   10228         1592 :   gimple *stmt = gsi_stmt (*gsi_p);
   10229         1592 :   bool has_clauses
   10230         1592 :     = gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt)) != NULL;
   10231         1592 :   tree lane = NULL_TREE;
   10232         1592 :   gimple_seq before = NULL;
   10233         1592 :   omp_context *octx = ctx->outer;
   10234         1592 :   gcc_assert (octx);
   10235         1592 :   if (octx->scan_exclusive && !has_clauses)
   10236              :     {
   10237          536 :       gimple_stmt_iterator gsi2 = *gsi_p;
   10238          536 :       gsi_next (&gsi2);
   10239          536 :       gimple *stmt2 = gsi_stmt (gsi2);
   10240              :       /* For exclusive scan, swap GIMPLE_OMP_SCAN without clauses
   10241              :          with following GIMPLE_OMP_SCAN with clauses, so that input_phase,
   10242              :          the one with exclusive clause(s), comes first.  */
   10243          536 :       if (stmt2
   10244          274 :           && gimple_code (stmt2) == GIMPLE_OMP_SCAN
   10245          804 :           && gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt2)) != NULL)
   10246              :         {
   10247          268 :           gsi_remove (gsi_p, false);
   10248          268 :           gsi_insert_after (gsi_p, stmt, GSI_SAME_STMT);
   10249          268 :           ctx = maybe_lookup_ctx (stmt2);
   10250          268 :           gcc_assert (ctx);
   10251          268 :           lower_omp_scan (gsi_p, ctx);
   10252          268 :           return;
   10253              :         }
   10254              :     }
   10255              : 
   10256         1324 :   bool input_phase = has_clauses ^ octx->scan_inclusive;
   10257         1324 :   bool is_simd = (gimple_code (octx->stmt) == GIMPLE_OMP_FOR
   10258         1324 :                   && gimple_omp_for_kind (octx->stmt) == GF_OMP_FOR_KIND_SIMD);
   10259         1324 :   bool is_for = (gimple_code (octx->stmt) == GIMPLE_OMP_FOR
   10260         1324 :                  && gimple_omp_for_kind (octx->stmt) == GF_OMP_FOR_KIND_FOR
   10261         1694 :                  && !gimple_omp_for_combined_p (octx->stmt));
   10262         1324 :   bool is_for_simd = is_simd && gimple_omp_for_combined_into_p (octx->stmt);
   10263          364 :   if (is_for_simd && octx->for_simd_scan_phase)
   10264              :     is_simd = false;
   10265         1142 :   if (is_simd)
   10266          772 :     if (tree c = omp_find_clause (gimple_omp_for_clauses (octx->stmt),
   10267              :                                   OMP_CLAUSE__SIMDUID_))
   10268              :       {
   10269          368 :         tree uid = OMP_CLAUSE__SIMDUID__DECL (c);
   10270          368 :         lane = create_tmp_var (unsigned_type_node);
   10271          368 :         tree t = build_int_cst (integer_type_node,
   10272          552 :                                 input_phase ? 1
   10273          184 :                                 : octx->scan_inclusive ? 2 : 3);
   10274          368 :         gimple *g
   10275          368 :           = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 2, uid, t);
   10276          368 :         gimple_call_set_lhs (g, lane);
   10277          368 :         gimple_seq_add_stmt (&before, g);
   10278              :       }
   10279              : 
   10280         1324 :   if (is_simd || is_for)
   10281              :     {
   10282         5100 :       for (tree c = gimple_omp_for_clauses (octx->stmt);
   10283         5100 :            c; c = OMP_CLAUSE_CHAIN (c))
   10284         4140 :         if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
   10285         4140 :             && OMP_CLAUSE_REDUCTION_INSCAN (c))
   10286              :           {
   10287         1184 :             location_t clause_loc = OMP_CLAUSE_LOCATION (c);
   10288         1184 :             tree var = OMP_CLAUSE_DECL (c);
   10289         1184 :             tree new_var = lookup_decl (var, octx);
   10290         1184 :             tree val = new_var;
   10291         1184 :             tree var2 = NULL_TREE;
   10292         1184 :             tree var3 = NULL_TREE;
   10293         1184 :             tree var4 = NULL_TREE;
   10294         1184 :             tree lane0 = NULL_TREE;
   10295         1184 :             tree new_vard = new_var;
   10296         1184 :             if (omp_privatize_by_reference (var))
   10297              :               {
   10298          184 :                 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
   10299          184 :                 val = new_var;
   10300              :               }
   10301         1184 :             if (DECL_HAS_VALUE_EXPR_P (new_vard))
   10302              :               {
   10303          464 :                 val = DECL_VALUE_EXPR (new_vard);
   10304          464 :                 if (new_vard != new_var)
   10305              :                   {
   10306           76 :                     gcc_assert (TREE_CODE (val) == ADDR_EXPR);
   10307           76 :                     val = TREE_OPERAND (val, 0);
   10308              :                   }
   10309          464 :                 if (TREE_CODE (val) == ARRAY_REF
   10310          464 :                     && VAR_P (TREE_OPERAND (val, 0)))
   10311              :                   {
   10312          464 :                     tree v = TREE_OPERAND (val, 0);
   10313          464 :                     if (lookup_attribute ("omp simd array",
   10314          464 :                                           DECL_ATTRIBUTES (v)))
   10315              :                       {
   10316          464 :                         val = unshare_expr (val);
   10317          464 :                         lane0 = TREE_OPERAND (val, 1);
   10318          464 :                         TREE_OPERAND (val, 1) = lane;
   10319          464 :                         var2 = lookup_decl (v, octx);
   10320          464 :                         if (octx->scan_exclusive)
   10321          228 :                           var4 = lookup_decl (var2, octx);
   10322          464 :                         if (input_phase
   10323          464 :                             && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
   10324          120 :                           var3 = maybe_lookup_decl (var4 ? var4 : var2, octx);
   10325          464 :                         if (!input_phase)
   10326              :                           {
   10327          232 :                             var2 = build4 (ARRAY_REF, TREE_TYPE (val),
   10328              :                                            var2, lane, NULL_TREE, NULL_TREE);
   10329          232 :                             TREE_THIS_NOTRAP (var2) = 1;
   10330          232 :                             if (octx->scan_exclusive)
   10331              :                               {
   10332          114 :                                 var4 = build4 (ARRAY_REF, TREE_TYPE (val),
   10333              :                                                var4, lane, NULL_TREE,
   10334              :                                                NULL_TREE);
   10335          114 :                                 TREE_THIS_NOTRAP (var4) = 1;
   10336              :                               }
   10337              :                           }
   10338              :                         else
   10339              :                           var2 = val;
   10340              :                       }
   10341              :                   }
   10342          464 :                 gcc_assert (var2);
   10343              :               }
   10344              :             else
   10345              :               {
   10346          720 :                 var2 = build_outer_var_ref (var, octx);
   10347          720 :                 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
   10348              :                   {
   10349          220 :                     var3 = maybe_lookup_decl (new_vard, octx);
   10350          220 :                     if (var3 == new_vard || var3 == NULL_TREE)
   10351              :                       var3 = NULL_TREE;
   10352          108 :                     else if (is_simd && octx->scan_exclusive && !input_phase)
   10353              :                       {
   10354           16 :                         var4 = maybe_lookup_decl (var3, octx);
   10355           16 :                         if (var4 == var3 || var4 == NULL_TREE)
   10356              :                           {
   10357            0 :                             if (TREE_ADDRESSABLE (TREE_TYPE (new_var)))
   10358              :                               {
   10359              :                                 var4 = var3;
   10360              :                                 var3 = NULL_TREE;
   10361              :                               }
   10362              :                             else
   10363           16 :                               var4 = NULL_TREE;
   10364              :                           }
   10365              :                       }
   10366              :                   }
   10367          688 :                 if (is_simd
   10368          500 :                     && octx->scan_exclusive
   10369          244 :                     && !input_phase
   10370          244 :                     && var4 == NULL_TREE)
   10371          106 :                   var4 = create_tmp_var (TREE_TYPE (val));
   10372              :               }
   10373         1184 :             if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
   10374              :               {
   10375          376 :                 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
   10376          376 :                 if (input_phase)
   10377              :                   {
   10378          188 :                     if (var3)
   10379              :                       {
   10380              :                         /* If we've added a separate identity element
   10381              :                            variable, copy it over into val.  */
   10382           92 :                         tree x = lang_hooks.decls.omp_clause_assign_op (c, val,
   10383              :                                                                         var3);
   10384           92 :                         gimplify_and_add (x, &before);
   10385              :                       }
   10386           96 :                     else if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
   10387              :                       {
   10388              :                         /* Otherwise, assign to it the identity element.  */
   10389           96 :                         gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
   10390           96 :                         if (is_for)
   10391           16 :                           tseq = copy_gimple_seq_and_replace_locals (tseq);
   10392           96 :                         tree ref = build_outer_var_ref (var, octx);
   10393           96 :                         tree x = (DECL_HAS_VALUE_EXPR_P (new_vard)
   10394           96 :                                   ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
   10395           40 :                         if (x)
   10396              :                           {
   10397           40 :                             if (new_vard != new_var)
   10398            8 :                               val = build_fold_addr_expr_loc (clause_loc, val);
   10399           40 :                             SET_DECL_VALUE_EXPR (new_vard, val);
   10400              :                           }
   10401           96 :                         SET_DECL_VALUE_EXPR (placeholder, ref);
   10402           96 :                         DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
   10403           96 :                         lower_omp (&tseq, octx);
   10404           96 :                         if (x)
   10405           40 :                           SET_DECL_VALUE_EXPR (new_vard, x);
   10406           96 :                         SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
   10407           96 :                         DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
   10408           96 :                         gimple_seq_add_seq (&before, tseq);
   10409           96 :                         if (is_simd)
   10410           80 :                           OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
   10411              :                       }
   10412              :                   }
   10413          188 :                 else if (is_simd)
   10414              :                   {
   10415          156 :                     tree x;
   10416          156 :                     if (octx->scan_exclusive)
   10417              :                       {
   10418           72 :                         tree v4 = unshare_expr (var4);
   10419           72 :                         tree v2 = unshare_expr (var2);
   10420           72 :                         x = lang_hooks.decls.omp_clause_assign_op (c, v4, v2);
   10421           72 :                         gimplify_and_add (x, &before);
   10422              :                       }
   10423          156 :                     gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
   10424          156 :                     x = (DECL_HAS_VALUE_EXPR_P (new_vard)
   10425          156 :                          ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
   10426          156 :                     tree vexpr = val;
   10427          156 :                     if (x && new_vard != new_var)
   10428           30 :                       vexpr = build_fold_addr_expr_loc (clause_loc, val);
   10429          156 :                     if (x)
   10430           78 :                       SET_DECL_VALUE_EXPR (new_vard, vexpr);
   10431          156 :                     SET_DECL_VALUE_EXPR (placeholder, var2);
   10432          156 :                     DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
   10433          156 :                     lower_omp (&tseq, octx);
   10434          156 :                     gimple_seq_add_seq (&before, tseq);
   10435          156 :                     OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
   10436          156 :                     if (x)
   10437           78 :                       SET_DECL_VALUE_EXPR (new_vard, x);
   10438          156 :                     SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
   10439          156 :                     DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
   10440          156 :                     if (octx->scan_inclusive)
   10441              :                       {
   10442           84 :                         x = lang_hooks.decls.omp_clause_assign_op (c, val,
   10443              :                                                                    var2);
   10444           84 :                         gimplify_and_add (x, &before);
   10445              :                       }
   10446           72 :                     else if (lane0 == NULL_TREE)
   10447              :                       {
   10448           36 :                         x = lang_hooks.decls.omp_clause_assign_op (c, val,
   10449              :                                                                    var4);
   10450           36 :                         gimplify_and_add (x, &before);
   10451              :                       }
   10452              :                   }
   10453              :               }
   10454              :             else
   10455              :               {
   10456          808 :                 if (input_phase)
   10457              :                   {
   10458              :                     /* input phase.  Set val to initializer before
   10459              :                        the body.  */
   10460          404 :                     tree x = omp_reduction_init (c, TREE_TYPE (new_var));
   10461          404 :                     gimplify_assign (val, x, &before);
   10462              :                   }
   10463          404 :                 else if (is_simd)
   10464              :                   {
   10465              :                     /* scan phase.  */
   10466          326 :                     enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
   10467          326 :                     if (code == MINUS_EXPR)
   10468            0 :                       code = PLUS_EXPR;
   10469              : 
   10470          326 :                     tree x = build2 (code, TREE_TYPE (var2),
   10471              :                                      unshare_expr (var2), unshare_expr (val));
   10472          326 :                     if (octx->scan_inclusive)
   10473              :                       {
   10474          162 :                         gimplify_assign (unshare_expr (var2), x, &before);
   10475          162 :                         gimplify_assign (val, var2, &before);
   10476              :                       }
   10477              :                     else
   10478              :                       {
   10479          164 :                         gimplify_assign (unshare_expr (var4),
   10480              :                                          unshare_expr (var2), &before);
   10481          164 :                         gimplify_assign (var2, x, &before);
   10482          164 :                         if (lane0 == NULL_TREE)
   10483           86 :                           gimplify_assign (val, var4, &before);
   10484              :                       }
   10485              :                   }
   10486              :               }
   10487         1184 :             if (octx->scan_exclusive && !input_phase && lane0)
   10488              :               {
   10489          114 :                 tree vexpr = unshare_expr (var4);
   10490          114 :                 TREE_OPERAND (vexpr, 1) = lane0;
   10491          114 :                 if (new_vard != new_var)
   10492           16 :                   vexpr = build_fold_addr_expr_loc (clause_loc, vexpr);
   10493          114 :                 SET_DECL_VALUE_EXPR (new_vard, vexpr);
   10494              :               }
   10495              :           }
   10496              :     }
   10497         1324 :   if (is_simd && !is_for_simd)
   10498              :     {
   10499          590 :       gsi_insert_seq_after (gsi_p, gimple_omp_body (stmt), GSI_SAME_STMT);
   10500          590 :       gsi_insert_seq_after (gsi_p, before, GSI_SAME_STMT);
   10501          590 :       gsi_replace (gsi_p, gimple_build_nop (), true);
   10502          590 :       return;
   10503              :     }
   10504          734 :   lower_omp (gimple_omp_body_ptr (stmt), octx);
   10505          734 :   if (before)
   10506              :     {
   10507          276 :       gimple_stmt_iterator gsi = gsi_start (*gimple_omp_body_ptr (stmt));
   10508          276 :       gsi_insert_seq_before (&gsi, before, GSI_SAME_STMT);
   10509              :     }
   10510              : }
   10511              : 
   10512              : 
   10513              : /* Gimplify a GIMPLE_OMP_CRITICAL statement.  This is a relatively simple
   10514              :    substitution of a couple of function calls.  But in the NAMED case,
   10515              :    requires that languages coordinate a symbol name.  It is therefore
   10516              :    best put here in common code.  */
   10517              : 
   10518              : static GTY(()) hash_map<tree, tree> *critical_name_mutexes;
   10519              : 
   10520              : static void
   10521          317 : lower_omp_critical (gimple_stmt_iterator *gsi_p, omp_context *ctx)
   10522              : {
   10523          317 :   tree block;
   10524          317 :   tree name, lock, unlock;
   10525          317 :   gomp_critical *stmt = as_a <gomp_critical *> (gsi_stmt (*gsi_p));
   10526          317 :   gbind *bind;
   10527          317 :   location_t loc = gimple_location (stmt);
   10528          317 :   gimple_seq tbody;
   10529              : 
   10530          317 :   name = gimple_omp_critical_name (stmt);
   10531          317 :   if (name)
   10532              :     {
   10533           92 :       tree decl;
   10534              : 
   10535           92 :       if (!critical_name_mutexes)
   10536           50 :         critical_name_mutexes = hash_map<tree, tree>::create_ggc (10);
   10537              : 
   10538           92 :       tree *n = critical_name_mutexes->get (name);
   10539           92 :       if (n == NULL)
   10540              :         {
   10541           74 :           char *new_str;
   10542              : 
   10543           74 :           decl = create_tmp_var_raw (ptr_type_node);
   10544              : 
   10545           74 :           new_str = ACONCAT ((".gomp_critical_user_",
   10546              :                               IDENTIFIER_POINTER (name), NULL));
   10547           74 :           DECL_NAME (decl) = get_identifier (new_str);
   10548           74 :           TREE_PUBLIC (decl) = 1;
   10549           74 :           TREE_STATIC (decl) = 1;
   10550           74 :           DECL_COMMON (decl) = 1;
   10551           74 :           DECL_ARTIFICIAL (decl) = 1;
   10552           74 :           DECL_IGNORED_P (decl) = 1;
   10553              : 
   10554           74 :           varpool_node::finalize_decl (decl);
   10555              : 
   10556           74 :           critical_name_mutexes->put (name, decl);
   10557              :         }
   10558              :       else
   10559           18 :         decl = *n;
   10560              : 
   10561              :       /* If '#pragma omp critical' is inside offloaded region or
   10562              :          inside function marked as offloadable, the symbol must be
   10563              :          marked as offloadable too.  */
   10564           92 :       omp_context *octx;
   10565           92 :       if (cgraph_node::get (current_function_decl)->offloadable)
   10566            1 :         varpool_node::get_create (decl)->offloadable = 1;
   10567              :       else
   10568          181 :         for (octx = ctx->outer; octx; octx = octx->outer)
   10569           94 :           if (is_gimple_omp_offloaded (octx->stmt))
   10570              :             {
   10571            4 :               varpool_node::get_create (decl)->offloadable = 1;
   10572            4 :               break;
   10573              :             }
   10574              : 
   10575           92 :       lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START);
   10576           92 :       lock = build_call_expr_loc (loc, lock, 1,
   10577              :                                   build_fold_addr_expr_loc (loc, decl));
   10578              : 
   10579           92 :       unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END);
   10580           92 :       unlock = build_call_expr_loc (loc, unlock, 1,
   10581              :                                 build_fold_addr_expr_loc (loc, decl));
   10582              :     }
   10583              :   else
   10584              :     {
   10585          225 :       lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START);
   10586          225 :       lock = build_call_expr_loc (loc, lock, 0);
   10587              : 
   10588          225 :       unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END);
   10589          225 :       unlock = build_call_expr_loc (loc, unlock, 0);
   10590              :     }
   10591              : 
   10592          317 :   push_gimplify_context ();
   10593              : 
   10594          317 :   block = make_node (BLOCK);
   10595          317 :   bind = gimple_build_bind (NULL, NULL, block);
   10596          317 :   gsi_replace (gsi_p, bind, true);
   10597          317 :   gimple_bind_add_stmt (bind, stmt);
   10598              : 
   10599          317 :   tbody = gimple_bind_body (bind);
   10600          317 :   gimplify_and_add (lock, &tbody);
   10601          317 :   gimple_bind_set_body (bind, tbody);
   10602              : 
   10603          317 :   lower_omp (gimple_omp_body_ptr (stmt), ctx);
   10604          317 :   gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
   10605          317 :   gimple_bind_add_seq (bind, gimple_omp_body (stmt));
   10606          317 :   gimple_omp_set_body (stmt, NULL);
   10607              : 
   10608          317 :   tbody = gimple_bind_body (bind);
   10609          317 :   gimplify_and_add (unlock, &tbody);
   10610          317 :   gimple_bind_set_body (bind, tbody);
   10611              : 
   10612          317 :   gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
   10613              : 
   10614          317 :   pop_gimplify_context (bind);
   10615          317 :   gimple_bind_append_vars (bind, ctx->block_vars);
   10616          317 :   BLOCK_VARS (block) = gimple_bind_vars (bind);
   10617          317 : }
   10618              : 
   10619              : /* A subroutine of lower_omp_for.  Generate code to emit the predicate
   10620              :    for a lastprivate clause.  Given a loop control predicate of (V
   10621              :    cond N2), we gate the clause on (!(V cond N2)).  The lowered form
   10622              :    is appended to *DLIST, iterator initialization is appended to
   10623              :    *BODY_P.  *CLIST is for lastprivate(conditional:) code that needs
   10624              :    to be emitted in a critical section.  */
   10625              : 
   10626              : static void
   10627        47102 : lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p,
   10628              :                            gimple_seq *dlist, gimple_seq *clist,
   10629              :                            struct omp_context *ctx)
   10630              : {
   10631        47102 :   tree clauses, cond, vinit;
   10632        47102 :   enum tree_code cond_code;
   10633        47102 :   gimple_seq stmts;
   10634              : 
   10635        47102 :   cond_code = fd->loop.cond_code;
   10636        47102 :   cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR;
   10637              : 
   10638              :   /* When possible, use a strict equality expression.  This can let VRP
   10639              :      type optimizations deduce the value and remove a copy.  */
   10640        47102 :   if (tree_fits_shwi_p (fd->loop.step))
   10641              :     {
   10642        44863 :       HOST_WIDE_INT step = tree_to_shwi (fd->loop.step);
   10643        44863 :       if (step == 1 || step == -1)
   10644        47102 :         cond_code = EQ_EXPR;
   10645              :     }
   10646              : 
   10647        47102 :   tree n2 = fd->loop.n2;
   10648        47102 :   if (fd->collapse > 1
   10649        11251 :       && TREE_CODE (n2) != INTEGER_CST
   10650        53316 :       && gimple_omp_for_combined_into_p (fd->for_stmt))
   10651              :     {
   10652         2692 :       struct omp_context *taskreg_ctx = NULL;
   10653         2692 :       if (gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR)
   10654              :         {
   10655         1226 :           gomp_for *gfor = as_a <gomp_for *> (ctx->outer->stmt);
   10656         1226 :           if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_FOR
   10657         1226 :               || gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_DISTRIBUTE)
   10658              :             {
   10659         1194 :               if (gimple_omp_for_combined_into_p (gfor))
   10660              :                 {
   10661          672 :                   gcc_assert (ctx->outer->outer
   10662              :                               && is_parallel_ctx (ctx->outer->outer));
   10663              :                   taskreg_ctx = ctx->outer->outer;
   10664              :                 }
   10665              :               else
   10666              :                 {
   10667          522 :                   struct omp_for_data outer_fd;
   10668          522 :                   omp_extract_for_data (gfor, &outer_fd, NULL);
   10669          522 :                   n2 = fold_convert (TREE_TYPE (n2), outer_fd.loop.n2);
   10670              :                 }
   10671              :             }
   10672           32 :           else if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_TASKLOOP)
   10673           32 :             taskreg_ctx = ctx->outer->outer;
   10674              :         }
   10675         1466 :       else if (is_taskreg_ctx (ctx->outer))
   10676              :         taskreg_ctx = ctx->outer;
   10677         2692 :       if (taskreg_ctx)
   10678              :         {
   10679         2170 :           int i;
   10680         2170 :           tree taskreg_clauses
   10681         2170 :             = gimple_omp_taskreg_clauses (taskreg_ctx->stmt);
   10682         2170 :           tree innerc = omp_find_clause (taskreg_clauses,
   10683              :                                          OMP_CLAUSE__LOOPTEMP_);
   10684         2170 :           gcc_assert (innerc);
   10685         2170 :           int count = fd->collapse;
   10686         2170 :           if (fd->non_rect
   10687           24 :               && fd->last_nonrect == fd->first_nonrect + 1)
   10688           12 :             if (tree v = gimple_omp_for_index (fd->for_stmt, fd->last_nonrect))
   10689           12 :               if (!TYPE_UNSIGNED (TREE_TYPE (v)))
   10690           12 :                 count += 4;
   10691         8582 :           for (i = 0; i < count; i++)
   10692              :             {
   10693         6412 :               innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
   10694              :                                         OMP_CLAUSE__LOOPTEMP_);
   10695         6412 :               gcc_assert (innerc);
   10696              :             }
   10697         2170 :           innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
   10698              :                                     OMP_CLAUSE__LOOPTEMP_);
   10699         2170 :           if (innerc)
   10700         1419 :             n2 = fold_convert (TREE_TYPE (n2),
   10701              :                                lookup_decl (OMP_CLAUSE_DECL (innerc),
   10702              :                                             taskreg_ctx));
   10703              :         }
   10704              :     }
   10705        47102 :   cond = build2 (cond_code, boolean_type_node, fd->loop.v, n2);
   10706              : 
   10707        47102 :   clauses = gimple_omp_for_clauses (fd->for_stmt);
   10708        47102 :   stmts = NULL;
   10709        47102 :   lower_lastprivate_clauses (clauses, cond, body_p, &stmts, clist, ctx);
   10710        47102 :   if (!gimple_seq_empty_p (stmts))
   10711              :     {
   10712        16376 :       gimple_seq_add_seq (&stmts, *dlist);
   10713        16376 :       *dlist = stmts;
   10714              : 
   10715              :       /* Optimize: v = 0; is usually cheaper than v = some_other_constant.  */
   10716        16376 :       vinit = fd->loop.n1;
   10717        16376 :       if (cond_code == EQ_EXPR
   10718        14389 :           && tree_fits_shwi_p (fd->loop.n2)
   10719        25951 :           && ! integer_zerop (fd->loop.n2))
   10720         8359 :         vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0);
   10721              :       else
   10722         8017 :         vinit = unshare_expr (vinit);
   10723              : 
   10724              :       /* Initialize the iterator variable, so that threads that don't execute
   10725              :          any iterations don't execute the lastprivate clauses by accident.  */
   10726        16376 :       gimplify_assign (fd->loop.v, vinit, body_p);
   10727              :     }
   10728        47102 : }
   10729              : 
   10730              : /* OpenACC privatization.
   10731              : 
   10732              :    Or, in other words, *sharing* at the respective OpenACC level of
   10733              :    parallelism.
   10734              : 
   10735              :    From a correctness perspective, a non-addressable variable can't be accessed
   10736              :    outside the current thread, so it can go in a (faster than shared memory)
   10737              :    register -- though that register may need to be broadcast in some
   10738              :    circumstances.  A variable can only meaningfully be "shared" across workers
   10739              :    or vector lanes if its address is taken, e.g. by a call to an atomic
   10740              :    builtin.
   10741              : 
   10742              :    From an optimisation perspective, the answer might be fuzzier: maybe
   10743              :    sometimes, using shared memory directly would be faster than
   10744              :    broadcasting.  */
   10745              : 
   10746              : static void
   10747        18997 : oacc_privatization_begin_diagnose_var (const dump_flags_t l_dump_flags,
   10748              :                                        const location_t loc, const tree c,
   10749              :                                        const tree decl)
   10750              : {
   10751        18997 :   const dump_user_location_t d_u_loc
   10752        18997 :     = dump_user_location_t::from_location_t (loc);
   10753              : /* PR100695 "Format decoder, quoting in 'dump_printf' etc." */
   10754              : #if __GNUC__ >= 10
   10755        18997 : # pragma GCC diagnostic push
   10756        18997 : # pragma GCC diagnostic ignored "-Wformat"
   10757              : #endif
   10758        18997 :   dump_printf_loc (l_dump_flags, d_u_loc,
   10759              :                    "variable %<%T%> ", decl);
   10760              : #if __GNUC__ >= 10
   10761        18997 : # pragma GCC diagnostic pop
   10762              : #endif
   10763        18997 :   if (c)
   10764         2797 :     dump_printf (l_dump_flags,
   10765              :                  "in %qs clause ",
   10766         2797 :                  omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
   10767              :   else
   10768        16200 :     dump_printf (l_dump_flags,
   10769              :                  "declared in block ");
   10770        18997 : }
   10771              : 
   10772              : static bool
   10773        41210 : oacc_privatization_candidate_p (const location_t loc, const tree c,
   10774              :                                 const tree decl)
   10775              : {
   10776        41210 :   dump_flags_t l_dump_flags = get_openacc_privatization_dump_flags ();
   10777              : 
   10778              :   /* There is some differentiation depending on block vs. clause.  */
   10779        41210 :   bool block = !c;
   10780              : 
   10781        41210 :   bool res = true;
   10782              : 
   10783        41210 :   if (res && !VAR_P (decl))
   10784              :     {
   10785              :       /* A PARM_DECL (appearing in a 'private' clause) is expected to have been
   10786              :          privatized into a new VAR_DECL.  */
   10787          712 :       gcc_checking_assert (TREE_CODE (decl) != PARM_DECL);
   10788              : 
   10789          712 :       res = false;
   10790              : 
   10791          712 :       if (dump_enabled_p ())
   10792              :         {
   10793          683 :           oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
   10794          683 :           dump_printf (l_dump_flags,
   10795              :                        "potentially has improper OpenACC privatization level: %qs\n",
   10796          683 :                        get_tree_code_name (TREE_CODE (decl)));
   10797              :         }
   10798              :     }
   10799              : 
   10800        41181 :   if (res && block && TREE_STATIC (decl))
   10801              :     {
   10802           53 :       res = false;
   10803              : 
   10804           53 :       if (dump_enabled_p ())
   10805              :         {
   10806           28 :           oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
   10807           28 :           dump_printf (l_dump_flags,
   10808              :                        "isn%'t candidate for adjusting OpenACC privatization level: %s\n",
   10809              :                        "static");
   10810              :         }
   10811              :     }
   10812              : 
   10813        41185 :   if (res && block && DECL_EXTERNAL (decl))
   10814              :     {
   10815           12 :       res = false;
   10816              : 
   10817           12 :       if (dump_enabled_p ())
   10818              :         {
   10819           12 :           oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
   10820           12 :           dump_printf (l_dump_flags,
   10821              :                        "isn%'t candidate for adjusting OpenACC privatization level: %s\n",
   10822              :                        "external");
   10823              :         }
   10824              :     }
   10825              : 
   10826        41210 :   if (res && !TREE_ADDRESSABLE (decl))
   10827              :     {
   10828        39341 :       res = false;
   10829              : 
   10830        39341 :       if (dump_enabled_p ())
   10831              :         {
   10832        17456 :           oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
   10833        17456 :           dump_printf (l_dump_flags,
   10834              :                        "isn%'t candidate for adjusting OpenACC privatization level: %s\n",
   10835              :                        "not addressable");
   10836              :         }
   10837              :     }
   10838              : 
   10839              :   /* If an artificial variable has been added to a bind, e.g.
   10840              :      a compiler-generated temporary structure used by the Fortran front-end, do
   10841              :      not consider it as a privatization candidate.  Note that variables on
   10842              :      the stack are private per-thread by default: making them "gang-private"
   10843              :      for OpenACC actually means to share a single instance of a variable
   10844              :      amongst all workers and threads spawned within each gang.
   10845              :      At present, no compiler-generated artificial variables require such
   10846              :      sharing semantics, so this is safe.  */
   10847              : 
   10848        18548 :   if (res && block && DECL_ARTIFICIAL (decl))
   10849              :     {
   10850          548 :       res = false;
   10851              : 
   10852          548 :       if (dump_enabled_p ())
   10853              :         {
   10854          360 :           oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
   10855          360 :           dump_printf (l_dump_flags,
   10856              :                        "isn%'t candidate for adjusting OpenACC privatization "
   10857              :                        "level: %s\n", "artificial");
   10858              :         }
   10859              :     }
   10860              : 
   10861        41022 :   if (res)
   10862              :     {
   10863          544 :       if (dump_enabled_p ())
   10864              :         {
   10865          458 :           oacc_privatization_begin_diagnose_var (l_dump_flags, loc, c, decl);
   10866          458 :           dump_printf (l_dump_flags,
   10867              :                        "is candidate for adjusting OpenACC privatization level\n");
   10868              :         }
   10869              :     }
   10870              : 
   10871        41210 :   if (dump_file && (dump_flags & TDF_DETAILS))
   10872              :     {
   10873            0 :       print_generic_decl (dump_file, decl, dump_flags);
   10874            0 :       fprintf (dump_file, "\n");
   10875              :     }
   10876              : 
   10877        41210 :   return res;
   10878              : }
   10879              : 
   10880              : /* Scan CLAUSES for candidates for adjusting OpenACC privatization level in
   10881              :    CTX.  */
   10882              : 
   10883              : static void
   10884        11325 : oacc_privatization_scan_clause_chain (omp_context *ctx, tree clauses)
   10885              : {
   10886        35223 :   for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
   10887        23898 :     if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE)
   10888              :       {
   10889        12986 :         tree decl = OMP_CLAUSE_DECL (c);
   10890              : 
   10891        12986 :         tree new_decl = lookup_decl (decl, ctx);
   10892              : 
   10893        12986 :         if (!oacc_privatization_candidate_p (OMP_CLAUSE_LOCATION (c), c,
   10894              :                                              new_decl))
   10895        12654 :           continue;
   10896              : 
   10897          332 :         gcc_checking_assert
   10898              :           (!ctx->oacc_privatization_candidates.contains (new_decl));
   10899          332 :         ctx->oacc_privatization_candidates.safe_push (new_decl);
   10900              :       }
   10901        11325 : }
   10902              : 
   10903              : /* Scan DECLS for candidates for adjusting OpenACC privatization level in
   10904              :    CTX.  */
   10905              : 
   10906              : static void
   10907        23090 : oacc_privatization_scan_decl_chain (omp_context *ctx, tree decls)
   10908              : {
   10909        51314 :   for (tree decl = decls; decl; decl = DECL_CHAIN (decl))
   10910              :     {
   10911        28224 :       tree new_decl = lookup_decl (decl, ctx);
   10912        28224 :       gcc_checking_assert (new_decl == decl);
   10913              : 
   10914        28224 :       if (!oacc_privatization_candidate_p (gimple_location (ctx->stmt), NULL,
   10915              :                                            new_decl))
   10916        28012 :         continue;
   10917              : 
   10918          212 :       gcc_checking_assert
   10919              :         (!ctx->oacc_privatization_candidates.contains (new_decl));
   10920          212 :       ctx->oacc_privatization_candidates.safe_push (new_decl);
   10921              :     }
   10922        23090 : }
   10923              : 
   10924              : /* Callback for walk_gimple_seq.  Find #pragma omp scan statement.  */
   10925              : 
   10926              : static tree
   10927         2341 : omp_find_scan (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
   10928              :                struct walk_stmt_info *wi)
   10929              : {
   10930         2341 :   gimple *stmt = gsi_stmt (*gsi_p);
   10931              : 
   10932         2341 :   *handled_ops_p = true;
   10933         2341 :   switch (gimple_code (stmt))
   10934              :     {
   10935          462 :     WALK_SUBSTMTS;
   10936              : 
   10937          182 :     case GIMPLE_OMP_FOR:
   10938          182 :       if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_SIMD
   10939          182 :           && gimple_omp_for_combined_into_p (stmt))
   10940          182 :         *handled_ops_p = false;
   10941              :       break;
   10942              : 
   10943          734 :     case GIMPLE_OMP_SCAN:
   10944          734 :       *(gimple_stmt_iterator *) (wi->info) = *gsi_p;
   10945          734 :       return integer_zero_node;
   10946              :     default:
   10947              :       break;
   10948              :     }
   10949              :   return NULL;
   10950              : }
   10951              : 
   10952              : /* Helper function for lower_omp_for, add transformations for a worksharing
   10953              :    loop with scan directives inside of it.
   10954              :    For worksharing loop not combined with simd, transform:
   10955              :    #pragma omp for reduction(inscan,+:r) private(i)
   10956              :    for (i = 0; i < n; i = i + 1)
   10957              :      {
   10958              :        {
   10959              :          update (r);
   10960              :        }
   10961              :        #pragma omp scan inclusive(r)
   10962              :        {
   10963              :          use (r);
   10964              :        }
   10965              :      }
   10966              : 
   10967              :    into two worksharing loops + code to merge results:
   10968              : 
   10969              :    num_threads = omp_get_num_threads ();
   10970              :    thread_num = omp_get_thread_num ();
   10971              :    if (thread_num == 0) goto <D.2099>; else goto <D.2100>;
   10972              :    <D.2099>:
   10973              :    var2 = r;
   10974              :    goto <D.2101>;
   10975              :    <D.2100>:
   10976              :    // For UDRs this is UDR init, or if ctors are needed, copy from
   10977              :    // var3 that has been constructed to contain the neutral element.
   10978              :    var2 = 0;
   10979              :    <D.2101>:
   10980              :    ivar = 0;
   10981              :    // The _scantemp_ clauses will arrange for rpriva to be initialized to
   10982              :    // a shared array with num_threads elements and rprivb to a local array
   10983              :    // number of elements equal to the number of (contiguous) iterations the
   10984              :    // current thread will perform.  controlb and controlp variables are
   10985              :    // temporaries to handle deallocation of rprivb at the end of second
   10986              :    // GOMP_FOR.
   10987              :    #pragma omp for _scantemp_(rpriva) _scantemp_(rprivb) _scantemp_(controlb) \
   10988              :      _scantemp_(controlp) reduction(inscan,+:r) private(i) nowait
   10989              :    for (i = 0; i < n; i = i + 1)
   10990              :      {
   10991              :        {
   10992              :          // For UDRs this is UDR init or copy from var3.
   10993              :          r = 0;
   10994              :          // This is the input phase from user code.
   10995              :          update (r);
   10996              :        }
   10997              :        {
   10998              :          // For UDRs this is UDR merge.
   10999              :          var2 = var2 + r;
   11000              :          // Rather than handing it over to the user, save to local thread's
   11001              :          // array.
   11002              :          rprivb[ivar] = var2;
   11003              :          // For exclusive scan, the above two statements are swapped.
   11004              :          ivar = ivar + 1;
   11005              :        }
   11006              :      }
   11007              :    // And remember the final value from this thread's into the shared
   11008              :    // rpriva array.
   11009              :    rpriva[(sizetype) thread_num] = var2;
   11010              :    // If more than one thread, compute using Work-Efficient prefix sum
   11011              :    // the inclusive parallel scan of the rpriva array.
   11012              :    if (num_threads > 1) goto <D.2102>; else goto <D.2103>;
   11013              :    <D.2102>:
   11014              :    GOMP_barrier ();
   11015              :    down = 0;
   11016              :    k = 1;
   11017              :    num_threadsu = (unsigned int) num_threads;
   11018              :    thread_numup1 = (unsigned int) thread_num + 1;
   11019              :    <D.2108>:
   11020              :    twok = k << 1;
   11021              :    if (twok > num_threadsu) goto <D.2110>; else goto <D.2111>;
   11022              :    <D.2110>:
   11023              :    down = 4294967295;
   11024              :    k = k >> 1;
   11025              :    if (k == num_threadsu) goto <D.2112>; else goto <D.2111>;
   11026              :    <D.2112>:
   11027              :    k = k >> 1;
   11028              :    <D.2111>:
   11029              :    twok = k << 1;
   11030              :    cplx = .MUL_OVERFLOW (thread_nump1, twok);
   11031              :    mul = REALPART_EXPR <cplx>;
   11032              :    ovf = IMAGPART_EXPR <cplx>;
   11033              :    if (ovf == 0) goto <D.2116>; else goto <D.2117>;
   11034              :    <D.2116>:
   11035              :    andv = k & down;
   11036              :    andvm1 = andv + 4294967295;
   11037              :    l = mul + andvm1;
   11038              :    if (l < num_threadsu) goto <D.2120>; else goto <D.2117>;
   11039              :    <D.2120>:
   11040              :    // For UDRs this is UDR merge, performed using var2 variable as temporary,
   11041              :    // i.e. var2 = rpriva[l - k]; UDR merge (var2, rpriva[l]); rpriva[l] = var2;
   11042              :    rpriva[l] = rpriva[l - k] + rpriva[l];
   11043              :    <D.2117>:
   11044              :    if (down == 0) goto <D.2121>; else goto <D.2122>;
   11045              :    <D.2121>:
   11046              :    k = k << 1;
   11047              :    goto <D.2123>;
   11048              :    <D.2122>:
   11049              :    k = k >> 1;
   11050              :    <D.2123>:
   11051              :    GOMP_barrier ();
   11052              :    if (k != 0) goto <D.2108>; else goto <D.2103>;
   11053              :    <D.2103>:
   11054              :    if (thread_num == 0) goto <D.2124>; else goto <D.2125>;
   11055              :    <D.2124>:
   11056              :    // For UDRs this is UDR init or copy from var3.
   11057              :    var2 = 0;
   11058              :    goto <D.2126>;
   11059              :    <D.2125>:
   11060              :    var2 = rpriva[thread_num - 1];
   11061              :    <D.2126>:
   11062              :    ivar = 0;
   11063              :    #pragma omp for _scantemp_(controlb) _scantemp_(controlp) \
   11064              :      reduction(inscan,+:r) private(i)
   11065              :    for (i = 0; i < n; i = i + 1)
   11066              :      {
   11067              :        {
   11068              :          // For UDRs, this is r = var2; UDR merge (r, rprivb[ivar]);
   11069              :          r = var2 + rprivb[ivar];
   11070              :        }
   11071              :        {
   11072              :          // This is the scan phase from user code.
   11073              :          use (r);
   11074              :          // Plus a bump of the iterator.
   11075              :          ivar = ivar + 1;
   11076              :        }
   11077              :      }  */
   11078              : 
   11079              : static void
   11080          185 : lower_omp_for_scan (gimple_seq *body_p, gimple_seq *dlist, gomp_for *stmt,
   11081              :                     struct omp_for_data *fd, omp_context *ctx)
   11082              : {
   11083          185 :   bool is_for_simd = gimple_omp_for_combined_p (stmt);
   11084          185 :   gcc_assert (ctx->scan_inclusive || ctx->scan_exclusive);
   11085              : 
   11086          185 :   gimple_seq body = gimple_omp_body (stmt);
   11087          185 :   gimple_stmt_iterator input1_gsi = gsi_none ();
   11088          185 :   struct walk_stmt_info wi;
   11089          185 :   memset (&wi, 0, sizeof (wi));
   11090          185 :   wi.val_only = true;
   11091          185 :   wi.info = (void *) &input1_gsi;
   11092          185 :   walk_gimple_seq_mod (&body, omp_find_scan, NULL, &wi);
   11093          185 :   gcc_assert (!gsi_end_p (input1_gsi));
   11094              : 
   11095          185 :   gimple *input_stmt1 = gsi_stmt (input1_gsi);
   11096          185 :   gimple_stmt_iterator gsi = input1_gsi;
   11097          185 :   gsi_next (&gsi);
   11098          185 :   gimple_stmt_iterator scan1_gsi = gsi;
   11099          185 :   gimple *scan_stmt1 = gsi_stmt (gsi);
   11100          185 :   gcc_assert (scan_stmt1 && gimple_code (scan_stmt1) == GIMPLE_OMP_SCAN);
   11101              : 
   11102          185 :   gimple_seq input_body = gimple_omp_body (input_stmt1);
   11103          185 :   gimple_seq scan_body = gimple_omp_body (scan_stmt1);
   11104          185 :   gimple_omp_set_body (input_stmt1, NULL);
   11105          185 :   gimple_omp_set_body (scan_stmt1, NULL);
   11106          185 :   gimple_omp_set_body (stmt, NULL);
   11107              : 
   11108          185 :   gomp_for *new_stmt = as_a <gomp_for *> (gimple_copy (stmt));
   11109          185 :   gimple_seq new_body = copy_gimple_seq_and_replace_locals (body);
   11110          185 :   gimple_omp_set_body (stmt, body);
   11111          185 :   gimple_omp_set_body (input_stmt1, input_body);
   11112              : 
   11113          185 :   gimple_stmt_iterator input2_gsi = gsi_none ();
   11114          185 :   memset (&wi, 0, sizeof (wi));
   11115          185 :   wi.val_only = true;
   11116          185 :   wi.info = (void *) &input2_gsi;
   11117          185 :   walk_gimple_seq_mod (&new_body, omp_find_scan, NULL, &wi);
   11118          185 :   gcc_assert (!gsi_end_p (input2_gsi));
   11119              : 
   11120          185 :   gimple *input_stmt2 = gsi_stmt (input2_gsi);
   11121          185 :   gsi = input2_gsi;
   11122          185 :   gsi_next (&gsi);
   11123          185 :   gimple_stmt_iterator scan2_gsi = gsi;
   11124          185 :   gimple *scan_stmt2 = gsi_stmt (gsi);
   11125          185 :   gcc_assert (scan_stmt2 && gimple_code (scan_stmt2) == GIMPLE_OMP_SCAN);
   11126          185 :   gimple_omp_set_body (scan_stmt2, scan_body);
   11127              : 
   11128          185 :   gimple_stmt_iterator input3_gsi = gsi_none ();
   11129          185 :   gimple_stmt_iterator scan3_gsi = gsi_none ();
   11130          185 :   gimple_stmt_iterator input4_gsi = gsi_none ();
   11131          185 :   gimple_stmt_iterator scan4_gsi = gsi_none ();
   11132          185 :   gimple *input_stmt3 = NULL, *scan_stmt3 = NULL;
   11133          185 :   gimple *input_stmt4 = NULL, *scan_stmt4 = NULL;
   11134          185 :   omp_context *input_simd_ctx = NULL, *scan_simd_ctx = NULL;
   11135          185 :   if (is_for_simd)
   11136              :     {
   11137           91 :       memset (&wi, 0, sizeof (wi));
   11138           91 :       wi.val_only = true;
   11139           91 :       wi.info = (void *) &input3_gsi;
   11140           91 :       walk_gimple_seq_mod (&input_body, omp_find_scan, NULL, &wi);
   11141           91 :       gcc_assert (!gsi_end_p (input3_gsi));
   11142              : 
   11143           91 :       input_stmt3 = gsi_stmt (input3_gsi);
   11144           91 :       gsi = input3_gsi;
   11145           91 :       gsi_next (&gsi);
   11146           91 :       scan3_gsi = gsi;
   11147           91 :       scan_stmt3 = gsi_stmt (gsi);
   11148           91 :       gcc_assert (scan_stmt3 && gimple_code (scan_stmt3) == GIMPLE_OMP_SCAN);
   11149              : 
   11150           91 :       memset (&wi, 0, sizeof (wi));
   11151           91 :       wi.val_only = true;
   11152           91 :       wi.info = (void *) &input4_gsi;
   11153           91 :       walk_gimple_seq_mod (&scan_body, omp_find_scan, NULL, &wi);
   11154           91 :       gcc_assert (!gsi_end_p (input4_gsi));
   11155              : 
   11156           91 :       input_stmt4 = gsi_stmt (input4_gsi);
   11157           91 :       gsi = input4_gsi;
   11158           91 :       gsi_next (&gsi);
   11159           91 :       scan4_gsi = gsi;
   11160           91 :       scan_stmt4 = gsi_stmt (gsi);
   11161           91 :       gcc_assert (scan_stmt4 && gimple_code (scan_stmt4) == GIMPLE_OMP_SCAN);
   11162              : 
   11163           91 :       input_simd_ctx = maybe_lookup_ctx (input_stmt3)->outer;
   11164           91 :       scan_simd_ctx = maybe_lookup_ctx (input_stmt4)->outer;
   11165              :     }
   11166              : 
   11167          185 :   tree num_threads = create_tmp_var (integer_type_node);
   11168          185 :   tree thread_num = create_tmp_var (integer_type_node);
   11169          185 :   tree nthreads_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
   11170          185 :   tree threadnum_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
   11171          185 :   gimple *g = gimple_build_call (nthreads_decl, 0);
   11172          185 :   gimple_call_set_lhs (g, num_threads);
   11173          185 :   gimple_seq_add_stmt (body_p, g);
   11174          185 :   g = gimple_build_call (threadnum_decl, 0);
   11175          185 :   gimple_call_set_lhs (g, thread_num);
   11176          185 :   gimple_seq_add_stmt (body_p, g);
   11177              : 
   11178          185 :   tree ivar = create_tmp_var (sizetype);
   11179          185 :   tree new_clauses1 = NULL_TREE, new_clauses2 = NULL_TREE;
   11180          185 :   tree *cp1 = &new_clauses1, *cp2 = &new_clauses2;
   11181          185 :   tree k = create_tmp_var (unsigned_type_node);
   11182          185 :   tree l = create_tmp_var (unsigned_type_node);
   11183              : 
   11184          185 :   gimple_seq clist = NULL, mdlist = NULL;
   11185          185 :   gimple_seq thr01_list = NULL, thrn1_list = NULL;
   11186          185 :   gimple_seq thr02_list = NULL, thrn2_list = NULL;
   11187          185 :   gimple_seq scan1_list = NULL, input2_list = NULL;
   11188          185 :   gimple_seq last_list = NULL, reduc_list = NULL;
   11189          718 :   for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
   11190          533 :     if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
   11191          533 :         && OMP_CLAUSE_REDUCTION_INSCAN (c))
   11192              :       {
   11193          217 :         location_t clause_loc = OMP_CLAUSE_LOCATION (c);
   11194          217 :         tree var = OMP_CLAUSE_DECL (c);
   11195          217 :         tree new_var = lookup_decl (var, ctx);
   11196          217 :         tree var3 = NULL_TREE;
   11197          217 :         tree new_vard = new_var;
   11198          217 :         if (omp_privatize_by_reference (var))
   11199           32 :           new_var = build_simple_mem_ref_loc (clause_loc, new_var);
   11200          217 :         if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
   11201              :           {
   11202           64 :             var3 = maybe_lookup_decl (new_vard, ctx);
   11203           64 :             if (var3 == new_vard)
   11204          185 :               var3 = NULL_TREE;
   11205              :           }
   11206              : 
   11207          217 :         tree ptype = build_pointer_type (TREE_TYPE (new_var));
   11208          217 :         tree rpriva = create_tmp_var (ptype);
   11209          217 :         tree nc = build_omp_clause (clause_loc, OMP_CLAUSE__SCANTEMP_);
   11210          217 :         OMP_CLAUSE_DECL (nc) = rpriva;
   11211          217 :         *cp1 = nc;
   11212          217 :         cp1 = &OMP_CLAUSE_CHAIN (nc);
   11213              : 
   11214          217 :         tree rprivb = create_tmp_var (ptype);
   11215          217 :         nc = build_omp_clause (clause_loc, OMP_CLAUSE__SCANTEMP_);
   11216          217 :         OMP_CLAUSE_DECL (nc) = rprivb;
   11217          217 :         OMP_CLAUSE__SCANTEMP__ALLOC (nc) = 1;
   11218          217 :         *cp1 = nc;
   11219          217 :         cp1 = &OMP_CLAUSE_CHAIN (nc);
   11220              : 
   11221          217 :         tree var2 = create_tmp_var_raw (TREE_TYPE (new_var));
   11222          217 :         if (new_vard != new_var)
   11223           32 :           TREE_ADDRESSABLE (var2) = 1;
   11224          217 :         gimple_add_tmp_var (var2);
   11225              : 
   11226          217 :         tree x = fold_convert_loc (clause_loc, sizetype, thread_num);
   11227          217 :         x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
   11228          217 :                              TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
   11229          217 :         x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
   11230          217 :         tree rpriva_ref = build_simple_mem_ref_loc (clause_loc, x);
   11231              : 
   11232          217 :         x = fold_build2_loc (clause_loc, PLUS_EXPR, integer_type_node,
   11233              :                              thread_num, integer_minus_one_node);
   11234          217 :         x = fold_convert_loc (clause_loc, sizetype, x);
   11235          217 :         x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
   11236          217 :                              TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
   11237          217 :         x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
   11238          217 :         tree rprivam1_ref = build_simple_mem_ref_loc (clause_loc, x);
   11239              : 
   11240          217 :         x = fold_convert_loc (clause_loc, sizetype, l);
   11241          217 :         x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
   11242          217 :                              TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
   11243          217 :         x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
   11244          217 :         tree rprival_ref = build_simple_mem_ref_loc (clause_loc, x);
   11245              : 
   11246          217 :         x = fold_build2_loc (clause_loc, MINUS_EXPR, unsigned_type_node, l, k);
   11247          217 :         x = fold_convert_loc (clause_loc, sizetype, x);
   11248          217 :         x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
   11249          217 :                              TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
   11250          217 :         x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
   11251          217 :         tree rprivalmk_ref = build_simple_mem_ref_loc (clause_loc, x);
   11252              : 
   11253          217 :         x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, ivar,
   11254          217 :                              TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
   11255          217 :         x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rprivb), rprivb, x);
   11256          217 :         tree rprivb_ref = build_simple_mem_ref_loc (clause_loc, x);
   11257              : 
   11258          217 :         tree var4 = is_for_simd ? new_var : var2;
   11259          107 :         tree var5 = NULL_TREE, var6 = NULL_TREE;
   11260          107 :         if (is_for_simd)
   11261              :           {
   11262          107 :             var5 = lookup_decl (var, input_simd_ctx);
   11263          107 :             var6 = lookup_decl (var, scan_simd_ctx);
   11264          107 :             if (new_vard != new_var)
   11265              :               {
   11266           16 :                 var5 = build_simple_mem_ref_loc (clause_loc, var5);
   11267           16 :                 var6 = build_simple_mem_ref_loc (clause_loc, var6);
   11268              :               }
   11269              :           }
   11270          217 :         if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
   11271              :           {
   11272           64 :             tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
   11273           64 :             tree val = var2;
   11274              : 
   11275           64 :             x = lang_hooks.decls.omp_clause_default_ctor
   11276           64 :                     (c, var2, build_outer_var_ref (var, ctx));
   11277           64 :             if (x)
   11278           32 :               gimplify_and_add (x, &clist);
   11279              : 
   11280           64 :             x = build_outer_var_ref (var, ctx);
   11281           64 :             x = lang_hooks.decls.omp_clause_assign_op (c, unshare_expr (var4),
   11282              :                                                        x);
   11283           64 :             gimplify_and_add (x, &thr01_list);
   11284              : 
   11285           64 :             tree y = (DECL_HAS_VALUE_EXPR_P (new_vard)
   11286           64 :                       ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
   11287           64 :             if (var3)
   11288              :               {
   11289           32 :                 x = unshare_expr (var4);
   11290           32 :                 x = lang_hooks.decls.omp_clause_assign_op (c, x, var3);
   11291           32 :                 gimplify_and_add (x, &thrn1_list);
   11292           32 :                 x = unshare_expr (var4);
   11293           32 :                 x = lang_hooks.decls.omp_clause_assign_op (c, x, var3);
   11294           32 :                 gimplify_and_add (x, &thr02_list);
   11295              :               }
   11296           32 :             else if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
   11297              :               {
   11298              :                 /* Otherwise, assign to it the identity element.  */
   11299           32 :                 gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
   11300           32 :                 tseq = copy_gimple_seq_and_replace_locals (tseq);
   11301           32 :                 if (!is_for_simd)
   11302              :                   {
   11303           16 :                     if (new_vard != new_var)
   11304            4 :                       val = build_fold_addr_expr_loc (clause_loc, val);
   11305           16 :                     SET_DECL_VALUE_EXPR (new_vard, val);
   11306           16 :                     DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
   11307              :                   }
   11308           32 :                 SET_DECL_VALUE_EXPR (placeholder, error_mark_node);
   11309           32 :                 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
   11310           32 :                 lower_omp (&tseq, ctx);
   11311           32 :                 gimple_seq_add_seq (&thrn1_list, tseq);
   11312           32 :                 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
   11313           32 :                 lower_omp (&tseq, ctx);
   11314           32 :                 gimple_seq_add_seq (&thr02_list, tseq);
   11315           32 :                 SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
   11316           32 :                 DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
   11317           32 :                 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
   11318           32 :                 if (y)
   11319            0 :                   SET_DECL_VALUE_EXPR (new_vard, y);
   11320              :                 else
   11321              :                   {
   11322           32 :                     DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
   11323           32 :                     SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
   11324              :                   }
   11325              :               }
   11326              : 
   11327           64 :             x = unshare_expr (var4);
   11328           64 :             x = lang_hooks.decls.omp_clause_assign_op (c, x, rprivam1_ref);
   11329           64 :             gimplify_and_add (x, &thrn2_list);
   11330              : 
   11331           64 :             if (is_for_simd)
   11332              :               {
   11333           32 :                 x = unshare_expr (rprivb_ref);
   11334           32 :                 x = lang_hooks.decls.omp_clause_assign_op (c, x, var5);
   11335           32 :                 gimplify_and_add (x, &scan1_list);
   11336              :               }
   11337              :             else
   11338              :               {
   11339           32 :                 if (ctx->scan_exclusive)
   11340              :                   {
   11341           16 :                     x = unshare_expr (rprivb_ref);
   11342           16 :                     x = lang_hooks.decls.omp_clause_assign_op (c, x, var2);
   11343           16 :                     gimplify_and_add (x, &scan1_list);
   11344              :                   }
   11345              : 
   11346           32 :                 gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
   11347           32 :                 tseq = copy_gimple_seq_and_replace_locals (tseq);
   11348           32 :                 SET_DECL_VALUE_EXPR (placeholder, var2);
   11349           32 :                 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
   11350           32 :                 lower_omp (&tseq, ctx);
   11351           32 :                 gimple_seq_add_seq (&scan1_list, tseq);
   11352              : 
   11353           32 :                 if (ctx->scan_inclusive)
   11354              :                   {
   11355           16 :                     x = unshare_expr (rprivb_ref);
   11356           16 :                     x = lang_hooks.decls.omp_clause_assign_op (c, x, var2);
   11357           16 :                     gimplify_and_add (x, &scan1_list);
   11358              :                   }
   11359              :               }
   11360              : 
   11361           64 :             x = unshare_expr (rpriva_ref);
   11362           64 :             x = lang_hooks.decls.omp_clause_assign_op (c, x,
   11363              :                                                        unshare_expr (var4));
   11364           64 :             gimplify_and_add (x, &mdlist);
   11365              : 
   11366           96 :             x = unshare_expr (is_for_simd ? var6 : new_var);
   11367           64 :             x = lang_hooks.decls.omp_clause_assign_op (c, x, var4);
   11368           64 :             gimplify_and_add (x, &input2_list);
   11369              : 
   11370           64 :             val = rprivb_ref;
   11371           64 :             if (new_vard != new_var)
   11372           24 :               val = build_fold_addr_expr_loc (clause_loc, val);
   11373              : 
   11374           64 :             gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
   11375           64 :             tseq = copy_gimple_seq_and_replace_locals (tseq);
   11376           64 :             SET_DECL_VALUE_EXPR (new_vard, val);
   11377           64 :             DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
   11378           64 :             if (is_for_simd)
   11379              :               {
   11380           32 :                 SET_DECL_VALUE_EXPR (placeholder, var6);
   11381           32 :                 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
   11382              :               }
   11383              :             else
   11384           32 :               DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
   11385           64 :             lower_omp (&tseq, ctx);
   11386           64 :             if (y)
   11387            0 :               SET_DECL_VALUE_EXPR (new_vard, y);
   11388              :             else
   11389              :               {
   11390           64 :                 DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
   11391           64 :                 SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
   11392              :               }
   11393           64 :             if (!is_for_simd)
   11394              :               {
   11395           32 :                 SET_DECL_VALUE_EXPR (placeholder, new_var);
   11396           32 :                 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
   11397           32 :                 lower_omp (&tseq, ctx);
   11398              :               }
   11399           64 :             gimple_seq_add_seq (&input2_list, tseq);
   11400              : 
   11401           64 :             x = build_outer_var_ref (var, ctx);
   11402           64 :             x = lang_hooks.decls.omp_clause_assign_op (c, x, rpriva_ref);
   11403           64 :             gimplify_and_add (x, &last_list);
   11404              : 
   11405           64 :             x = lang_hooks.decls.omp_clause_assign_op (c, var2, rprivalmk_ref);
   11406           64 :             gimplify_and_add (x, &reduc_list);
   11407           64 :             tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
   11408           64 :             tseq = copy_gimple_seq_and_replace_locals (tseq);
   11409           64 :             val = rprival_ref;
   11410           64 :             if (new_vard != new_var)
   11411           24 :               val = build_fold_addr_expr_loc (clause_loc, val);
   11412           64 :             SET_DECL_VALUE_EXPR (new_vard, val);
   11413           64 :             DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
   11414           64 :             SET_DECL_VALUE_EXPR (placeholder, var2);
   11415           64 :             lower_omp (&tseq, ctx);
   11416           64 :             OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
   11417           64 :             SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
   11418           64 :             DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
   11419           64 :             if (y)
   11420            0 :               SET_DECL_VALUE_EXPR (new_vard, y);
   11421              :             else
   11422              :               {
   11423           64 :                 DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
   11424           64 :                 SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
   11425              :               }
   11426           64 :             gimple_seq_add_seq (&reduc_list, tseq);
   11427           64 :             x = lang_hooks.decls.omp_clause_assign_op (c, rprival_ref, var2);
   11428           64 :             gimplify_and_add (x, &reduc_list);
   11429              : 
   11430           64 :             x = lang_hooks.decls.omp_clause_dtor (c, var2);
   11431           64 :             if (x)
   11432           32 :               gimplify_and_add (x, dlist);
   11433              :           }
   11434              :         else
   11435              :           {
   11436          153 :             x = build_outer_var_ref (var, ctx);
   11437          153 :             gimplify_assign (unshare_expr (var4), x, &thr01_list);
   11438              : 
   11439          153 :             x = omp_reduction_init (c, TREE_TYPE (new_var));
   11440          153 :             gimplify_assign (unshare_expr (var4), unshare_expr (x),
   11441              :                              &thrn1_list);
   11442          153 :             gimplify_assign (unshare_expr (var4), x, &thr02_list);
   11443              : 
   11444          153 :             gimplify_assign (unshare_expr (var4), rprivam1_ref, &thrn2_list);
   11445              : 
   11446          153 :             enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
   11447          153 :             if (code == MINUS_EXPR)
   11448            0 :               code = PLUS_EXPR;
   11449              : 
   11450          153 :             if (is_for_simd)
   11451           75 :               gimplify_assign (unshare_expr (rprivb_ref), var5, &scan1_list);
   11452              :             else
   11453              :               {
   11454           78 :                 if (ctx->scan_exclusive)
   11455           28 :                   gimplify_assign (unshare_expr (rprivb_ref), var2,
   11456              :                                    &scan1_list);
   11457           78 :                 x = build2 (code, TREE_TYPE (new_var), var2, new_var);
   11458           78 :                 gimplify_assign (var2, x, &scan1_list);
   11459           78 :                 if (ctx->scan_inclusive)
   11460           50 :                   gimplify_assign (unshare_expr (rprivb_ref), var2,
   11461              :                                    &scan1_list);
   11462              :               }
   11463              : 
   11464          153 :             gimplify_assign (unshare_expr (rpriva_ref), unshare_expr (var4),
   11465              :                              &mdlist);
   11466              : 
   11467          153 :             x = build2 (code, TREE_TYPE (new_var), var4, rprivb_ref);
   11468          231 :             gimplify_assign (is_for_simd ? var6 : new_var, x, &input2_list);
   11469              : 
   11470          153 :             gimplify_assign (build_outer_var_ref (var, ctx), rpriva_ref,
   11471              :                              &last_list);
   11472              : 
   11473          153 :             x = build2 (code, TREE_TYPE (new_var), rprivalmk_ref,
   11474              :                         unshare_expr (rprival_ref));
   11475          153 :             gimplify_assign (rprival_ref, x, &reduc_list);
   11476              :           }
   11477              :       }
   11478              : 
   11479          185 :   g = gimple_build_assign (ivar, PLUS_EXPR, ivar, size_one_node);
   11480          185 :   gimple_seq_add_stmt (&scan1_list, g);
   11481          185 :   g = gimple_build_assign (ivar, PLUS_EXPR, ivar, size_one_node);
   11482          279 :   gimple_seq_add_stmt (gimple_omp_body_ptr (is_for_simd
   11483              :                                             ? scan_stmt4 : scan_stmt2), g);
   11484              : 
   11485          185 :   tree controlb = create_tmp_var (boolean_type_node);
   11486          185 :   tree controlp = create_tmp_var (ptr_type_node);
   11487          185 :   tree nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
   11488          185 :   OMP_CLAUSE_DECL (nc) = controlb;
   11489          185 :   OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
   11490          185 :   *cp1 = nc;
   11491          185 :   cp1 = &OMP_CLAUSE_CHAIN (nc);
   11492          185 :   nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
   11493          185 :   OMP_CLAUSE_DECL (nc) = controlp;
   11494          185 :   OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
   11495          185 :   *cp1 = nc;
   11496          185 :   cp1 = &OMP_CLAUSE_CHAIN (nc);
   11497          185 :   nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
   11498          185 :   OMP_CLAUSE_DECL (nc) = controlb;
   11499          185 :   OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
   11500          185 :   *cp2 = nc;
   11501          185 :   cp2 = &OMP_CLAUSE_CHAIN (nc);
   11502          185 :   nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
   11503          185 :   OMP_CLAUSE_DECL (nc) = controlp;
   11504          185 :   OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
   11505          185 :   *cp2 = nc;
   11506          185 :   cp2 = &OMP_CLAUSE_CHAIN (nc);
   11507              : 
   11508          185 :   *cp1 = gimple_omp_for_clauses (stmt);
   11509          185 :   gimple_omp_for_set_clauses (stmt, new_clauses1);
   11510          185 :   *cp2 = gimple_omp_for_clauses (new_stmt);
   11511          185 :   gimple_omp_for_set_clauses (new_stmt, new_clauses2);
   11512              : 
   11513          185 :   if (is_for_simd)
   11514              :     {
   11515           91 :       gimple_seq_add_seq (gimple_omp_body_ptr (scan_stmt3), scan1_list);
   11516           91 :       gimple_seq_add_seq (gimple_omp_body_ptr (input_stmt4), input2_list);
   11517              : 
   11518           91 :       gsi_insert_seq_after (&input3_gsi, gimple_omp_body (input_stmt3),
   11519              :                             GSI_SAME_STMT);
   11520           91 :       gsi_remove (&input3_gsi, true);
   11521           91 :       gsi_insert_seq_after (&scan3_gsi, gimple_omp_body (scan_stmt3),
   11522              :                             GSI_SAME_STMT);
   11523           91 :       gsi_remove (&scan3_gsi, true);
   11524           91 :       gsi_insert_seq_after (&input4_gsi, gimple_omp_body (input_stmt4),
   11525              :                             GSI_SAME_STMT);
   11526           91 :       gsi_remove (&input4_gsi, true);
   11527           91 :       gsi_insert_seq_after (&scan4_gsi, gimple_omp_body (scan_stmt4),
   11528              :                             GSI_SAME_STMT);
   11529           91 :       gsi_remove (&scan4_gsi, true);
   11530              :     }
   11531              :   else
   11532              :     {
   11533           94 :       gimple_omp_set_body (scan_stmt1, scan1_list);
   11534           94 :       gimple_omp_set_body (input_stmt2, input2_list);
   11535              :     }
   11536              : 
   11537          185 :   gsi_insert_seq_after (&input1_gsi, gimple_omp_body (input_stmt1),
   11538              :                         GSI_SAME_STMT);
   11539          185 :   gsi_remove (&input1_gsi, true);
   11540          185 :   gsi_insert_seq_after (&scan1_gsi, gimple_omp_body (scan_stmt1),
   11541              :                         GSI_SAME_STMT);
   11542          185 :   gsi_remove (&scan1_gsi, true);
   11543          185 :   gsi_insert_seq_after (&input2_gsi, gimple_omp_body (input_stmt2),
   11544              :                         GSI_SAME_STMT);
   11545          185 :   gsi_remove (&input2_gsi, true);
   11546          185 :   gsi_insert_seq_after (&scan2_gsi, gimple_omp_body (scan_stmt2),
   11547              :                         GSI_SAME_STMT);
   11548          185 :   gsi_remove (&scan2_gsi, true);
   11549              : 
   11550          185 :   gimple_seq_add_seq (body_p, clist);
   11551              : 
   11552          185 :   tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
   11553          185 :   tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
   11554          185 :   tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
   11555          185 :   g = gimple_build_cond (EQ_EXPR, thread_num, integer_zero_node, lab1, lab2);
   11556          185 :   gimple_seq_add_stmt (body_p, g);
   11557          185 :   g = gimple_build_label (lab1);
   11558          185 :   gimple_seq_add_stmt (body_p, g);
   11559          185 :   gimple_seq_add_seq (body_p, thr01_list);
   11560          185 :   g = gimple_build_goto (lab3);
   11561          185 :   gimple_seq_add_stmt (body_p, g);
   11562          185 :   g = gimple_build_label (lab2);
   11563          185 :   gimple_seq_add_stmt (body_p, g);
   11564          185 :   gimple_seq_add_seq (body_p, thrn1_list);
   11565          185 :   g = gimple_build_label (lab3);
   11566          185 :   gimple_seq_add_stmt (body_p, g);
   11567              : 
   11568          185 :   g = gimple_build_assign (ivar, size_zero_node);
   11569          185 :   gimple_seq_add_stmt (body_p, g);
   11570              : 
   11571          185 :   gimple_seq_add_stmt (body_p, stmt);
   11572          185 :   gimple_seq_add_seq (body_p, body);
   11573          185 :   gimple_seq_add_stmt (body_p, gimple_build_omp_continue (fd->loop.v,
   11574              :                                                           fd->loop.v));
   11575              : 
   11576          185 :   g = gimple_build_omp_return (true);
   11577          185 :   gimple_seq_add_stmt (body_p, g);
   11578          185 :   gimple_seq_add_seq (body_p, mdlist);
   11579              : 
   11580          185 :   lab1 = create_artificial_label (UNKNOWN_LOCATION);
   11581          185 :   lab2 = create_artificial_label (UNKNOWN_LOCATION);
   11582          185 :   g = gimple_build_cond (GT_EXPR, num_threads, integer_one_node, lab1, lab2);
   11583          185 :   gimple_seq_add_stmt (body_p, g);
   11584          185 :   g = gimple_build_label (lab1);
   11585          185 :   gimple_seq_add_stmt (body_p, g);
   11586              : 
   11587          185 :   g = omp_build_barrier (NULL, GOMP_BARRIER_IMPLICIT_WORKSHARE);
   11588          185 :   gimple_seq_add_stmt (body_p, g);
   11589              : 
   11590          185 :   tree down = create_tmp_var (unsigned_type_node);
   11591          185 :   g = gimple_build_assign (down, build_zero_cst (unsigned_type_node));
   11592          185 :   gimple_seq_add_stmt (body_p, g);
   11593              : 
   11594          185 :   g = gimple_build_assign (k, build_one_cst (unsigned_type_node));
   11595          185 :   gimple_seq_add_stmt (body_p, g);
   11596              : 
   11597          185 :   tree num_threadsu = create_tmp_var (unsigned_type_node);
   11598          185 :   g = gimple_build_assign (num_threadsu, NOP_EXPR, num_threads);
   11599          185 :   gimple_seq_add_stmt (body_p, g);
   11600              : 
   11601          185 :   tree thread_numu = create_tmp_var (unsigned_type_node);
   11602          185 :   g = gimple_build_assign (thread_numu, NOP_EXPR, thread_num);
   11603          185 :   gimple_seq_add_stmt (body_p, g);
   11604              : 
   11605          185 :   tree thread_nump1 = create_tmp_var (unsigned_type_node);
   11606          185 :   g = gimple_build_assign (thread_nump1, PLUS_EXPR, thread_numu,
   11607              :                            build_int_cst (unsigned_type_node, 1));
   11608          185 :   gimple_seq_add_stmt (body_p, g);
   11609              : 
   11610          185 :   lab3 = create_artificial_label (UNKNOWN_LOCATION);
   11611          185 :   g = gimple_build_label (lab3);
   11612          185 :   gimple_seq_add_stmt (body_p, g);
   11613              : 
   11614          185 :   tree twok = create_tmp_var (unsigned_type_node);
   11615          185 :   g = gimple_build_assign (twok, LSHIFT_EXPR, k, integer_one_node);
   11616          185 :   gimple_seq_add_stmt (body_p, g);
   11617              : 
   11618          185 :   tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
   11619          185 :   tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
   11620          185 :   tree lab6 = create_artificial_label (UNKNOWN_LOCATION);
   11621          185 :   g = gimple_build_cond (GT_EXPR, twok, num_threadsu, lab4, lab5);
   11622          185 :   gimple_seq_add_stmt (body_p, g);
   11623          185 :   g = gimple_build_label (lab4);
   11624          185 :   gimple_seq_add_stmt (body_p, g);
   11625          185 :   g = gimple_build_assign (down, build_all_ones_cst (unsigned_type_node));
   11626          185 :   gimple_seq_add_stmt (body_p, g);
   11627          185 :   g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
   11628          185 :   gimple_seq_add_stmt (body_p, g);
   11629              : 
   11630          185 :   g = gimple_build_cond (EQ_EXPR, k, num_threadsu, lab6, lab5);
   11631          185 :   gimple_seq_add_stmt (body_p, g);
   11632          185 :   g = gimple_build_label (lab6);
   11633          185 :   gimple_seq_add_stmt (body_p, g);
   11634              : 
   11635          185 :   g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
   11636          185 :   gimple_seq_add_stmt (body_p, g);
   11637              : 
   11638          185 :   g = gimple_build_label (lab5);
   11639          185 :   gimple_seq_add_stmt (body_p, g);
   11640              : 
   11641          185 :   g = gimple_build_assign (twok, LSHIFT_EXPR, k, integer_one_node);
   11642          185 :   gimple_seq_add_stmt (body_p, g);
   11643              : 
   11644          185 :   tree cplx = create_tmp_var (build_complex_type (unsigned_type_node, false));
   11645          185 :   g = gimple_build_call_internal (IFN_MUL_OVERFLOW, 2, thread_nump1, twok);
   11646          185 :   gimple_call_set_lhs (g, cplx);
   11647          185 :   gimple_seq_add_stmt (body_p, g);
   11648          185 :   tree mul = create_tmp_var (unsigned_type_node);
   11649          185 :   g = gimple_build_assign (mul, REALPART_EXPR,
   11650              :                            build1 (REALPART_EXPR, unsigned_type_node, cplx));
   11651          185 :   gimple_seq_add_stmt (body_p, g);
   11652          185 :   tree ovf = create_tmp_var (unsigned_type_node);
   11653          185 :   g = gimple_build_assign (ovf, IMAGPART_EXPR,
   11654              :                            build1 (IMAGPART_EXPR, unsigned_type_node, cplx));
   11655          185 :   gimple_seq_add_stmt (body_p, g);
   11656              : 
   11657          185 :   tree lab7 = create_artificial_label (UNKNOWN_LOCATION);
   11658          185 :   tree lab8 = create_artificial_label (UNKNOWN_LOCATION);
   11659          185 :   g = gimple_build_cond (EQ_EXPR, ovf, build_zero_cst (unsigned_type_node),
   11660              :                          lab7, lab8);
   11661          185 :   gimple_seq_add_stmt (body_p, g);
   11662          185 :   g = gimple_build_label (lab7);
   11663          185 :   gimple_seq_add_stmt (body_p, g);
   11664              : 
   11665          185 :   tree andv = create_tmp_var (unsigned_type_node);
   11666          185 :   g = gimple_build_assign (andv, BIT_AND_EXPR, k, down);
   11667          185 :   gimple_seq_add_stmt (body_p, g);
   11668          185 :   tree andvm1 = create_tmp_var (unsigned_type_node);
   11669          185 :   g = gimple_build_assign (andvm1, PLUS_EXPR, andv,
   11670              :                            build_minus_one_cst (unsigned_type_node));
   11671          185 :   gimple_seq_add_stmt (body_p, g);
   11672              : 
   11673          185 :   g = gimple_build_assign (l, PLUS_EXPR, mul, andvm1);
   11674          185 :   gimple_seq_add_stmt (body_p, g);
   11675              : 
   11676          185 :   tree lab9 = create_artificial_label (UNKNOWN_LOCATION);
   11677          185 :   g = gimple_build_cond (LT_EXPR, l, num_threadsu, lab9, lab8);
   11678          185 :   gimple_seq_add_stmt (body_p, g);
   11679          185 :   g = gimple_build_label (lab9);
   11680          185 :   gimple_seq_add_stmt (body_p, g);
   11681          185 :   gimple_seq_add_seq (body_p, reduc_list);
   11682          185 :   g = gimple_build_label (lab8);
   11683          185 :   gimple_seq_add_stmt (body_p, g);
   11684              : 
   11685          185 :   tree lab10 = create_artificial_label (UNKNOWN_LOCATION);
   11686          185 :   tree lab11 = create_artificial_label (UNKNOWN_LOCATION);
   11687          185 :   tree lab12 = create_artificial_label (UNKNOWN_LOCATION);
   11688          185 :   g = gimple_build_cond (EQ_EXPR, down, build_zero_cst (unsigned_type_node),
   11689              :                          lab10, lab11);
   11690          185 :   gimple_seq_add_stmt (body_p, g);
   11691          185 :   g = gimple_build_label (lab10);
   11692          185 :   gimple_seq_add_stmt (body_p, g);
   11693          185 :   g = gimple_build_assign (k, LSHIFT_EXPR, k, integer_one_node);
   11694          185 :   gimple_seq_add_stmt (body_p, g);
   11695          185 :   g = gimple_build_goto (lab12);
   11696          185 :   gimple_seq_add_stmt (body_p, g);
   11697          185 :   g = gimple_build_label (lab11);
   11698          185 :   gimple_seq_add_stmt (body_p, g);
   11699          185 :   g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
   11700          185 :   gimple_seq_add_stmt (body_p, g);
   11701          185 :   g = gimple_build_label (lab12);
   11702          185 :   gimple_seq_add_stmt (body_p, g);
   11703              : 
   11704          185 :   g = omp_build_barrier (NULL, GOMP_BARRIER_IMPLICIT_WORKSHARE);
   11705          185 :   gimple_seq_add_stmt (body_p, g);
   11706              : 
   11707          185 :   g = gimple_build_cond (NE_EXPR, k, build_zero_cst (unsigned_type_node),
   11708              :                          lab3, lab2);
   11709          185 :   gimple_seq_add_stmt (body_p, g);
   11710              : 
   11711          185 :   g = gimple_build_label (lab2);
   11712          185 :   gimple_seq_add_stmt (body_p, g);
   11713              : 
   11714          185 :   lab1 = create_artificial_label (UNKNOWN_LOCATION);
   11715          185 :   lab2 = create_artificial_label (UNKNOWN_LOCATION);
   11716          185 :   lab3 = create_artificial_label (UNKNOWN_LOCATION);
   11717          185 :   g = gimple_build_cond (EQ_EXPR, thread_num, integer_zero_node, lab1, lab2);
   11718          185 :   gimple_seq_add_stmt (body_p, g);
   11719          185 :   g = gimple_build_label (lab1);
   11720          185 :   gimple_seq_add_stmt (body_p, g);
   11721          185 :   gimple_seq_add_seq (body_p, thr02_list);
   11722          185 :   g = gimple_build_goto (lab3);
   11723          185 :   gimple_seq_add_stmt (body_p, g);
   11724          185 :   g = gimple_build_label (lab2);
   11725          185 :   gimple_seq_add_stmt (body_p, g);
   11726          185 :   gimple_seq_add_seq (body_p, thrn2_list);
   11727          185 :   g = gimple_build_label (lab3);
   11728          185 :   gimple_seq_add_stmt (body_p, g);
   11729              : 
   11730          185 :   g = gimple_build_assign (ivar, size_zero_node);
   11731          185 :   gimple_seq_add_stmt (body_p, g);
   11732          185 :   gimple_seq_add_stmt (body_p, new_stmt);
   11733          185 :   gimple_seq_add_seq (body_p, new_body);
   11734              : 
   11735          185 :   gimple_seq new_dlist = NULL;
   11736          185 :   lab1 = create_artificial_label (UNKNOWN_LOCATION);
   11737          185 :   lab2 = create_artificial_label (UNKNOWN_LOCATION);
   11738          185 :   tree num_threadsm1 = create_tmp_var (integer_type_node);
   11739          185 :   g = gimple_build_assign (num_threadsm1, PLUS_EXPR, num_threads,
   11740              :                            integer_minus_one_node);
   11741          185 :   gimple_seq_add_stmt (&new_dlist, g);
   11742          185 :   g = gimple_build_cond (EQ_EXPR, thread_num, num_threadsm1, lab1, lab2);
   11743          185 :   gimple_seq_add_stmt (&new_dlist, g);
   11744          185 :   g = gimple_build_label (lab1);
   11745          185 :   gimple_seq_add_stmt (&new_dlist, g);
   11746          185 :   gimple_seq_add_seq (&new_dlist, last_list);
   11747          185 :   g = gimple_build_label (lab2);
   11748          185 :   gimple_seq_add_stmt (&new_dlist, g);
   11749          185 :   gimple_seq_add_seq (&new_dlist, *dlist);
   11750          185 :   *dlist = new_dlist;
   11751          185 : }
   11752              : 
   11753              : /* Build an internal UNIQUE function with type IFN_UNIQUE_OACC_PRIVATE listing
   11754              :    the addresses of variables to be made private at the surrounding
   11755              :    parallelism level.  Such functions appear in the gimple code stream in two
   11756              :    forms, e.g. for a partitioned loop:
   11757              : 
   11758              :       .data_dep.6 = .UNIQUE (OACC_HEAD_MARK, .data_dep.6, 1, 68);
   11759              :       .data_dep.6 = .UNIQUE (OACC_PRIVATE, .data_dep.6, -1, &w);
   11760              :       .data_dep.6 = .UNIQUE (OACC_FORK, .data_dep.6, -1);
   11761              :       .data_dep.6 = .UNIQUE (OACC_HEAD_MARK, .data_dep.6);
   11762              : 
   11763              :    or alternatively, OACC_PRIVATE can appear at the top level of a parallel,
   11764              :    not as part of a HEAD_MARK sequence:
   11765              : 
   11766              :       .UNIQUE (OACC_PRIVATE, 0, 0, &w);
   11767              : 
   11768              :    For such stand-alone appearances, the 3rd argument is always 0, denoting
   11769              :    gang partitioning.  */
   11770              : 
   11771              : static gcall *
   11772        19728 : lower_oacc_private_marker (omp_context *ctx)
   11773              : {
   11774        19728 :   if (ctx->oacc_privatization_candidates.length () == 0)
   11775              :     return NULL;
   11776              : 
   11777          308 :   auto_vec<tree, 5> args;
   11778              : 
   11779          308 :   args.quick_push (build_int_cst (integer_type_node, IFN_UNIQUE_OACC_PRIVATE));
   11780          308 :   args.quick_push (integer_zero_node);
   11781          308 :   args.quick_push (integer_minus_one_node);
   11782              : 
   11783          308 :   int i;
   11784          308 :   tree decl;
   11785         1004 :   FOR_EACH_VEC_ELT (ctx->oacc_privatization_candidates, i, decl)
   11786              :     {
   11787          388 :       gcc_checking_assert (TREE_ADDRESSABLE (decl));
   11788          388 :       tree addr = build_fold_addr_expr (decl);
   11789          388 :       args.safe_push (addr);
   11790              :     }
   11791              : 
   11792          308 :   return gimple_build_call_internal_vec (IFN_UNIQUE, args);
   11793          308 : }
   11794              : 
   11795              : /* Lower code for an OMP loop directive.  */
   11796              : 
   11797              : static void
   11798        47102 : lower_omp_for (gimple_stmt_iterator *gsi_p, omp_context *ctx)
   11799              : {
   11800        47102 :   tree *rhs_p, block;
   11801        47102 :   struct omp_for_data fd, *fdp = NULL;
   11802        47102 :   gomp_for *stmt = as_a <gomp_for *> (gsi_stmt (*gsi_p));
   11803        47102 :   gbind *new_stmt;
   11804        47102 :   gimple_seq omp_for_body, body, dlist, tred_ilist = NULL, tred_dlist = NULL;
   11805        47102 :   gimple_seq cnt_list = NULL, clist = NULL;
   11806        47102 :   gimple_seq oacc_head = NULL, oacc_tail = NULL;
   11807        47102 :   size_t i;
   11808              : 
   11809        47102 :   push_gimplify_context ();
   11810              : 
   11811        47102 :   if (is_gimple_omp_oacc (ctx->stmt))
   11812        11325 :     oacc_privatization_scan_clause_chain (ctx, gimple_omp_for_clauses (stmt));
   11813              : 
   11814        47102 :   lower_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
   11815              : 
   11816        47102 :   block = make_node (BLOCK);
   11817        47102 :   new_stmt = gimple_build_bind (NULL, NULL, block);
   11818              :   /* Replace at gsi right away, so that 'stmt' is no member
   11819              :      of a sequence anymore as we're going to add to a different
   11820              :      one below.  */
   11821        47102 :   gsi_replace (gsi_p, new_stmt, true);
   11822              : 
   11823              :   /* Move declaration of temporaries in the loop body before we make
   11824              :      it go away.  */
   11825        47102 :   omp_for_body = gimple_omp_body (stmt);
   11826        47102 :   if (!gimple_seq_empty_p (omp_for_body)
   11827        47102 :       && gimple_code (gimple_seq_first_stmt (omp_for_body)) == GIMPLE_BIND)
   11828              :     {
   11829        19656 :       gbind *inner_bind
   11830        19656 :         = as_a <gbind *> (gimple_seq_first_stmt (omp_for_body));
   11831        19656 :       tree vars = gimple_bind_vars (inner_bind);
   11832        19656 :       if (is_gimple_omp_oacc (ctx->stmt))
   11833         5305 :         oacc_privatization_scan_decl_chain (ctx, vars);
   11834        19656 :       gimple_bind_append_vars (new_stmt, vars);
   11835              :       /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
   11836              :          keep them on the inner_bind and it's block.  */
   11837        19656 :       gimple_bind_set_vars (inner_bind, NULL_TREE);
   11838        19656 :       if (gimple_bind_block (inner_bind))
   11839        17087 :         BLOCK_VARS (gimple_bind_block (inner_bind)) = NULL_TREE;
   11840              :     }
   11841              : 
   11842        47102 :   if (gimple_omp_for_combined_into_p (stmt))
   11843              :     {
   11844        14656 :       omp_extract_for_data (stmt, &fd, NULL);
   11845        14656 :       fdp = &fd;
   11846              : 
   11847              :       /* We need two temporaries with fd.loop.v type (istart/iend)
   11848              :          and then (fd.collapse - 1) temporaries with the same
   11849              :          type for count2 ... countN-1 vars if not constant.  */
   11850        14656 :       size_t count = 2;
   11851        14656 :       tree type = fd.iter_type;
   11852        14656 :       if (fd.collapse > 1
   11853         4867 :           && TREE_CODE (fd.loop.n2) != INTEGER_CST)
   11854         2692 :         count += fd.collapse - 1;
   11855        14656 :       size_t count2 = 0;
   11856        14656 :       tree type2 = NULL_TREE;
   11857        14656 :       bool taskreg_for
   11858        14656 :         = (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR
   11859        14656 :            || gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_TASKLOOP);
   11860        14656 :       tree outerc = NULL, *pc = gimple_omp_for_clauses_ptr (stmt);
   11861        14656 :       tree simtc = NULL;
   11862        14656 :       tree clauses = *pc;
   11863        14656 :       if (fd.collapse > 1
   11864         4867 :           && fd.non_rect
   11865          137 :           && fd.last_nonrect == fd.first_nonrect + 1
   11866           89 :           && TREE_CODE (fd.loop.n2) != INTEGER_CST)
   11867           64 :         if (tree v = gimple_omp_for_index (stmt, fd.last_nonrect))
   11868           64 :           if (!TYPE_UNSIGNED (TREE_TYPE (v)))
   11869              :             {
   11870           60 :               v = gimple_omp_for_index (stmt, fd.first_nonrect);
   11871           60 :               type2 = TREE_TYPE (v);
   11872           60 :               count++;
   11873           60 :               count2 = 3;
   11874              :             }
   11875        14656 :       if (taskreg_for)
   11876         7634 :         outerc
   11877         7634 :           = omp_find_clause (gimple_omp_taskreg_clauses (ctx->outer->stmt),
   11878              :                              OMP_CLAUSE__LOOPTEMP_);
   11879        14656 :       if (ctx->simt_stmt)
   11880            0 :         simtc = omp_find_clause (gimple_omp_for_clauses (ctx->simt_stmt),
   11881              :                                  OMP_CLAUSE__LOOPTEMP_);
   11882        49377 :       for (i = 0; i < count + count2; i++)
   11883              :         {
   11884        34721 :           tree temp;
   11885        34721 :           if (taskreg_for)
   11886              :             {
   11887        18149 :               gcc_assert (outerc);
   11888        18149 :               temp = lookup_decl (OMP_CLAUSE_DECL (outerc), ctx->outer);
   11889        18149 :               outerc = omp_find_clause (OMP_CLAUSE_CHAIN (outerc),
   11890              :                                         OMP_CLAUSE__LOOPTEMP_);
   11891              :             }
   11892              :           else
   11893              :             {
   11894              :               /* If there are 2 adjacent SIMD stmts, one with _simt_
   11895              :                  clause, another without, make sure they have the same
   11896              :                  decls in _looptemp_ clauses, because the outer stmt
   11897              :                  they are combined into will look up just one inner_stmt.  */
   11898        16572 :               if (ctx->simt_stmt)
   11899            0 :                 temp = OMP_CLAUSE_DECL (simtc);
   11900              :               else
   11901        33000 :                 temp = create_tmp_var (i >= count ? type2 : type);
   11902        16572 :               insert_decl_map (&ctx->outer->cb, temp, temp);
   11903              :             }
   11904        34721 :           *pc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
   11905        34721 :           OMP_CLAUSE_DECL (*pc) = temp;
   11906        34721 :           pc = &OMP_CLAUSE_CHAIN (*pc);
   11907        34721 :           if (ctx->simt_stmt)
   11908            0 :             simtc = omp_find_clause (OMP_CLAUSE_CHAIN (simtc),
   11909              :                                      OMP_CLAUSE__LOOPTEMP_);
   11910              :         }
   11911        14656 :       *pc = clauses;
   11912              :     }
   11913              : 
   11914              :   /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR.  */
   11915        47102 :   dlist = NULL;
   11916        47102 :   body = NULL;
   11917        47102 :   tree rclauses
   11918        47102 :     = omp_task_reductions_find_first (gimple_omp_for_clauses (stmt), OMP_FOR,
   11919              :                                       OMP_CLAUSE_REDUCTION);
   11920        47102 :   tree rtmp = NULL_TREE;
   11921        47102 :   if (rclauses)
   11922              :     {
   11923          228 :       tree type = build_pointer_type (pointer_sized_int_node);
   11924          228 :       tree temp = create_tmp_var (type);
   11925          228 :       tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
   11926          228 :       OMP_CLAUSE_DECL (c) = temp;
   11927          228 :       OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (stmt);
   11928          228 :       gimple_omp_for_set_clauses (stmt, c);
   11929          228 :       lower_omp_task_reductions (ctx, OMP_FOR,
   11930              :                                  gimple_omp_for_clauses (stmt),
   11931              :                                  &tred_ilist, &tred_dlist);
   11932          228 :       rclauses = c;
   11933          228 :       rtmp = make_ssa_name (type);
   11934          228 :       gimple_seq_add_stmt (&body, gimple_build_assign (rtmp, temp));
   11935              :     }
   11936              : 
   11937        47102 :   lower_lastprivate_conditional_clauses (gimple_omp_for_clauses_ptr (stmt),
   11938              :                                          ctx);
   11939              : 
   11940        47102 :   lower_rec_input_clauses (gimple_omp_for_clauses (stmt), &body, &dlist, ctx,
   11941              :                            fdp);
   11942        93976 :   gimple_seq_add_seq (rclauses ? &tred_ilist : &body,
   11943              :                       gimple_omp_for_pre_body (stmt));
   11944              : 
   11945        47102 :   lower_omp (gimple_omp_body_ptr (stmt), ctx);
   11946              : 
   11947        47102 :   gcall *private_marker = NULL;
   11948        47102 :   if (is_gimple_omp_oacc (ctx->stmt)
   11949        47102 :       && !gimple_seq_empty_p (omp_for_body))
   11950        10333 :     private_marker = lower_oacc_private_marker (ctx);
   11951              : 
   11952              :   /* Lower the header expressions.  At this point, we can assume that
   11953              :      the header is of the form:
   11954              : 
   11955              :         #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
   11956              : 
   11957              :      We just need to make sure that VAL1, VAL2 and VAL3 are lowered
   11958              :      using the .omp_data_s mapping, if needed.  */
   11959       115488 :   for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
   11960              :     {
   11961        68386 :       rhs_p = gimple_omp_for_initial_ptr (stmt, i);
   11962        68386 :       if (TREE_CODE (*rhs_p) == TREE_VEC)
   11963              :         {
   11964          639 :           if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 1)))
   11965          149 :             TREE_VEC_ELT (*rhs_p, 1)
   11966          298 :               = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 1), &cnt_list);
   11967          639 :           if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 2)))
   11968          161 :             TREE_VEC_ELT (*rhs_p, 2)
   11969          322 :               = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 2), &cnt_list);
   11970              :         }
   11971        67747 :       else if (!is_gimple_min_invariant (*rhs_p))
   11972        11465 :         *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
   11973        56282 :       else if (TREE_CODE (*rhs_p) == ADDR_EXPR)
   11974         4713 :         recompute_tree_invariant_for_addr_expr (*rhs_p);
   11975              : 
   11976        68386 :       rhs_p = gimple_omp_for_final_ptr (stmt, i);
   11977        68386 :       if (TREE_CODE (*rhs_p) == TREE_VEC)
   11978              :         {
   11979          519 :           if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 1)))
   11980          142 :             TREE_VEC_ELT (*rhs_p, 1)
   11981          284 :               = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 1), &cnt_list);
   11982          519 :           if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 2)))
   11983          157 :             TREE_VEC_ELT (*rhs_p, 2)
   11984          314 :               = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 2), &cnt_list);
   11985              :         }
   11986        67867 :       else if (!is_gimple_min_invariant (*rhs_p))
   11987        16893 :         *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
   11988        50974 :       else if (TREE_CODE (*rhs_p) == ADDR_EXPR)
   11989         4630 :         recompute_tree_invariant_for_addr_expr (*rhs_p);
   11990              : 
   11991        68386 :       rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1);
   11992        68386 :       if (!is_gimple_min_invariant (*rhs_p))
   11993         6318 :         *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
   11994              :     }
   11995        47102 :   if (rclauses)
   11996          228 :     gimple_seq_add_seq (&tred_ilist, cnt_list);
   11997              :   else
   11998        46874 :     gimple_seq_add_seq (&body, cnt_list);
   11999              : 
   12000              :   /* Once lowered, extract the bounds and clauses.  */
   12001        47102 :   omp_extract_for_data (stmt, &fd, NULL);
   12002              : 
   12003        47102 :   if (is_gimple_omp_oacc (ctx->stmt)
   12004        47102 :       && !ctx_in_oacc_kernels_region (ctx))
   12005         9665 :     lower_oacc_head_tail (gimple_location (stmt),
   12006              :                           gimple_omp_for_clauses (stmt), private_marker,
   12007              :                           &oacc_head, &oacc_tail, ctx);
   12008              : 
   12009              :   /* Add OpenACC partitioning and reduction markers just before the loop.  */
   12010        47102 :   if (oacc_head)
   12011         9665 :     gimple_seq_add_seq (&body, oacc_head);
   12012              : 
   12013        47102 :   lower_omp_for_lastprivate (&fd, &body, &dlist, &clist, ctx);
   12014              : 
   12015        47102 :   if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
   12016        86180 :     for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
   12017        70509 :       if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
   12018        70509 :           && !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
   12019              :         {
   12020          227 :           OMP_CLAUSE_DECL (c) = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
   12021          227 :           if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c)))
   12022           42 :             OMP_CLAUSE_LINEAR_STEP (c)
   12023           84 :               = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c),
   12024              :                                                 ctx);
   12025              :         }
   12026              : 
   12027        46708 :   if ((ctx->scan_inclusive || ctx->scan_exclusive)
   12028        47370 :       && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
   12029          185 :     lower_omp_for_scan (&body, &dlist, stmt, &fd, ctx);
   12030              :   else
   12031              :     {
   12032        46917 :       gimple_seq_add_stmt (&body, stmt);
   12033        46917 :       gimple_seq_add_seq (&body, gimple_omp_body (stmt));
   12034              :     }
   12035              : 
   12036        47102 :   gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v,
   12037              :                                                          fd.loop.v));
   12038              : 
   12039              :   /* After the loop, add exit clauses.  */
   12040        47102 :   lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, &clist, ctx);
   12041              : 
   12042        47102 :   if (clist)
   12043              :     {
   12044          179 :       tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_START);
   12045          179 :       gcall *g = gimple_build_call (fndecl, 0);
   12046          179 :       gimple_seq_add_stmt (&body, g);
   12047          179 :       gimple_seq_add_seq (&body, clist);
   12048          179 :       fndecl = builtin_decl_explicit (BUILT_IN_GOMP_REDUCTION_END);
   12049          179 :       g = gimple_build_call (fndecl, 0);
   12050          179 :       gimple_seq_add_stmt (&body, g);
   12051              :     }
   12052              : 
   12053        47102 :   if (ctx->cancellable)
   12054           84 :     gimple_seq_add_stmt (&body, gimple_build_label (ctx->cancel_label));
   12055              : 
   12056        47102 :   gimple_seq_add_seq (&body, dlist);
   12057              : 
   12058        47102 :   if (rclauses)
   12059              :     {
   12060          228 :       gimple_seq_add_seq (&tred_ilist, body);
   12061          228 :       body = tred_ilist;
   12062              :     }
   12063              : 
   12064        47102 :   body = maybe_catch_exception (body);
   12065              : 
   12066              :   /* Region exit marker goes at the end of the loop body.  */
   12067        47102 :   gimple *g = gimple_build_omp_return (fd.have_nowait);
   12068        47102 :   gimple_seq_add_stmt (&body, g);
   12069              : 
   12070        47102 :   gimple_seq_add_seq (&body, tred_dlist);
   12071              : 
   12072        47102 :   maybe_add_implicit_barrier_cancel (ctx, g, &body);
   12073              : 
   12074        47102 :   if (rclauses)
   12075          228 :     OMP_CLAUSE_DECL (rclauses) = rtmp;
   12076              : 
   12077              :   /* Add OpenACC joining and reduction markers just after the loop.  */
   12078        47102 :   if (oacc_tail)
   12079         9665 :     gimple_seq_add_seq (&body, oacc_tail);
   12080              : 
   12081        47102 :   pop_gimplify_context (new_stmt);
   12082              : 
   12083        47102 :   gimple_bind_append_vars (new_stmt, ctx->block_vars);
   12084        47102 :   maybe_remove_omp_member_access_dummy_vars (new_stmt);
   12085        47102 :   BLOCK_VARS (block) = gimple_bind_vars (new_stmt);
   12086        47102 :   if (BLOCK_VARS (block))
   12087        46453 :     TREE_USED (block) = 1;
   12088              : 
   12089        47102 :   gimple_bind_set_body (new_stmt, body);
   12090        47102 :   gimple_omp_set_body (stmt, NULL);
   12091        47102 :   gimple_omp_for_set_pre_body (stmt, NULL);
   12092        47102 : }
   12093              : 
   12094              : /* Callback for walk_stmts.  Check if the current statement only contains
   12095              :    GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS.  */
   12096              : 
   12097              : static tree
   12098       217552 : check_combined_parallel (gimple_stmt_iterator *gsi_p,
   12099              :                          bool *handled_ops_p,
   12100              :                          struct walk_stmt_info *wi)
   12101              : {
   12102       217552 :   int *info = (int *) wi->info;
   12103       217552 :   gimple *stmt = gsi_stmt (*gsi_p);
   12104              : 
   12105       217552 :   *handled_ops_p = true;
   12106       217552 :   switch (gimple_code (stmt))
   12107              :     {
   12108        12030 :     WALK_SUBSTMTS;
   12109              : 
   12110              :     case GIMPLE_DEBUG:
   12111              :       break;
   12112         1914 :     case GIMPLE_OMP_FOR:
   12113         1914 :     case GIMPLE_OMP_SECTIONS:
   12114         1914 :       *info = *info == 0 ? 1 : -1;
   12115         1914 :       break;
   12116       203569 :     default:
   12117       203569 :       *info = -1;
   12118       203569 :       break;
   12119              :     }
   12120       217552 :   return NULL;
   12121              : }
   12122              : 
   12123              : struct omp_taskcopy_context
   12124              : {
   12125              :   /* This field must be at the beginning, as we do "inheritance": Some
   12126              :      callback functions for tree-inline.cc (e.g., omp_copy_decl)
   12127              :      receive a copy_body_data pointer that is up-casted to an
   12128              :      omp_context pointer.  */
   12129              :   copy_body_data cb;
   12130              :   omp_context *ctx;
   12131              : };
   12132              : 
   12133              : static tree
   12134          366 : task_copyfn_copy_decl (tree var, copy_body_data *cb)
   12135              : {
   12136          366 :   struct omp_taskcopy_context *tcctx = (struct omp_taskcopy_context *) cb;
   12137              : 
   12138          366 :   if (splay_tree_lookup (tcctx->ctx->sfield_map, (splay_tree_key) var))
   12139          158 :     return create_tmp_var (TREE_TYPE (var));
   12140              : 
   12141              :   return var;
   12142              : }
   12143              : 
   12144              : static tree
   12145           50 : task_copyfn_remap_type (struct omp_taskcopy_context *tcctx, tree orig_type)
   12146              : {
   12147           50 :   tree name, new_fields = NULL, type, f;
   12148              : 
   12149           50 :   type = lang_hooks.types.make_type (RECORD_TYPE);
   12150           50 :   name = DECL_NAME (TYPE_NAME (orig_type));
   12151           50 :   name = build_decl (gimple_location (tcctx->ctx->stmt),
   12152              :                      TYPE_DECL, name, type);
   12153           50 :   TYPE_NAME (type) = name;
   12154              : 
   12155         1140 :   for (f = TYPE_FIELDS (orig_type); f ; f = TREE_CHAIN (f))
   12156              :     {
   12157         1090 :       tree new_f = copy_node (f);
   12158         1090 :       DECL_CONTEXT (new_f) = type;
   12159         1090 :       TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &tcctx->cb);
   12160         1090 :       TREE_CHAIN (new_f) = new_fields;
   12161         1090 :       walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &tcctx->cb, NULL);
   12162         1090 :       walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, &tcctx->cb, NULL);
   12163         1090 :       walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
   12164              :                  &tcctx->cb, NULL);
   12165         1090 :       new_fields = new_f;
   12166         1090 :       tcctx->cb.decl_map->put (f, new_f);
   12167              :     }
   12168           50 :   TYPE_FIELDS (type) = nreverse (new_fields);
   12169           50 :   layout_type (type);
   12170           50 :   return type;
   12171              : }
   12172              : 
   12173              : /* Create task copyfn.  */
   12174              : 
   12175              : static void
   12176          513 : create_task_copyfn (gomp_task *task_stmt, omp_context *ctx)
   12177              : {
   12178          513 :   struct function *child_cfun;
   12179          513 :   tree child_fn, t, c, src, dst, f, sf, arg, sarg, decl;
   12180          513 :   tree record_type, srecord_type, bind, list;
   12181          513 :   bool record_needs_remap = false, srecord_needs_remap = false;
   12182          513 :   splay_tree_node n;
   12183          513 :   struct omp_taskcopy_context tcctx;
   12184          513 :   location_t loc = gimple_location (task_stmt);
   12185          513 :   size_t looptempno = 0;
   12186              : 
   12187          513 :   child_fn = gimple_omp_task_copy_fn (task_stmt);
   12188          513 :   task_cpyfns.safe_push (task_stmt);
   12189          513 :   child_cfun = DECL_STRUCT_FUNCTION (child_fn);
   12190          513 :   gcc_assert (child_cfun->cfg == NULL);
   12191          513 :   DECL_SAVED_TREE (child_fn) = alloc_stmt_list ();
   12192              : 
   12193              :   /* Reset DECL_CONTEXT on function arguments.  */
   12194         1539 :   for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
   12195         1026 :     DECL_CONTEXT (t) = child_fn;
   12196              : 
   12197              :   /* Populate the function.  */
   12198          513 :   push_gimplify_context ();
   12199          513 :   push_cfun (child_cfun);
   12200              : 
   12201          513 :   bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
   12202          513 :   TREE_SIDE_EFFECTS (bind) = 1;
   12203          513 :   list = NULL;
   12204          513 :   DECL_SAVED_TREE (child_fn) = bind;
   12205          513 :   DECL_SOURCE_LOCATION (child_fn) = gimple_location (task_stmt);
   12206              : 
   12207              :   /* Remap src and dst argument types if needed.  */
   12208          513 :   record_type = ctx->record_type;
   12209          513 :   srecord_type = ctx->srecord_type;
   12210         3132 :   for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
   12211         2644 :     if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
   12212              :       {
   12213              :         record_needs_remap = true;
   12214              :         break;
   12215              :       }
   12216         2871 :   for (f = TYPE_FIELDS (srecord_type); f ; f = DECL_CHAIN (f))
   12217         2383 :     if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
   12218              :       {
   12219              :         srecord_needs_remap = true;
   12220              :         break;
   12221              :       }
   12222              : 
   12223          513 :   if (record_needs_remap || srecord_needs_remap)
   12224              :     {
   12225           25 :       memset (&tcctx, '\0', sizeof (tcctx));
   12226           25 :       tcctx.cb.src_fn = ctx->cb.src_fn;
   12227           25 :       tcctx.cb.dst_fn = child_fn;
   12228           25 :       tcctx.cb.src_node = cgraph_node::get (tcctx.cb.src_fn);
   12229           25 :       gcc_checking_assert (tcctx.cb.src_node);
   12230           25 :       tcctx.cb.dst_node = tcctx.cb.src_node;
   12231           25 :       tcctx.cb.src_cfun = ctx->cb.src_cfun;
   12232           25 :       tcctx.cb.copy_decl = task_copyfn_copy_decl;
   12233           25 :       tcctx.cb.eh_lp_nr = 0;
   12234           25 :       tcctx.cb.transform_call_graph_edges = CB_CGE_MOVE;
   12235           25 :       tcctx.cb.decl_map = new hash_map<tree, tree>;
   12236           25 :       tcctx.ctx = ctx;
   12237              : 
   12238           25 :       if (record_needs_remap)
   12239           25 :         record_type = task_copyfn_remap_type (&tcctx, record_type);
   12240           25 :       if (srecord_needs_remap)
   12241           25 :         srecord_type = task_copyfn_remap_type (&tcctx, srecord_type);
   12242              :     }
   12243              :   else
   12244          488 :     tcctx.cb.decl_map = NULL;
   12245              : 
   12246          513 :   arg = DECL_ARGUMENTS (child_fn);
   12247          513 :   TREE_TYPE (arg) = build_pointer_type (record_type);
   12248          513 :   sarg = DECL_CHAIN (arg);
   12249          513 :   TREE_TYPE (sarg) = build_pointer_type (srecord_type);
   12250              : 
   12251              :   /* First pass: initialize temporaries used in record_type and srecord_type
   12252              :      sizes and field offsets.  */
   12253          513 :   if (tcctx.cb.decl_map)
   12254          631 :     for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
   12255          606 :       if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
   12256              :         {
   12257          497 :           tree *p;
   12258              : 
   12259          497 :           decl = OMP_CLAUSE_DECL (c);
   12260          497 :           p = tcctx.cb.decl_map->get (decl);
   12261          497 :           if (p == NULL)
   12262          339 :             continue;
   12263          158 :           n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
   12264          158 :           sf = (tree) n->value;
   12265          158 :           sf = *tcctx.cb.decl_map->get (sf);
   12266          158 :           src = build_simple_mem_ref_loc (loc, sarg);
   12267          158 :           src = omp_build_component_ref (src, sf);
   12268          158 :           t = build2 (MODIFY_EXPR, TREE_TYPE (*p), *p, src);
   12269          158 :           append_to_statement_list (t, &list);
   12270              :         }
   12271              : 
   12272              :   /* Second pass: copy shared var pointers and copy construct non-VLA
   12273              :      firstprivate vars.  */
   12274         5895 :   for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
   12275         5382 :     switch (OMP_CLAUSE_CODE (c))
   12276              :       {
   12277          583 :         splay_tree_key key;
   12278          583 :       case OMP_CLAUSE_SHARED:
   12279          583 :         decl = OMP_CLAUSE_DECL (c);
   12280          583 :         key = (splay_tree_key) decl;
   12281          583 :         if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
   12282          273 :           key = (splay_tree_key) &DECL_UID (decl);
   12283          583 :         n = splay_tree_lookup (ctx->field_map, key);
   12284          583 :         if (n == NULL)
   12285              :           break;
   12286          149 :         f = (tree) n->value;
   12287          149 :         if (tcctx.cb.decl_map)
   12288           12 :           f = *tcctx.cb.decl_map->get (f);
   12289          149 :         n = splay_tree_lookup (ctx->sfield_map, key);
   12290          149 :         sf = (tree) n->value;
   12291          149 :         if (tcctx.cb.decl_map)
   12292           12 :           sf = *tcctx.cb.decl_map->get (sf);
   12293          149 :         src = build_simple_mem_ref_loc (loc, sarg);
   12294          149 :         src = omp_build_component_ref (src, sf);
   12295          149 :         dst = build_simple_mem_ref_loc (loc, arg);
   12296          149 :         dst = omp_build_component_ref (dst, f);
   12297          149 :         t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
   12298          149 :         append_to_statement_list (t, &list);
   12299          149 :         break;
   12300          460 :       case OMP_CLAUSE_REDUCTION:
   12301          460 :       case OMP_CLAUSE_IN_REDUCTION:
   12302          460 :         decl = OMP_CLAUSE_DECL (c);
   12303          460 :         if (TREE_CODE (decl) == MEM_REF)
   12304              :           {
   12305          160 :             decl = TREE_OPERAND (decl, 0);
   12306          160 :             if (TREE_CODE (decl) == POINTER_PLUS_EXPR)
   12307           38 :               decl = TREE_OPERAND (decl, 0);
   12308          160 :             if (TREE_CODE (decl) == INDIRECT_REF
   12309          128 :                 || TREE_CODE (decl) == ADDR_EXPR)
   12310           95 :               decl = TREE_OPERAND (decl, 0);
   12311              :           }
   12312          460 :         key = (splay_tree_key) decl;
   12313          460 :         n = splay_tree_lookup (ctx->field_map, key);
   12314          460 :         if (n == NULL)
   12315              :           break;
   12316          101 :         f = (tree) n->value;
   12317          101 :         if (tcctx.cb.decl_map)
   12318           20 :           f = *tcctx.cb.decl_map->get (f);
   12319          101 :         n = splay_tree_lookup (ctx->sfield_map, key);
   12320          101 :         sf = (tree) n->value;
   12321          101 :         if (tcctx.cb.decl_map)
   12322           20 :           sf = *tcctx.cb.decl_map->get (sf);
   12323          101 :         src = build_simple_mem_ref_loc (loc, sarg);
   12324          101 :         src = omp_build_component_ref (src, sf);
   12325          101 :         if (decl != OMP_CLAUSE_DECL (c)
   12326           95 :             && TREE_CODE (TREE_TYPE (decl)) == REFERENCE_TYPE
   12327          173 :             && TREE_CODE (TREE_TYPE (TREE_TYPE (decl))) == POINTER_TYPE)
   12328           32 :           src = build_simple_mem_ref_loc (loc, src);
   12329          101 :         dst = build_simple_mem_ref_loc (loc, arg);
   12330          101 :         dst = omp_build_component_ref (dst, f);
   12331          101 :         t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
   12332          101 :         append_to_statement_list (t, &list);
   12333          101 :         break;
   12334          766 :       case OMP_CLAUSE__LOOPTEMP_:
   12335              :         /* Fields for first two _looptemp_ clauses are initialized by
   12336              :            GOMP_taskloop*, the rest are handled like firstprivate.  */
   12337          766 :         if (looptempno < 2)
   12338              :           {
   12339          760 :             looptempno++;
   12340          760 :             break;
   12341              :           }
   12342              :         /* FALLTHRU */
   12343         1740 :       case OMP_CLAUSE__REDUCTEMP_:
   12344         1740 :       case OMP_CLAUSE_FIRSTPRIVATE:
   12345         1740 :         decl = OMP_CLAUSE_DECL (c);
   12346         1740 :         if (is_variable_sized (decl))
   12347              :           break;
   12348         1726 :         n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
   12349         1726 :         if (n == NULL)
   12350              :           break;
   12351         1726 :         f = (tree) n->value;
   12352         1726 :         if (tcctx.cb.decl_map)
   12353          490 :           f = *tcctx.cb.decl_map->get (f);
   12354         1726 :         n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
   12355         1726 :         if (n != NULL)
   12356              :           {
   12357         1476 :             sf = (tree) n->value;
   12358         1476 :             if (tcctx.cb.decl_map)
   12359          490 :               sf = *tcctx.cb.decl_map->get (sf);
   12360         1476 :             src = build_simple_mem_ref_loc (loc, sarg);
   12361         1476 :             src = omp_build_component_ref (src, sf);
   12362         1476 :             if (use_pointer_for_field (decl, NULL)
   12363         1476 :                 || omp_privatize_by_reference (decl))
   12364          326 :               src = build_simple_mem_ref_loc (loc, src);
   12365              :           }
   12366              :         else
   12367              :           src = decl;
   12368         1726 :         dst = build_simple_mem_ref_loc (loc, arg);
   12369         1726 :         dst = omp_build_component_ref (dst, f);
   12370         1726 :         if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FIRSTPRIVATE)
   12371          192 :           t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
   12372              :         else
   12373              :           {
   12374         1534 :             if (ctx->allocate_map)
   12375           93 :               if (tree *allocatorp = ctx->allocate_map->get (decl))
   12376              :                 {
   12377           59 :                   tree allocator = *allocatorp;
   12378           59 :                   HOST_WIDE_INT ialign = 0;
   12379           59 :                   if (TREE_CODE (allocator) == TREE_LIST)
   12380              :                     {
   12381            4 :                       ialign = tree_to_uhwi (TREE_VALUE (allocator));
   12382            4 :                       allocator = TREE_PURPOSE (allocator);
   12383              :                     }
   12384           59 :                   if (TREE_CODE (allocator) != INTEGER_CST)
   12385              :                     {
   12386           22 :                       n = splay_tree_lookup (ctx->sfield_map,
   12387              :                                              (splay_tree_key) allocator);
   12388           22 :                       allocator = (tree) n->value;
   12389           22 :                       if (tcctx.cb.decl_map)
   12390            0 :                         allocator = *tcctx.cb.decl_map->get (allocator);
   12391           22 :                       tree a = build_simple_mem_ref_loc (loc, sarg);
   12392           22 :                       allocator = omp_build_component_ref (a, allocator);
   12393              :                     }
   12394           59 :                   allocator = fold_convert (pointer_sized_int_node, allocator);
   12395           59 :                   tree a = builtin_decl_explicit (BUILT_IN_GOMP_ALLOC);
   12396           59 :                   tree align = build_int_cst (size_type_node,
   12397           59 :                                               MAX (ialign,
   12398              :                                                    DECL_ALIGN_UNIT (decl)));
   12399           59 :                   tree sz = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (dst)));
   12400           59 :                   tree ptr = build_call_expr_loc (loc, a, 3, align, sz,
   12401              :                                                   allocator);
   12402           59 :                   ptr = fold_convert (TREE_TYPE (dst), ptr);
   12403           59 :                   t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, ptr);
   12404           59 :                   append_to_statement_list (t, &list);
   12405           59 :                   dst = build_simple_mem_ref_loc (loc, dst);
   12406              :                 }
   12407         1534 :             t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
   12408              :           }
   12409         1726 :         append_to_statement_list (t, &list);
   12410         1726 :         break;
   12411          495 :       case OMP_CLAUSE_PRIVATE:
   12412          495 :         if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c))
   12413              :           break;
   12414           12 :         decl = OMP_CLAUSE_DECL (c);
   12415           12 :         n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
   12416           12 :         f = (tree) n->value;
   12417           12 :         if (tcctx.cb.decl_map)
   12418            0 :           f = *tcctx.cb.decl_map->get (f);
   12419           12 :         n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
   12420           12 :         if (n != NULL)
   12421              :           {
   12422           12 :             sf = (tree) n->value;
   12423           12 :             if (tcctx.cb.decl_map)
   12424            0 :               sf = *tcctx.cb.decl_map->get (sf);
   12425           12 :             src = build_simple_mem_ref_loc (loc, sarg);
   12426           12 :             src = omp_build_component_ref (src, sf);
   12427           12 :             if (use_pointer_for_field (decl, NULL))
   12428           12 :               src = build_simple_mem_ref_loc (loc, src);
   12429              :           }
   12430              :         else
   12431              :           src = decl;
   12432           12 :         dst = build_simple_mem_ref_loc (loc, arg);
   12433           12 :         dst = omp_build_component_ref (dst, f);
   12434           12 :         t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
   12435           12 :         append_to_statement_list (t, &list);
   12436           12 :         break;
   12437              :       default:
   12438              :         break;
   12439              :       }
   12440              : 
   12441              :   /* Last pass: handle VLA firstprivates.  */
   12442          513 :   if (tcctx.cb.decl_map)
   12443          631 :     for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
   12444          606 :       if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
   12445              :         {
   12446          497 :           tree ind, ptr, df;
   12447              : 
   12448          497 :           decl = OMP_CLAUSE_DECL (c);
   12449          497 :           if (!is_variable_sized (decl))
   12450          483 :             continue;
   12451           14 :           n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
   12452           14 :           if (n == NULL)
   12453            0 :             continue;
   12454           14 :           f = (tree) n->value;
   12455           14 :           f = *tcctx.cb.decl_map->get (f);
   12456           14 :           gcc_assert (DECL_HAS_VALUE_EXPR_P (decl));
   12457           14 :           ind = DECL_VALUE_EXPR (decl);
   12458           14 :           gcc_assert (TREE_CODE (ind) == INDIRECT_REF);
   12459           14 :           gcc_assert (DECL_P (TREE_OPERAND (ind, 0)));
   12460           42 :           n = splay_tree_lookup (ctx->sfield_map,
   12461           14 :                                  (splay_tree_key) TREE_OPERAND (ind, 0));
   12462           14 :           sf = (tree) n->value;
   12463           14 :           sf = *tcctx.cb.decl_map->get (sf);
   12464           14 :           src = build_simple_mem_ref_loc (loc, sarg);
   12465           14 :           src = omp_build_component_ref (src, sf);
   12466           14 :           src = build_simple_mem_ref_loc (loc, src);
   12467           14 :           dst = build_simple_mem_ref_loc (loc, arg);
   12468           14 :           dst = omp_build_component_ref (dst, f);
   12469           14 :           t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
   12470           14 :           append_to_statement_list (t, &list);
   12471           42 :           n = splay_tree_lookup (ctx->field_map,
   12472           14 :                                  (splay_tree_key) TREE_OPERAND (ind, 0));
   12473           14 :           df = (tree) n->value;
   12474           14 :           df = *tcctx.cb.decl_map->get (df);
   12475           14 :           ptr = build_simple_mem_ref_loc (loc, arg);
   12476           14 :           ptr = omp_build_component_ref (ptr, df);
   12477           14 :           t = build2 (MODIFY_EXPR, TREE_TYPE (ptr), ptr,
   12478              :                       build_fold_addr_expr_loc (loc, dst));
   12479           14 :           append_to_statement_list (t, &list);
   12480              :         }
   12481              : 
   12482          513 :   t = build1 (RETURN_EXPR, void_type_node, NULL);
   12483          513 :   append_to_statement_list (t, &list);
   12484              : 
   12485          513 :   if (tcctx.cb.decl_map)
   12486           25 :     delete tcctx.cb.decl_map;
   12487          513 :   pop_gimplify_context (NULL);
   12488          513 :   BIND_EXPR_BODY (bind) = list;
   12489          513 :   pop_cfun ();
   12490          513 : }
   12491              : 
   12492              : static void
   12493         1553 : lower_depend_clauses (tree *pclauses, gimple_seq *iseq, gimple_seq *oseq)
   12494              : {
   12495         1553 :   tree c, clauses;
   12496         1553 :   gimple *g;
   12497         1553 :   size_t cnt[5] = { 0, 0, 0, 0, 0 }, idx = 2, i;
   12498              : 
   12499         1553 :   clauses = omp_find_clause (*pclauses, OMP_CLAUSE_DEPEND);
   12500         1553 :   gcc_assert (clauses);
   12501         6316 :   for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
   12502         4888 :     if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
   12503         1713 :       switch (OMP_CLAUSE_DEPEND_KIND (c))
   12504              :         {
   12505          125 :         case OMP_CLAUSE_DEPEND_LAST:
   12506              :           /* Lowering already done at gimplification.  */
   12507          125 :           return;
   12508          481 :         case OMP_CLAUSE_DEPEND_IN:
   12509          481 :           cnt[2]++;
   12510          481 :           break;
   12511          969 :         case OMP_CLAUSE_DEPEND_OUT:
   12512          969 :         case OMP_CLAUSE_DEPEND_INOUT:
   12513          969 :           cnt[0]++;
   12514          969 :           break;
   12515           30 :         case OMP_CLAUSE_DEPEND_MUTEXINOUTSET:
   12516           30 :           cnt[1]++;
   12517           30 :           break;
   12518           76 :         case OMP_CLAUSE_DEPEND_DEPOBJ:
   12519           76 :           cnt[3]++;
   12520           76 :           break;
   12521           32 :         case OMP_CLAUSE_DEPEND_INOUTSET:
   12522           32 :           cnt[4]++;
   12523           32 :           break;
   12524            0 :         default:
   12525            0 :           gcc_unreachable ();
   12526              :         }
   12527         1428 :   if (cnt[1] || cnt[3] || cnt[4])
   12528          135 :     idx = 5;
   12529         1428 :   size_t total = cnt[0] + cnt[1] + cnt[2] + cnt[3] + cnt[4];
   12530         1428 :   size_t inoutidx = total + idx;
   12531         1428 :   tree type = build_array_type_nelts (ptr_type_node, total + idx + 2 * cnt[4]);
   12532         1428 :   tree array = create_tmp_var (type);
   12533         1428 :   TREE_ADDRESSABLE (array) = 1;
   12534         1428 :   tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE,
   12535              :                    NULL_TREE);
   12536         1428 :   if (idx == 5)
   12537              :     {
   12538          135 :       g = gimple_build_assign (r, build_int_cst (ptr_type_node, 0));
   12539          135 :       gimple_seq_add_stmt (iseq, g);
   12540          135 :       r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE,
   12541              :                   NULL_TREE);
   12542              :     }
   12543         1428 :   g = gimple_build_assign (r, build_int_cst (ptr_type_node, total));
   12544         1428 :   gimple_seq_add_stmt (iseq, g);
   12545         7140 :   for (i = 0; i < (idx == 5 ? 3 : 1); i++)
   12546              :     {
   12547         1698 :       r = build4 (ARRAY_REF, ptr_type_node, array,
   12548         1698 :                   size_int (i + 1 + (idx == 5)), NULL_TREE, NULL_TREE);
   12549         1698 :       g = gimple_build_assign (r, build_int_cst (ptr_type_node, cnt[i]));
   12550         1698 :       gimple_seq_add_stmt (iseq, g);
   12551              :     }
   12552         8568 :   for (i = 0; i < 5; i++)
   12553              :     {
   12554         7140 :       if (cnt[i] == 0)
   12555         5656 :         continue;
   12556         6565 :       for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
   12557         5081 :         if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DEPEND)
   12558         3354 :           continue;
   12559              :         else
   12560              :           {
   12561         1727 :             switch (OMP_CLAUSE_DEPEND_KIND (c))
   12562              :               {
   12563          547 :               case OMP_CLAUSE_DEPEND_IN:
   12564          547 :                 if (i != 2)
   12565          139 :                   continue;
   12566              :                 break;
   12567         1025 :               case OMP_CLAUSE_DEPEND_OUT:
   12568         1025 :               case OMP_CLAUSE_DEPEND_INOUT:
   12569         1025 :                 if (i != 0)
   12570           56 :                   continue;
   12571              :                 break;
   12572           44 :               case OMP_CLAUSE_DEPEND_MUTEXINOUTSET:
   12573           44 :                 if (i != 1)
   12574           14 :                   continue;
   12575              :                 break;
   12576           79 :               case OMP_CLAUSE_DEPEND_DEPOBJ:
   12577           79 :                 if (i != 3)
   12578            3 :                   continue;
   12579              :                 break;
   12580           32 :               case OMP_CLAUSE_DEPEND_INOUTSET:
   12581           32 :                 if (i != 4)
   12582            0 :                    continue;
   12583              :                 break;
   12584            0 :               default:
   12585            0 :                 gcc_unreachable ();
   12586              :               }
   12587         1588 :             tree t = OMP_CLAUSE_DECL (c);
   12588         1588 :             if (i == 4)
   12589              :               {
   12590           32 :                 t = build4 (ARRAY_REF, ptr_type_node, array,
   12591           32 :                             size_int (inoutidx), NULL_TREE, NULL_TREE);
   12592           32 :                 t = build_fold_addr_expr (t);
   12593           32 :                 inoutidx += 2;
   12594              :               }
   12595         1588 :             t = fold_convert (ptr_type_node, t);
   12596         1588 :             gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
   12597         1588 :             r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
   12598              :                         NULL_TREE, NULL_TREE);
   12599         1588 :             g = gimple_build_assign (r, t);
   12600         1588 :             gimple_seq_add_stmt (iseq, g);
   12601              :           }
   12602              :     }
   12603         1428 :   if (cnt[4])
   12604           81 :     for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
   12605           49 :       if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
   12606           49 :           && OMP_CLAUSE_DEPEND_KIND (c) == OMP_CLAUSE_DEPEND_INOUTSET)
   12607              :         {
   12608           32 :           tree t = OMP_CLAUSE_DECL (c);
   12609           32 :           t = fold_convert (ptr_type_node, t);
   12610           32 :           gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
   12611           32 :           r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
   12612              :                       NULL_TREE, NULL_TREE);
   12613           32 :           g = gimple_build_assign (r, t);
   12614           32 :           gimple_seq_add_stmt (iseq, g);
   12615           32 :           t = build_int_cst (ptr_type_node, GOMP_DEPEND_INOUTSET);
   12616           32 :           r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
   12617              :                       NULL_TREE, NULL_TREE);
   12618           32 :           g = gimple_build_assign (r, t);
   12619           32 :           gimple_seq_add_stmt (iseq, g);
   12620              :         }
   12621              : 
   12622         1428 :   c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND);
   12623         1428 :   OMP_CLAUSE_DEPEND_KIND (c) = OMP_CLAUSE_DEPEND_LAST;
   12624         1428 :   OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array);
   12625         1428 :   OMP_CLAUSE_CHAIN (c) = *pclauses;
   12626         1428 :   *pclauses = c;
   12627         1428 :   tree clobber = build_clobber (type);
   12628         1428 :   g = gimple_build_assign (array, clobber);
   12629         1428 :   gimple_seq_add_stmt (oseq, g);
   12630              : }
   12631              : 
   12632              : /* Lower the OpenMP parallel or task directive in the current statement
   12633              :    in GSI_P.  CTX holds context information for the directive.  */
   12634              : 
   12635              : static void
   12636        22809 : lower_omp_taskreg (gimple_stmt_iterator *gsi_p, omp_context *ctx)
   12637              : {
   12638        22809 :   tree clauses;
   12639        22809 :   tree child_fn, t;
   12640        22809 :   gimple *stmt = gsi_stmt (*gsi_p);
   12641        22809 :   gbind *par_bind, *bind, *dep_bind = NULL;
   12642        22809 :   gimple_seq par_body;
   12643        22809 :   location_t loc = gimple_location (stmt);
   12644              : 
   12645        22809 :   clauses = gimple_omp_taskreg_clauses (stmt);
   12646        22809 :   if (gimple_code (stmt) == GIMPLE_OMP_TASK
   12647        22809 :       && gimple_omp_task_taskwait_p (stmt))
   12648              :     {
   12649           84 :       par_bind = NULL;
   12650           84 :       par_body = NULL;
   12651              :     }
   12652              :   else
   12653              :     {
   12654        22725 :       par_bind
   12655        22725 :         = as_a <gbind *> (gimple_seq_first_stmt (gimple_omp_body (stmt)));
   12656        22725 :       par_body = gimple_bind_body (par_bind);
   12657              :     }
   12658        22809 :   child_fn = ctx->cb.dst_fn;
   12659        22809 :   if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
   12660        22809 :       && !gimple_omp_parallel_combined_p (stmt))
   12661              :     {
   12662         4349 :       struct walk_stmt_info wi;
   12663         4349 :       int ws_num = 0;
   12664              : 
   12665         4349 :       memset (&wi, 0, sizeof (wi));
   12666         4349 :       wi.info = &ws_num;
   12667         4349 :       wi.val_only = true;
   12668         4349 :       walk_gimple_seq (par_body, check_combined_parallel, NULL, &wi);
   12669         4349 :       if (ws_num == 1)
   12670          385 :         gimple_omp_parallel_set_combined_p (stmt, true);
   12671              :     }
   12672        22809 :   gimple_seq dep_ilist = NULL;
   12673        22809 :   gimple_seq dep_olist = NULL;
   12674        22809 :   if (gimple_code (stmt) == GIMPLE_OMP_TASK
   12675        22809 :       && omp_find_clause (clauses, OMP_CLAUSE_DEPEND))
   12676              :     {
   12677         1189 :       push_gimplify_context ();
   12678         1189 :       dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
   12679         1189 :       lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt),
   12680              :                             &dep_ilist, &dep_olist);
   12681              :     }
   12682              : 
   12683        22809 :   if (gimple_code (stmt) == GIMPLE_OMP_TASK
   12684        22809 :       && gimple_omp_task_taskwait_p (stmt))
   12685              :     {
   12686           84 :       if (dep_bind)
   12687              :         {
   12688           84 :           gsi_replace (gsi_p, dep_bind, true);
   12689           84 :           gimple_bind_add_seq (dep_bind, dep_ilist);
   12690           84 :           gimple_bind_add_stmt (dep_bind, stmt);
   12691           84 :           gimple_bind_add_seq (dep_bind, dep_olist);
   12692           84 :           pop_gimplify_context (dep_bind);
   12693              :         }
   12694           84 :       return;
   12695              :     }
   12696              : 
   12697        22725 :   if (ctx->srecord_type)
   12698          513 :     create_task_copyfn (as_a <gomp_task *> (stmt), ctx);
   12699              : 
   12700        22725 :   gimple_seq tskred_ilist = NULL;
   12701        22725 :   gimple_seq tskred_olist = NULL;
   12702        22725 :   if ((is_task_ctx (ctx)
   12703         3842 :        && gimple_omp_task_taskloop_p (ctx->stmt)
   12704         1336 :        && omp_find_clause (gimple_omp_task_clauses (ctx->stmt),
   12705              :                            OMP_CLAUSE_REDUCTION))
   12706        26070 :       || (is_parallel_ctx (ctx)
   12707        16306 :           && omp_find_clause (gimple_omp_parallel_clauses (stmt),
   12708              :                               OMP_CLAUSE__REDUCTEMP_)))
   12709              :     {
   12710          559 :       if (dep_bind == NULL)
   12711              :         {
   12712          559 :           push_gimplify_context ();
   12713          559 :           dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
   12714              :         }
   12715          621 :       lower_omp_task_reductions (ctx, is_task_ctx (ctx) ? OMP_TASKLOOP
   12716              :                                                         : OMP_PARALLEL,
   12717          559 :                                  gimple_omp_taskreg_clauses (ctx->stmt),
   12718              :                                  &tskred_ilist, &tskred_olist);
   12719              :     }
   12720              : 
   12721        22725 :   push_gimplify_context ();
   12722              : 
   12723        22725 :   gimple_seq par_olist = NULL;
   12724        22725 :   gimple_seq par_ilist = NULL;
   12725        22725 :   gimple_seq par_rlist = NULL;
   12726        22725 :   lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx, NULL);
   12727        22721 :   lower_omp (&par_body, ctx);
   12728        22721 :   if (gimple_code (stmt) != GIMPLE_OMP_TASK)
   12729        18879 :     lower_reduction_clauses (clauses, &par_rlist, NULL, ctx);
   12730              : 
   12731              :   /* Declare all the variables created by mapping and the variables
   12732              :      declared in the scope of the parallel body.  */
   12733        22721 :   record_vars_into (ctx->block_vars, child_fn);
   12734        22721 :   maybe_remove_omp_member_access_dummy_vars (par_bind);
   12735        22721 :   record_vars_into (gimple_bind_vars (par_bind), child_fn);
   12736              : 
   12737        22721 :   if (ctx->record_type)
   12738              :     {
   12739        18181 :       ctx->sender_decl
   12740        35849 :         = create_tmp_var (ctx->srecord_type ? ctx->srecord_type
   12741              :                           : ctx->record_type, ".omp_data_o");
   12742        18181 :       DECL_NAMELESS (ctx->sender_decl) = 1;
   12743        18181 :       TREE_ADDRESSABLE (ctx->sender_decl) = 1;
   12744        18181 :       gimple_omp_taskreg_set_data_arg (stmt, ctx->sender_decl);
   12745              :     }
   12746              : 
   12747        22721 :   gimple_seq olist = NULL;
   12748        22721 :   gimple_seq ilist = NULL;
   12749        22721 :   lower_send_clauses (clauses, &ilist, &olist, ctx);
   12750        22721 :   lower_send_shared_vars (&ilist, &olist, ctx);
   12751              : 
   12752        22721 :   if (ctx->record_type)
   12753              :     {
   12754        18181 :       tree clobber = build_clobber (TREE_TYPE (ctx->sender_decl));
   12755        18181 :       gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
   12756              :                                                         clobber));
   12757              :     }
   12758              : 
   12759              :   /* Once all the expansions are done, sequence all the different
   12760              :      fragments inside gimple_omp_body.  */
   12761              : 
   12762        22721 :   gimple_seq new_body = NULL;
   12763              : 
   12764        22721 :   if (ctx->record_type)
   12765              :     {
   12766        18181 :       t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
   12767              :       /* fixup_child_record_type might have changed receiver_decl's type.  */
   12768        18181 :       t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
   12769        18181 :       gimple_seq_add_stmt (&new_body,
   12770        18181 :                            gimple_build_assign (ctx->receiver_decl, t));
   12771              :     }
   12772              : 
   12773        22721 :   gimple_seq_add_seq (&new_body, par_ilist);
   12774        22721 :   gimple_seq_add_seq (&new_body, par_body);
   12775        22721 :   gimple_seq_add_seq (&new_body, par_rlist);
   12776        22721 :   if (ctx->cancellable)
   12777          146 :     gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
   12778        22721 :   gimple_seq_add_seq (&new_body, par_olist);
   12779        22721 :   new_body = maybe_catch_exception (new_body);
   12780        22721 :   if (gimple_code (stmt) == GIMPLE_OMP_TASK)
   12781         3842 :     gimple_seq_add_stmt (&new_body,
   12782         3842 :                          gimple_build_omp_continue (integer_zero_node,
   12783              :                                                     integer_zero_node));
   12784        22721 :   gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
   12785        22721 :   gimple_omp_set_body (stmt, new_body);
   12786              : 
   12787        22721 :   if (dep_bind && gimple_bind_block (par_bind) == NULL_TREE)
   12788          497 :     bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
   12789              :   else
   12790        22224 :     bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind));
   12791        43778 :   gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
   12792        22721 :   gimple_bind_add_seq (bind, ilist);
   12793        22721 :   gimple_bind_add_stmt (bind, stmt);
   12794        22721 :   gimple_bind_add_seq (bind, olist);
   12795              : 
   12796        22721 :   pop_gimplify_context (NULL);
   12797              : 
   12798        22721 :   if (dep_bind)
   12799              :     {
   12800         1664 :       gimple_bind_add_seq (dep_bind, dep_ilist);
   12801         1664 :       gimple_bind_add_seq (dep_bind, tskred_ilist);
   12802         1664 :       gimple_bind_add_stmt (dep_bind, bind);
   12803         1664 :       gimple_bind_add_seq (dep_bind, tskred_olist);
   12804         1664 :       gimple_bind_add_seq (dep_bind, dep_olist);
   12805         1664 :       pop_gimplify_context (dep_bind);
   12806              :     }
   12807              : }
   12808              : 
   12809              :  /* Set EXPR as the hostaddr expression that should result from the clause C
   12810              :     in the target statement STMT.  Returns the tree that should be
   12811              :     passed as the hostaddr (a pointer to the array containing the expanded
   12812              :     hostaddrs and sizes of the clause).  */
   12813              : 
   12814              : static tree
   12815        37413 : lower_omp_map_iterator_expr (tree expr, tree c, gomp_target *stmt)
   12816              : {
   12817        37413 :   if (!OMP_CLAUSE_HAS_ITERATORS (c))
   12818              :     return expr;
   12819              : 
   12820          215 :   tree iterator = OMP_CLAUSE_ITERATORS (c);
   12821          215 :   tree index = OMP_ITERATOR_INDEX (iterator);
   12822          215 :   tree elems = OMP_ITERATOR_ELEMS (iterator);
   12823          215 :   gimple_seq *loop_body_p = enter_omp_iterator_loop_context (c, stmt);
   12824              : 
   12825              :    /* IN LOOP BODY:  */
   12826              :    /* elems[idx] = <expr>;  */
   12827          215 :   tree lhs = build4 (ARRAY_REF, ptr_type_node, elems, index,
   12828              :                      NULL_TREE, NULL_TREE);
   12829          215 :   tree mod_expr = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
   12830              :                               void_type_node, lhs, expr);
   12831          215 :   gimplify_and_add (mod_expr, loop_body_p);
   12832          215 :   exit_omp_iterator_loop_context (c);
   12833              : 
   12834          215 :   return build_fold_addr_expr_with_type (elems, ptr_type_node);
   12835              : }
   12836              : 
   12837              : /* Set SIZE as the size expression that should result from the clause C
   12838              :    in the target statement STMT.  Returns the tree that should be
   12839              :    passed as the clause size (a size_int with the value SIZE_MAX, indicating
   12840              :    that the clause uses an iterator).  */
   12841              : 
   12842              : static tree
   12843        80263 : lower_omp_map_iterator_size (tree size, tree c, gomp_target *stmt)
   12844              : {
   12845        80263 :   if (!OMP_CLAUSE_HAS_ITERATORS (c))
   12846              :     return size;
   12847              : 
   12848          219 :   tree iterator = OMP_CLAUSE_ITERATORS (c);
   12849          219 :   tree index = OMP_ITERATOR_INDEX (iterator);
   12850          219 :   tree elems = OMP_ITERATOR_ELEMS (iterator);
   12851          219 :   gimple_seq *loop_body_p = enter_omp_iterator_loop_context (c, stmt);
   12852              : 
   12853              :   /* IN LOOP BODY:  */
   12854              :   /* elems[idx+1] = <size>;  */
   12855          219 :   tree lhs = build4 (ARRAY_REF, ptr_type_node, elems,
   12856              :                      size_binop (PLUS_EXPR, index, size_int (1)),
   12857              :                      NULL_TREE, NULL_TREE);
   12858          219 :   tree mod_expr = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
   12859              :                               void_type_node, lhs, size);
   12860          219 :   gimplify_and_add (mod_expr, loop_body_p);
   12861          219 :   exit_omp_iterator_loop_context (c);
   12862              : 
   12863          219 :   return size_int (SIZE_MAX);
   12864              : }
   12865              : 
   12866              : /* Lower the GIMPLE_OMP_TARGET in the current statement
   12867              :    in GSI_P.  CTX holds context information for the directive.  */
   12868              : 
   12869              : static void
   12870        37314 : lower_omp_target (gimple_stmt_iterator *gsi_p, omp_context *ctx)
   12871              : {
   12872        37314 :   tree clauses;
   12873        37314 :   tree child_fn, t, c;
   12874        37314 :   gomp_target *stmt = as_a <gomp_target *> (gsi_stmt (*gsi_p));
   12875        37314 :   gbind *tgt_bind, *bind, *dep_bind = NULL;
   12876        37314 :   gimple_seq tgt_body, olist, ilist, fplist, new_body;
   12877        37314 :   location_t loc = gimple_location (stmt);
   12878        37314 :   bool offloaded, data_region;
   12879        37314 :   unsigned int map_cnt = 0;
   12880        37314 :   tree in_reduction_clauses = NULL_TREE;
   12881              : 
   12882        37314 :   tree deep_map_cnt = NULL_TREE;
   12883        37314 :   tree deep_map_data = NULL_TREE;
   12884        37314 :   tree deep_map_offset_data = NULL_TREE;
   12885        37314 :   tree deep_map_offset = NULL_TREE;
   12886              : 
   12887        37314 :   offloaded = is_gimple_omp_offloaded (stmt);
   12888        37314 :   switch (gimple_omp_target_kind (stmt))
   12889              :     {
   12890        11916 :     case GF_OMP_TARGET_KIND_REGION:
   12891        11916 :       tree *p, *q;
   12892        11916 :       q = &in_reduction_clauses;
   12893        79969 :       for (p = gimple_omp_target_clauses_ptr (stmt); *p; )
   12894        68053 :         if (OMP_CLAUSE_CODE (*p) == OMP_CLAUSE_IN_REDUCTION)
   12895              :           {
   12896          422 :             *q = *p;
   12897          422 :             q = &OMP_CLAUSE_CHAIN (*q);
   12898          422 :             *p = OMP_CLAUSE_CHAIN (*p);
   12899              :           }
   12900              :         else
   12901        67631 :           p = &OMP_CLAUSE_CHAIN (*p);
   12902        11916 :       *q = NULL_TREE;
   12903        11916 :       *p = in_reduction_clauses;
   12904              :       /* FALLTHRU */
   12905              :     case GF_OMP_TARGET_KIND_UPDATE:
   12906              :     case GF_OMP_TARGET_KIND_ENTER_DATA:
   12907              :     case GF_OMP_TARGET_KIND_EXIT_DATA:
   12908              :     case GF_OMP_TARGET_KIND_OACC_PARALLEL:
   12909              :     case GF_OMP_TARGET_KIND_OACC_KERNELS:
   12910              :     case GF_OMP_TARGET_KIND_OACC_SERIAL:
   12911              :     case GF_OMP_TARGET_KIND_OACC_UPDATE:
   12912              :     case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
   12913              :     case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
   12914              :     case GF_OMP_TARGET_KIND_OACC_DECLARE:
   12915              :     case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
   12916              :     case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
   12917              :       data_region = false;
   12918              :       break;
   12919              :     case GF_OMP_TARGET_KIND_DATA:
   12920              :     case GF_OMP_TARGET_KIND_OACC_DATA:
   12921              :     case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
   12922              :     case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
   12923        37314 :       data_region = true;
   12924              :       break;
   12925            0 :     default:
   12926            0 :       gcc_unreachable ();
   12927              :     }
   12928              : 
   12929              :   /* Ensure that requires map is written via output_offload_tables, even if only
   12930              :      'target (enter/exit) data' is used in the translation unit.  */
   12931        37314 :   if (ENABLE_OFFLOADING && (omp_requires_mask & OMP_REQUIRES_TARGET_USED))
   12932              :     g->have_offload = true;
   12933              : 
   12934        37314 :   clauses = gimple_omp_target_clauses (stmt);
   12935              : 
   12936        37314 :   gimple_seq dep_ilist = NULL;
   12937        37314 :   gimple_seq dep_olist = NULL;
   12938        37314 :   bool has_depend = omp_find_clause (clauses, OMP_CLAUSE_DEPEND) != NULL_TREE;
   12939        37314 :   if (has_depend || in_reduction_clauses)
   12940              :     {
   12941          392 :       push_gimplify_context ();
   12942          392 :       dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
   12943          392 :       if (has_depend)
   12944          329 :         lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt),
   12945              :                               &dep_ilist, &dep_olist);
   12946          392 :       if (in_reduction_clauses)
   12947          314 :         lower_rec_input_clauses (in_reduction_clauses, &dep_ilist, &dep_olist,
   12948              :                                  ctx, NULL);
   12949              :     }
   12950              : 
   12951        37314 :   tgt_bind = NULL;
   12952        37314 :   tgt_body = NULL;
   12953        37314 :   if (offloaded)
   12954              :     {
   12955        21311 :       tgt_bind = gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt));
   12956        21311 :       tgt_body = gimple_bind_body (tgt_bind);
   12957              :     }
   12958        16003 :   else if (data_region)
   12959         4024 :     tgt_body = gimple_omp_body (stmt);
   12960        37314 :   child_fn = ctx->cb.dst_fn;
   12961              : 
   12962        37314 :   push_gimplify_context ();
   12963        37314 :   fplist = NULL;
   12964              : 
   12965        37314 :   ilist = NULL;
   12966        37314 :   olist = NULL;
   12967              : 
   12968        37314 :   gimple_seq alloc_dlist = NULL;
   12969        37314 :   hash_map<tree, tree> alloc_map;
   12970        74628 :   hash_map<tree, gimple_seq> alloc_seq_map;
   12971              : 
   12972              :   /* The value expression for USES_ALLOCATORS needs to be setup before the
   12973              :      allocator is used via lower_private_allocate.  */
   12974       214424 :   for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
   12975       139796 :     if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USES_ALLOCATORS)
   12976              :       {
   12977           86 :         tree allocator = OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c);
   12978           86 :         tree new_allocator = lookup_decl (allocator, ctx);
   12979           86 :         tree x = build_receiver_ref (allocator, false, ctx);
   12980           86 :         SET_DECL_VALUE_EXPR (new_allocator, x);
   12981           86 :         DECL_HAS_VALUE_EXPR_P (new_allocator) = 1;
   12982           86 :         map_cnt++;
   12983              :       }
   12984              : 
   12985       177110 :   for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
   12986       139796 :     switch (OMP_CLAUSE_CODE (c))
   12987              :       {
   12988              :         tree var, x, new_var, allocator, allocate_ptr, size;
   12989              :         gimple_seq alloc_seq;
   12990              :         bool by_ref;
   12991              : 
   12992              :       default:
   12993        93255 :         break;
   12994        77986 :       case OMP_CLAUSE_MAP:
   12995              : #if CHECKING_P
   12996              :         /* First check what we're prepared to handle in the following.  */
   12997        77986 :         switch (OMP_CLAUSE_MAP_KIND (c))
   12998              :           {
   12999              :           case GOMP_MAP_ALLOC:
   13000              :           case GOMP_MAP_TO:
   13001              :           case GOMP_MAP_FROM:
   13002              :           case GOMP_MAP_TOFROM:
   13003              :           case GOMP_MAP_POINTER:
   13004              :           case GOMP_MAP_TO_PSET:
   13005              :           case GOMP_MAP_DELETE:
   13006              :           case GOMP_MAP_RELEASE:
   13007              :           case GOMP_MAP_ALWAYS_TO:
   13008              :           case GOMP_MAP_ALWAYS_FROM:
   13009              :           case GOMP_MAP_ALWAYS_TOFROM:
   13010              :           case GOMP_MAP_FORCE_PRESENT:
   13011              :           case GOMP_MAP_ALWAYS_PRESENT_FROM:
   13012              :           case GOMP_MAP_ALWAYS_PRESENT_TO:
   13013              :           case GOMP_MAP_ALWAYS_PRESENT_TOFROM:
   13014              : 
   13015              :           case GOMP_MAP_FIRSTPRIVATE_POINTER:
   13016              :           case GOMP_MAP_FIRSTPRIVATE_REFERENCE:
   13017              :           case GOMP_MAP_STRUCT:
   13018              :           case GOMP_MAP_STRUCT_UNORD:
   13019              :           case GOMP_MAP_ALWAYS_POINTER:
   13020              :           case GOMP_MAP_ATTACH:
   13021              :           case GOMP_MAP_DETACH:
   13022              :           case GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION:
   13023              :           case GOMP_MAP_POINTER_TO_ZERO_LENGTH_ARRAY_SECTION:
   13024              :             break;
   13025         2587 :           case GOMP_MAP_IF_PRESENT:
   13026         2587 :           case GOMP_MAP_FORCE_ALLOC:
   13027         2587 :           case GOMP_MAP_FORCE_TO:
   13028         2587 :           case GOMP_MAP_FORCE_FROM:
   13029         2587 :           case GOMP_MAP_FORCE_TOFROM:
   13030         2587 :           case GOMP_MAP_FORCE_DEVICEPTR:
   13031         2587 :           case GOMP_MAP_DEVICE_RESIDENT:
   13032         2587 :           case GOMP_MAP_LINK:
   13033         2587 :           case GOMP_MAP_FORCE_DETACH:
   13034         2587 :             gcc_assert (is_gimple_omp_oacc (stmt));
   13035              :             break;
   13036            0 :           default:
   13037            0 :             gcc_unreachable ();
   13038              :           }
   13039              : #endif
   13040              :           /* FALLTHRU */
   13041        89575 :       case OMP_CLAUSE_TO:
   13042        89575 :       case OMP_CLAUSE_FROM:
   13043        89575 :       oacc_firstprivate:
   13044        89575 :         var = OMP_CLAUSE_DECL (c);
   13045        89575 :         {
   13046        89575 :           tree extra = lang_hooks.decls.omp_deep_mapping_cnt (stmt, c, &ilist);
   13047        89575 :           if (extra != NULL_TREE && deep_map_cnt != NULL_TREE)
   13048           59 :             deep_map_cnt = fold_build2_loc (OMP_CLAUSE_LOCATION (c), PLUS_EXPR,
   13049              :                                             size_type_node, deep_map_cnt,
   13050              :                                             extra);
   13051        89516 :           else if (extra != NULL_TREE)
   13052              :             deep_map_cnt = extra;
   13053              :         }
   13054              : 
   13055        89431 :         if (deep_map_cnt
   13056        89797 :             && OMP_CLAUSE_HAS_ITERATORS (c))
   13057            0 :           sorry ("iterators used together with deep mapping are not "
   13058              :                  "supported yet");
   13059              : 
   13060        89575 :         if (!DECL_P (var))
   13061              :           {
   13062        36988 :             if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
   13063        36988 :                 || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
   13064        35376 :                     && (OMP_CLAUSE_MAP_KIND (c)
   13065              :                         != GOMP_MAP_FIRSTPRIVATE_POINTER)))
   13066        36988 :               map_cnt++;
   13067        46541 :             continue;
   13068              :           }
   13069              : 
   13070        52587 :         if (DECL_SIZE (var)
   13071        52587 :             && !poly_int_tree_p (DECL_SIZE (var)))
   13072              :           {
   13073          719 :             tree var2 = DECL_VALUE_EXPR (var);
   13074          719 :             gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
   13075          719 :             var2 = TREE_OPERAND (var2, 0);
   13076          719 :             gcc_assert (DECL_P (var2));
   13077              :             var = var2;
   13078              :           }
   13079              : 
   13080        52587 :         if (offloaded
   13081        36688 :             && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
   13082        85379 :             && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
   13083        29549 :                 || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE))
   13084              :           {
   13085         3672 :             if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
   13086              :               {
   13087          415 :                 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx))
   13088          415 :                     && varpool_node::get_create (var)->offloadable)
   13089            2 :                   continue;
   13090              : 
   13091          413 :                 tree type = build_pointer_type (TREE_TYPE (var));
   13092          413 :                 tree new_var = lookup_decl (var, ctx);
   13093          413 :                 x = create_tmp_var_raw (type, get_name (new_var));
   13094          413 :                 gimple_add_tmp_var (x);
   13095          413 :                 x = build_simple_mem_ref (x);
   13096          413 :                 SET_DECL_VALUE_EXPR (new_var, x);
   13097          413 :                 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
   13098              :               }
   13099         3670 :             continue;
   13100         3670 :           }
   13101              : 
   13102        48915 :         if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
   13103        38938 :             && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
   13104        38727 :                 || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
   13105        49180 :             && is_omp_target (stmt))
   13106              :           {
   13107          241 :             gcc_assert (maybe_lookup_field (c, ctx));
   13108          241 :             map_cnt++;
   13109          241 :             continue;
   13110              :           }
   13111              : 
   13112        48674 :         if (!maybe_lookup_field (c, ctx))
   13113         5640 :           continue;
   13114              : 
   13115              :         /* Don't remap compute constructs' reduction variables, because the
   13116              :            intermediate result must be local to each gang.  */
   13117        70864 :         if (offloaded && !(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
   13118        23934 :                            && is_gimple_omp_oacc (ctx->stmt)
   13119        13572 :                            && OMP_CLAUSE_MAP_IN_REDUCTION (c)))
   13120              :           {
   13121        27050 :             x = build_receiver_ref (c, true, ctx);
   13122        27050 :             tree new_var = lookup_decl (var, ctx);
   13123              : 
   13124        27050 :             if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
   13125        23154 :                 && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
   13126         3002 :                 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
   13127        30052 :                 && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
   13128          425 :               x = build_simple_mem_ref (x);
   13129        27050 :             if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
   13130              :               {
   13131         3896 :                 gcc_assert (is_gimple_omp_oacc (ctx->stmt));
   13132         3896 :                 if (omp_privatize_by_reference (new_var)
   13133         3896 :                     && (TREE_CODE (TREE_TYPE (new_var)) != POINTER_TYPE
   13134           44 :                         || DECL_BY_REFERENCE (var)))
   13135              :                   {
   13136              :                     /* Create a local object to hold the instance
   13137              :                        value.  */
   13138          455 :                     tree type = TREE_TYPE (TREE_TYPE (new_var));
   13139          455 :                     const char *id = IDENTIFIER_POINTER (DECL_NAME (new_var));
   13140          455 :                     tree inst = create_tmp_var (type, id);
   13141          455 :                     gimplify_assign (inst, fold_indirect_ref (x), &fplist);
   13142          455 :                     x = build_fold_addr_expr (inst);
   13143              :                   }
   13144         3896 :                 gimplify_assign (new_var, x, &fplist);
   13145              :               }
   13146        23154 :             else if (DECL_P (new_var))
   13147              :               {
   13148        23154 :                 SET_DECL_VALUE_EXPR (new_var, x);
   13149        23154 :                 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
   13150              :               }
   13151              :             else
   13152            0 :               gcc_unreachable ();
   13153              :           }
   13154        43034 :         map_cnt++;
   13155        43034 :         break;
   13156              : 
   13157        15467 :       case OMP_CLAUSE_FIRSTPRIVATE:
   13158        15467 :       omp_firstprivate_recv:
   13159        15467 :         gcc_checking_assert (offloaded);
   13160        15467 :         if (is_gimple_omp_oacc (ctx->stmt))
   13161              :           {
   13162              :             /* No 'firstprivate' clauses on OpenACC 'kernels'.  */
   13163         3896 :             gcc_checking_assert (!is_oacc_kernels (ctx));
   13164              :             /* Likewise, on OpenACC 'kernels' decomposed parts.  */
   13165         3896 :             gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
   13166              : 
   13167         3896 :             goto oacc_firstprivate;
   13168              :           }
   13169        11571 :         map_cnt++;
   13170        11571 :         var = OMP_CLAUSE_DECL (c);
   13171        11571 :         new_var = lookup_decl (var, ctx);
   13172        11571 :         allocator = NULL_TREE;
   13173        11571 :         allocate_ptr = NULL_TREE;
   13174        11571 :         size = TREE_TYPE (var);
   13175        11571 :         by_ref = omp_privatize_by_reference (var);
   13176        11571 :         if (by_ref)
   13177          812 :           size = TREE_TYPE (size);
   13178        11571 :         size = TYPE_SIZE_UNIT (size);
   13179        11571 :         if (is_variable_sized (var, by_ref))
   13180           39 :           size = lookup_decl (size, ctx);
   13181        11571 :         alloc_seq = NULL;
   13182        11571 :         if (lower_private_allocate (var, new_var, allocator, allocate_ptr,
   13183              :                                     &alloc_seq, ctx, by_ref, size))
   13184              :           {
   13185           68 :             alloc_map.put (new_var, allocate_ptr);
   13186           68 :             alloc_seq_map.put (new_var, alloc_seq);
   13187              :           }
   13188        11571 :         if (!by_ref && !is_gimple_reg_type (TREE_TYPE (var)))
   13189              :           {
   13190          156 :             if (is_variable_sized (var))
   13191              :               {
   13192           10 :                 tree pvar = DECL_VALUE_EXPR (var);
   13193           10 :                 gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
   13194           10 :                 pvar = TREE_OPERAND (pvar, 0);
   13195           10 :                 gcc_assert (DECL_P (pvar));
   13196           10 :                 tree new_pvar = lookup_decl (pvar, ctx);
   13197           10 :                 x = build_fold_indirect_ref (new_pvar);
   13198           10 :                 TREE_THIS_NOTRAP (x) = 1;
   13199              :               }
   13200          146 :             else if (allocate_ptr)
   13201           22 :               x = build_fold_indirect_ref (allocate_ptr);
   13202              :             else
   13203          124 :               x = build_receiver_ref (var, true, ctx);
   13204          156 :             SET_DECL_VALUE_EXPR (new_var, x);
   13205          156 :             DECL_HAS_VALUE_EXPR_P (new_var) = 1;
   13206              :           }
   13207              :           /* Fortran array descriptors: firstprivate of data + attach.  */
   13208        11571 :           if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
   13209        11552 :               && lang_hooks.decls.omp_array_data (var, true)
   13210        11602 :               && lang_hooks.decls.omp_array_data_privatize (var))
   13211           23 :             map_cnt += 2;
   13212              : 
   13213        11719 :       do_dtor:
   13214        11719 :         if (allocator)
   13215              :           {
   13216          125 :             if (!is_gimple_val (allocator))
   13217              :               {
   13218            0 :                 tree avar = create_tmp_var (TREE_TYPE (allocator));
   13219            0 :                 gimplify_assign (avar, allocator, &alloc_dlist);
   13220            0 :                 allocator = avar;
   13221              :               }
   13222          125 :             if (!is_gimple_val (allocate_ptr))
   13223              :               {
   13224            0 :                 tree apvar = create_tmp_var (TREE_TYPE (allocate_ptr));
   13225            0 :                 gimplify_assign (apvar, allocate_ptr, &alloc_dlist);
   13226            0 :                 allocate_ptr = apvar;
   13227              :               }
   13228          125 :             tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
   13229          125 :             gimple *g = gimple_build_call (f, 2, allocate_ptr, allocator);
   13230          125 :             gimple_seq_add_stmt (&alloc_dlist, g);
   13231              :           }
   13232              :         break;
   13233              : 
   13234          228 :       case OMP_CLAUSE_PRIVATE:
   13235          228 :         gcc_checking_assert (offloaded);
   13236          228 :         if (is_gimple_omp_oacc (ctx->stmt))
   13237              :           {
   13238              :             /* No 'private' clauses on OpenACC 'kernels'.  */
   13239           80 :             gcc_checking_assert (!is_oacc_kernels (ctx));
   13240              :             /* Likewise, on OpenACC 'kernels' decomposed parts.  */
   13241           80 :             gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
   13242              : 
   13243              :             break;
   13244              :           }
   13245          148 :         var = OMP_CLAUSE_DECL (c);
   13246          148 :         new_var = lookup_decl (var, ctx);
   13247          148 :         allocator = NULL_TREE;
   13248          148 :         allocate_ptr = NULL_TREE;
   13249          148 :         alloc_seq = NULL;
   13250          148 :         size = TREE_TYPE (var);
   13251          148 :         by_ref = omp_privatize_by_reference (var);
   13252          148 :         if (by_ref)
   13253           22 :           size = TREE_TYPE (size);
   13254          148 :         size = TYPE_SIZE_UNIT (size);
   13255          148 :         if (is_variable_sized (var, by_ref))
   13256           21 :           size = lookup_decl (size, ctx);
   13257          148 :         lower_private_allocate (var, new_var, allocator, allocate_ptr,
   13258              :                                 &alloc_seq, ctx, by_ref, size);
   13259          148 :         if (allocate_ptr)
   13260              :           {
   13261           57 :             alloc_map.put (new_var, allocate_ptr);
   13262           57 :             alloc_seq_map.put (new_var, alloc_seq);
   13263              :           }
   13264          148 :         if (!allocate_ptr && is_variable_sized (var))
   13265              :           {
   13266            1 :             tree new_var = lookup_decl (var, ctx);
   13267            1 :             tree pvar = DECL_VALUE_EXPR (var);
   13268            1 :             gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
   13269            1 :             pvar = TREE_OPERAND (pvar, 0);
   13270            1 :             gcc_assert (DECL_P (pvar));
   13271            1 :             tree new_pvar = lookup_decl (pvar, ctx);
   13272            1 :             x = build_fold_indirect_ref (new_pvar);
   13273            1 :             TREE_THIS_NOTRAP (x) = 1;
   13274            1 :             SET_DECL_VALUE_EXPR (new_var, x);
   13275            1 :             DECL_HAS_VALUE_EXPR_P (new_var) = 1;
   13276              :           }
   13277          148 :         goto do_dtor;
   13278              : 
   13279         2431 :       case OMP_CLAUSE_USE_DEVICE_PTR:
   13280         2431 :       case OMP_CLAUSE_USE_DEVICE_ADDR:
   13281         2431 :       case OMP_CLAUSE_HAS_DEVICE_ADDR:
   13282         2431 :       case OMP_CLAUSE_IS_DEVICE_PTR:
   13283         2431 :         var = OMP_CLAUSE_DECL (c);
   13284         2431 :         if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
   13285              :           {
   13286          253 :             while (TREE_CODE (var) == INDIRECT_REF
   13287          253 :                    || TREE_CODE (var) == ARRAY_REF)
   13288           17 :               var = TREE_OPERAND (var, 0);
   13289          236 :             if (lang_hooks.decls.omp_array_data (var, true))
   13290           19 :               goto omp_firstprivate_recv;
   13291              :           }
   13292         2412 :         map_cnt++;
   13293         2412 :         if (is_variable_sized (var))
   13294              :           {
   13295           12 :             tree new_var = lookup_decl (var, ctx);
   13296           12 :             tree pvar = DECL_VALUE_EXPR (var);
   13297           12 :             gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
   13298           12 :             pvar = TREE_OPERAND (pvar, 0);
   13299           12 :             gcc_assert (DECL_P (pvar));
   13300           12 :             tree new_pvar = lookup_decl (pvar, ctx);
   13301           12 :             x = build_fold_indirect_ref (new_pvar);
   13302           12 :             TREE_THIS_NOTRAP (x) = 1;
   13303           12 :             SET_DECL_VALUE_EXPR (new_var, x);
   13304           12 :             DECL_HAS_VALUE_EXPR_P (new_var) = 1;
   13305              :           }
   13306         2400 :         else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
   13307          543 :                    || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
   13308         2072 :                   && !omp_privatize_by_reference (var)
   13309          703 :                   && !omp_is_allocatable_or_ptr (var)
   13310          564 :                   && !lang_hooks.decls.omp_array_data (var, true))
   13311         4435 :                  || TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
   13312              :           {
   13313          389 :             tree new_var = lookup_decl (var, ctx);
   13314          389 :             tree type = build_pointer_type (TREE_TYPE (var));
   13315          389 :             x = create_tmp_var_raw (type, get_name (new_var));
   13316          389 :             gimple_add_tmp_var (x);
   13317          389 :             x = build_simple_mem_ref (x);
   13318          389 :             SET_DECL_VALUE_EXPR (new_var, x);
   13319          389 :             DECL_HAS_VALUE_EXPR_P (new_var) = 1;
   13320              :           }
   13321              :         else
   13322              :           {
   13323         2011 :             tree new_var = lookup_decl (var, ctx);
   13324         2011 :             x = create_tmp_var_raw (TREE_TYPE (new_var), get_name (new_var));
   13325         2011 :             gimple_add_tmp_var (x);
   13326         2011 :             SET_DECL_VALUE_EXPR (new_var, x);
   13327         2011 :             DECL_HAS_VALUE_EXPR_P (new_var) = 1;
   13328              :           }
   13329              :         break;
   13330              :       }
   13331              : 
   13332        37314 :   if (offloaded)
   13333              :     {
   13334        21311 :       target_nesting_level++;
   13335        21311 :       lower_omp (&tgt_body, ctx);
   13336        21311 :       target_nesting_level--;
   13337              :     }
   13338        16003 :   else if (data_region)
   13339         4024 :     lower_omp (&tgt_body, ctx);
   13340              : 
   13341        37314 :   if (offloaded)
   13342              :     {
   13343              :       /* Declare all the variables created by mapping and the variables
   13344              :          declared in the scope of the target body.  */
   13345        21311 :       record_vars_into (ctx->block_vars, child_fn);
   13346        21311 :       maybe_remove_omp_member_access_dummy_vars (tgt_bind);
   13347        21311 :       record_vars_into (gimple_bind_vars (tgt_bind), child_fn);
   13348              :     }
   13349              : 
   13350        37314 :   if (ctx->record_type)
   13351              :     {
   13352        32403 :       if (deep_map_cnt && TREE_CODE (deep_map_cnt) == INTEGER_CST)
   13353              :         /* map_cnt = map_cnt + tree_to_hwi (deep_map_cnt); */
   13354              :         /* deep_map_cnt = NULL_TREE; */
   13355            0 :         gcc_unreachable ();
   13356          144 :       else if (deep_map_cnt)
   13357              :         {
   13358          144 :           gcc_assert (flexible_array_type_p (ctx->record_type));
   13359          144 :           tree n = create_tmp_var_raw (size_type_node, "nn_map");
   13360          144 :           gimple_add_tmp_var (n);
   13361          144 :           gimplify_assign (n, deep_map_cnt, &ilist);
   13362          144 :           deep_map_cnt = n;
   13363              :         }
   13364          144 :       ctx->sender_decl
   13365        32403 :         = create_tmp_var (deep_map_cnt ? build_pointer_type (ctx->record_type)
   13366              :                                        : ctx->record_type, ".omp_data_arr");
   13367        32403 :       DECL_NAMELESS (ctx->sender_decl) = 1;
   13368        32403 :       TREE_ADDRESSABLE (ctx->sender_decl) = 1;
   13369        64662 :       t = make_tree_vec (deep_map_cnt ? 4 : 3);
   13370        32403 :       TREE_VEC_ELT (t, 0) = ctx->sender_decl;
   13371        32403 :       TREE_VEC_ELT (t, 1)
   13372        64806 :         = create_tmp_var (deep_map_cnt
   13373          144 :                           ? build_pointer_type (size_type_node)
   13374        32259 :                           : build_array_type_nelts (size_type_node, map_cnt),
   13375              :                           ".omp_data_sizes");
   13376        32403 :       DECL_NAMELESS (TREE_VEC_ELT (t, 1)) = 1;
   13377        32403 :       TREE_ADDRESSABLE (TREE_VEC_ELT (t, 1)) = 1;
   13378        32403 :       TREE_STATIC (TREE_VEC_ELT (t, 1)) = 1;
   13379        32403 :       tree tkind_type = short_unsigned_type_node;
   13380        32403 :       int talign_shift = 8;
   13381        32403 :       TREE_VEC_ELT (t, 2)
   13382        64806 :         = create_tmp_var (deep_map_cnt
   13383          144 :                           ? build_pointer_type (tkind_type)
   13384        32259 :                           : build_array_type_nelts (tkind_type, map_cnt),
   13385              :                           ".omp_data_kinds");
   13386        32403 :       DECL_NAMELESS (TREE_VEC_ELT (t, 2)) = 1;
   13387        32403 :       TREE_ADDRESSABLE (TREE_VEC_ELT (t, 2)) = 1;
   13388        32403 :       TREE_STATIC (TREE_VEC_ELT (t, 2)) = 1;
   13389        32403 :       gimple_omp_target_set_data_arg (stmt, t);
   13390              : 
   13391        32403 :       if (deep_map_cnt)
   13392              :         {
   13393          144 :           tree tmp, size;
   13394          144 :           size = create_tmp_var (size_type_node, NULL);
   13395          144 :           DECL_NAMELESS (size) = 1;
   13396          144 :           gimplify_assign (size,
   13397              :                            fold_build2_loc (UNKNOWN_LOCATION, PLUS_EXPR,
   13398              :                                             size_type_node, deep_map_cnt,
   13399              :                                             build_int_cst (size_type_node,
   13400          144 :                                                            map_cnt)), &ilist);
   13401          144 :           TREE_VEC_ELT (t, 3) = size;
   13402              : 
   13403          144 :           tree call = builtin_decl_explicit (BUILT_IN_MALLOC);
   13404          144 :           size = fold_build2_loc (UNKNOWN_LOCATION, MULT_EXPR,
   13405              :                                   size_type_node, deep_map_cnt,
   13406          144 :                                   TYPE_SIZE_UNIT (ptr_type_node));
   13407          144 :           size = fold_build2_loc (UNKNOWN_LOCATION, PLUS_EXPR,
   13408              :                                   size_type_node, size,
   13409          144 :                                   TYPE_SIZE_UNIT (ctx->record_type));
   13410          144 :           tmp = build_call_expr_loc (input_location, call, 1, size);
   13411          144 :           gimplify_assign (ctx->sender_decl, tmp, &ilist);
   13412              : 
   13413          144 :           size = fold_build2_loc (UNKNOWN_LOCATION, MULT_EXPR,
   13414          144 :                                   size_type_node, TREE_VEC_ELT (t, 3),
   13415          144 :                                   TYPE_SIZE_UNIT (size_type_node));
   13416          144 :           tmp = build_call_expr_loc (input_location, call, 1, size);
   13417          144 :           gimplify_assign (TREE_VEC_ELT (t, 1), tmp, &ilist);
   13418              : 
   13419          144 :           size = fold_build2_loc (UNKNOWN_LOCATION, MULT_EXPR,
   13420          144 :                                   size_type_node, TREE_VEC_ELT (t, 3),
   13421          144 :                                   TYPE_SIZE_UNIT (tkind_type));
   13422          144 :           tmp = build_call_expr_loc (input_location, call, 1, size);
   13423          144 :           gimplify_assign (TREE_VEC_ELT (t, 2), tmp, &ilist);
   13424          144 :           tree field = TYPE_FIELDS (TREE_TYPE (TREE_TYPE (ctx->sender_decl)));
   13425          595 :           for ( ; DECL_CHAIN (field) != NULL_TREE; field = DECL_CHAIN (field))
   13426              :             ;
   13427          144 :           gcc_assert (TREE_CODE (TREE_TYPE (field)));
   13428          144 :           tmp = build_fold_indirect_ref (ctx->sender_decl);
   13429          144 :           deep_map_data = omp_build_component_ref (tmp, field);
   13430          144 :           deep_map_offset_data = create_tmp_var_raw (size_type_node,
   13431              :                                                      "map_offset_data");
   13432          144 :           deep_map_offset = create_tmp_var_raw (size_type_node, "map_offset");
   13433          144 :           gimple_add_tmp_var (deep_map_offset_data);
   13434          144 :           gimple_add_tmp_var (deep_map_offset);
   13435          144 :           gimplify_assign (deep_map_offset_data, build_int_cst (size_type_node,
   13436              :                                                                 0), &ilist);
   13437          144 :           gimplify_assign (deep_map_offset, build_int_cst (size_type_node,
   13438          144 :                                                            map_cnt), &ilist);
   13439              :         }
   13440              : 
   13441        32403 :       vec<constructor_elt, va_gc> *vsize;
   13442        32403 :       vec<constructor_elt, va_gc> *vkind;
   13443        32403 :       vec_alloc (vsize, map_cnt);
   13444        32403 :       vec_alloc (vkind, map_cnt);
   13445        32403 :       unsigned int map_idx = 0;
   13446              : 
   13447       195168 :       for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
   13448       130362 :         switch (OMP_CLAUSE_CODE (c))
   13449              :           {
   13450              :             tree ovar, nc, s, purpose, var, x, type;
   13451              :             unsigned int talign;
   13452              : 
   13453              :           default:
   13454       126391 :             break;
   13455              : 
   13456        88074 :           case OMP_CLAUSE_MAP:
   13457        88074 :           case OMP_CLAUSE_TO:
   13458        88074 :           case OMP_CLAUSE_FROM:
   13459        88074 :           oacc_firstprivate_map:
   13460        88074 :             nc = c;
   13461        88074 :             ovar = OMP_CLAUSE_DECL (c);
   13462        88074 :             if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
   13463        88074 :                 && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
   13464        73074 :                     || (OMP_CLAUSE_MAP_KIND (c)
   13465              :                         == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
   13466              :               break;
   13467        84234 :             if (deep_map_cnt)
   13468              :               {
   13469          428 :                 unsigned HOST_WIDE_INT tkind2;
   13470          428 :                 switch (OMP_CLAUSE_CODE (c))
   13471              :                   {
   13472          424 :                   case OMP_CLAUSE_MAP:
   13473          424 :                     tkind2 = OMP_CLAUSE_MAP_KIND (c);
   13474          424 :                     if (OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (c)
   13475          424 :                         && (((tkind2 & GOMP_MAP_FLAG_SPECIAL_BITS)
   13476              :                              & ~GOMP_MAP_IMPLICIT)
   13477              :                             == 0))
   13478              :                       {
   13479              :                         /* If this is an implicit map, and the GOMP_MAP_IMPLICIT
   13480              :                            bits are not interfered by other special bit
   13481              :                            encodings, then turn the GOMP_IMPLICIT_BIT flag on
   13482              :                            for the runtime to see.  */
   13483          120 :                         tkind2 |= GOMP_MAP_IMPLICIT;
   13484              :                       }
   13485              :                     break;
   13486              :                   case OMP_CLAUSE_FIRSTPRIVATE: tkind2 = GOMP_MAP_TO; break;
   13487              :                   case OMP_CLAUSE_TO: tkind2 = GOMP_MAP_TO; break;
   13488            2 :                   case OMP_CLAUSE_FROM: tkind2 = GOMP_MAP_FROM; break;
   13489            0 :                   default: gcc_unreachable ();
   13490              :                   }
   13491          428 :                 lang_hooks.decls.omp_deep_mapping (stmt, c, tkind2,
   13492              :                                                    deep_map_data,
   13493          428 :                                                    TREE_VEC_ELT (t, 1),
   13494          428 :                                                    TREE_VEC_ELT (t, 2),
   13495              :                                                    deep_map_offset_data,
   13496              :                                                    deep_map_offset, &ilist);
   13497              :               }
   13498        84234 :             if (!DECL_P (ovar))
   13499              :               {
   13500        36988 :                 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
   13501        36988 :                     && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
   13502              :                   {
   13503            0 :                     nc = OMP_CLAUSE_CHAIN (c);
   13504            0 :                     gcc_checking_assert (OMP_CLAUSE_DECL (nc)
   13505              :                                          == get_base_address (ovar));
   13506            0 :                     ovar = OMP_CLAUSE_DECL (nc);
   13507              :                   }
   13508              :                 else
   13509              :                   {
   13510        36988 :                     tree x = build_sender_ref (c, ctx);
   13511        36988 :                     tree v = ovar;
   13512        36988 :                     if (in_reduction_clauses
   13513          163 :                         && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
   13514        37151 :                         && OMP_CLAUSE_MAP_IN_REDUCTION (c))
   13515              :                       {
   13516          104 :                         v = unshare_expr (v);
   13517          104 :                         tree *p = &v;
   13518          104 :                         while (handled_component_p (*p)
   13519              :                                || TREE_CODE (*p) == INDIRECT_REF
   13520              :                                || TREE_CODE (*p) == ADDR_EXPR
   13521              :                                || TREE_CODE (*p) == MEM_REF
   13522          256 :                                || TREE_CODE (*p) == NON_LVALUE_EXPR)
   13523          152 :                           p = &TREE_OPERAND (*p, 0);
   13524          104 :                         tree d = *p;
   13525          104 :                         if (is_variable_sized (d))
   13526              :                           {
   13527           16 :                             gcc_assert (DECL_HAS_VALUE_EXPR_P (d));
   13528           16 :                             d = DECL_VALUE_EXPR (d);
   13529           16 :                             gcc_assert (TREE_CODE (d) == INDIRECT_REF);
   13530           16 :                             d = TREE_OPERAND (d, 0);
   13531           16 :                             gcc_assert (DECL_P (d));
   13532              :                           }
   13533          104 :                         splay_tree_key key
   13534          104 :                           = (splay_tree_key) &DECL_CONTEXT (d);
   13535          104 :                         tree nd = (tree) splay_tree_lookup (ctx->field_map,
   13536          104 :                                                             key)->value;
   13537          104 :                         if (d == *p)
   13538           88 :                           *p = nd;
   13539              :                         else
   13540           16 :                           *p = build_fold_indirect_ref (nd);
   13541              :                       }
   13542        36988 :                     v = build_fold_addr_expr_with_type (v, ptr_type_node);
   13543        36988 :                     v = lower_omp_map_iterator_expr (v, c, stmt);
   13544        36988 :                     gimplify_assign (x, v, &ilist);
   13545        36988 :                     nc = NULL_TREE;
   13546              :                   }
   13547              :               }
   13548              :             else
   13549              :               {
   13550        47246 :                 if (DECL_SIZE (ovar)
   13551        47246 :                     && !poly_int_tree_p (DECL_SIZE (ovar)))
   13552              :                   {
   13553            7 :                     tree ovar2 = DECL_VALUE_EXPR (ovar);
   13554            7 :                     gcc_assert (TREE_CODE (ovar2) == INDIRECT_REF);
   13555            7 :                     ovar2 = TREE_OPERAND (ovar2, 0);
   13556            7 :                     gcc_assert (DECL_P (ovar2));
   13557              :                     ovar = ovar2;
   13558              :                   }
   13559        47246 :                 if (!maybe_lookup_field (c, ctx)
   13560        51217 :                     && !(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
   13561         3971 :                          && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
   13562         3971 :                              || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)))
   13563         3971 :                   continue;
   13564              :               }
   13565              : 
   13566        80263 :             talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar));
   13567        80263 :             if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign)
   13568         6416 :               talign = DECL_ALIGN_UNIT (ovar);
   13569              : 
   13570        80263 :             var = NULL_TREE;
   13571        80263 :             if (nc)
   13572              :               {
   13573        43275 :                 if (in_reduction_clauses
   13574          879 :                     && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
   13575        44154 :                     && OMP_CLAUSE_MAP_IN_REDUCTION (c))
   13576              :                   {
   13577          268 :                     tree d = ovar;
   13578          268 :                     if (is_variable_sized (d))
   13579              :                       {
   13580            0 :                         gcc_assert (DECL_HAS_VALUE_EXPR_P (d));
   13581            0 :                         d = DECL_VALUE_EXPR (d);
   13582            0 :                         gcc_assert (TREE_CODE (d) == INDIRECT_REF);
   13583            0 :                         d = TREE_OPERAND (d, 0);
   13584            0 :                         gcc_assert (DECL_P (d));
   13585              :                       }
   13586          268 :                     splay_tree_key key
   13587          268 :                       = (splay_tree_key) &DECL_CONTEXT (d);
   13588          268 :                     tree nd = (tree) splay_tree_lookup (ctx->field_map,
   13589          268 :                                                         key)->value;
   13590          268 :                     if (d == ovar)
   13591              :                       var = nd;
   13592              :                     else
   13593            0 :                       var = build_fold_indirect_ref (nd);
   13594              :                   }
   13595              :                 else
   13596        43007 :                   var = lookup_decl_in_outer_ctx (ovar, ctx);
   13597              :               }
   13598        43275 :             if (nc
   13599        43275 :                 && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
   13600        33298 :                 && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
   13601        33087 :                     || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
   13602          265 :                 && is_omp_target (stmt))
   13603              :               {
   13604          241 :                 x = build_sender_ref (c, ctx);
   13605          241 :                 gimplify_assign (x, build_fold_addr_expr (var), &ilist);
   13606              :               }
   13607        80022 :             else if (nc)
   13608              :               {
   13609        43034 :                 x = build_sender_ref (nc, ctx);
   13610              : 
   13611        43034 :                 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
   13612        33057 :                     && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
   13613         5172 :                     && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
   13614        48206 :                     && TREE_CODE (TREE_TYPE (ovar)) == ARRAY_TYPE)
   13615              :                   {
   13616          425 :                     gcc_assert (offloaded);
   13617          425 :                     tree avar = build_fold_addr_expr (var);
   13618          425 :                     if (!OMP_CLAUSE_ITERATORS (c))
   13619              :                       {
   13620          425 :                         tree tmp = create_tmp_var (TREE_TYPE (TREE_TYPE (x)));
   13621          425 :                         mark_addressable (tmp);
   13622          425 :                         gimplify_assign (tmp, avar, &ilist);
   13623          425 :                         avar = tmp;
   13624              :                       }
   13625          425 :                     talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (x)));
   13626          425 :                     avar = build_fold_addr_expr (avar);
   13627          425 :                     avar = lower_omp_map_iterator_expr (avar, c, stmt);
   13628          425 :                     gimplify_assign (x, avar, &ilist);
   13629              :                   }
   13630        42609 :                 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
   13631              :                   {
   13632         3896 :                     gcc_assert (is_gimple_omp_oacc (ctx->stmt));
   13633         3896 :                     if (!omp_privatize_by_reference (var))
   13634              :                       {
   13635         3441 :                         if (is_gimple_reg (var)
   13636         3441 :                             && OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
   13637         2578 :                           suppress_warning (var);
   13638         3441 :                         var = build_fold_addr_expr (var);
   13639              :                       }
   13640              :                     else
   13641          455 :                       talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
   13642         3896 :                     gimplify_assign (x, var, &ilist);
   13643              :                   }
   13644        38713 :                 else if (is_gimple_reg (var))
   13645              :                   {
   13646         5197 :                     gcc_assert (offloaded);
   13647         5197 :                     tree avar = create_tmp_var (TREE_TYPE (var));
   13648         5197 :                     mark_addressable (avar);
   13649         5197 :                     enum gomp_map_kind map_kind = OMP_CLAUSE_MAP_KIND (c);
   13650         5197 :                     if (GOMP_MAP_COPY_TO_P (map_kind)
   13651              :                         || map_kind == GOMP_MAP_POINTER
   13652          475 :                         || map_kind == GOMP_MAP_TO_PSET
   13653            2 :                         || map_kind == GOMP_MAP_FORCE_DEVICEPTR)
   13654              :                       {
   13655              :                         /* If we need to initialize a temporary
   13656              :                            with VAR because it is not addressable, and
   13657              :                            the variable hasn't been initialized yet, then
   13658              :                            we'll get a warning for the store to avar.
   13659              :                            Don't warn in that case, the mapping might
   13660              :                            be implicit.  */
   13661         5195 :                         suppress_warning (var, OPT_Wuninitialized);
   13662         5195 :                         gimplify_assign (avar, var, &ilist);
   13663              :                       }
   13664         5197 :                     avar = build_fold_addr_expr (avar);
   13665         5197 :                     gimplify_assign (x, avar, &ilist);
   13666         5197 :                     if ((GOMP_MAP_COPY_FROM_P (map_kind)
   13667         1404 :                          || map_kind == GOMP_MAP_FORCE_DEVICEPTR)
   13668         8990 :                         && !TYPE_READONLY (TREE_TYPE (var)))
   13669              :                       {
   13670         3793 :                         x = unshare_expr (x);
   13671         3793 :                         x = build_simple_mem_ref (x);
   13672         3793 :                         gimplify_assign (var, x, &olist);
   13673              :                       }
   13674              :                   }
   13675              :                 else
   13676              :                   {
   13677              :                     /* While MAP is handled explicitly by the FE,
   13678              :                        for 'target update', only the identified is passed.  */
   13679        33516 :                     if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FROM
   13680        28385 :                          || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TO)
   13681        34466 :                         && (omp_is_allocatable_or_ptr (var)
   13682          120 :                             && omp_check_optional_argument (var, false)))
   13683            0 :                       var = build_fold_indirect_ref (var);
   13684        33516 :                     else if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FROM
   13685        28385 :                               && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_TO)
   13686        34466 :                              || (!omp_is_allocatable_or_ptr (var)
   13687         5961 :                                  && !omp_check_optional_argument (var, false)))
   13688        33396 :                       var = build_fold_addr_expr (var);
   13689        33516 :                     gimplify_assign (x, var, &ilist);
   13690              :                   }
   13691              :               }
   13692        80263 :             s = NULL_TREE;
   13693        80263 :             if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
   13694              :               {
   13695         3896 :                 gcc_checking_assert (is_gimple_omp_oacc (ctx->stmt));
   13696         3896 :                 s = TREE_TYPE (ovar);
   13697         3896 :                 if (TREE_CODE (s) == REFERENCE_TYPE
   13698         3896 :                     || omp_check_optional_argument (ovar, false))
   13699          449 :                   s = TREE_TYPE (s);
   13700         3896 :                 s = TYPE_SIZE_UNIT (s);
   13701              :               }
   13702              :             else
   13703        76367 :               s = OMP_CLAUSE_SIZE (c);
   13704        80263 :             if (s == NULL_TREE)
   13705          155 :               s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
   13706        80263 :             s = fold_convert (size_type_node, s);
   13707        80263 :             s = lower_omp_map_iterator_size (s, c, stmt);
   13708        80263 :             purpose = size_int (map_idx++);
   13709        80263 :             CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
   13710        80263 :             if (TREE_CODE (s) != INTEGER_CST)
   13711        16158 :               TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
   13712              : 
   13713        80263 :             unsigned HOST_WIDE_INT tkind, tkind_zero;
   13714        80263 :             switch (OMP_CLAUSE_CODE (c))
   13715              :               {
   13716        68674 :               case OMP_CLAUSE_MAP:
   13717        68674 :                 tkind = OMP_CLAUSE_MAP_KIND (c);
   13718        68674 :                 tkind_zero = tkind;
   13719        68674 :                 if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c))
   13720         5326 :                   switch (tkind)
   13721              :                     {
   13722              :                     case GOMP_MAP_ALLOC:
   13723              :                     case GOMP_MAP_IF_PRESENT:
   13724              :                     case GOMP_MAP_TO:
   13725              :                     case GOMP_MAP_FROM:
   13726              :                     case GOMP_MAP_TOFROM:
   13727              :                     case GOMP_MAP_ALWAYS_TO:
   13728              :                     case GOMP_MAP_ALWAYS_FROM:
   13729              :                     case GOMP_MAP_ALWAYS_TOFROM:
   13730              :                     case GOMP_MAP_ALWAYS_PRESENT_TO:
   13731              :                     case GOMP_MAP_ALWAYS_PRESENT_FROM:
   13732              :                     case GOMP_MAP_ALWAYS_PRESENT_TOFROM:
   13733              :                     case GOMP_MAP_RELEASE:
   13734              :                     case GOMP_MAP_FORCE_TO:
   13735              :                     case GOMP_MAP_FORCE_FROM:
   13736              :                     case GOMP_MAP_FORCE_TOFROM:
   13737              :                     case GOMP_MAP_FORCE_PRESENT:
   13738        68674 :                       tkind_zero = GOMP_MAP_ZERO_LEN_ARRAY_SECTION;
   13739              :                       break;
   13740           21 :                     case GOMP_MAP_DELETE:
   13741           21 :                       tkind_zero = GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION;
   13742              :                     default:
   13743              :                       break;
   13744              :                     }
   13745        68674 :                 if (tkind_zero != tkind)
   13746              :                   {
   13747         5326 :                     if (integer_zerop (s))
   13748              :                       tkind = tkind_zero;
   13749         4594 :                     else if (integer_nonzerop (s))
   13750        68674 :                       tkind_zero = tkind;
   13751              :                   }
   13752        68674 :                 if (tkind_zero == tkind
   13753        67211 :                     && OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (c)
   13754        68674 :                     && (((tkind & GOMP_MAP_FLAG_SPECIAL_BITS)
   13755         5164 :                          & ~GOMP_MAP_IMPLICIT)
   13756              :                         == 0))
   13757              :                   {
   13758              :                     /* If this is an implicit map, and the GOMP_MAP_IMPLICIT
   13759              :                        bits are not interfered by other special bit encodings,
   13760              :                        then turn the GOMP_IMPLICIT_BIT flag on for the runtime
   13761              :                        to see.  */
   13762         5121 :                     tkind |= GOMP_MAP_IMPLICIT;
   13763         5121 :                     tkind_zero = tkind;
   13764              :                   }
   13765              :                 break;
   13766         3896 :               case OMP_CLAUSE_FIRSTPRIVATE:
   13767         3896 :                 gcc_checking_assert (is_gimple_omp_oacc (ctx->stmt));
   13768              :                 tkind = GOMP_MAP_TO;
   13769              :                 tkind_zero = tkind;
   13770              :                 break;
   13771         1659 :               case OMP_CLAUSE_TO:
   13772         1659 :                 tkind
   13773         1659 :                   = (OMP_CLAUSE_MOTION_PRESENT (c)
   13774         1659 :                      ? GOMP_MAP_ALWAYS_PRESENT_TO : GOMP_MAP_TO);
   13775              :                 tkind_zero = tkind;
   13776              :                 break;
   13777         6034 :               case OMP_CLAUSE_FROM:
   13778         6034 :                 tkind
   13779         6034 :                   = (OMP_CLAUSE_MOTION_PRESENT (c)
   13780         6034 :                      ? GOMP_MAP_ALWAYS_PRESENT_FROM : GOMP_MAP_FROM);
   13781              :                 tkind_zero = tkind;
   13782              :                 break;
   13783            0 :               default:
   13784            0 :                 gcc_unreachable ();
   13785              :               }
   13786        68674 :             gcc_checking_assert (tkind
   13787              :                                  < (HOST_WIDE_INT_C (1U) << talign_shift));
   13788        80263 :             gcc_checking_assert (tkind_zero
   13789              :                                  < (HOST_WIDE_INT_C (1U) << talign_shift));
   13790        80263 :             talign = ceil_log2 (talign);
   13791        80263 :             tkind |= talign << talign_shift;
   13792        80263 :             tkind_zero |= talign << talign_shift;
   13793        80263 :             gcc_checking_assert (tkind
   13794              :                                  <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
   13795        80263 :             gcc_checking_assert (tkind_zero
   13796              :                                  <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
   13797        80263 :             if (tkind == tkind_zero)
   13798        78800 :               x = build_int_cstu (tkind_type, tkind);
   13799              :             else
   13800              :               {
   13801         1463 :                 TREE_STATIC (TREE_VEC_ELT (t, 2)) = 0;
   13802         1463 :                 x = build3 (COND_EXPR, tkind_type,
   13803              :                             fold_build2 (EQ_EXPR, boolean_type_node,
   13804              :                                          unshare_expr (s), size_zero_node),
   13805         1463 :                             build_int_cstu (tkind_type, tkind_zero),
   13806         1463 :                             build_int_cstu (tkind_type, tkind));
   13807              :               }
   13808        80263 :             CONSTRUCTOR_APPEND_ELT (vkind, purpose, x);
   13809        80263 :             if (nc && nc != c)
   13810            0 :               c = nc;
   13811              :             break;
   13812              : 
   13813        15467 :           case OMP_CLAUSE_FIRSTPRIVATE:
   13814        15467 :           omp_has_device_addr_descr:
   13815        15467 :             if (is_gimple_omp_oacc (ctx->stmt))
   13816         3896 :               goto oacc_firstprivate_map;
   13817        11571 :             ovar = OMP_CLAUSE_DECL (c);
   13818        11571 :             if (omp_privatize_by_reference (ovar))
   13819          812 :               talign = TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
   13820              :             else
   13821        10759 :               talign = DECL_ALIGN_UNIT (ovar);
   13822        11571 :             var = lookup_decl_in_outer_ctx (ovar, ctx);
   13823        11571 :             x = build_sender_ref (ovar, ctx);
   13824        11571 :             tkind = GOMP_MAP_FIRSTPRIVATE;
   13825        11571 :             type = TREE_TYPE (ovar);
   13826        11571 :             if (omp_privatize_by_reference (ovar))
   13827          812 :               type = TREE_TYPE (type);
   13828        11571 :             if ((INTEGRAL_TYPE_P (type)
   13829        11148 :                  && TYPE_PRECISION (type) <= POINTER_SIZE)
   13830        11581 :                 || TREE_CODE (type) == POINTER_TYPE)
   13831              :               {
   13832        11297 :                 tkind = GOMP_MAP_FIRSTPRIVATE_INT;
   13833        11297 :                 tree t = var;
   13834        11297 :                 if (omp_privatize_by_reference (var))
   13835          757 :                   t = build_simple_mem_ref (var);
   13836        10540 :                 else if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
   13837         9331 :                   suppress_warning (var);
   13838        11297 :                 if (TREE_CODE (type) != POINTER_TYPE)
   13839        11138 :                   t = fold_convert (pointer_sized_int_node, t);
   13840        11297 :                 t = fold_convert (TREE_TYPE (x), t);
   13841        11297 :                 gimplify_assign (x, t, &ilist);
   13842              :               }
   13843          274 :             else if (omp_privatize_by_reference (var))
   13844           55 :               gimplify_assign (x, var, &ilist);
   13845          219 :             else if (is_gimple_reg (var))
   13846              :               {
   13847           51 :                 tree avar = create_tmp_var (TREE_TYPE (var));
   13848           51 :                 mark_addressable (avar);
   13849           51 :                 if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
   13850           44 :                   suppress_warning (var);
   13851           51 :                 gimplify_assign (avar, var, &ilist);
   13852           51 :                 avar = build_fold_addr_expr (avar);
   13853           51 :                 gimplify_assign (x, avar, &ilist);
   13854              :               }
   13855              :             else
   13856              :               {
   13857          168 :                 var = build_fold_addr_expr (var);
   13858          168 :                 gimplify_assign (x, var, &ilist);
   13859              :               }
   13860          274 :             if (tkind == GOMP_MAP_FIRSTPRIVATE_INT)
   13861        11297 :               s = size_int (0);
   13862          274 :             else if (omp_privatize_by_reference (ovar))
   13863           55 :               s = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar)));
   13864              :             else
   13865          219 :               s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
   13866        11571 :             s = fold_convert (size_type_node, s);
   13867        11571 :             purpose = size_int (map_idx++);
   13868        11571 :             CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
   13869        11571 :             if (TREE_CODE (s) != INTEGER_CST)
   13870           39 :               TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
   13871              : 
   13872        11571 :             gcc_checking_assert (tkind
   13873              :                                  < (HOST_WIDE_INT_C (1U) << talign_shift));
   13874        11571 :             talign = ceil_log2 (talign);
   13875        11571 :             tkind |= talign << talign_shift;
   13876        11571 :             gcc_checking_assert (tkind
   13877              :                                  <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
   13878        11571 :             CONSTRUCTOR_APPEND_ELT (vkind, purpose,
   13879              :                                     build_int_cstu (tkind_type, tkind));
   13880              :             /* Fortran array descriptors: firstprivate of data + attach.  */
   13881        11571 :             if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
   13882        11552 :                 && lang_hooks.decls.omp_array_data (ovar, true)
   13883        11602 :                 && lang_hooks.decls.omp_array_data_privatize (ovar))
   13884              :               {
   13885           23 :                 tree not_null_lb, null_lb, after_lb;
   13886           23 :                 tree var1, var2, size1, size2;
   13887           23 :                 tree present = omp_check_optional_argument (ovar, true);
   13888           23 :                 if (present)
   13889              :                   {
   13890            1 :                     location_t clause_loc = OMP_CLAUSE_LOCATION (c);
   13891            1 :                     not_null_lb = create_artificial_label (clause_loc);
   13892            1 :                     null_lb = create_artificial_label (clause_loc);
   13893            1 :                     after_lb = create_artificial_label (clause_loc);
   13894            1 :                     gimple_seq seq = NULL;
   13895            1 :                     present = force_gimple_operand (present, &seq, true,
   13896              :                                                     NULL_TREE);
   13897            1 :                     gimple_seq_add_seq (&ilist, seq);
   13898            1 :                     gimple_seq_add_stmt (&ilist,
   13899            1 :                       gimple_build_cond_from_tree (present,
   13900              :                                                    not_null_lb, null_lb));
   13901            1 :                     gimple_seq_add_stmt (&ilist,
   13902            1 :                                          gimple_build_label (not_null_lb));
   13903              :                   }
   13904           23 :                 var1 = lang_hooks.decls.omp_array_data (var, false);
   13905           23 :                 size1 = lang_hooks.decls.omp_array_size (var, &ilist);
   13906           23 :                 var2 = build_fold_addr_expr (x);
   13907           23 :                 if (!POINTER_TYPE_P (TREE_TYPE (var)))
   13908            0 :                   var = build_fold_addr_expr (var);
   13909           23 :                 size2 = fold_build2 (POINTER_DIFF_EXPR, ssizetype,
   13910              :                                    build_fold_addr_expr (var1), var);
   13911           23 :                 size2 = fold_convert (sizetype, size2);
   13912           23 :                 if (present)
   13913              :                   {
   13914            1 :                     tree tmp = create_tmp_var (TREE_TYPE (var1));
   13915            1 :                     gimplify_assign (tmp, var1, &ilist);
   13916            1 :                     var1 = tmp;
   13917            1 :                     tmp = create_tmp_var (TREE_TYPE (var2));
   13918            1 :                     gimplify_assign (tmp, var2, &ilist);
   13919            1 :                     var2 = tmp;
   13920            1 :                     tmp = create_tmp_var (TREE_TYPE (size1));
   13921            1 :                     gimplify_assign (tmp, size1, &ilist);
   13922            1 :                     size1 = tmp;
   13923            1 :                     tmp = create_tmp_var (TREE_TYPE (size2));
   13924            1 :                     gimplify_assign (tmp, size2, &ilist);
   13925            1 :                     size2 = tmp;
   13926            1 :                     gimple_seq_add_stmt (&ilist, gimple_build_goto (after_lb));
   13927            1 :                     gimple_seq_add_stmt (&ilist, gimple_build_label (null_lb));
   13928            1 :                     gimplify_assign (var1, null_pointer_node, &ilist);
   13929            1 :                     gimplify_assign (var2, null_pointer_node, &ilist);
   13930            1 :                     gimplify_assign (size1, size_zero_node, &ilist);
   13931            1 :                     gimplify_assign (size2, size_zero_node, &ilist);
   13932            1 :                     gimple_seq_add_stmt (&ilist, gimple_build_label (after_lb));
   13933              :                   }
   13934           23 :                 x = build_sender_ref ((splay_tree_key) &DECL_NAME (ovar), ctx);
   13935           23 :                 gimplify_assign (x, var1, &ilist);
   13936           23 :                 tkind = GOMP_MAP_FIRSTPRIVATE;
   13937           23 :                 talign = DECL_ALIGN_UNIT (ovar);
   13938           23 :                 talign = ceil_log2 (talign);
   13939           23 :                 tkind |= talign << talign_shift;
   13940           23 :                 gcc_checking_assert (tkind
   13941              :                                      <= tree_to_uhwi (
   13942              :                                           TYPE_MAX_VALUE (tkind_type)));
   13943           23 :                 purpose = size_int (map_idx++);
   13944           23 :                 CONSTRUCTOR_APPEND_ELT (vsize, purpose, size1);
   13945           23 :                 if (TREE_CODE (size1) != INTEGER_CST)
   13946           23 :                   TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
   13947           23 :                 CONSTRUCTOR_APPEND_ELT (vkind, purpose,
   13948              :                                         build_int_cstu (tkind_type, tkind));
   13949           23 :                 x = build_sender_ref ((splay_tree_key) &DECL_UID (ovar), ctx);
   13950           23 :                 gimplify_assign (x, var2, &ilist);
   13951           23 :                 tkind = GOMP_MAP_ATTACH;
   13952           23 :                 purpose = size_int (map_idx++);
   13953           23 :                 CONSTRUCTOR_APPEND_ELT (vsize, purpose, size2);
   13954           23 :                 CONSTRUCTOR_APPEND_ELT (vkind, purpose,
   13955              :                                         build_int_cstu (tkind_type, tkind));
   13956              :               }
   13957              :             break;
   13958              : 
   13959         2431 :           case OMP_CLAUSE_USE_DEVICE_PTR:
   13960         2431 :           case OMP_CLAUSE_USE_DEVICE_ADDR:
   13961         2431 :           case OMP_CLAUSE_HAS_DEVICE_ADDR:
   13962         2431 :           case OMP_CLAUSE_IS_DEVICE_PTR:
   13963         2431 :             ovar = OMP_CLAUSE_DECL (c);
   13964         2431 :             if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
   13965              :               {
   13966          236 :                 if (lang_hooks.decls.omp_array_data (ovar, true))
   13967           19 :                   goto omp_has_device_addr_descr;
   13968          234 :                 while (TREE_CODE (ovar) == INDIRECT_REF
   13969          234 :                        || TREE_CODE (ovar) == ARRAY_REF)
   13970           17 :                   ovar = TREE_OPERAND (ovar, 0);
   13971              :               }
   13972         2412 :             var = lookup_decl_in_outer_ctx (ovar, ctx);
   13973              : 
   13974         2412 :             if (lang_hooks.decls.omp_array_data (ovar, true))
   13975              :               {
   13976          601 :                 tkind = ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
   13977          601 :                           && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
   13978          601 :                          ? GOMP_MAP_USE_DEVICE_PTR : GOMP_MAP_FIRSTPRIVATE_INT);
   13979          601 :                 x = build_sender_ref ((splay_tree_key) &DECL_NAME (ovar), ctx);
   13980              :               }
   13981         1811 :             else if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
   13982         1811 :                      && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
   13983              :               {
   13984         1453 :                 tkind = GOMP_MAP_USE_DEVICE_PTR;
   13985         1453 :                 x = build_sender_ref ((splay_tree_key) &DECL_UID (ovar), ctx);
   13986              :               }
   13987              :             else
   13988              :               {
   13989          358 :                 tkind = GOMP_MAP_FIRSTPRIVATE_INT;
   13990          358 :                 x = build_sender_ref (ovar, ctx);
   13991              :               }
   13992              : 
   13993         2412 :             if (is_gimple_omp_oacc (ctx->stmt))
   13994              :               {
   13995          147 :                 gcc_assert (tkind == GOMP_MAP_USE_DEVICE_PTR);
   13996              : 
   13997          147 :                 if (OMP_CLAUSE_USE_DEVICE_PTR_IF_PRESENT (c))
   13998           57 :                   tkind = GOMP_MAP_USE_DEVICE_PTR_IF_PRESENT;
   13999              :               }
   14000              : 
   14001         2412 :             type = TREE_TYPE (ovar);
   14002         2412 :             if (lang_hooks.decls.omp_array_data (ovar, true))
   14003          601 :               var = lang_hooks.decls.omp_array_data (var, false);
   14004         1811 :             else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
   14005          543 :                       || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
   14006         1485 :                       && !omp_privatize_by_reference (ovar)
   14007          512 :                       && !omp_is_allocatable_or_ptr (ovar))
   14008         3249 :                      || TREE_CODE (type) == ARRAY_TYPE)
   14009          401 :               var = build_fold_addr_expr (var);
   14010              :             else
   14011              :               {
   14012         1410 :                 if (omp_privatize_by_reference (ovar)
   14013          370 :                     || omp_check_optional_argument (ovar, false)
   14014         1780 :                     || omp_is_allocatable_or_ptr (ovar))
   14015              :                   {
   14016         1213 :                     type = TREE_TYPE (type);
   14017         1213 :                     if (POINTER_TYPE_P (type)
   14018         1213 :                         && TREE_CODE (type) != ARRAY_TYPE
   14019         1572 :                         && ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_USE_DEVICE_ADDR
   14020           36 :                             && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
   14021           28 :                             && !omp_is_allocatable_or_ptr (ovar))
   14022          337 :                            || (omp_privatize_by_reference (ovar)
   14023          337 :                                && omp_is_allocatable_or_ptr (ovar))))
   14024          357 :                       var = build_simple_mem_ref (var);
   14025         1213 :                     var = fold_convert (TREE_TYPE (x), var);
   14026              :                   }
   14027              :               }
   14028         2412 :             tree present;
   14029         2412 :             present = omp_check_optional_argument (ovar, true);
   14030         2412 :             if (present)
   14031              :               {
   14032          878 :                 tree null_label = create_artificial_label (UNKNOWN_LOCATION);
   14033          878 :                 tree notnull_label = create_artificial_label (UNKNOWN_LOCATION);
   14034          878 :                 tree opt_arg_label = create_artificial_label (UNKNOWN_LOCATION);
   14035          878 :                 tree new_x = unshare_expr (x);
   14036          878 :                 gimplify_expr (&present, &ilist, NULL, is_gimple_val,
   14037              :                                fb_rvalue);
   14038          878 :                 gcond *cond = gimple_build_cond_from_tree (present,
   14039              :                                                            notnull_label,
   14040              :                                                            null_label);
   14041          878 :                 gimple_seq_add_stmt (&ilist, cond);
   14042          878 :                 gimple_seq_add_stmt (&ilist, gimple_build_label (null_label));
   14043          878 :                 gimplify_assign (new_x, null_pointer_node, &ilist);
   14044          878 :                 gimple_seq_add_stmt (&ilist, gimple_build_goto (opt_arg_label));
   14045          878 :                 gimple_seq_add_stmt (&ilist,
   14046          878 :                                      gimple_build_label (notnull_label));
   14047          878 :                 gimplify_assign (x, var, &ilist);
   14048          878 :                 gimple_seq_add_stmt (&ilist,
   14049          878 :                                      gimple_build_label (opt_arg_label));
   14050              :               }
   14051              :             else
   14052         1534 :               gimplify_assign (x, var, &ilist);
   14053         2412 :             s = size_int (0);
   14054         2412 :             purpose = size_int (map_idx++);
   14055         2412 :             CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
   14056         2412 :             gcc_checking_assert (tkind
   14057              :                                  < (HOST_WIDE_INT_C (1U) << talign_shift));
   14058         2412 :             gcc_checking_assert (tkind
   14059              :                                  <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
   14060         2412 :             CONSTRUCTOR_APPEND_ELT (vkind, purpose,
   14061              :                                     build_int_cstu (tkind_type, tkind));
   14062         2412 :             break;
   14063              : 
   14064           86 :           case OMP_CLAUSE_USES_ALLOCATORS:
   14065           86 :             tree allocator = OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c);
   14066           86 :             tree memspace = OMP_CLAUSE_USES_ALLOCATORS_MEMSPACE (c);
   14067           86 :             tree traits = OMP_CLAUSE_USES_ALLOCATORS_TRAITS (c);
   14068              : 
   14069           86 :             tree ntraits, traits_var;
   14070           86 :             if (traits == NULL_TREE)
   14071              :               {
   14072           29 :                 ntraits = integer_zero_node;
   14073           29 :                 traits_var = null_pointer_node;
   14074              :               }
   14075              :             else
   14076              :               {
   14077           57 :                 location_t loc = OMP_CLAUSE_LOCATION (c);
   14078           57 :                 ntraits = array_type_nelts_top (TREE_TYPE (traits));
   14079           57 :                 traits_var = build_fold_addr_expr_loc (loc, traits);
   14080              :               }
   14081              : 
   14082           86 :             if (memspace == NULL_TREE)
   14083           61 :               memspace = build_int_cst (pointer_sized_int_node, 0);
   14084              :             else
   14085           25 :               memspace = fold_convert (pointer_sized_int_node, memspace);
   14086              : 
   14087           86 :             tree arr_type
   14088           86 :               = build_array_type_nelts (pointer_sized_int_node, 3);
   14089           86 :             tree uses_allocators_descr
   14090           86 :               = create_tmp_var (arr_type, "uses_allocator_descr");
   14091           86 :             tree ua_descr[3];
   14092          430 :             for (int i = 0; i < 3; i++)
   14093          258 :               ua_descr[i] = build4 (ARRAY_REF, pointer_sized_int_node,
   14094              :                                     uses_allocators_descr,
   14095          258 :                                     build_int_cst (size_type_node, i),
   14096              :                                     NULL_TREE, NULL_TREE);
   14097           86 :             gimplify_assign (ua_descr[0],
   14098              :                              fold_convert (pointer_sized_int_node, memspace),
   14099              :                              &ilist);
   14100           86 :             gimplify_assign (ua_descr[1],
   14101              :                              fold_convert (pointer_sized_int_node, ntraits),
   14102              :                              &ilist);
   14103           86 :             gimplify_assign (ua_descr[2],
   14104              :                              fold_convert (pointer_sized_int_node, traits_var),
   14105              :                              &ilist);
   14106              : 
   14107           86 :             x = build_sender_ref (allocator, ctx);
   14108           86 :             tree ptr = build_fold_addr_expr (uses_allocators_descr);
   14109           86 :             gimplify_assign (x, fold_convert (TREE_TYPE (x), ptr), &ilist);
   14110              : 
   14111           86 :             s = size_int (0);
   14112           86 :             purpose = size_int (map_idx++);
   14113           86 :             CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
   14114           86 :             tkind = GOMP_MAP_USES_ALLOCATORS;
   14115           86 :             gcc_checking_assert (tkind
   14116              :                                  < (HOST_WIDE_INT_C (1U) << talign_shift));
   14117           86 :             talign = TYPE_ALIGN_UNIT (pointer_sized_int_node);
   14118           86 :             talign = ceil_log2 (talign);
   14119           86 :             tkind |= talign << talign_shift;
   14120           86 :             gcc_checking_assert (tkind
   14121              :                                  <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
   14122           86 :             CONSTRUCTOR_APPEND_ELT (vkind, purpose,
   14123              :                                     build_int_cstu (tkind_type, tkind));
   14124           86 :             break;
   14125              :           }
   14126              : 
   14127        32403 :       gcc_assert (map_idx == map_cnt);
   14128              : 
   14129        32403 :       if (!deep_map_cnt)
   14130              :         {
   14131        32259 :           DECL_INITIAL (TREE_VEC_ELT (t, 1))
   14132        32259 :             = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), vsize);
   14133        32259 :           DECL_INITIAL (TREE_VEC_ELT (t, 2))
   14134        64518 :             = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 2)), vkind);
   14135              :         }
   14136        97209 :       for (int i = 1; i <= 2; i++)
   14137        64806 :         if (deep_map_cnt || !TREE_STATIC (TREE_VEC_ELT (t, i)))
   14138              :           {
   14139         9667 :             tree tmp = TREE_VEC_ELT (t, i);
   14140         9667 :             if (deep_map_cnt)
   14141              :               {
   14142          288 :                 const char *prefix = (i == 1 ? ".omp_data_sizes0"
   14143              :                                              : ".omp_data_kinds0");
   14144          144 :                 tree type = (i == 1) ? size_type_node : tkind_type;
   14145          288 :                 type = build_array_type_nelts (type, map_cnt);
   14146          288 :                 tree var = create_tmp_var (type, prefix);
   14147          288 :                 DECL_NAMELESS (var) = 1;
   14148          288 :                 TREE_ADDRESSABLE (var) = 1;
   14149          288 :                 TREE_STATIC (var) = TREE_STATIC (tmp);
   14150          288 :                 DECL_INITIAL (var) = build_constructor (type, i == 1
   14151              :                                                               ? vsize : vkind);
   14152          288 :                 tmp = var;
   14153          288 :                 TREE_STATIC (TREE_VEC_ELT (t, i)) = 0;
   14154              :               }
   14155              : 
   14156         9667 :             gimple_seq initlist = NULL;
   14157         9667 :             force_gimple_operand (build1 (DECL_EXPR, void_type_node, tmp),
   14158              :                                   &initlist, true, NULL_TREE);
   14159         9667 :             gimple_seq_add_seq (&ilist, initlist);
   14160              : 
   14161         9667 :             if (deep_map_cnt)
   14162              :               {
   14163          288 :                 tree tmp2;
   14164          288 :                 tree call = builtin_decl_explicit (BUILT_IN_MEMCPY);
   14165          288 :                 tmp2 = TYPE_SIZE_UNIT (TREE_TYPE (tmp));
   14166          576 :                 call = build_call_expr_loc (input_location, call, 3,
   14167          288 :                                             TREE_VEC_ELT (t, i),
   14168              :                                             build_fold_addr_expr (tmp), tmp2);
   14169          288 :                 gimplify_and_add (call, &ilist);
   14170              :               }
   14171              : 
   14172         9667 :             if (!TREE_STATIC (tmp))
   14173              :               {
   14174         9449 :                 tree clobber = build_clobber (TREE_TYPE (tmp));
   14175         9449 :                 gimple_seq_add_stmt (&olist,
   14176         9449 :                                      gimple_build_assign (tmp, clobber));
   14177              :               }
   14178         9667 :             if (deep_map_cnt)
   14179              :               {
   14180          288 :                 tmp = TREE_VEC_ELT (t, i);
   14181          288 :                 tree call = builtin_decl_explicit (BUILT_IN_FREE);
   14182          288 :                 call = build_call_expr_loc (input_location, call, 1, tmp);
   14183          288 :                 gimplify_and_add (call, &olist);
   14184          288 :                 tree clobber = build_clobber (TREE_TYPE (tmp));
   14185          288 :                 gimple_seq_add_stmt (&olist,
   14186          288 :                                      gimple_build_assign (tmp, clobber));
   14187              :               }
   14188              :           }
   14189        55139 :         else if (omp_maybe_offloaded_ctx (ctx->outer))
   14190              :           {
   14191         3296 :             tree id = get_identifier ("omp declare target");
   14192         3296 :             tree decl = TREE_VEC_ELT (t, i);
   14193         3296 :             DECL_ATTRIBUTES (decl)
   14194         3296 :               = tree_cons (id, NULL_TREE, DECL_ATTRIBUTES (decl));
   14195         3296 :             varpool_node *node = varpool_node::get (decl);
   14196         3296 :             if (node)
   14197              :               {
   14198            0 :                 node->offloadable = 1;
   14199            0 :                 if (ENABLE_OFFLOADING)
   14200              :                   {
   14201              :                     g->have_offload = true;
   14202              :                     vec_safe_push (offload_vars, t);
   14203              :                   }
   14204              :               }
   14205              :           }
   14206              : 
   14207        32403 :       if (deep_map_cnt)
   14208              :         {
   14209          144 :           tree call = builtin_decl_explicit (BUILT_IN_FREE);
   14210          144 :           call = build_call_expr_loc (input_location, call, 1,
   14211          144 :                                       TREE_VEC_ELT (t, 0));
   14212          144 :           gimplify_and_add (call, &olist);
   14213              : 
   14214          144 :           gimplify_expr (&TREE_VEC_ELT (t, 1), &ilist, NULL, is_gimple_val,
   14215              :                          fb_rvalue);
   14216              :         }
   14217              : 
   14218        32403 :       tree clobber = build_clobber (TREE_TYPE (ctx->sender_decl));
   14219        32403 :       gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
   14220              :                                                         clobber));
   14221              :     }
   14222              : 
   14223              :   /* Once all the expansions are done, sequence all the different
   14224              :      fragments inside gimple_omp_body.  */
   14225              : 
   14226        37314 :   new_body = NULL;
   14227              : 
   14228        37314 :   if (offloaded
   14229        21311 :       && ctx->record_type)
   14230              :     {
   14231        16633 :       t = ctx->sender_decl;
   14232        16633 :       if (!deep_map_cnt)
   14233        16526 :         t = build_fold_addr_expr_loc (loc, t);
   14234              :       /* fixup_child_record_type might have changed receiver_decl's type.  */
   14235        16633 :       t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
   14236        16633 :       if (!AGGREGATE_TYPE_P (TREE_TYPE (ctx->sender_decl)))
   14237          107 :         gimplify_assign (ctx->receiver_decl, t, &new_body);
   14238              :       else
   14239        16526 :         gimple_seq_add_stmt (&new_body,
   14240        16526 :                              gimple_build_assign (ctx->receiver_decl, t));
   14241              :     }
   14242        37314 :   gimple_seq_add_seq (&new_body, fplist);
   14243              : 
   14244        37314 :   if (offloaded || data_region)
   14245              :     {
   14246       141697 :       tree prev = NULL_TREE;
   14247              :       bool by_ref;
   14248       141697 :       for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
   14249       116362 :         switch (OMP_CLAUSE_CODE (c))
   14250              :           {
   14251              :             tree var, x, new_var, *allocate_ptr;
   14252              :           default:
   14253              :             break;
   14254        15467 :           case OMP_CLAUSE_FIRSTPRIVATE:
   14255        15467 :           omp_firstprivatize_data_region:
   14256        15467 :             if (is_gimple_omp_oacc (ctx->stmt))
   14257              :               break;
   14258        11571 :             var = OMP_CLAUSE_DECL (c);
   14259        11571 :             new_var = lookup_decl (var, ctx);
   14260        11571 :             allocate_ptr = alloc_map.get (new_var);
   14261        11571 :             by_ref = omp_privatize_by_reference (var);
   14262        11571 :             if (allocate_ptr)
   14263              :               {
   14264           68 :                 if (is_variable_sized (var, by_ref))
   14265              :                   /* Handle this in the next pass when the size is
   14266              :                      available.  */
   14267              :                   break;
   14268              : 
   14269           53 :                 gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
   14270           53 :                 gcc_assert (allocate_seq);
   14271           53 :                 gimple_seq_add_seq (&new_body, *allocate_seq);
   14272              : 
   14273           53 :                 if (by_ref)
   14274              :                   {
   14275            4 :                     x = fold_convert (TREE_TYPE (new_var), *allocate_ptr);
   14276            4 :                     gimplify_assign (new_var, x, &new_body);
   14277            4 :                     new_var = build_fold_indirect_ref (new_var);
   14278              :                   }
   14279              :                 else
   14280           49 :                   new_var = build_simple_mem_ref (*allocate_ptr);
   14281              :               }
   14282        11556 :             if (by_ref || is_gimple_reg_type (TREE_TYPE (var)))
   14283              :               {
   14284        11402 :                 tree type;
   14285        11402 :                 type = TREE_TYPE (var);
   14286        11402 :                 if (by_ref)
   14287          799 :                   type = TREE_TYPE (type);
   14288        11402 :                 if ((INTEGRAL_TYPE_P (type)
   14289        11148 :                      && TYPE_PRECISION (type) <= POINTER_SIZE)
   14290        11412 :                     || TREE_CODE (type) == POINTER_TYPE)
   14291              :                   {
   14292        11297 :                     x = build_receiver_ref (var, false, ctx);
   14293        11297 :                     if (TREE_CODE (type) != POINTER_TYPE)
   14294        11138 :                       x = fold_convert (pointer_sized_int_node, x);
   14295        11297 :                     x = fold_convert (type, x);
   14296        11297 :                     gimplify_expr (&x, &new_body, NULL, is_gimple_val,
   14297              :                                    fb_rvalue);
   14298        11297 :                     if (by_ref && !allocate_ptr)
   14299              :                       {
   14300          756 :                         tree v = create_tmp_var_raw (type, get_name (var));
   14301          756 :                         gimple_add_tmp_var (v);
   14302          756 :                         TREE_ADDRESSABLE (v) = 1;
   14303          756 :                         gimple_seq_add_stmt (&new_body,
   14304          756 :                                              gimple_build_assign (v, x));
   14305          756 :                         x = build_fold_addr_expr (v);
   14306              :                       }
   14307        11297 :                     gimplify_assign (new_var, x, &new_body);
   14308              :                   }
   14309              :                 else
   14310              :                   {
   14311          105 :                     x = build_receiver_ref (var, allocate_ptr || !by_ref, ctx);
   14312          105 :                     gimplify_expr (&x, &new_body, NULL, is_gimple_val,
   14313              :                                    fb_rvalue);
   14314          105 :                     gimplify_assign (new_var, x, &new_body);
   14315              :                   }
   14316              :               }
   14317          154 :             else if (is_variable_sized (var))
   14318              :               {
   14319            8 :                 tree pvar = DECL_VALUE_EXPR (var);
   14320            8 :                 gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
   14321            8 :                 pvar = TREE_OPERAND (pvar, 0);
   14322            8 :                 gcc_assert (DECL_P (pvar));
   14323            8 :                 tree new_var = lookup_decl (pvar, ctx);
   14324            8 :                 x = build_receiver_ref (var, false, ctx);
   14325            8 :                 gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
   14326            8 :                 gimple_seq_add_stmt (&new_body,
   14327            8 :                                      gimple_build_assign (new_var, x));
   14328              :               }
   14329          146 :             else if (allocate_ptr)
   14330              :               {
   14331           22 :                 x = build_receiver_ref (var, true, ctx);
   14332           22 :                 new_var = unshare_expr (new_var);
   14333           22 :                 x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
   14334           22 :                 gimplify_and_add (x, &new_body);
   14335              :               }
   14336              :             break;
   14337          228 :           case OMP_CLAUSE_PRIVATE:
   14338          228 :             if (is_gimple_omp_oacc (ctx->stmt))
   14339              :               break;
   14340          148 :             var = OMP_CLAUSE_DECL (c);
   14341          148 :             new_var = lookup_decl (var, ctx);
   14342          148 :             allocate_ptr = alloc_map.get (new_var);
   14343          148 :             by_ref = omp_privatize_by_reference (var);
   14344          148 :             if (allocate_ptr)
   14345              :               {
   14346           57 :                 if (is_variable_sized (var, by_ref))
   14347              :                   /* Handle this in the next pass when the size is
   14348              :                      available.  */
   14349              :                   break;
   14350              : 
   14351           45 :                 gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
   14352           45 :                 gcc_assert (allocate_seq);
   14353           45 :                 gimple_seq_add_seq (&new_body, *allocate_seq);
   14354              : 
   14355           45 :                 if (!by_ref)
   14356           44 :                   new_var = build_simple_mem_ref (*allocate_ptr);
   14357              :               }
   14358          135 :             if (by_ref)
   14359              :               {
   14360           12 :                 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
   14361           12 :                 if (!allocate_ptr)
   14362              :                   {
   14363           11 :                     x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
   14364           11 :                     if (TREE_CONSTANT (x))
   14365              :                       {
   14366            3 :                         x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
   14367              :                                                 get_name (var));
   14368            3 :                         gimple_add_tmp_var (x);
   14369            3 :                         TREE_ADDRESSABLE (x) = 1;
   14370            3 :                         x = build_fold_addr_expr_loc (clause_loc, x);
   14371              :                       }
   14372              :                     else
   14373              :                       break;
   14374              : 
   14375            3 :                     x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
   14376            3 :                     gimplify_expr (&x, &new_body, NULL, is_gimple_val,
   14377              :                                    fb_rvalue);
   14378              :                   }
   14379              :                 else
   14380            1 :                   x = *allocate_ptr;
   14381              : 
   14382            4 :                 gimple_seq_add_stmt (&new_body,
   14383            4 :                                      gimple_build_assign (new_var, x));
   14384              :               }
   14385              :             break;
   14386         2431 :           case OMP_CLAUSE_USE_DEVICE_PTR:
   14387         2431 :           case OMP_CLAUSE_USE_DEVICE_ADDR:
   14388         2431 :           case OMP_CLAUSE_HAS_DEVICE_ADDR:
   14389         2431 :           case OMP_CLAUSE_IS_DEVICE_PTR:
   14390         2431 :             gimple_seq assign_body;
   14391         2431 :             bool is_array_data;
   14392         2431 :             bool do_optional_check;
   14393         2431 :             assign_body = NULL;
   14394         2431 :             do_optional_check = false;
   14395         2431 :             var = OMP_CLAUSE_DECL (c);
   14396         2431 :             is_array_data = lang_hooks.decls.omp_array_data (var, true) != NULL;
   14397         2431 :             if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR && is_array_data)
   14398           19 :               goto omp_firstprivatize_data_region;
   14399              : 
   14400         2412 :             if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
   14401         2412 :                 && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
   14402         4108 :               x = build_sender_ref (is_array_data
   14403          601 :                                     ? (splay_tree_key) &DECL_NAME (var)
   14404         1453 :                                     : (splay_tree_key) &DECL_UID (var), ctx);
   14405              :             else
   14406              :               {
   14407          358 :                 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
   14408              :                   {
   14409          234 :                     while (TREE_CODE (var) == INDIRECT_REF
   14410          234 :                            || TREE_CODE (var) == ARRAY_REF)
   14411           17 :                       var = TREE_OPERAND (var, 0);
   14412              :                   }
   14413          358 :                 x = build_receiver_ref (var, false, ctx);
   14414              :               }
   14415              : 
   14416         2412 :             if (is_array_data)
   14417              :               {
   14418          601 :                 bool is_ref = omp_privatize_by_reference (var);
   14419          601 :                 do_optional_check = true;
   14420              :                 /* First, we copy the descriptor data from the host; then
   14421              :                    we update its data to point to the target address.  */
   14422          601 :                 new_var = lookup_decl (var, ctx);
   14423          601 :                 new_var = DECL_VALUE_EXPR (new_var);
   14424          601 :                 tree v = new_var;
   14425          601 :                 tree v2 = var;
   14426          601 :                 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_PTR
   14427          601 :                     || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR)
   14428          601 :                   v2 = maybe_lookup_decl_in_outer_ctx (var, ctx);
   14429              : 
   14430          601 :                 if (is_ref)
   14431              :                   {
   14432          396 :                     v2 = build_fold_indirect_ref (v2);
   14433          396 :                     v = create_tmp_var_raw (TREE_TYPE (v2), get_name (var));
   14434          396 :                     gimple_add_tmp_var (v);
   14435          396 :                     TREE_ADDRESSABLE (v) = 1;
   14436          396 :                     gimplify_assign (v, v2, &assign_body);
   14437          396 :                     tree rhs = build_fold_addr_expr (v);
   14438          396 :                     gimple_seq_add_stmt (&assign_body,
   14439          396 :                                          gimple_build_assign (new_var, rhs));
   14440              :                   }
   14441              :                 else
   14442          205 :                   gimplify_assign (new_var, v2, &assign_body);
   14443              : 
   14444          601 :                 v2 = lang_hooks.decls.omp_array_data (unshare_expr (v), false);
   14445          601 :                 gcc_assert (v2);
   14446          601 :                 gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
   14447         1202 :                 gimple_seq_add_stmt (&assign_body,
   14448          601 :                                      gimple_build_assign (v2, x));
   14449              :               }
   14450         1811 :             else if (is_variable_sized (var))
   14451              :               {
   14452           12 :                 tree pvar = DECL_VALUE_EXPR (var);
   14453           12 :                 gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
   14454           12 :                 pvar = TREE_OPERAND (pvar, 0);
   14455           12 :                 gcc_assert (DECL_P (pvar));
   14456           12 :                 new_var = lookup_decl (pvar, ctx);
   14457           12 :                 gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
   14458           24 :                 gimple_seq_add_stmt (&assign_body,
   14459           12 :                                      gimple_build_assign (new_var, x));
   14460              :               }
   14461         1799 :             else if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
   14462          537 :                       || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
   14463         1477 :                       && !omp_privatize_by_reference (var)
   14464          504 :                       && !omp_is_allocatable_or_ptr (var))
   14465         3233 :                      || TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
   14466              :               {
   14467          389 :                 new_var = lookup_decl (var, ctx);
   14468          389 :                 new_var = DECL_VALUE_EXPR (new_var);
   14469          389 :                 gcc_assert (TREE_CODE (new_var) == MEM_REF);
   14470          389 :                 new_var = TREE_OPERAND (new_var, 0);
   14471          389 :                 gcc_assert (DECL_P (new_var));
   14472          389 :                 gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
   14473          778 :                 gimple_seq_add_stmt (&assign_body,
   14474          389 :                                      gimple_build_assign (new_var, x));
   14475              :               }
   14476              :             else
   14477              :               {
   14478         1410 :                 tree type = TREE_TYPE (var);
   14479         1410 :                 new_var = lookup_decl (var, ctx);
   14480         1410 :                 if (omp_privatize_by_reference (var))
   14481              :                   {
   14482         1040 :                     type = TREE_TYPE (type);
   14483         1040 :                     if (POINTER_TYPE_P (type)
   14484         1040 :                         && TREE_CODE (type) != ARRAY_TYPE
   14485         1399 :                         && ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_USE_DEVICE_ADDR
   14486           36 :                             && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR)
   14487          331 :                             || (omp_privatize_by_reference (var)
   14488          331 :                                 && omp_is_allocatable_or_ptr (var))))
   14489              :                       {
   14490          357 :                         tree v = create_tmp_var_raw (type, get_name (var));
   14491          357 :                         gimple_add_tmp_var (v);
   14492          357 :                         TREE_ADDRESSABLE (v) = 1;
   14493          357 :                         x = fold_convert (type, x);
   14494          357 :                         gimplify_expr (&x, &assign_body, NULL, is_gimple_val,
   14495              :                                        fb_rvalue);
   14496          357 :                         gimple_seq_add_stmt (&assign_body,
   14497          357 :                                              gimple_build_assign (v, x));
   14498          357 :                         x = build_fold_addr_expr (v);
   14499          357 :                         do_optional_check = true;
   14500              :                       }
   14501              :                   }
   14502         1410 :                 new_var = DECL_VALUE_EXPR (new_var);
   14503         1410 :                 x = fold_convert (TREE_TYPE (new_var), x);
   14504         1410 :                 gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
   14505         1410 :                 gimple_seq_add_stmt (&assign_body,
   14506         1410 :                                      gimple_build_assign (new_var, x));
   14507              :               }
   14508         2412 :             tree present;
   14509         1002 :             present = ((do_optional_check
   14510          958 :                         && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_HAS_DEVICE_ADDR
   14511          952 :                         && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR)
   14512         2343 :                        ? omp_check_optional_argument (OMP_CLAUSE_DECL (c), true)
   14513              :                        : NULL_TREE);
   14514         2412 :             if (present)
   14515              :               {
   14516          480 :                 tree null_label = create_artificial_label (UNKNOWN_LOCATION);
   14517          480 :                 tree notnull_label = create_artificial_label (UNKNOWN_LOCATION);
   14518          480 :                 tree opt_arg_label = create_artificial_label (UNKNOWN_LOCATION);
   14519          480 :                 glabel *null_glabel = gimple_build_label (null_label);
   14520          480 :                 glabel *notnull_glabel = gimple_build_label (notnull_label);
   14521          480 :                 ggoto *opt_arg_ggoto = gimple_build_goto (opt_arg_label);
   14522          480 :                 gimplify_expr (&x, &new_body, NULL, is_gimple_val,
   14523              :                                            fb_rvalue);
   14524          480 :                 gimplify_expr (&present, &new_body, NULL, is_gimple_val,
   14525              :                                fb_rvalue);
   14526          480 :                 gcond *cond = gimple_build_cond_from_tree (present,
   14527              :                                                            notnull_label,
   14528              :                                                            null_label);
   14529          480 :                 gimple_seq_add_stmt (&new_body, cond);
   14530          480 :                 gimple_seq_add_stmt (&new_body, null_glabel);
   14531          480 :                 gimplify_assign (new_var, null_pointer_node, &new_body);
   14532          480 :                 gimple_seq_add_stmt (&new_body, opt_arg_ggoto);
   14533          480 :                 gimple_seq_add_stmt (&new_body, notnull_glabel);
   14534          480 :                 gimple_seq_add_seq (&new_body, assign_body);
   14535          480 :                 gimple_seq_add_stmt (&new_body,
   14536          480 :                                      gimple_build_label (opt_arg_label));
   14537              :               }
   14538              :             else
   14539         1932 :               gimple_seq_add_seq (&new_body, assign_body);
   14540              :             break;
   14541              :           }
   14542              :       /* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass,
   14543              :          so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
   14544              :          are already handled.  Similarly OMP_CLAUSE_PRIVATE for VLAs
   14545              :          or references to VLAs.  */
   14546       141697 :       for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
   14547       116362 :         switch (OMP_CLAUSE_CODE (c))
   14548              :           {
   14549              :             tree var, new_var, *allocate_ptr;
   14550              :           default:
   14551       100938 :             break;
   14552        63606 :           case OMP_CLAUSE_MAP:
   14553        63606 :             if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
   14554        63606 :                 || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
   14555              :               {
   14556         3672 :                 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
   14557         3672 :                 poly_int64 offset = 0;
   14558         3672 :                 gcc_assert (prev);
   14559         3672 :                 var = OMP_CLAUSE_DECL (c);
   14560         3672 :                 if (DECL_P (var)
   14561         3672 :                     && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
   14562         1127 :                     && is_global_var (maybe_lookup_decl_in_outer_ctx (var,
   14563              :                                                                       ctx))
   14564         3770 :                     && varpool_node::get_create (var)->offloadable)
   14565              :                   break;
   14566         3670 :                 if (TREE_CODE (var) == INDIRECT_REF
   14567         3670 :                     && TREE_CODE (TREE_OPERAND (var, 0)) == COMPONENT_REF)
   14568            0 :                   var = TREE_OPERAND (var, 0);
   14569         3670 :                 if (TREE_CODE (var) == COMPONENT_REF)
   14570              :                   {
   14571            0 :                     var = get_addr_base_and_unit_offset (var, &offset);
   14572            0 :                     gcc_assert (var != NULL_TREE && DECL_P (var));
   14573              :                   }
   14574         3670 :                 else if (DECL_SIZE (var)
   14575         3670 :                          && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST)
   14576              :                   {
   14577          712 :                     tree var2 = DECL_VALUE_EXPR (var);
   14578          712 :                     gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
   14579          712 :                     var2 = TREE_OPERAND (var2, 0);
   14580          712 :                     gcc_assert (DECL_P (var2));
   14581              :                     var = var2;
   14582              :                   }
   14583         3670 :                 tree new_var = lookup_decl (var, ctx), x;
   14584         3670 :                 tree type = TREE_TYPE (new_var);
   14585         3670 :                 bool is_ref;
   14586         3670 :                 if (TREE_CODE (OMP_CLAUSE_DECL (c)) == INDIRECT_REF
   14587         3670 :                     && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c), 0))
   14588              :                         == COMPONENT_REF))
   14589              :                   {
   14590            0 :                     type = TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c), 0));
   14591            0 :                     is_ref = true;
   14592            0 :                     new_var = build2 (MEM_REF, type,
   14593              :                                       build_fold_addr_expr (new_var),
   14594              :                                       build_int_cst (build_pointer_type (type),
   14595              :                                                      offset));
   14596              :                   }
   14597         3670 :                 else if (TREE_CODE (OMP_CLAUSE_DECL (c)) == COMPONENT_REF)
   14598              :                   {
   14599            0 :                     type = TREE_TYPE (OMP_CLAUSE_DECL (c));
   14600            0 :                     is_ref = TREE_CODE (type) == REFERENCE_TYPE;
   14601            0 :                     new_var = build2 (MEM_REF, type,
   14602              :                                       build_fold_addr_expr (new_var),
   14603              :                                       build_int_cst (build_pointer_type (type),
   14604              :                                                      offset));
   14605              :                   }
   14606              :                 else
   14607         3670 :                   is_ref = omp_privatize_by_reference (var);
   14608         3670 :                 if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
   14609              :                   is_ref = false;
   14610         6711 :                 bool ref_to_array = false;
   14611         6711 :                 bool ref_to_ptr = false;
   14612         3241 :                 if (is_ref)
   14613              :                   {
   14614          200 :                     type = TREE_TYPE (type);
   14615          200 :                     if (TREE_CODE (type) == ARRAY_TYPE)
   14616              :                       {
   14617          194 :                         type = build_pointer_type (type);
   14618          194 :                         ref_to_array = true;
   14619              :                       }
   14620              :                   }
   14621         3470 :                 else if (TREE_CODE (type) == ARRAY_TYPE)
   14622              :                   {
   14623          413 :                     tree decl2 = DECL_VALUE_EXPR (new_var);
   14624          413 :                     gcc_assert (TREE_CODE (decl2) == MEM_REF);
   14625          413 :                     decl2 = TREE_OPERAND (decl2, 0);
   14626          413 :                     gcc_assert (DECL_P (decl2));
   14627          413 :                     new_var = decl2;
   14628          413 :                     type = TREE_TYPE (new_var);
   14629              :                   }
   14630         3057 :                 else if (TREE_CODE (type) == REFERENCE_TYPE
   14631         3057 :                          && TREE_CODE (TREE_TYPE (type)) == POINTER_TYPE)
   14632              :                   {
   14633          156 :                     type = TREE_TYPE (type);
   14634          156 :                     ref_to_ptr = true;
   14635              :                   }
   14636         3670 :                 x = build_receiver_ref (prev, false, ctx);
   14637         3670 :                 x = fold_convert_loc (clause_loc, type, x);
   14638         3670 :                 if (!integer_zerop (OMP_CLAUSE_SIZE (c)))
   14639              :                   {
   14640          762 :                     tree bias = OMP_CLAUSE_SIZE (c);
   14641          762 :                     if (DECL_P (bias))
   14642          364 :                       bias = lookup_decl (bias, ctx);
   14643          762 :                     bias = fold_convert_loc (clause_loc, sizetype, bias);
   14644          762 :                     bias = fold_build1_loc (clause_loc, NEGATE_EXPR, sizetype,
   14645              :                                             bias);
   14646          762 :                     x = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
   14647          762 :                                          TREE_TYPE (x), x, bias);
   14648              :                   }
   14649         3670 :                 if (ref_to_array)
   14650          194 :                   x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
   14651         3670 :                 gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
   14652         3670 :                 if ((is_ref && !ref_to_array)
   14653         3664 :                     || ref_to_ptr)
   14654              :                   {
   14655          162 :                     tree t = create_tmp_var_raw (type, get_name (var));
   14656          162 :                     gimple_add_tmp_var (t);
   14657          162 :                     TREE_ADDRESSABLE (t) = 1;
   14658          162 :                     gimple_seq_add_stmt (&new_body,
   14659          162 :                                          gimple_build_assign (t, x));
   14660          162 :                     x = build_fold_addr_expr_loc (clause_loc, t);
   14661              :                   }
   14662         3670 :                 gimple_seq_add_stmt (&new_body,
   14663         3670 :                                      gimple_build_assign (new_var, x));
   14664         3670 :                 prev = NULL_TREE;
   14665              :               }
   14666        59934 :             else if (OMP_CLAUSE_CHAIN (c)
   14667        43281 :                      && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c))
   14668              :                         == OMP_CLAUSE_MAP
   14669        97780 :                      && (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c))
   14670              :                          == GOMP_MAP_FIRSTPRIVATE_POINTER
   14671        34603 :                          || (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c))
   14672              :                              == GOMP_MAP_FIRSTPRIVATE_REFERENCE)))
   14673              :               prev = c;
   14674              :             break;
   14675          228 :           case OMP_CLAUSE_PRIVATE:
   14676          228 :             var = OMP_CLAUSE_DECL (c);
   14677          228 :             by_ref = omp_privatize_by_reference (var);
   14678          228 :             new_var = lookup_decl (var, ctx);
   14679          228 :             allocate_ptr = alloc_map.get (new_var);
   14680          228 :             if (is_variable_sized (var, by_ref)
   14681          228 :                 || (!allocate_ptr && by_ref && !is_gimple_omp_oacc (ctx->stmt)))
   14682              :               {
   14683           24 :                 if (allocate_ptr)
   14684              :                   {
   14685           12 :                     gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
   14686           12 :                     gcc_assert (allocate_seq);
   14687           12 :                     gimple_seq_add_seq (&new_body, *allocate_seq);
   14688              :                   }
   14689           24 :                 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
   14690           24 :                 tree pvar = var;
   14691           24 :                 if (!by_ref)
   14692              :                   {
   14693            3 :                     pvar = DECL_VALUE_EXPR (var);
   14694            3 :                     gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
   14695            3 :                     pvar = TREE_OPERAND (pvar, 0);
   14696              :                   }
   14697           24 :                 gcc_assert (DECL_P (pvar));
   14698           24 :                 tree new_pvar = lookup_decl (pvar, ctx);
   14699           24 :                 tree x;
   14700           24 :                 if (!allocate_ptr)
   14701              :                   {
   14702           12 :                     tree atmp = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
   14703           12 :                     tree ty = TREE_TYPE (new_var);
   14704           12 :                     if (by_ref)
   14705           11 :                       ty = TREE_TYPE (ty);
   14706           12 :                     x = TYPE_SIZE_UNIT (ty);
   14707           12 :                     if (TREE_CONSTANT (x))
   14708              :                       break;
   14709            9 :                     tree al = size_int (TYPE_ALIGN (ty));
   14710            9 :                     x = build_call_expr_loc (clause_loc, atmp, 2, x, al);
   14711            9 :                     x = fold_convert_loc (clause_loc, TREE_TYPE (new_pvar), x);
   14712              :                   }
   14713              :                 else
   14714           12 :                   x = *allocate_ptr;
   14715           21 :                 gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
   14716           21 :                 gimple_seq_add_stmt (&new_body,
   14717           21 :                                      gimple_build_assign (new_pvar, x));
   14718              :               }
   14719              :             break;
   14720        15448 :           case OMP_CLAUSE_FIRSTPRIVATE:
   14721        15448 :             var = OMP_CLAUSE_DECL (c);
   14722        15448 :             by_ref = omp_privatize_by_reference (var);
   14723        15448 :             if (is_variable_sized (var, by_ref))
   14724              :               {
   14725           39 :                 tree new_var = lookup_decl (var, ctx);
   14726           39 :                 tree *allocate_ptr = alloc_map.get (new_var);
   14727           39 :                 if (!allocate_ptr)
   14728              :                   break;
   14729           15 :                 gimple_seq *allocate_seq = alloc_seq_map.get (new_var);
   14730           15 :                 gcc_assert (allocate_seq);
   14731           15 :                 gimple_seq_add_seq (&new_body, *allocate_seq);
   14732              : 
   14733           15 :                 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
   14734           15 :                 tree pvar = var;
   14735           15 :                 if (!by_ref)
   14736              :                   {
   14737            2 :                     pvar = DECL_VALUE_EXPR (var);
   14738            2 :                     gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
   14739            2 :                     pvar = TREE_OPERAND (pvar, 0);
   14740              :                   }
   14741           15 :                 gcc_assert (DECL_P (pvar));
   14742           15 :                 tree new_pvar = lookup_decl (pvar, ctx);
   14743           15 :                 tree x = fold_convert_loc (clause_loc, TREE_TYPE (new_pvar),
   14744           15 :                                            *allocate_ptr);
   14745           15 :                 gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
   14746           15 :                 gimple_seq_add_stmt (&new_body,
   14747           15 :                                      gimple_build_assign (new_pvar, x));
   14748              : 
   14749           15 :                 x = build_receiver_ref (var, true, ctx);
   14750           15 :                 new_var = unshare_expr (new_var);
   14751           15 :                 if (by_ref)
   14752           13 :                   new_var = build_fold_indirect_ref (new_var);
   14753           15 :                 x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
   14754           15 :                 gimplify_and_add (x, &new_body);
   14755              :               }
   14756              :           }
   14757              : 
   14758        25335 :       gimple_seq fork_seq = NULL;
   14759        25335 :       gimple_seq join_seq = NULL;
   14760              : 
   14761        25335 :       if (offloaded && is_gimple_omp_oacc (ctx->stmt))
   14762              :         {
   14763              :           /* If there are reductions on the offloaded region itself, treat
   14764              :              them as a dummy GANG loop.  */
   14765         9395 :           tree level = build_int_cst (integer_type_node, GOMP_DIM_GANG);
   14766              : 
   14767         9395 :           gcall *private_marker = lower_oacc_private_marker (ctx);
   14768              : 
   14769         9395 :           if (private_marker)
   14770           30 :             gimple_call_set_arg (private_marker, 2, level);
   14771              : 
   14772         9395 :           lower_oacc_reductions (gimple_location (ctx->stmt), clauses, level,
   14773              :                                  false, NULL, private_marker, NULL, &fork_seq,
   14774              :                                  &join_seq, ctx);
   14775              :         }
   14776              : 
   14777        25335 :       gimple_seq_add_seq (&new_body, fork_seq);
   14778        25335 :       gimple_seq_add_seq (&new_body, tgt_body);
   14779        25335 :       gimple_seq_add_seq (&new_body, join_seq);
   14780        25335 :       gimple_seq_add_seq (&new_body, alloc_dlist);
   14781              : 
   14782        25335 :       if (offloaded)
   14783              :         {
   14784        21311 :           new_body = maybe_catch_exception (new_body);
   14785        21311 :           gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
   14786              :         }
   14787        25335 :       gimple_omp_set_body (stmt, new_body);
   14788              :     }
   14789              : 
   14790        37314 :   gsi_insert_seq_before (gsi_p, gimple_omp_target_iterator_loops (stmt),
   14791              :                          GSI_SAME_STMT);
   14792        37314 :   gimple_omp_target_set_iterator_loops (stmt, NULL);
   14793        58625 :   bind = gimple_build_bind (NULL, NULL,
   14794        21311 :                             tgt_bind ? gimple_bind_block (tgt_bind)
   14795              :                                      : NULL_TREE);
   14796        74236 :   gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
   14797        37314 :   gimple_bind_add_seq (bind, ilist);
   14798        37314 :   gimple_bind_add_stmt (bind, stmt);
   14799        37314 :   gimple_bind_add_seq (bind, olist);
   14800              : 
   14801        37314 :   pop_gimplify_context (NULL);
   14802              : 
   14803        37314 :   if (dep_bind)
   14804              :     {
   14805          392 :       gimple_bind_add_seq (dep_bind, dep_ilist);
   14806          392 :       gimple_bind_add_stmt (dep_bind, bind);
   14807          392 :       gimple_bind_add_seq (dep_bind, dep_olist);
   14808          392 :       pop_gimplify_context (dep_bind);
   14809              :     }
   14810        37314 : }
   14811              : 
   14812              : /* Generate code to implement the action-clauses (destroy, init, use) of an
   14813              :    OpenMP interop construct.  */
   14814              : 
   14815              : static void
   14816          492 : lower_omp_interop_action_clauses (gimple_seq *seq, vec<tree> &objs,
   14817              :                                   vec<tree> *interop_types = NULL,
   14818              :                                   vec<tree> *prefer_types = NULL)
   14819              : {
   14820          492 :   if (objs.length () == 0)
   14821          229 :     return;
   14822              : 
   14823          263 :   enum omp_clause_code action = OMP_CLAUSE_CODE (objs[0]);
   14824          263 :   if (action == OMP_CLAUSE_INIT)
   14825          402 :     gcc_checking_assert (objs.length () == interop_types->length ()
   14826              :                          && objs.length () == prefer_types->length ());
   14827              :   else
   14828          129 :     gcc_assert (prefer_types == NULL && interop_types == NULL);
   14829              : 
   14830          263 :   tree ret_objs = NULL_TREE, ret_interop_types = NULL_TREE,
   14831          263 :        ret_prefer_types = NULL_TREE;
   14832              : 
   14833              :   /* Build an array of interop objects. */
   14834              : 
   14835          263 :   tree type_obj_pref = build_array_type_nelts (ptr_type_node, objs.length ());
   14836          263 :   ret_objs = create_tmp_var (type_obj_pref, "interopobjs");
   14837              : 
   14838          263 :   bool have_pref_type = false;
   14839          263 :   if (action == OMP_CLAUSE_INIT)
   14840              :     {
   14841          592 :       for (tree pref_type : prefer_types)
   14842          260 :         if (pref_type != NULL_TREE)
   14843              :           {
   14844              :             have_pref_type = true;
   14845              :             break;
   14846              :           }
   14847          134 :       tree type_tgtsync
   14848          134 :         = build_array_type_nelts (integer_type_node, objs.length ());
   14849          134 :       ret_interop_types = create_tmp_var (type_tgtsync, "tgt_tgtsync");
   14850          134 :       if (have_pref_type)
   14851           70 :         ret_prefer_types = create_tmp_var (type_obj_pref, "pref_type");
   14852              :       else
   14853              :         {
   14854           64 :           ret_prefer_types = null_pointer_node;
   14855           64 :           prefer_types->truncate (0);
   14856              :         }
   14857              :     }
   14858              : 
   14859          856 :   for (size_t i = 0; !objs.is_empty (); i++)
   14860              :     {
   14861          593 :       tree offset = build_int_cst (integer_type_node, i);
   14862          593 :       tree init = build4 (ARRAY_REF, ptr_type_node, ret_objs, offset, NULL_TREE,
   14863              :                           NULL_TREE);
   14864          593 :       tree obj = OMP_CLAUSE_DECL (objs.pop ());
   14865          593 :       if (TREE_CODE (TREE_TYPE (obj)) == REFERENCE_TYPE)
   14866            6 :         obj = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (obj)), obj);
   14867          593 :       if (action != OMP_CLAUSE_USE
   14868          593 :           && TREE_CODE (TREE_TYPE (obj)) != POINTER_TYPE)
   14869              :         /* For modifying actions, we need a pointer. */
   14870          483 :         obj = build_fold_addr_expr (obj);
   14871          110 :       else if (action == OMP_CLAUSE_USE
   14872          110 :                && TREE_CODE (TREE_TYPE (obj)) == POINTER_TYPE)
   14873              :         /* For use action, we need the value. */
   14874            2 :         obj = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (obj)), obj);
   14875          593 :       init = build2 (MODIFY_EXPR, ptr_type_node, init,
   14876              :                      fold_convert (ptr_type_node, obj));
   14877          593 :       gimplify_and_add (init, seq);
   14878              : 
   14879          593 :       if (action == OMP_CLAUSE_INIT)
   14880              :         {
   14881          384 :           init = build4 (ARRAY_REF, integer_type_node, ret_interop_types,
   14882              :                          offset, NULL_TREE, NULL_TREE);
   14883          384 :           init = build2 (MODIFY_EXPR, integer_type_node, init,
   14884          384 :                          interop_types->pop ());
   14885          384 :           gimplify_and_add (init, seq);
   14886              : 
   14887          384 :           if (have_pref_type)
   14888              :             {
   14889          194 :               tree prefer_type = prefer_types->pop ();
   14890          194 :               tree pref = (prefer_type == NULL_TREE
   14891          194 :                              ? null_pointer_node
   14892          194 :                              : build_fold_addr_expr (prefer_type));
   14893          194 :               init = build4 (ARRAY_REF, ptr_type_node, ret_prefer_types, offset,
   14894              :                              NULL_TREE, NULL_TREE);
   14895          194 :               init = build2 (MODIFY_EXPR, ptr_type_node, init, pref);
   14896          194 :               gimplify_and_add (init, seq);
   14897              :             }
   14898              :         }
   14899              :     }
   14900          263 :   if (action == OMP_CLAUSE_INIT)
   14901              :     {
   14902          134 :       if (have_pref_type)
   14903           70 :         ret_prefer_types = build_fold_addr_expr (ret_prefer_types);
   14904          134 :       ret_interop_types = build_fold_addr_expr (ret_interop_types);
   14905              :     }
   14906          263 :   ret_objs = build_fold_addr_expr (ret_objs);
   14907              : 
   14908          660 :   gcc_assert (objs.is_empty ()
   14909              :               && (!interop_types || interop_types->is_empty ())
   14910              :               && (!prefer_types || prefer_types->is_empty ()));
   14911              : 
   14912          263 :   objs.safe_push (ret_objs);
   14913          263 :   if (action == OMP_CLAUSE_INIT)
   14914              :     {
   14915          134 :       interop_types->safe_push (ret_interop_types);
   14916          134 :       prefer_types->safe_push (ret_prefer_types);
   14917              :     }
   14918              : }
   14919              : 
   14920              : /* Lower code for an OpenMP interop directive.  */
   14921              : 
   14922              : static void
   14923          164 : lower_omp_interop (gimple_stmt_iterator *gsi_p, omp_context *ctx)
   14924              : {
   14925          164 :   push_gimplify_context ();
   14926              : 
   14927          164 :   tree block = make_node (BLOCK);
   14928          164 :   gbind *bind = gimple_build_bind (NULL, NULL, block);
   14929          164 :   gimple_seq bind_body = NULL;
   14930              : 
   14931              :   /* Emit call to GOMP_interop:
   14932              :       void
   14933              :       GOMP_interop (int device_num, int n_init, omp_interop_t **init,
   14934              :                     const void *target_targetsync, const void *prefer_type,
   14935              :                     int n_use, omp_interop_t *use, int n_destroy,
   14936              :                     omp_interop_t **destroy, unsigned int flags,
   14937              :                     void **depend)  */
   14938              : 
   14939          164 :   tree flags = NULL_TREE;
   14940          164 :   tree depend = null_pointer_node;
   14941          164 :   tree device_num = NULL_TREE;
   14942              : 
   14943          164 :   auto_vec<tree> init_objs, use_objs, destroy_objs, prefer_type,
   14944          164 :     target_targetsync;
   14945          164 :   gimple_seq dep_ilist = NULL, dep_olist = NULL;
   14946          164 :   tree clauses = gimple_omp_interop_clauses (gsi_stmt (*gsi_p));
   14947          868 :   for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
   14948              :     {
   14949          704 :       switch (OMP_CLAUSE_CODE (c))
   14950              :         {
   14951          384 :         case OMP_CLAUSE_INIT:
   14952          384 :           {
   14953          384 :             init_objs.safe_push (c);
   14954          384 :             int target_targetsync_bits = 0;
   14955          384 :             if (OMP_CLAUSE_INIT_TARGET (c))
   14956          229 :               target_targetsync_bits |= GOMP_INTEROP_TARGET;
   14957          384 :             if (OMP_CLAUSE_INIT_TARGETSYNC (c))
   14958          204 :               target_targetsync_bits |= GOMP_INTEROP_TARGETSYNC;
   14959          384 :             tree t = build_int_cst (integer_type_node, target_targetsync_bits);
   14960          384 :             target_targetsync.safe_push (t);
   14961          384 :             prefer_type.safe_push (OMP_CLAUSE_INIT_PREFER_TYPE (c));
   14962              :           }
   14963          384 :           break;
   14964          106 :         case OMP_CLAUSE_USE:
   14965          106 :           use_objs.safe_push (c);
   14966          106 :           break;
   14967          103 :         case OMP_CLAUSE_DESTROY:
   14968          103 :           destroy_objs.safe_push (c);
   14969          103 :           break;
   14970           50 :         case OMP_CLAUSE_NOWAIT:
   14971           50 :           flags = build_int_cst (integer_type_node, GOMP_INTEROP_FLAG_NOWAIT);
   14972           50 :           break;
   14973           35 :         case OMP_CLAUSE_DEPEND:
   14974           35 :           {
   14975           35 :             tree *cp = gimple_omp_interop_clauses_ptr (gsi_stmt (*gsi_p));
   14976           35 :             lower_depend_clauses (cp, &dep_ilist, &dep_olist);
   14977           35 :             depend = OMP_CLAUSE_DECL (*cp);
   14978              :           }
   14979           35 :           break;
   14980           26 :         case OMP_CLAUSE_DEVICE:
   14981           26 :           device_num = OMP_CLAUSE_DEVICE_ID (c);
   14982           26 :           break;
   14983            0 :         default:
   14984            0 :           gcc_unreachable ();
   14985              :         }
   14986              :     }
   14987              : 
   14988          164 :   if (flags == NULL_TREE)
   14989          114 :     flags = build_int_cst (integer_type_node, 0);
   14990              : 
   14991          164 :   if (device_num == NULL_TREE)
   14992          138 :     device_num = build_int_cst (integer_type_node, GOMP_DEVICE_DEFAULT_OMP_61);
   14993              : 
   14994          298 :   tree n_init = build_int_cst (integer_type_node, init_objs.length ());
   14995          234 :   tree n_use = build_int_cst (integer_type_node, use_objs.length ());
   14996          223 :   tree n_destroy = build_int_cst (integer_type_node, destroy_objs.length ());
   14997              : 
   14998          164 :   lower_omp_interop_action_clauses (&bind_body, init_objs, &target_targetsync,
   14999              :                                     &prefer_type);
   15000          164 :   lower_omp_interop_action_clauses (&bind_body, use_objs);
   15001          164 :   lower_omp_interop_action_clauses (&bind_body, destroy_objs);
   15002              : 
   15003          164 :   gimple_seq_add_seq (&bind_body, dep_ilist);
   15004          164 :   tree fn = builtin_decl_explicit (BUILT_IN_GOMP_INTEROP);
   15005          298 :   tree init_arg = init_objs.length () ? init_objs.pop () : null_pointer_node;
   15006          164 :   tree target_targetsync_arg = target_targetsync.length ()
   15007          134 :                                  ? target_targetsync.pop ()
   15008          164 :                                  : null_pointer_node;
   15009          164 :   tree prefer_type_arg
   15010          298 :     = prefer_type.length () ? prefer_type.pop () : null_pointer_node;
   15011          234 :   tree use_arg = use_objs.length () ? use_objs.pop () : null_pointer_node;
   15012          164 :   tree destroy_arg
   15013          223 :     = destroy_objs.length () ? destroy_objs.pop () : null_pointer_node;
   15014          164 :   gcall *call
   15015          164 :     = gimple_build_call (fn, 11, device_num, n_init, init_arg,
   15016              :                          target_targetsync_arg, prefer_type_arg, n_use, use_arg,
   15017              :                          n_destroy, destroy_arg, flags, depend);
   15018          164 :   gimple_seq_add_stmt (&bind_body, call);
   15019          164 :   gimple_seq_add_seq (&bind_body, dep_olist);
   15020              : 
   15021          164 :   gsi_replace (gsi_p, bind, true);
   15022          164 :   gimple_bind_set_body (bind, bind_body);
   15023          164 :   pop_gimplify_context (bind);
   15024          164 :   gimple_bind_append_vars (bind, ctx->block_vars);
   15025          164 :   BLOCK_VARS (block) = ctx->block_vars;
   15026          164 : }
   15027              : 
   15028              : /* Expand code for an OpenMP teams directive.  */
   15029              : 
   15030              : static void
   15031         5938 : lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx)
   15032              : {
   15033         5938 :   gomp_teams *teams_stmt = as_a <gomp_teams *> (gsi_stmt (*gsi_p));
   15034         5938 :   push_gimplify_context ();
   15035              : 
   15036         5938 :   tree block = make_node (BLOCK);
   15037         5938 :   gbind *bind = gimple_build_bind (NULL, NULL, block);
   15038         5938 :   gsi_replace (gsi_p, bind, true);
   15039         5938 :   gimple_seq bind_body = NULL;
   15040         5938 :   gimple_seq dlist = NULL;
   15041         5938 :   gimple_seq olist = NULL;
   15042              : 
   15043         5938 :   tree num_teams = omp_find_clause (gimple_omp_teams_clauses (teams_stmt),
   15044         5938 :                                     OMP_CLAUSE_NUM_TEAMS);
   15045         5938 :   tree num_teams_lower = NULL_TREE;
   15046         5938 :   if (num_teams == NULL_TREE)
   15047         5308 :     num_teams = build_int_cst (unsigned_type_node, 0);
   15048              :   else
   15049              :     {
   15050          630 :       num_teams_lower = OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (num_teams);
   15051          630 :       if (num_teams_lower)
   15052              :         {
   15053          150 :           num_teams_lower = fold_convert (unsigned_type_node, num_teams_lower);
   15054          150 :           gimplify_expr (&num_teams_lower, &bind_body, NULL, is_gimple_val,
   15055              :                          fb_rvalue);
   15056              :         }
   15057          630 :       num_teams = OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (num_teams);
   15058              :       // FIXME: Handle dim(x), cf. omp-expand.cc's get_target_arguments.  */
   15059          630 :       if (TREE_CODE (num_teams) == TREE_LIST && TREE_CHAIN (num_teams))
   15060              :         {
   15061            3 :           num_teams_lower = TREE_VALUE (num_teams);
   15062            3 :           num_teams = TREE_VALUE (TREE_CHAIN (num_teams));
   15063              :         }
   15064          627 :       else if (TREE_CODE (num_teams) == TREE_LIST)
   15065            2 :         num_teams = TREE_VALUE (num_teams);
   15066          630 :       num_teams = fold_convert (unsigned_type_node, num_teams);
   15067          630 :       gimplify_expr (&num_teams, &bind_body, NULL, is_gimple_val, fb_rvalue);
   15068              :     }
   15069         5938 :   if (num_teams_lower == NULL_TREE)
   15070         5785 :     num_teams_lower = num_teams;
   15071         5938 :   tree thread_limit = omp_find_clause (gimple_omp_teams_clauses (teams_stmt),
   15072         5938 :                                        OMP_CLAUSE_THREAD_LIMIT);
   15073         5938 :   if (thread_limit == NULL_TREE)
   15074         5457 :     thread_limit = build_int_cst (unsigned_type_node, 0);
   15075              :   else
   15076              :     {
   15077          481 :       thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit);
   15078              :       // FIXME: Handle dim(x), cf. omp-expand.cc's get_target_arguments.  */
   15079          481 :       if (TREE_CODE (thread_limit) == TREE_LIST)
   15080           69 :         thread_limit = TREE_VALUE (thread_limit);
   15081          481 :       thread_limit = fold_convert (unsigned_type_node, thread_limit);
   15082          481 :       gimplify_expr (&thread_limit, &bind_body, NULL, is_gimple_val,
   15083              :                      fb_rvalue);
   15084              :     }
   15085         5938 :   location_t loc = gimple_location (teams_stmt);
   15086         5938 :   tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS4);
   15087         5938 :   tree rettype = TREE_TYPE (TREE_TYPE (decl));
   15088         5938 :   tree first = create_tmp_var (rettype);
   15089         5938 :   gimple_seq_add_stmt (&bind_body,
   15090         5938 :                        gimple_build_assign (first, build_one_cst (rettype)));
   15091         5938 :   tree llabel = create_artificial_label (loc);
   15092         5938 :   gimple_seq_add_stmt (&bind_body, gimple_build_label (llabel));
   15093         5938 :   gimple *call
   15094         5938 :     = gimple_build_call (decl, 4, num_teams_lower, num_teams, thread_limit,
   15095              :                          first);
   15096         5938 :   gimple_set_location (call, loc);
   15097         5938 :   tree temp = create_tmp_var (rettype);
   15098         5938 :   gimple_call_set_lhs (call, temp);
   15099         5938 :   gimple_seq_add_stmt (&bind_body, call);
   15100              : 
   15101         5938 :   tree tlabel = create_artificial_label (loc);
   15102         5938 :   tree flabel = create_artificial_label (loc);
   15103         5938 :   gimple *cond = gimple_build_cond (NE_EXPR, temp, build_zero_cst (rettype),
   15104              :                                     tlabel, flabel);
   15105         5938 :   gimple_seq_add_stmt (&bind_body, cond);
   15106         5938 :   gimple_seq_add_stmt (&bind_body, gimple_build_label (tlabel));
   15107         5938 :   gimple_seq_add_stmt (&bind_body,
   15108         5938 :                        gimple_build_assign (first, build_zero_cst (rettype)));
   15109              : 
   15110         5938 :   lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt),
   15111              :                            &bind_body, &dlist, ctx, NULL);
   15112         5938 :   lower_omp (gimple_omp_body_ptr (teams_stmt), ctx);
   15113         5938 :   lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt), &olist,
   15114              :                            NULL, ctx);
   15115         5938 :   gimple_seq_add_stmt (&bind_body, teams_stmt);
   15116              : 
   15117         5938 :   gimple_seq_add_seq (&bind_body, gimple_omp_body (teams_stmt));
   15118         5938 :   gimple_omp_set_body (teams_stmt, NULL);
   15119         5938 :   gimple_seq_add_seq (&bind_body, olist);
   15120         5938 :   gimple_seq_add_seq (&bind_body, dlist);
   15121         5938 :   gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true));
   15122         5938 :   gimple_seq_add_stmt (&bind_body, gimple_build_goto (llabel));
   15123         5938 :   gimple_seq_add_stmt (&bind_body, gimple_build_label (flabel));
   15124         5938 :   gimple_bind_set_body (bind, bind_body);
   15125              : 
   15126         5938 :   pop_gimplify_context (bind);
   15127              : 
   15128         5938 :   gimple_bind_append_vars (bind, ctx->block_vars);
   15129         5938 :   BLOCK_VARS (block) = ctx->block_vars;
   15130         5938 :   if (BLOCK_VARS (block))
   15131          689 :     TREE_USED (block) = 1;
   15132         5938 : }
   15133              : 
   15134              : /* Callback for lower_omp_1.  Return non-NULL if *tp needs to be
   15135              :    regimplified.  If DATA is non-NULL, lower_omp_1 is outside
   15136              :    of OMP context, but with make_addressable_vars set.  */
   15137              : 
   15138              : static tree
   15139      3226433 : lower_omp_regimplify_p (tree *tp, int *walk_subtrees,
   15140              :                         void *data)
   15141              : {
   15142      3226433 :   tree t = *tp;
   15143              : 
   15144              :   /* Any variable with DECL_VALUE_EXPR needs to be regimplified.  */
   15145      3226433 :   if ((VAR_P (t) || TREE_CODE (t) == PARM_DECL || TREE_CODE (t) == RESULT_DECL)
   15146      1500736 :       && data == NULL
   15147      1440235 :       && DECL_HAS_VALUE_EXPR_P (t))
   15148              :     return t;
   15149              : 
   15150      3120264 :   if (make_addressable_vars
   15151       423226 :       && DECL_P (t)
   15152      3325655 :       && bitmap_bit_p (make_addressable_vars, DECL_UID (t)))
   15153              :     return t;
   15154              : 
   15155              :   /* If a global variable has been privatized, TREE_CONSTANT on
   15156              :      ADDR_EXPR might be wrong.  */
   15157      3117675 :   if (data == NULL && TREE_CODE (t) == ADDR_EXPR)
   15158       110505 :     recompute_tree_invariant_for_addr_expr (t);
   15159              : 
   15160      3117675 :   *walk_subtrees = !IS_TYPE_OR_DECL_P (t);
   15161      3117675 :   return NULL_TREE;
   15162              : }
   15163              : 
   15164              : /* Data to be communicated between lower_omp_regimplify_operands and
   15165              :    lower_omp_regimplify_operands_p.  */
   15166              : 
   15167              : struct lower_omp_regimplify_operands_data
   15168              : {
   15169              :   omp_context *ctx;
   15170              :   vec<tree> *decls;
   15171              : };
   15172              : 
   15173              : /* Helper function for lower_omp_regimplify_operands.  Find
   15174              :    omp_member_access_dummy_var vars and adjust temporarily their
   15175              :    DECL_VALUE_EXPRs if needed.  */
   15176              : 
   15177              : static tree
   15178       428301 : lower_omp_regimplify_operands_p (tree *tp, int *walk_subtrees,
   15179              :                                  void *data)
   15180              : {
   15181       428301 :   tree t = omp_member_access_dummy_var (*tp);
   15182       428301 :   if (t)
   15183              :     {
   15184            9 :       struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
   15185            9 :       lower_omp_regimplify_operands_data *ldata
   15186              :         = (lower_omp_regimplify_operands_data *) wi->info;
   15187            9 :       tree o = maybe_lookup_decl (t, ldata->ctx);
   15188            9 :       if (o == NULL_TREE)
   15189            5 :         o = maybe_lookup_decl_in_outer_ctx (t, ldata->ctx);
   15190            9 :       if (o != t)
   15191              :         {
   15192            9 :           ldata->decls->safe_push (DECL_VALUE_EXPR (*tp));
   15193            9 :           ldata->decls->safe_push (*tp);
   15194            9 :           tree v = unshare_and_remap (DECL_VALUE_EXPR (*tp), t, o);
   15195            9 :           SET_DECL_VALUE_EXPR (*tp, v);
   15196              :         }
   15197              :     }
   15198       428301 :   *walk_subtrees = !IS_TYPE_OR_DECL_P (*tp);
   15199       428301 :   return NULL_TREE;
   15200              : }
   15201              : 
   15202              : /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
   15203              :    of omp_member_access_dummy_var vars during regimplification.  */
   15204              : 
   15205              : static void
   15206       108601 : lower_omp_regimplify_operands (omp_context *ctx, gimple *stmt,
   15207              :                                gimple_stmt_iterator *gsi_p)
   15208              : {
   15209       108601 :   auto_vec<tree, 10> decls;
   15210       108601 :   if (ctx)
   15211              :     {
   15212       106079 :       struct walk_stmt_info wi;
   15213       106079 :       memset (&wi, '\0', sizeof (wi));
   15214       106079 :       struct lower_omp_regimplify_operands_data data;
   15215       106079 :       data.ctx = ctx;
   15216       106079 :       data.decls = &decls;
   15217       106079 :       wi.info = &data;
   15218       106079 :       walk_gimple_op (stmt, lower_omp_regimplify_operands_p, &wi);
   15219              :     }
   15220       108601 :   gimple_regimplify_operands (stmt, gsi_p);
   15221       325812 :   while (!decls.is_empty ())
   15222              :     {
   15223            9 :       tree t = decls.pop ();
   15224            9 :       tree v = decls.pop ();
   15225            9 :       SET_DECL_VALUE_EXPR (t, v);
   15226              :     }
   15227       108601 : }
   15228              : 
   15229              : static void
   15230      2524622 : lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx)
   15231              : {
   15232      2524622 :   gimple *stmt = gsi_stmt (*gsi_p);
   15233      2524622 :   struct walk_stmt_info wi;
   15234      2524622 :   gcall *call_stmt;
   15235              : 
   15236      2524622 :   if (gimple_has_location (stmt))
   15237      2071578 :     input_location = gimple_location (stmt);
   15238              : 
   15239      2524622 :   if (make_addressable_vars)
   15240       154287 :     memset (&wi, '\0', sizeof (wi));
   15241              : 
   15242              :   /* If we have issued syntax errors, avoid doing any heavy lifting.
   15243              :      Just replace the OMP directives with a NOP to avoid
   15244              :      confusing RTL expansion.  */
   15245      2524622 :   if (seen_error () && is_gimple_omp (stmt))
   15246              :     {
   15247        11728 :       gsi_replace (gsi_p, gimple_build_nop (), true);
   15248        11728 :       return;
   15249              :     }
   15250              : 
   15251      2512894 :   switch (gimple_code (stmt))
   15252              :     {
   15253       157605 :     case GIMPLE_COND:
   15254       157605 :       {
   15255       157605 :         gcond *cond_stmt = as_a <gcond *> (stmt);
   15256        89792 :         if ((ctx || make_addressable_vars)
   15257       161218 :             && (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
   15258              :                            lower_omp_regimplify_p,
   15259              :                            ctx ? NULL : &wi, NULL)
   15260       137292 :                 || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
   15261              :                               lower_omp_regimplify_p,
   15262              :                               ctx ? NULL : &wi, NULL)))
   15263         1247 :           lower_omp_regimplify_operands (ctx, cond_stmt, gsi_p);
   15264              :       }
   15265              :       break;
   15266           20 :     case GIMPLE_CATCH:
   15267           20 :       lower_omp (gimple_catch_handler_ptr (as_a <gcatch *> (stmt)), ctx);
   15268           20 :       break;
   15269            0 :     case GIMPLE_EH_FILTER:
   15270            0 :       lower_omp (gimple_eh_filter_failure_ptr (stmt), ctx);
   15271            0 :       break;
   15272        26211 :     case GIMPLE_TRY:
   15273        26211 :       lower_omp (gimple_try_eval_ptr (stmt), ctx);
   15274        26206 :       lower_omp (gimple_try_cleanup_ptr (stmt), ctx);
   15275        26206 :       break;
   15276            2 :     case GIMPLE_ASSUME:
   15277            2 :       lower_omp (gimple_assume_body_ptr (stmt), ctx);
   15278            2 :       break;
   15279            5 :     case GIMPLE_TRANSACTION:
   15280            5 :       lower_omp (gimple_transaction_body_ptr (as_a <gtransaction *> (stmt)),
   15281              :                  ctx);
   15282            5 :       break;
   15283       169478 :     case GIMPLE_BIND:
   15284       169478 :       if (ctx && is_gimple_omp_oacc (ctx->stmt))
   15285              :         {
   15286        17785 :           tree vars = gimple_bind_vars (as_a <gbind *> (stmt));
   15287        17785 :           oacc_privatization_scan_decl_chain (ctx, vars);
   15288              :         }
   15289       169478 :       lower_omp (gimple_bind_body_ptr (as_a <gbind *> (stmt)), ctx);
   15290       169471 :       maybe_remove_omp_member_access_dummy_vars (as_a <gbind *> (stmt));
   15291       169471 :       break;
   15292        20232 :     case GIMPLE_OMP_PARALLEL:
   15293        20232 :     case GIMPLE_OMP_TASK:
   15294        20232 :       ctx = maybe_lookup_ctx (stmt);
   15295        20232 :       gcc_assert (ctx);
   15296        20232 :       if (ctx->cancellable)
   15297          146 :         ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
   15298        20232 :       lower_omp_taskreg (gsi_p, ctx);
   15299        20232 :       break;
   15300        47102 :     case GIMPLE_OMP_FOR:
   15301        47102 :       ctx = maybe_lookup_ctx (stmt);
   15302        47102 :       gcc_assert (ctx);
   15303        47102 :       if (ctx->cancellable)
   15304           84 :         ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
   15305        47102 :       lower_omp_for (gsi_p, ctx);
   15306        47102 :       break;
   15307          386 :     case GIMPLE_OMP_SECTIONS:
   15308          386 :       ctx = maybe_lookup_ctx (stmt);
   15309          386 :       gcc_assert (ctx);
   15310          386 :       if (ctx->cancellable)
   15311           19 :         ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
   15312          386 :       lower_omp_sections (gsi_p, ctx);
   15313          386 :       break;
   15314          161 :     case GIMPLE_OMP_SCOPE:
   15315          161 :       ctx = maybe_lookup_ctx (stmt);
   15316          161 :       gcc_assert (ctx);
   15317          161 :       lower_omp_scope (gsi_p, ctx);
   15318          161 :       break;
   15319          659 :     case GIMPLE_OMP_DISPATCH:
   15320          659 :       ctx = maybe_lookup_ctx (stmt);
   15321          659 :       gcc_assert (ctx);
   15322          659 :       lower_omp_dispatch (gsi_p, ctx);
   15323          659 :       break;
   15324          164 :     case GIMPLE_OMP_INTEROP:
   15325          164 :       ctx = maybe_lookup_ctx (stmt);
   15326          164 :       gcc_assert (ctx);
   15327          164 :       lower_omp_interop (gsi_p, ctx);
   15328          164 :       break;
   15329         1140 :     case GIMPLE_OMP_SINGLE:
   15330         1140 :       ctx = maybe_lookup_ctx (stmt);
   15331         1140 :       gcc_assert (ctx);
   15332         1140 :       lower_omp_single (gsi_p, ctx);
   15333         1140 :       break;
   15334          658 :     case GIMPLE_OMP_STRUCTURED_BLOCK:
   15335              :       /* We have already done error checking at this point, so these nodes
   15336              :          can be completely removed and replaced with their body.  */
   15337          658 :       ctx = maybe_lookup_ctx (stmt);
   15338          658 :       gcc_assert (ctx);
   15339          658 :       lower_omp (gimple_omp_body_ptr (stmt), ctx);
   15340          658 :       gsi_replace_with_seq (gsi_p, gimple_omp_body (stmt), true);
   15341          658 :       break;
   15342         1127 :     case GIMPLE_OMP_MASTER:
   15343         1127 :     case GIMPLE_OMP_MASKED:
   15344         1127 :       ctx = maybe_lookup_ctx (stmt);
   15345         1127 :       gcc_assert (ctx);
   15346         1127 :       lower_omp_master (gsi_p, ctx);
   15347         1127 :       break;
   15348          536 :     case GIMPLE_OMP_TASKGROUP:
   15349          536 :       ctx = maybe_lookup_ctx (stmt);
   15350          536 :       gcc_assert (ctx);
   15351          536 :       lower_omp_taskgroup (gsi_p, ctx);
   15352          536 :       break;
   15353         1128 :     case GIMPLE_OMP_ORDERED:
   15354         1128 :       ctx = maybe_lookup_ctx (stmt);
   15355         1128 :       gcc_assert (ctx);
   15356         1128 :       lower_omp_ordered (gsi_p, ctx);
   15357         1128 :       break;
   15358         1324 :     case GIMPLE_OMP_SCAN:
   15359         1324 :       ctx = maybe_lookup_ctx (stmt);
   15360         1324 :       gcc_assert (ctx);
   15361         1324 :       lower_omp_scan (gsi_p, ctx);
   15362         1324 :       break;
   15363          317 :     case GIMPLE_OMP_CRITICAL:
   15364          317 :       ctx = maybe_lookup_ctx (stmt);
   15365          317 :       gcc_assert (ctx);
   15366          317 :       lower_omp_critical (gsi_p, ctx);
   15367          317 :       break;
   15368         2566 :     case GIMPLE_OMP_ATOMIC_LOAD:
   15369           90 :       if ((ctx || make_addressable_vars)
   15370         2566 :           && walk_tree (gimple_omp_atomic_load_rhs_ptr (
   15371              :                           as_a <gomp_atomic_load *> (stmt)),
   15372              :                         lower_omp_regimplify_p, ctx ? NULL : &wi, NULL))
   15373         1442 :         lower_omp_regimplify_operands (ctx, stmt, gsi_p);
   15374              :       break;
   15375        37314 :     case GIMPLE_OMP_TARGET:
   15376        37314 :       ctx = maybe_lookup_ctx (stmt);
   15377        37314 :       gcc_assert (ctx);
   15378        37314 :       lower_omp_target (gsi_p, ctx);
   15379        37314 :       break;
   15380         8515 :     case GIMPLE_OMP_TEAMS:
   15381         8515 :       ctx = maybe_lookup_ctx (stmt);
   15382         8515 :       gcc_assert (ctx);
   15383         8515 :       if (gimple_omp_teams_host (as_a <gomp_teams *> (stmt)))
   15384         2577 :         lower_omp_taskreg (gsi_p, ctx);
   15385              :       else
   15386         5938 :         lower_omp_teams (gsi_p, ctx);
   15387              :       break;
   15388       136307 :     case GIMPLE_CALL:
   15389       136307 :       tree fndecl;
   15390       136307 :       call_stmt = as_a <gcall *> (stmt);
   15391       136307 :       fndecl = gimple_call_fndecl (call_stmt);
   15392       136307 :       if (fndecl
   15393       136307 :           && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL))
   15394        39123 :         switch (DECL_FUNCTION_CODE (fndecl))
   15395              :           {
   15396          742 :           case BUILT_IN_GOMP_BARRIER:
   15397          742 :             if (ctx == NULL)
   15398              :               break;
   15399              :             /* FALLTHRU */
   15400         1070 :           case BUILT_IN_GOMP_CANCEL:
   15401         1070 :           case BUILT_IN_GOMP_CANCELLATION_POINT:
   15402         1070 :             omp_context *cctx;
   15403         1070 :             cctx = ctx;
   15404         1070 :             if (gimple_code (cctx->stmt) == GIMPLE_OMP_SECTION)
   15405           43 :               cctx = cctx->outer;
   15406         1070 :             gcc_assert (gimple_call_lhs (call_stmt) == NULL_TREE);
   15407         1070 :             if (!cctx->cancellable)
   15408              :               {
   15409          688 :                 if (DECL_FUNCTION_CODE (fndecl)
   15410              :                     == BUILT_IN_GOMP_CANCELLATION_POINT)
   15411              :                   {
   15412            0 :                     stmt = gimple_build_nop ();
   15413            0 :                     gsi_replace (gsi_p, stmt, false);
   15414              :                   }
   15415              :                 break;
   15416              :               }
   15417          382 :             if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_GOMP_BARRIER)
   15418              :               {
   15419           20 :                 fndecl = builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL);
   15420           20 :                 gimple_call_set_fndecl (call_stmt, fndecl);
   15421           20 :                 gimple_call_set_fntype (call_stmt, TREE_TYPE (fndecl));
   15422              :               }
   15423          382 :             tree lhs;
   15424          382 :             lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl)));
   15425          382 :             gimple_call_set_lhs (call_stmt, lhs);
   15426          382 :             tree fallthru_label;
   15427          382 :             fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
   15428          382 :             gimple *g;
   15429          382 :             g = gimple_build_label (fallthru_label);
   15430          382 :             gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
   15431          382 :             g = gimple_build_cond (NE_EXPR, lhs,
   15432          382 :                                    fold_convert (TREE_TYPE (lhs),
   15433              :                                                  boolean_false_node),
   15434              :                                    cctx->cancel_label, fallthru_label);
   15435          382 :             gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
   15436          382 :             break;
   15437              :           default:
   15438              :             break;
   15439              :           }
   15440       136272 :       goto regimplify;
   15441              : 
   15442              :     case GIMPLE_ASSIGN:
   15443      1461389 :       for (omp_context *up = ctx; up; up = up->outer)
   15444              :         {
   15445       687431 :           if (gimple_code (up->stmt) == GIMPLE_OMP_ORDERED
   15446              :               || gimple_code (up->stmt) == GIMPLE_OMP_CRITICAL
   15447              :               || gimple_code (up->stmt) == GIMPLE_OMP_TASKGROUP
   15448              :               || gimple_code (up->stmt) == GIMPLE_OMP_SCOPE
   15449              :               || gimple_code (up->stmt) == GIMPLE_OMP_SECTION
   15450              :               || gimple_code (up->stmt) == GIMPLE_OMP_SCAN
   15451              :               || (gimple_code (up->stmt) == GIMPLE_OMP_TARGET
   15452       230338 :                   && (gimple_omp_target_kind (up->stmt)
   15453              :                       == GF_OMP_TARGET_KIND_DATA)))
   15454        90764 :             continue;
   15455       596667 :           else if (!up->lastprivate_conditional_map)
   15456              :             break;
   15457         4204 :           tree lhs = get_base_address (gimple_assign_lhs (stmt));
   15458         4204 :           if (TREE_CODE (lhs) == MEM_REF
   15459           26 :               && DECL_P (TREE_OPERAND (lhs, 0))
   15460         4227 :               && TREE_CODE (TREE_TYPE (TREE_OPERAND (lhs,
   15461              :                                                      0))) == REFERENCE_TYPE)
   15462           19 :             lhs = TREE_OPERAND (lhs, 0);
   15463         4204 :           if (DECL_P (lhs))
   15464         3161 :             if (tree *v = up->lastprivate_conditional_map->get (lhs))
   15465              :               {
   15466          382 :                 tree clauses;
   15467          382 :                 if (up->combined_into_simd_safelen1)
   15468              :                   {
   15469           54 :                     up = up->outer;
   15470           54 :                     if (gimple_code (up->stmt) == GIMPLE_OMP_SCAN)
   15471            3 :                       up = up->outer;
   15472              :                   }
   15473          382 :                 if (gimple_code (up->stmt) == GIMPLE_OMP_FOR)
   15474          298 :                   clauses = gimple_omp_for_clauses (up->stmt);
   15475              :                 else
   15476           84 :                   clauses = gimple_omp_sections_clauses (up->stmt);
   15477          382 :                 tree c = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_);
   15478          382 :                 if (!OMP_CLAUSE__CONDTEMP__ITER (c))
   15479          348 :                   c = omp_find_clause (OMP_CLAUSE_CHAIN (c),
   15480              :                                        OMP_CLAUSE__CONDTEMP_);
   15481          382 :                 gcc_assert (OMP_CLAUSE__CONDTEMP__ITER (c));
   15482          382 :                 gimple *g = gimple_build_assign (*v, OMP_CLAUSE_DECL (c));
   15483          382 :                 gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
   15484              :               }
   15485              :         }
   15486              :       /* FALLTHRU */
   15487              : 
   15488      2036244 :     default:
   15489      2036244 :     regimplify:
   15490      1074761 :       if ((ctx || make_addressable_vars)
   15491      2112159 :           && walk_gimple_op (stmt, lower_omp_regimplify_p,
   15492              :                              ctx ? NULL : &wi))
   15493              :         {
   15494              :           /* Just remove clobbers, this should happen only if we have
   15495              :              "privatized" local addressable variables in SIMD regions,
   15496              :              the clobber isn't needed in that case and gimplifying address
   15497              :              of the ARRAY_REF into a pointer and creating MEM_REF based
   15498              :              clobber would create worse code than we get with the clobber
   15499              :              dropped.  */
   15500       106069 :           if (gimple_clobber_p (stmt))
   15501              :             {
   15502          157 :               gsi_replace (gsi_p, gimple_build_nop (), true);
   15503          157 :               break;
   15504              :             }
   15505       105912 :           lower_omp_regimplify_operands (ctx, stmt, gsi_p);
   15506              :         }
   15507              :       break;
   15508              :     }
   15509              : }
   15510              : 
   15511              : static void
   15512       414005 : lower_omp (gimple_seq *body, omp_context *ctx)
   15513              : {
   15514       414005 :   location_t saved_location = input_location;
   15515       414005 :   gimple_stmt_iterator gsi;
   15516      3304132 :   for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
   15517      2524622 :     lower_omp_1 (&gsi, ctx);
   15518              :   /* During gimplification, we haven't folded statements inside offloading
   15519              :      or taskreg regions (gimplify.cc:maybe_fold_stmt); do that now.  */
   15520       413989 :   if (target_nesting_level || taskreg_nesting_level)
   15521       658434 :     for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
   15522       444775 :       fold_stmt (&gsi);
   15523       413989 :   input_location = saved_location;
   15524       413989 : }
   15525              : 
   15526              : /* Main entry point.  */
   15527              : 
   15528              : static unsigned int
   15529      3010141 : execute_lower_omp (void)
   15530              : {
   15531      3010141 :   gimple_seq body;
   15532      3010141 :   int i;
   15533      3010141 :   omp_context *ctx;
   15534              : 
   15535              :   /* This pass always runs, to provide PROP_gimple_lomp.
   15536              :      But often, there is nothing to do.  */
   15537      3010141 :   if (flag_openacc == 0 && flag_openmp == 0
   15538      2952884 :       && flag_openmp_simd == 0)
   15539              :     return 0;
   15540              : 
   15541        59553 :   all_contexts = splay_tree_new (splay_tree_compare_pointers, 0,
   15542              :                                  delete_omp_context);
   15543              : 
   15544        59553 :   body = gimple_body (current_function_decl);
   15545              : 
   15546        59553 :   scan_omp (&body, NULL);
   15547        59553 :   gcc_assert (taskreg_nesting_level == 0);
   15548        86163 :   FOR_EACH_VEC_ELT (taskreg_contexts, i, ctx)
   15549        26610 :     finish_taskreg_scan (ctx);
   15550        59553 :   taskreg_contexts.release ();
   15551              : 
   15552        59553 :   if (all_contexts->root)
   15553              :     {
   15554        26058 :       if (make_addressable_vars)
   15555          624 :         push_gimplify_context ();
   15556        26058 :       lower_omp (&body, NULL);
   15557        26054 :       if (make_addressable_vars)
   15558          624 :         pop_gimplify_context (NULL);
   15559              :     }
   15560              : 
   15561        59549 :   if (all_contexts)
   15562              :     {
   15563        59549 :       splay_tree_delete (all_contexts);
   15564        59549 :       all_contexts = NULL;
   15565              :     }
   15566        59549 :   BITMAP_FREE (make_addressable_vars);
   15567        59549 :   BITMAP_FREE (global_nonaddressable_vars);
   15568              : 
   15569              :   /* If current function is a method, remove artificial dummy VAR_DECL created
   15570              :      for non-static data member privatization, they aren't needed for
   15571              :      debuginfo nor anything else, have been already replaced everywhere in the
   15572              :      IL and cause problems with LTO.  */
   15573        59549 :   if (DECL_ARGUMENTS (current_function_decl)
   15574        34084 :       && DECL_ARTIFICIAL (DECL_ARGUMENTS (current_function_decl))
   15575        69950 :       && (TREE_CODE (TREE_TYPE (DECL_ARGUMENTS (current_function_decl)))
   15576              :           == POINTER_TYPE))
   15577         9838 :     remove_member_access_dummy_vars (DECL_INITIAL (current_function_decl));
   15578              : 
   15579        60540 :   for (auto task_stmt : task_cpyfns)
   15580          513 :     finalize_task_copyfn (task_stmt);
   15581        59549 :   task_cpyfns.release ();
   15582        59549 :   return 0;
   15583              : }
   15584              : 
   15585              : namespace {
   15586              : 
   15587              : const pass_data pass_data_lower_omp =
   15588              : {
   15589              :   GIMPLE_PASS, /* type */
   15590              :   "omplower", /* name */
   15591              :   OPTGROUP_OMP, /* optinfo_flags */
   15592              :   TV_NONE, /* tv_id */
   15593              :   PROP_gimple_any, /* properties_required */
   15594              :   PROP_gimple_lomp | PROP_gimple_lomp_dev, /* properties_provided */
   15595              :   0, /* properties_destroyed */
   15596              :   0, /* todo_flags_start */
   15597              :   0, /* todo_flags_finish */
   15598              : };
   15599              : 
   15600              : class pass_lower_omp : public gimple_opt_pass
   15601              : {
   15602              : public:
   15603       297180 :   pass_lower_omp (gcc::context *ctxt)
   15604       594360 :     : gimple_opt_pass (pass_data_lower_omp, ctxt)
   15605              :   {}
   15606              : 
   15607              :   /* opt_pass methods: */
   15608      3010141 :   unsigned int execute (function *) final override
   15609              :   {
   15610      3010141 :     return execute_lower_omp ();
   15611              :   }
   15612              : 
   15613              : }; // class pass_lower_omp
   15614              : 
   15615              : } // anon namespace
   15616              : 
   15617              : gimple_opt_pass *
   15618       297180 : make_pass_lower_omp (gcc::context *ctxt)
   15619              : {
   15620       297180 :   return new pass_lower_omp (ctxt);
   15621              : }
   15622              : 
   15623              : /* The following is a utility to diagnose structured block violations.
   15624              :    It is not part of the "omplower" pass, as that's invoked too late.  It
   15625              :    should be invoked by the respective front ends after gimplification.  */
   15626              : 
   15627              : static splay_tree all_labels;
   15628              : 
   15629              : /* Check for mismatched contexts and generate an error if needed.  Return
   15630              :    true if an error is detected.  */
   15631              : 
   15632              : static bool
   15633       568384 : diagnose_sb_0 (gimple_stmt_iterator *gsi_p,
   15634              :                gimple *branch_ctx, gimple *label_ctx)
   15635              : {
   15636       568384 :   gcc_checking_assert (!branch_ctx || is_gimple_omp (branch_ctx));
   15637       568384 :   gcc_checking_assert (!label_ctx || is_gimple_omp (label_ctx));
   15638              : 
   15639       568384 :   if (label_ctx == branch_ctx)
   15640              :     return false;
   15641              : 
   15642          180 :   const char* kind = NULL;
   15643              : 
   15644          180 :   if (flag_openacc)
   15645              :     {
   15646            7 :       if ((branch_ctx && is_gimple_omp_oacc (branch_ctx))
   15647           16 :           || (label_ctx && is_gimple_omp_oacc (label_ctx)))
   15648              :         {
   15649              :           gcc_checking_assert (kind == NULL);
   15650              :           kind = "OpenACC";
   15651              :         }
   15652              :     }
   15653              :   if (kind == NULL)
   15654              :     {
   15655          164 :       gcc_checking_assert (flag_openmp || flag_openmp_simd);
   15656              :       kind = "OpenMP";
   15657              :     }
   15658              : 
   15659              :   /* Previously we kept track of the label's entire context in diagnose_sb_[12]
   15660              :      so we could traverse it and issue a correct "exit" or "enter" error
   15661              :      message upon a structured block violation.
   15662              : 
   15663              :      We built the context by building a list with tree_cons'ing, but there is
   15664              :      no easy counterpart in gimple tuples.  It seems like far too much work
   15665              :      for issuing exit/enter error messages.  If someone really misses the
   15666              :      distinct error message... patches welcome.  */
   15667              : 
   15668              : #if 0
   15669              :   /* Try to avoid confusing the user by producing and error message
   15670              :      with correct "exit" or "enter" verbiage.  We prefer "exit"
   15671              :      unless we can show that LABEL_CTX is nested within BRANCH_CTX.  */
   15672              :   if (branch_ctx == NULL)
   15673              :     exit_p = false;
   15674              :   else
   15675              :     {
   15676              :       while (label_ctx)
   15677              :         {
   15678              :           if (TREE_VALUE (label_ctx) == branch_ctx)
   15679              :             {
   15680              :               exit_p = false;
   15681              :               break;
   15682              :             }
   15683              :           label_ctx = TREE_CHAIN (label_ctx);
   15684              :         }
   15685              :     }
   15686              : 
   15687              :   if (exit_p)
   15688              :     error ("invalid exit from %s structured block", kind);
   15689              :   else
   15690              :     error ("invalid entry to %s structured block", kind);
   15691              : #endif
   15692              : 
   15693              :   /* If it's obvious we have an invalid entry, be specific about the error.  */
   15694          180 :   if (branch_ctx == NULL)
   15695           53 :     error ("invalid entry to %s structured block", kind);
   15696              :   else
   15697              :     {
   15698              :       /* Otherwise, be vague and lazy, but efficient.  */
   15699          127 :       error ("invalid branch to/from %s structured block", kind);
   15700              :     }
   15701              : 
   15702          180 :   gsi_replace (gsi_p, gimple_build_nop (), false);
   15703          180 :   return true;
   15704              : }
   15705              : 
   15706              : /* Pass 1: Create a minimal tree of structured blocks, and record
   15707              :    where each label is found.  */
   15708              : 
   15709              : static tree
   15710      3395322 : diagnose_sb_1 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
   15711              :                struct walk_stmt_info *wi)
   15712              : {
   15713      3395322 :   gimple *context = (gimple *) wi->info;
   15714      3395322 :   gimple *inner_context;
   15715      3395322 :   gimple *stmt = gsi_stmt (*gsi_p);
   15716              : 
   15717      3395322 :   *handled_ops_p = true;
   15718              : 
   15719      3395322 :   switch (gimple_code (stmt))
   15720              :     {
   15721       327782 :     WALK_SUBSTMTS;
   15722              : 
   15723        84560 :     case GIMPLE_OMP_PARALLEL:
   15724        84560 :     case GIMPLE_OMP_TASK:
   15725        84560 :     case GIMPLE_OMP_SCOPE:
   15726        84560 :     case GIMPLE_OMP_SECTIONS:
   15727        84560 :     case GIMPLE_OMP_SINGLE:
   15728        84560 :     case GIMPLE_OMP_SECTION:
   15729        84560 :     case GIMPLE_OMP_STRUCTURED_BLOCK:
   15730        84560 :     case GIMPLE_OMP_MASTER:
   15731        84560 :     case GIMPLE_OMP_MASKED:
   15732        84560 :     case GIMPLE_OMP_ORDERED:
   15733        84560 :     case GIMPLE_OMP_SCAN:
   15734        84560 :     case GIMPLE_OMP_CRITICAL:
   15735        84560 :     case GIMPLE_OMP_TARGET:
   15736        84560 :     case GIMPLE_OMP_TEAMS:
   15737        84560 :     case GIMPLE_OMP_TASKGROUP:
   15738              :       /* The minimal context here is just the current OMP construct.  */
   15739        84560 :       inner_context = stmt;
   15740        84560 :       wi->info = inner_context;
   15741        84560 :       walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
   15742        84560 :       wi->info = context;
   15743        84560 :       break;
   15744              : 
   15745        53086 :     case GIMPLE_OMP_FOR:
   15746        53086 :       inner_context = stmt;
   15747        53086 :       wi->info = inner_context;
   15748              :       /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
   15749              :          walk them.  */
   15750        53086 :       walk_gimple_seq (gimple_omp_for_pre_body (stmt),
   15751              :                        diagnose_sb_1, NULL, wi);
   15752        53086 :       walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
   15753        53086 :       wi->info = context;
   15754        53086 :       break;
   15755              : 
   15756       563857 :     case GIMPLE_LABEL:
   15757      1127714 :       splay_tree_insert (all_labels,
   15758       563857 :                          (splay_tree_key) gimple_label_label (
   15759       563857 :                                             as_a <glabel *> (stmt)),
   15760              :                          (splay_tree_value) context);
   15761       563857 :       break;
   15762              : 
   15763              :     default:
   15764              :       break;
   15765              :     }
   15766              : 
   15767      3395322 :   return NULL_TREE;
   15768              : }
   15769              : 
   15770              : /* Pass 2: Check each branch and see if its context differs from that of
   15771              :    the destination label's context.  */
   15772              : 
   15773              : static tree
   15774      3395322 : diagnose_sb_2 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
   15775              :                struct walk_stmt_info *wi)
   15776              : {
   15777      3395322 :   gimple *context = (gimple *) wi->info;
   15778      3395322 :   splay_tree_node n;
   15779      3395322 :   gimple *stmt = gsi_stmt (*gsi_p);
   15780              : 
   15781      3395322 :   *handled_ops_p = true;
   15782              : 
   15783      3395322 :   switch (gimple_code (stmt))
   15784              :     {
   15785       327782 :     WALK_SUBSTMTS;
   15786              : 
   15787        84560 :     case GIMPLE_OMP_PARALLEL:
   15788        84560 :     case GIMPLE_OMP_TASK:
   15789        84560 :     case GIMPLE_OMP_SCOPE:
   15790        84560 :     case GIMPLE_OMP_SECTIONS:
   15791        84560 :     case GIMPLE_OMP_SINGLE:
   15792        84560 :     case GIMPLE_OMP_SECTION:
   15793        84560 :     case GIMPLE_OMP_STRUCTURED_BLOCK:
   15794        84560 :     case GIMPLE_OMP_MASTER:
   15795        84560 :     case GIMPLE_OMP_MASKED:
   15796        84560 :     case GIMPLE_OMP_ORDERED:
   15797        84560 :     case GIMPLE_OMP_SCAN:
   15798        84560 :     case GIMPLE_OMP_CRITICAL:
   15799        84560 :     case GIMPLE_OMP_TARGET:
   15800        84560 :     case GIMPLE_OMP_TEAMS:
   15801        84560 :     case GIMPLE_OMP_TASKGROUP:
   15802        84560 :       wi->info = stmt;
   15803        84560 :       walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
   15804        84560 :       wi->info = context;
   15805        84560 :       break;
   15806              : 
   15807        53086 :     case GIMPLE_OMP_FOR:
   15808        53086 :       wi->info = stmt;
   15809              :       /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
   15810              :          walk them.  */
   15811        53086 :       walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt),
   15812              :                            diagnose_sb_2, NULL, wi);
   15813        53086 :       walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
   15814        53086 :       wi->info = context;
   15815        53086 :       break;
   15816              : 
   15817       208786 :     case GIMPLE_COND:
   15818       208786 :         {
   15819       208786 :           gcond *cond_stmt = as_a <gcond *> (stmt);
   15820       208786 :           tree lab = gimple_cond_true_label (cond_stmt);
   15821       208786 :           if (lab)
   15822              :             {
   15823       208786 :               n = splay_tree_lookup (all_labels,
   15824              :                                      (splay_tree_key) lab);
   15825       417572 :               diagnose_sb_0 (gsi_p, context,
   15826       208786 :                              n ? (gimple *) n->value : NULL);
   15827              :             }
   15828       208786 :           lab = gimple_cond_false_label (cond_stmt);
   15829       208786 :           if (lab)
   15830              :             {
   15831       208786 :               n = splay_tree_lookup (all_labels,
   15832              :                                      (splay_tree_key) lab);
   15833       417572 :               diagnose_sb_0 (gsi_p, context,
   15834       208786 :                              n ? (gimple *) n->value : NULL);
   15835              :             }
   15836              :         }
   15837              :       break;
   15838              : 
   15839       113245 :     case GIMPLE_GOTO:
   15840       113245 :       {
   15841       113245 :         tree lab = gimple_goto_dest (stmt);
   15842       113245 :         if (TREE_CODE (lab) != LABEL_DECL)
   15843              :           break;
   15844              : 
   15845       113245 :         n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
   15846       113245 :         diagnose_sb_0 (gsi_p, context, n ? (gimple *) n->value : NULL);
   15847              :       }
   15848       113245 :       break;
   15849              : 
   15850          433 :     case GIMPLE_SWITCH:
   15851          433 :       {
   15852          433 :         gswitch *switch_stmt = as_a <gswitch *> (stmt);
   15853          433 :         unsigned int i;
   15854         2205 :         for (i = 0; i < gimple_switch_num_labels (switch_stmt); ++i)
   15855              :           {
   15856         1355 :             tree lab = CASE_LABEL (gimple_switch_label (switch_stmt, i));
   15857         1355 :             n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
   15858         1355 :             if (n && diagnose_sb_0 (gsi_p, context, (gimple *) n->value))
   15859              :               break;
   15860              :           }
   15861              :       }
   15862              :       break;
   15863              : 
   15864          756 :     case GIMPLE_ASM:
   15865          756 :       {
   15866          756 :         gasm *asm_stmt = as_a <gasm *> (stmt);
   15867         1527 :         for (unsigned i = 0; i < gimple_asm_nlabels (asm_stmt); ++i)
   15868              :           {
   15869           21 :             tree lab = TREE_VALUE (gimple_asm_label_op (asm_stmt, i));
   15870           21 :             n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
   15871           21 :             if (n && diagnose_sb_0 (gsi_p, context, (gimple *) n->value))
   15872              :               break;
   15873              :           }
   15874              :       }
   15875              :       break;
   15876              : 
   15877        36191 :     case GIMPLE_RETURN:
   15878        36191 :       diagnose_sb_0 (gsi_p, context, NULL);
   15879        36191 :       break;
   15880              : 
   15881              :     default:
   15882              :       break;
   15883              :     }
   15884              : 
   15885      3395322 :   return NULL_TREE;
   15886              : }
   15887              : 
   15888              : static unsigned int
   15889        59562 : diagnose_omp_structured_block_errors (void)
   15890              : {
   15891        59562 :   struct walk_stmt_info wi;
   15892        59562 :   gimple_seq body = gimple_body (current_function_decl);
   15893              : 
   15894        59562 :   all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0);
   15895              : 
   15896        59562 :   memset (&wi, 0, sizeof (wi));
   15897        59562 :   walk_gimple_seq (body, diagnose_sb_1, NULL, &wi);
   15898              : 
   15899        59562 :   memset (&wi, 0, sizeof (wi));
   15900        59562 :   wi.want_locations = true;
   15901        59562 :   walk_gimple_seq_mod (&body, diagnose_sb_2, NULL, &wi);
   15902              : 
   15903        59562 :   gimple_set_body (current_function_decl, body);
   15904              : 
   15905        59562 :   splay_tree_delete (all_labels);
   15906        59562 :   all_labels = NULL;
   15907              : 
   15908        59562 :   return 0;
   15909              : }
   15910              : 
   15911              : namespace {
   15912              : 
   15913              : const pass_data pass_data_diagnose_omp_blocks =
   15914              : {
   15915              :   GIMPLE_PASS, /* type */
   15916              :   "*diagnose_omp_blocks", /* name */
   15917              :   OPTGROUP_OMP, /* optinfo_flags */
   15918              :   TV_NONE, /* tv_id */
   15919              :   PROP_gimple_any, /* properties_required */
   15920              :   0, /* properties_provided */
   15921              :   0, /* properties_destroyed */
   15922              :   0, /* todo_flags_start */
   15923              :   0, /* todo_flags_finish */
   15924              : };
   15925              : 
   15926              : class pass_diagnose_omp_blocks : public gimple_opt_pass
   15927              : {
   15928              : public:
   15929       297180 :   pass_diagnose_omp_blocks (gcc::context *ctxt)
   15930       594360 :     : gimple_opt_pass (pass_data_diagnose_omp_blocks, ctxt)
   15931              :   {}
   15932              : 
   15933              :   /* opt_pass methods: */
   15934      3010155 :   bool gate (function *) final override
   15935              :   {
   15936      3010155 :     return flag_openacc || flag_openmp || flag_openmp_simd;
   15937              :   }
   15938        59562 :   unsigned int execute (function *) final override
   15939              :     {
   15940        59562 :       return diagnose_omp_structured_block_errors ();
   15941              :     }
   15942              : 
   15943              : }; // class pass_diagnose_omp_blocks
   15944              : 
   15945              : } // anon namespace
   15946              : 
   15947              : gimple_opt_pass *
   15948       297180 : make_pass_diagnose_omp_blocks (gcc::context *ctxt)
   15949              : {
   15950       297180 :   return new pass_diagnose_omp_blocks (ctxt);
   15951              : }
   15952              : 
   15953              : 
   15954              : #include "gt-omp-low.h"
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.