LCOV - code coverage report
Current view: top level - gcc - gimplify.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 94.5 % 10810 10219
Test Date: 2026-11-14 16:39:50 Functions: 97.9 % 242 237
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Tree lowering pass.  This pass converts the GENERIC functions-as-trees
       2              :    tree representation into the GIMPLE form.
       3              :    Copyright (C) 2002-2026 Free Software Foundation, Inc.
       4              :    Major work done by Sebastian Pop <s.pop@laposte.net>,
       5              :    Diego Novillo <dnovillo@redhat.com> and Jason Merrill <jason@redhat.com>.
       6              : 
       7              : This file is part of GCC.
       8              : 
       9              : GCC is free software; you can redistribute it and/or modify it under
      10              : the terms of the GNU General Public License as published by the Free
      11              : Software Foundation; either version 3, or (at your option) any later
      12              : version.
      13              : 
      14              : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      15              : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      16              : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      17              : for more details.
      18              : 
      19              : You should have received a copy of the GNU General Public License
      20              : along with GCC; see the file COPYING3.  If not see
      21              : <http://www.gnu.org/licenses/>.  */
      22              : 
      23              : #include "config.h"
      24              : #include "system.h"
      25              : #include "coretypes.h"
      26              : #include "backend.h"
      27              : #include "target.h"
      28              : #include "rtl.h"
      29              : #include "tree.h"
      30              : #include "memmodel.h"
      31              : #include "tm_p.h"
      32              : #include "gimple.h"
      33              : #include "gimple-predict.h"
      34              : #include "tree-pass.h"                /* FIXME: only for PROP_gimple_any */
      35              : #include "ssa.h"
      36              : #include "cgraph.h"
      37              : #include "tree-pretty-print.h"
      38              : #include "diagnostic-core.h"
      39              : #include "diagnostic.h"               /* For errorcount.  */
      40              : #include "alias.h"
      41              : #include "fold-const.h"
      42              : #include "calls.h"
      43              : #include "varasm.h"
      44              : #include "stmt.h"
      45              : #include "expr.h"
      46              : #include "gimple-iterator.h"
      47              : #include "gimple-fold.h"
      48              : #include "tree-eh.h"
      49              : #include "gimplify.h"
      50              : #include "stor-layout.h"
      51              : #include "print-tree.h"
      52              : #include "tree-iterator.h"
      53              : #include "tree-inline.h"
      54              : #include "langhooks.h"
      55              : #include "tree-cfg.h"
      56              : #include "tree-ssa.h"
      57              : #include "tree-hash-traits.h"
      58              : #include "omp-general.h"
      59              : #include "omp-low.h"
      60              : #include "gimple-low.h"
      61              : #include "gomp-constants.h"
      62              : #include "splay-tree.h"
      63              : #include "gimple-walk.h"
      64              : #include "langhooks-def.h"    /* FIXME: for lhd_set_decl_assembler_name */
      65              : #include "builtins.h"
      66              : #include "stringpool.h"
      67              : #include "attribs.h"
      68              : #include "asan.h"
      69              : #include "dbgcnt.h"
      70              : #include "omp-offload.h"
      71              : #include "context.h"
      72              : #include "tree-nested.h"
      73              : #include "gcc-urlifier.h"
      74              : #include "insn-config.h"
      75              : #include "recog.h"
      76              : #include "output.h"
      77              : #include "gimplify_reg_info.h"
      78              : #include "tree-ssa-loop-niter.h" /* For simplify_replace_tree.  */
      79              : 
      80              : /* Identifier for a basic condition, mapping it to other basic conditions of
      81              :    its Boolean expression.  Basic conditions given the same uid (in the same
      82              :    function) are parts of the same ANDIF/ORIF expression.  Used for condition
      83              :    coverage.  */
      84              : static unsigned nextconduid = 1;
      85              : 
      86              : /* Annotated gconds so that basic conditions in the same expression map to
      87              :    the same uid.  This is used for condition coverage.  */
      88              : static hash_map <tree, unsigned> *cond_uids;
      89              : 
      90              : /* Get a fresh identifier for a new condition expression.  This is used for
      91              :    condition coverage.  */
      92              : static unsigned
      93      5906125 : next_cond_uid ()
      94              : {
      95      5906125 :   return nextconduid++;
      96              : }
      97              : 
      98              : /* Reset the condition uid to the value it should have when compiling a new
      99              :    function.  0 is already the default/untouched value, so start at non-zero.
     100              :    A valid and set id should always be > 0.  This is used for condition
     101              :    coverage.  */
     102              : static void
     103      3010285 : reset_cond_uid ()
     104              : {
     105      3010285 :   nextconduid = 1;
     106            0 : }
     107              : 
     108              : /* Associate the condition STMT with the discriminator UID.  STMTs that are
     109              :    broken down with ANDIF/ORIF from the same Boolean expression should be given
     110              :    the same UID; 'if (a && b && c) { if (d || e) ... } ...' should yield the
     111              :    { a: 1, b: 1, c: 1, d: 2, e: 2 } when gimplification is done.  This is used
     112              :    for condition coverage.  */
     113              : static void
     114      1231009 : tree_associate_condition_with_expr (tree stmt, unsigned uid)
     115              : {
     116      1231009 :   if (!condition_coverage_flag)
     117              :     return;
     118              : 
     119          542 :   if (!cond_uids)
     120           68 :     cond_uids = new hash_map <tree, unsigned> ();
     121              : 
     122          542 :   cond_uids->put (stmt, uid);
     123              : }
     124              : 
     125              : /* Hash set of poisoned variables in a bind expr.  */
     126              : static hash_set<tree> *asan_poisoned_variables = NULL;
     127              : 
     128              : /* Hash set of already-resolved calls to OpenMP "declare variant"
     129              :    functions.  A call can resolve to the original function and
     130              :    we don't want to repeat the resolution multiple times.  */
     131              : static hash_set<tree> *omp_resolved_variant_calls = NULL;
     132              : 
     133              : enum gimplify_omp_var_data
     134              : {
     135              :   GOVD_SEEN = 0x000001,
     136              :   GOVD_EXPLICIT = 0x000002,
     137              :   GOVD_SHARED = 0x000004,
     138              :   GOVD_PRIVATE = 0x000008,
     139              :   GOVD_FIRSTPRIVATE = 0x000010,
     140              :   GOVD_LASTPRIVATE = 0x000020,
     141              :   GOVD_REDUCTION = 0x000040,
     142              :   GOVD_LOCAL = 0x00080,
     143              :   GOVD_MAP = 0x000100,
     144              :   GOVD_DEBUG_PRIVATE = 0x000200,
     145              :   GOVD_PRIVATE_OUTER_REF = 0x000400,
     146              :   GOVD_LINEAR = 0x000800,
     147              :   GOVD_ALIGNED = 0x001000,
     148              : 
     149              :   /* Flag for GOVD_MAP: don't copy back.  */
     150              :   GOVD_MAP_TO_ONLY = 0x002000,
     151              : 
     152              :   /* Flag for GOVD_LINEAR or GOVD_LASTPRIVATE: no outer reference.  */
     153              :   GOVD_LINEAR_LASTPRIVATE_NO_OUTER = 0x004000,
     154              : 
     155              :   GOVD_MAP_0LEN_ARRAY = 0x008000,
     156              : 
     157              :   /* Flag for GOVD_MAP, if it is always, to or always, tofrom mapping.  */
     158              :   GOVD_MAP_ALWAYS_TO = 0x010000,
     159              : 
     160              :   /* Flag for shared vars that are or might be stored to in the region.  */
     161              :   GOVD_WRITTEN = 0x020000,
     162              : 
     163              :   /* Flag for GOVD_MAP, if it is a forced mapping.  */
     164              :   GOVD_MAP_FORCE = 0x040000,
     165              : 
     166              :   /* Flag for GOVD_MAP: must be present already.  */
     167              :   GOVD_MAP_FORCE_PRESENT = 0x080000,
     168              : 
     169              :   /* Flag for GOVD_MAP: only allocate.  */
     170              :   GOVD_MAP_ALLOC_ONLY = 0x100000,
     171              : 
     172              :   /* Flag for GOVD_MAP: only copy back.  */
     173              :   GOVD_MAP_FROM_ONLY = 0x200000,
     174              : 
     175              :   GOVD_NONTEMPORAL = 0x400000,
     176              : 
     177              :   /* Flag for GOVD_LASTPRIVATE: conditional modifier.  */
     178              :   GOVD_LASTPRIVATE_CONDITIONAL = 0x800000,
     179              : 
     180              :   GOVD_CONDTEMP = 0x1000000,
     181              : 
     182              :   /* Flag for GOVD_REDUCTION: inscan seen in {in,ex}clusive clause.  */
     183              :   GOVD_REDUCTION_INSCAN = 0x2000000,
     184              : 
     185              :   /* Flag for GOVD_FIRSTPRIVATE: OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT.  */
     186              :   GOVD_FIRSTPRIVATE_IMPLICIT = 0x4000000,
     187              : 
     188              :   /* Flag to indicate this an allocator for the uses_allocators clause.  */
     189              :   GOVD_USES_ALLOCATORS_ALLOCATOR = 0x8000000,
     190              : 
     191              :   GOVD_DATA_SHARE_CLASS = (GOVD_SHARED | GOVD_PRIVATE | GOVD_FIRSTPRIVATE
     192              :                            | GOVD_LASTPRIVATE | GOVD_REDUCTION | GOVD_LINEAR
     193              :                            | GOVD_LOCAL)
     194              : };
     195              : 
     196              : 
     197              : enum omp_region_type
     198              : {
     199              :   ORT_WORKSHARE = 0x00,
     200              :   ORT_TASKGROUP = 0x01,
     201              :   ORT_DISPATCH  = 0x02,
     202              :   ORT_SIMD      = 0x04,
     203              : 
     204              :   ORT_PARALLEL  = 0x08,
     205              :   ORT_COMBINED_PARALLEL = ORT_PARALLEL | 1,
     206              : 
     207              :   ORT_TASK      = 0x10,
     208              :   ORT_UNTIED_TASK = ORT_TASK | 1,
     209              :   ORT_TASKLOOP  = ORT_TASK | 2,
     210              :   ORT_UNTIED_TASKLOOP = ORT_UNTIED_TASK | 2,
     211              : 
     212              :   ORT_TEAMS     = 0x20,
     213              :   ORT_COMBINED_TEAMS = ORT_TEAMS | 1,
     214              :   ORT_HOST_TEAMS = ORT_TEAMS | 2,
     215              :   ORT_COMBINED_HOST_TEAMS = ORT_COMBINED_TEAMS | 2,
     216              : 
     217              :   /* Data region.  */
     218              :   ORT_TARGET_DATA = 0x40,
     219              : 
     220              :   /* Data region with offloading.  */
     221              :   ORT_TARGET    = 0x80,
     222              :   ORT_COMBINED_TARGET = ORT_TARGET | 1,
     223              :   ORT_IMPLICIT_TARGET = ORT_TARGET | 2,
     224              : 
     225              :   /* OpenACC variants.  */
     226              :   ORT_ACC       = 0x100,  /* A generic OpenACC region.  */
     227              :   ORT_ACC_DATA  = ORT_ACC | ORT_TARGET_DATA, /* Data construct.  */
     228              :   ORT_ACC_PARALLEL = ORT_ACC | ORT_TARGET,  /* Parallel construct */
     229              :   ORT_ACC_KERNELS  = ORT_ACC | ORT_TARGET | 2,  /* Kernels construct.  */
     230              :   ORT_ACC_SERIAL   = ORT_ACC | ORT_TARGET | 4,  /* Serial construct.  */
     231              :   ORT_ACC_HOST_DATA = ORT_ACC | ORT_TARGET_DATA | 2,  /* Host data.  */
     232              : 
     233              :   /* Dummy OpenMP region, used to disable expansion of
     234              :      DECL_VALUE_EXPRs in taskloop pre body.  */
     235              :   ORT_NONE      = 0x200
     236              : };
     237              : 
     238              : /* Gimplify hashtable helper.  */
     239              : 
     240              : struct gimplify_hasher : free_ptr_hash <elt_t>
     241              : {
     242              :   static inline hashval_t hash (const elt_t *);
     243              :   static inline bool equal (const elt_t *, const elt_t *);
     244              : };
     245              : 
     246              : struct gimplify_ctx
     247              : {
     248              :   struct gimplify_ctx *prev_context;
     249              : 
     250              :   vec<gbind *> bind_expr_stack;
     251              :   tree temps;
     252              :   gimple_seq conditional_cleanups;
     253              :   tree exit_label;
     254              :   tree return_temp;
     255              : 
     256              :   vec<tree> case_labels;
     257              :   hash_set<tree> *live_switch_vars;
     258              :   /* The formal temporary table.  Should this be persistent?  */
     259              :   hash_table<gimplify_hasher> *temp_htab;
     260              : 
     261              :   int conditions;
     262              :   unsigned into_ssa : 1;
     263              :   unsigned allow_rhs_cond_expr : 1;
     264              :   unsigned in_cleanup_point_expr : 1;
     265              :   unsigned keep_stack : 1;
     266              :   unsigned save_stack : 1;
     267              :   unsigned in_switch_expr : 1;
     268              :   unsigned in_handler_expr : 1;
     269              : };
     270              : 
     271              : enum gimplify_defaultmap_kind
     272              : {
     273              :   GDMK_SCALAR,
     274              :   GDMK_SCALAR_TARGET, /* w/ Fortran's target attr, implicit mapping, only.  */
     275              :   GDMK_AGGREGATE,
     276              :   GDMK_ALLOCATABLE,
     277              :   GDMK_POINTER
     278              : };
     279              : 
     280              : struct gimplify_omp_ctx
     281              : {
     282              :   struct gimplify_omp_ctx *outer_context;
     283              :   splay_tree variables;
     284              :   hash_map<omp_name_type<tree>, tree> *implicit_mappers;
     285              :   hash_set<tree> *privatized_types;
     286              :   tree clauses;
     287              :   /* Iteration variables in an OMP_FOR.  */
     288              :   vec<tree> loop_iter_var;
     289              :   location_t location;
     290              :   enum omp_clause_default_kind default_kind;
     291              :   enum omp_region_type region_type;
     292              :   enum tree_code code;
     293              :   bool combined_loop;
     294              :   bool distribute;
     295              :   bool target_firstprivatize_array_bases;
     296              :   bool add_safelen1;
     297              :   bool order_concurrent;
     298              :   bool has_depend;
     299              :   bool in_for_exprs;
     300              :   bool in_call_args;
     301              :   int defaultmap[5];
     302              : };
     303              : 
     304              : static struct gimplify_ctx *gimplify_ctxp;
     305              : static struct gimplify_omp_ctx *gimplify_omp_ctxp;
     306              : static bool in_omp_construct;
     307              : 
     308              : /* Forward declaration.  */
     309              : static enum gimplify_status gimplify_compound_expr (tree *, gimple_seq *, bool);
     310              : static hash_map<tree, tree> *oacc_declare_returns;
     311              : static enum gimplify_status gimplify_expr (tree *, gimple_seq *, gimple_seq *,
     312              :                                            bool (*) (tree), fallback_t, bool);
     313              : static void prepare_gimple_addressable (tree *, gimple_seq *);
     314              : 
     315              : /* Shorter alias name for the above function for use in gimplify.cc
     316              :    only.  */
     317              : 
     318              : static inline void
     319     94954106 : gimplify_seq_add_stmt (gimple_seq *seq_p, gimple *gs)
     320              : {
     321     94954106 :   gimple_seq_add_stmt_without_update (seq_p, gs);
     322              : }
     323              : 
     324              : /* Append sequence SRC to the end of sequence *DST_P.  If *DST_P is
     325              :    NULL, a new sequence is allocated.   This function is
     326              :    similar to gimple_seq_add_seq, but does not scan the operands.
     327              :    During gimplification, we need to manipulate statement sequences
     328              :    before the def/use vectors have been constructed.  */
     329              : 
     330              : static void
     331      9571033 : gimplify_seq_add_seq (gimple_seq *dst_p, gimple_seq src)
     332              : {
     333      9571033 :   gimple_stmt_iterator si;
     334              : 
     335      9571033 :   if (src == NULL)
     336      4117489 :     return;
     337              : 
     338      5453544 :   si = gsi_last (*dst_p);
     339      5453544 :   gsi_insert_seq_after_without_update (&si, src, GSI_NEW_STMT);
     340              : }
     341              : 
     342              : 
     343              : /* Pointer to a list of allocated gimplify_ctx structs to be used for pushing
     344              :    and popping gimplify contexts.  */
     345              : 
     346              : static struct gimplify_ctx *ctx_pool = NULL;
     347              : 
     348              : /* Return a gimplify context struct from the pool.  */
     349              : 
     350              : static inline struct gimplify_ctx *
     351      9164794 : ctx_alloc (void)
     352              : {
     353      9164794 :   struct gimplify_ctx * c = ctx_pool;
     354              : 
     355      9164794 :   if (c)
     356      8892149 :     ctx_pool = c->prev_context;
     357              :   else
     358       272645 :     c = XNEW (struct gimplify_ctx);
     359              : 
     360      9164794 :   memset (c, '\0', sizeof (*c));
     361      9164794 :   return c;
     362              : }
     363              : 
     364              : /* Put gimplify context C back into the pool.  */
     365              : 
     366              : static inline void
     367      9164790 : ctx_free (struct gimplify_ctx *c)
     368              : {
     369      9164790 :   c->prev_context = ctx_pool;
     370      9164790 :   ctx_pool = c;
     371              : }
     372              : 
     373              : /* Free allocated ctx stack memory.  */
     374              : 
     375              : void
     376       239732 : free_gimplify_stack (void)
     377              : {
     378       239732 :   struct gimplify_ctx *c;
     379              : 
     380       481739 :   while ((c = ctx_pool))
     381              :     {
     382       242007 :       ctx_pool = c->prev_context;
     383       242007 :       free (c);
     384              :     }
     385       239732 : }
     386              : 
     387              : 
     388              : /* Set up a context for the gimplifier.  */
     389              : 
     390              : void
     391      9164794 : push_gimplify_context (bool in_ssa, bool rhs_cond_ok)
     392              : {
     393      9164794 :   struct gimplify_ctx *c = ctx_alloc ();
     394              : 
     395      9164794 :   c->prev_context = gimplify_ctxp;
     396      9164794 :   gimplify_ctxp = c;
     397      9164794 :   gimplify_ctxp->into_ssa = in_ssa;
     398      9164794 :   gimplify_ctxp->allow_rhs_cond_expr = rhs_cond_ok;
     399      9164794 : }
     400              : 
     401              : /* Tear down a context for the gimplifier.  If BODY is non-null, then
     402              :    put the temporaries into the outer BIND_EXPR.  Otherwise, put them
     403              :    in the local_decls.
     404              : 
     405              :    BODY is not a sequence, but the first tuple in a sequence.  */
     406              : 
     407              : void
     408      9164790 : pop_gimplify_context (gimple *body)
     409              : {
     410      9164790 :   struct gimplify_ctx *c = gimplify_ctxp;
     411              : 
     412      9164790 :   gcc_assert (c
     413              :               && (!c->bind_expr_stack.exists ()
     414              :                   || c->bind_expr_stack.is_empty ()));
     415      9164790 :   c->bind_expr_stack.release ();
     416      9164790 :   gimplify_ctxp = c->prev_context;
     417              : 
     418      9164790 :   if (body)
     419      3135985 :     declare_vars (c->temps, body, false);
     420              :   else
     421      6028805 :     record_vars (c->temps);
     422              : 
     423      9164790 :   delete c->temp_htab;
     424      9164790 :   c->temp_htab = NULL;
     425      9164790 :   ctx_free (c);
     426      9164790 : }
     427              : 
     428              : /* Push a GIMPLE_BIND tuple onto the stack of bindings.  */
     429              : 
     430              : static void
     431      6097656 : gimple_push_bind_expr (gbind *bind_stmt)
     432              : {
     433      6097656 :   gimplify_ctxp->bind_expr_stack.reserve (8);
     434      6097656 :   gimplify_ctxp->bind_expr_stack.safe_push (bind_stmt);
     435      6097656 : }
     436              : 
     437              : /* Pop the first element off the stack of bindings.  */
     438              : 
     439              : static void
     440      6097656 : gimple_pop_bind_expr (void)
     441              : {
     442            0 :   gimplify_ctxp->bind_expr_stack.pop ();
     443            0 : }
     444              : 
     445              : /* Return the first element of the stack of bindings.  */
     446              : 
     447              : gbind *
     448            0 : gimple_current_bind_expr (void)
     449              : {
     450            0 :   return gimplify_ctxp->bind_expr_stack.last ();
     451              : }
     452              : 
     453              : /* Return the stack of bindings created during gimplification.  */
     454              : 
     455              : vec<gbind *>
     456          370 : gimple_bind_expr_stack (void)
     457              : {
     458          370 :   return gimplify_ctxp->bind_expr_stack;
     459              : }
     460              : 
     461              : /* Return true iff there is a COND_EXPR between us and the innermost
     462              :    CLEANUP_POINT_EXPR.  This info is used by gimple_push_cleanup.  */
     463              : 
     464              : static bool
     465      2948668 : gimple_conditional_context (void)
     466              : {
     467      2948668 :   return gimplify_ctxp->conditions > 0;
     468              : }
     469              : 
     470              : /* Note that we've entered a COND_EXPR.  */
     471              : 
     472              : static void
     473      5906572 : gimple_push_condition (void)
     474              : {
     475              : #ifdef ENABLE_GIMPLE_CHECKING
     476      5906572 :   if (gimplify_ctxp->conditions == 0)
     477      3887862 :     gcc_assert (gimple_seq_empty_p (gimplify_ctxp->conditional_cleanups));
     478              : #endif
     479      5906572 :   ++(gimplify_ctxp->conditions);
     480      5906572 : }
     481              : 
     482              : /* Note that we've left a COND_EXPR.  If we're back at unconditional scope
     483              :    now, add any conditional cleanups we've seen to the prequeue.  */
     484              : 
     485              : static void
     486      5906572 : gimple_pop_condition (gimple_seq *pre_p)
     487              : {
     488      5906572 :   int conds = --(gimplify_ctxp->conditions);
     489              : 
     490      5906572 :   gcc_assert (conds >= 0);
     491      5906572 :   if (conds == 0)
     492              :     {
     493      3887862 :       gimplify_seq_add_seq (pre_p, gimplify_ctxp->conditional_cleanups);
     494      3887862 :       gimplify_ctxp->conditional_cleanups = NULL;
     495              :     }
     496      5906572 : }
     497              : 
     498              : /* A stable comparison routine for use with splay trees and DECLs.  */
     499              : 
     500              : static int
     501     18398552 : splay_tree_compare_decl_uid (splay_tree_key xa, splay_tree_key xb)
     502              : {
     503     18398552 :   tree a = (tree) xa;
     504     18398552 :   tree b = (tree) xb;
     505              : 
     506     18398552 :   return DECL_UID (a) - DECL_UID (b);
     507              : }
     508              : 
     509              : /* Create a new omp construct that deals with variable remapping.  */
     510              : 
     511              : static struct gimplify_omp_ctx *
     512       141692 : new_omp_context (enum omp_region_type region_type)
     513              : {
     514       141692 :   struct gimplify_omp_ctx *c;
     515              : 
     516       141692 :   c = XCNEW (struct gimplify_omp_ctx);
     517       141692 :   c->outer_context = gimplify_omp_ctxp;
     518       141692 :   c->variables = splay_tree_new (splay_tree_compare_decl_uid, 0, 0);
     519       141692 :   c->implicit_mappers = new hash_map<omp_name_type<tree>, tree>;
     520       141692 :   c->privatized_types = new hash_set<tree>;
     521       141692 :   c->location = input_location;
     522       141692 :   c->region_type = region_type;
     523       141692 :   if ((region_type & ORT_TASK) == 0)
     524              :     c->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
     525              :   else
     526         6325 :     c->default_kind = OMP_CLAUSE_DEFAULT_UNSPECIFIED;
     527       141692 :   c->defaultmap[GDMK_SCALAR] = GOVD_MAP;
     528       141692 :   c->defaultmap[GDMK_SCALAR_TARGET] = GOVD_MAP;
     529       141692 :   c->defaultmap[GDMK_AGGREGATE] = GOVD_MAP;
     530       141692 :   c->defaultmap[GDMK_ALLOCATABLE] = GOVD_MAP;
     531       141692 :   c->defaultmap[GDMK_POINTER] = GOVD_MAP;
     532              : 
     533       141692 :   return c;
     534              : }
     535              : 
     536              : /* Destroy an omp construct that deals with variable remapping.  */
     537              : 
     538              : static void
     539       141133 : delete_omp_context (struct gimplify_omp_ctx *c)
     540              : {
     541       141133 :   splay_tree_delete (c->variables);
     542       282266 :   delete c->privatized_types;
     543       282266 :   delete c->implicit_mappers;
     544       141133 :   c->loop_iter_var.release ();
     545       141133 :   XDELETE (c);
     546       141133 : }
     547              : 
     548              : static void omp_add_variable (struct gimplify_omp_ctx *, tree, unsigned int);
     549              : static bool omp_notice_variable (struct gimplify_omp_ctx *, tree, bool);
     550              : 
     551              : /* Both gimplify the statement T and append it to *SEQ_P.  This function
     552              :    behaves exactly as gimplify_stmt, but you don't have to pass T as a
     553              :    reference.  */
     554              : 
     555              : void
     556     39015182 : gimplify_and_add (tree t, gimple_seq *seq_p)
     557              : {
     558     39015182 :   gimplify_stmt (&t, seq_p);
     559     39015182 : }
     560              : 
     561              : /* Gimplify statement T into sequence *SEQ_P, and return the first
     562              :    tuple in the sequence of generated tuples for this statement.
     563              :    Return NULL if gimplifying T produced no tuples.  */
     564              : 
     565              : static gimple *
     566       106468 : gimplify_and_return_first (tree t, gimple_seq *seq_p)
     567              : {
     568       106468 :   gimple_stmt_iterator last = gsi_last (*seq_p);
     569              : 
     570       106468 :   gimplify_and_add (t, seq_p);
     571              : 
     572       106468 :   if (!gsi_end_p (last))
     573              :     {
     574         4882 :       gsi_next (&last);
     575         4882 :       return gsi_stmt (last);
     576              :     }
     577              :   else
     578       101586 :     return gimple_seq_first_stmt (*seq_p);
     579              : }
     580              : 
     581              : /* Returns true iff T is a valid RHS for an assignment to an un-renamed
     582              :    LHS, or for a call argument.  */
     583              : 
     584              : static bool
     585       238801 : is_gimple_mem_rhs (tree t)
     586              : {
     587              :   /* If we're dealing with a renamable type, either source or dest must be
     588              :      a renamed variable.  */
     589       238801 :   if (is_gimple_reg_type (TREE_TYPE (t)))
     590       235423 :     return is_gimple_val (t);
     591              :   else
     592         3378 :     return is_gimple_val (t) || is_gimple_lvalue (t);
     593              : }
     594              : 
     595              : /* Return true if T is a CALL_EXPR or an expression that can be
     596              :    assigned to a temporary.  Note that this predicate should only be
     597              :    used during gimplification.  See the rationale for this in
     598              :    gimplify_modify_expr.  */
     599              : 
     600              : static bool
     601    100184754 : is_gimple_reg_rhs_or_call (tree t)
     602              : {
     603     73591981 :   return (get_gimple_rhs_class (TREE_CODE (t)) != GIMPLE_INVALID_RHS
     604    100184754 :           || TREE_CODE (t) == CALL_EXPR);
     605              : }
     606              : 
     607              : /* Return true if T is a valid memory RHS or a CALL_EXPR.  Note that
     608              :    this predicate should only be used during gimplification.  See the
     609              :    rationale for this in gimplify_modify_expr.  */
     610              : 
     611              : static bool
     612     14733282 : is_gimple_mem_rhs_or_call (tree t)
     613              : {
     614              :   /* If we're dealing with a renamable type, either source or dest must be
     615              :      a renamed variable.  */
     616     14733282 :   if (is_gimple_reg_type (TREE_TYPE (t)))
     617     11214775 :     return is_gimple_val (t);
     618              :   else
     619      3518507 :     return (is_gimple_val (t)
     620      1777893 :             || is_gimple_lvalue (t)
     621      1245957 :             || (TREE_CODE (t) == CONSTRUCTOR && CONSTRUCTOR_NELTS (t) == 0)
     622      4764072 :             || TREE_CODE (t) == CALL_EXPR);
     623              : }
     624              : 
     625              : /* Create a temporary with a name derived from VAL.  Subroutine of
     626              :    lookup_tmp_var; nobody else should call this function.  */
     627              : 
     628              : static inline tree
     629      2201135 : create_tmp_from_val (tree val)
     630              : {
     631              :   /* Drop all qualifiers and address-space information from the value type.  */
     632      2201135 :   tree type = TYPE_MAIN_VARIANT (TREE_TYPE (val));
     633      2201135 :   tree var = create_tmp_var (type, get_name (val));
     634      2201135 :   return var;
     635              : }
     636              : 
     637              : /* Create a temporary to hold the value of VAL.  If IS_FORMAL, try to reuse
     638              :    an existing expression temporary.  If NOT_GIMPLE_REG, mark it as such.  */
     639              : 
     640              : static tree
     641      2440854 : lookup_tmp_var (tree val, bool is_formal, bool not_gimple_reg)
     642              : {
     643      2440854 :   tree ret;
     644              : 
     645              :   /* We cannot mark a formal temporary with DECL_NOT_GIMPLE_REG_P.  */
     646      2440854 :   gcc_assert (!is_formal || !not_gimple_reg);
     647              : 
     648              :   /* If not optimizing, never really reuse a temporary.  local-alloc
     649              :      won't allocate any variable that is used in more than one basic
     650              :      block, which means it will go into memory, causing much extra
     651              :      work in reload and final and poorer code generation, outweighing
     652              :      the extra memory allocation here.  */
     653      2440854 :   if (!optimize || !is_formal || TREE_SIDE_EFFECTS (val))
     654              :     {
     655      1109464 :       ret = create_tmp_from_val (val);
     656      1109464 :       DECL_NOT_GIMPLE_REG_P (ret) = not_gimple_reg;
     657              :     }
     658              :   else
     659              :     {
     660      1331390 :       elt_t elt, *elt_p;
     661      1331390 :       elt_t **slot;
     662              : 
     663      1331390 :       elt.val = val;
     664      1331390 :       if (!gimplify_ctxp->temp_htab)
     665       420946 :         gimplify_ctxp->temp_htab = new hash_table<gimplify_hasher> (1000);
     666      1331390 :       slot = gimplify_ctxp->temp_htab->find_slot (&elt, INSERT);
     667      1331390 :       if (*slot == NULL)
     668              :         {
     669      1091671 :           elt_p = XNEW (elt_t);
     670      1091671 :           elt_p->val = val;
     671      1091671 :           elt_p->temp = ret = create_tmp_from_val (val);
     672      1091671 :           *slot = elt_p;
     673              :         }
     674              :       else
     675              :         {
     676       239719 :           elt_p = *slot;
     677       239719 :           ret = elt_p->temp;
     678              :         }
     679              :     }
     680              : 
     681      2440854 :   return ret;
     682              : }
     683              : 
     684              : /* Helper for get_formal_tmp_var and get_initialized_tmp_var.  */
     685              : 
     686              : static tree
     687     27134571 : internal_get_tmp_var (tree val, gimple_seq *pre_p, gimple_seq *post_p,
     688              :                       bool is_formal, bool allow_ssa, bool not_gimple_reg)
     689              : {
     690     27134571 :   tree t, mod;
     691              : 
     692              :   /* Notice that we explicitly allow VAL to be a CALL_EXPR so that we
     693              :      can create an INIT_EXPR and convert it into a GIMPLE_CALL below.  */
     694     27134571 :   gimplify_expr (&val, pre_p, post_p, is_gimple_reg_rhs_or_call,
     695              :                  fb_rvalue);
     696              : 
     697     27134571 :   if (allow_ssa
     698     26864064 :       && gimplify_ctxp->into_ssa
     699     52520452 :       && is_gimple_reg_type (TREE_TYPE (val)))
     700              :     {
     701     24693718 :       t = make_ssa_name (TYPE_MAIN_VARIANT (TREE_TYPE (val)));
     702     24693718 :       if (! gimple_in_ssa_p (cfun))
     703              :         {
     704     21287363 :           const char *name = get_name (val);
     705     21287363 :           if (name)
     706      6874418 :             SET_SSA_NAME_VAR_OR_IDENTIFIER (t, create_tmp_var_name (name));
     707              :         }
     708              :     }
     709              :   else
     710      2440853 :     t = lookup_tmp_var (val, is_formal, not_gimple_reg);
     711              : 
     712     27134571 :   mod = build2 (INIT_EXPR, TREE_TYPE (t), t, unshare_expr (val));
     713              : 
     714     27134571 :   SET_EXPR_LOCATION (mod, EXPR_LOC_OR_LOC (val, input_location));
     715              : 
     716              :   /* gimplify_modify_expr might want to reduce this further.  */
     717     27134571 :   gimplify_and_add (mod, pre_p);
     718     27134571 :   ggc_free (mod);
     719              : 
     720              :   /* If we failed to gimplify VAL then we can end up with the temporary
     721              :      SSA name not having a definition.  In this case return a decl.  */
     722     27134571 :   if (TREE_CODE (t) == SSA_NAME && ! SSA_NAME_DEF_STMT (t))
     723            1 :     return lookup_tmp_var (val, is_formal, not_gimple_reg);
     724              : 
     725              :   return t;
     726              : }
     727              : 
     728              : /* Return a formal temporary variable initialized with VAL.  PRE_P is as
     729              :    in gimplify_expr.  Only use this function if:
     730              : 
     731              :    1) The value of the unfactored expression represented by VAL will not
     732              :       change between the initialization and use of the temporary, and
     733              :    2) The temporary will not be otherwise modified.
     734              : 
     735              :    For instance, #1 means that this is inappropriate for SAVE_EXPR temps,
     736              :    and #2 means it is inappropriate for && temps.
     737              : 
     738              :    For other cases, use get_initialized_tmp_var instead.  */
     739              : 
     740              : tree
     741     26631491 : get_formal_tmp_var (tree val, gimple_seq *pre_p)
     742              : {
     743     26631491 :   return internal_get_tmp_var (val, pre_p, NULL, true, true, false);
     744              : }
     745              : 
     746              : /* Return a temporary variable initialized with VAL.  PRE_P and POST_P
     747              :    are as in gimplify_expr.  */
     748              : 
     749              : tree
     750       490946 : get_initialized_tmp_var (tree val, gimple_seq *pre_p,
     751              :                          gimple_seq *post_p /* = NULL */,
     752              :                          bool allow_ssa /* = true */)
     753              : {
     754       490946 :   return internal_get_tmp_var (val, pre_p, post_p, false, allow_ssa, false);
     755              : }
     756              : 
     757              : /* Declare all the variables in VARS in SCOPE.  If DEBUG_INFO is true,
     758              :    generate debug info for them; otherwise don't.  */
     759              : 
     760              : void
     761      3144314 : declare_vars (tree vars, gimple *gs, bool debug_info)
     762              : {
     763      3144314 :   tree last = vars;
     764      3144314 :   if (last)
     765              :     {
     766      1978013 :       tree temps, block;
     767              : 
     768      1978013 :       gbind *scope = as_a <gbind *> (gs);
     769              : 
     770      1978013 :       temps = nreverse (last);
     771              : 
     772      1978013 :       block = gimple_bind_block (scope);
     773      1978013 :       gcc_assert (!block || TREE_CODE (block) == BLOCK);
     774      1978013 :       if (!block || !debug_info)
     775              :         {
     776      1974131 :           DECL_CHAIN (last) = gimple_bind_vars (scope);
     777      1974131 :           gimple_bind_set_vars (scope, temps);
     778              :         }
     779              :       else
     780              :         {
     781              :           /* We need to attach the nodes both to the BIND_EXPR and to its
     782              :              associated BLOCK for debugging purposes.  The key point here
     783              :              is that the BLOCK_VARS of the BIND_EXPR_BLOCK of a BIND_EXPR
     784              :              is a subchain of the BIND_EXPR_VARS of the BIND_EXPR.  */
     785         3882 :           if (BLOCK_VARS (block))
     786         3689 :             BLOCK_VARS (block) = chainon (BLOCK_VARS (block), temps);
     787              :           else
     788              :             {
     789          193 :               gimple_bind_set_vars (scope,
     790              :                                     chainon (gimple_bind_vars (scope), temps));
     791          193 :               BLOCK_VARS (block) = temps;
     792              :             }
     793              :         }
     794              :     }
     795      3144314 : }
     796              : 
     797              : /* For VAR a VAR_DECL of variable size, try to find a constant upper bound
     798              :    for the size and adjust DECL_SIZE/DECL_SIZE_UNIT accordingly.  Abort if
     799              :    no such upper bound can be obtained.  */
     800              : 
     801              : static void
     802            0 : force_constant_size (tree var)
     803              : {
     804              :   /* The only attempt we make is by querying the maximum size of objects
     805              :      of the variable's type.  */
     806              : 
     807            0 :   HOST_WIDE_INT max_size;
     808              : 
     809            0 :   gcc_assert (VAR_P (var));
     810              : 
     811            0 :   max_size = max_int_size_in_bytes (TREE_TYPE (var));
     812              : 
     813            0 :   gcc_assert (max_size >= 0);
     814              : 
     815            0 :   DECL_SIZE_UNIT (var)
     816            0 :     = build_int_cst (TREE_TYPE (DECL_SIZE_UNIT (var)), max_size);
     817            0 :   DECL_SIZE (var)
     818            0 :     = build_int_cst (TREE_TYPE (DECL_SIZE (var)), max_size * BITS_PER_UNIT);
     819            0 : }
     820              : 
     821              : /* Push the temporary variable TMP into the current binding.  */
     822              : 
     823              : void
     824        29869 : gimple_add_tmp_var_fn (struct function *fn, tree tmp)
     825              : {
     826        29869 :   gcc_assert (!DECL_CHAIN (tmp) && !DECL_SEEN_IN_BIND_EXPR_P (tmp));
     827              : 
     828              :   /* Later processing assumes that the object size is constant, which might
     829              :      not be true at this point.  Force the use of a constant upper bound in
     830              :      this case.  */
     831        29869 :   if (!tree_fits_poly_uint64_p (DECL_SIZE_UNIT (tmp)))
     832            0 :     force_constant_size (tmp);
     833              : 
     834        29869 :   DECL_CONTEXT (tmp) = fn->decl;
     835        29869 :   DECL_SEEN_IN_BIND_EXPR_P (tmp) = 1;
     836              : 
     837        29869 :   record_vars_into (tmp, fn->decl);
     838        29869 : }
     839              : 
     840              : /* Push the temporary variable TMP into the current binding.  */
     841              : 
     842              : void
     843     16803620 : gimple_add_tmp_var (tree tmp)
     844              : {
     845     16803620 :   gcc_assert (!DECL_CHAIN (tmp) && !DECL_SEEN_IN_BIND_EXPR_P (tmp));
     846              : 
     847              :   /* Later processing assumes that the object size is constant, which might
     848              :      not be true at this point.  Force the use of a constant upper bound in
     849              :      this case.  */
     850     16803620 :   if (!tree_fits_poly_uint64_p (DECL_SIZE_UNIT (tmp)))
     851            0 :     force_constant_size (tmp);
     852              : 
     853     16803620 :   DECL_CONTEXT (tmp) = current_function_decl;
     854     16803620 :   DECL_SEEN_IN_BIND_EXPR_P (tmp) = 1;
     855              : 
     856     16803620 :   if (gimplify_ctxp)
     857              :     {
     858      5819274 :       DECL_CHAIN (tmp) = gimplify_ctxp->temps;
     859      5819274 :       gimplify_ctxp->temps = tmp;
     860              : 
     861              :       /* Mark temporaries local within the nearest enclosing parallel.  */
     862      5819274 :       if (gimplify_omp_ctxp)
     863              :         {
     864              :           struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
     865       555515 :           int flag = GOVD_LOCAL | GOVD_SEEN;
     866              :           while (ctx
     867       555515 :                  && (ctx->region_type == ORT_WORKSHARE
     868              :                      || ctx->region_type == ORT_TASKGROUP
     869       407360 :                      || ctx->region_type == ORT_SIMD
     870       356791 :                      || ctx->region_type == ORT_ACC))
     871              :             {
     872       205152 :               if (ctx->region_type == ORT_SIMD
     873        50569 :                   && TREE_ADDRESSABLE (tmp)
     874           95 :                   && !TREE_STATIC (tmp))
     875              :                 {
     876           95 :                   if (TREE_CODE (DECL_SIZE_UNIT (tmp)) != INTEGER_CST)
     877            0 :                     ctx->add_safelen1 = true;
     878           95 :                   else if (ctx->in_for_exprs)
     879              :                     flag = GOVD_PRIVATE;
     880              :                   else
     881              :                     flag = GOVD_PRIVATE | GOVD_SEEN;
     882              :                   break;
     883              :                 }
     884       205057 :               ctx = ctx->outer_context;
     885              :             }
     886       350363 :           if (ctx)
     887       322943 :             omp_add_variable (ctx, tmp, flag);
     888              :         }
     889              :     }
     890     10984346 :   else if (cfun)
     891     10984346 :     record_vars (tmp);
     892              :   else
     893              :     {
     894            0 :       gimple_seq body_seq;
     895              : 
     896              :       /* This case is for nested functions.  We need to expose the locals
     897              :          they create.  */
     898            0 :       body_seq = gimple_body (current_function_decl);
     899            0 :       declare_vars (tmp, gimple_seq_first_stmt (body_seq), false);
     900              :     }
     901     16803620 : }
     902              : 
     903              : /* Unshare all the trees in the body of FNDECL, as well as in the bodies of
     904              :    any nested functions.  */
     905              : 
     906              : static void
     907      3038012 : unshare_body (tree fndecl)
     908              : {
     909      3038012 :   struct cgraph_node *cgn = cgraph_node::get (fndecl);
     910              :   /* If the language requires deep unsharing, we need a pointer set to make
     911              :      sure we don't repeatedly unshare subtrees of unshareable nodes.  */
     912      3038012 :   hash_set<tree> *visited
     913      3038012 :     = lang_hooks.deep_unsharing ? new hash_set<tree> : NULL;
     914              : 
     915      3038012 :   copy_if_shared (&DECL_SAVED_TREE (fndecl), visited);
     916      3038012 :   copy_if_shared (&DECL_SIZE (DECL_RESULT (fndecl)), visited);
     917      3038012 :   copy_if_shared (&DECL_SIZE_UNIT (DECL_RESULT (fndecl)), visited);
     918              : 
     919      3042188 :   delete visited;
     920              : 
     921      3038012 :   if (cgn)
     922      6122058 :     for (cgn = first_nested_function (cgn); cgn;
     923        27214 :          cgn = next_nested_function (cgn))
     924        27214 :       unshare_body (cgn->decl);
     925      3038012 : }
     926              : 
     927              : /* Callback for walk_tree to unmark the visited trees rooted at *TP.
     928              :    Subtrees are walked until the first unvisited node is encountered.  */
     929              : 
     930              : static tree
     931    288873512 : unmark_visited_r (tree *tp, int *walk_subtrees, void *data ATTRIBUTE_UNUSED)
     932              : {
     933    288873512 :   tree t = *tp;
     934              : 
     935              :   /* If this node has been visited, unmark it and keep looking.  */
     936    288873512 :   if (TREE_VISITED (t))
     937    202824863 :     TREE_VISITED (t) = 0;
     938              : 
     939              :   /* Otherwise, don't look any deeper.  */
     940              :   else
     941     86048649 :     *walk_subtrees = 0;
     942              : 
     943    288873512 :   return NULL_TREE;
     944              : }
     945              : 
     946              : /* Unmark the visited trees rooted at *TP.  */
     947              : 
     948              : static inline void
     949      9114036 : unmark_visited (tree *tp)
     950              : {
     951      9114036 :   walk_tree (tp, unmark_visited_r, NULL, NULL);
     952      9114036 : }
     953              : 
     954              : /* Likewise, but mark all trees as not visited.  */
     955              : 
     956              : static void
     957      3038012 : unvisit_body (tree fndecl)
     958              : {
     959      3038012 :   struct cgraph_node *cgn = cgraph_node::get (fndecl);
     960              : 
     961      3038012 :   unmark_visited (&DECL_SAVED_TREE (fndecl));
     962      3038012 :   unmark_visited (&DECL_SIZE (DECL_RESULT (fndecl)));
     963      3038012 :   unmark_visited (&DECL_SIZE_UNIT (DECL_RESULT (fndecl)));
     964              : 
     965      3038012 :   if (cgn)
     966      6094844 :     for (cgn = first_nested_function (cgn);
     967      3061029 :          cgn; cgn = next_nested_function (cgn))
     968        27214 :       unvisit_body (cgn->decl);
     969      3038012 : }
     970              : 
     971              : /* Return the EXPR_LOCATION of EXPR, if it (maybe recursively) has
     972              :    one, OR_ELSE otherwise.  The location of a STATEMENT_LISTs
     973              :    comprising at least one DEBUG_BEGIN_STMT followed by exactly one
     974              :    EXPR is the location of the EXPR.  */
     975              : 
     976              : static location_t
     977      1194475 : rexpr_location (tree expr, location_t or_else = UNKNOWN_LOCATION)
     978              : {
     979      1194475 :   if (!expr)
     980              :     return or_else;
     981              : 
     982      1194475 :   if (EXPR_HAS_LOCATION (expr))
     983       830631 :     return EXPR_LOCATION (expr);
     984              : 
     985       363844 :   if (TREE_CODE (expr) != STATEMENT_LIST)
     986              :     return or_else;
     987              : 
     988            0 :   tree_stmt_iterator i = tsi_start (expr);
     989              : 
     990            0 :   bool found = false;
     991            0 :   while (!tsi_end_p (i) && TREE_CODE (tsi_stmt (i)) == DEBUG_BEGIN_STMT)
     992              :     {
     993            0 :       found = true;
     994            0 :       tsi_next (&i);
     995              :     }
     996              : 
     997            0 :   if (!found || !tsi_one_before_end_p (i))
     998              :     return or_else;
     999              : 
    1000            0 :   return rexpr_location (tsi_stmt (i), or_else);
    1001              : }
    1002              : 
    1003              : /* Return TRUE iff EXPR (maybe recursively) has a location; see
    1004              :    rexpr_location for the potential recursion.  */
    1005              : 
    1006              : static inline bool
    1007       522035 : rexpr_has_location (tree expr)
    1008              : {
    1009       313100 :   return rexpr_location (expr) != UNKNOWN_LOCATION;
    1010              : }
    1011              : 
    1012              : 
    1013              : /* WRAPPER is a code such as BIND_EXPR or CLEANUP_POINT_EXPR which can both
    1014              :    contain statements and have a value.  Assign its value to a temporary
    1015              :    and give it void_type_node.  Return the temporary, or NULL_TREE if
    1016              :    WRAPPER was already void.  */
    1017              : 
    1018              : tree
    1019     20802473 : voidify_wrapper_expr (tree wrapper, tree temp)
    1020              : {
    1021     20802473 :   tree type = TREE_TYPE (wrapper);
    1022     20802473 :   if (type && !VOID_TYPE_P (type))
    1023              :     {
    1024              :       tree *p;
    1025              : 
    1026              :       /* Set p to point to the body of the wrapper.  Loop until we find
    1027              :          something that isn't a wrapper.  */
    1028       797075 :       for (p = &wrapper; p && *p; )
    1029              :         {
    1030       797075 :           switch (TREE_CODE (*p))
    1031              :             {
    1032         3148 :             case BIND_EXPR:
    1033         3148 :               TREE_SIDE_EFFECTS (*p) = 1;
    1034         3148 :               TREE_TYPE (*p) = void_type_node;
    1035              :               /* For a BIND_EXPR, the body is operand 1.  */
    1036         3148 :               p = &BIND_EXPR_BODY (*p);
    1037         3148 :               break;
    1038              : 
    1039       368878 :             case CLEANUP_POINT_EXPR:
    1040       368878 :             case TRY_FINALLY_EXPR:
    1041       368878 :             case TRY_CATCH_EXPR:
    1042       368878 :               TREE_SIDE_EFFECTS (*p) = 1;
    1043       368878 :               TREE_TYPE (*p) = void_type_node;
    1044       368878 :               p = &TREE_OPERAND (*p, 0);
    1045       368878 :               break;
    1046              : 
    1047        17984 :             case STATEMENT_LIST:
    1048        17984 :               {
    1049        17984 :                 tree_stmt_iterator i = tsi_last (*p);
    1050        17984 :                 TREE_SIDE_EFFECTS (*p) = 1;
    1051        17984 :                 TREE_TYPE (*p) = void_type_node;
    1052        17984 :                 p = tsi_end_p (i) ? NULL : tsi_stmt_ptr (i);
    1053              :               }
    1054        17984 :               break;
    1055              : 
    1056              :             case COMPOUND_EXPR:
    1057              :               /* Advance to the last statement.  Set all container types to
    1058              :                  void.  */
    1059        64081 :               for (; TREE_CODE (*p) == COMPOUND_EXPR; p = &TREE_OPERAND (*p, 1))
    1060              :                 {
    1061        32048 :                   TREE_SIDE_EFFECTS (*p) = 1;
    1062        32048 :                   TREE_TYPE (*p) = void_type_node;
    1063              :                 }
    1064              :               break;
    1065              : 
    1066           76 :             case TRANSACTION_EXPR:
    1067           76 :               TREE_SIDE_EFFECTS (*p) = 1;
    1068           76 :               TREE_TYPE (*p) = void_type_node;
    1069           76 :               p = &TRANSACTION_EXPR_BODY (*p);
    1070           76 :               break;
    1071              : 
    1072       374956 :             default:
    1073              :               /* Assume that any tree upon which voidify_wrapper_expr is
    1074              :                  directly called is a wrapper, and that its body is op0.  */
    1075       374956 :               if (p == &wrapper)
    1076              :                 {
    1077           33 :                   TREE_SIDE_EFFECTS (*p) = 1;
    1078           33 :                   TREE_TYPE (*p) = void_type_node;
    1079           33 :                   p = &TREE_OPERAND (*p, 0);
    1080           33 :                   break;
    1081              :                 }
    1082       374923 :               goto out;
    1083              :             }
    1084              :         }
    1085              : 
    1086            0 :     out:
    1087       374923 :       if (p == NULL || IS_EMPTY_STMT (*p))
    1088              :         temp = NULL_TREE;
    1089       374923 :       else if (temp)
    1090              :         {
    1091              :           /* The wrapper is on the RHS of an assignment that we're pushing
    1092              :              down.  */
    1093         2021 :           gcc_assert (TREE_CODE (temp) == INIT_EXPR
    1094              :                       || TREE_CODE (temp) == MODIFY_EXPR);
    1095         2021 :           TREE_OPERAND (temp, 1) = *p;
    1096         2021 :           *p = temp;
    1097              :         }
    1098              :       else
    1099              :         {
    1100       372902 :           temp = create_tmp_var (type, "retval");
    1101       372902 :           *p = build2 (INIT_EXPR, type, temp, *p);
    1102              :         }
    1103              : 
    1104              :       return temp;
    1105              :     }
    1106              : 
    1107              :   return NULL_TREE;
    1108              : }
    1109              : 
    1110              : /* Prepare calls to builtins to SAVE and RESTORE the stack as well as
    1111              :    a temporary through which they communicate.  */
    1112              : 
    1113              : static void
    1114         7935 : build_stack_save_restore (gcall **save, gcall **restore)
    1115              : {
    1116         7935 :   tree tmp_var;
    1117              : 
    1118        15870 :   *save = gimple_build_call (builtin_decl_implicit (BUILT_IN_STACK_SAVE), 0);
    1119         7935 :   tmp_var = create_tmp_var (ptr_type_node, "saved_stack");
    1120         7935 :   gimple_call_set_lhs (*save, tmp_var);
    1121              : 
    1122         7935 :   *restore
    1123         7935 :     = gimple_build_call (builtin_decl_implicit (BUILT_IN_STACK_RESTORE),
    1124              :                          1, tmp_var);
    1125         7935 : }
    1126              : 
    1127              : /* Generate IFN_ASAN_MARK call that poisons shadow memory of the DECL
    1128              :    variable.  */
    1129              : 
    1130              : static tree
    1131          472 : build_asan_poison_call_expr (tree decl)
    1132              : {
    1133              :   /* Do not poison variables that have size equal to zero.  */
    1134          472 :   tree unit_size = DECL_SIZE_UNIT (decl);
    1135          472 :   if (zerop (unit_size))
    1136              :     return NULL_TREE;
    1137              : 
    1138          472 :   tree base = build_fold_addr_expr (decl);
    1139              : 
    1140          472 :   return build_call_expr_internal_loc (UNKNOWN_LOCATION, IFN_ASAN_MARK,
    1141              :                                        void_type_node, 3,
    1142              :                                        build_int_cst (integer_type_node,
    1143              :                                                       ASAN_MARK_POISON),
    1144              :                                        base, unit_size);
    1145              : }
    1146              : 
    1147              : /* Generate IFN_ASAN_MARK call that would poison or unpoison, depending
    1148              :    on POISON flag, shadow memory of a DECL variable.  The call will be
    1149              :    put on location identified by IT iterator, where BEFORE flag drives
    1150              :    position where the stmt will be put.  */
    1151              : 
    1152              : static void
    1153         5091 : asan_poison_variable (tree decl, bool poison, gimple_stmt_iterator *it,
    1154              :                       bool before)
    1155              : {
    1156         5091 :   tree unit_size = DECL_SIZE_UNIT (decl);
    1157         5091 :   tree base = build_fold_addr_expr (decl);
    1158              : 
    1159              :   /* Do not poison variables that have size equal to zero.  */
    1160         5091 :   if (zerop (unit_size))
    1161         5091 :     return;
    1162              : 
    1163              :   /* It's necessary to have all stack variables aligned to ASAN granularity
    1164              :      bytes.  */
    1165         5077 :   gcc_assert (!hwassist_sanitize_p () || hwassist_sanitize_stack_p ());
    1166         5077 :   unsigned shadow_granularity
    1167         5077 :     = (hwassist_sanitize_p ()
    1168         5077 :        ? HWASAN_TAG_GRANULE_SIZE : ASAN_SHADOW_GRANULARITY);
    1169         5077 :   if (DECL_ALIGN_UNIT (decl) <= shadow_granularity)
    1170         4875 :     SET_DECL_ALIGN (decl, BITS_PER_UNIT * shadow_granularity);
    1171              : 
    1172         5077 :   HOST_WIDE_INT flags = poison ? ASAN_MARK_POISON : ASAN_MARK_UNPOISON;
    1173              : 
    1174         5077 :   gimple *g
    1175         5077 :     = gimple_build_call_internal (IFN_ASAN_MARK, 3,
    1176         5077 :                                   build_int_cst (integer_type_node, flags),
    1177              :                                   base, unit_size);
    1178              : 
    1179         5077 :   if (before)
    1180         2648 :     gsi_insert_before (it, g, GSI_NEW_STMT);
    1181              :   else
    1182         2429 :     gsi_insert_after (it, g, GSI_NEW_STMT);
    1183              : }
    1184              : 
    1185              : /* Generate IFN_ASAN_MARK internal call that depending on POISON flag
    1186              :    either poisons or unpoisons a DECL.  Created statement is appended
    1187              :    to SEQ_P gimple sequence.  */
    1188              : 
    1189              : static void
    1190         4619 : asan_poison_variable (tree decl, bool poison, gimple_seq *seq_p)
    1191              : {
    1192         4619 :   gimple_stmt_iterator it = gsi_last (*seq_p);
    1193         4619 :   bool before = false;
    1194              : 
    1195         4619 :   if (gsi_end_p (it))
    1196         2439 :     before = true;
    1197              : 
    1198         4619 :   asan_poison_variable (decl, poison, &it, before);
    1199         4619 : }
    1200              : 
    1201              : /* Sort pair of VAR_DECLs A and B by DECL_UID.  */
    1202              : 
    1203              : static int
    1204          135 : sort_by_decl_uid (const void *a, const void *b)
    1205              : {
    1206          135 :   const tree *t1 = (const tree *)a;
    1207          135 :   const tree *t2 = (const tree *)b;
    1208              : 
    1209          135 :   int uid1 = DECL_UID (*t1);
    1210          135 :   int uid2 = DECL_UID (*t2);
    1211              : 
    1212          135 :   if (uid1 < uid2)
    1213              :     return -1;
    1214           54 :   else if (uid1 > uid2)
    1215              :     return 1;
    1216              :   else
    1217            0 :     return 0;
    1218              : }
    1219              : 
    1220              : /* Generate IFN_ASAN_MARK internal call for all VARIABLES
    1221              :    depending on POISON flag.  Created statement is appended
    1222              :    to SEQ_P gimple sequence.  */
    1223              : 
    1224              : static void
    1225      1046658 : asan_poison_variables (hash_set<tree> *variables, bool poison, gimple_seq *seq_p)
    1226              : {
    1227      1046658 :   unsigned c = variables->elements ();
    1228      1046658 :   if (c == 0)
    1229      1046384 :     return;
    1230              : 
    1231          274 :   auto_vec<tree> sorted_variables (c);
    1232              : 
    1233          274 :   for (hash_set<tree>::iterator it = variables->begin ();
    1234          569 :        it != variables->end (); ++it)
    1235          295 :     sorted_variables.safe_push (*it);
    1236              : 
    1237          274 :   sorted_variables.qsort (sort_by_decl_uid);
    1238              : 
    1239          274 :   unsigned i;
    1240          274 :   tree var;
    1241          843 :   FOR_EACH_VEC_ELT (sorted_variables, i, var)
    1242              :     {
    1243          295 :       asan_poison_variable (var, poison, seq_p);
    1244              : 
    1245              :       /* Add use_after_scope_memory attribute for the variable in order
    1246              :          to prevent re-written into SSA.  */
    1247          295 :       if (!lookup_attribute (ASAN_USE_AFTER_SCOPE_ATTRIBUTE,
    1248          295 :                              DECL_ATTRIBUTES (var)))
    1249           97 :         DECL_ATTRIBUTES (var)
    1250          194 :           = tree_cons (get_identifier (ASAN_USE_AFTER_SCOPE_ATTRIBUTE),
    1251              :                        integer_one_node,
    1252           97 :                        DECL_ATTRIBUTES (var));
    1253              :     }
    1254          274 : }
    1255              : 
    1256              : /* Gimplify a BIND_EXPR.  Just voidify and recurse.  */
    1257              : 
    1258              : static enum gimplify_status
    1259      6095400 : gimplify_bind_expr (tree *expr_p, gimple_seq *pre_p)
    1260              : {
    1261      6095400 :   tree bind_expr = *expr_p;
    1262      6095400 :   bool old_keep_stack = gimplify_ctxp->keep_stack;
    1263      6095400 :   bool old_save_stack = gimplify_ctxp->save_stack;
    1264      6095400 :   tree t;
    1265      6095400 :   gbind *bind_stmt;
    1266      6095400 :   gimple_seq body, cleanup;
    1267      6095400 :   gcall *stack_save;
    1268      6095400 :   location_t start_locus = 0, end_locus = 0;
    1269      6095400 :   tree ret_clauses = NULL;
    1270              : 
    1271      6095400 :   tree temp = voidify_wrapper_expr (bind_expr, NULL);
    1272              : 
    1273              :   /* Mark variables seen in this bind expr.  */
    1274     13145929 :   for (t = BIND_EXPR_VARS (bind_expr); t ; t = DECL_CHAIN (t))
    1275              :     {
    1276      7050529 :       if (VAR_P (t))
    1277              :         {
    1278      6376094 :           struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
    1279      6376094 :           tree attr;
    1280              : 
    1281      6376094 :           if (flag_openmp
    1282       264787 :               && !is_global_var (t)
    1283       255747 :               && !TREE_STATIC (t)
    1284       255747 :               && DECL_CONTEXT (t) == current_function_decl
    1285       255747 :               && TREE_USED (t)
    1286       253641 :               && (attr = lookup_attribute ("omp allocate", DECL_ATTRIBUTES (t)))
    1287              :                  != NULL_TREE
    1288              :               /* In C++, bad attributes are marked instead of removed.  */
    1289      6377371 :               && TREE_PURPOSE (TREE_VALUE (attr)) != error_mark_node)
    1290              :             {
    1291         1253 :               gcc_assert (!DECL_HAS_VALUE_EXPR_P (t));
    1292         1253 :               tree alloc = TREE_PURPOSE (TREE_VALUE (attr));
    1293         1253 :               tree align = TREE_VALUE (TREE_VALUE (attr));
    1294              :               /* The C++ front end smuggles a location through the chain field,
    1295              :                  this conflicts with Fortran handling, clear it.  */
    1296         1253 :               if (TREE_CHAIN (TREE_VALUE (attr)) != NULL_TREE
    1297         1253 :                   && TREE_CODE (TREE_CHAIN (TREE_VALUE (attr))) == NOP_EXPR)
    1298         1175 :                 TREE_CHAIN (TREE_VALUE (attr)) = NULL_TREE;
    1299              : 
    1300              :               /* Allocate directives that appear in a target region must specify
    1301              :                  an allocator clause unless a requires directive with the
    1302              :                  dynamic_allocators clause is present in the same compilation
    1303              :                  unit.  */
    1304         1253 :               bool missing_dyn_alloc = false;
    1305         1253 :               if ((omp_requires_mask & OMP_REQUIRES_DYNAMIC_ALLOCATORS) == 0)
    1306              :                 {
    1307              :                   /* This comes too early for omp_discover_declare_target...,
    1308              :                      but should at least catch the most common cases.  */
    1309         1241 :                   missing_dyn_alloc
    1310         1241 :                     = (alloc == NULL_TREE
    1311         1241 :                        && cgraph_node::get (current_function_decl)->offloadable);
    1312         1241 :                   for (struct gimplify_omp_ctx *ctx2 = ctx;
    1313         1251 :                        ctx2 && !missing_dyn_alloc; ctx2 = ctx2->outer_context)
    1314           29 :                     if (ctx2->code == OMP_TARGET)
    1315              :                       {
    1316           19 :                         if (alloc == NULL_TREE)
    1317              :                           missing_dyn_alloc = true;
    1318           14 :                         else if (TREE_CODE (alloc) != INTEGER_CST)
    1319              :                           {
    1320            4 :                             tree alloc2 = alloc;
    1321            4 :                             if (TREE_CODE (alloc2) == MEM_REF
    1322            4 :                                 || TREE_CODE (alloc2) == INDIRECT_REF)
    1323            0 :                               alloc2 = TREE_OPERAND (alloc2, 0);
    1324            4 :                             tree c2;
    1325            4 :                             for (c2 = ctx2->clauses; c2;
    1326            0 :                                  c2 = OMP_CLAUSE_CHAIN (c2))
    1327            2 :                               if (OMP_CLAUSE_CODE (c2)
    1328              :                                   == OMP_CLAUSE_USES_ALLOCATORS)
    1329              :                                 {
    1330            2 :                                   tree t2
    1331            2 :                                     = OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c2);
    1332            2 :                                   if (operand_equal_p (alloc2, t2))
    1333              :                                     break;
    1334              :                                 }
    1335            4 :                             if (c2 == NULL_TREE)
    1336            2 :                               error_at (EXPR_LOC_OR_LOC (
    1337              :                                           alloc, DECL_SOURCE_LOCATION (t)),
    1338              :                                         "%qE in %<allocator%> clause inside a "
    1339              :                                         "target region must be specified in an "
    1340              :                                         "%<uses_allocators%> clause on the "
    1341              :                                         "%<target%> directive", alloc2);
    1342              :                           }
    1343              :                         break;
    1344              :                       }
    1345              :                 }
    1346         1236 :               if (missing_dyn_alloc)
    1347           10 :                 error_at (DECL_SOURCE_LOCATION (t),
    1348              :                           "%<allocate%> directive for %qD inside a target "
    1349              :                           "region must specify an %<allocator%> clause", t);
    1350              :               /* Skip for omp_default_mem_alloc (= 1),
    1351              :                  unless align is present.  For C/C++, there should be always a
    1352              :                  statement list following if TREE_USED, except for, e.g., using
    1353              :                  this decl in a static_assert; in that case, only a single
    1354              :                  DECL_EXPR remains, which can be skipped here.  */
    1355         1243 :               else if (!errorcount
    1356         1186 :                        && (align != NULL_TREE
    1357         1186 :                            || alloc == NULL_TREE
    1358          758 :                            || !integer_onep (alloc))
    1359         2362 :                        && (lang_GNU_Fortran ()
    1360         1091 :                            || (TREE_CODE (BIND_EXPR_BODY (bind_expr))
    1361              :                                != DECL_EXPR)))
    1362              :                 {
    1363              :                   /* Fortran might already use a pointer type internally;
    1364              :                      use that pointer except for type(C_ptr) and type(C_funptr);
    1365              :                      note that normal proc pointers are rejected.  */
    1366         1119 :                   tree type = TREE_TYPE (t);
    1367         1119 :                   tree tmp, v;
    1368         1119 :                   if (lang_GNU_Fortran ()
    1369           28 :                       && POINTER_TYPE_P (type)
    1370            8 :                       && TREE_TYPE (type) != void_type_node
    1371         1125 :                       && TREE_CODE (TREE_TYPE (type)) != FUNCTION_TYPE)
    1372              :                     {
    1373            6 :                       type = TREE_TYPE (type);
    1374            6 :                       v = t;
    1375              :                     }
    1376              :                   else
    1377              :                     {
    1378         1113 :                       tmp = build_pointer_type (type);
    1379         1113 :                       v = create_tmp_var (tmp, get_name (t));
    1380         1113 :                       DECL_IGNORED_P (v) = 0;
    1381         1113 :                       DECL_ATTRIBUTES (v)
    1382         1113 :                         = tree_cons (get_identifier ("omp allocate var"),
    1383              :                                      build_tree_list (NULL_TREE, t),
    1384              :                                      remove_attribute ("omp allocate",
    1385         1113 :                                                        DECL_ATTRIBUTES (t)));
    1386         1113 :                       tmp = build_fold_indirect_ref (v);
    1387         1113 :                       TREE_THIS_NOTRAP (tmp) = 1;
    1388         1113 :                       SET_DECL_VALUE_EXPR (t, tmp);
    1389         1113 :                       DECL_HAS_VALUE_EXPR_P (t) = 1;
    1390              :                     }
    1391         1119 :                   tree sz = TYPE_SIZE_UNIT (type);
    1392              :                   /* The size to use in Fortran might not match TYPE_SIZE_UNIT;
    1393              :                      hence, for some decls, a size variable is saved in the
    1394              :                      attributes; use it, if available.  */
    1395         1119 :                   if (TREE_CHAIN (TREE_VALUE (attr))
    1396           28 :                       && TREE_CHAIN (TREE_CHAIN (TREE_VALUE (attr)))
    1397         1125 :                       && TREE_PURPOSE (
    1398              :                            TREE_CHAIN (TREE_CHAIN (TREE_VALUE (attr)))))
    1399              :                     {
    1400            6 :                       sz = TREE_CHAIN (TREE_CHAIN (TREE_VALUE (attr)));
    1401            6 :                       sz = TREE_PURPOSE (sz);
    1402              :                     }
    1403         1119 :                   if (alloc == NULL_TREE)
    1404          227 :                     alloc = build_zero_cst (ptr_type_node);
    1405         1119 :                   if (align == NULL_TREE)
    1406          801 :                     align = build_int_cst (size_type_node, DECL_ALIGN_UNIT (t));
    1407              :                   else
    1408          318 :                     align = build_int_cst (size_type_node,
    1409          318 :                                            MAX (tree_to_uhwi (align),
    1410              :                                                 DECL_ALIGN_UNIT (t)));
    1411         1119 :                   location_t loc = DECL_SOURCE_LOCATION (t);
    1412         1119 :                   tmp = builtin_decl_explicit (BUILT_IN_GOMP_ALLOC);
    1413         1119 :                   tmp = build_call_expr_loc (loc, tmp, 3, align, sz, alloc);
    1414         1119 :                   tmp = fold_build2_loc (loc, MODIFY_EXPR, TREE_TYPE (v), v,
    1415         1119 :                                          fold_convert (TREE_TYPE (v), tmp));
    1416         1119 :                   gcc_assert (BIND_EXPR_BODY (bind_expr) != NULL_TREE);
    1417              :                   /* Ensure that either TREE_CHAIN (TREE_VALUE (attr) is set
    1418              :                      and GOMP_FREE added here or that DECL_HAS_VALUE_EXPR_P (t)
    1419              :                      is set, using in a condition much further below.  */
    1420         1125 :                   gcc_assert (DECL_HAS_VALUE_EXPR_P (t)
    1421              :                               || TREE_CHAIN (TREE_VALUE (attr)));
    1422         1119 :                   if (TREE_CHAIN (TREE_VALUE (attr)))
    1423              :                     {
    1424              :                       /* Fortran is special as it does not have properly nest
    1425              :                          declarations in blocks.  And as there is no
    1426              :                          initializer, there is also no expression to look for.
    1427              :                          Hence, the FE makes the statement list of the
    1428              :                          try-finally block available. We can put the GOMP_alloc
    1429              :                          at the top, unless an allocator or size expression
    1430              :                          requires to put it afterward; note that the size is
    1431              :                          always later in generated code; for strings, no
    1432              :                          size expr but still an expr might be available.
    1433              :                          As LTO does not handle a statement list, 'sl' has
    1434              :                          to be removed; done so by removing the attribute.  */
    1435           28 :                       DECL_ATTRIBUTES (t)
    1436           28 :                         = remove_attribute ("omp allocate",
    1437           28 :                                             DECL_ATTRIBUTES (t));
    1438           28 :                       tree sl = TREE_PURPOSE (TREE_CHAIN (TREE_VALUE (attr)));
    1439           28 :                       tree_stmt_iterator e = tsi_start (sl);
    1440           28 :                       tree needle = NULL_TREE;
    1441           28 :                       if (TREE_CHAIN (TREE_CHAIN (TREE_VALUE (attr))))
    1442              :                         {
    1443            6 :                           needle = TREE_CHAIN (TREE_CHAIN (TREE_VALUE (attr)));
    1444            6 :                           needle = (TREE_VALUE (needle) ? TREE_VALUE (needle)
    1445              :                                                         : sz);
    1446              :                         }
    1447           22 :                       else if (TREE_CHAIN (TREE_CHAIN (TREE_VALUE (attr))))
    1448              :                         needle = sz;
    1449           22 :                       else if (DECL_P (alloc) && DECL_ARTIFICIAL (alloc))
    1450              :                         needle = alloc;
    1451              : 
    1452           10 :                       if (needle != NULL_TREE)
    1453              :                         {
    1454           49 :                           while (!tsi_end_p (e))
    1455              :                             {
    1456           49 :                               if (*e == needle
    1457           49 :                                   || (TREE_CODE (*e) == MODIFY_EXPR
    1458           49 :                                       && TREE_OPERAND (*e, 0) == needle))
    1459              :                                 break;
    1460           39 :                               ++e;
    1461              :                             }
    1462           10 :                           gcc_assert (!tsi_end_p (e));
    1463              :                         }
    1464           28 :                       tsi_link_after (&e, tmp, TSI_SAME_STMT);
    1465              : 
    1466              :                       /* As the cleanup is in BIND_EXPR_BODY, GOMP_free is added
    1467              :                          here; for C/C++ it will be added in the 'cleanup'
    1468              :                          section after gimplification. But Fortran already has
    1469              :                          a try-finally block.  */
    1470           28 :                       sl = TREE_VALUE (TREE_CHAIN (TREE_VALUE (attr)));
    1471           28 :                       e = tsi_last (sl);
    1472           28 :                       tmp = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
    1473           28 :                       tmp = build_call_expr_loc (EXPR_LOCATION (*e), tmp, 2, v,
    1474              :                                                  build_zero_cst (ptr_type_node));
    1475           28 :                       tsi_link_after (&e, tmp, TSI_SAME_STMT);
    1476           28 :                       tmp = build_clobber (TREE_TYPE (v), CLOBBER_STORAGE_END);
    1477           28 :                       tmp = fold_build2_loc (loc, MODIFY_EXPR, TREE_TYPE (v), v,
    1478           28 :                                              fold_convert (TREE_TYPE (v), tmp));
    1479           28 :                       ++e;
    1480           28 :                       tsi_link_after (&e, tmp, TSI_SAME_STMT);
    1481              :                     }
    1482              :                   else
    1483              :                     {
    1484         1091 :                       gcc_assert (TREE_CODE (BIND_EXPR_BODY (bind_expr))
    1485              :                                   == STATEMENT_LIST);
    1486         1091 :                       tree_stmt_iterator e;
    1487         1091 :                       e = tsi_start (BIND_EXPR_BODY (bind_expr));
    1488         2344 :                       while (!tsi_end_p (e))
    1489              :                         {
    1490         1253 :                           if ((TREE_CODE (*e) == DECL_EXPR
    1491          818 :                                && TREE_OPERAND (*e, 0) == t)
    1492         1309 :                               || (TREE_CODE (*e) == CLEANUP_POINT_EXPR
    1493          373 :                                   && (TREE_CODE (TREE_OPERAND (*e, 0))
    1494              :                                       == DECL_EXPR)
    1495          371 :                                   && (TREE_OPERAND (TREE_OPERAND (*e, 0), 0)
    1496              :                                       == t)))
    1497              :                             break;
    1498          162 :                           ++e;
    1499              :                         }
    1500         1091 :                       gcc_assert (!tsi_end_p (e));
    1501         1091 :                       tsi_link_before (&e, tmp, TSI_SAME_STMT);
    1502              :                    }
    1503              :                 }
    1504              :             }
    1505              : 
    1506              :           /* Mark variable as local.  */
    1507      6376094 :           if (ctx && ctx->region_type != ORT_NONE && !DECL_EXTERNAL (t))
    1508              :             {
    1509       101221 :               if (! DECL_SEEN_IN_BIND_EXPR_P (t)
    1510       101221 :                   || splay_tree_lookup (ctx->variables,
    1511              :                                         (splay_tree_key) t) == NULL)
    1512              :                 {
    1513       101221 :                   int flag = GOVD_LOCAL;
    1514       101221 :                   if (ctx->region_type == ORT_SIMD
    1515         4388 :                       && TREE_ADDRESSABLE (t)
    1516           83 :                       && !TREE_STATIC (t))
    1517              :                     {
    1518           71 :                       if (TREE_CODE (DECL_SIZE_UNIT (t)) != INTEGER_CST)
    1519            2 :                         ctx->add_safelen1 = true;
    1520              :                       else
    1521              :                         flag = GOVD_PRIVATE;
    1522              :                     }
    1523       101221 :                   omp_add_variable (ctx, t, flag | GOVD_SEEN);
    1524              :                 }
    1525              :               /* Static locals inside of target construct or offloaded
    1526              :                  routines need to be "omp declare target".  */
    1527       101221 :               if (TREE_STATIC (t))
    1528         1399 :                 for (; ctx; ctx = ctx->outer_context)
    1529         1214 :                   if ((ctx->region_type & ORT_TARGET) != 0)
    1530              :                     {
    1531          970 :                       if (!lookup_attribute ("omp declare target",
    1532          970 :                                              DECL_ATTRIBUTES (t)))
    1533              :                         {
    1534          958 :                           tree id = get_identifier ("omp declare target");
    1535          958 :                           DECL_ATTRIBUTES (t)
    1536          958 :                             = tree_cons (id, NULL_TREE, DECL_ATTRIBUTES (t));
    1537          958 :                           varpool_node *node = varpool_node::get (t);
    1538          958 :                           if (node)
    1539              :                             {
    1540           33 :                               node->offloadable = 1;
    1541           33 :                               if (ENABLE_OFFLOADING && !DECL_EXTERNAL (t))
    1542              :                                 {
    1543              :                                   g->have_offload = true;
    1544              :                                   if (!in_lto_p)
    1545              :                                     vec_safe_push (offload_vars, t);
    1546              :                                 }
    1547              :                             }
    1548              :                         }
    1549              :                       break;
    1550              :                     }
    1551              :             }
    1552              : 
    1553      6376094 :           DECL_SEEN_IN_BIND_EXPR_P (t) = 1;
    1554              : 
    1555      6376094 :           if (DECL_HARD_REGISTER (t) && !is_global_var (t) && cfun)
    1556         1093 :             cfun->has_local_explicit_reg_vars = true;
    1557              :         }
    1558              :     }
    1559              : 
    1560     12190800 :   bind_stmt = gimple_build_bind (BIND_EXPR_VARS (bind_expr), NULL,
    1561      6095400 :                                  BIND_EXPR_BLOCK (bind_expr));
    1562      6095400 :   gimple_push_bind_expr (bind_stmt);
    1563              : 
    1564      6095400 :   gimplify_ctxp->keep_stack = false;
    1565      6095400 :   gimplify_ctxp->save_stack = false;
    1566              : 
    1567              :   /* Gimplify the body into the GIMPLE_BIND tuple's body.  */
    1568      6095400 :   body = NULL;
    1569      6095400 :   gimplify_stmt (&BIND_EXPR_BODY (bind_expr), &body);
    1570      6095400 :   gimple_bind_set_body (bind_stmt, body);
    1571              : 
    1572              :   /* Source location wise, the cleanup code (stack_restore and clobbers)
    1573              :      belongs to the end of the block, so propagate what we have.  The
    1574              :      stack_save operation belongs to the beginning of block, which we can
    1575              :      infer from the bind_expr directly if the block has no explicit
    1576              :      assignment.  */
    1577      6095400 :   if (BIND_EXPR_BLOCK (bind_expr))
    1578              :     {
    1579      5963657 :       end_locus = BLOCK_SOURCE_END_LOCATION (BIND_EXPR_BLOCK (bind_expr));
    1580      5963657 :       start_locus = BLOCK_SOURCE_LOCATION (BIND_EXPR_BLOCK (bind_expr));
    1581              :     }
    1582      5963657 :   if (start_locus == 0)
    1583      6095400 :     start_locus = EXPR_LOCATION (bind_expr);
    1584              : 
    1585      6095400 :   cleanup = NULL;
    1586      6095400 :   stack_save = NULL;
    1587              : 
    1588              :   /* Add clobbers for all variables that go out of scope.  */
    1589     13145929 :   for (t = BIND_EXPR_VARS (bind_expr); t ; t = DECL_CHAIN (t))
    1590              :     {
    1591      7050529 :       if (VAR_P (t)
    1592      6376094 :           && !is_global_var (t)
    1593     13200929 :           && DECL_CONTEXT (t) == current_function_decl)
    1594              :         {
    1595      6150400 :           if (flag_openmp
    1596       255710 :               && DECL_HAS_VALUE_EXPR_P (t)
    1597         2800 :               && TREE_USED (t)
    1598      6153180 :               && lookup_attribute ("omp allocate", DECL_ATTRIBUTES (t)))
    1599              :             {
    1600              :               /* For Fortran, TREE_CHAIN (TREE_VALUE (attr)) is set, which
    1601              :                  causes that the GOMP_free call is already added above;
    1602              :                  and "omp allocate" is removed from DECL_ATTRIBUTES.  */
    1603         1091 :               tree v = TREE_OPERAND (DECL_VALUE_EXPR (t), 0);
    1604         1091 :               tree tmp = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
    1605         1091 :               tmp = build_call_expr_loc (end_locus, tmp, 2, v,
    1606              :                                          build_zero_cst (ptr_type_node));
    1607         1091 :               gimplify_and_add (tmp, &cleanup);
    1608         1091 :               gimple *clobber_stmt;
    1609         1091 :               tmp = build_clobber (TREE_TYPE (v), CLOBBER_STORAGE_END);
    1610         1091 :               clobber_stmt = gimple_build_assign (v, tmp);
    1611         1091 :               gimple_set_location (clobber_stmt, end_locus);
    1612         1091 :               gimplify_seq_add_stmt (&cleanup, clobber_stmt);
    1613              :             }
    1614      6150400 :           if (!DECL_HARD_REGISTER (t)
    1615      6149307 :               && !TREE_THIS_VOLATILE (t)
    1616      6109448 :               && !DECL_HAS_VALUE_EXPR_P (t)
    1617              :               /* Only care for variables that have to be in memory.  Others
    1618              :                  will be rewritten into SSA names, hence moved to the
    1619              :                  top-level.  */
    1620      5980045 :               && !is_gimple_reg (t)
    1621      7440690 :               && flag_stack_reuse != SR_NONE)
    1622              :             {
    1623      1286209 :               tree clobber = build_clobber (TREE_TYPE (t), CLOBBER_STORAGE_END);
    1624      1286209 :               gimple *clobber_stmt;
    1625      1286209 :               clobber_stmt = gimple_build_assign (t, clobber);
    1626      1286209 :               gimple_set_location (clobber_stmt, end_locus);
    1627      1286209 :               gimplify_seq_add_stmt (&cleanup, clobber_stmt);
    1628              :             }
    1629              : 
    1630      6150400 :           if (flag_openacc && oacc_declare_returns != NULL)
    1631              :             {
    1632          207 :               tree key = t;
    1633          207 :               if (DECL_HAS_VALUE_EXPR_P (key))
    1634              :                 {
    1635            8 :                   key = DECL_VALUE_EXPR (key);
    1636            8 :                   if (INDIRECT_REF_P (key))
    1637            8 :                     key = TREE_OPERAND (key, 0);
    1638              :                 }
    1639          207 :               tree *c = oacc_declare_returns->get (key);
    1640          207 :               if (c != NULL)
    1641              :                 {
    1642          116 :                   if (ret_clauses)
    1643           64 :                     OMP_CLAUSE_CHAIN (*c) = ret_clauses;
    1644              : 
    1645          116 :                   ret_clauses = unshare_expr (*c);
    1646              : 
    1647          116 :                   oacc_declare_returns->remove (key);
    1648              : 
    1649          116 :                   if (oacc_declare_returns->is_empty ())
    1650              :                     {
    1651           40 :                       delete oacc_declare_returns;
    1652           40 :                       oacc_declare_returns = NULL;
    1653              :                     }
    1654              :                 }
    1655              :             }
    1656              :         }
    1657              : 
    1658      7050529 :       if (asan_poisoned_variables != NULL
    1659      7050529 :           && asan_poisoned_variables->contains (t))
    1660              :         {
    1661         2162 :           asan_poisoned_variables->remove (t);
    1662         2162 :           asan_poison_variable (t, true, &cleanup);
    1663              :         }
    1664              : 
    1665      7050529 :       if (gimplify_ctxp->live_switch_vars != NULL
    1666      7050529 :           && gimplify_ctxp->live_switch_vars->contains (t))
    1667           55 :         gimplify_ctxp->live_switch_vars->remove (t);
    1668              :     }
    1669              : 
    1670              :   /* If the code both contains VLAs and calls alloca, then we cannot reclaim
    1671              :      the stack space allocated to the VLAs.  */
    1672      6095400 :   if (gimplify_ctxp->save_stack && !gimplify_ctxp->keep_stack)
    1673              :     {
    1674         7935 :       gcall *stack_restore;
    1675              : 
    1676              :       /* Save stack on entry and restore it on exit.  Add a try_finally
    1677              :          block to achieve this.  */
    1678         7935 :       build_stack_save_restore (&stack_save, &stack_restore);
    1679              : 
    1680         7935 :       gimple_set_location (stack_save, start_locus);
    1681         7935 :       gimple_set_location (stack_restore, end_locus);
    1682              : 
    1683         7935 :       gimplify_seq_add_stmt (&cleanup, stack_restore);
    1684              :     }
    1685              : 
    1686      6095400 :   if (ret_clauses)
    1687              :     {
    1688           52 :       gomp_target *stmt;
    1689           52 :       gimple_stmt_iterator si = gsi_start (cleanup);
    1690              : 
    1691           52 :       stmt = gimple_build_omp_target (NULL, GF_OMP_TARGET_KIND_OACC_DECLARE,
    1692              :                                       ret_clauses);
    1693           52 :       gsi_insert_seq_before_without_update (&si, stmt, GSI_NEW_STMT);
    1694              :     }
    1695              : 
    1696      6095400 :   if (cleanup)
    1697              :     {
    1698       793917 :       gtry *gs;
    1699       793917 :       gimple_seq new_body;
    1700              : 
    1701       793917 :       new_body = NULL;
    1702       793917 :       gs = gimple_build_try (gimple_bind_body (bind_stmt), cleanup,
    1703              :                              GIMPLE_TRY_FINALLY);
    1704              : 
    1705       793917 :       if (stack_save)
    1706         7935 :         gimplify_seq_add_stmt (&new_body, stack_save);
    1707       793917 :       gimplify_seq_add_stmt (&new_body, gs);
    1708       793917 :       gimple_bind_set_body (bind_stmt, new_body);
    1709              :     }
    1710              : 
    1711              :   /* keep_stack propagates all the way up to the outermost BIND_EXPR.  */
    1712      6095400 :   if (!gimplify_ctxp->keep_stack)
    1713      6080288 :     gimplify_ctxp->keep_stack = old_keep_stack;
    1714      6095400 :   gimplify_ctxp->save_stack = old_save_stack;
    1715              : 
    1716      6095400 :   gimple_pop_bind_expr ();
    1717              : 
    1718      6095400 :   gimplify_seq_add_stmt (pre_p, bind_stmt);
    1719              : 
    1720      6095400 :   if (temp)
    1721              :     {
    1722         1709 :       *expr_p = temp;
    1723         1709 :       return GS_OK;
    1724              :     }
    1725              : 
    1726              :   *expr_p = NULL_TREE;
    1727              :   return GS_ALL_DONE;
    1728              : }
    1729              : 
    1730              : /* Maybe add early return predict statement to PRE_P sequence.  */
    1731              : 
    1732              : static void
    1733      2298177 : maybe_add_early_return_predict_stmt (gimple_seq *pre_p)
    1734              : {
    1735              :   /* If we are not in a conditional context, add PREDICT statement.  */
    1736      2298177 :   if (gimple_conditional_context ())
    1737              :     {
    1738       448885 :       gimple *predict = gimple_build_predict (PRED_TREE_EARLY_RETURN,
    1739              :                                               NOT_TAKEN);
    1740       448885 :       gimplify_seq_add_stmt (pre_p, predict);
    1741              :     }
    1742      2298177 : }
    1743              : 
    1744              : /* Gimplify a RETURN_EXPR.  If the expression to be returned is not a
    1745              :    GIMPLE value, it is assigned to a new temporary and the statement is
    1746              :    re-written to return the temporary.
    1747              : 
    1748              :    PRE_P points to the sequence where side effects that must happen before
    1749              :    STMT should be stored.  */
    1750              : 
    1751              : static enum gimplify_status
    1752      2298186 : gimplify_return_expr (tree stmt, gimple_seq *pre_p)
    1753              : {
    1754      2298186 :   greturn *ret;
    1755      2298186 :   tree ret_expr = TREE_OPERAND (stmt, 0);
    1756      2298186 :   tree result_decl, result;
    1757              : 
    1758      2298186 :   if (ret_expr == error_mark_node)
    1759              :     return GS_ERROR;
    1760              : 
    1761      2298177 :   if (!ret_expr
    1762      2207316 :       || TREE_CODE (ret_expr) == RESULT_DECL)
    1763              :     {
    1764       106817 :       maybe_add_early_return_predict_stmt (pre_p);
    1765       106817 :       greturn *ret = gimple_build_return (ret_expr);
    1766       106817 :       copy_warning (ret, stmt);
    1767       106817 :       gimplify_seq_add_stmt (pre_p, ret);
    1768       106817 :       return GS_ALL_DONE;
    1769              :     }
    1770              : 
    1771      2191360 :   if (VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))))
    1772              :     result_decl = NULL_TREE;
    1773      2191038 :   else if (TREE_CODE (ret_expr) == COMPOUND_EXPR)
    1774              :     {
    1775              :       /* Used in C++ for handling EH cleanup of the return value if a local
    1776              :          cleanup throws.  Assume the front-end knows what it's doing.  */
    1777         4969 :       result_decl = DECL_RESULT (current_function_decl);
    1778              :       /* But crash if we end up trying to modify ret_expr below.  */
    1779         4969 :       ret_expr = NULL_TREE;
    1780              :     }
    1781              :   else
    1782              :     {
    1783      2186069 :       result_decl = TREE_OPERAND (ret_expr, 0);
    1784              : 
    1785              :       /* See through a return by reference.  */
    1786      2186069 :       if (INDIRECT_REF_P (result_decl))
    1787        43455 :         result_decl = TREE_OPERAND (result_decl, 0);
    1788              : 
    1789      2186069 :       gcc_assert ((TREE_CODE (ret_expr) == MODIFY_EXPR
    1790              :                    || TREE_CODE (ret_expr) == INIT_EXPR)
    1791              :                   && TREE_CODE (result_decl) == RESULT_DECL);
    1792              :     }
    1793              : 
    1794              :   /* If aggregate_value_p is true, then we can return the bare RESULT_DECL.
    1795              :      Recall that aggregate_value_p is FALSE for any aggregate type that is
    1796              :      returned in registers.  If we're returning values in registers, then
    1797              :      we don't want to extend the lifetime of the RESULT_DECL, particularly
    1798              :      across another call.  In addition, for those aggregates for which
    1799              :      hard_function_value generates a PARALLEL, we'll die during normal
    1800              :      expansion of structure assignments; there's special code in expand_return
    1801              :      to handle this case that does not exist in expand_expr.  */
    1802         4969 :   if (!result_decl)
    1803              :     result = NULL_TREE;
    1804      2191038 :   else if (aggregate_value_p (result_decl, TREE_TYPE (current_function_decl)))
    1805              :     {
    1806       182134 :       if (!poly_int_tree_p (DECL_SIZE (result_decl)))
    1807              :         {
    1808           48 :           if (!TYPE_SIZES_GIMPLIFIED (TREE_TYPE (result_decl)))
    1809            0 :             gimplify_type_sizes (TREE_TYPE (result_decl), pre_p);
    1810              :           /* Note that we don't use gimplify_vla_decl because the RESULT_DECL
    1811              :              should be effectively allocated by the caller, i.e. all calls to
    1812              :              this function must be subject to the Return Slot Optimization.  */
    1813           48 :           gimplify_one_sizepos (&DECL_SIZE (result_decl), pre_p);
    1814           48 :           gimplify_one_sizepos (&DECL_SIZE_UNIT (result_decl), pre_p);
    1815              :         }
    1816              :       result = result_decl;
    1817              :     }
    1818      2008904 :   else if (gimplify_ctxp->return_temp)
    1819              :     result = gimplify_ctxp->return_temp;
    1820              :   else
    1821              :     {
    1822      1509153 :       result = create_tmp_reg (TREE_TYPE (result_decl));
    1823              : 
    1824              :       /* ??? With complex control flow (usually involving abnormal edges),
    1825              :          we can wind up warning about an uninitialized value for this.  Due
    1826              :          to how this variable is constructed and initialized, this is never
    1827              :          true.  Give up and never warn.  */
    1828      1509153 :       suppress_warning (result, OPT_Wuninitialized);
    1829              : 
    1830      1509153 :       gimplify_ctxp->return_temp = result;
    1831              :     }
    1832              : 
    1833              :   /* Smash the lhs of the MODIFY_EXPR to the temporary we plan to use.
    1834              :      Then gimplify the whole thing.  */
    1835      2191360 :   if (result != result_decl)
    1836      2008904 :     TREE_OPERAND (ret_expr, 0) = result;
    1837              : 
    1838      2191360 :   gimplify_and_add (TREE_OPERAND (stmt, 0), pre_p);
    1839              : 
    1840      2191360 :   maybe_add_early_return_predict_stmt (pre_p);
    1841      2191360 :   ret = gimple_build_return (result);
    1842      2191360 :   copy_warning (ret, stmt);
    1843      2191360 :   gimplify_seq_add_stmt (pre_p, ret);
    1844              : 
    1845      2191360 :   return GS_ALL_DONE;
    1846              : }
    1847              : 
    1848              : /* Gimplify a variable-length array DECL.  */
    1849              : 
    1850              : static void
    1851         9053 : gimplify_vla_decl (tree decl, gimple_seq *seq_p)
    1852              : {
    1853              :   /* This is a variable-sized decl.  Simplify its size and mark it
    1854              :      for deferred expansion.  */
    1855         9053 :   tree t, addr, ptr_type;
    1856              : 
    1857         9053 :   gimplify_one_sizepos (&DECL_SIZE (decl), seq_p);
    1858         9053 :   gimplify_one_sizepos (&DECL_SIZE_UNIT (decl), seq_p);
    1859              : 
    1860              :   /* Don't mess with a DECL_VALUE_EXPR set by the front-end.  */
    1861         9053 :   if (DECL_HAS_VALUE_EXPR_P (decl))
    1862              :     return;
    1863              : 
    1864              :   /* All occurrences of this decl in final gimplified code will be
    1865              :      replaced by indirection.  Setting DECL_VALUE_EXPR does two
    1866              :      things: First, it lets the rest of the gimplifier know what
    1867              :      replacement to use.  Second, it lets the debug info know
    1868              :      where to find the value.  */
    1869         9041 :   ptr_type = build_pointer_type (TREE_TYPE (decl));
    1870         9041 :   addr = create_tmp_var (ptr_type, get_name (decl));
    1871         9041 :   DECL_IGNORED_P (addr) = 0;
    1872         9041 :   t = build_fold_indirect_ref (addr);
    1873         9041 :   TREE_THIS_NOTRAP (t) = 1;
    1874         9041 :   SET_DECL_VALUE_EXPR (decl, t);
    1875         9041 :   DECL_HAS_VALUE_EXPR_P (decl) = 1;
    1876              : 
    1877         9041 :   t = build_alloca_call_expr (DECL_SIZE_UNIT (decl), DECL_ALIGN (decl),
    1878         9041 :                               max_int_size_in_bytes (TREE_TYPE (decl)));
    1879              :   /* The call has been built for a variable-sized object.  */
    1880         9041 :   CALL_ALLOCA_FOR_VAR_P (t) = 1;
    1881         9041 :   t = fold_convert (ptr_type, t);
    1882         9041 :   t = build2 (MODIFY_EXPR, TREE_TYPE (addr), addr, t);
    1883              : 
    1884         9041 :   gimplify_and_add (t, seq_p);
    1885              : 
    1886              :   /* Record the dynamic allocation associated with DECL if requested.  */
    1887         9041 :   if (flag_callgraph_info & CALLGRAPH_INFO_DYNAMIC_ALLOC)
    1888            0 :     record_dynamic_alloc (decl);
    1889              : }
    1890              : 
    1891              : /* A helper function to be called via walk_tree.  Mark all labels under *TP
    1892              :    as being forced.  To be called for DECL_INITIAL of static variables.  */
    1893              : 
    1894              : static tree
    1895       871680 : force_labels_r (tree *tp, int *walk_subtrees, void *data ATTRIBUTE_UNUSED)
    1896              : {
    1897       871680 :   if (TYPE_P (*tp))
    1898            0 :     *walk_subtrees = 0;
    1899       871680 :   if (TREE_CODE (*tp) == LABEL_DECL)
    1900              :     {
    1901         1026 :       FORCED_LABEL (*tp) = 1;
    1902         1026 :       cfun->has_forced_label_in_static = 1;
    1903              :     }
    1904              : 
    1905       871680 :   return NULL_TREE;
    1906              : }
    1907              : 
    1908              : /* Generate an initialization to automatic variable DECL based on INIT_TYPE.
    1909              :    Build a call to internal const function DEFERRED_INIT:
    1910              :    1st argument: SIZE of the DECL;
    1911              :    2nd argument: INIT_TYPE;
    1912              :    3rd argument: NAME of the DECL;
    1913              : 
    1914              :    as LHS = DEFERRED_INIT (SIZE of the DECL, INIT_TYPE, NAME of the DECL).  */
    1915              : 
    1916              : static void
    1917       161928 : gimple_add_init_for_auto_var (tree decl,
    1918              :                               enum auto_init_type init_type,
    1919              :                               gimple_seq *seq_p)
    1920              : {
    1921       161928 :   gcc_assert (auto_var_p (decl));
    1922       161928 :   gcc_assert (init_type > AUTO_INIT_UNINITIALIZED);
    1923              : 
    1924       161928 :   const location_t loc = DECL_SOURCE_LOCATION (decl);
    1925       161928 :   tree decl_size = TYPE_SIZE_UNIT (TREE_TYPE (decl));
    1926       161928 :   tree init_type_node = build_int_cst (integer_type_node, (int) init_type);
    1927       161928 :   tree decl_name;
    1928              : 
    1929       161928 :   if (DECL_NAME (decl))
    1930        92364 :     decl_name = build_string_literal (DECL_NAME (decl));
    1931              :   else
    1932              :     {
    1933        69564 :       char decl_name_anonymous[3 + (HOST_BITS_PER_INT + 2) / 3];
    1934        69564 :       sprintf (decl_name_anonymous, "D.%u", DECL_UID (decl));
    1935        69564 :       decl_name = build_string_literal (decl_name_anonymous);
    1936              :     }
    1937              : 
    1938       161928 :   tree call = build_call_expr_internal_loc (loc, IFN_DEFERRED_INIT,
    1939       161928 :                                             TREE_TYPE (decl), 3,
    1940              :                                             decl_size, init_type_node,
    1941              :                                             decl_name);
    1942              : 
    1943       161928 :   gimplify_assign (decl, call, seq_p);
    1944       161928 : }
    1945              : 
    1946              : /* Generate padding initialization for automatic variable DECL.
    1947              :    C guarantees that brace-init with fewer initializers than members
    1948              :    aggregate will initialize the rest of the aggregate as-if it were
    1949              :    static initialization.  In turn static initialization guarantees
    1950              :    that padding is initialized to zero. So, we always initialize paddings
    1951              :    to zeroes regardless INIT_TYPE.
    1952              :    To do the padding initialization, we insert a call to
    1953              :    __builtin_clear_padding (&decl, 0, for_auto_init = true).
    1954              :    Note, we add an additional dummy argument for __builtin_clear_padding,
    1955              :    'for_auto_init' to distinguish whether this call is for automatic
    1956              :    variable initialization or not.
    1957              :    */
    1958              : static void
    1959          116 : gimple_add_padding_init_for_auto_var (tree decl, bool is_vla,
    1960              :                                       gimple_seq *seq_p)
    1961              : {
    1962          116 :   tree addr_of_decl = NULL_TREE;
    1963          116 :   tree fn = builtin_decl_explicit (BUILT_IN_CLEAR_PADDING);
    1964              : 
    1965          116 :   if (is_vla)
    1966              :     {
    1967              :       /* The temporary address variable for this vla should be
    1968              :          created in gimplify_vla_decl.  */
    1969            0 :       gcc_assert (DECL_HAS_VALUE_EXPR_P (decl));
    1970            0 :       gcc_assert (INDIRECT_REF_P (DECL_VALUE_EXPR (decl)));
    1971            0 :       addr_of_decl = TREE_OPERAND (DECL_VALUE_EXPR (decl), 0);
    1972              :     }
    1973              :   else
    1974              :     {
    1975          116 :       mark_addressable (decl);
    1976          116 :       addr_of_decl = build_fold_addr_expr (decl);
    1977              :     }
    1978              : 
    1979          116 :   gimple *call = gimple_build_call (fn, 2, addr_of_decl,
    1980          116 :                                     build_one_cst (TREE_TYPE (addr_of_decl)));
    1981          116 :   gimplify_seq_add_stmt (seq_p, call);
    1982          116 : }
    1983              : 
    1984              : /* Return true if the DECL need to be automatically initialized by the
    1985              :    compiler.  */
    1986              : static bool
    1987      4362821 : var_needs_auto_init_p (tree decl)
    1988              : {
    1989      4362821 :   if (auto_var_p (decl)
    1990      4263883 :       && (TREE_CODE (decl) != VAR_DECL || !DECL_HARD_REGISTER (decl))
    1991      4263175 :       && flag_auto_var_init > AUTO_INIT_UNINITIALIZED
    1992       363999 :       && !lookup_attribute ("uninitialized", DECL_ATTRIBUTES (decl))
    1993       363991 :       && !lookup_attribute ("indeterminate", DECL_ATTRIBUTES (decl))
    1994       363982 :       && !OPAQUE_TYPE_P (TREE_TYPE (decl))
    1995      4726733 :       && !is_empty_type (TREE_TYPE (decl)))
    1996       336174 :     return true;
    1997              :   return false;
    1998              : }
    1999              : 
    2000              : /* Gimplify a DECL_EXPR node *STMT_P by making any necessary allocation
    2001              :    and initialization explicit.  */
    2002              : 
    2003              : static enum gimplify_status
    2004      6653752 : gimplify_decl_expr (tree *stmt_p, gimple_seq *seq_p)
    2005              : {
    2006      6653752 :   tree stmt = *stmt_p;
    2007      6653752 :   tree decl = DECL_EXPR_DECL (stmt);
    2008              : 
    2009      6653752 :   *stmt_p = NULL_TREE;
    2010              : 
    2011      6653752 :   if (TREE_TYPE (decl) == error_mark_node)
    2012              :     return GS_ERROR;
    2013              : 
    2014      6653665 :   if ((TREE_CODE (decl) == TYPE_DECL
    2015      6609274 :        || VAR_P (decl))
    2016     13259799 :       && !TYPE_SIZES_GIMPLIFIED (TREE_TYPE (decl)))
    2017              :     {
    2018       717863 :       gimplify_type_sizes (TREE_TYPE (decl), seq_p);
    2019       717863 :       if (TREE_CODE (TREE_TYPE (decl)) == REFERENCE_TYPE)
    2020        23547 :         gimplify_type_sizes (TREE_TYPE (TREE_TYPE (decl)), seq_p);
    2021              :     }
    2022              : 
    2023              :   /* ??? DECL_ORIGINAL_TYPE is streamed for LTO so it needs to be gimplified
    2024              :      in case its size expressions contain problematic nodes like CALL_EXPR.  */
    2025      6653665 :   if (TREE_CODE (decl) == TYPE_DECL
    2026        44391 :       && DECL_ORIGINAL_TYPE (decl)
    2027      6658986 :       && !TYPE_SIZES_GIMPLIFIED (DECL_ORIGINAL_TYPE (decl)))
    2028              :     {
    2029            0 :       gimplify_type_sizes (DECL_ORIGINAL_TYPE (decl), seq_p);
    2030            0 :       if (TREE_CODE (DECL_ORIGINAL_TYPE (decl)) == REFERENCE_TYPE)
    2031            0 :         gimplify_type_sizes (TREE_TYPE (DECL_ORIGINAL_TYPE (decl)), seq_p);
    2032              :     }
    2033              : 
    2034      6653665 :   if (VAR_P (decl) && !DECL_EXTERNAL (decl))
    2035              :     {
    2036      6606134 :       tree init = DECL_INITIAL (decl);
    2037      6606134 :       bool is_vla = false;
    2038              :       /* Check whether a decl has FE created VALUE_EXPR here BEFORE
    2039              :          gimplify_vla_decl creates VALUE_EXPR for a vla decl.
    2040              :          If the decl has VALUE_EXPR that was created by FE (usually
    2041              :          C++FE), it's a proxy variable, and FE already initialized
    2042              :          the VALUE_EXPR of it, we should not initialize it anymore.  */
    2043      6606134 :       bool decl_had_value_expr_p = DECL_HAS_VALUE_EXPR_P (decl);
    2044              : 
    2045      6606134 :       poly_uint64 size;
    2046      6606134 :       if (!poly_int_tree_p (DECL_SIZE_UNIT (decl), &size)
    2047      6606134 :           || (!TREE_STATIC (decl)
    2048      6478389 :               && flag_stack_check == GENERIC_STACK_CHECK
    2049          234 :               && maybe_gt (size,
    2050              :                            (unsigned HOST_WIDE_INT) STACK_CHECK_MAX_VAR_SIZE)))
    2051              :         {
    2052         8926 :           gimplify_vla_decl (decl, seq_p);
    2053         8926 :           is_vla = true;
    2054              :         }
    2055              : 
    2056      6606134 :       if (asan_poisoned_variables
    2057         5241 :           && !is_vla
    2058         5055 :           && TREE_ADDRESSABLE (decl)
    2059         2457 :           && !TREE_STATIC (decl)
    2060         2247 :           && !DECL_HAS_VALUE_EXPR_P (decl)
    2061         2170 :           && DECL_ALIGN (decl) <= MAX_SUPPORTED_STACK_ALIGNMENT
    2062         2170 :           && dbg_cnt (asan_use_after_scope)
    2063         2170 :           && !gimplify_omp_ctxp
    2064              :           /* GNAT introduces temporaries to hold return values of calls in
    2065              :              initializers of variables defined in other units, so the
    2066              :              declaration of the variable is discarded completely.  We do not
    2067              :              want to issue poison calls for such dropped variables.  */
    2068      6608296 :           && (DECL_SEEN_IN_BIND_EXPR_P (decl)
    2069            0 :               || (DECL_ARTIFICIAL (decl) && DECL_NAME (decl) == NULL_TREE)))
    2070              :         {
    2071         2162 :           asan_poisoned_variables->add (decl);
    2072         2162 :           asan_poison_variable (decl, false, seq_p);
    2073         2162 :           if (!DECL_ARTIFICIAL (decl) && gimplify_ctxp->live_switch_vars)
    2074           55 :             gimplify_ctxp->live_switch_vars->add (decl);
    2075              :         }
    2076              : 
    2077              :       /* Some front ends do not explicitly declare all anonymous
    2078              :          artificial variables.  We compensate here by declaring the
    2079              :          variables, though it would be better if the front ends would
    2080              :          explicitly declare them.  */
    2081      6606134 :       if (!DECL_SEEN_IN_BIND_EXPR_P (decl)
    2082      6606134 :           && DECL_ARTIFICIAL (decl) && DECL_NAME (decl) == NULL_TREE)
    2083        18346 :         gimple_add_tmp_var (decl);
    2084              : 
    2085      6606134 :       if (init && init != error_mark_node)
    2086              :         {
    2087      3754065 :           if (!TREE_STATIC (decl))
    2088              :             {
    2089      3723468 :               DECL_INITIAL (decl) = NULL_TREE;
    2090      3723468 :               init = build2 (INIT_EXPR, void_type_node, decl, init);
    2091      3723468 :               gimplify_and_add (init, seq_p);
    2092      3723468 :               ggc_free (init);
    2093              :               /* Clear TREE_READONLY if we really have an initialization.  */
    2094      3723468 :               if (!DECL_INITIAL (decl)
    2095      3723468 :                   && !omp_privatize_by_reference (decl))
    2096      3699182 :                 TREE_READONLY (decl) = 0;
    2097              :             }
    2098              :           else
    2099              :             /* We must still examine initializers for static variables
    2100              :                as they may contain a label address.  */
    2101        30597 :             walk_tree (&init, force_labels_r, NULL, NULL);
    2102              :         }
    2103              :       /* When there is no explicit initializer, if the user requested,
    2104              :          We should insert an artificial initializer for this automatic
    2105              :          variable.  */
    2106      2852069 :       else if (var_needs_auto_init_p (decl)
    2107      2852069 :                && !decl_had_value_expr_p)
    2108              :         {
    2109       101878 :           gimple_add_init_for_auto_var (decl,
    2110              :                                         flag_auto_var_init,
    2111              :                                         seq_p);
    2112              :           /* The expanding of a call to the above .DEFERRED_INIT will apply
    2113              :              block initialization to the whole space covered by this variable.
    2114              :              As a result, all the paddings will be initialized to zeroes
    2115              :              for zero initialization and 0xFE byte-repeatable patterns for
    2116              :              pattern initialization.
    2117              :              In order to make the paddings as zeroes for pattern init, We
    2118              :              should add a call to __builtin_clear_padding to clear the
    2119              :              paddings to zero in compatible with CLANG.
    2120              :              We cannot insert this call if the variable is a gimple register
    2121              :              since __builtin_clear_padding will take the address of the
    2122              :              variable.  As a result, if a long double/_Complex long double
    2123              :              variable will spilled into stack later, its padding is 0XFE.  */
    2124       101878 :           if ((flag_auto_var_init & ~AUTO_INIT_CXX26) == AUTO_INIT_PATTERN
    2125          186 :               && !is_gimple_reg (decl)
    2126       101976 :               && clear_padding_type_may_have_padding_p (TREE_TYPE (decl)))
    2127           75 :             gimple_add_padding_init_for_auto_var (decl, is_vla, seq_p);
    2128              :         }
    2129              :     }
    2130              : 
    2131              :   return GS_ALL_DONE;
    2132              : }
    2133              : 
    2134              : /* Gimplify a LOOP_EXPR.  Normally this just involves gimplifying the body
    2135              :    and replacing the LOOP_EXPR with goto, but if the loop contains an
    2136              :    EXIT_EXPR, we need to append a label for it to jump to.  */
    2137              : 
    2138              : static enum gimplify_status
    2139       210185 : gimplify_loop_expr (tree *expr_p, gimple_seq *pre_p)
    2140              : {
    2141       210185 :   tree saved_label = gimplify_ctxp->exit_label;
    2142       210185 :   tree start_label = create_artificial_label (UNKNOWN_LOCATION);
    2143              : 
    2144       210185 :   gimplify_seq_add_stmt (pre_p, gimple_build_label (start_label));
    2145              : 
    2146       210185 :   gimplify_ctxp->exit_label = NULL_TREE;
    2147              : 
    2148       210185 :   gimplify_and_add (LOOP_EXPR_BODY (*expr_p), pre_p);
    2149              : 
    2150       210185 :   gimplify_seq_add_stmt (pre_p, gimple_build_goto (start_label));
    2151              : 
    2152       210185 :   if (gimplify_ctxp->exit_label)
    2153         6810 :     gimplify_seq_add_stmt (pre_p,
    2154         3405 :                            gimple_build_label (gimplify_ctxp->exit_label));
    2155              : 
    2156       210185 :   gimplify_ctxp->exit_label = saved_label;
    2157              : 
    2158       210185 :   *expr_p = NULL;
    2159       210185 :   return GS_ALL_DONE;
    2160              : }
    2161              : 
    2162              : /* Gimplify a statement list onto a sequence.  These may be created either
    2163              :    by an enlightened front-end, or by shortcut_cond_expr.  */
    2164              : 
    2165              : static enum gimplify_status
    2166      8689877 : gimplify_statement_list (tree *expr_p, gimple_seq *pre_p)
    2167              : {
    2168      8689877 :   tree temp = voidify_wrapper_expr (*expr_p, NULL);
    2169              : 
    2170      8689877 :   tree_stmt_iterator i = tsi_start (*expr_p);
    2171              : 
    2172     56130874 :   while (!tsi_end_p (i))
    2173              :     {
    2174     38751120 :       gimplify_stmt (tsi_stmt_ptr (i), pre_p);
    2175     38751120 :       tsi_delink (&i);
    2176              :     }
    2177              : 
    2178      8689877 :   if (temp)
    2179              :     {
    2180        14836 :       *expr_p = temp;
    2181        14836 :       return GS_OK;
    2182              :     }
    2183              : 
    2184              :   return GS_ALL_DONE;
    2185              : }
    2186              : 
    2187              : 
    2188              : /* Emit warning for the unreachable statement STMT if needed.
    2189              :    Return the gimple itself when the warning is emitted, otherwise
    2190              :    return NULL.  */
    2191              : static gimple *
    2192          180 : emit_warn_switch_unreachable (gimple *stmt)
    2193              : {
    2194          180 :   if (gimple_code (stmt) == GIMPLE_GOTO
    2195           50 :       && TREE_CODE (gimple_goto_dest (stmt)) == LABEL_DECL
    2196          230 :       && DECL_ARTIFICIAL (gimple_goto_dest (stmt)))
    2197              :   /* Don't warn for compiler-generated gotos.  These occur
    2198              :      in Duff's devices, for example.  */
    2199              :     return NULL;
    2200              :   else
    2201          130 :     warning_at (gimple_location (stmt), OPT_Wswitch_unreachable,
    2202              :                 "statement will never be executed");
    2203          130 :   return stmt;
    2204              : }
    2205              : 
    2206              : /* Callback for walk_gimple_seq.  */
    2207              : 
    2208              : static tree
    2209        50544 : warn_switch_unreachable_and_auto_init_r (gimple_stmt_iterator *gsi_p,
    2210              :                                          bool *handled_ops_p,
    2211              :                                          struct walk_stmt_info *wi)
    2212              : {
    2213        50544 :   gimple *stmt = gsi_stmt (*gsi_p);
    2214        50544 :   bool unreachable_issued = wi->info != NULL;
    2215              : 
    2216        50544 :   *handled_ops_p = true;
    2217        50544 :   switch (gimple_code (stmt))
    2218              :     {
    2219          445 :     case GIMPLE_TRY:
    2220              :       /* A compiler-generated cleanup or a user-written try block.
    2221              :          If it's empty, don't dive into it--that would result in
    2222              :          worse location info.  */
    2223          445 :       if (gimple_try_eval (stmt) == NULL)
    2224              :         {
    2225            3 :           if (warn_switch_unreachable && !unreachable_issued)
    2226            3 :             wi->info = emit_warn_switch_unreachable (stmt);
    2227              : 
    2228              :           /* Stop when auto var init warning is not on.  */
    2229            3 :           if (!warn_trivial_auto_var_init)
    2230            3 :             return integer_zero_node;
    2231              :         }
    2232              :       /* Fall through.  */
    2233         2677 :     case GIMPLE_BIND:
    2234         2677 :     case GIMPLE_CATCH:
    2235         2677 :     case GIMPLE_EH_FILTER:
    2236         2677 :     case GIMPLE_TRANSACTION:
    2237              :       /* Walk the sub-statements.  */
    2238         2677 :       *handled_ops_p = false;
    2239         2677 :       break;
    2240              : 
    2241              :     case GIMPLE_DEBUG:
    2242              :       /* Ignore these.  We may generate them before declarations that
    2243              :          are never executed.  If there's something to warn about,
    2244              :          there will be non-debug stmts too, and we'll catch those.  */
    2245              :       break;
    2246              : 
    2247           60 :     case GIMPLE_ASSIGN:
    2248              :       /* See comment below in the GIMPLE_CALL case.  */
    2249           60 :       if (flag_auto_var_init > AUTO_INIT_UNINITIALIZED
    2250           20 :           && gimple_assign_single_p (stmt)
    2251           79 :           && TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME)
    2252              :         {
    2253            6 :           gimple *g = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmt));
    2254            6 :           if (gimple_call_internal_p (g, IFN_DEFERRED_INIT))
    2255              :             break;
    2256              :         }
    2257           54 :       goto do_default;
    2258              : 
    2259        46683 :     case GIMPLE_LABEL:
    2260              :       /* Stop till the first Label.  */
    2261        46683 :       return integer_zero_node;
    2262           61 :     case GIMPLE_CALL:
    2263           61 :       if (gimple_call_internal_p (stmt, IFN_ASAN_MARK))
    2264              :         {
    2265            8 :           *handled_ops_p = false;
    2266            8 :           break;
    2267              :         }
    2268              :       /* Don't warn for compiler-generated initializations for
    2269              :          -ftrivial-auto-var-init for -Wswitch-unreachable.  Though
    2270              :          do warn for -Wtrivial-auto-var-init.
    2271              :          There are 3 cases:
    2272              :          case 1: a call to .DEFERRED_INIT;
    2273              :          case 2: a call to __builtin_clear_padding with the 2nd argument is
    2274              :                  present and non-zero;
    2275              :          case 3: a gimple assign store right after the call to .DEFERRED_INIT
    2276              :                  that has the LHS of .DEFERRED_INIT as the RHS as following:
    2277              :                   _1 = .DEFERRED_INIT (4, 2, &"i1"[0]);
    2278              :                   i1 = _1.
    2279              :          case 3 is handled above in the GIMPLE_ASSIGN case.  */
    2280           53 :       if (flag_auto_var_init > AUTO_INIT_UNINITIALIZED
    2281           53 :           && gimple_call_internal_p (stmt, IFN_DEFERRED_INIT))
    2282              :         {
    2283           33 :           if (warn_trivial_auto_var_init)
    2284              :             {
    2285              :               /* Get the variable name from the 3rd argument of call.  */
    2286           12 :               tree var_name = gimple_call_arg (stmt, 2);
    2287           12 :               var_name = TREE_OPERAND (TREE_OPERAND (var_name, 0), 0);
    2288           12 :               const char *var_name_str = TREE_STRING_POINTER (var_name);
    2289              : 
    2290           12 :               warning_at (gimple_location (stmt), OPT_Wtrivial_auto_var_init,
    2291              :                           "%qs cannot be initialized with "
    2292              :                           "%<-ftrivial-auto-var_init%>", var_name_str);
    2293              :             }
    2294              :           break;
    2295              :        }
    2296           20 :       if (flag_auto_var_init > AUTO_INIT_UNINITIALIZED
    2297            7 :           && gimple_call_builtin_p (stmt, BUILT_IN_CLEAR_PADDING)
    2298           23 :           && (bool) TREE_INT_CST_LOW (gimple_call_arg (stmt, 1)))
    2299              :         break;
    2300              :       /* Fall through.  */
    2301          177 :     default:
    2302          177 :     do_default:
    2303              :       /* check the first "real" statement (not a decl/lexical scope/...), issue
    2304              :          warning if needed.  */
    2305          177 :       if (warn_switch_unreachable && !unreachable_issued)
    2306          177 :         wi->info = emit_warn_switch_unreachable (stmt);
    2307              :       /* Stop when auto var init warning is not on.  */
    2308          177 :       if (!warn_trivial_auto_var_init)
    2309          177 :         return integer_zero_node;
    2310              :       break;
    2311              :     }
    2312              :   return NULL_TREE;
    2313              : }
    2314              : 
    2315              : 
    2316              : /* Possibly warn about unreachable statements between switch's controlling
    2317              :    expression and the first case.  Also warn about -ftrivial-auto-var-init
    2318              :    cannot initialize the auto variable under such situation.
    2319              :    SEQ is the body of a switch expression.  */
    2320              : 
    2321              : static void
    2322        50807 : maybe_warn_switch_unreachable_and_auto_init (gimple_seq seq)
    2323              : {
    2324           64 :   if ((!warn_switch_unreachable && !warn_trivial_auto_var_init)
    2325              :       /* This warning doesn't play well with Fortran when optimizations
    2326              :          are on.  */
    2327        50743 :       || lang_GNU_Fortran ()
    2328        97844 :       || seq == NULL)
    2329         3941 :     return;
    2330              : 
    2331        46866 :   struct walk_stmt_info wi;
    2332              : 
    2333        46866 :   memset (&wi, 0, sizeof (wi));
    2334        46866 :   walk_gimple_seq (seq, warn_switch_unreachable_and_auto_init_r, NULL, &wi);
    2335              : }
    2336              : 
    2337              : 
    2338              : /* A label entry that pairs label and a location.  */
    2339              : struct label_entry
    2340              : {
    2341              :   tree label;
    2342              :   location_t loc;
    2343              : };
    2344              : 
    2345              : /* Find LABEL in vector of label entries VEC.  */
    2346              : 
    2347              : static struct label_entry *
    2348        12352 : find_label_entry (const auto_vec<struct label_entry> *vec, tree label)
    2349              : {
    2350        12352 :   unsigned int i;
    2351        12352 :   struct label_entry *l;
    2352              : 
    2353        23634 :   FOR_EACH_VEC_ELT (*vec, i, l)
    2354        20270 :     if (l->label == label)
    2355              :       return l;
    2356              :   return NULL;
    2357              : }
    2358              : 
    2359              : /* Return true if LABEL, a LABEL_DECL, represents a case label
    2360              :    in a vector of labels CASES.  */
    2361              : 
    2362              : static bool
    2363        16478 : case_label_p (const vec<tree> *cases, tree label)
    2364              : {
    2365        16478 :   unsigned int i;
    2366        16478 :   tree l;
    2367              : 
    2368       165417 :   FOR_EACH_VEC_ELT (*cases, i, l)
    2369       165083 :     if (CASE_LABEL (l) == label)
    2370              :       return true;
    2371              :   return false;
    2372              : }
    2373              : 
    2374              : /* Find the last nondebug statement in a scope STMT.  */
    2375              : 
    2376              : static gimple *
    2377         2257 : last_stmt_in_scope (gimple *stmt)
    2378              : {
    2379         4196 :   if (!stmt)
    2380              :     return NULL;
    2381              : 
    2382         6834 :   auto last_stmt_in_seq = [] (gimple_seq s) 
    2383              :     {
    2384         2649 :       gimple_seq_node n;
    2385         5289 :       for (n = gimple_seq_last (s);
    2386         2654 :            n && (is_gimple_debug (n)
    2387         2641 :                  || (flag_auto_var_init > AUTO_INIT_UNINITIALIZED
    2388          109 :                      && gimple_call_internal_p (n, IFN_DEFERRED_INIT)));
    2389            5 :         n = n->prev)
    2390            7 :       if (n == s)
    2391              :         return (gimple *) NULL;
    2392              :       return (gimple *) n;
    2393              :     };
    2394              : 
    2395         4185 :   switch (gimple_code (stmt))
    2396              :     {
    2397         1391 :     case GIMPLE_BIND:
    2398         1391 :       {
    2399         1391 :         gbind *bind = as_a <gbind *> (stmt);
    2400         1391 :         stmt = last_stmt_in_seq (gimple_bind_body (bind));
    2401         1391 :         return last_stmt_in_scope (stmt);
    2402              :       }
    2403              : 
    2404          710 :     case GIMPLE_TRY:
    2405          710 :       {
    2406          710 :         gtry *try_stmt = as_a <gtry *> (stmt);
    2407          710 :         stmt = last_stmt_in_seq (gimple_try_eval (try_stmt));
    2408          710 :         gimple *last_eval = last_stmt_in_scope (stmt);
    2409          710 :         if (gimple_stmt_may_fallthru (last_eval)
    2410          567 :             && (last_eval == NULL
    2411          564 :                 || !gimple_call_internal_p (last_eval, IFN_FALLTHROUGH))
    2412         1270 :             && gimple_try_kind (try_stmt) == GIMPLE_TRY_FINALLY)
    2413              :           {
    2414          548 :             stmt = last_stmt_in_seq (gimple_try_cleanup (try_stmt));
    2415          548 :             return last_stmt_in_scope (stmt);
    2416              :           }
    2417              :         else
    2418              :           return last_eval;
    2419              :       }
    2420              : 
    2421            0 :     case GIMPLE_DEBUG:
    2422            0 :       gcc_unreachable ();
    2423              : 
    2424              :     default:
    2425              :       return stmt;
    2426              :     }
    2427              : }
    2428              : 
    2429              : /* Collect labels that may fall through into LABELS and return the statement
    2430              :    preceding another case label, or a user-defined label.  Store a location
    2431              :    useful to give warnings at *PREVLOC (usually the location of the returned
    2432              :    statement or of its surrounding scope).  */
    2433              : 
    2434              : static gimple *
    2435        21093 : collect_fallthrough_labels (gimple_stmt_iterator *gsi_p,
    2436              :                             auto_vec <struct label_entry> *labels,
    2437              :                             location_t *prevloc)
    2438              : {
    2439        21093 :   gimple *prev = NULL;
    2440              : 
    2441        21093 :   *prevloc = UNKNOWN_LOCATION;
    2442       102348 :   do
    2443              :     {
    2444       102348 :       if (gimple_code (gsi_stmt (*gsi_p)) == GIMPLE_BIND)
    2445              :         {
    2446              :           /* Recognize the special GIMPLE_BIND added by gimplify_switch_expr,
    2447              :              which starts on a GIMPLE_SWITCH and ends with a break label.
    2448              :              Handle that as a single statement that can fall through.  */
    2449         1452 :           gbind *bind = as_a <gbind *> (gsi_stmt (*gsi_p));
    2450         1452 :           gimple *first = gimple_seq_first_stmt (gimple_bind_body (bind));
    2451         1452 :           gimple *last = gimple_seq_last_stmt (gimple_bind_body (bind));
    2452         1446 :           if (last
    2453         1446 :               && gimple_code (first) == GIMPLE_SWITCH
    2454         1548 :               && gimple_code (last) == GIMPLE_LABEL)
    2455              :             {
    2456          102 :               tree label = gimple_label_label (as_a <glabel *> (last));
    2457          102 :               if (SWITCH_BREAK_LABEL_P (label))
    2458              :                 {
    2459          102 :                   prev = bind;
    2460          102 :                   gsi_next (gsi_p);
    2461          102 :                   continue;
    2462              :                 }
    2463              :             }
    2464              :         }
    2465       102246 :       if (gimple_code (gsi_stmt (*gsi_p)) == GIMPLE_BIND
    2466       102246 :           || gimple_code (gsi_stmt (*gsi_p)) == GIMPLE_TRY)
    2467              :         {
    2468              :           /* Nested scope.  Only look at the last statement of
    2469              :              the innermost scope.  */
    2470         1547 :           location_t bind_loc = gimple_location (gsi_stmt (*gsi_p));
    2471         1547 :           gimple *last = last_stmt_in_scope (gsi_stmt (*gsi_p));
    2472         1547 :           if (last)
    2473              :             {
    2474         1539 :               prev = last;
    2475              :               /* It might be a label without a location.  Use the
    2476              :                  location of the scope then.  */
    2477         1539 :               if (!gimple_has_location (prev))
    2478          671 :                 *prevloc = bind_loc;
    2479              :             }
    2480         1547 :           gsi_next (gsi_p);
    2481         1547 :           continue;
    2482         1547 :         }
    2483              : 
    2484              :       /* Ifs are tricky.  */
    2485       100699 :       if (gimple_code (gsi_stmt (*gsi_p)) == GIMPLE_COND)
    2486              :         {
    2487         7268 :           gcond *cond_stmt = as_a <gcond *> (gsi_stmt (*gsi_p));
    2488         7268 :           tree false_lab = gimple_cond_false_label (cond_stmt);
    2489         7268 :           location_t if_loc = gimple_location (cond_stmt);
    2490              : 
    2491              :           /* If we have e.g.
    2492              :                if (i > 1) goto <D.2259>; else goto D;
    2493              :              we can't do much with the else-branch.  */
    2494         7268 :           if (!DECL_ARTIFICIAL (false_lab))
    2495              :             break;
    2496              : 
    2497              :           /* Go on until the false label, then one step back.  */
    2498        55328 :           for (; !gsi_end_p (*gsi_p); gsi_next (gsi_p))
    2499              :             {
    2500        55328 :               gimple *stmt = gsi_stmt (*gsi_p);
    2501        55328 :               if (gimple_code (stmt) == GIMPLE_LABEL
    2502        55328 :                   && gimple_label_label (as_a <glabel *> (stmt)) == false_lab)
    2503              :                 break;
    2504              :             }
    2505              : 
    2506              :           /* Not found?  Oops.  */
    2507         7268 :           if (gsi_end_p (*gsi_p))
    2508              :             break;
    2509              : 
    2510              :           /* A dead label can't fall through.  */
    2511         7268 :           if (!UNUSED_LABEL_P (false_lab))
    2512              :             {
    2513         7221 :               struct label_entry l = { false_lab, if_loc };
    2514         7221 :               labels->safe_push (l);
    2515              :             }
    2516              : 
    2517              :           /* Go to the last statement of the then branch.  */
    2518         7268 :           gsi_prev (gsi_p);
    2519              : 
    2520              :           /* if (i != 0) goto <D.1759>; else goto <D.1760>;
    2521              :              <D.1759>:
    2522              :              <stmt>;
    2523              :              goto <D.1761>;
    2524              :              <D.1760>:
    2525              :            */
    2526         7268 :           if (gimple_code (gsi_stmt (*gsi_p)) == GIMPLE_GOTO
    2527         7268 :               && !gimple_has_location (gsi_stmt (*gsi_p)))
    2528              :             {
    2529              :               /* Look at the statement before, it might be
    2530              :                  attribute fallthrough, in which case don't warn.  */
    2531         1626 :               gsi_prev (gsi_p);
    2532         1626 :               bool fallthru_before_dest
    2533         1626 :                 = gimple_call_internal_p (gsi_stmt (*gsi_p), IFN_FALLTHROUGH);
    2534         1626 :               gsi_next (gsi_p);
    2535         1626 :               tree goto_dest = gimple_goto_dest (gsi_stmt (*gsi_p));
    2536         1626 :               if (!fallthru_before_dest)
    2537              :                 {
    2538         1569 :                   struct label_entry l = { goto_dest, if_loc };
    2539         1569 :                   labels->safe_push (l);
    2540              :                 }
    2541              :             }
    2542              :           /* This case is about
    2543              :               if (1 != 0) goto <D.2022>; else goto <D.2023>;
    2544              :               <D.2022>:
    2545              :               n = n + 1; // #1
    2546              :               <D.2023>:  // #2
    2547              :               <D.1988>:  // #3
    2548              :              where #2 is UNUSED_LABEL_P and we want to warn about #1 falling
    2549              :              through to #3.  So set PREV to #1.  */
    2550         5642 :           else if (UNUSED_LABEL_P (false_lab))
    2551           47 :             prev = gsi_stmt (*gsi_p);
    2552              : 
    2553              :           /* And move back.  */
    2554         7268 :           gsi_next (gsi_p);
    2555              :         }
    2556              : 
    2557       100699 :       tree lab;
    2558              :       /* Remember the last statement.  Skip labels that are of no interest
    2559              :          to us.  */
    2560       100699 :       if (gimple_code (gsi_stmt (*gsi_p)) == GIMPLE_LABEL)
    2561              :         {
    2562        12134 :           tree label = gimple_label_label (as_a <glabel *> (gsi_stmt (*gsi_p)));
    2563        12134 :           if (find_label_entry (labels, label))
    2564         8790 :             prev = gsi_stmt (*gsi_p);
    2565              :         }
    2566        88565 :       else if (gimple_call_internal_p (gsi_stmt (*gsi_p), IFN_ASAN_MARK))
    2567              :         ;
    2568        88565 :       else if (flag_auto_var_init > AUTO_INIT_UNINITIALIZED
    2569        88565 :                && gimple_call_internal_p (gsi_stmt (*gsi_p),
    2570              :                                           IFN_DEFERRED_INIT))
    2571              :         ;
    2572        88559 :       else if (gimple_code (gsi_stmt (*gsi_p)) == GIMPLE_PREDICT)
    2573              :         ;
    2574        87331 :       else if (flag_auto_var_init > AUTO_INIT_UNINITIALIZED
    2575         2382 :                && gimple_code (gsi_stmt (*gsi_p)) == GIMPLE_GOTO
    2576          348 :                && (lab = gimple_goto_dest (gsi_stmt (*gsi_p)))
    2577          348 :                && TREE_CODE (lab) == LABEL_DECL
    2578        87678 :                && VACUOUS_INIT_LABEL_P (lab))
    2579              :         ;
    2580        87331 :       else if (!is_gimple_debug (gsi_stmt (*gsi_p)))
    2581        67015 :         prev = gsi_stmt (*gsi_p);
    2582       100699 :       gsi_next (gsi_p);
    2583              :     }
    2584       102348 :   while (!gsi_end_p (*gsi_p)
    2585              :          /* Stop if we find a case or a user-defined label.  */
    2586       200609 :          && (gimple_code (gsi_stmt (*gsi_p)) != GIMPLE_LABEL
    2587        21872 :              || !gimple_has_location (gsi_stmt (*gsi_p))));
    2588              : 
    2589        21093 :   if (prev && gimple_has_location (prev))
    2590        20640 :     *prevloc = gimple_location (prev);
    2591        21093 :   return prev;
    2592              : }
    2593              : 
    2594              : /* Return true if the switch fallthrough warning should occur.  LABEL is
    2595              :    the label statement that we're falling through to.  */
    2596              : 
    2597              : static bool
    2598        16999 : should_warn_for_implicit_fallthrough (gimple_stmt_iterator *gsi_p, tree label)
    2599              : {
    2600        16999 :   gimple_stmt_iterator gsi = *gsi_p;
    2601              : 
    2602              :   /* Don't warn if the label is marked with a "falls through" comment.  */
    2603        16999 :   if (FALLTHROUGH_LABEL_P (label))
    2604              :     return false;
    2605              : 
    2606              :   /* Don't warn for non-case labels followed by a statement:
    2607              :        case 0:
    2608              :          foo ();
    2609              :        label:
    2610              :          bar ();
    2611              :      as these are likely intentional.  */
    2612        16219 :   if (!case_label_p (&gimplify_ctxp->case_labels, label))
    2613              :     {
    2614              :       tree l;
    2615          334 :       while (!gsi_end_p (gsi)
    2616          334 :              && ((gimple_code (gsi_stmt (gsi)) == GIMPLE_LABEL
    2617          259 :                   && (l
    2618          259 :                       = gimple_label_label (as_a <glabel *> (gsi_stmt (gsi))))
    2619          259 :                   && !case_label_p (&gimplify_ctxp->case_labels, l))
    2620          153 :                  || (flag_auto_var_init > AUTO_INIT_UNINITIALIZED
    2621           31 :                      && gimple_call_internal_p (gsi_stmt (gsi),
    2622              :                                                 IFN_DEFERRED_INIT))))
    2623          173 :         gsi_next_nondebug (&gsi);
    2624          161 :       if (gsi_end_p (gsi) || gimple_code (gsi_stmt (gsi)) != GIMPLE_LABEL)
    2625              :         return false;
    2626              :     }
    2627              : 
    2628              :   /* Don't warn for terminated branches, i.e. when the subsequent case labels
    2629              :      immediately breaks.  */
    2630        16144 :   gsi = *gsi_p;
    2631              : 
    2632              :   /* Skip all immediately following labels.  */
    2633        35963 :   while (!gsi_end_p (gsi)
    2634        35963 :          && (gimple_code (gsi_stmt (gsi)) == GIMPLE_LABEL
    2635        16293 :              || gimple_code (gsi_stmt (gsi)) == GIMPLE_PREDICT
    2636        16084 :              || (flag_auto_var_init > AUTO_INIT_UNINITIALIZED
    2637          775 :                  && gimple_call_internal_p (gsi_stmt (gsi),
    2638              :                                             IFN_DEFERRED_INIT))))
    2639        19819 :     gsi_next_nondebug (&gsi);
    2640              : 
    2641              :   /* { ... something; default:; } */
    2642        16144 :   if (gsi_end_p (gsi)
    2643              :       /* { ... something; default: break; } or
    2644              :          { ... something; default: goto L; } */
    2645        16078 :       || gimple_code (gsi_stmt (gsi)) == GIMPLE_GOTO
    2646              :       /* { ... something; default: return; } */
    2647        31123 :       || gimple_code (gsi_stmt (gsi)) == GIMPLE_RETURN)
    2648         1192 :     return false;
    2649              : 
    2650              :   return true;
    2651              : }
    2652              : 
    2653              : /* Callback for walk_gimple_seq.  */
    2654              : 
    2655              : static tree
    2656        22101 : warn_implicit_fallthrough_r (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
    2657              :                              struct walk_stmt_info *)
    2658              : {
    2659        22101 :   gimple *stmt = gsi_stmt (*gsi_p);
    2660              : 
    2661        22101 :   *handled_ops_p = true;
    2662        22101 :   switch (gimple_code (stmt))
    2663              :     {
    2664          299 :     case GIMPLE_TRY:
    2665          299 :     case GIMPLE_BIND:
    2666          299 :     case GIMPLE_CATCH:
    2667          299 :     case GIMPLE_EH_FILTER:
    2668          299 :     case GIMPLE_TRANSACTION:
    2669              :       /* Walk the sub-statements.  */
    2670          299 :       *handled_ops_p = false;
    2671          299 :       break;
    2672              : 
    2673              :     /* Find a sequence of form:
    2674              : 
    2675              :        GIMPLE_LABEL
    2676              :        [...]
    2677              :        <may fallthru stmt>
    2678              :        GIMPLE_LABEL
    2679              : 
    2680              :        and possibly warn.  */
    2681              :     case GIMPLE_LABEL:
    2682              :       {
    2683              :         /* Found a label.  Skip all immediately following labels.  */
    2684        49612 :         while (!gsi_end_p (*gsi_p)
    2685        49612 :                && gimple_code (gsi_stmt (*gsi_p)) == GIMPLE_LABEL)
    2686        28285 :           gsi_next_nondebug (gsi_p);
    2687              : 
    2688              :         /* There might be no more statements.  */
    2689        21327 :         if (gsi_end_p (*gsi_p))
    2690         4321 :           return integer_zero_node;
    2691              : 
    2692              :         /* Vector of labels that fall through.  */
    2693        21093 :         auto_vec <struct label_entry> labels;
    2694        21093 :         location_t prevloc;
    2695        21093 :         gimple *prev = collect_fallthrough_labels (gsi_p, &labels, &prevloc);
    2696              : 
    2697              :         /* There might be no more statements.  */
    2698        21093 :         if (gsi_end_p (*gsi_p))
    2699         4087 :           return integer_zero_node;
    2700              : 
    2701        17006 :         gimple *next = gsi_stmt (*gsi_p);
    2702        17006 :         tree label;
    2703              :         /* If what follows is a label, then we may have a fallthrough.  */
    2704        17006 :         if (gimple_code (next) == GIMPLE_LABEL
    2705        17006 :             && gimple_has_location (next)
    2706        17006 :             && (label = gimple_label_label (as_a <glabel *> (next)))
    2707        34012 :             && prev != NULL)
    2708              :           {
    2709        16999 :             struct label_entry *l;
    2710        16999 :             bool warned_p = false;
    2711        16999 :             auto_diagnostic_group d;
    2712        16999 :             if (!should_warn_for_implicit_fallthrough (gsi_p, label))
    2713              :               /* Quiet.  */;
    2714        14952 :             else if (gimple_code (prev) == GIMPLE_LABEL
    2715          218 :                      && (label = gimple_label_label (as_a <glabel *> (prev)))
    2716        15170 :                      && (l = find_label_entry (&labels, label)))
    2717          198 :               warned_p = warning_at (l->loc, OPT_Wimplicit_fallthrough_,
    2718              :                                      "this statement may fall through");
    2719        14754 :             else if (!gimple_call_internal_p (prev, IFN_FALLTHROUGH)
    2720              :                      /* Try to be clever and don't warn when the statement
    2721              :                         can't actually fall through.  */
    2722        14174 :                      && gimple_stmt_may_fallthru (prev)
    2723        15055 :                      && prevloc != UNKNOWN_LOCATION)
    2724          301 :               warned_p = warning_at (prevloc,
    2725          301 :                                      OPT_Wimplicit_fallthrough_,
    2726              :                                      "this statement may fall through");
    2727          499 :             if (warned_p)
    2728          499 :               inform (gimple_location (next), "here");
    2729              : 
    2730              :             /* Mark this label as processed so as to prevent multiple
    2731              :                warnings in nested switches.  */
    2732        16999 :             FALLTHROUGH_LABEL_P (label) = true;
    2733              : 
    2734              :             /* So that next warn_implicit_fallthrough_r will start looking for
    2735              :                a new sequence starting with this label.  */
    2736        16999 :             gsi_prev (gsi_p);
    2737        16999 :           }
    2738         4087 :       }
    2739        17006 :       break;
    2740              :    default:
    2741              :       break;
    2742              :     }
    2743              :   return NULL_TREE;
    2744              : }
    2745              : 
    2746              : /* Warn when a switch case falls through.  */
    2747              : 
    2748              : static void
    2749        50807 : maybe_warn_implicit_fallthrough (gimple_seq seq)
    2750              : {
    2751        50807 :   if (!warn_implicit_fallthrough)
    2752        46627 :     return;
    2753              : 
    2754              :   /* This warning is meant for C/C++/ObjC/ObjC++ only.  */
    2755         4180 :   if (!(lang_GNU_C ()
    2756         1232 :         || lang_GNU_CXX ()
    2757            0 :         || lang_GNU_OBJC ()))
    2758              :     return;
    2759              : 
    2760         4180 :   struct walk_stmt_info wi;
    2761         4180 :   memset (&wi, 0, sizeof (wi));
    2762         4180 :   walk_gimple_seq (seq, warn_implicit_fallthrough_r, NULL, &wi);
    2763              : }
    2764              : 
    2765              : /* Callback for walk_gimple_seq.  */
    2766              : 
    2767              : static tree
    2768      3360457 : expand_FALLTHROUGH_r (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
    2769              :                       struct walk_stmt_info *wi)
    2770              : {
    2771      3360457 :   gimple *stmt = gsi_stmt (*gsi_p);
    2772              : 
    2773      3360457 :   *handled_ops_p = true;
    2774      3360457 :   switch (gimple_code (stmt))
    2775              :     {
    2776       148871 :     case GIMPLE_TRY:
    2777       148871 :     case GIMPLE_BIND:
    2778       148871 :     case GIMPLE_CATCH:
    2779       148871 :     case GIMPLE_EH_FILTER:
    2780       148871 :     case GIMPLE_TRANSACTION:
    2781              :       /* Walk the sub-statements.  */
    2782       148871 :       *handled_ops_p = false;
    2783       148871 :       break;
    2784       259130 :     case GIMPLE_CALL:
    2785       259130 :       static_cast<location_t *>(wi->info)[0] = UNKNOWN_LOCATION;
    2786       259130 :       if (gimple_call_internal_p (stmt, IFN_FALLTHROUGH))
    2787              :         {
    2788         3962 :           location_t loc = gimple_location (stmt);
    2789         3962 :           gsi_remove (gsi_p, true);
    2790         3962 :           wi->removed_stmt = true;
    2791              : 
    2792              :           /* nothrow flag is added by genericize_c_loop to mark fallthrough
    2793              :              statement at the end of some loop's body.  Those should be
    2794              :              always diagnosed, either because they indeed don't precede
    2795              :              a case label or default label, or because the next statement
    2796              :              is not within the same iteration statement.  */
    2797         3962 :           if ((stmt->subcode & GF_CALL_NOTHROW) != 0)
    2798              :             {
    2799           12 :               pedwarn (loc, 0, "attribute %<fallthrough%> not preceding "
    2800              :                                "a case label or default label");
    2801           12 :               break;
    2802              :             }
    2803              : 
    2804         3950 :           if (gsi_end_p (*gsi_p))
    2805              :             {
    2806           32 :               static_cast<location_t *>(wi->info)[0] = BUILTINS_LOCATION;
    2807           32 :               static_cast<location_t *>(wi->info)[1] = loc;
    2808           32 :               break;
    2809              :             }
    2810              : 
    2811         3918 :           bool found = false;
    2812              : 
    2813         3918 :           gimple_stmt_iterator gsi2 = *gsi_p;
    2814         3918 :           stmt = gsi_stmt (gsi2);
    2815         3918 :           tree lab;
    2816         3918 :           if (flag_auto_var_init > AUTO_INIT_UNINITIALIZED
    2817         1704 :               && gimple_code (stmt) == GIMPLE_GOTO
    2818           20 :               && (lab = gimple_goto_dest (stmt))
    2819           20 :               && TREE_CODE (lab) == LABEL_DECL
    2820         3938 :               && VACUOUS_INIT_LABEL_P (lab))
    2821              :             {
    2822              :               /* Handle for C++ artificial -ftrivial-auto-var-init=
    2823              :                  sequences.  Those look like:
    2824              :                  goto lab1;
    2825              :                  lab2:;
    2826              :                  v1 = .DEFERRED_INIT (...);
    2827              :                  v2 = .DEFERRED_INIT (...);
    2828              :                  lab3:;
    2829              :                  v3 = .DEFERRED_INIT (...);
    2830              :                  lab1:;
    2831              :                  In this case, a case/default label can be either in between
    2832              :                  the GIMPLE_GOTO and the corresponding GIMPLE_LABEL, if jumps
    2833              :                  from the switch condition to the case/default label cross
    2834              :                  vacuous initialization of some variables, or after the
    2835              :                  corresponding GIMPLE_LABEL, if those jumps don't cross
    2836              :                  any such initialization but there is an adjacent named label
    2837              :                  which crosses such initialization.  So, for the purpose of
    2838              :                  this function, just ignore the goto but until reaching the
    2839              :                  corresponding GIMPLE_LABEL allow also .DEFERRED_INIT
    2840              :                  calls.  */
    2841            5 :               gsi_next (&gsi2);
    2842              :             }
    2843         3913 :           else if (gimple_code (stmt) == GIMPLE_GOTO
    2844         3913 :                    && !gimple_has_location (stmt))
    2845              :             {
    2846              :               /* Go on until the artificial label.  */
    2847           57 :               tree goto_dest = gimple_goto_dest (stmt);
    2848          414 :               for (; !gsi_end_p (gsi2); gsi_next (&gsi2))
    2849              :                 {
    2850          357 :                   if (gimple_code (gsi_stmt (gsi2)) == GIMPLE_LABEL
    2851          357 :                       && gimple_label_label (as_a <glabel *> (gsi_stmt (gsi2)))
    2852              :                            == goto_dest)
    2853              :                     break;
    2854              :                 }
    2855              : 
    2856              :               /* Not found?  Stop.  */
    2857           57 :               if (gsi_end_p (gsi2))
    2858              :                 break;
    2859              : 
    2860              :               /* Look one past it.  */
    2861           57 :               gsi_next (&gsi2);
    2862              :             }
    2863              : 
    2864              :           /* We're looking for a case label or default label here.  */
    2865         4016 :           while (!gsi_end_p (gsi2))
    2866              :             {
    2867         4016 :               stmt = gsi_stmt (gsi2);
    2868         4016 :               if (gimple_code (stmt) == GIMPLE_LABEL)
    2869              :                 {
    2870         3973 :                   tree label = gimple_label_label (as_a <glabel *> (stmt));
    2871         3973 :                   if (gimple_has_location (stmt) && DECL_ARTIFICIAL (label))
    2872              :                     {
    2873              :                       found = true;
    2874              :                       break;
    2875              :                     }
    2876              :                 }
    2877           43 :               else if (gimple_call_internal_p (stmt, IFN_ASAN_MARK))
    2878              :                 ;
    2879           43 :               else if (flag_auto_var_init > AUTO_INIT_UNINITIALIZED
    2880           43 :                        && gimple_call_internal_p (stmt, IFN_DEFERRED_INIT))
    2881              :                 ;
    2882           43 :               else if (!is_gimple_debug (stmt))
    2883              :                 /* Anything else is not expected.  */
    2884              :                 break;
    2885           98 :               gsi_next (&gsi2);
    2886              :             }
    2887         3918 :           if (!found)
    2888           23 :             pedwarn (loc, 0, "attribute %<fallthrough%> not preceding "
    2889              :                      "a case label or default label");
    2890              :         }
    2891              :       break;
    2892      2952456 :     default:
    2893      2952456 :       static_cast<location_t *>(wi->info)[0] = UNKNOWN_LOCATION;
    2894      2952456 :       break;
    2895              :     }
    2896      3360457 :   return NULL_TREE;
    2897              : }
    2898              : 
    2899              : /* Expand all FALLTHROUGH () calls in SEQ.  */
    2900              : 
    2901              : static void
    2902        48976 : expand_FALLTHROUGH (gimple_seq *seq_p)
    2903              : {
    2904        48976 :   auto_urlify_attributes sentinel;
    2905              : 
    2906        48976 :   struct walk_stmt_info wi;
    2907        48976 :   location_t loc[2];
    2908        48976 :   memset (&wi, 0, sizeof (wi));
    2909        48976 :   loc[0] = UNKNOWN_LOCATION;
    2910        48976 :   loc[1] = UNKNOWN_LOCATION;
    2911        48976 :   wi.info = (void *) &loc[0];
    2912        48976 :   walk_gimple_seq_mod (seq_p, expand_FALLTHROUGH_r, NULL, &wi);
    2913        48976 :   if (loc[0] != UNKNOWN_LOCATION)
    2914              :     /* We've found [[fallthrough]]; at the end of a switch, which the C++
    2915              :        standard says is ill-formed; see [dcl.attr.fallthrough].  */
    2916            8 :     pedwarn (loc[1], 0, "attribute %<fallthrough%> not preceding "
    2917              :              "a case label or default label");
    2918        48976 : }
    2919              : 
    2920              : 
    2921              : /* Gimplify a SWITCH_EXPR, and collect the vector of labels it can
    2922              :    branch to.  */
    2923              : 
    2924              : static enum gimplify_status
    2925        50807 : gimplify_switch_expr (tree *expr_p, gimple_seq *pre_p)
    2926              : {
    2927        50807 :   tree switch_expr = *expr_p;
    2928        50807 :   gimple_seq switch_body_seq = NULL;
    2929        50807 :   enum gimplify_status ret;
    2930        50807 :   tree index_type = TREE_TYPE (switch_expr);
    2931        50807 :   if (index_type == NULL_TREE)
    2932        11452 :     index_type = TREE_TYPE (SWITCH_COND (switch_expr));
    2933              : 
    2934        50807 :   ret = gimplify_expr (&SWITCH_COND (switch_expr), pre_p, NULL, is_gimple_val,
    2935              :                        fb_rvalue);
    2936        50807 :   if (ret == GS_ERROR || ret == GS_UNHANDLED)
    2937              :     return ret;
    2938              : 
    2939        50807 :   if (SWITCH_BODY (switch_expr))
    2940              :     {
    2941        50807 :       vec<tree> labels;
    2942        50807 :       vec<tree> saved_labels;
    2943        50807 :       hash_set<tree> *saved_live_switch_vars = NULL;
    2944        50807 :       tree default_case = NULL_TREE;
    2945        50807 :       gswitch *switch_stmt;
    2946              : 
    2947              :       /* Save old labels, get new ones from body, then restore the old
    2948              :          labels.  Save all the things from the switch body to append after.  */
    2949        50807 :       saved_labels = gimplify_ctxp->case_labels;
    2950        50807 :       gimplify_ctxp->case_labels.create (8);
    2951              : 
    2952              :       /* Do not create live_switch_vars if SWITCH_BODY is not a BIND_EXPR.  */
    2953        50807 :       saved_live_switch_vars = gimplify_ctxp->live_switch_vars;
    2954        50807 :       tree_code body_type = TREE_CODE (SWITCH_BODY (switch_expr));
    2955        50807 :       if (body_type == BIND_EXPR || body_type == STATEMENT_LIST)
    2956        50523 :         gimplify_ctxp->live_switch_vars = new hash_set<tree> (4);
    2957              :       else
    2958          284 :         gimplify_ctxp->live_switch_vars = NULL;
    2959              : 
    2960        50807 :       bool old_in_switch_expr = gimplify_ctxp->in_switch_expr;
    2961        50807 :       gimplify_ctxp->in_switch_expr = true;
    2962              : 
    2963        50807 :       gimplify_stmt (&SWITCH_BODY (switch_expr), &switch_body_seq);
    2964              : 
    2965        50807 :       gimplify_ctxp->in_switch_expr = old_in_switch_expr;
    2966        50807 :       maybe_warn_switch_unreachable_and_auto_init (switch_body_seq);
    2967        50807 :       maybe_warn_implicit_fallthrough (switch_body_seq);
    2968              :       /* Only do this for the outermost GIMPLE_SWITCH.  */
    2969        50807 :       if (!gimplify_ctxp->in_switch_expr)
    2970        48976 :         expand_FALLTHROUGH (&switch_body_seq);
    2971              : 
    2972        50807 :       labels = gimplify_ctxp->case_labels;
    2973        50807 :       gimplify_ctxp->case_labels = saved_labels;
    2974              : 
    2975        50807 :       if (gimplify_ctxp->live_switch_vars)
    2976              :         {
    2977        50523 :           gcc_assert (gimplify_ctxp->live_switch_vars->is_empty ());
    2978        50523 :           delete gimplify_ctxp->live_switch_vars;
    2979              :         }
    2980        50807 :       gimplify_ctxp->live_switch_vars = saved_live_switch_vars;
    2981              : 
    2982        50807 :       preprocess_case_label_vec_for_gimple (labels, index_type,
    2983              :                                             &default_case);
    2984              : 
    2985        50807 :       bool add_bind = false;
    2986        50807 :       if (!default_case)
    2987              :         {
    2988        15068 :           glabel *new_default;
    2989              : 
    2990        15068 :           default_case
    2991        15068 :             = build_case_label (NULL_TREE, NULL_TREE,
    2992              :                                 create_artificial_label (UNKNOWN_LOCATION));
    2993        15068 :           if (old_in_switch_expr)
    2994              :             {
    2995         1085 :               SWITCH_BREAK_LABEL_P (CASE_LABEL (default_case)) = 1;
    2996         1085 :               add_bind = true;
    2997              :             }
    2998        15068 :           new_default = gimple_build_label (CASE_LABEL (default_case));
    2999        15068 :           gimplify_seq_add_stmt (&switch_body_seq, new_default);
    3000              :         }
    3001        35739 :       else if (old_in_switch_expr)
    3002              :         {
    3003          746 :           gimple *last = gimple_seq_last_stmt (switch_body_seq);
    3004          746 :           if (last && gimple_code (last) == GIMPLE_LABEL)
    3005              :             {
    3006          481 :               tree label = gimple_label_label (as_a <glabel *> (last));
    3007          481 :               if (SWITCH_BREAK_LABEL_P (label))
    3008        50807 :                 add_bind = true;
    3009              :             }
    3010              :         }
    3011              : 
    3012        50807 :       switch_stmt = gimple_build_switch (SWITCH_COND (switch_expr),
    3013              :                                          default_case, labels);
    3014        50807 :       gimple_set_location (switch_stmt, EXPR_LOCATION (switch_expr));
    3015              :       /* For the benefit of -Wimplicit-fallthrough, if switch_body_seq
    3016              :          ends with a GIMPLE_LABEL holding SWITCH_BREAK_LABEL_P LABEL_DECL,
    3017              :          wrap the GIMPLE_SWITCH up to that GIMPLE_LABEL into a GIMPLE_BIND,
    3018              :          so that we can easily find the start and end of the switch
    3019              :          statement.  */
    3020        50807 :       if (add_bind)
    3021              :         {
    3022         1565 :           gimple_seq bind_body = NULL;
    3023         1565 :           gimplify_seq_add_stmt (&bind_body, switch_stmt);
    3024         1565 :           gimple_seq_add_seq (&bind_body, switch_body_seq);
    3025         1565 :           gbind *bind = gimple_build_bind (NULL_TREE, bind_body, NULL_TREE);
    3026         1565 :           gimple_set_location (bind, EXPR_LOCATION (switch_expr));
    3027         1565 :           gimplify_seq_add_stmt (pre_p, bind);
    3028              :         }
    3029              :       else
    3030              :         {
    3031        49242 :           gimplify_seq_add_stmt (pre_p, switch_stmt);
    3032        49242 :           gimplify_seq_add_seq (pre_p, switch_body_seq);
    3033              :         }
    3034        50807 :       labels.release ();
    3035              :     }
    3036              :   else
    3037            0 :     gcc_unreachable ();
    3038              : 
    3039        50807 :   return GS_ALL_DONE;
    3040              : }
    3041              : 
    3042              : /* Gimplify the LABEL_EXPR pointed to by EXPR_P.  */
    3043              : 
    3044              : static enum gimplify_status
    3045      2628015 : gimplify_label_expr (tree *expr_p, gimple_seq *pre_p)
    3046              : {
    3047      2628015 :   gcc_assert (decl_function_context (LABEL_EXPR_LABEL (*expr_p))
    3048              :               == current_function_decl);
    3049              : 
    3050      2628015 :   tree label = LABEL_EXPR_LABEL (*expr_p);
    3051      2628015 :   glabel *label_stmt = gimple_build_label (label);
    3052      2628015 :   gimple_set_location (label_stmt, EXPR_LOCATION (*expr_p));
    3053      2628015 :   gimplify_seq_add_stmt (pre_p, label_stmt);
    3054              : 
    3055      2628015 :   if (lookup_attribute ("cold", DECL_ATTRIBUTES (label)))
    3056           21 :     gimple_seq_add_stmt (pre_p, gimple_build_predict (PRED_COLD_LABEL,
    3057              :                                                       NOT_TAKEN));
    3058      2627994 :   else if (lookup_attribute ("hot", DECL_ATTRIBUTES (label)))
    3059           11 :     gimple_seq_add_stmt (pre_p, gimple_build_predict (PRED_HOT_LABEL,
    3060              :                                                       TAKEN));
    3061              : 
    3062      2628015 :   return GS_ALL_DONE;
    3063              : }
    3064              : 
    3065              : /* Gimplify the CASE_LABEL_EXPR pointed to by EXPR_P.  */
    3066              : 
    3067              : static enum gimplify_status
    3068      1045941 : gimplify_case_label_expr (tree *expr_p, gimple_seq *pre_p)
    3069              : {
    3070      1045941 :   struct gimplify_ctx *ctxp;
    3071      1045941 :   glabel *label_stmt;
    3072              : 
    3073              :   /* Invalid programs can play Duff's Device type games with, for example,
    3074              :      #pragma omp parallel.  At least in the C front end, we don't
    3075              :      detect such invalid branches until after gimplification, in the
    3076              :      diagnose_omp_blocks pass.  */
    3077      1045951 :   for (ctxp = gimplify_ctxp; ; ctxp = ctxp->prev_context)
    3078      1045951 :     if (ctxp->case_labels.exists ())
    3079              :       break;
    3080              : 
    3081      1045941 :   tree label = CASE_LABEL (*expr_p);
    3082      1045941 :   label_stmt = gimple_build_label (label);
    3083      1045941 :   gimple_set_location (label_stmt, EXPR_LOCATION (*expr_p));
    3084      1045941 :   ctxp->case_labels.safe_push (*expr_p);
    3085      1045941 :   gimplify_seq_add_stmt (pre_p, label_stmt);
    3086              : 
    3087      1045941 :   if (lookup_attribute ("cold", DECL_ATTRIBUTES (label)))
    3088           16 :     gimple_seq_add_stmt (pre_p, gimple_build_predict (PRED_COLD_LABEL,
    3089              :                                                       NOT_TAKEN));
    3090      1045925 :   else if (lookup_attribute ("hot", DECL_ATTRIBUTES (label)))
    3091           20 :     gimple_seq_add_stmt (pre_p, gimple_build_predict (PRED_HOT_LABEL,
    3092              :                                                       TAKEN));
    3093              : 
    3094      1045941 :   return GS_ALL_DONE;
    3095              : }
    3096              : 
    3097              : /* Build a GOTO to the LABEL_DECL pointed to by LABEL_P, building it first
    3098              :    if necessary.  */
    3099              : 
    3100              : tree
    3101      1806598 : build_and_jump (tree *label_p)
    3102              : {
    3103      1806598 :   if (label_p == NULL)
    3104              :     /* If there's nowhere to jump, just fall through.  */
    3105              :     return NULL_TREE;
    3106              : 
    3107      1225151 :   if (*label_p == NULL_TREE)
    3108              :     {
    3109       701927 :       tree label = create_artificial_label (UNKNOWN_LOCATION);
    3110       701927 :       *label_p = label;
    3111              :     }
    3112              : 
    3113      1225151 :   return build1 (GOTO_EXPR, void_type_node, *label_p);
    3114              : }
    3115              : 
    3116              : /* Gimplify an EXIT_EXPR by converting to a GOTO_EXPR inside a COND_EXPR.
    3117              :    This also involves building a label to jump to and communicating it to
    3118              :    gimplify_loop_expr through gimplify_ctxp->exit_label.  */
    3119              : 
    3120              : static enum gimplify_status
    3121         3411 : gimplify_exit_expr (tree *expr_p)
    3122              : {
    3123         3411 :   tree cond = TREE_OPERAND (*expr_p, 0);
    3124         3411 :   tree expr;
    3125              : 
    3126         3411 :   expr = build_and_jump (&gimplify_ctxp->exit_label);
    3127         3411 :   expr = build3 (COND_EXPR, void_type_node, cond, expr, NULL_TREE);
    3128         3411 :   *expr_p = expr;
    3129              : 
    3130         3411 :   return GS_OK;
    3131              : }
    3132              : 
    3133              : /* *EXPR_P is a COMPONENT_REF being used as an rvalue.  If its type is
    3134              :    different from its canonical type, wrap the whole thing inside a
    3135              :    NOP_EXPR and force the type of the COMPONENT_REF to be the canonical
    3136              :    type.
    3137              : 
    3138              :    The canonical type of a COMPONENT_REF is the type of the field being
    3139              :    referenced--unless the field is a bit-field which can be read directly
    3140              :    in a smaller mode, in which case the canonical type is the
    3141              :    sign-appropriate type corresponding to that mode.  */
    3142              : 
    3143              : static void
    3144     19075803 : canonicalize_component_ref (tree *expr_p)
    3145              : {
    3146     19075803 :   tree expr = *expr_p;
    3147     19075803 :   tree type;
    3148              : 
    3149     19075803 :   gcc_assert (TREE_CODE (expr) == COMPONENT_REF);
    3150              : 
    3151     19075803 :   if (INTEGRAL_TYPE_P (TREE_TYPE (expr)))
    3152      7679641 :     type = TREE_TYPE (get_unwidened (expr, NULL_TREE));
    3153              :   else
    3154     11396162 :     type = TREE_TYPE (TREE_OPERAND (expr, 1));
    3155              : 
    3156              :   /* One could argue that all the stuff below is not necessary for
    3157              :      the non-bitfield case and declare it a FE error if type
    3158              :      adjustment would be needed.  */
    3159     19075803 :   if (TREE_TYPE (expr) != type)
    3160              :     {
    3161              : #ifdef ENABLE_TYPES_CHECKING
    3162       802912 :       tree old_type = TREE_TYPE (expr);
    3163              : #endif
    3164       802912 :       auto type_quals = TYPE_QUALS (type);
    3165       802912 :       gcc_checking_assert (/* Fields should lack address space
    3166              :                               qualification.  */
    3167              :                            ADDR_SPACE_GENERIC_P (type_quals.addr_space ()));
    3168              : 
    3169              :       /* We need to preserve qualifiers and propagate them from
    3170              :          operand 0.  */
    3171       802912 :       addr_space_t op_as;
    3172       802912 :       cv_qualifier op_cv;
    3173       802912 :       std::tie (op_cv, op_as)
    3174       802912 :         = TYPE_QUALS (TREE_TYPE (TREE_OPERAND (expr, 0))).split ();
    3175       802912 :       type_quals |= op_cv;
    3176       802912 :       type_quals.set_as (op_as);
    3177       802912 :       if (TYPE_QUALS (type) != type_quals)
    3178       802655 :         type = build_qualified_type (TYPE_MAIN_VARIANT (type), type_quals);
    3179              : 
    3180              :       /* Set the type of the COMPONENT_REF to the underlying type.  */
    3181       802912 :       TREE_TYPE (expr) = type;
    3182              : 
    3183              : #ifdef ENABLE_TYPES_CHECKING
    3184              :       /* It is now a FE error, if the conversion from the canonical
    3185              :          type to the original expression type is not useless.  */
    3186       802912 :       gcc_assert (useless_type_conversion_p (old_type, type));
    3187              : #endif
    3188              :     }
    3189     19075803 : }
    3190              : 
    3191              : /* If a NOP conversion is changing a pointer to array of foo to a pointer
    3192              :    to foo, embed that change in the ADDR_EXPR by converting
    3193              :       T array[U];
    3194              :       (T *)&array
    3195              :    ==>
    3196              :       &array[L]
    3197              :    where L is the lower bound.  For simplicity, only do this for constant
    3198              :    lower bound.
    3199              :    The constraint is that the type of &array[L] is trivially convertible
    3200              :    to T *.  */
    3201              : 
    3202              : static void
    3203       568442 : canonicalize_addr_expr (tree *expr_p)
    3204              : {
    3205       568442 :   tree expr = *expr_p;
    3206       568442 :   tree addr_expr = TREE_OPERAND (expr, 0);
    3207       568442 :   tree datype, ddatype, pddatype;
    3208              : 
    3209              :   /* We simplify only conversions from an ADDR_EXPR to a pointer type.  */
    3210      1136538 :   if (!POINTER_TYPE_P (TREE_TYPE (expr))
    3211       568442 :       || TREE_CODE (addr_expr) != ADDR_EXPR)
    3212              :     return;
    3213              : 
    3214              :   /* The addr_expr type should be a pointer to an array.  */
    3215          346 :   datype = TREE_TYPE (TREE_TYPE (addr_expr));
    3216          346 :   if (TREE_CODE (datype) != ARRAY_TYPE)
    3217              :     return;
    3218              : 
    3219              :   /* The pointer to element type shall be trivially convertible to
    3220              :      the expression pointer type.  */
    3221           28 :   ddatype = TREE_TYPE (datype);
    3222           28 :   pddatype = build_pointer_type (ddatype);
    3223           28 :   if (!useless_type_conversion_p (TYPE_MAIN_VARIANT (TREE_TYPE (expr)),
    3224              :                                   pddatype))
    3225              :     return;
    3226              : 
    3227              :   /* The lower bound and element sizes must be constant.  */
    3228            0 :   if (!TYPE_SIZE_UNIT (ddatype)
    3229            0 :       || TREE_CODE (TYPE_SIZE_UNIT (ddatype)) != INTEGER_CST
    3230            0 :       || !TYPE_DOMAIN (datype) || !TYPE_MIN_VALUE (TYPE_DOMAIN (datype))
    3231            0 :       || TREE_CODE (TYPE_MIN_VALUE (TYPE_DOMAIN (datype))) != INTEGER_CST)
    3232              :     return;
    3233              : 
    3234              :   /* All checks succeeded.  Build a new node to merge the cast.  */
    3235            0 :   *expr_p = build4 (ARRAY_REF, ddatype, TREE_OPERAND (addr_expr, 0),
    3236            0 :                     TYPE_MIN_VALUE (TYPE_DOMAIN (datype)),
    3237              :                     NULL_TREE, NULL_TREE);
    3238            0 :   *expr_p = build1 (ADDR_EXPR, pddatype, *expr_p);
    3239              : 
    3240              :   /* We can have stripped a required restrict qualifier above.  */
    3241            0 :   if (!useless_type_conversion_p (TREE_TYPE (expr), TREE_TYPE (*expr_p)))
    3242            0 :     *expr_p = fold_convert (TREE_TYPE (expr), *expr_p);
    3243              : }
    3244              : 
    3245              : /* *EXPR_P is a NOP_EXPR or CONVERT_EXPR.  Remove it and/or other conversions
    3246              :    underneath as appropriate.  */
    3247              : 
    3248              : static enum gimplify_status
    3249     13327082 : gimplify_conversion (tree *expr_p)
    3250              : {
    3251     13327082 :   location_t loc = EXPR_LOCATION (*expr_p);
    3252     13327082 :   gcc_assert (CONVERT_EXPR_P (*expr_p));
    3253              : 
    3254              :   /* Then strip away all but the outermost conversion.  */
    3255     13327082 :   STRIP_SIGN_NOPS (TREE_OPERAND (*expr_p, 0));
    3256              : 
    3257              :   /* And remove the outermost conversion if it's useless.  */
    3258     13327082 :   if (tree_ssa_useless_type_conversion (*expr_p))
    3259            0 :     *expr_p = TREE_OPERAND (*expr_p, 0);
    3260              : 
    3261              :   /* If we still have a conversion at the toplevel,
    3262              :      then canonicalize some constructs.  */
    3263     13327082 :   if (CONVERT_EXPR_P (*expr_p))
    3264              :     {
    3265     13327082 :       tree sub = TREE_OPERAND (*expr_p, 0);
    3266              : 
    3267              :       /* If a NOP conversion is changing the type of a COMPONENT_REF
    3268              :          expression, then canonicalize its type now in order to expose more
    3269              :          redundant conversions.  */
    3270     13327082 :       if (TREE_CODE (sub) == COMPONENT_REF)
    3271       279252 :         canonicalize_component_ref (&TREE_OPERAND (*expr_p, 0));
    3272              : 
    3273              :       /* If a NOP conversion is changing a pointer to array of foo
    3274              :          to a pointer to foo, embed that change in the ADDR_EXPR.  */
    3275     13047830 :       else if (TREE_CODE (sub) == ADDR_EXPR)
    3276       568442 :         canonicalize_addr_expr (expr_p);
    3277              :     }
    3278              : 
    3279              :   /* If we have a conversion to a non-register type force the
    3280              :      use of a VIEW_CONVERT_EXPR instead.  */
    3281     13327082 :   if (CONVERT_EXPR_P (*expr_p) && !is_gimple_reg_type (TREE_TYPE (*expr_p)))
    3282           90 :     *expr_p = fold_build1_loc (loc, VIEW_CONVERT_EXPR, TREE_TYPE (*expr_p),
    3283           90 :                                TREE_OPERAND (*expr_p, 0));
    3284              : 
    3285              :   /* Canonicalize CONVERT_EXPR to NOP_EXPR.  */
    3286     13327082 :   if (TREE_CODE (*expr_p) == CONVERT_EXPR)
    3287       235100 :     TREE_SET_CODE (*expr_p, NOP_EXPR);
    3288              : 
    3289     13327082 :   return GS_OK;
    3290              : }
    3291              : 
    3292              : /* Gimplify a VAR_DECL or PARM_DECL.  Return GS_OK if we expanded a
    3293              :    DECL_VALUE_EXPR, and it's worth re-examining things.  */
    3294              : 
    3295              : static enum gimplify_status
    3296    130604262 : gimplify_var_or_parm_decl (tree *expr_p)
    3297              : {
    3298    130604262 :   tree decl = *expr_p;
    3299              : 
    3300              :   /* ??? If this is a local variable, and it has not been seen in any
    3301              :      outer BIND_EXPR, then it's probably the result of a duplicate
    3302              :      declaration, for which we've already issued an error.  It would
    3303              :      be really nice if the front end wouldn't leak these at all.
    3304              :      Currently the only known culprit is C++ destructors, as seen
    3305              :      in g++.old-deja/g++.jason/binding.C.
    3306              :      Another possible culpit are size expressions for variably modified
    3307              :      types which are lost in the FE or not gimplified correctly.  */
    3308    130604262 :   if (VAR_P (decl)
    3309    101868211 :       && !DECL_SEEN_IN_BIND_EXPR_P (decl)
    3310     17462040 :       && !TREE_STATIC (decl) && !DECL_EXTERNAL (decl)
    3311    130664610 :       && decl_function_context (decl) == current_function_decl)
    3312              :     {
    3313           77 :       gcc_assert (seen_error ());
    3314              :       return GS_ERROR;
    3315              :     }
    3316              : 
    3317              :   /* When within an OMP context, notice uses of variables.  */
    3318    130604185 :   if (gimplify_omp_ctxp && omp_notice_variable (gimplify_omp_ctxp, decl, true))
    3319              :     return GS_ALL_DONE;
    3320              : 
    3321              :   /* If the decl is an alias for another expression, substitute it now.  */
    3322    130598365 :   if (DECL_HAS_VALUE_EXPR_P (decl))
    3323              :     {
    3324       470952 :       *expr_p = unshare_expr (DECL_VALUE_EXPR (decl));
    3325       470952 :       return GS_OK;
    3326              :     }
    3327              : 
    3328              :   return GS_ALL_DONE;
    3329              : }
    3330              : 
    3331              : /* Recalculate the value of the TREE_SIDE_EFFECTS flag for T.  */
    3332              : 
    3333              : static void
    3334    113741919 : recalculate_side_effects (tree t)
    3335              : {
    3336    113741919 :   enum tree_code code = TREE_CODE (t);
    3337    113741919 :   int len = TREE_OPERAND_LENGTH (t);
    3338    113741919 :   int i;
    3339              : 
    3340    113741919 :   switch (TREE_CODE_CLASS (code))
    3341              :     {
    3342       673421 :     case tcc_expression:
    3343       673421 :       switch (code)
    3344              :         {
    3345              :         case INIT_EXPR:
    3346              :         case MODIFY_EXPR:
    3347              :         case VA_ARG_EXPR:
    3348              :         case PREDECREMENT_EXPR:
    3349              :         case PREINCREMENT_EXPR:
    3350              :         case POSTDECREMENT_EXPR:
    3351              :         case POSTINCREMENT_EXPR:
    3352              :           /* All of these have side-effects, no matter what their
    3353              :              operands are.  */
    3354              :           return;
    3355              : 
    3356              :         default:
    3357              :           break;
    3358              :         }
    3359              :       /* Fall through.  */
    3360              : 
    3361    113741915 :     case tcc_comparison:  /* a comparison expression */
    3362    113741915 :     case tcc_unary:       /* a unary arithmetic expression */
    3363    113741915 :     case tcc_binary:      /* a binary arithmetic expression */
    3364    113741915 :     case tcc_reference:   /* a reference */
    3365    113741915 :     case tcc_vl_exp:        /* a function call */
    3366    113741915 :       TREE_SIDE_EFFECTS (t) = TREE_THIS_VOLATILE (t);
    3367    370670918 :       for (i = 0; i < len; ++i)
    3368              :         {
    3369    256929003 :           tree op = TREE_OPERAND (t, i);
    3370    256929003 :           if (op && TREE_SIDE_EFFECTS (op))
    3371      1360453 :             TREE_SIDE_EFFECTS (t) = 1;
    3372              :         }
    3373              :       break;
    3374              : 
    3375              :     case tcc_constant:
    3376              :       /* No side-effects.  */
    3377              :       return;
    3378              : 
    3379              :     case tcc_declaration:
    3380              :       /* These can have side-effects if TREE_THIS_VOLATILE,
    3381              :          but those should be set elsewhere, not in
    3382              :          recalculate_side_effects.  Can be triggered e.g. if
    3383              :          a comparison is folded into one of its operands.  */
    3384              :       return;
    3385              : 
    3386            2 :     default:
    3387            2 :       if (code == SSA_NAME)
    3388              :         /* No side-effects.  */
    3389              :         return;
    3390            0 :       gcc_unreachable ();
    3391              :    }
    3392              : }
    3393              : 
    3394              : /* Gimplify the COMPONENT_REF, ARRAY_REF, REALPART_EXPR or IMAGPART_EXPR
    3395              :    node *EXPR_P.
    3396              : 
    3397              :       compound_lval
    3398              :               : min_lval '[' val ']'
    3399              :               | min_lval '.' ID
    3400              :               | compound_lval '[' val ']'
    3401              :               | compound_lval '.' ID
    3402              : 
    3403              :    This is not part of the original SIMPLE definition, which separates
    3404              :    array and member references, but it seems reasonable to handle them
    3405              :    together.  Also, this way we don't run into problems with union
    3406              :    aliasing; gcc requires that for accesses through a union to alias, the
    3407              :    union reference must be explicit, which was not always the case when we
    3408              :    were splitting up array and member refs.
    3409              : 
    3410              :    PRE_P points to the sequence where side effects that must happen before
    3411              :      *EXPR_P should be stored.
    3412              : 
    3413              :    POST_P points to the sequence where side effects that must happen after
    3414              :      *EXPR_P should be stored.  */
    3415              : 
    3416              : static enum gimplify_status
    3417     30970513 : gimplify_compound_lval (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
    3418              :                         fallback_t fallback)
    3419              : {
    3420     30970513 :   tree *p;
    3421     30970513 :   enum gimplify_status ret = GS_ALL_DONE, tret;
    3422     30970513 :   int i;
    3423     30970513 :   location_t loc = EXPR_LOCATION (*expr_p);
    3424     30970513 :   tree expr = *expr_p;
    3425              : 
    3426              :   /* Create a stack of the subexpressions so later we can walk them in
    3427              :      order from inner to outer.  */
    3428     30970513 :   auto_vec<tree, 10> expr_stack;
    3429              : 
    3430              :   /* We can handle anything that get_inner_reference can deal with.  */
    3431     74743404 :   for (p = expr_p; ; p = &TREE_OPERAND (*p, 0))
    3432              :     {
    3433     43839314 :     restart:
    3434              :       /* Fold INDIRECT_REFs now to turn them into ARRAY_REFs.  */
    3435     74809827 :       if (TREE_CODE (*p) == INDIRECT_REF)
    3436      4069469 :         *p = fold_indirect_ref_loc (loc, *p);
    3437              : 
    3438     74809827 :       if (handled_component_p (*p))
    3439              :         ;
    3440              :       /* Expand DECL_VALUE_EXPR now.  In some cases that may expose
    3441              :          additional COMPONENT_REFs.  */
    3442     13638119 :       else if ((VAR_P (*p) || TREE_CODE (*p) == PARM_DECL)
    3443     32490803 :                && gimplify_var_or_parm_decl (p) == GS_OK)
    3444        66423 :         goto restart;
    3445              :       else
    3446              :         break;
    3447              : 
    3448     43772891 :       expr_stack.safe_push (*p);
    3449              :     }
    3450              : 
    3451     30970513 :   gcc_assert (expr_stack.length ());
    3452              : 
    3453              :   /* Now EXPR_STACK is a stack of pointers to all the refs we've
    3454              :      walked through and P points to the innermost expression.
    3455              : 
    3456              :      Java requires that we elaborated nodes in source order.  That
    3457              :      means we must gimplify the inner expression followed by each of
    3458              :      the indices, in order.  But we can't gimplify the inner
    3459              :      expression until we deal with any variable bounds, sizes, or
    3460              :      positions in order to deal with PLACEHOLDER_EXPRs.
    3461              : 
    3462              :      The base expression may contain a statement expression that
    3463              :      has declarations used in size expressions, so has to be
    3464              :      gimplified before gimplifying the size expressions.
    3465              : 
    3466              :      So we do this in three steps.  First we deal with variable
    3467              :      bounds, sizes, and positions, then we gimplify the base and
    3468              :      ensure it is memory if needed, then we deal with the annotations
    3469              :      for any variables in the components and any indices, from left
    3470              :      to right.  */
    3471              : 
    3472     30970513 :   bool need_non_reg = false;
    3473     74743401 :   for (i = expr_stack.length () - 1; i >= 0; i--)
    3474              :     {
    3475     43772891 :       tree t = expr_stack[i];
    3476              : 
    3477     43772891 :       if (error_operand_p (TREE_OPERAND (t, 0)))
    3478              :         return GS_ERROR;
    3479              : 
    3480     43772888 :       if (TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
    3481              :         {
    3482              :           /* Deal with the low bound and element type size and put them into
    3483              :              the ARRAY_REF.  If these values are set, they have already been
    3484              :              gimplified.  */
    3485      9207450 :           if (TREE_OPERAND (t, 2) == NULL_TREE)
    3486              :             {
    3487      9142094 :               tree low = unshare_expr (array_ref_low_bound (t));
    3488      9142094 :               if (!is_gimple_min_invariant (low))
    3489              :                 {
    3490            0 :                   TREE_OPERAND (t, 2) = low;
    3491              :                 }
    3492              :             }
    3493              : 
    3494      9207450 :           if (TREE_OPERAND (t, 3) == NULL_TREE)
    3495              :             {
    3496      9193395 :               tree elmt_size = array_ref_element_size (t);
    3497      9193395 :               if (!is_gimple_min_invariant (elmt_size))
    3498              :                 {
    3499         8793 :                   elmt_size = unshare_expr (elmt_size);
    3500         8793 :                   tree elmt_type = TREE_TYPE (TREE_TYPE (TREE_OPERAND (t, 0)));
    3501         8793 :                   tree factor = size_int (TYPE_ALIGN_UNIT (elmt_type));
    3502              : 
    3503              :                   /* Divide the element size by the alignment of the element
    3504              :                      type (above).  */
    3505         8793 :                   elmt_size = size_binop_loc (loc, EXACT_DIV_EXPR,
    3506              :                                               elmt_size, factor);
    3507              : 
    3508         8793 :                   TREE_OPERAND (t, 3) = elmt_size;
    3509              :                 }
    3510              :             }
    3511              :           need_non_reg = true;
    3512              :         }
    3513     34565438 :       else if (TREE_CODE (t) == COMPONENT_REF)
    3514              :         {
    3515              :           /* Set the field offset into T and gimplify it.  */
    3516     33242333 :           if (TREE_OPERAND (t, 2) == NULL_TREE)
    3517              :             {
    3518     33241954 :               tree offset = component_ref_field_offset (t);
    3519     33241954 :               if (!is_gimple_min_invariant (offset))
    3520              :                 {
    3521          287 :                   offset = unshare_expr (offset);
    3522          287 :                   tree field = TREE_OPERAND (t, 1);
    3523          287 :                   tree factor
    3524          287 :                     = size_int (DECL_OFFSET_ALIGN (field) / BITS_PER_UNIT);
    3525              : 
    3526              :                   /* Divide the offset by its alignment.  */
    3527          287 :                   offset = size_binop_loc (loc, EXACT_DIV_EXPR,
    3528              :                                            offset, factor);
    3529              : 
    3530          287 :                   TREE_OPERAND (t, 2) = offset;
    3531              :                 }
    3532              :             }
    3533              :           need_non_reg = true;
    3534              :         }
    3535      1323105 :       else if (!is_gimple_reg_type (TREE_TYPE (t)))
    3536              :         /* When the result of an operation, in particular a VIEW_CONVERT_EXPR
    3537              :            is a non-register type then require the base object to be a
    3538              :            non-register as well.  */
    3539     42745261 :         need_non_reg = true;
    3540              :     }
    3541              : 
    3542              :   /* Step 2 is to gimplify the base expression.  Make sure lvalue is set
    3543              :      so as to match the min_lval predicate.  Failure to do so may result
    3544              :      in the creation of large aggregate temporaries.  */
    3545     30970510 :   tret = gimplify_expr (p, pre_p, post_p, is_gimple_min_lval,
    3546              :                         fallback | fb_lvalue);
    3547     30970510 :   ret = MIN (ret, tret);
    3548     30970510 :   if (ret == GS_ERROR)
    3549              :     return GS_ERROR;
    3550              : 
    3551              :   /* Step 2a: if we have component references we do not support on
    3552              :      registers then make sure the base isn't a register.  Of course
    3553              :      we can only do so if an rvalue is OK.  */
    3554     30970510 :   if (need_non_reg && (fallback & fb_rvalue))
    3555     23040829 :     prepare_gimple_addressable (p, pre_p);
    3556              : 
    3557              : 
    3558              :   /* Step 3: gimplify size expressions and the indices and operands of
    3559              :      ARRAY_REF.  During this loop we also remove any useless conversions.
    3560              :      If we operate on a register also make sure to properly gimplify
    3561              :      to individual operations.  */
    3562              : 
    3563     30970510 :   bool reg_operations = is_gimple_reg (*p);
    3564    105713908 :   for (; expr_stack.length () > 0; )
    3565              :     {
    3566     43772888 :       tree t = expr_stack.pop ();
    3567              : 
    3568     43772888 :       if (TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
    3569              :         {
    3570      9207450 :           gcc_assert (!reg_operations);
    3571              : 
    3572              :           /* Gimplify the low bound and element type size. */
    3573      9207450 :           tret = gimplify_expr (&TREE_OPERAND (t, 2), pre_p, post_p,
    3574              :                                 is_gimple_reg, fb_rvalue);
    3575      9207450 :           ret = MIN (ret, tret);
    3576              : 
    3577      9207450 :           tret = gimplify_expr (&TREE_OPERAND (t, 3), pre_p, post_p,
    3578              :                                 is_gimple_reg, fb_rvalue);
    3579      9207450 :           ret = MIN (ret, tret);
    3580              : 
    3581              :           /* Gimplify the dimension.  */
    3582      9207450 :           tret = gimplify_expr (&TREE_OPERAND (t, 1), pre_p, post_p,
    3583              :                                 is_gimple_val, fb_rvalue);
    3584      9207450 :           ret = MIN (ret, tret);
    3585              :         }
    3586     34565438 :       else if (TREE_CODE (t) == COMPONENT_REF)
    3587              :         {
    3588     33242333 :           gcc_assert (!reg_operations);
    3589              : 
    3590     33242333 :           tret = gimplify_expr (&TREE_OPERAND (t, 2), pre_p, post_p,
    3591              :                                 is_gimple_reg, fb_rvalue);
    3592     33242333 :           ret = MIN (ret, tret);
    3593              :         }
    3594      1323105 :       else if (reg_operations)
    3595              :         {
    3596       908632 :           tret = gimplify_expr (&TREE_OPERAND (t, 0), pre_p, post_p,
    3597              :                                 is_gimple_val, fb_rvalue);
    3598      1323105 :           ret = MIN (ret, tret);
    3599              :         }
    3600              : 
    3601     43772888 :       STRIP_USELESS_TYPE_CONVERSION (TREE_OPERAND (t, 0));
    3602              : 
    3603              :       /* The innermost expression P may have originally had
    3604              :          TREE_SIDE_EFFECTS set which would have caused all the outer
    3605              :          expressions in *EXPR_P leading to P to also have had
    3606              :          TREE_SIDE_EFFECTS set.  */
    3607     43772888 :       recalculate_side_effects (t);
    3608              :     }
    3609              : 
    3610              :   /* If the outermost expression is a COMPONENT_REF, canonicalize its type.  */
    3611     30970510 :   if ((fallback & fb_rvalue) && TREE_CODE (*expr_p) == COMPONENT_REF)
    3612              :     {
    3613     18796551 :       canonicalize_component_ref (expr_p);
    3614              :     }
    3615              : 
    3616     30970510 :   expr_stack.release ();
    3617              : 
    3618     30970510 :   gcc_assert (*expr_p == expr || ret != GS_ALL_DONE);
    3619              : 
    3620              :   return ret;
    3621     30970513 : }
    3622              : 
    3623              : /*  Gimplify the self modifying expression pointed to by EXPR_P
    3624              :     (++, --, +=, -=).
    3625              : 
    3626              :     PRE_P points to the list where side effects that must happen before
    3627              :         *EXPR_P should be stored.
    3628              : 
    3629              :     POST_P points to the list where side effects that must happen after
    3630              :         *EXPR_P should be stored.
    3631              : 
    3632              :     WANT_VALUE is nonzero iff we want to use the value of this expression
    3633              :         in another expression.
    3634              : 
    3635              :     ARITH_TYPE is the type the computation should be performed in.  */
    3636              : 
    3637              : enum gimplify_status
    3638      1153902 : gimplify_self_mod_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
    3639              :                         bool want_value, tree arith_type)
    3640              : {
    3641      1153902 :   enum tree_code code;
    3642      1153902 :   tree lhs, lvalue, rhs, t1;
    3643      1153902 :   gimple_seq post = NULL, *orig_post_p = post_p;
    3644      1153902 :   bool postfix;
    3645      1153902 :   enum tree_code arith_code;
    3646      1153902 :   enum gimplify_status ret;
    3647      1153902 :   location_t loc = EXPR_LOCATION (*expr_p);
    3648              : 
    3649      1153902 :   code = TREE_CODE (*expr_p);
    3650              : 
    3651      1153902 :   gcc_assert (code == POSTINCREMENT_EXPR || code == POSTDECREMENT_EXPR
    3652              :               || code == PREINCREMENT_EXPR || code == PREDECREMENT_EXPR);
    3653              : 
    3654              :   /* Prefix or postfix?  */
    3655      1153902 :   if (code == POSTINCREMENT_EXPR || code == POSTDECREMENT_EXPR)
    3656              :     /* Faster to treat as prefix if result is not used.  */
    3657              :     postfix = want_value;
    3658              :   else
    3659              :     postfix = false;
    3660              : 
    3661              :   /* For postfix, make sure the inner expression's post side effects
    3662              :      are executed after side effects from this expression.  */
    3663       414001 :   if (postfix)
    3664      1153902 :     post_p = &post;
    3665              : 
    3666              :   /* Add or subtract?  */
    3667      1153902 :   if (code == PREINCREMENT_EXPR || code == POSTINCREMENT_EXPR)
    3668              :     arith_code = PLUS_EXPR;
    3669              :   else
    3670        45867 :     arith_code = MINUS_EXPR;
    3671              : 
    3672              :   /* Gimplify the LHS into a GIMPLE lvalue.  */
    3673      1153902 :   lvalue = TREE_OPERAND (*expr_p, 0);
    3674      1153902 :   ret = gimplify_expr (&lvalue, pre_p, post_p, is_gimple_lvalue, fb_lvalue);
    3675      1153902 :   if (ret == GS_ERROR)
    3676              :     return ret;
    3677              : 
    3678              :   /* Extract the operands to the arithmetic operation.  */
    3679      1153899 :   lhs = lvalue;
    3680      1153899 :   rhs = TREE_OPERAND (*expr_p, 1);
    3681              : 
    3682              :   /* For postfix operator, we evaluate the LHS to an rvalue and then use
    3683              :      that as the result value and in the postqueue operation.  */
    3684      1153899 :   if (postfix)
    3685              :     {
    3686       189998 :       ret = gimplify_expr (&lhs, pre_p, post_p, is_gimple_val, fb_rvalue);
    3687       189998 :       if (ret == GS_ERROR)
    3688              :         return ret;
    3689              : 
    3690       189998 :       lhs = get_initialized_tmp_var (lhs, pre_p);
    3691              :     }
    3692              : 
    3693              :   /* For POINTERs increment, use POINTER_PLUS_EXPR.  */
    3694      1153899 :   if (POINTER_TYPE_P (TREE_TYPE (lhs)))
    3695              :     {
    3696       237635 :       rhs = convert_to_ptrofftype_loc (loc, rhs);
    3697       237635 :       if (arith_code == MINUS_EXPR)
    3698        10236 :         rhs = fold_build1_loc (loc, NEGATE_EXPR, TREE_TYPE (rhs), rhs);
    3699       237635 :       t1 = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (*expr_p), lhs, rhs);
    3700              :     }
    3701              :   else
    3702       916264 :     t1 = fold_convert (TREE_TYPE (*expr_p),
    3703              :                        fold_build2 (arith_code, arith_type,
    3704              :                                     fold_convert (arith_type, lhs),
    3705              :                                     fold_convert (arith_type, rhs)));
    3706              : 
    3707      1153899 :   if (postfix)
    3708              :     {
    3709       189998 :       gimplify_assign (lvalue, t1, pre_p);
    3710       189998 :       gimplify_seq_add_seq (orig_post_p, post);
    3711       189998 :       *expr_p = lhs;
    3712       189998 :       return GS_ALL_DONE;
    3713              :     }
    3714              :   else
    3715              :     {
    3716       963901 :       *expr_p = build2 (MODIFY_EXPR, TREE_TYPE (lvalue), lvalue, t1);
    3717       963901 :       return GS_OK;
    3718              :     }
    3719              : }
    3720              : 
    3721              : /* If *EXPR_P has a variable sized type, wrap it in a WITH_SIZE_EXPR.  */
    3722              : 
    3723              : static void
    3724     83768394 : maybe_with_size_expr (tree *expr_p)
    3725              : {
    3726     83768394 :   tree expr = *expr_p;
    3727     83768394 :   tree type = TREE_TYPE (expr);
    3728     83768394 :   tree size;
    3729              : 
    3730              :   /* If we've already wrapped this or the type is error_mark_node, we can't do
    3731              :      anything.  */
    3732     83768394 :   if (TREE_CODE (expr) == WITH_SIZE_EXPR
    3733     83768269 :       || type == error_mark_node)
    3734              :     return;
    3735              : 
    3736              :   /* If the size isn't known or is a constant, we have nothing to do.  */
    3737     83768121 :   size = TYPE_SIZE_UNIT (type);
    3738     83768121 :   if (!size || poly_int_tree_p (size))
    3739              :     return;
    3740              : 
    3741              :   /* Otherwise, make a WITH_SIZE_EXPR.  */
    3742         1704 :   size = unshare_expr (size);
    3743         1704 :   size = SUBSTITUTE_PLACEHOLDER_IN_EXPR (size, expr);
    3744         1704 :   *expr_p = build2 (WITH_SIZE_EXPR, type, expr, size);
    3745              : }
    3746              : 
    3747              : /* Helper for gimplify_call_expr.  Gimplify a single argument *ARG_P
    3748              :    Store any side-effects in PRE_P.  CALL_LOCATION is the location of
    3749              :    the CALL_EXPR.  If ALLOW_SSA is set the actual parameter may be
    3750              :    gimplified to an SSA name.  */
    3751              : 
    3752              : enum gimplify_status
    3753     34894209 : gimplify_arg (tree *arg_p, gimple_seq *pre_p, location_t call_location,
    3754              :               bool allow_ssa)
    3755              : {
    3756     34894209 :   bool (*test) (tree);
    3757     34894209 :   fallback_t fb;
    3758              : 
    3759              :   /* In general, we allow lvalues for function arguments to avoid
    3760              :      extra overhead of copying large aggregates out of even larger
    3761              :      aggregates into temporaries only to copy the temporaries to
    3762              :      the argument list.  Make optimizers happy by pulling out to
    3763              :      temporaries those types that fit in registers.  */
    3764     34894209 :   if (is_gimple_reg_type (TREE_TYPE (*arg_p)))
    3765              :     test = is_gimple_val, fb = fb_rvalue;
    3766              :   else
    3767              :     {
    3768      1624582 :       test = is_gimple_lvalue, fb = fb_either;
    3769              :       /* Also strip a TARGET_EXPR that would force an extra copy.  */
    3770      1624582 :       if (TREE_CODE (*arg_p) == TARGET_EXPR)
    3771              :         {
    3772       340662 :           tree init = TARGET_EXPR_INITIAL (*arg_p);
    3773       340662 :           if (init
    3774       340659 :               && !VOID_TYPE_P (TREE_TYPE (init))
    3775              :               /* Currently, due to c++/116015, it is not desirable to
    3776              :                  strip a TARGET_EXPR whose initializer is a {}.  The
    3777              :                  problem is that if we do elide it, we also have to
    3778              :                  replace all the occurrences of the slot temporary in the
    3779              :                  initializer with the temporary created for the argument.
    3780              :                  But we do not have that temporary yet so the replacement
    3781              :                  would be quite awkward and it might be needed to resort
    3782              :                  back to a PLACEHOLDER_EXPR.  Note that stripping the
    3783              :                  TARGET_EXPR wouldn't help anyway, as gimplify_expr would
    3784              :                  just allocate a temporary to store the CONSTRUCTOR into.
    3785              :                  (FIXME PR116375.)
    3786              : 
    3787              :                  See convert_for_arg_passing for the C++ code that marks
    3788              :                  the TARGET_EXPR as eliding or not.  */
    3789       639238 :               && TREE_CODE (init) != CONSTRUCTOR)
    3790       259070 :             *arg_p = init;
    3791              :         }
    3792              :     }
    3793              : 
    3794              :   /* If this is a variable sized type, we must remember the size.  */
    3795     34894209 :   maybe_with_size_expr (arg_p);
    3796              : 
    3797              :   /* FIXME diagnostics: This will mess up gcc.dg/Warray-bounds.c.  */
    3798              :   /* Make sure arguments have the same location as the function call
    3799              :      itself.  */
    3800     34894209 :   protected_set_expr_location (*arg_p, call_location);
    3801              : 
    3802              :   /* There is a sequence point before a function call.  Side effects in
    3803              :      the argument list must occur before the actual call. So, when
    3804              :      gimplifying arguments, force gimplify_expr to use an internal
    3805              :      post queue which is then appended to the end of PRE_P.  */
    3806     34894209 :   return gimplify_expr (arg_p, pre_p, NULL, test, fb, allow_ssa);
    3807              : }
    3808              : 
    3809              : /* Don't fold inside offloading or taskreg regions: it can break code by
    3810              :    adding decl references that weren't in the source.  We'll do it during
    3811              :    omplower pass instead.  */
    3812              : 
    3813              : static bool
    3814     60674731 : maybe_fold_stmt (gimple_stmt_iterator *gsi)
    3815              : {
    3816     60674731 :   struct gimplify_omp_ctx *ctx;
    3817     61204114 :   for (ctx = gimplify_omp_ctxp; ctx; ctx = ctx->outer_context)
    3818      1204114 :     if ((ctx->region_type & (ORT_TARGET | ORT_PARALLEL | ORT_TASK)) != 0)
    3819              :       return false;
    3820       534402 :     else if ((ctx->region_type & ORT_HOST_TEAMS) == ORT_HOST_TEAMS)
    3821              :       return false;
    3822              :   /* Delay folding of builtins until the IL is in consistent state
    3823              :      so the diagnostic machinery can do a better job.  */
    3824     60000000 :   if (gimple_call_builtin_p (gsi_stmt (*gsi)))
    3825              :     return false;
    3826     57586038 :   return fold_stmt (gsi);
    3827              : }
    3828              : 
    3829              : static tree
    3830              : expand_late_variant_directive (vec<struct omp_variant> all_candidates,
    3831              :                                tree construct_context);
    3832              : 
    3833              : 
    3834              : /* Helper function for calls to omp_dynamic_cond: find the current
    3835              :    enclosing block in the gimplification context.  */
    3836              : static tree
    3837          370 : find_supercontext (void)
    3838              : {
    3839          370 :   vec<gbind *>stack = gimple_bind_expr_stack ();
    3840          708 :   for (int i = stack.length () - 1; i >= 0; i++)
    3841              :     {
    3842          338 :       gbind *b = stack[i];
    3843          338 :       if (b->block)
    3844              :         return b->block;
    3845              :     }
    3846              :   return NULL_TREE;
    3847              : }
    3848              : 
    3849              : /* OpenMP: Handle the append_args and adjust_args clauses of
    3850              :    declare_variant for EXPR, which is a CALL_EXPR whose CALL_EXPR_FN
    3851              :    is the variant, within a dispatch construct with clauses DISPATCH_CLAUSES.
    3852              :    WANT_VALUE and POINTERIZE are as for expand_variant_call_expr.
    3853              : 
    3854              :    'append_args' causes interop objects are added after the last regular
    3855              :    (nonhidden, nonvariadic) arguments of the variant function.
    3856              :    'adjust_args' with need_device_{addr,ptr} converts the pointer target of
    3857              :    a pointer from a host to a device address. This uses either the default
    3858              :    device or the passed device number, which then sets the default device
    3859              :    address.  */
    3860              : static tree
    3861          471 : modify_call_for_omp_dispatch (tree expr, tree dispatch_clauses,
    3862              :                               bool want_value, bool pointerize)
    3863              : {
    3864          471 :   location_t loc = EXPR_LOCATION (expr);
    3865          471 :   tree fndecl = get_callee_fndecl (expr);
    3866              : 
    3867              :   /* Skip processing if we don't get the expected call form.  */
    3868          471 :   if (!fndecl)
    3869              :     return expr;
    3870              : 
    3871          471 :   tree init_code = NULL_TREE;
    3872          471 :   tree cleanup = NULL_TREE;
    3873          471 :   tree clobbers = NULL_TREE;
    3874          471 :   int nargs = call_expr_nargs (expr);
    3875          471 :   tree dispatch_device_num = NULL_TREE;
    3876          471 :   tree dispatch_interop = NULL_TREE;
    3877          471 :   tree dispatch_append_args = NULL_TREE;
    3878          471 :   int nfirst_args = 0;
    3879          471 :   tree dispatch_adjust_args_list
    3880          471 :     = lookup_attribute ("omp declare variant variant args",
    3881          471 :                         DECL_ATTRIBUTES (fndecl));
    3882              : 
    3883          471 :   if (dispatch_adjust_args_list)
    3884              :     {
    3885          389 :       dispatch_adjust_args_list = TREE_VALUE (dispatch_adjust_args_list);
    3886          389 :       dispatch_append_args = TREE_CHAIN (dispatch_adjust_args_list);
    3887          389 :       if (TREE_PURPOSE (dispatch_adjust_args_list) == NULL_TREE
    3888          389 :           && TREE_VALUE (dispatch_adjust_args_list) == NULL_TREE)
    3889              :         dispatch_adjust_args_list = NULL_TREE;
    3890              :     }
    3891          389 :   if (dispatch_append_args)
    3892              :     {
    3893          245 :       nfirst_args = tree_to_shwi (TREE_PURPOSE (dispatch_append_args));
    3894          245 :       dispatch_append_args = TREE_VALUE (dispatch_append_args);
    3895              :     }
    3896          471 :   dispatch_device_num = omp_find_clause (dispatch_clauses, OMP_CLAUSE_DEVICE);
    3897          471 :   if (dispatch_device_num)
    3898          287 :     dispatch_device_num = OMP_CLAUSE_DEVICE_ID (dispatch_device_num);
    3899          471 :   dispatch_interop = omp_find_clause (dispatch_clauses, OMP_CLAUSE_INTEROP);
    3900          471 :   int nappend = 0, ninterop = 0;
    3901         1351 :   for (tree t = dispatch_append_args; t; t = TREE_CHAIN (t))
    3902          409 :     nappend++;
    3903              : 
    3904              :   /* FIXME: error checking should be taken out of this function and
    3905              :      handled before any attempt at filtering or resolution happens.
    3906              :      Otherwise whether or not diagnostics appear is determined by
    3907              :      GCC internals, how good the front ends are at constant-folding,
    3908              :      the split between early/late resolution, etc instead of the code
    3909              :      as written by the user.  */
    3910          471 :   if (dispatch_interop)
    3911          861 :     for (tree t = dispatch_interop; t; t = TREE_CHAIN (t))
    3912          605 :       if (OMP_CLAUSE_CODE (t) == OMP_CLAUSE_INTEROP)
    3913          409 :         ninterop++;
    3914          471 :   if (dispatch_interop && !dispatch_device_num)
    3915              :     {
    3916            7 :       gcc_checking_assert (ninterop > 1);
    3917            7 :       error_at (OMP_CLAUSE_LOCATION (dispatch_interop),
    3918              :                 "the %<device%> clause must be present if the %<interop%> "
    3919              :                 "clause has more than one list item");
    3920              :     }
    3921          471 :   if (nappend < ninterop)
    3922              :     {
    3923           48 :       error_at (OMP_CLAUSE_LOCATION (dispatch_interop),
    3924              :                 "number of list items in %<interop%> clause (%d) "
    3925              :                 "exceeds the number of %<append_args%> items (%d) for "
    3926              :                 "%<declare variant%> candidate %qD", ninterop, nappend, fndecl);
    3927           88 :       inform (dispatch_append_args
    3928            8 :               ? EXPR_LOCATION (TREE_PURPOSE (dispatch_append_args))
    3929           40 :               : DECL_SOURCE_LOCATION (fndecl),
    3930              :               "%<declare variant%> candidate %qD declared here", fndecl);
    3931           48 :       ninterop = nappend;
    3932              :     }
    3933          471 :   if (dispatch_append_args)
    3934              :     {
    3935          245 :       tree *buffer = XALLOCAVEC (tree, nargs + nappend);
    3936          245 :       tree arg = TYPE_ARG_TYPES (TREE_TYPE (fndecl));
    3937              :       /* Copy the first arguments; insert then the interop objects,
    3938              :          and then copy the rest (nargs - nfirst_args) args.  */
    3939          245 :       int i;
    3940          679 :       for (i = 0; i < nfirst_args; i++)
    3941              :         {
    3942          434 :           arg = TREE_CHAIN (arg);
    3943          434 :           buffer[i] = CALL_EXPR_ARG (expr, i);
    3944              :         }
    3945              :       int j = ninterop;
    3946          583 :       for (tree t = dispatch_interop; t && j > 0; t = TREE_CHAIN (t))
    3947          338 :         if (OMP_CLAUSE_CODE (t) == OMP_CLAUSE_INTEROP)
    3948          338 :           buffer[i + --j] = OMP_CLAUSE_DECL (t);
    3949          245 :       gcc_checking_assert (j == 0);
    3950              : 
    3951              :       /* Do we need to create additional interop objects?  */
    3952          245 :       if (ninterop < nappend)
    3953              :         {
    3954           56 :           if (dispatch_device_num == NULL_TREE)
    3955              :             /* Not remapping device number.  */
    3956           22 :             dispatch_device_num = build_int_cst (integer_type_node,
    3957              :                                                  GOMP_DEVICE_DEFAULT_OMP_61);
    3958           56 :           int nnew = nappend - ninterop;
    3959           56 :           tree nobjs = build_int_cst (integer_type_node, nnew);
    3960           56 :           tree a, t;
    3961              : 
    3962              :           /* Skip to the append_args clause for the first constructed
    3963              :              interop argument.  */
    3964           56 :           tree apparg = dispatch_append_args;
    3965           83 :           for (j = 0; j < ninterop; j++)
    3966           27 :             apparg = TREE_CHAIN (apparg);
    3967              : 
    3968              :           /* omp_interop_t *objs[n]; */
    3969           56 :           tree objtype = build_pointer_type (pointer_sized_int_node);
    3970           56 :           t = build_array_type_nelts (objtype, nnew);
    3971           56 :           tree objs = create_tmp_var (t, "interopobjs");
    3972              : 
    3973              :           /* int target_tgtsync[n]; */
    3974           56 :           t = build_array_type_nelts (integer_type_node, nnew);
    3975           56 :           tree target_tgtsync = create_tmp_var (t, "tgt_tgtsync");
    3976              : 
    3977              :           /* Scan first to determine if we need a prefer_type array.  */
    3978           56 :           tree prefer_type = NULL_TREE;
    3979           56 :           tree prefer_type_type = NULL_TREE;
    3980          141 :           for (j = ninterop, a = apparg; j < nappend; j++, a = TREE_CHAIN (a))
    3981           64 :             if (TREE_VALUE (a) != NULL_TREE)
    3982              :               {
    3983              :                 /* const char *prefer_type[n];  */
    3984           35 :                 t = build_qualified_type (char_type_node, TYPE_QUAL_CONST);
    3985           35 :                 prefer_type_type = build_pointer_type (t);
    3986           35 :                 t = build_array_type_nelts (prefer_type_type, nnew);
    3987           35 :                 prefer_type = create_tmp_var (t, "pref_type");
    3988           35 :                 break;
    3989              :               }
    3990              : 
    3991              :           /* Initialize the arrays, generating temp vars and clobbers for
    3992              :              the interop objects.  (The constructed array holding the
    3993              :              pointers to these objects shouldn't need clobbering as there's
    3994              :              no reason for GOMP_interop to modify its contents.)  */
    3995          127 :           for (j = ninterop, a = apparg; j < nappend; j++, a = TREE_CHAIN (a))
    3996              :             {
    3997              :               /* The allocated temporaries for the interop objects
    3998              :                  have type omp_interop_t, which is an integer type that
    3999              :                  can encode a pointer.  */
    4000           71 :               tree objvar = create_tmp_var (pointer_sized_int_node, "interop");
    4001           71 :               buffer[i + j] = objvar;
    4002           71 :               TREE_ADDRESSABLE (objvar) = 1;
    4003              :               /* Generate a clobber for the temporary for when we're done
    4004              :                  with it.  */
    4005           71 :               tree c = build_clobber (pointer_sized_int_node,
    4006              :                                       CLOBBER_OBJECT_END);
    4007           71 :               c = build2 (MODIFY_EXPR, pointer_sized_int_node, objvar, c);
    4008           71 :               if (clobbers)
    4009           15 :                 clobbers = build2 (COMPOUND_EXPR, TREE_TYPE (clobbers),
    4010              :                                    c, clobbers);
    4011              :               else
    4012              :                 clobbers = c;
    4013              : 
    4014              :               /* objs[offset] = &objvar;  */
    4015           71 :               tree offset = build_int_cst (integer_type_node, j - ninterop);
    4016           71 :               tree init = build4 (ARRAY_REF, objtype, objs, offset,
    4017              :                                   NULL_TREE, NULL_TREE);
    4018           71 :               init = build2 (MODIFY_EXPR, objtype, init,
    4019              :                              build_fold_addr_expr (objvar));
    4020           71 :               if (init_code)
    4021           15 :                 init_code = build2 (COMPOUND_EXPR, TREE_TYPE (init),
    4022              :                                     init_code, init);
    4023              :               else
    4024              :                 init_code = init;
    4025              : 
    4026              :               /* target_tgtsync[offset] = tgt;
    4027              :                  (Don't blame me, I didn't design the encoding of this
    4028              :                  info into the dispatch interop clause data structure,
    4029              :                  but the runtime wants a bit mask.)  */
    4030           71 :               tree tree_tgt = TREE_OPERAND (TREE_PURPOSE (a), 0);
    4031           71 :               int tgt = 0;
    4032           71 :               if (TREE_PURPOSE (tree_tgt) == boolean_true_node)
    4033           42 :                 tgt |= GOMP_INTEROP_TARGET;
    4034           71 :               if (TREE_VALUE (tree_tgt) == boolean_true_node)
    4035           36 :                 tgt |= GOMP_INTEROP_TARGETSYNC;
    4036           71 :               init = build4 (ARRAY_REF, integer_type_node,
    4037              :                              target_tgtsync, offset, NULL_TREE, NULL_TREE);
    4038           71 :               init = build2 (MODIFY_EXPR, integer_type_node, init,
    4039           71 :                              build_int_cst (integer_type_node, tgt));
    4040           71 :               init_code = build2 (COMPOUND_EXPR, TREE_TYPE (init),
    4041              :                                   init_code, init);
    4042              : 
    4043           71 :               if (prefer_type)
    4044              :                 {
    4045           45 :                   tree pref = TREE_VALUE (a);
    4046           45 :                   if (pref == NULL_TREE)
    4047            6 :                     pref = null_pointer_node;
    4048              :                   else
    4049           39 :                     pref = build_fold_addr_expr (pref);
    4050           45 :                   init = build4 (ARRAY_REF, prefer_type_type, prefer_type,
    4051              :                                  offset, NULL_TREE, NULL_TREE);
    4052           45 :                   init = build2 (MODIFY_EXPR, prefer_type_type, init,
    4053              :                                  pref);
    4054           45 :                   init_code = build2 (COMPOUND_EXPR, TREE_TYPE (init),
    4055              :                                       init_code, init);
    4056              :                 }
    4057              :             }
    4058              : 
    4059           56 :           objs = build_fold_addr_expr (objs);
    4060           56 :           target_tgtsync = build_fold_addr_expr (target_tgtsync);
    4061           56 :           prefer_type = prefer_type ? build_fold_addr_expr (prefer_type)
    4062              :                                     : null_pointer_node;
    4063           56 :           tree fn = builtin_decl_explicit (BUILT_IN_GOMP_INTEROP);
    4064           56 :           tree create
    4065           56 :             = build_call_expr_loc (loc, fn, 11, dispatch_device_num,
    4066              :                                    nobjs, objs, target_tgtsync, prefer_type,
    4067              :                                    integer_zero_node, null_pointer_node,
    4068              :                                    integer_zero_node, null_pointer_node,
    4069              :                                    integer_zero_node, null_pointer_node);
    4070           56 :           if (init_code)
    4071           56 :             init_code = build2 (COMPOUND_EXPR, TREE_TYPE (create),
    4072              :                                 init_code, create);
    4073              :           else
    4074              :             init_code = create;
    4075              : 
    4076           56 :           cleanup
    4077           56 :             = build_call_expr_loc (loc, fn, 11, dispatch_device_num,
    4078              :                                    integer_zero_node, null_pointer_node,
    4079              :                                    null_pointer_node, null_pointer_node,
    4080              :                                    integer_zero_node, null_pointer_node,
    4081              :                                    nobjs, objs,
    4082              :                                    integer_zero_node, null_pointer_node);
    4083           56 :           if (clobbers)
    4084           56 :             cleanup = build2 (COMPOUND_EXPR, TREE_TYPE (clobbers),
    4085              :                               cleanup, clobbers);
    4086              :         }
    4087              : 
    4088          654 :       for (j = 0; j < nappend; j++)
    4089              :         {
    4090              :           /* Fortran permits by-reference or by-value for the dummy arg
    4091              :              and by-value, by-reference, ptr by-reference as actual
    4092              :              argument. Handle this.  */
    4093          409 :           tree obj = buffer[i + j];  // interop object
    4094          409 :           tree a2 = TREE_VALUE (arg);  // parameter type
    4095          766 :           if (POINTER_TYPE_P (TREE_TYPE (obj))
    4096          436 :               && POINTER_TYPE_P (TREE_TYPE (TREE_TYPE (obj))))
    4097              :             {
    4098           32 :               tree t = TREE_TYPE (TREE_TYPE (obj));
    4099           32 :               gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (t)));
    4100           32 :               obj = fold_build1 (INDIRECT_REF, t, obj);
    4101              :             }
    4102          750 :           if (POINTER_TYPE_P (TREE_TYPE (obj))
    4103          420 :               && INTEGRAL_TYPE_P (a2))
    4104              :             {
    4105           43 :               tree t = TREE_TYPE (TREE_TYPE (obj));
    4106           43 :               gcc_checking_assert (INTEGRAL_TYPE_P (t));
    4107           43 :               obj = fold_build1 (INDIRECT_REF, t, obj);
    4108              :             }
    4109          497 :           else if (INTEGRAL_TYPE_P (TREE_TYPE (obj))
    4110          461 :                    && POINTER_TYPE_P (a2))
    4111              :             {
    4112           18 :               gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (a2)));
    4113           18 :               obj = build_fold_addr_expr (obj);
    4114              :             }
    4115          348 :           else if (!INTEGRAL_TYPE_P (a2)
    4116          348 :                    || !INTEGRAL_TYPE_P (TREE_TYPE (obj)))
    4117              :             {
    4118           36 :               tree t = TREE_TYPE (obj);
    4119           36 :               gcc_checking_assert (POINTER_TYPE_P (t)
    4120              :                                    && POINTER_TYPE_P (a2)
    4121              :                                    && INTEGRAL_TYPE_P (TREE_TYPE (t))
    4122              :                                    && INTEGRAL_TYPE_P (TREE_TYPE (a2)));
    4123              :             }
    4124          409 :           buffer[i + j] = obj;
    4125          409 :           arg = TREE_CHAIN (arg);
    4126              :         }
    4127          245 :       i += nappend;
    4128          534 :       for (j = nfirst_args; j < nargs; j++)
    4129          289 :         buffer[i++] = CALL_EXPR_ARG (expr, j);
    4130          245 :       nargs += nappend;
    4131          245 :       tree call = expr;
    4132          490 :       expr = build_call_array_loc (EXPR_LOCATION (expr), TREE_TYPE (call),
    4133          245 :                                    CALL_EXPR_FN (call), nargs, buffer);
    4134              : 
    4135              :       /* Copy all CALL_EXPR flags.  */
    4136          245 :       CALL_EXPR_STATIC_CHAIN (expr) = CALL_EXPR_STATIC_CHAIN (call);
    4137          245 :       CALL_EXPR_TAILCALL (expr) = CALL_EXPR_TAILCALL (call);
    4138          245 :       CALL_EXPR_RETURN_SLOT_OPT (expr)
    4139          245 :         = CALL_EXPR_RETURN_SLOT_OPT (call);
    4140          245 :       CALL_FROM_THUNK_P (expr) = CALL_FROM_THUNK_P (call);
    4141          245 :       SET_EXPR_LOCATION (expr, EXPR_LOCATION (call));
    4142          245 :       CALL_EXPR_VA_ARG_PACK (expr) = CALL_EXPR_VA_ARG_PACK (call);
    4143              :     }
    4144              : 
    4145              :   /* Nothing to do for adjust_args?  */
    4146          471 :   if (!dispatch_adjust_args_list || !TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
    4147          260 :     goto add_cleanup;
    4148              : 
    4149              :   /* Handle adjust_args.  */
    4150          966 :   for (int i = 0; i < nargs; i++)
    4151              :     {
    4152          755 :       tree *arg_p = &CALL_EXPR_ARG (expr, i);
    4153              : 
    4154              :       /* Nothing to do if arg is constant null pointer.  */
    4155          755 :       if (integer_zerop (*arg_p))
    4156            6 :         continue;
    4157              : 
    4158              :       bool need_device_ptr = false;
    4159              :       bool need_device_addr = false;
    4160         2247 :       for (int need_addr = 0; need_addr <= 1; need_addr++)
    4161         2301 :         for (tree arg = (need_addr
    4162         1498 :                          ? TREE_VALUE (dispatch_adjust_args_list)
    4163         1498 :                          : TREE_PURPOSE (dispatch_adjust_args_list));
    4164         2301 :              arg != NULL; arg = TREE_CHAIN (arg))
    4165              :           {
    4166         1152 :             if (TREE_VALUE (arg)
    4167         1152 :                 && TREE_CODE (TREE_VALUE (arg)) == INTEGER_CST
    4168         2304 :                 && wi::eq_p (i, wi::to_wide (TREE_VALUE (arg))))
    4169              :               {
    4170          349 :                 if (need_addr)
    4171              :                   need_device_addr = true;
    4172              :                 else
    4173          349 :                   need_device_ptr = true;
    4174              :                 break;
    4175              :               }
    4176              :           }
    4177              : 
    4178          749 :       if (need_device_ptr || need_device_addr)
    4179              :         {
    4180          740 :           bool is_device_ptr = false;
    4181          740 :           bool has_device_addr = false;
    4182              : 
    4183          740 :           for (tree c = dispatch_clauses; c; c = TREE_CHAIN (c))
    4184              :             {
    4185          454 :               if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR
    4186          454 :                   || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_HAS_DEVICE_ADDR)
    4187              :                 {
    4188          164 :                   tree decl1 = DECL_NAME (OMP_CLAUSE_DECL (c));
    4189          164 :                   tree decl2 = tree_strip_nop_conversions (*arg_p);
    4190          164 :                   if (TREE_CODE (decl2) == ADDR_EXPR)
    4191           19 :                     decl2 = TREE_OPERAND (decl2, 0);
    4192          164 :                   if (VAR_P (decl2) || TREE_CODE (decl2) == PARM_DECL)
    4193              :                     {
    4194          140 :                       decl2 = DECL_NAME (decl2);
    4195          140 :                       if (decl1 == decl2
    4196          203 :                           && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IS_DEVICE_PTR)
    4197              :                         {
    4198           58 :                           if (need_device_addr)
    4199            0 :                             warning_at (OMP_CLAUSE_LOCATION (c),
    4200            0 :                                         OPT_Wopenmp,
    4201              :                                         "%<is_device_ptr%> for %qD does"
    4202              :                                         " not imply %<has_device_addr%> "
    4203              :                                         "required for %<need_device_addr%>",
    4204            0 :                                         OMP_CLAUSE_DECL (c));
    4205              :                           is_device_ptr = true;
    4206              :                           break;
    4207              :                         }
    4208           82 :                       else if (decl1 == decl2)
    4209              :                         {
    4210            5 :                           if (need_device_ptr)
    4211           10 :                             warning_at (OMP_CLAUSE_LOCATION (c),
    4212            5 :                                         OPT_Wopenmp,
    4213              :                                         "%<has_device_addr%> for %qD does"
    4214              :                                         " not imply %<is_device_ptr%> "
    4215              :                                         "required for %<need_device_ptr%>",
    4216            5 :                                         OMP_CLAUSE_DECL (c));
    4217              :                           has_device_addr = true;
    4218              :                           break;
    4219              :                         }
    4220              :                     }
    4221              :                 }
    4222              :             }
    4223              : 
    4224          349 :           if ((need_device_ptr && !is_device_ptr)
    4225           58 :               || (need_device_addr && !has_device_addr))
    4226              :             {
    4227          291 :               if (dispatch_device_num == NULL_TREE)
    4228              :                 {
    4229              :                   // device_num = omp_get_default_device ()
    4230          105 :                   tree fn
    4231          105 :                     = builtin_decl_explicit (BUILT_IN_OMP_GET_DEFAULT_DEVICE);
    4232          105 :                   tree call = build_call_expr (fn, 0);
    4233          105 :                   dispatch_device_num = create_tmp_var_raw (TREE_TYPE (call));
    4234          105 :                   tree init
    4235          105 :                     = build4 (TARGET_EXPR, TREE_TYPE (call),
    4236              :                               dispatch_device_num, call, NULL_TREE, NULL_TREE);
    4237          105 :                   if (init_code)
    4238            0 :                     init_code = build2 (COMPOUND_EXPR, TREE_TYPE (init),
    4239              :                                         init_code, init);
    4240              :                   else
    4241              :                     init_code = init;
    4242              :                 }
    4243              : 
    4244              :               // We want to emit the following statement:
    4245              :               //   mapped_arg = omp_get_mapped_ptr (arg,
    4246              :               //                device_num)
    4247              :               // but arg has to be the actual pointer, not a
    4248              :               // reference or a conversion expression.
    4249          291 :               tree actual_ptr
    4250          291 :                 = ((TREE_CODE (*arg_p) == ADDR_EXPR)
    4251          291 :                    ? TREE_OPERAND (*arg_p, 0)
    4252           38 :                    : *arg_p);
    4253          291 :               if (TREE_CODE (actual_ptr) == NOP_EXPR
    4254          291 :                   && (TREE_CODE (TREE_TYPE (TREE_OPERAND (actual_ptr, 0)))
    4255              :                       == REFERENCE_TYPE))
    4256              :                 {
    4257            6 :                   actual_ptr = TREE_OPERAND (actual_ptr, 0);
    4258            6 :                   actual_ptr = build1 (INDIRECT_REF,
    4259            6 :                                        TREE_TYPE (actual_ptr),
    4260              :                                        actual_ptr);
    4261              :                 }
    4262          291 :               tree fn = builtin_decl_explicit (BUILT_IN_OMP_GET_MAPPED_PTR);
    4263          291 :               tree mapped_arg = build_call_expr_loc (loc, fn, 2, actual_ptr,
    4264              :                                                      dispatch_device_num);
    4265              : 
    4266          291 :               if (TREE_CODE (*arg_p) == ADDR_EXPR
    4267          291 :                   || (TREE_CODE (TREE_TYPE (actual_ptr)) == REFERENCE_TYPE))
    4268           47 :                 mapped_arg = build_fold_addr_expr (mapped_arg);
    4269          244 :               else if (TREE_CODE (*arg_p) == NOP_EXPR)
    4270           35 :                 mapped_arg = build1 (NOP_EXPR, TREE_TYPE (*arg_p),
    4271              :                                      mapped_arg);
    4272          291 :               *arg_p = mapped_arg;
    4273              :             }
    4274              :         }
    4275              :     }
    4276              : 
    4277          471 :  add_cleanup:
    4278          471 :   if (cleanup)
    4279              :     {
    4280           56 :       tree result = NULL_TREE;
    4281           56 :       if (want_value && pointerize)
    4282              :         {
    4283            0 :           tree tmp = create_tmp_var (build_pointer_type (TREE_TYPE (expr)),
    4284              :                                      "cleanuptmp");
    4285            0 :           result = build_simple_mem_ref (tmp);
    4286            0 :           expr = build2 (INIT_EXPR, TREE_TYPE (tmp), tmp,
    4287              :                          build_fold_addr_expr (expr));
    4288              :         }
    4289           56 :       else if (want_value)
    4290              :         {
    4291           18 :           tree tmp = create_tmp_var (TREE_TYPE (expr), "cleanuptmp");
    4292           18 :           result = tmp;
    4293           18 :           expr = build2 (INIT_EXPR, TREE_TYPE (tmp), tmp, expr);
    4294              :         }
    4295           56 :       if (init_code)
    4296           56 :         expr = build2 (COMPOUND_EXPR, TREE_TYPE (expr), init_code, expr);
    4297           56 :       expr = build2 (TRY_FINALLY_EXPR, void_type_node, expr, cleanup);
    4298              : 
    4299           56 :       if (result)
    4300           18 :         expr = build2 (COMPOUND_EXPR, TREE_TYPE (result), expr, result);
    4301              :     }
    4302          415 :   else if (init_code)
    4303          105 :     expr = build2 (COMPOUND_EXPR, TREE_TYPE (expr), init_code, expr);
    4304              : 
    4305              :   return expr;
    4306              : }
    4307              : 
    4308              : /* Helper function for gimplify_call_expr: handle "declare variant"
    4309              :    resolution and expansion of the CALL_EXPR EXPR.  WANT_VALUE is true
    4310              :    if the result value of the call is needed; POINTERIZE is true if it
    4311              :    also needs to be pointerized.  If OMP_DISPATCH_P is true, apply
    4312              :    associated transformations using DISPATCH_CLAUSES.
    4313              :    This function may return either the original call or some other
    4314              :    expression such as a conditional to select one of multiple calls.
    4315              : 
    4316              :    FIXME: this function is written to be independent of gimplifier internals
    4317              :    so that it could be moved to omp-general.cc and invoked from the
    4318              :    front ends instead, per PR115076.  */
    4319              : 
    4320              : static tree
    4321         1355 : expand_variant_call_expr (tree expr, bool want_value, bool pointerize,
    4322              :                           bool omp_dispatch_p, tree dispatch_clauses)
    4323              : {
    4324              :   /* If we've already processed this call, stop now.  This can happen
    4325              :      if the variant call resolves to the original function, or to
    4326              :      a dynamic conditional that includes the default call to the original
    4327              :      function.  */
    4328         1355 :   gcc_assert (omp_resolved_variant_calls != NULL);
    4329         1355 :   if (omp_resolved_variant_calls->contains (expr))
    4330          131 :     return expr;
    4331              : 
    4332         1224 :   tree fndecl = get_callee_fndecl (expr);
    4333         1224 :   tree fnptrtype = TREE_TYPE (CALL_EXPR_FN (expr));
    4334         1224 :   location_t loc = EXPR_LOCATION (expr);
    4335         1224 :   tree construct_context = omp_get_construct_context ();
    4336         1224 :   vec<struct omp_variant> all_candidates
    4337         1224 :     = omp_declare_variant_candidates (fndecl, construct_context);
    4338         1224 :   gcc_assert (!all_candidates.is_empty ());
    4339         1224 :   vec<struct omp_variant> candidates
    4340         1224 :     = omp_get_dynamic_candidates (all_candidates, construct_context);
    4341              : 
    4342              :   /* If the variant call could be resolved now, build a nest of COND_EXPRs
    4343              :      if there are dynamic candidates, and/or a new CALL_EXPR for each
    4344              :      candidate call.  */
    4345         1224 :   if (!candidates.is_empty ())
    4346              :     {
    4347         1212 :       int n = candidates.length ();
    4348         1212 :       tree tail = NULL_TREE;
    4349              : 
    4350         2462 :       for (int i = n - 1; i >= 0; i--)
    4351              :         {
    4352         1250 :           if (tail)
    4353           38 :             gcc_assert (candidates[i].dynamic_selector);
    4354              :           else
    4355         1212 :             gcc_assert (!candidates[i].dynamic_selector);
    4356         1250 :           if (candidates[i].alternative == fndecl)
    4357              :             {
    4358              :               /* We should only get the original function back as the
    4359              :                  default.  */
    4360          284 :               gcc_assert (!tail);
    4361          284 :               omp_resolved_variant_calls->add (expr);
    4362          284 :               tail = expr;
    4363              :             }
    4364              :           else
    4365              :             {
    4366              :               /* For the final static selector, we can re-use the old
    4367              :                  CALL_EXPR and just replace the function, unless it may
    4368              :                  need dispatch argument modification.  Otherwise,
    4369              :                  make a copy of it.  */
    4370          966 :               tree thiscall = (tail || omp_dispatch_p
    4371          966 :                                ? unshare_expr (expr) : expr);
    4372          966 :               CALL_EXPR_FN (thiscall) = build1 (ADDR_EXPR, fnptrtype,
    4373          966 :                                                 candidates[i].alternative);
    4374          966 :               if (omp_dispatch_p)
    4375          471 :                 thiscall = modify_call_for_omp_dispatch (thiscall,
    4376              :                                                          dispatch_clauses,
    4377              :                                                          want_value,
    4378              :                                                          pointerize);
    4379          966 :               if (!tail)
    4380              :                 tail = thiscall;
    4381              :               else
    4382           76 :                 tail = build3 (COND_EXPR, TREE_TYPE (expr),
    4383           38 :                                omp_dynamic_cond (candidates[i].selector,
    4384              :                                                  find_supercontext ()),
    4385              :                                thiscall, tail);
    4386              :             }
    4387              :         }
    4388              :       return tail;
    4389              :     }
    4390              : 
    4391              :   /* If we couldn't resolve the variant call now, expand it into a loop using
    4392              :      a switch and OMP_NEXT_VARIANT for dispatch.  The ompdevlow pass will
    4393              :      handle OMP_NEXT_VARIANT expansion.  */
    4394              :   else
    4395              :     {
    4396              :       /* If we need a usable return value, we need a temporary
    4397              :          and an assignment in each alternative.  This logic was borrowed
    4398              :          from gimplify_cond_expr.  */
    4399           12 :       tree type = TREE_TYPE (expr);
    4400           12 :       tree tmp = NULL_TREE, result = NULL_TREE;
    4401              : 
    4402           12 :       if (want_value)
    4403              :         {
    4404           12 :           if (pointerize)
    4405              :             {
    4406            0 :               type = build_pointer_type (type);
    4407            0 :               tmp = create_tmp_var (type, "iftmp");
    4408            0 :               result = build_simple_mem_ref_loc (loc, tmp);
    4409              :             }
    4410              :           else
    4411              :             {
    4412           12 :               tmp = create_tmp_var (type, "iftmp");
    4413           12 :               result = tmp;
    4414              :             }
    4415              :         }
    4416              : 
    4417              :       /* Preprocess the all_candidates array so that the alternative field of
    4418              :          each element holds the actual function call expression and possible
    4419              :          assignment, instead of just the decl for the variant function.  */
    4420           60 :       for (unsigned int i = 0; i < all_candidates.length (); i++)
    4421              :         {
    4422           48 :           tree decl = all_candidates[i].alternative;
    4423           48 :           tree thiscall;
    4424              : 
    4425              :           /* We need to turn the decl from the candidate into a function
    4426              :              call and possible assignment, and stuff that in
    4427              :              the directive seq of the gomp_variant.  */
    4428           48 :           if (decl == fndecl)
    4429              :             {
    4430           12 :               thiscall = expr;
    4431           12 :               omp_resolved_variant_calls->add (expr);
    4432              :             }
    4433              :           else
    4434              :             {
    4435           36 :               thiscall = unshare_expr (expr);
    4436           36 :               CALL_EXPR_FN (thiscall) = build1 (ADDR_EXPR, fnptrtype, decl);
    4437           36 :               if (omp_dispatch_p)
    4438            0 :                 thiscall = modify_call_for_omp_dispatch (thiscall,
    4439              :                                                          dispatch_clauses,
    4440              :                                                          want_value,
    4441              :                                                          pointerize);
    4442              :             }
    4443           48 :           if (pointerize)
    4444            0 :             thiscall = build_fold_addr_expr_loc (loc, thiscall);
    4445           48 :           if (want_value)
    4446           48 :             thiscall = build2 (INIT_EXPR, type, tmp, thiscall);
    4447           48 :           all_candidates[i].alternative = thiscall;
    4448              :         }
    4449              : 
    4450           12 :       cgraph_node::get (cfun->decl)->has_omp_variant_constructs = 1;
    4451           12 :       tree expansion = expand_late_variant_directive (all_candidates,
    4452              :                                                       construct_context);
    4453           12 :       if (result)
    4454           12 :         expansion = build2 (COMPOUND_EXPR, TREE_TYPE (result),
    4455              :                             expansion, result);
    4456              :       return expansion;
    4457              :     }
    4458              : }
    4459              : 
    4460              : /* Wrapper around expand_variant_call_expr to interface with gimplifier
    4461              :    state.  EXPR and OMP_DISPATCH_P are as for expand_variant_call_expr,
    4462              :    FALLBACK is used to compute the WANT_VALUE and POINTERIZE arguments.  */
    4463              : static tree
    4464         1355 : gimplify_variant_call_expr (tree expr, fallback_t fallback,
    4465              :                             bool omp_dispatch_p)
    4466              : {
    4467         1355 :   tree type = TREE_TYPE (expr);
    4468         1355 :   bool want_value = (fallback != fb_none && !VOID_TYPE_P (type));
    4469          673 :   bool pointerize = false;
    4470              :   /* If the result value must be an lvalue or the result type must
    4471              :      live in memory, then we have to pointerize it if we need a temporary.  */
    4472          673 :   if (want_value
    4473          673 :       && ((!(fallback & fb_rvalue) && (fallback & fb_lvalue))
    4474          673 :           || TREE_ADDRESSABLE (type)))
    4475            0 :     pointerize = true;
    4476              : 
    4477         1867 :   return expand_variant_call_expr (expr, want_value, pointerize,
    4478              :                                    omp_dispatch_p,
    4479              :                                    (omp_dispatch_p
    4480          512 :                                     ? gimplify_omp_ctxp->clauses
    4481         1355 :                                     : NULL_TREE));
    4482              : }
    4483              : 
    4484              : 
    4485              : /* Helper function for gimplify_call_expr, called via walk_tree.
    4486              :    Find used user labels.  */
    4487              : 
    4488              : static tree
    4489          766 : find_used_user_labels (tree *tp, int *, void *)
    4490              : {
    4491          766 :   if (TREE_CODE (*tp) == LABEL_EXPR
    4492           15 :       && !DECL_ARTIFICIAL (LABEL_EXPR_LABEL (*tp))
    4493           15 :       && DECL_NAME (LABEL_EXPR_LABEL (*tp))
    4494          781 :       && TREE_USED (LABEL_EXPR_LABEL (*tp)))
    4495           15 :     return *tp;
    4496              :   return NULL_TREE;
    4497              : }
    4498              : 
    4499              : 
    4500              : /* Gimplify the CALL_EXPR node *EXPR_P into the GIMPLE sequence PRE_P.
    4501              :    WANT_VALUE is true if the result of the call is desired.  */
    4502              : 
    4503              : static enum gimplify_status
    4504     17413258 : gimplify_call_expr (tree *expr_p, gimple_seq *pre_p, fallback_t fallback)
    4505              : {
    4506     17413258 :   bool want_value = (fallback != fb_none);
    4507     17413258 :   tree fndecl, parms, p, fnptrtype;
    4508     17413258 :   enum gimplify_status ret;
    4509     17413258 :   int i, nargs;
    4510     17413258 :   gcall *call;
    4511     17413258 :   bool builtin_va_start_p = false, omp_dispatch_p = false;
    4512     17413258 :   location_t loc = EXPR_LOCATION (*expr_p);
    4513              : 
    4514     17413258 :   gcc_assert (TREE_CODE (*expr_p) == CALL_EXPR);
    4515              : 
    4516              :   /* For reliable diagnostics during inlining, it is necessary that
    4517              :      every call_expr be annotated with file and line.  */
    4518     17413258 :   if (! EXPR_HAS_LOCATION (*expr_p))
    4519       519719 :     SET_EXPR_LOCATION (*expr_p, input_location);
    4520              : 
    4521              :   /* Gimplify internal functions created in the FEs.  */
    4522     17413258 :   if (CALL_EXPR_FN (*expr_p) == NULL_TREE)
    4523              :     {
    4524       609304 :       enum internal_fn ifn = CALL_EXPR_IFN (*expr_p);
    4525       609304 :       if (ifn == IFN_GOMP_DISPATCH)
    4526              :         {
    4527          884 :           gcc_assert (flag_openmp
    4528              :                       && gimplify_omp_ctxp
    4529              :                       && gimplify_omp_ctxp->code == OMP_DISPATCH);
    4530          884 :           *expr_p = CALL_EXPR_ARG (*expr_p, 0);
    4531          884 :           gcc_assert (TREE_CODE (*expr_p) == CALL_EXPR);
    4532          884 :           if (! EXPR_HAS_LOCATION (*expr_p))
    4533           94 :             SET_EXPR_LOCATION (*expr_p, input_location);
    4534              :           omp_dispatch_p = true;
    4535              :         }
    4536              :       else
    4537              :         {
    4538       608420 :           if (want_value)
    4539              :             return GS_ALL_DONE;
    4540              : 
    4541        21214 :           nargs = call_expr_nargs (*expr_p);
    4542        21214 :           auto_vec<tree> vargs (nargs);
    4543              : 
    4544        21214 :           if (ifn == IFN_ASSUME)
    4545              :             {
    4546          574 :               if (simple_condition_p (CALL_EXPR_ARG (*expr_p, 0)))
    4547              :                 {
    4548              :                   /* If the [[assume (cond)]]; condition is simple
    4549              :                      enough and can be evaluated unconditionally
    4550              :                      without side-effects, expand it as
    4551              :                      if (!cond) __builtin_unreachable ();  */
    4552          312 :                   tree fndecl = builtin_decl_explicit (BUILT_IN_UNREACHABLE);
    4553          312 :                   *expr_p
    4554          624 :                     = build3 (COND_EXPR, void_type_node,
    4555          312 :                               CALL_EXPR_ARG (*expr_p, 0), void_node,
    4556          312 :                               build_call_expr_loc (EXPR_LOCATION (*expr_p),
    4557              :                                                    fndecl, 0));
    4558          312 :                   return GS_OK;
    4559              :                 }
    4560              :               /* If not optimizing, ignore the assumptions unless there
    4561              :                  are used user labels in it.  */
    4562          262 :               if ((!optimize
    4563           98 :                    && !walk_tree_without_duplicates (&CALL_EXPR_ARG (*expr_p,
    4564              :                                                                      0),
    4565              :                                                      find_used_user_labels,
    4566              :                                                      NULL))
    4567          277 :                   || seen_error ())
    4568              :                 {
    4569           94 :                   *expr_p = NULL_TREE;
    4570           94 :                   return GS_ALL_DONE;
    4571              :                 }
    4572              :               /* Temporarily, until gimple lowering, transform
    4573              :                  .ASSUME (cond);
    4574              :                  into:
    4575              :                  [[assume (guard)]]
    4576              :                  {
    4577              :                    guard = cond;
    4578              :                  }
    4579              :                  such that gimple lowering can outline the condition into
    4580              :                  a separate function easily.  */
    4581          168 :               tree guard = create_tmp_var (boolean_type_node);
    4582          168 :               *expr_p = build2 (MODIFY_EXPR, void_type_node, guard,
    4583          168 :                                 gimple_boolify (CALL_EXPR_ARG (*expr_p, 0)));
    4584          168 :               *expr_p = build3 (BIND_EXPR, void_type_node, NULL, *expr_p, NULL);
    4585          168 :               push_gimplify_context ();
    4586          168 :               gimple_seq body = NULL;
    4587          168 :               gimple *g = gimplify_and_return_first (*expr_p, &body);
    4588          168 :               pop_gimplify_context (g);
    4589          168 :               g = gimple_build_assume (guard, body);
    4590          168 :               gimple_set_location (g, loc);
    4591          168 :               gimplify_seq_add_stmt (pre_p, g);
    4592          168 :               *expr_p = NULL_TREE;
    4593          168 :               return GS_ALL_DONE;
    4594              :             }
    4595        20640 :           else if (ifn == IFN_UBSAN_BOUNDS
    4596        20640 :                    && nargs == 3
    4597        20640 :                    && integer_onep (CALL_EXPR_ARG (*expr_p, 0)))
    4598              :             {
    4599              :               /* If first argument is one, add TYPE_MAX_VALUE (TYPE_DOMAIN (t))
    4600              :                  to 3rd argument and change first argument to 0.  This is
    4601              :                  done by ubsan_instrument_bounds so that we can use the
    4602              :                  max value from gimplify_type_sizes here instead of original
    4603              :                  expression for VLAs.  */
    4604          406 :               tree type = TREE_TYPE (CALL_EXPR_ARG (*expr_p, 0));
    4605          406 :               CALL_EXPR_ARG (*expr_p, 0) = build_int_cst (type, 0);
    4606          406 :               gcc_assert (TREE_CODE (type) == POINTER_TYPE);
    4607          406 :               type = TREE_TYPE (type);
    4608          406 :               gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
    4609          406 :               tree maxv = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
    4610          406 :               gcc_assert (maxv);
    4611          406 :               tree arg3 = CALL_EXPR_ARG (*expr_p, 2);
    4612          406 :               CALL_EXPR_ARG (*expr_p, 2)
    4613          812 :                 = fold_build2 (PLUS_EXPR, TREE_TYPE (arg3), maxv, arg3);
    4614              :             }
    4615              : 
    4616        46429 :           for (i = 0; i < nargs; i++)
    4617              :             {
    4618        25789 :               gimplify_arg (&CALL_EXPR_ARG (*expr_p, i), pre_p,
    4619        25789 :                             EXPR_LOCATION (*expr_p));
    4620        25789 :               vargs.quick_push (CALL_EXPR_ARG (*expr_p, i));
    4621              :             }
    4622              : 
    4623        20640 :           gcall *call = gimple_build_call_internal_vec (ifn, vargs);
    4624        20640 :           gimple_call_set_nothrow (call, TREE_NOTHROW (*expr_p));
    4625        20640 :           gimplify_seq_add_stmt (pre_p, call);
    4626        20640 :           return GS_ALL_DONE;
    4627        21214 :         }
    4628              :     }
    4629              : 
    4630              :   /* This may be a call to a builtin function.
    4631              : 
    4632              :      Builtin function calls may be transformed into different
    4633              :      (and more efficient) builtin function calls under certain
    4634              :      circumstances.  Unfortunately, gimplification can muck things
    4635              :      up enough that the builtin expanders are not aware that certain
    4636              :      transformations are still valid.
    4637              : 
    4638              :      So we attempt transformation/gimplification of the call before
    4639              :      we gimplify the CALL_EXPR.  At this time we do not manage to
    4640              :      transform all calls in the same manner as the expanders do, but
    4641              :      we do transform most of them.  */
    4642     16804838 :   fndecl = get_callee_fndecl (*expr_p);
    4643     16804838 :   if (fndecl && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL))
    4644      3998389 :     switch (DECL_FUNCTION_CODE (fndecl))
    4645              :       {
    4646        95112 :       CASE_BUILT_IN_ALLOCA:
    4647              :         /* If the call has been built for a variable-sized object, then we
    4648              :            want to restore the stack level when the enclosing BIND_EXPR is
    4649              :            exited to reclaim the allocated space; otherwise, we precisely
    4650              :            need to do the opposite and preserve the latest stack level.  */
    4651        95112 :         if (CALL_ALLOCA_FOR_VAR_P (*expr_p))
    4652         9041 :           gimplify_ctxp->save_stack = true;
    4653              :         else
    4654        86071 :           gimplify_ctxp->keep_stack = true;
    4655              :         break;
    4656              : 
    4657        20905 :       case BUILT_IN_VA_START:
    4658        20905 :         {
    4659        20905 :           builtin_va_start_p = true;
    4660        20905 :           if (call_expr_nargs (*expr_p) < 2)
    4661              :             {
    4662            1 :               error ("too few arguments to function %<va_start%>");
    4663            1 :               *expr_p = build_empty_stmt (EXPR_LOCATION (*expr_p));
    4664            1 :               return GS_OK;
    4665              :             }
    4666              : 
    4667        20904 :           if (fold_builtin_next_arg (*expr_p, true))
    4668              :             {
    4669            8 :               *expr_p = build_empty_stmt (EXPR_LOCATION (*expr_p));
    4670            8 :               return GS_OK;
    4671              :             }
    4672              :           break;
    4673              :         }
    4674              : 
    4675           32 :       case BUILT_IN_EH_RETURN:
    4676           32 :         cfun->calls_eh_return = true;
    4677           32 :         break;
    4678              : 
    4679         1036 :       case BUILT_IN_CLEAR_PADDING:
    4680         1036 :         if (call_expr_nargs (*expr_p) == 1)
    4681              :           {
    4682              :             /* Remember the original type of the argument in an internal
    4683              :                dummy second argument, as in GIMPLE pointer conversions are
    4684              :                useless.  Also mark this call as not for automatic
    4685              :                initialization in the internal dummy third argument.  */
    4686          518 :             p = CALL_EXPR_ARG (*expr_p, 0);
    4687          518 :             *expr_p
    4688          518 :               = build_call_expr_loc (EXPR_LOCATION (*expr_p), fndecl, 2, p,
    4689          518 :                                      build_zero_cst (TREE_TYPE (p)));
    4690          518 :             return GS_OK;
    4691              :           }
    4692              :         break;
    4693              : 
    4694              :       default:
    4695              :         ;
    4696              :       }
    4697     16804311 :   if (fndecl && fndecl_built_in_p (fndecl))
    4698              :     {
    4699      4239413 :       tree new_tree = fold_call_expr (input_location, *expr_p, !want_value);
    4700      4239413 :       if (new_tree && new_tree != *expr_p)
    4701              :         {
    4702              :           /* There was a transformation of this call which computes the
    4703              :              same value, but in a more efficient way.  Return and try
    4704              :              again.  */
    4705          773 :           *expr_p = new_tree;
    4706          773 :           return GS_OK;
    4707              :         }
    4708              :     }
    4709              : 
    4710              :   /* Remember the original function pointer type.  */
    4711     16803538 :   fnptrtype = TREE_TYPE (CALL_EXPR_FN (*expr_p));
    4712              : 
    4713              :   /* Handle "declare variant" resolution and arglist processing.  */
    4714     16803538 :   if (flag_openmp
    4715       399306 :       && fndecl
    4716       396621 :       && cfun
    4717       396621 :       && (cfun->curr_properties & PROP_gimple_any) == 0
    4718       257165 :       && !omp_has_novariants ()
    4719     17060591 :       && lookup_attribute ("omp declare variant base",
    4720       257053 :                            DECL_ATTRIBUTES (fndecl)))
    4721              :     {
    4722         1355 :       tree orig = *expr_p;
    4723         1355 :       *expr_p = gimplify_variant_call_expr (*expr_p, fallback,
    4724              :                                             omp_dispatch_p);
    4725              : 
    4726              :       /* This may resolve to the same call, or the call expr with just
    4727              :          the function replaced, in which case we should just continue to
    4728              :          gimplify it normally.  Otherwise, if we get something else back,
    4729              :          stop here and re-gimplify the whole replacement expr.  */
    4730         1355 :       if (*expr_p != orig)
    4731              :         return GS_OK;
    4732              :     }
    4733              : 
    4734              :   /* There is a sequence point before the call, so any side effects in
    4735              :      the calling expression must occur before the actual call.  Force
    4736              :      gimplify_expr to use an internal post queue.  */
    4737     16803049 :   ret = gimplify_expr (&CALL_EXPR_FN (*expr_p), pre_p, NULL,
    4738              :                        is_gimple_call_addr, fb_rvalue);
    4739              : 
    4740     16803049 :   if (ret == GS_ERROR)
    4741              :     return GS_ERROR;
    4742              : 
    4743     16803047 :   nargs = call_expr_nargs (*expr_p);
    4744              : 
    4745              :   /* Get argument types for verification.  */
    4746     16803047 :   fndecl = get_callee_fndecl (*expr_p);
    4747     16803047 :   parms = NULL_TREE;
    4748     16803047 :   if (fndecl)
    4749     16576636 :     parms = TYPE_ARG_TYPES (TREE_TYPE (fndecl));
    4750              :   else
    4751       226411 :     parms = TYPE_ARG_TYPES (TREE_TYPE (fnptrtype));
    4752              : 
    4753     33379683 :   if (fndecl && DECL_ARGUMENTS (fndecl))
    4754              :     p = DECL_ARGUMENTS (fndecl);
    4755      7424923 :   else if (parms)
    4756              :     p = parms;
    4757              :   else
    4758              :     p = NULL_TREE;
    4759     44764715 :   for (i = 0; i < nargs && p; i++, p = TREE_CHAIN (p))
    4760              :     ;
    4761              : 
    4762              :   /* If the last argument is __builtin_va_arg_pack () and it is not
    4763              :      passed as a named argument, decrease the number of CALL_EXPR
    4764              :      arguments and set instead the CALL_EXPR_VA_ARG_PACK flag.  */
    4765     16803047 :   if (!p
    4766     16803047 :       && i < nargs
    4767     16803047 :       && TREE_CODE (CALL_EXPR_ARG (*expr_p, nargs - 1)) == CALL_EXPR)
    4768              :     {
    4769         5971 :       tree last_arg = CALL_EXPR_ARG (*expr_p, nargs - 1);
    4770         5971 :       tree last_arg_fndecl = get_callee_fndecl (last_arg);
    4771              : 
    4772         5971 :       if (last_arg_fndecl
    4773         5971 :           && fndecl_built_in_p (last_arg_fndecl, BUILT_IN_VA_ARG_PACK))
    4774              :         {
    4775          132 :           tree call = *expr_p;
    4776              : 
    4777          132 :           --nargs;
    4778          264 :           *expr_p = build_call_array_loc (loc, TREE_TYPE (call),
    4779          132 :                                           CALL_EXPR_FN (call),
    4780          132 :                                           nargs, CALL_EXPR_ARGP (call));
    4781              : 
    4782              :           /* Copy all CALL_EXPR flags, location and block, except
    4783              :              CALL_EXPR_VA_ARG_PACK flag.  */
    4784          132 :           CALL_EXPR_STATIC_CHAIN (*expr_p) = CALL_EXPR_STATIC_CHAIN (call);
    4785          132 :           CALL_EXPR_TAILCALL (*expr_p) = CALL_EXPR_TAILCALL (call);
    4786          132 :           CALL_EXPR_RETURN_SLOT_OPT (*expr_p)
    4787          132 :             = CALL_EXPR_RETURN_SLOT_OPT (call);
    4788          132 :           CALL_FROM_THUNK_P (*expr_p) = CALL_FROM_THUNK_P (call);
    4789          132 :           SET_EXPR_LOCATION (*expr_p, EXPR_LOCATION (call));
    4790              : 
    4791              :           /* Set CALL_EXPR_VA_ARG_PACK.  */
    4792          132 :           CALL_EXPR_VA_ARG_PACK (*expr_p) = 1;
    4793              :         }
    4794              :     }
    4795              : 
    4796              :   /* If the call returns twice then after building the CFG the call
    4797              :      argument computations will no longer dominate the call because
    4798              :      we add an abnormal incoming edge to the call.  So do not use SSA
    4799              :      vars there.  */
    4800     16803047 :   bool returns_twice = call_expr_flags (*expr_p) & ECF_RETURNS_TWICE;
    4801              : 
    4802              : 
    4803              :   /* Gimplify the function arguments.  */
    4804     16803047 :   if (nargs > 0)
    4805              :     {
    4806     14691794 :       for (i = (PUSH_ARGS_REVERSED ? nargs - 1 : 0);
    4807     44142974 :            PUSH_ARGS_REVERSED ? i >= 0 : i < nargs;
    4808     29451180 :            PUSH_ARGS_REVERSED ? i-- : i++)
    4809              :         {
    4810     29451180 :           enum gimplify_status t;
    4811              : 
    4812              :           /* Avoid gimplifying the second argument to va_start, which needs to
    4813              :              be the plain PARM_DECL.  */
    4814     29451180 :           if ((i != 1) || !builtin_va_start_p)
    4815              :             {
    4816     29430284 :               tree *arg_p = &CALL_EXPR_ARG (*expr_p, i);
    4817              : 
    4818     29430284 :               if (gimplify_omp_ctxp && gimplify_omp_ctxp->code == OMP_DISPATCH)
    4819         6687 :                 gimplify_omp_ctxp->in_call_args = true;
    4820     29430284 :               t = gimplify_arg (arg_p, pre_p, EXPR_LOCATION (*expr_p),
    4821              :                                 !returns_twice);
    4822     29430284 :               if (gimplify_omp_ctxp && gimplify_omp_ctxp->code == OMP_DISPATCH)
    4823         6687 :                 gimplify_omp_ctxp->in_call_args = false;
    4824              : 
    4825     29430284 :               if (t == GS_ERROR)
    4826     29451180 :                 ret = GS_ERROR;
    4827              :             }
    4828              :         }
    4829              :     }
    4830              : 
    4831              :   /* Gimplify the static chain.  */
    4832     16803047 :   if (CALL_EXPR_STATIC_CHAIN (*expr_p))
    4833              :     {
    4834        28853 :       if (fndecl && !DECL_STATIC_CHAIN (fndecl))
    4835            3 :         CALL_EXPR_STATIC_CHAIN (*expr_p) = NULL;
    4836              :       else
    4837              :         {
    4838        28174 :           enum gimplify_status t;
    4839        28174 :           t = gimplify_arg (&CALL_EXPR_STATIC_CHAIN (*expr_p), pre_p,
    4840        28174 :                             EXPR_LOCATION (*expr_p), ! returns_twice);
    4841        28174 :           if (t == GS_ERROR)
    4842     16803047 :             ret = GS_ERROR;
    4843              :         }
    4844              :     }
    4845              : 
    4846              :   /* Verify the function result.  */
    4847     16803047 :   if (want_value && fndecl
    4848     16803047 :       && VOID_TYPE_P (TREE_TYPE (TREE_TYPE (fnptrtype))))
    4849              :     {
    4850            0 :       error_at (loc, "using result of function returning %<void%>");
    4851            0 :       ret = GS_ERROR;
    4852              :     }
    4853              : 
    4854              :   /* Try this again in case gimplification exposed something.  */
    4855     16803047 :   if (ret != GS_ERROR)
    4856              :     {
    4857     16802897 :       tree new_tree = fold_call_expr (input_location, *expr_p, !want_value);
    4858              : 
    4859     16802897 :       if (new_tree && new_tree != *expr_p)
    4860              :         {
    4861              :           /* There was a transformation of this call which computes the
    4862              :              same value, but in a more efficient way.  Return and try
    4863              :              again.  */
    4864        11459 :           *expr_p = new_tree;
    4865        11459 :           return GS_OK;
    4866              :         }
    4867              :     }
    4868              :   else
    4869              :     {
    4870          150 :       *expr_p = error_mark_node;
    4871          150 :       return GS_ERROR;
    4872              :     }
    4873              : 
    4874              :   /* If the function is "const" or "pure", then clear TREE_SIDE_EFFECTS on its
    4875              :      decl.  This allows us to eliminate redundant or useless
    4876              :      calls to "const" functions.  */
    4877     16791438 :   if (TREE_CODE (*expr_p) == CALL_EXPR)
    4878              :     {
    4879     16791438 :       int flags = call_expr_flags (*expr_p);
    4880     16791438 :       if (flags & (ECF_CONST | ECF_PURE)
    4881              :           /* An infinite loop is considered a side effect.  */
    4882      2450925 :           && !(flags & (ECF_LOOPING_CONST_OR_PURE)))
    4883      2050326 :         TREE_SIDE_EFFECTS (*expr_p) = 0;
    4884              :     }
    4885              : 
    4886              :   /* If the value is not needed by the caller, emit a new GIMPLE_CALL
    4887              :      and clear *EXPR_P.  Otherwise, leave *EXPR_P in its gimplified
    4888              :      form and delegate the creation of a GIMPLE_CALL to
    4889              :      gimplify_modify_expr.  This is always possible because when
    4890              :      WANT_VALUE is true, the caller wants the result of this call into
    4891              :      a temporary, which means that we will emit an INIT_EXPR in
    4892              :      internal_get_tmp_var which will then be handled by
    4893              :      gimplify_modify_expr.  */
    4894     16791438 :   if (!want_value)
    4895              :     {
    4896              :       /* The CALL_EXPR in *EXPR_P is already in GIMPLE form, so all we
    4897              :          have to do is replicate it as a GIMPLE_CALL tuple.  */
    4898      6398332 :       gimple_stmt_iterator gsi;
    4899      6398332 :       call = gimple_build_call_from_tree (*expr_p, fnptrtype);
    4900      6398332 :       notice_special_calls (call);
    4901      6398332 :       gimplify_seq_add_stmt (pre_p, call);
    4902      6398332 :       gsi = gsi_last (*pre_p);
    4903      6398332 :       maybe_fold_stmt (&gsi);
    4904      6398332 :       *expr_p = NULL_TREE;
    4905              :     }
    4906              :   else
    4907              :     /* Remember the original function type.  */
    4908     10393106 :     CALL_EXPR_FN (*expr_p) = build1 (NOP_EXPR, fnptrtype,
    4909     10393106 :                                      CALL_EXPR_FN (*expr_p));
    4910              : 
    4911              :   return ret;
    4912              : }
    4913              : 
    4914              : /* Handle shortcut semantics in the predicate operand of a COND_EXPR by
    4915              :    rewriting it into multiple COND_EXPRs, and possibly GOTO_EXPRs.
    4916              : 
    4917              :    TRUE_LABEL_P and FALSE_LABEL_P point to the labels to jump to if the
    4918              :    condition is true or false, respectively.  If null, we should generate
    4919              :    our own to skip over the evaluation of this specific expression.
    4920              : 
    4921              :    LOCUS is the source location of the COND_EXPR.
    4922              : 
    4923              :    The condition_uid is a discriminator tag for condition coverage used to map
    4924              :    conditions to its corresponding full Boolean function.
    4925              : 
    4926              :    This function is the tree equivalent of do_jump.
    4927              : 
    4928              :    shortcut_cond_r should only be called by shortcut_cond_expr.  */
    4929              : 
    4930              : static tree
    4931      1333345 : shortcut_cond_r (tree pred, tree *true_label_p, tree *false_label_p,
    4932              :                  location_t locus, unsigned condition_uid)
    4933              : {
    4934      1333345 :   tree local_label = NULL_TREE;
    4935      1333345 :   tree t, expr = NULL;
    4936              : 
    4937              :   /* OK, it's not a simple case; we need to pull apart the COND_EXPR to
    4938              :      retain the shortcut semantics.  Just insert the gotos here;
    4939              :      shortcut_cond_expr will append the real blocks later.  */
    4940      1333345 :   if (TREE_CODE (pred) == TRUTH_ANDIF_EXPR)
    4941              :     {
    4942       124417 :       location_t new_locus;
    4943              : 
    4944              :       /* Turn if (a && b) into
    4945              : 
    4946              :          if (a); else goto no;
    4947              :          if (b) goto yes; else goto no;
    4948              :          (no:) */
    4949              : 
    4950       124417 :       if (false_label_p == NULL)
    4951         4022 :         false_label_p = &local_label;
    4952              : 
    4953              :       /* Keep the original source location on the first 'if'.  */
    4954       124417 :       t = shortcut_cond_r (TREE_OPERAND (pred, 0), NULL, false_label_p, locus,
    4955              :                            condition_uid);
    4956       124417 :       append_to_statement_list (t, &expr);
    4957              : 
    4958              :       /* Set the source location of the && on the second 'if'.  */
    4959       124417 :       new_locus = rexpr_location (pred, locus);
    4960       124417 :       t = shortcut_cond_r (TREE_OPERAND (pred, 1), true_label_p, false_label_p,
    4961              :                            new_locus, condition_uid);
    4962       124417 :       append_to_statement_list (t, &expr);
    4963              :     }
    4964      1208928 :   else if (TREE_CODE (pred) == TRUTH_ORIF_EXPR)
    4965              :     {
    4966       386308 :       location_t new_locus;
    4967              : 
    4968              :       /* Turn if (a || b) into
    4969              : 
    4970              :          if (a) goto yes;
    4971              :          if (b) goto yes; else goto no;
    4972              :          (yes:) */
    4973              : 
    4974       386308 :       if (true_label_p == NULL)
    4975       231597 :         true_label_p = &local_label;
    4976              : 
    4977              :       /* Keep the original source location on the first 'if'.  */
    4978       386308 :       t = shortcut_cond_r (TREE_OPERAND (pred, 0), true_label_p, NULL, locus,
    4979              :                            condition_uid);
    4980       386308 :       append_to_statement_list (t, &expr);
    4981              : 
    4982              :       /* Set the source location of the || on the second 'if'.  */
    4983       386308 :       new_locus = rexpr_location (pred, locus);
    4984       386308 :       t = shortcut_cond_r (TREE_OPERAND (pred, 1), true_label_p, false_label_p,
    4985              :                            new_locus, condition_uid);
    4986       386308 :       append_to_statement_list (t, &expr);
    4987              :     }
    4988       822620 :   else if (TREE_CODE (pred) == COND_EXPR
    4989          852 :            && !VOID_TYPE_P (TREE_TYPE (TREE_OPERAND (pred, 1)))
    4990       823472 :            && !VOID_TYPE_P (TREE_TYPE (TREE_OPERAND (pred, 2))))
    4991              :     {
    4992          846 :       location_t new_locus;
    4993              : 
    4994              :       /* As long as we're messing with gotos, turn if (a ? b : c) into
    4995              :          if (a)
    4996              :            if (b) goto yes; else goto no;
    4997              :          else
    4998              :            if (c) goto yes; else goto no;
    4999              : 
    5000              :          Don't do this if one of the arms has void type, which can happen
    5001              :          in C++ when the arm is throw.  */
    5002              : 
    5003              :       /* Keep the original source location on the first 'if'.  Set the source
    5004              :          location of the ? on the second 'if'.  */
    5005          846 :       new_locus = rexpr_location (pred, locus);
    5006         1692 :       expr = build3 (COND_EXPR, void_type_node, TREE_OPERAND (pred, 0),
    5007          846 :                      shortcut_cond_r (TREE_OPERAND (pred, 1), true_label_p,
    5008              :                                       false_label_p, locus, condition_uid),
    5009          846 :                      shortcut_cond_r (TREE_OPERAND (pred, 2), true_label_p,
    5010              :                                       false_label_p, new_locus,
    5011              :                                       condition_uid));
    5012          846 :       tree_associate_condition_with_expr (expr, condition_uid);
    5013              :     }
    5014              :   else
    5015              :     {
    5016       821774 :       expr = build3 (COND_EXPR, void_type_node, pred,
    5017              :                      build_and_jump (true_label_p),
    5018              :                      build_and_jump (false_label_p));
    5019       821774 :       SET_EXPR_LOCATION (expr, locus);
    5020       821774 :       tree_associate_condition_with_expr (expr, condition_uid);
    5021              :     }
    5022              : 
    5023      1333345 :   if (local_label)
    5024              :     {
    5025       235619 :       t = build1 (LABEL_EXPR, void_type_node, local_label);
    5026       235619 :       append_to_statement_list (t, &expr);
    5027              :     }
    5028              : 
    5029      1333345 :   return expr;
    5030              : }
    5031              : 
    5032              : /* If EXPR is a GOTO_EXPR, return it.  If it is a STATEMENT_LIST, skip
    5033              :    any of its leading DEBUG_BEGIN_STMTS and recurse on the subsequent
    5034              :    statement, if it is the last one.  Otherwise, return NULL.  */
    5035              : 
    5036              : static tree
    5037     11722832 : find_goto (tree expr)
    5038              : {
    5039     12051906 :   if (!expr)
    5040              :     return NULL_TREE;
    5041              : 
    5042     10083063 :   if (TREE_CODE (expr) == GOTO_EXPR)
    5043              :     return expr;
    5044              : 
    5045      8038118 :   if (TREE_CODE (expr) != STATEMENT_LIST)
    5046              :     return NULL_TREE;
    5047              : 
    5048      1050456 :   tree_stmt_iterator i = tsi_start (expr);
    5049              : 
    5050      1407118 :   while (!tsi_end_p (i) && TREE_CODE (tsi_stmt (i)) == DEBUG_BEGIN_STMT)
    5051       356662 :     tsi_next (&i);
    5052              : 
    5053      1050456 :   if (!tsi_one_before_end_p (i))
    5054              :     return NULL_TREE;
    5055              : 
    5056       329074 :   return find_goto (tsi_stmt (i));
    5057              : }
    5058              : 
    5059              : /* Same as find_goto, except that it returns NULL if the destination
    5060              :    is not a LABEL_DECL.  */
    5061              : 
    5062              : static inline tree
    5063     11722832 : find_goto_label (tree expr)
    5064              : {
    5065     11722832 :   tree dest = find_goto (expr);
    5066     13767777 :   if (dest && TREE_CODE (GOTO_DESTINATION (dest)) == LABEL_DECL)
    5067      2044834 :     return dest;
    5068              :   return NULL_TREE;
    5069              : }
    5070              : 
    5071              : 
    5072              : /* Given a multi-term condition (ANDIF, ORIF), walk the predicate PRED and tag
    5073              :    every basic condition with CONDITION_UID.  Two basic conditions share the
    5074              :    CONDITION_UID discriminator when they belong to the same predicate, which is
    5075              :    used by the condition coverage.  Doing this as an explicit step makes for a
    5076              :    simpler implementation than weaving it into the splitting code as the
    5077              :    splitting code eventually calls the entry point gimplfiy_expr which makes
    5078              :    bookkeeping complicated.  */
    5079              : static void
    5080       576682 : tag_shortcut_cond (tree pred, unsigned condition_uid)
    5081              : {
    5082       616243 :   if (TREE_CODE (pred) == TRUTH_ANDIF_EXPR
    5083       616243 :       || TREE_CODE (pred) == TRUTH_ORIF_EXPR)
    5084              :     {
    5085       565752 :       tree fst = TREE_OPERAND (pred, 0);
    5086       565752 :       tree lst = TREE_OPERAND (pred, 1);
    5087              : 
    5088       565752 :       if (TREE_CODE (fst) == TRUTH_ANDIF_EXPR
    5089       565752 :           || TREE_CODE (fst) == TRUTH_ORIF_EXPR)
    5090       168982 :         tag_shortcut_cond (fst, condition_uid);
    5091       396770 :       else if (TREE_CODE (fst) == COND_EXPR)
    5092          244 :         tree_associate_condition_with_expr (fst, condition_uid);
    5093              : 
    5094       565752 :       if (TREE_CODE (lst) == TRUTH_ANDIF_EXPR
    5095       565752 :           || TREE_CODE (lst) == TRUTH_ORIF_EXPR)
    5096              :         tag_shortcut_cond (lst, condition_uid);
    5097       526191 :       else if (TREE_CODE (lst) == COND_EXPR)
    5098          445 :         tree_associate_condition_with_expr (lst, condition_uid);
    5099              :     }
    5100       576682 : }
    5101              : 
    5102              : /* Given a conditional expression EXPR with short-circuit boolean
    5103              :    predicates using TRUTH_ANDIF_EXPR or TRUTH_ORIF_EXPR, break the
    5104              :    predicate apart into the equivalent sequence of conditionals.  CONDITION_UID
    5105              :    is a the tag/discriminator for this EXPR - all basic conditions in the
    5106              :    expression will be given the same CONDITION_UID.  */
    5107              : static tree
    5108       407700 : shortcut_cond_expr (tree expr, unsigned condition_uid)
    5109              : {
    5110       407700 :   tree pred = TREE_OPERAND (expr, 0);
    5111       407700 :   tree then_ = TREE_OPERAND (expr, 1);
    5112       407700 :   tree else_ = TREE_OPERAND (expr, 2);
    5113       407700 :   tree true_label, false_label, end_label, t;
    5114       407700 :   tree *true_label_p;
    5115       407700 :   tree *false_label_p;
    5116       407700 :   bool emit_end, emit_false, jump_over_else;
    5117       407700 :   bool then_se = then_ && TREE_SIDE_EFFECTS (then_);
    5118       407700 :   bool else_se = else_ && TREE_SIDE_EFFECTS (else_);
    5119              : 
    5120       407700 :   tag_shortcut_cond (pred, condition_uid);
    5121              : 
    5122              :   /* First do simple transformations.  */
    5123       407700 :   if (!else_se)
    5124              :     {
    5125              :       /* If there is no 'else', turn
    5126              :            if (a && b) then c
    5127              :          into
    5128              :            if (a) if (b) then c.  */
    5129       294911 :       while (TREE_CODE (pred) == TRUTH_ANDIF_EXPR)
    5130              :         {
    5131              :           /* Keep the original source location on the first 'if'.  */
    5132        51544 :           location_t locus = EXPR_LOC_OR_LOC (expr, input_location);
    5133        51544 :           TREE_OPERAND (expr, 0) = TREE_OPERAND (pred, 1);
    5134              :           /* Set the source location of the && on the second 'if'.  */
    5135        51544 :           if (rexpr_has_location (pred))
    5136        50520 :             SET_EXPR_LOCATION (expr, rexpr_location (pred));
    5137        51544 :           then_ = shortcut_cond_expr (expr, condition_uid);
    5138        51544 :           then_se = then_ && TREE_SIDE_EFFECTS (then_);
    5139        51544 :           pred = TREE_OPERAND (pred, 0);
    5140        51544 :           expr = build3 (COND_EXPR, void_type_node, pred, then_, NULL_TREE);
    5141        51544 :           SET_EXPR_LOCATION (expr, locus);
    5142              :         }
    5143              :     }
    5144              : 
    5145       407700 :   if (!then_se)
    5146              :     {
    5147              :       /* If there is no 'then', turn
    5148              :            if (a || b); else d
    5149              :          into
    5150              :            if (a); else if (b); else d.  */
    5151        25589 :       while (TREE_CODE (pred) == TRUTH_ORIF_EXPR)
    5152              :         {
    5153              :           /* Keep the original source location on the first 'if'.  */
    5154          797 :           location_t locus = EXPR_LOC_OR_LOC (expr, input_location);
    5155          797 :           TREE_OPERAND (expr, 0) = TREE_OPERAND (pred, 1);
    5156              :           /* Set the source location of the || on the second 'if'.  */
    5157          797 :           if (rexpr_has_location (pred))
    5158          796 :             SET_EXPR_LOCATION (expr, rexpr_location (pred));
    5159          797 :           else_ = shortcut_cond_expr (expr, condition_uid);
    5160          797 :           else_se = else_ && TREE_SIDE_EFFECTS (else_);
    5161          797 :           pred = TREE_OPERAND (pred, 0);
    5162          797 :           expr = build3 (COND_EXPR, void_type_node, pred, NULL_TREE, else_);
    5163          797 :           SET_EXPR_LOCATION (expr, locus);
    5164              :         }
    5165              :     }
    5166              : 
    5167              :   /* The expr tree should also have the expression id set.  */
    5168       407700 :   tree_associate_condition_with_expr (expr, condition_uid);
    5169              : 
    5170              :   /* If we're done, great.  */
    5171       407700 :   if (TREE_CODE (pred) != TRUTH_ANDIF_EXPR
    5172       407700 :       && TREE_CODE (pred) != TRUTH_ORIF_EXPR)
    5173        97497 :     return expr;
    5174              : 
    5175              :   /* Otherwise we need to mess with gotos.  Change
    5176              :        if (a) c; else d;
    5177              :      to
    5178              :        if (a); else goto no;
    5179              :        c; goto end;
    5180              :        no: d; end:
    5181              :      and recursively gimplify the condition.  */
    5182              : 
    5183       310203 :   true_label = false_label = end_label = NULL_TREE;
    5184              : 
    5185              :   /* If our arms just jump somewhere, hijack those labels so we don't
    5186              :      generate jumps to jumps.  */
    5187              : 
    5188       310203 :   if (tree then_goto = find_goto_label (then_))
    5189              :     {
    5190         4632 :       true_label = GOTO_DESTINATION (then_goto);
    5191         4632 :       then_ = NULL;
    5192         4632 :       then_se = false;
    5193              :     }
    5194              : 
    5195       310203 :   if (tree else_goto = find_goto_label (else_))
    5196              :     {
    5197         4225 :       false_label = GOTO_DESTINATION (else_goto);
    5198         4225 :       else_ = NULL;
    5199         4225 :       else_se = false;
    5200              :     }
    5201              : 
    5202              :   /* If we aren't hijacking a label for the 'then' branch, it falls through.  */
    5203       310203 :   if (true_label)
    5204              :     true_label_p = &true_label;
    5205              :   else
    5206       305571 :     true_label_p = NULL;
    5207              : 
    5208              :   /* The 'else' branch also needs a label if it contains interesting code.  */
    5209       310203 :   if (false_label || else_se)
    5210              :     false_label_p = &false_label;
    5211              :   else
    5212              :     false_label_p = NULL;
    5213              : 
    5214              :   /* If there was nothing else in our arms, just forward the label(s).  */
    5215       151066 :   if (!then_se && !else_se)
    5216         8594 :     return shortcut_cond_r (pred, true_label_p, false_label_p,
    5217        12891 :                             EXPR_LOC_OR_LOC (expr, input_location), condition_uid);
    5218              : 
    5219              :   /* If our last subexpression already has a terminal label, reuse it.  */
    5220       305906 :   if (else_se)
    5221       159137 :     t = expr_last (else_);
    5222       146769 :   else if (then_se)
    5223       146769 :     t = expr_last (then_);
    5224              :   else
    5225              :     t = NULL;
    5226       305906 :   if (t && TREE_CODE (t) == LABEL_EXPR)
    5227          958 :     end_label = LABEL_EXPR_LABEL (t);
    5228              : 
    5229              :   /* If we don't care about jumping to the 'else' branch, jump to the end
    5230              :      if the condition is false.  */
    5231       305906 :   if (!false_label_p)
    5232       146527 :     false_label_p = &end_label;
    5233              : 
    5234              :   /* We only want to emit these labels if we aren't hijacking them.  */
    5235       305906 :   emit_end = (end_label == NULL_TREE);
    5236       305906 :   emit_false = (false_label == NULL_TREE);
    5237              : 
    5238              :   /* We only emit the jump over the else clause if we have to--if the
    5239              :      then clause may fall through.  Otherwise we can wind up with a
    5240              :      useless jump and a useless label at the end of gimplified code,
    5241              :      which will cause us to think that this conditional as a whole
    5242              :      falls through even if it doesn't.  If we then inline a function
    5243              :      which ends with such a condition, that can cause us to issue an
    5244              :      inappropriate warning about control reaching the end of a
    5245              :      non-void function.  */
    5246       305906 :   jump_over_else = block_may_fallthru (then_);
    5247              : 
    5248       611181 :   pred = shortcut_cond_r (pred, true_label_p, false_label_p,
    5249       611181 :                           EXPR_LOC_OR_LOC (expr, input_location),
    5250              :                           condition_uid);
    5251              : 
    5252       305906 :   expr = NULL;
    5253       305906 :   append_to_statement_list (pred, &expr);
    5254              : 
    5255       305906 :   append_to_statement_list (then_, &expr);
    5256       305906 :   if (else_se)
    5257              :     {
    5258       159137 :       if (jump_over_else)
    5259              :         {
    5260       156594 :           tree last = expr_last (expr);
    5261       156594 :           t = build_and_jump (&end_label);
    5262       156594 :           if (rexpr_has_location (last))
    5263        37944 :             SET_EXPR_LOCATION (t, rexpr_location (last));
    5264       156594 :           append_to_statement_list (t, &expr);
    5265              :         }
    5266       159137 :       if (emit_false)
    5267              :         {
    5268       159137 :           t = build1 (LABEL_EXPR, void_type_node, false_label);
    5269       159137 :           append_to_statement_list (t, &expr);
    5270              :         }
    5271       159137 :       append_to_statement_list (else_, &expr);
    5272              :     }
    5273       305906 :   if (emit_end && end_label)
    5274              :     {
    5275       302169 :       t = build1 (LABEL_EXPR, void_type_node, end_label);
    5276       302169 :       append_to_statement_list (t, &expr);
    5277              :     }
    5278              : 
    5279       305906 :   return expr;
    5280              : }
    5281              : 
    5282              : /* EXPR is used in a boolean context; make sure it has BOOLEAN_TYPE.  */
    5283              : 
    5284              : tree
    5285     17062538 : gimple_boolify (tree expr)
    5286              : {
    5287     17062538 :   tree type = TREE_TYPE (expr);
    5288     17062538 :   location_t loc = EXPR_LOCATION (expr);
    5289              : 
    5290     17062538 :   if (TREE_CODE (expr) == NE_EXPR
    5291      5808348 :       && TREE_CODE (TREE_OPERAND (expr, 0)) == CALL_EXPR
    5292     18020289 :       && integer_zerop (TREE_OPERAND (expr, 1)))
    5293              :     {
    5294       496109 :       tree call = TREE_OPERAND (expr, 0);
    5295       496109 :       tree fn = get_callee_fndecl (call);
    5296              : 
    5297              :       /* For __builtin_expect ((long) (x), y) recurse into x as well
    5298              :          if x is truth_value_p.  */
    5299       496109 :       if (fn
    5300       495126 :           && fndecl_built_in_p (fn, BUILT_IN_EXPECT)
    5301       634012 :           && call_expr_nargs (call) == 2)
    5302              :         {
    5303       137903 :           tree arg = CALL_EXPR_ARG (call, 0);
    5304       137903 :           if (arg)
    5305              :             {
    5306       137903 :               if (TREE_CODE (arg) == NOP_EXPR
    5307       137903 :                   && TREE_TYPE (arg) == TREE_TYPE (call))
    5308        65113 :                 arg = TREE_OPERAND (arg, 0);
    5309       137903 :               if (truth_value_p (TREE_CODE (arg)))
    5310              :                 {
    5311       114411 :                   arg = gimple_boolify (arg);
    5312       114411 :                   CALL_EXPR_ARG (call, 0)
    5313       228822 :                     = fold_convert_loc (loc, TREE_TYPE (call), arg);
    5314              :                 }
    5315              :             }
    5316              :         }
    5317              :     }
    5318              : 
    5319     17062538 :   switch (TREE_CODE (expr))
    5320              :     {
    5321      1398191 :     case TRUTH_AND_EXPR:
    5322      1398191 :     case TRUTH_OR_EXPR:
    5323      1398191 :     case TRUTH_XOR_EXPR:
    5324      1398191 :     case TRUTH_ANDIF_EXPR:
    5325      1398191 :     case TRUTH_ORIF_EXPR:
    5326              :       /* Also boolify the arguments of truth exprs.  */
    5327      1398191 :       TREE_OPERAND (expr, 1) = gimple_boolify (TREE_OPERAND (expr, 1));
    5328              :       /* FALLTHRU */
    5329              : 
    5330      1830789 :     case TRUTH_NOT_EXPR:
    5331      1830789 :       TREE_OPERAND (expr, 0) = gimple_boolify (TREE_OPERAND (expr, 0));
    5332              : 
    5333              :       /* These expressions always produce boolean results.  */
    5334      1830789 :       if (TREE_CODE (type) != BOOLEAN_TYPE)
    5335       310923 :         TREE_TYPE (expr) = boolean_type_node;
    5336              :       return expr;
    5337              : 
    5338         5482 :     case ANNOTATE_EXPR:
    5339         5482 :       switch ((enum annot_expr_kind) TREE_INT_CST_LOW (TREE_OPERAND (expr, 1)))
    5340              :         {
    5341         5482 :         case annot_expr_ivdep_kind:
    5342         5482 :         case annot_expr_unroll_kind:
    5343         5482 :         case annot_expr_no_vector_kind:
    5344         5482 :         case annot_expr_vector_kind:
    5345         5482 :         case annot_expr_parallel_kind:
    5346         5482 :         case annot_expr_maybe_infinite_kind:
    5347         5482 :           TREE_OPERAND (expr, 0) = gimple_boolify (TREE_OPERAND (expr, 0));
    5348         5482 :           if (TREE_CODE (type) != BOOLEAN_TYPE)
    5349         3437 :             TREE_TYPE (expr) = boolean_type_node;
    5350              :           return expr;
    5351            0 :         default:
    5352            0 :           gcc_unreachable ();
    5353              :         }
    5354              : 
    5355     15226267 :     default:
    5356     15226267 :       if (COMPARISON_CLASS_P (expr))
    5357              :         {
    5358              :           /* These expressions always produce boolean results.  */
    5359     13332826 :           if (TREE_CODE (type) != BOOLEAN_TYPE)
    5360      1614594 :             TREE_TYPE (expr) = boolean_type_node;
    5361              :           return expr;
    5362              :         }
    5363              :       /* Other expressions that get here must have boolean values, but
    5364              :          might need to be converted to the appropriate mode.  */
    5365      1893441 :       if (TREE_CODE (type) == BOOLEAN_TYPE)
    5366              :         return expr;
    5367       205921 :       return fold_convert_loc (loc, boolean_type_node, expr);
    5368              :     }
    5369              : }
    5370              : 
    5371              : /* Given a conditional expression *EXPR_P without side effects, gimplify
    5372              :    its operands.  New statements are inserted to PRE_P.  */
    5373              : 
    5374              : static enum gimplify_status
    5375        46122 : gimplify_pure_cond_expr (tree *expr_p, gimple_seq *pre_p)
    5376              : {
    5377        46122 :   tree expr = *expr_p, cond;
    5378        46122 :   enum gimplify_status ret, tret;
    5379        46122 :   enum tree_code code;
    5380              : 
    5381        46122 :   cond = gimple_boolify (COND_EXPR_COND (expr));
    5382              : 
    5383              :   /* We need to handle && and || specially, as their gimplification
    5384              :      creates pure cond_expr, thus leading to an infinite cycle otherwise.  */
    5385        46122 :   code = TREE_CODE (cond);
    5386        46122 :   if (code == TRUTH_ANDIF_EXPR)
    5387           39 :     TREE_SET_CODE (cond, TRUTH_AND_EXPR);
    5388        46083 :   else if (code == TRUTH_ORIF_EXPR)
    5389         4655 :     TREE_SET_CODE (cond, TRUTH_OR_EXPR);
    5390        46122 :   ret = gimplify_expr (&cond, pre_p, NULL, is_gimple_val, fb_rvalue);
    5391        46122 :   COND_EXPR_COND (*expr_p) = cond;
    5392              : 
    5393        46122 :   tret = gimplify_expr (&COND_EXPR_THEN (expr), pre_p, NULL,
    5394              :                         is_gimple_val, fb_rvalue);
    5395        46122 :   ret = MIN (ret, tret);
    5396        46122 :   tret = gimplify_expr (&COND_EXPR_ELSE (expr), pre_p, NULL,
    5397              :                         is_gimple_val, fb_rvalue);
    5398              : 
    5399        46122 :   return MIN (ret, tret);
    5400              : }
    5401              : 
    5402              : /* Return true if evaluating EXPR could trap.
    5403              :    EXPR is GENERIC, while tree_could_trap_p can be called
    5404              :    only on GIMPLE.  */
    5405              : 
    5406              : bool
    5407     16490848 : generic_expr_could_trap_p (tree expr)
    5408              : {
    5409     16490848 :   unsigned i, n;
    5410              : 
    5411     16490848 :   if (!expr || is_gimple_val (expr))
    5412              :     return false;
    5413              : 
    5414      9204361 :   if (!EXPR_P (expr) || tree_could_trap_p (expr))
    5415              :     return true;
    5416              : 
    5417      6754415 :   n = TREE_OPERAND_LENGTH (expr);
    5418     16120128 :   for (i = 0; i < n; i++)
    5419     11686275 :     if (generic_expr_could_trap_p (TREE_OPERAND (expr, i)))
    5420              :       return true;
    5421              : 
    5422              :   return false;
    5423              : }
    5424              : 
    5425              : /* Associate the condition STMT with the discriminator UID.  STMTs that are
    5426              :    broken down with ANDIF/ORIF from the same Boolean expression should be given
    5427              :    the same UID; 'if (a && b && c) { if (d || e) ... } ...' should yield the
    5428              :    { a: 1, b: 1, c: 1, d: 2, e: 2 } when gimplification is done.  This is used
    5429              :    for condition coverage.  */
    5430              : static void
    5431      5551213 : gimple_associate_condition_with_expr (struct function *fn, gcond *stmt,
    5432              :                                       unsigned uid)
    5433              : {
    5434      5551213 :   if (!condition_coverage_flag)
    5435              :     return;
    5436              : 
    5437          657 :   if (!fn->cond_uids)
    5438          144 :     fn->cond_uids = new hash_map <gcond*, unsigned> ();
    5439              : 
    5440          657 :   fn->cond_uids->put (stmt, uid);
    5441              : }
    5442              : 
    5443              : /*  Convert the conditional expression pointed to by EXPR_P '(p) ? a : b;'
    5444              :     into
    5445              : 
    5446              :     if (p)                      if (p)
    5447              :       t1 = a;                     a;
    5448              :     else                or      else
    5449              :       t1 = b;                     b;
    5450              :     t1;
    5451              : 
    5452              :     The second form is used when *EXPR_P is of type void.
    5453              : 
    5454              :     PRE_P points to the list where side effects that must happen before
    5455              :       *EXPR_P should be stored.  */
    5456              : 
    5457              : static enum gimplify_status
    5458      6326952 : gimplify_cond_expr (tree *expr_p, gimple_seq *pre_p, fallback_t fallback)
    5459              : {
    5460      6326952 :   tree expr = *expr_p;
    5461      6326952 :   tree type = TREE_TYPE (expr);
    5462      6326952 :   location_t loc = EXPR_LOCATION (expr);
    5463      6326952 :   tree tmp, arm1, arm2;
    5464      6326952 :   enum gimplify_status ret;
    5465      6326952 :   tree label_true, label_false, label_cont;
    5466      6326952 :   bool have_then_clause_p, have_else_clause_p;
    5467      6326952 :   gcond *cond_stmt;
    5468      6326952 :   enum tree_code pred_code;
    5469      6326952 :   gimple_seq seq = NULL;
    5470              : 
    5471              :   /* If this COND_EXPR has a value, copy the values into a temporary within
    5472              :      the arms.  */
    5473      6326952 :   if (!VOID_TYPE_P (type))
    5474              :     {
    5475       419598 :       tree then_ = TREE_OPERAND (expr, 1), else_ = TREE_OPERAND (expr, 2);
    5476       419598 :       tree result;
    5477              : 
    5478              :       /* If either an rvalue is ok or we do not require an lvalue, create the
    5479              :          temporary.  But we cannot do that if the type is addressable.  */
    5480       419598 :       if (((fallback & fb_rvalue) || !(fallback & fb_lvalue))
    5481       419591 :           && !TREE_ADDRESSABLE (type))
    5482              :         {
    5483       419591 :           if (gimplify_ctxp->allow_rhs_cond_expr
    5484              :               /* If either branch has side effects or could trap, it can't be
    5485              :                  evaluated unconditionally.  */
    5486        46122 :               && !TREE_SIDE_EFFECTS (then_)
    5487        46122 :               && !generic_expr_could_trap_p (then_)
    5488        46122 :               && !TREE_SIDE_EFFECTS (else_)
    5489       465713 :               && !generic_expr_could_trap_p (else_))
    5490        46122 :             return gimplify_pure_cond_expr (expr_p, pre_p);
    5491              : 
    5492       373469 :           tmp = create_tmp_var (type, "iftmp");
    5493       373469 :           result = tmp;
    5494              :         }
    5495              : 
    5496              :       /* Otherwise, only create and copy references to the values.  */
    5497              :       else
    5498              :         {
    5499            7 :           type = build_pointer_type (type);
    5500              : 
    5501            7 :           if (!VOID_TYPE_P (TREE_TYPE (then_)))
    5502            7 :             then_ = build_fold_addr_expr_loc (loc, then_);
    5503              : 
    5504            7 :           if (!VOID_TYPE_P (TREE_TYPE (else_)))
    5505            7 :             else_ = build_fold_addr_expr_loc (loc, else_);
    5506              : 
    5507            7 :           expr
    5508            7 :             = build3 (COND_EXPR, type, TREE_OPERAND (expr, 0), then_, else_);
    5509              : 
    5510            7 :           tmp = create_tmp_var (type, "iftmp");
    5511            7 :           result = build_simple_mem_ref_loc (loc, tmp);
    5512              :         }
    5513              : 
    5514              :       /* Build the new then clause, `tmp = then_;'.  But don't build the
    5515              :          assignment if the value is void; in C++ it can be if it's a throw.  */
    5516       373476 :       if (!VOID_TYPE_P (TREE_TYPE (then_)))
    5517       291900 :         TREE_OPERAND (expr, 1) = build2 (INIT_EXPR, type, tmp, then_);
    5518              : 
    5519              :       /* Similarly, build the new else clause, `tmp = else_;'.  */
    5520       373476 :       if (!VOID_TYPE_P (TREE_TYPE (else_)))
    5521       362804 :         TREE_OPERAND (expr, 2) = build2 (INIT_EXPR, type, tmp, else_);
    5522              : 
    5523       373476 :       TREE_TYPE (expr) = void_type_node;
    5524       373476 :       recalculate_side_effects (expr);
    5525              : 
    5526              :       /* Move the COND_EXPR to the prequeue.  */
    5527       373476 :       gimplify_stmt (&expr, pre_p);
    5528              : 
    5529       373476 :       *expr_p = result;
    5530       373476 :       return GS_ALL_DONE;
    5531              :     }
    5532              : 
    5533              :   /* Remove any COMPOUND_EXPR so the following cases will be caught.  */
    5534      5922405 :   STRIP_TYPE_NOPS (TREE_OPERAND (expr, 0));
    5535      5907354 :   if (TREE_CODE (TREE_OPERAND (expr, 0)) == COMPOUND_EXPR)
    5536        81163 :     gimplify_compound_expr (&TREE_OPERAND (expr, 0), pre_p, true);
    5537              : 
    5538              :   /* Make sure the condition has BOOLEAN_TYPE.  */
    5539      5907354 :   TREE_OPERAND (expr, 0) = gimple_boolify (TREE_OPERAND (expr, 0));
    5540              : 
    5541              :   /* Break apart && and || conditions.  */
    5542      5907354 :   if (TREE_CODE (TREE_OPERAND (expr, 0)) == TRUTH_ANDIF_EXPR
    5543      5907354 :       || TREE_CODE (TREE_OPERAND (expr, 0)) == TRUTH_ORIF_EXPR)
    5544              :     {
    5545       355359 :       expr = shortcut_cond_expr (expr, next_cond_uid ());
    5546              : 
    5547       355359 :       if (expr != *expr_p)
    5548              :         {
    5549       355359 :           *expr_p = expr;
    5550              : 
    5551              :           /* We can't rely on gimplify_expr to re-gimplify the expanded
    5552              :              form properly, as cleanups might cause the target labels to be
    5553              :              wrapped in a TRY_FINALLY_EXPR.  To prevent that, we need to
    5554              :              set up a conditional context.  */
    5555       355359 :           gimple_push_condition ();
    5556       355359 :           gimplify_stmt (expr_p, &seq);
    5557       355359 :           gimple_pop_condition (pre_p);
    5558       355359 :           gimple_seq_add_seq (pre_p, seq);
    5559              : 
    5560       355359 :           return GS_ALL_DONE;
    5561              :         }
    5562              :     }
    5563              : 
    5564              :   /* Now do the normal gimplification.  */
    5565              : 
    5566              :   /* Gimplify condition.  */
    5567      5551995 :   ret = gimplify_expr (&TREE_OPERAND (expr, 0), pre_p, NULL,
    5568              :                        is_gimple_condexpr_for_cond, fb_rvalue);
    5569      5551995 :   if (ret == GS_ERROR)
    5570              :     return GS_ERROR;
    5571      5551213 :   gcc_assert (TREE_OPERAND (expr, 0) != NULL_TREE);
    5572              : 
    5573      5551213 :   gimple_push_condition ();
    5574              : 
    5575      5551213 :   have_then_clause_p = have_else_clause_p = false;
    5576      5551213 :   label_true = find_goto_label (TREE_OPERAND (expr, 1));
    5577      5551213 :   if (label_true
    5578      1301603 :       && DECL_CONTEXT (GOTO_DESTINATION (label_true)) == current_function_decl
    5579              :       /* For -O0 avoid this optimization if the COND_EXPR and GOTO_EXPR
    5580              :          have different locations, otherwise we end up with incorrect
    5581              :          location information on the branches.  */
    5582      6852816 :       && (optimize
    5583       235531 :           || !EXPR_HAS_LOCATION (expr)
    5584       208584 :           || !rexpr_has_location (label_true)
    5585        71498 :           || EXPR_LOCATION (expr) == rexpr_location (label_true)))
    5586              :     {
    5587      1300424 :       have_then_clause_p = true;
    5588      1300424 :       label_true = GOTO_DESTINATION (label_true);
    5589              :     }
    5590              :   else
    5591      4250789 :     label_true = create_artificial_label (UNKNOWN_LOCATION);
    5592      5551213 :   label_false = find_goto_label (TREE_OPERAND (expr, 2));
    5593      5551213 :   if (label_false
    5594       734374 :       && DECL_CONTEXT (GOTO_DESTINATION (label_false)) == current_function_decl
    5595              :       /* For -O0 avoid this optimization if the COND_EXPR and GOTO_EXPR
    5596              :          have different locations, otherwise we end up with incorrect
    5597              :          location information on the branches.  */
    5598      6285587 :       && (optimize
    5599       105404 :           || !EXPR_HAS_LOCATION (expr)
    5600       104516 :           || !rexpr_has_location (label_false)
    5601          111 :           || EXPR_LOCATION (expr) == rexpr_location (label_false)))
    5602              :     {
    5603       734275 :       have_else_clause_p = true;
    5604       734275 :       label_false = GOTO_DESTINATION (label_false);
    5605              :     }
    5606              :   else
    5607      4816938 :     label_false = create_artificial_label (UNKNOWN_LOCATION);
    5608              : 
    5609      5551213 :   unsigned cond_uid = 0;
    5610      5551213 :   if (cond_uids)
    5611          467 :     if (unsigned *v = cond_uids->get (expr))
    5612          447 :       cond_uid = *v;
    5613          447 :   if (cond_uid == 0)
    5614      5550766 :     cond_uid = next_cond_uid ();
    5615              : 
    5616      5551213 :   gimple_cond_get_ops_from_tree (COND_EXPR_COND (expr), &pred_code, &arm1,
    5617              :                                  &arm2);
    5618      5551213 :   cond_stmt = gimple_build_cond (pred_code, arm1, arm2, label_true,
    5619              :                                  label_false);
    5620      5551213 :   gimple_set_location (cond_stmt, EXPR_LOCATION (expr));
    5621      5551213 :   gimple_associate_condition_with_expr (cfun, cond_stmt, cond_uid);
    5622      5551213 :   copy_warning (cond_stmt, COND_EXPR_COND (expr));
    5623      5551213 :   gimplify_seq_add_stmt (&seq, cond_stmt);
    5624      5551213 :   gimple_stmt_iterator gsi = gsi_last (seq);
    5625      5551213 :   maybe_fold_stmt (&gsi);
    5626              : 
    5627      5551213 :   label_cont = NULL_TREE;
    5628      5551213 :   if (!have_then_clause_p)
    5629              :     {
    5630              :       /* For if (...) {} else { code; } put label_true after
    5631              :          the else block.  */
    5632      4250789 :       if (TREE_OPERAND (expr, 1) == NULL_TREE
    5633       200524 :           && !have_else_clause_p
    5634      4251559 :           && TREE_OPERAND (expr, 2) != NULL_TREE)
    5635              :         {
    5636              :           /* For if (0) {} else { code; } tell -Wimplicit-fallthrough
    5637              :              handling that label_cont == label_true can be only reached
    5638              :              through fallthrough from { code; }.  */
    5639          770 :           if (integer_zerop (COND_EXPR_COND (expr)))
    5640            0 :             UNUSED_LABEL_P (label_true) = 1;
    5641              :           label_cont = label_true;
    5642              :         }
    5643              :       else
    5644              :         {
    5645      4250019 :           bool then_side_effects
    5646      4250019 :             = (TREE_OPERAND (expr, 1)
    5647      4250019 :                && TREE_SIDE_EFFECTS (TREE_OPERAND (expr, 1)));
    5648      4250019 :           gimplify_seq_add_stmt (&seq, gimple_build_label (label_true));
    5649      4250019 :           have_then_clause_p = gimplify_stmt (&TREE_OPERAND (expr, 1), &seq);
    5650              :           /* For if (...) { code; } else {} or
    5651              :              if (...) { code; } else goto label; or
    5652              :              if (...) { code; return; } else { ... }
    5653              :              label_cont isn't needed.  */
    5654      4250019 :           if (!have_else_clause_p
    5655      4049119 :               && TREE_OPERAND (expr, 2) != NULL_TREE
    5656      7068219 :               && gimple_seq_may_fallthru (seq))
    5657              :             {
    5658      1960119 :               gimple *g;
    5659      1960119 :               label_cont = create_artificial_label (UNKNOWN_LOCATION);
    5660              : 
    5661              :               /* For if (0) { non-side-effect-code } else { code }
    5662              :                  tell -Wimplicit-fallthrough handling that label_cont can
    5663              :                  be only reached through fallthrough from { code }.  */
    5664      1960119 :               if (integer_zerop (COND_EXPR_COND (expr)))
    5665              :                 {
    5666        75896 :                   UNUSED_LABEL_P (label_true) = 1;
    5667        75896 :                   if (!then_side_effects)
    5668        12378 :                     UNUSED_LABEL_P (label_cont) = 1;
    5669              :                 }
    5670              : 
    5671      1960119 :               g = gimple_build_goto (label_cont);
    5672              : 
    5673              :               /* GIMPLE_COND's are very low level; they have embedded
    5674              :                  gotos.  This particular embedded goto should not be marked
    5675              :                  with the location of the original COND_EXPR, as it would
    5676              :                  correspond to the COND_EXPR's condition, not the ELSE or the
    5677              :                  THEN arms.  To avoid marking it with the wrong location, flag
    5678              :                  it as "no location".  */
    5679      1960119 :               gimple_set_do_not_emit_location (g);
    5680              : 
    5681      1960119 :               gimplify_seq_add_stmt (&seq, g);
    5682              :             }
    5683              :         }
    5684              :     }
    5685      5551213 :   if (!have_else_clause_p)
    5686              :     {
    5687              :       /* For if (1) { code } or if (1) { code } else { non-side-effect-code }
    5688              :          tell -Wimplicit-fallthrough handling that label_false can be only
    5689              :          reached through fallthrough from { code }.  */
    5690      4816938 :       if (integer_nonzerop (COND_EXPR_COND (expr))
    5691      4816938 :           && (TREE_OPERAND (expr, 2) == NULL_TREE
    5692        57500 :               || !TREE_SIDE_EFFECTS (TREE_OPERAND (expr, 2))))
    5693         7938 :         UNUSED_LABEL_P (label_false) = 1;
    5694      4816938 :       gimplify_seq_add_stmt (&seq, gimple_build_label (label_false));
    5695      4816938 :       have_else_clause_p = gimplify_stmt (&TREE_OPERAND (expr, 2), &seq);
    5696              :     }
    5697      5551213 :   if (label_cont)
    5698      1960889 :     gimplify_seq_add_stmt (&seq, gimple_build_label (label_cont));
    5699              : 
    5700      5551213 :   gimple_pop_condition (pre_p);
    5701      5551213 :   gimple_seq_add_seq (pre_p, seq);
    5702              : 
    5703      5551213 :   if (ret == GS_ERROR)
    5704              :     ; /* Do nothing.  */
    5705      5551213 :   else if (have_then_clause_p || have_else_clause_p)
    5706              :     ret = GS_ALL_DONE;
    5707              :   else
    5708              :     {
    5709              :       /* Both arms are empty; replace the COND_EXPR with its predicate.  */
    5710         2960 :       expr = TREE_OPERAND (expr, 0);
    5711         2960 :       gimplify_stmt (&expr, pre_p);
    5712              :     }
    5713              : 
    5714      5551213 :   *expr_p = NULL;
    5715      5551213 :   return ret;
    5716              : }
    5717              : 
    5718              : /* Prepare the node pointed to by EXPR_P, an is_gimple_addressable expression,
    5719              :    to be marked addressable.
    5720              : 
    5721              :    We cannot rely on such an expression being directly markable if a temporary
    5722              :    has been created by the gimplification.  In this case, we create another
    5723              :    temporary and initialize it with a copy, which will become a store after we
    5724              :    mark it addressable.  This can happen if the front-end passed us something
    5725              :    that it could not mark addressable yet, like a Fortran pass-by-reference
    5726              :    parameter (int) floatvar.  */
    5727              : 
    5728              : static void
    5729     57115190 : prepare_gimple_addressable (tree *expr_p, gimple_seq *seq_p)
    5730              : {
    5731     62674923 :   while (handled_component_p (*expr_p))
    5732      5559733 :     expr_p = &TREE_OPERAND (*expr_p, 0);
    5733              : 
    5734              :   /* Do not allow an SSA name as the temporary.  */
    5735     57115190 :   if (is_gimple_reg (*expr_p))
    5736        12134 :     *expr_p = internal_get_tmp_var (*expr_p, seq_p, NULL, false, false, true);
    5737     57115190 : }
    5738              : 
    5739              : /* A subroutine of gimplify_modify_expr.  Replace a MODIFY_EXPR with
    5740              :    a call to __builtin_memcpy.  */
    5741              : 
    5742              : static enum gimplify_status
    5743         1110 : gimplify_modify_expr_to_memcpy (tree *expr_p, tree size, bool want_value,
    5744              :                                 gimple_seq *seq_p)
    5745              : {
    5746         1110 :   tree t, to, to_ptr, from, from_ptr;
    5747         1110 :   gcall *gs;
    5748         1110 :   location_t loc = EXPR_LOCATION (*expr_p);
    5749              : 
    5750         1110 :   to = TREE_OPERAND (*expr_p, 0);
    5751         1110 :   from = TREE_OPERAND (*expr_p, 1);
    5752         1110 :   gcc_assert (ADDR_SPACE_GENERIC_P (TYPE_ADDR_SPACE (TREE_TYPE (to)))
    5753              :               && ADDR_SPACE_GENERIC_P (TYPE_ADDR_SPACE (TREE_TYPE (from))));
    5754              : 
    5755              :   /* Mark the RHS addressable.  Beware that it may not be possible to do so
    5756              :      directly if a temporary has been created by the gimplification.  */
    5757         1110 :   prepare_gimple_addressable (&from, seq_p);
    5758              : 
    5759         1110 :   mark_addressable (from);
    5760         1110 :   from_ptr = build_fold_addr_expr_loc (loc, from);
    5761         1110 :   gimplify_arg (&from_ptr, seq_p, loc);
    5762              : 
    5763         1110 :   mark_addressable (to);
    5764         1110 :   to_ptr = build_fold_addr_expr_loc (loc, to);
    5765         1110 :   gimplify_arg (&to_ptr, seq_p, loc);
    5766              : 
    5767         1110 :   t = builtin_decl_implicit (BUILT_IN_MEMCPY);
    5768              : 
    5769         1110 :   gs = gimple_build_call (t, 3, to_ptr, from_ptr, size);
    5770         1110 :   gimple_call_set_alloca_for_var (gs, true);
    5771              : 
    5772         1110 :   if (want_value)
    5773              :     {
    5774              :       /* tmp = memcpy() */
    5775            8 :       t = create_tmp_var (TREE_TYPE (to_ptr));
    5776            8 :       gimple_call_set_lhs (gs, t);
    5777            8 :       gimplify_seq_add_stmt (seq_p, gs);
    5778              : 
    5779            8 :       *expr_p = build_simple_mem_ref (t);
    5780            8 :       return GS_ALL_DONE;
    5781              :     }
    5782              : 
    5783         1102 :   gimplify_seq_add_stmt (seq_p, gs);
    5784         1102 :   *expr_p = NULL;
    5785         1102 :   return GS_ALL_DONE;
    5786              : }
    5787              : 
    5788              : /* A subroutine of gimplify_modify_expr.  Replace a MODIFY_EXPR with
    5789              :    a call to __builtin_memset.  In this case we know that the RHS is
    5790              :    a CONSTRUCTOR with an empty element list.  */
    5791              : 
    5792              : static enum gimplify_status
    5793           84 : gimplify_modify_expr_to_memset (tree *expr_p, tree size, bool want_value,
    5794              :                                 gimple_seq *seq_p)
    5795              : {
    5796           84 :   tree t, from, to, to_ptr;
    5797           84 :   gcall *gs;
    5798           84 :   location_t loc = EXPR_LOCATION (*expr_p);
    5799              : 
    5800              :   /* Assert our assumptions, to abort instead of producing wrong code
    5801              :      silently if they are not met.  Beware that the RHS CONSTRUCTOR might
    5802              :      not be immediately exposed.  */
    5803           84 :   from = TREE_OPERAND (*expr_p, 1);
    5804           84 :   if (TREE_CODE (from) == WITH_SIZE_EXPR)
    5805           84 :     from = TREE_OPERAND (from, 0);
    5806              : 
    5807           84 :   gcc_assert (TREE_CODE (from) == CONSTRUCTOR
    5808              :               && vec_safe_is_empty (CONSTRUCTOR_ELTS (from)));
    5809              : 
    5810              :   /* Now proceed.  */
    5811           84 :   to = TREE_OPERAND (*expr_p, 0);
    5812           84 :   gcc_assert (ADDR_SPACE_GENERIC_P (TYPE_ADDR_SPACE (TREE_TYPE (to))));
    5813              : 
    5814           84 :   to_ptr = build_fold_addr_expr_loc (loc, to);
    5815           84 :   gimplify_arg (&to_ptr, seq_p, loc);
    5816           84 :   t = builtin_decl_implicit (BUILT_IN_MEMSET);
    5817              : 
    5818           84 :   gs = gimple_build_call (t, 3, to_ptr, integer_zero_node, size);
    5819              : 
    5820           84 :   if (want_value)
    5821              :     {
    5822              :       /* tmp = memset() */
    5823            0 :       t = create_tmp_var (TREE_TYPE (to_ptr));
    5824            0 :       gimple_call_set_lhs (gs, t);
    5825            0 :       gimplify_seq_add_stmt (seq_p, gs);
    5826              : 
    5827            0 :       *expr_p = build1 (INDIRECT_REF, TREE_TYPE (to), t);
    5828            0 :       return GS_ALL_DONE;
    5829              :     }
    5830              : 
    5831           84 :   gimplify_seq_add_stmt (seq_p, gs);
    5832           84 :   *expr_p = NULL;
    5833           84 :   return GS_ALL_DONE;
    5834              : }
    5835              : 
    5836              : /* A subroutine of gimplify_init_ctor_preeval.  Called via walk_tree,
    5837              :    determine, cautiously, if a CONSTRUCTOR overlaps the lhs of an
    5838              :    assignment.  Return non-null if we detect a potential overlap.  */
    5839              : 
    5840              : struct gimplify_init_ctor_preeval_data
    5841              : {
    5842              :   /* The base decl of the lhs object.  May be NULL, in which case we
    5843              :      have to assume the lhs is indirect.  */
    5844              :   tree lhs_base_decl;
    5845              : 
    5846              :   /* The alias set of the lhs object.  */
    5847              :   alias_set_type lhs_alias_set;
    5848              : };
    5849              : 
    5850              : static tree
    5851       116633 : gimplify_init_ctor_preeval_1 (tree *tp, int *walk_subtrees, void *xdata)
    5852              : {
    5853       116633 :   struct gimplify_init_ctor_preeval_data *data
    5854              :     = (struct gimplify_init_ctor_preeval_data *) xdata;
    5855       116633 :   tree t = *tp;
    5856              : 
    5857              :   /* If we find the base object, obviously we have overlap.  */
    5858       116633 :   if (data->lhs_base_decl == t)
    5859              :     return t;
    5860              : 
    5861              :   /* If the constructor component is indirect, determine if we have a
    5862              :      potential overlap with the lhs.  The only bits of information we
    5863              :      have to go on at this point are addressability and alias sets.  */
    5864       116632 :   if ((INDIRECT_REF_P (t)
    5865       116632 :        || TREE_CODE (t) == MEM_REF)
    5866          651 :       && (!data->lhs_base_decl || TREE_ADDRESSABLE (data->lhs_base_decl))
    5867       117148 :       && alias_sets_conflict_p (data->lhs_alias_set, get_alias_set (t)))
    5868              :     return t;
    5869              : 
    5870              :   /* If the constructor component is a call, determine if it can hide a
    5871              :      potential overlap with the lhs through an INDIRECT_REF like above.
    5872              :      ??? Ugh - this is completely broken.  In fact this whole analysis
    5873              :      doesn't look conservative.  */
    5874       116116 :   if (TREE_CODE (t) == CALL_EXPR)
    5875              :     {
    5876            0 :       tree type, fntype = TREE_TYPE (TREE_TYPE (CALL_EXPR_FN (t)));
    5877              : 
    5878            0 :       for (type = TYPE_ARG_TYPES (fntype); type; type = TREE_CHAIN (type))
    5879            0 :         if (POINTER_TYPE_P (TREE_VALUE (type))
    5880            0 :             && (!data->lhs_base_decl || TREE_ADDRESSABLE (data->lhs_base_decl))
    5881            0 :             && alias_sets_conflict_p (data->lhs_alias_set,
    5882              :                                       get_alias_set
    5883            0 :                                         (TREE_TYPE (TREE_VALUE (type)))))
    5884              :           return t;
    5885              :     }
    5886              : 
    5887       116116 :   if (IS_TYPE_OR_DECL_P (t))
    5888        10968 :     *walk_subtrees = 0;
    5889              :   return NULL;
    5890              : }
    5891              : 
    5892              : /* A subroutine of gimplify_init_constructor.  Pre-evaluate EXPR,
    5893              :    force values that overlap with the lhs (as described by *DATA)
    5894              :    into temporaries.  */
    5895              : 
    5896              : static void
    5897       420773 : gimplify_init_ctor_preeval (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
    5898              :                             struct gimplify_init_ctor_preeval_data *data)
    5899              : {
    5900       420773 :   enum gimplify_status one;
    5901              : 
    5902              :   /* If the value is constant, then there's nothing to pre-evaluate.  */
    5903       420773 :   if (TREE_CONSTANT (*expr_p))
    5904              :     {
    5905              :       /* Ensure it does not have side effects, it might contain a reference to
    5906              :          the object we're initializing.  */
    5907       201964 :       gcc_assert (!TREE_SIDE_EFFECTS (*expr_p));
    5908              :       return;
    5909              :     }
    5910              : 
    5911              :   /* If the type has non-trivial constructors, we can't pre-evaluate.  */
    5912       218809 :   if (TREE_ADDRESSABLE (TREE_TYPE (*expr_p)))
    5913              :     return;
    5914              : 
    5915              :   /* Recurse for nested constructors.  */
    5916       218809 :   if (TREE_CODE (*expr_p) == CONSTRUCTOR)
    5917              :     {
    5918        74158 :       unsigned HOST_WIDE_INT ix;
    5919        74158 :       constructor_elt *ce;
    5920        74158 :       vec<constructor_elt, va_gc> *v = CONSTRUCTOR_ELTS (*expr_p);
    5921              : 
    5922       272888 :       FOR_EACH_VEC_SAFE_ELT (v, ix, ce)
    5923       198730 :         gimplify_init_ctor_preeval (&ce->value, pre_p, post_p, data);
    5924              : 
    5925              :       return;
    5926              :     }
    5927              : 
    5928              :   /* If this is a variable sized type, we must remember the size.  */
    5929       144651 :   maybe_with_size_expr (expr_p);
    5930              : 
    5931              :   /* Gimplify the constructor element to something appropriate for the rhs
    5932              :      of a MODIFY_EXPR.  Given that we know the LHS is an aggregate, we know
    5933              :      the gimplifier will consider this a store to memory.  Doing this
    5934              :      gimplification now means that we won't have to deal with complicated
    5935              :      language-specific trees, nor trees like SAVE_EXPR that can induce
    5936              :      exponential search behavior.  */
    5937       144651 :   one = gimplify_expr (expr_p, pre_p, post_p, is_gimple_mem_rhs, fb_rvalue);
    5938       144651 :   if (one == GS_ERROR)
    5939              :     {
    5940            0 :       *expr_p = NULL;
    5941            0 :       return;
    5942              :     }
    5943              : 
    5944              :   /* If we gimplified to a bare decl, we can be sure that it doesn't overlap
    5945              :      with the lhs, since "a = { .x=a }" doesn't make sense.  This will
    5946              :      always be true for all scalars, since is_gimple_mem_rhs insists on a
    5947              :      temporary variable for them.  */
    5948       144651 :   if (DECL_P (*expr_p))
    5949              :     return;
    5950              : 
    5951              :   /* If this is of variable size, we have no choice but to assume it doesn't
    5952              :      overlap since we can't make a temporary for it.  */
    5953       105069 :   if (TREE_CODE (TYPE_SIZE (TREE_TYPE (*expr_p))) != INTEGER_CST)
    5954              :     return;
    5955              : 
    5956              :   /* Otherwise, we must search for overlap ...  */
    5957       105069 :   if (!walk_tree (expr_p, gimplify_init_ctor_preeval_1, data, NULL))
    5958              :     return;
    5959              : 
    5960              :   /* ... and if found, force the value into a temporary.  */
    5961          517 :   *expr_p = get_formal_tmp_var (*expr_p, pre_p);
    5962              : }
    5963              : 
    5964              : /* A subroutine of gimplify_init_ctor_eval.  Create a loop for
    5965              :    a RANGE_EXPR in a CONSTRUCTOR for an array.
    5966              : 
    5967              :       var = lower;
    5968              :     loop_entry:
    5969              :       object[var] = value;
    5970              :       if (var == upper)
    5971              :         goto loop_exit;
    5972              :       var = var + 1;
    5973              :       goto loop_entry;
    5974              :     loop_exit:
    5975              : 
    5976              :    We increment var _after_ the loop exit check because we might otherwise
    5977              :    fail if upper == TYPE_MAX_VALUE (type for upper).
    5978              : 
    5979              :    Note that we never have to deal with SAVE_EXPRs here, because this has
    5980              :    already been taken care of for us, in gimplify_init_ctor_preeval().  */
    5981              : 
    5982              : static void gimplify_init_ctor_eval (tree, vec<constructor_elt, va_gc> *,
    5983              :                                      gimple_seq *, bool);
    5984              : 
    5985              : static void
    5986          579 : gimplify_init_ctor_eval_range (tree object, tree lower, tree upper,
    5987              :                                tree value, tree array_elt_type,
    5988              :                                gimple_seq *pre_p, bool cleared)
    5989              : {
    5990          579 :   tree loop_entry_label, loop_exit_label, fall_thru_label;
    5991          579 :   tree var, var_type, cref, tmp;
    5992              : 
    5993          579 :   loop_entry_label = create_artificial_label (UNKNOWN_LOCATION);
    5994          579 :   loop_exit_label = create_artificial_label (UNKNOWN_LOCATION);
    5995          579 :   fall_thru_label = create_artificial_label (UNKNOWN_LOCATION);
    5996              : 
    5997              :   /* Create and initialize the index variable.  */
    5998          579 :   var_type = TREE_TYPE (upper);
    5999          579 :   var = create_tmp_var (var_type);
    6000          579 :   gimplify_seq_add_stmt (pre_p, gimple_build_assign (var, lower));
    6001              : 
    6002              :   /* Add the loop entry label.  */
    6003          579 :   gimplify_seq_add_stmt (pre_p, gimple_build_label (loop_entry_label));
    6004              : 
    6005              :   /* Build the reference.  */
    6006          579 :   cref = build4 (ARRAY_REF, array_elt_type, unshare_expr (object),
    6007              :                  var, NULL_TREE, NULL_TREE);
    6008              : 
    6009              :   /* If we are a constructor, just call gimplify_init_ctor_eval to do
    6010              :      the store.  Otherwise just assign value to the reference.  */
    6011              : 
    6012          579 :   if (TREE_CODE (value) == CONSTRUCTOR)
    6013              :     /* NB we might have to call ourself recursively through
    6014              :        gimplify_init_ctor_eval if the value is a constructor.  */
    6015          129 :     gimplify_init_ctor_eval (cref, CONSTRUCTOR_ELTS (value),
    6016              :                              pre_p, cleared);
    6017              :   else
    6018              :     {
    6019          450 :       if (gimplify_expr (&value, pre_p, NULL, is_gimple_val, fb_rvalue)
    6020              :           != GS_ERROR)
    6021          450 :         gimplify_seq_add_stmt (pre_p, gimple_build_assign (cref, value));
    6022              :     }
    6023              : 
    6024              :   /* We exit the loop when the index var is equal to the upper bound.  */
    6025         1158 :   gimplify_seq_add_stmt (pre_p,
    6026          579 :                          gimple_build_cond (EQ_EXPR, var, upper,
    6027              :                                             loop_exit_label, fall_thru_label));
    6028              : 
    6029          579 :   gimplify_seq_add_stmt (pre_p, gimple_build_label (fall_thru_label));
    6030              : 
    6031              :   /* Otherwise, increment the index var...  */
    6032          579 :   tmp = build2 (PLUS_EXPR, var_type, var,
    6033              :                 fold_convert (var_type, integer_one_node));
    6034          579 :   gimplify_seq_add_stmt (pre_p, gimple_build_assign (var, tmp));
    6035              : 
    6036              :   /* ...and jump back to the loop entry.  */
    6037          579 :   gimplify_seq_add_stmt (pre_p, gimple_build_goto (loop_entry_label));
    6038              : 
    6039              :   /* Add the loop exit label.  */
    6040          579 :   gimplify_seq_add_stmt (pre_p, gimple_build_label (loop_exit_label));
    6041          579 : }
    6042              : 
    6043              : /* A subroutine of gimplify_init_constructor.  Generate individual
    6044              :    MODIFY_EXPRs for a CONSTRUCTOR.  OBJECT is the LHS against which the
    6045              :    assignments should happen.  ELTS is the CONSTRUCTOR_ELTS of the
    6046              :    CONSTRUCTOR.  CLEARED is true if the entire LHS object has been
    6047              :    zeroed first.  */
    6048              : 
    6049              : static void
    6050      1277901 : gimplify_init_ctor_eval (tree object, vec<constructor_elt, va_gc> *elts,
    6051              :                          gimple_seq *pre_p, bool cleared)
    6052              : {
    6053      1277901 :   tree array_elt_type = NULL;
    6054      1277901 :   unsigned HOST_WIDE_INT ix;
    6055      1277901 :   tree purpose, value;
    6056              : 
    6057      1277901 :   if (TREE_CODE (TREE_TYPE (object)) == ARRAY_TYPE)
    6058       142468 :     array_elt_type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (object)));
    6059              : 
    6060      4722449 :   FOR_EACH_CONSTRUCTOR_ELT (elts, ix, purpose, value)
    6061              :     {
    6062      3444548 :       tree cref;
    6063              : 
    6064              :       /* NULL values are created above for gimplification errors.  */
    6065      3444548 :       if (value == NULL)
    6066       330004 :         continue;
    6067              : 
    6068      3444548 :       if (cleared && initializer_zerop (value))
    6069       329257 :         continue;
    6070              : 
    6071              :       /* ??? Here's to hoping the front end fills in all of the indices,
    6072              :          so we don't have to figure out what's missing ourselves.  */
    6073      3115291 :       gcc_assert (purpose);
    6074              : 
    6075              :       /* Skip zero-sized fields, unless value has side-effects.  This can
    6076              :          happen with calls to functions returning a empty type, which
    6077              :          we shouldn't discard.  As a number of downstream passes don't
    6078              :          expect sets of empty type fields, we rely on the gimplification of
    6079              :          the MODIFY_EXPR we make below to drop the assignment statement.  */
    6080      3115291 :       if (!TREE_SIDE_EFFECTS (value)
    6081      2819751 :           && TREE_CODE (purpose) == FIELD_DECL
    6082      5461460 :           && is_empty_type (TREE_TYPE (purpose)))
    6083          168 :         continue;
    6084              : 
    6085              :       /* If we have a RANGE_EXPR, we have to build a loop to assign the
    6086              :          whole range.  */
    6087      3115123 :       if (TREE_CODE (purpose) == RANGE_EXPR)
    6088              :         {
    6089          595 :           tree lower = TREE_OPERAND (purpose, 0);
    6090          595 :           tree upper = TREE_OPERAND (purpose, 1);
    6091              : 
    6092              :           /* If the lower bound is equal to upper, just treat it as if
    6093              :              upper was the index.  */
    6094          595 :           if (simple_cst_equal (lower, upper))
    6095              :             purpose = upper;
    6096              :           else
    6097              :             {
    6098          579 :               gimplify_init_ctor_eval_range (object, lower, upper, value,
    6099              :                                              array_elt_type, pre_p, cleared);
    6100          579 :               continue;
    6101              :             }
    6102              :         }
    6103              : 
    6104      3114544 :       if (array_elt_type)
    6105              :         {
    6106              :           /* Do not use bitsizetype for ARRAY_REF indices.  */
    6107       563825 :           if (TYPE_DOMAIN (TREE_TYPE (object)))
    6108       563825 :             purpose
    6109       563825 :               = fold_convert (TREE_TYPE (TYPE_DOMAIN (TREE_TYPE (object))),
    6110              :                               purpose);
    6111       563825 :           cref = build4 (ARRAY_REF, array_elt_type, unshare_expr (object),
    6112              :                          purpose, NULL_TREE, NULL_TREE);
    6113              :         }
    6114              :       else
    6115              :         {
    6116      2550719 :           gcc_assert (TREE_CODE (purpose) == FIELD_DECL);
    6117      2550719 :           cref = build3 (COMPONENT_REF, TREE_TYPE (purpose),
    6118              :                          unshare_expr (object), purpose, NULL_TREE);
    6119              :         }
    6120              : 
    6121      3114544 :       if (TREE_CODE (value) == CONSTRUCTOR
    6122      3114544 :           && TREE_CODE (TREE_TYPE (value)) != VECTOR_TYPE)
    6123       303102 :         gimplify_init_ctor_eval (cref, CONSTRUCTOR_ELTS (value),
    6124              :                                  pre_p, cleared);
    6125      2811442 :       else if (TREE_CODE (value) == RAW_DATA_CST)
    6126              :         {
    6127           43 :           if (RAW_DATA_LENGTH (value) <= 32)
    6128              :             {
    6129           13 :               for (unsigned int i = 0; i < (unsigned) RAW_DATA_LENGTH (value);
    6130              :                    ++i)
    6131           12 :                 if (!cleared || RAW_DATA_POINTER (value)[i])
    6132              :                   {
    6133           12 :                     if (i)
    6134              :                       {
    6135           11 :                         tree p
    6136           11 :                           = fold_build2 (PLUS_EXPR, TREE_TYPE (purpose),
    6137              :                                          purpose,
    6138              :                                          build_int_cst (TREE_TYPE (purpose),
    6139              :                                                         i));
    6140           11 :                         cref = build4 (ARRAY_REF, array_elt_type,
    6141              :                                        unshare_expr (object), p, NULL_TREE,
    6142              :                                        NULL_TREE);
    6143              :                       }
    6144           12 :                     tree init
    6145           12 :                       = build2 (INIT_EXPR, TREE_TYPE (cref), cref,
    6146           12 :                                 build_int_cst (TREE_TYPE (value),
    6147           12 :                                                RAW_DATA_UCHAR_ELT (value, i)));
    6148           12 :                     gimplify_and_add (init, pre_p);
    6149           12 :                     ggc_free (init);
    6150              :                   }
    6151              :             }
    6152              :           else
    6153              :             {
    6154           42 :               tree rtype = build_array_type_nelts (TREE_TYPE (value),
    6155           42 :                                                    RAW_DATA_LENGTH (value));
    6156           42 :               tree rctor = build_constructor_single (rtype, bitsize_zero_node,
    6157              :                                                      value);
    6158           42 :               tree addr = build_fold_addr_expr (cref);
    6159           42 :               cref = build2 (MEM_REF, rtype, addr,
    6160              :                              build_int_cst (ptr_type_node, 0));
    6161           42 :               rctor = tree_output_constant_def (rctor);
    6162           42 :               if (!useless_type_conversion_p (rtype, TREE_TYPE (rctor)))
    6163            4 :                 rctor = build1 (VIEW_CONVERT_EXPR, rtype, rctor);
    6164           42 :               if (gimplify_expr (&cref, pre_p, NULL, is_gimple_lvalue,
    6165              :                                  fb_lvalue) != GS_ERROR)
    6166           84 :                 gimplify_seq_add_stmt (pre_p,
    6167           42 :                                        gimple_build_assign (cref, rctor));
    6168              :             }
    6169              :         }
    6170              :       else
    6171              :         {
    6172      2811399 :           tree init = build2 (INIT_EXPR, TREE_TYPE (cref), cref, value);
    6173      2811399 :           gimplify_and_add (init, pre_p);
    6174      2811399 :           ggc_free (init);
    6175              :         }
    6176              :     }
    6177      1277901 : }
    6178              : 
    6179              : /* Return the appropriate RHS predicate for this LHS.  */
    6180              : 
    6181              : gimple_predicate
    6182     49951361 : rhs_predicate_for (tree lhs)
    6183              : {
    6184     49951361 :   if (is_gimple_reg (lhs))
    6185              :     return is_gimple_reg_rhs_or_call;
    6186              :   else
    6187     11828632 :     return is_gimple_mem_rhs_or_call;
    6188              : }
    6189              : 
    6190              : /* Return the initial guess for an appropriate RHS predicate for this LHS,
    6191              :    before the LHS has been gimplified.  */
    6192              : 
    6193              : static gimple_predicate
    6194     48729450 : initial_rhs_predicate_for (tree lhs)
    6195              : {
    6196     48729450 :   if (is_gimple_reg_type (TREE_TYPE (lhs)))
    6197              :     return is_gimple_reg_rhs_or_call;
    6198              :   else
    6199      2463097 :     return is_gimple_mem_rhs_or_call;
    6200              : }
    6201              : 
    6202              : /* Gimplify a C99 compound literal expression.  This just means adding
    6203              :    the DECL_EXPR before the current statement and using its anonymous
    6204              :    decl instead.  */
    6205              : 
    6206              : static enum gimplify_status
    6207        37437 : gimplify_compound_literal_expr (tree *expr_p, gimple_seq *pre_p,
    6208              :                                 bool (*gimple_test_f) (tree),
    6209              :                                 fallback_t fallback)
    6210              : {
    6211        37437 :   tree decl_s = COMPOUND_LITERAL_EXPR_DECL_EXPR (*expr_p);
    6212        37437 :   tree decl = DECL_EXPR_DECL (decl_s);
    6213        37437 :   tree init = DECL_INITIAL (decl);
    6214              :   /* Mark the decl as addressable if the compound literal
    6215              :      expression is addressable now, otherwise it is marked too late
    6216              :      after we gimplify the initialization expression.  */
    6217        37437 :   if (TREE_ADDRESSABLE (*expr_p))
    6218          549 :     TREE_ADDRESSABLE (decl) = 1;
    6219              :   /* Otherwise, if we don't need an lvalue and have a literal directly
    6220              :      substitute it.  Check if it matches the gimple predicate, as
    6221              :      otherwise we'd generate a new temporary, and we can as well just
    6222              :      use the decl we already have.  */
    6223        36888 :   else if (!TREE_ADDRESSABLE (decl)
    6224        36888 :            && !TREE_THIS_VOLATILE (decl)
    6225        36883 :            && init
    6226        36875 :            && (fallback & fb_lvalue) == 0
    6227        73598 :            && gimple_test_f (init))
    6228              :     {
    6229        35617 :       *expr_p = init;
    6230        35617 :       return GS_OK;
    6231              :     }
    6232              : 
    6233              :   /* If the decl is not addressable, then it is being used in some
    6234              :      expression or on the right hand side of a statement, and it can
    6235              :      be put into a readonly data section.  */
    6236         1820 :   if (!TREE_ADDRESSABLE (decl) && (fallback & fb_lvalue) == 0)
    6237         1098 :     TREE_READONLY (decl) = 1;
    6238              : 
    6239              :   /* This decl isn't mentioned in the enclosing block, so add it to the
    6240              :      list of temps.  FIXME it seems a bit of a kludge to say that
    6241              :      anonymous artificial vars aren't pushed, but everything else is.  */
    6242         1820 :   if (DECL_NAME (decl) == NULL_TREE && !DECL_SEEN_IN_BIND_EXPR_P (decl))
    6243            2 :     gimple_add_tmp_var (decl);
    6244              : 
    6245         1820 :   gimplify_and_add (decl_s, pre_p);
    6246         1820 :   *expr_p = decl;
    6247         1820 :   return GS_OK;
    6248              : }
    6249              : 
    6250              : /* Optimize embedded COMPOUND_LITERAL_EXPRs within a CONSTRUCTOR,
    6251              :    return a new CONSTRUCTOR if something changed.  */
    6252              : 
    6253              : static tree
    6254      1905654 : optimize_compound_literals_in_ctor (tree orig_ctor)
    6255              : {
    6256      1905654 :   tree ctor = orig_ctor;
    6257      1905654 :   vec<constructor_elt, va_gc> *elts = CONSTRUCTOR_ELTS (ctor);
    6258      1905654 :   unsigned int idx, num = vec_safe_length (elts);
    6259              : 
    6260      6314315 :   for (idx = 0; idx < num; idx++)
    6261              :     {
    6262      4408661 :       tree value = (*elts)[idx].value;
    6263      4408661 :       tree newval = value;
    6264      4408661 :       if (TREE_CODE (value) == CONSTRUCTOR)
    6265       545595 :         newval = optimize_compound_literals_in_ctor (value);
    6266      3863066 :       else if (TREE_CODE (value) == COMPOUND_LITERAL_EXPR)
    6267              :         {
    6268          103 :           tree decl_s = COMPOUND_LITERAL_EXPR_DECL_EXPR (value);
    6269          103 :           tree decl = DECL_EXPR_DECL (decl_s);
    6270          103 :           tree init = DECL_INITIAL (decl);
    6271              : 
    6272          103 :           if (!TREE_ADDRESSABLE (value)
    6273          103 :               && !TREE_ADDRESSABLE (decl)
    6274          103 :               && init
    6275          103 :               && TREE_CODE (init) == CONSTRUCTOR)
    6276           52 :             newval = optimize_compound_literals_in_ctor (init);
    6277              :         }
    6278      4408661 :       if (newval == value)
    6279      4408601 :         continue;
    6280              : 
    6281           60 :       if (ctor == orig_ctor)
    6282              :         {
    6283           49 :           ctor = copy_node (orig_ctor);
    6284           98 :           CONSTRUCTOR_ELTS (ctor) = vec_safe_copy (elts);
    6285           49 :           elts = CONSTRUCTOR_ELTS (ctor);
    6286              :         }
    6287           60 :       (*elts)[idx].value = newval;
    6288              :     }
    6289      1905654 :   return ctor;
    6290              : }
    6291              : 
    6292              : /* A subroutine of gimplify_modify_expr.  Break out elements of a
    6293              :    CONSTRUCTOR used as an initializer into separate MODIFY_EXPRs.
    6294              : 
    6295              :    Note that we still need to clear any elements that don't have explicit
    6296              :    initializers, so if not all elements are initialized we keep the
    6297              :    original MODIFY_EXPR, we just remove all of the constructor elements.
    6298              : 
    6299              :    If NOTIFY_TEMP_CREATION is true, do not gimplify, just return
    6300              :    GS_ERROR if we would have to create a temporary when gimplifying
    6301              :    this constructor.  Otherwise, return GS_OK.
    6302              : 
    6303              :    If NOTIFY_TEMP_CREATION is false, just do the gimplification.  */
    6304              : 
    6305              : static enum gimplify_status
    6306      1360007 : gimplify_init_constructor (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
    6307              :                            bool want_value, bool notify_temp_creation)
    6308              : {
    6309      1360007 :   tree object, ctor, type;
    6310      1360007 :   enum gimplify_status ret;
    6311      1360007 :   vec<constructor_elt, va_gc> *elts;
    6312      1360007 :   bool cleared = false;
    6313      1360007 :   bool is_empty_ctor = false;
    6314      1360007 :   bool is_init_expr = (TREE_CODE (*expr_p) == INIT_EXPR);
    6315              : 
    6316      1360007 :   gcc_assert (TREE_CODE (TREE_OPERAND (*expr_p, 1)) == CONSTRUCTOR);
    6317              : 
    6318      1360007 :   if (!notify_temp_creation)
    6319              :     {
    6320      1351341 :       ret = gimplify_expr (&TREE_OPERAND (*expr_p, 0), pre_p, post_p,
    6321              :                            is_gimple_lvalue, fb_lvalue);
    6322      1351341 :       if (ret == GS_ERROR)
    6323              :         return ret;
    6324              :     }
    6325              : 
    6326      1360007 :   object = TREE_OPERAND (*expr_p, 0);
    6327      1360007 :   ctor = TREE_OPERAND (*expr_p, 1)
    6328      1360007 :     = optimize_compound_literals_in_ctor (TREE_OPERAND (*expr_p, 1));
    6329      1360007 :   type = TREE_TYPE (ctor);
    6330      1360007 :   elts = CONSTRUCTOR_ELTS (ctor);
    6331      1360007 :   ret = GS_ALL_DONE;
    6332              : 
    6333      1360007 :   switch (TREE_CODE (type))
    6334              :     {
    6335      1351272 :     case RECORD_TYPE:
    6336      1351272 :     case UNION_TYPE:
    6337      1351272 :     case QUAL_UNION_TYPE:
    6338      1351272 :     case ARRAY_TYPE:
    6339      1351272 :       {
    6340              :         /* Use readonly data for initializers of this or smaller size
    6341              :            regardless of the num_nonzero_elements / num_unique_nonzero_elements
    6342              :            ratio.  */
    6343      1351272 :         const HOST_WIDE_INT min_unique_size = 64;
    6344              :         /* If num_nonzero_elements / num_unique_nonzero_elements ratio
    6345              :            is smaller than this, use readonly data.  */
    6346      1351272 :         const int unique_nonzero_ratio = 8;
    6347              :         /* True if a single access of the object must be ensured.  This is the
    6348              :            case if the target is volatile, the type is non-addressable and more
    6349              :            than one field need to be assigned.  */
    6350      1351272 :         const bool ensure_single_access
    6351      1351272 :           = TREE_THIS_VOLATILE (object)
    6352          275 :             && !TREE_ADDRESSABLE (type)
    6353      1351485 :             && vec_safe_length (elts) > 1;
    6354      1351272 :         struct gimplify_init_ctor_preeval_data preeval_data;
    6355      1351272 :         HOST_WIDE_INT num_ctor_elements, num_nonzero_elements;
    6356      1351272 :         HOST_WIDE_INT num_unique_nonzero_elements;
    6357      1351272 :         ctor_completeness complete_p;
    6358      1351272 :         bool valid_const_initializer;
    6359              : 
    6360              :         /* Aggregate types must lower constructors to initialization of
    6361              :            individual elements.  The exception is that a CONSTRUCTOR node
    6362              :            with no elements indicates zero-initialization of the whole.  */
    6363      1351272 :         if (vec_safe_is_empty (elts))
    6364              :           {
    6365       341598 :             if (notify_temp_creation)
    6366        10202 :               return GS_OK;
    6367              : 
    6368              :             /* The var will be initialized and so appear on lhs of
    6369              :                assignment, it can't be TREE_READONLY anymore.  */
    6370       341550 :             if (VAR_P (object))
    6371       189571 :               TREE_READONLY (object) = 0;
    6372              : 
    6373              :             is_empty_ctor = true;
    6374       346116 :             break;
    6375              :           }
    6376              : 
    6377              :         /* Fetch information about the constructor to direct later processing.
    6378              :            We might want to make static versions of it in various cases, and
    6379              :            can only do so if it known to be a valid constant initializer.  */
    6380      1009674 :         valid_const_initializer
    6381      1009674 :           = categorize_ctor_elements (ctor, &num_nonzero_elements,
    6382              :                                       &num_unique_nonzero_elements,
    6383              :                                       &num_ctor_elements, &complete_p);
    6384              : 
    6385              :         /* If a const aggregate variable is being initialized, then it
    6386              :            should never be a lose to promote the variable to be static.  */
    6387      1009674 :         if (valid_const_initializer
    6388       626037 :             && num_nonzero_elements > 1
    6389       458588 :             && TREE_READONLY (object)
    6390         8284 :             && VAR_P (object)
    6391         7907 :             && !DECL_REGISTER (object)
    6392         7900 :             && (flag_merge_constants >= 2 || !TREE_ADDRESSABLE (object)
    6393         5749 :                 || DECL_MERGEABLE (object))
    6394              :             /* For ctors that have many repeated nonzero elements
    6395              :                represented through RANGE_EXPRs, prefer initializing
    6396              :                those through runtime loops over copies of large amounts
    6397              :                of data from readonly data section.  */
    6398      1009674 :             && (num_unique_nonzero_elements
    6399         4566 :                 > num_nonzero_elements / unique_nonzero_ratio
    6400            0 :                 || ((unsigned HOST_WIDE_INT) int_size_in_bytes (type)
    6401              :                     <= (unsigned HOST_WIDE_INT) min_unique_size)))
    6402              :           {
    6403         4566 :             if (notify_temp_creation)
    6404              :               return GS_ERROR;
    6405              : 
    6406         4566 :             DECL_INITIAL (object) = ctor;
    6407         4566 :             TREE_STATIC (object) = 1;
    6408         4566 :             if (!DECL_NAME (object) || DECL_NAMELESS (object))
    6409         3600 :               DECL_NAME (object) = create_tmp_var_name ("C");
    6410         4566 :             walk_tree (&DECL_INITIAL (object), force_labels_r, NULL, NULL);
    6411              : 
    6412              :             /* ??? C++ doesn't automatically append a .<number> to the
    6413              :                assembler name, and even when it does, it looks at FE private
    6414              :                data structures to figure out what that number should be,
    6415              :                which are not set for this variable.  I suppose this is
    6416              :                important for local statics for inline functions, which aren't
    6417              :                "local" in the object file sense.  So in order to get a unique
    6418              :                TU-local symbol, we must invoke the lhd version now.  */
    6419         4566 :             lhd_set_decl_assembler_name (object);
    6420              : 
    6421         4566 :             *expr_p = NULL_TREE;
    6422         4566 :             break;
    6423              :           }
    6424              : 
    6425              :         /* The var will be initialized and so appear on lhs of
    6426              :            assignment, it can't be TREE_READONLY anymore.  */
    6427      1005108 :         if (VAR_P (object) && !notify_temp_creation)
    6428       791868 :           TREE_READONLY (object) = 0;
    6429              : 
    6430              :         /* If there are "lots" of initialized elements, even discounting
    6431              :            those that are not address constants (and thus *must* be
    6432              :            computed at runtime), then partition the constructor into
    6433              :            constant and non-constant parts.  Block copy the constant
    6434              :            parts in, then generate code for the non-constant parts.  */
    6435              :         /* TODO.  There's code in cp/typeck.cc to do this.  */
    6436              : 
    6437      1005108 :         if (int_size_in_bytes (TREE_TYPE (ctor)) < 0)
    6438              :           /* store_constructor will ignore the clearing of variable-sized
    6439              :              objects.  Initializers for such objects must explicitly set
    6440              :              every field that needs to be set.  */
    6441              :           cleared = false;
    6442      1005088 :         else if (complete_p.sparse)
    6443              :           /* If the constructor isn't complete, clear the whole object
    6444              :              beforehand, unless CONSTRUCTOR_NO_CLEARING is set on it.
    6445              : 
    6446              :              ??? This ought not to be needed.  For any element not present
    6447              :              in the initializer, we should simply set them to zero.  Except
    6448              :              we'd need to *find* the elements that are not present, and that
    6449              :              requires trickery to avoid quadratic compile-time behavior in
    6450              :              large cases or excessive memory use in small cases.  */
    6451       194525 :           cleared = !CONSTRUCTOR_NO_CLEARING (ctor);
    6452       810563 :         else if (num_ctor_elements - num_nonzero_elements
    6453       810563 :                  > CLEAR_RATIO (optimize_function_for_speed_p (cfun))
    6454       810563 :                  && num_nonzero_elements < num_ctor_elements / 4)
    6455              :           /* If there are "lots" of zeros, it's more efficient to clear
    6456              :              the memory and then set the nonzero elements.  */
    6457              :           cleared = true;
    6458       809532 :         else if (ensure_single_access && num_nonzero_elements == 0)
    6459              :           /* If a single access to the target must be ensured and all elements
    6460              :              are zero, then it's optimal to clear whatever their number.  */
    6461              :           cleared = true;
    6462              :         /* If the object is small enough to go in registers, and it's
    6463              :            not required to be constructed in memory, clear it first.
    6464              :            That will avoid wasting cycles preserving any padding bits
    6465              :            that might be there, and if there aren't any, the compiler
    6466              :            is smart enough to optimize the clearing out.  */
    6467       809523 :         else if ((complete_p.sparse || complete_p.padded_union
    6468       809133 :                   || complete_p.padded_non_union)
    6469        15240 :                  && !TREE_ADDRESSABLE (ctor)
    6470        15226 :                  && !TREE_THIS_VOLATILE (object)
    6471        15223 :                  && (TYPE_MODE (type) != BLKmode || TYPE_NO_FORCE_BLK (type))
    6472       816846 :                  && optimize)
    6473              :           cleared = true;
    6474              :         else
    6475              :           cleared = false;
    6476              : 
    6477              :         /* If there are "lots" of initialized elements, and all of them
    6478              :            are valid address constants, then the entire initializer can
    6479              :            be dropped to memory, and then memcpy'd out.  Don't do this
    6480              :            for sparse arrays, though, as it's more efficient to follow
    6481              :            the standard CONSTRUCTOR behavior of memset followed by
    6482              :            individual element initialization.  Also don't do this for small
    6483              :            all-zero initializers (which aren't big enough to merit
    6484              :            clearing), and don't try to make bitwise copies of
    6485              :            TREE_ADDRESSABLE types.  */
    6486      1005108 :         if (valid_const_initializer
    6487       621471 :             && !complete_p.sparse
    6488       442129 :             && !(cleared || num_nonzero_elements == 0)
    6489       345225 :             && !TREE_ADDRESSABLE (type))
    6490              :           {
    6491       343339 :             HOST_WIDE_INT size = int_size_in_bytes (type);
    6492       343339 :             unsigned int align;
    6493              : 
    6494              :             /* ??? We can still get unbounded array types, at least
    6495              :                from the C++ front end.  This seems wrong, but attempt
    6496              :                to work around it for now.  */
    6497       343339 :             if (size < 0)
    6498              :               {
    6499            9 :                 size = int_size_in_bytes (TREE_TYPE (object));
    6500            9 :                 if (size >= 0)
    6501            0 :                   TREE_TYPE (ctor) = type = TREE_TYPE (object);
    6502              :               }
    6503              : 
    6504              :             /* Find the maximum alignment we can assume for the object.  */
    6505              :             /* ??? Make use of DECL_OFFSET_ALIGN.  */
    6506       343339 :             if (DECL_P (object))
    6507       328979 :               align = DECL_ALIGN (object);
    6508              :             else
    6509        14360 :               align = TYPE_ALIGN (type);
    6510              : 
    6511              :             /* Do a block move either if the size is so small as to make
    6512              :                each individual move a sub-unit move on average, or if it
    6513              :                is so large as to make individual moves inefficient.  */
    6514       343339 :             if (size > 0
    6515       343330 :                 && num_nonzero_elements > 1
    6516              :                 /* For ctors that have many repeated nonzero elements
    6517              :                    represented through RANGE_EXPRs, prefer initializing
    6518              :                    those through runtime loops over copies of large amounts
    6519              :                    of data from readonly data section.  */
    6520       303944 :                 && (num_unique_nonzero_elements
    6521       303944 :                     > num_nonzero_elements / unique_nonzero_ratio
    6522           56 :                     || size <= min_unique_size)
    6523       647227 :                 && (size < num_nonzero_elements
    6524       303782 :                     || !can_move_by_pieces (size, align)))
    6525              :               {
    6526         2746 :                 if (notify_temp_creation)
    6527              :                   return GS_ERROR;
    6528              : 
    6529         1375 :                 walk_tree (&ctor, force_labels_r, NULL, NULL);
    6530         1375 :                 ctor = tree_output_constant_def (ctor);
    6531         1375 :                 if (!useless_type_conversion_p (type, TREE_TYPE (ctor)))
    6532            0 :                   ctor = build1 (VIEW_CONVERT_EXPR, type, ctor);
    6533         1375 :                 TREE_OPERAND (*expr_p, 1) = ctor;
    6534              : 
    6535              :                 /* This is no longer an assignment of a CONSTRUCTOR, but
    6536              :                    we still may have processing to do on the LHS.  So
    6537              :                    pretend we didn't do anything here to let that happen.  */
    6538         1375 :                 return GS_UNHANDLED;
    6539              :               }
    6540              :           }
    6541              : 
    6542       997563 :         if (!cleared)
    6543              :           {
    6544       800238 :             if (complete_p.padded_non_union
    6545         8125 :                 && warn_zero_init_padding_bits >= ZERO_INIT_PADDING_BITS_ALL)
    6546           12 :                 warning (OPT_Wzero_init_padding_bits_,
    6547              :                          "padding might not be initialized to zero; "
    6548              :                          "if code relies on it being zero, consider "
    6549              :                          "using %<-fzero-init-padding-bits=all%>");
    6550       800226 :             else if (complete_p.padded_union
    6551          122 :                      && warn_zero_init_padding_bits
    6552              :                         >= ZERO_INIT_PADDING_BITS_UNIONS)
    6553           14 :                 warning (OPT_Wzero_init_padding_bits_,
    6554              :                          "padding might not be initialized to zero; "
    6555              :                          "if code relies on it being zero, consider "
    6556              :                          "using %<-fzero-init-padding-bits=unions%> "
    6557              :                          "or %<-fzero-init-padding-bits=all%>");
    6558              :           }
    6559              : 
    6560              :         /* If a single access to the target must be ensured and there are
    6561              :            nonzero elements or the zero elements are not assigned en masse,
    6562              :            initialize the target from a temporary.  */
    6563      1002362 :         if (ensure_single_access && (num_nonzero_elements > 0 || !cleared))
    6564              :           {
    6565          167 :             if (notify_temp_creation)
    6566              :               return GS_ERROR;
    6567              : 
    6568          161 :             tree temp = create_tmp_var (TYPE_MAIN_VARIANT (type));
    6569          161 :             TREE_OPERAND (*expr_p, 0) = temp;
    6570          161 :             *expr_p = build2 (COMPOUND_EXPR, TREE_TYPE (*expr_p),
    6571              :                               *expr_p,
    6572              :                               build2 (MODIFY_EXPR, void_type_node,
    6573              :                                       object, temp));
    6574          161 :             return GS_OK;
    6575              :           }
    6576              : 
    6577      1002195 :         if (notify_temp_creation)
    6578              :           return GS_OK;
    6579              : 
    6580              :         /* If there are nonzero elements and if needed, pre-evaluate to capture
    6581              :            elements overlapping with the lhs into temporaries.  We must do this
    6582              :            before clearing to fetch the values before they are zeroed-out.  */
    6583       994954 :         if (num_nonzero_elements > 0 && TREE_CODE (*expr_p) != INIT_EXPR)
    6584              :           {
    6585       222043 :             preeval_data.lhs_base_decl = get_base_address (object);
    6586       222043 :             if (!DECL_P (preeval_data.lhs_base_decl))
    6587        41731 :               preeval_data.lhs_base_decl = NULL;
    6588       222043 :             preeval_data.lhs_alias_set = get_alias_set (object);
    6589              : 
    6590       222043 :             gimplify_init_ctor_preeval (&TREE_OPERAND (*expr_p, 1),
    6591              :                                         pre_p, post_p, &preeval_data);
    6592              :           }
    6593              : 
    6594       994954 :         bool ctor_has_side_effects_p
    6595       994954 :           = TREE_SIDE_EFFECTS (TREE_OPERAND (*expr_p, 1));
    6596              : 
    6597       994954 :         if (cleared)
    6598              :           {
    6599              :             /* Zap the CONSTRUCTOR element list, which simplifies this case.
    6600              :                Note that we still have to gimplify, in order to handle the
    6601              :                case of variable sized types.  Avoid shared tree structures.  */
    6602       201879 :             CONSTRUCTOR_ELTS (ctor) = NULL;
    6603       201879 :             TREE_SIDE_EFFECTS (ctor) = 0;
    6604       201879 :             object = unshare_expr (object);
    6605       201879 :             gimplify_stmt (expr_p, pre_p);
    6606              :           }
    6607              : 
    6608              :         /* If we have not block cleared the object, or if there are nonzero
    6609              :            elements in the constructor, or if the constructor has side effects,
    6610              :            add assignments to the individual scalar fields of the object.  */
    6611       201879 :         if (!cleared
    6612       201879 :             || num_nonzero_elements > 0
    6613        20680 :             || ctor_has_side_effects_p)
    6614       974670 :           gimplify_init_ctor_eval (object, elts, pre_p, cleared);
    6615              : 
    6616       994954 :         *expr_p = NULL_TREE;
    6617              :       }
    6618       994954 :       break;
    6619              : 
    6620            0 :     case COMPLEX_TYPE:
    6621            0 :       {
    6622            0 :         tree r, i;
    6623              : 
    6624            0 :         if (notify_temp_creation)
    6625              :           return GS_OK;
    6626              : 
    6627              :         /* Extract the real and imaginary parts out of the ctor.  */
    6628            0 :         gcc_assert (elts->length () == 2);
    6629            0 :         r = (*elts)[0].value;
    6630            0 :         i = (*elts)[1].value;
    6631            0 :         if (r == NULL || i == NULL)
    6632              :           {
    6633            0 :             tree zero = build_zero_cst (TREE_TYPE (type));
    6634            0 :             if (r == NULL)
    6635            0 :               r = zero;
    6636            0 :             if (i == NULL)
    6637            0 :               i = zero;
    6638              :           }
    6639              : 
    6640              :         /* Complex types have either COMPLEX_CST or COMPLEX_EXPR to
    6641              :            represent creation of a complex value.  */
    6642            0 :         if (TREE_CONSTANT (r) && TREE_CONSTANT (i))
    6643              :           {
    6644            0 :             ctor = build_complex (type, r, i);
    6645            0 :             TREE_OPERAND (*expr_p, 1) = ctor;
    6646              :           }
    6647              :         else
    6648              :           {
    6649            0 :             ctor = build2 (COMPLEX_EXPR, type, r, i);
    6650            0 :             TREE_OPERAND (*expr_p, 1) = ctor;
    6651            0 :             ret = gimplify_expr (&TREE_OPERAND (*expr_p, 1),
    6652              :                                  pre_p,
    6653              :                                  post_p,
    6654            0 :                                  rhs_predicate_for (TREE_OPERAND (*expr_p, 0)),
    6655              :                                  fb_rvalue);
    6656              :           }
    6657              :       }
    6658              :       break;
    6659              : 
    6660         8735 :     case VECTOR_TYPE:
    6661         8735 :       {
    6662         8735 :         unsigned HOST_WIDE_INT ix;
    6663         8735 :         constructor_elt *ce;
    6664              : 
    6665         8735 :         if (notify_temp_creation)
    6666      1360007 :           return GS_OK;
    6667              : 
    6668              :         /* Vector types use CONSTRUCTOR all the way through gimple
    6669              :            compilation as a general initializer.  */
    6670        76113 :         FOR_EACH_VEC_SAFE_ELT (elts, ix, ce)
    6671              :           {
    6672        67378 :             enum gimplify_status tret;
    6673        67378 :             tret = gimplify_expr (&ce->value, pre_p, post_p, is_gimple_val,
    6674              :                                   fb_rvalue);
    6675        67378 :             if (tret == GS_ERROR)
    6676              :               ret = GS_ERROR;
    6677        67377 :             else if (TREE_STATIC (ctor)
    6678        67410 :                      && !initializer_constant_valid_p (ce->value,
    6679           33 :                                                        TREE_TYPE (ce->value)))
    6680           33 :               TREE_STATIC (ctor) = 0;
    6681              :           }
    6682         8735 :         recompute_constructor_flags (ctor);
    6683              : 
    6684              :         /* Go ahead and simplify constant constructors to VECTOR_CST.  */
    6685         8735 :         if (TREE_CONSTANT (ctor))
    6686              :           {
    6687         2705 :             bool constant_p = true;
    6688              :             tree value;
    6689              : 
    6690              :             /* Even when ctor is constant, it might contain non-*_CST
    6691              :                elements, such as addresses or trapping values like
    6692              :                1.0/0.0 - 1.0/0.0.  Such expressions don't belong
    6693              :                in VECTOR_CST nodes.  */
    6694         2705 :             FOR_EACH_CONSTRUCTOR_VALUE (elts, ix, value)
    6695         2483 :               if (!CONSTANT_CLASS_P (value))
    6696              :                 {
    6697              :                   constant_p = false;
    6698              :                   break;
    6699              :                 }
    6700              : 
    6701          222 :             if (constant_p)
    6702              :               {
    6703          222 :                 TREE_OPERAND (*expr_p, 1) = build_vector_from_ctor (type, elts);
    6704          222 :                 break;
    6705              :               }
    6706              :           }
    6707              : 
    6708         8513 :         if (!is_gimple_reg (TREE_OPERAND (*expr_p, 0)))
    6709          956 :           TREE_OPERAND (*expr_p, 1) = get_formal_tmp_var (ctor, pre_p);
    6710              :       }
    6711              :       break;
    6712              : 
    6713            0 :     default:
    6714              :       /* So how did we get a CONSTRUCTOR for a scalar type?  */
    6715            0 :       gcc_unreachable ();
    6716              :     }
    6717              : 
    6718      1349805 :   if (ret == GS_ERROR)
    6719              :     return GS_ERROR;
    6720              :   /* If we have gimplified both sides of the initializer but have
    6721              :      not emitted an assignment, do so now.   */
    6722      1349804 :   if (*expr_p
    6723              :       /* If the type is an empty type, we don't need to emit the
    6724              :          assignment. */
    6725      1349804 :       && !is_empty_type (TREE_TYPE (TREE_OPERAND (*expr_p, 0))))
    6726              :     {
    6727       346120 :       tree lhs = TREE_OPERAND (*expr_p, 0);
    6728       346120 :       tree rhs = TREE_OPERAND (*expr_p, 1);
    6729       346120 :       if (want_value && object == lhs)
    6730           12 :         lhs = unshare_expr (lhs);
    6731       346120 :       gassign *init = gimple_build_assign (lhs, rhs);
    6732       346120 :       gimplify_seq_add_stmt (pre_p, init);
    6733              :     }
    6734      1349804 :   if (want_value)
    6735              :     {
    6736              :       *expr_p = object;
    6737      1349804 :       ret = GS_OK;
    6738              :     }
    6739              :   else
    6740              :     {
    6741      1349785 :       *expr_p = NULL;
    6742      1349785 :       ret = GS_ALL_DONE;
    6743              :     }
    6744              : 
    6745              :   /* If the user requests to initialize automatic variables, we
    6746              :      should initialize paddings inside the variable.  Add a call to
    6747              :      __builtin_clear_pading (&object, 0, for_auto_init = true) to
    6748              :      initialize paddings of object always to zero regardless of
    6749              :      INIT_TYPE.  Note, we will not insert this call if the aggregate
    6750              :      variable has be completely cleared already or it's initialized
    6751              :      with an empty constructor.  We cannot insert this call if the
    6752              :      variable is a gimple register since __builtin_clear_padding will take
    6753              :      the address of the variable.  As a result, if a long double/_Complex long
    6754              :      double variable will be spilled into stack later, its padding cannot
    6755              :      be cleared with __builtin_clear_padding.  We should clear its padding
    6756              :      when it is spilled into memory.  */
    6757      1349804 :   if (is_init_expr
    6758       957952 :       && !is_gimple_reg (object)
    6759       952504 :       && clear_padding_type_may_have_padding_p (type)
    6760       906521 :       && ((AGGREGATE_TYPE_P (type) && !cleared && !is_empty_ctor)
    6761       230357 :           || !AGGREGATE_TYPE_P (type))
    6762       676164 :       && var_needs_auto_init_p (object)
    6763      1386471 :       && flag_auto_var_init != AUTO_INIT_CXX26)
    6764           41 :     gimple_add_padding_init_for_auto_var (object, false, pre_p);
    6765              : 
    6766              :   return ret;
    6767              : }
    6768              : 
    6769              : /* Given a pointer value OP0, return a simplified version of an
    6770              :    indirection through OP0, or NULL_TREE if no simplification is
    6771              :    possible.  This may only be applied to a rhs of an expression.
    6772              :    Note that the resulting type may be different from the type pointed
    6773              :    to in the sense that it is still compatible from the langhooks
    6774              :    point of view. */
    6775              : 
    6776              : static tree
    6777       532872 : gimple_fold_indirect_ref_rhs (tree t)
    6778              : {
    6779            0 :   return gimple_fold_indirect_ref (t);
    6780              : }
    6781              : 
    6782              : /* Subroutine of gimplify_modify_expr to do simplifications of
    6783              :    MODIFY_EXPRs based on the code of the RHS.  We loop for as long as
    6784              :    something changes.  */
    6785              : 
    6786              : static enum gimplify_status
    6787     99545631 : gimplify_modify_expr_rhs (tree *expr_p, tree *from_p, tree *to_p,
    6788              :                           gimple_seq *pre_p, gimple_seq *post_p,
    6789              :                           bool want_value)
    6790              : {
    6791     99545631 :   enum gimplify_status ret = GS_UNHANDLED;
    6792    100200151 :   bool changed;
    6793              : 
    6794     98695159 :   do
    6795              :     {
    6796    100200151 :       changed = false;
    6797    100200151 :       switch (TREE_CODE (*from_p))
    6798              :         {
    6799     12074957 :         case VAR_DECL:
    6800              :           /* If we're assigning from a read-only variable initialized with
    6801              :              a constructor and not volatile, do the direct assignment from
    6802              :              the constructor, but only if the target is not volatile either
    6803              :              since this latter assignment might end up being done on a per
    6804              :              field basis.  However, if the target is volatile and the type
    6805              :              is aggregate and non-addressable, gimplify_init_constructor
    6806              :              knows that it needs to ensure a single access to the target
    6807              :              and it will return GS_OK only in this case.  */
    6808     12074957 :           if (TREE_READONLY (*from_p)
    6809        95315 :               && DECL_INITIAL (*from_p)
    6810        61563 :               && TREE_CODE (DECL_INITIAL (*from_p)) == CONSTRUCTOR
    6811         8666 :               && !TREE_THIS_VOLATILE (*from_p)
    6812     12083623 :               && (!TREE_THIS_VOLATILE (*to_p)
    6813            6 :                   || (AGGREGATE_TYPE_P (TREE_TYPE (*to_p))
    6814            6 :                       && !TREE_ADDRESSABLE (TREE_TYPE (*to_p)))))
    6815              :             {
    6816         8666 :               tree old_from = *from_p;
    6817         8666 :               enum gimplify_status subret;
    6818              : 
    6819              :               /* Move the constructor into the RHS.  */
    6820         8666 :               *from_p = unshare_expr (DECL_INITIAL (*from_p));
    6821              : 
    6822              :               /* Let's see if gimplify_init_constructor will need to put
    6823              :                  it in memory.  */
    6824         8666 :               subret = gimplify_init_constructor (expr_p, NULL, NULL,
    6825              :                                                   false, true);
    6826         8666 :               if (subret == GS_ERROR)
    6827              :                 {
    6828              :                   /* If so, revert the change.  */
    6829         1377 :                   *from_p = old_from;
    6830              :                 }
    6831              :               else
    6832              :                 {
    6833              :                   ret = GS_OK;
    6834              :                   changed = true;
    6835              :                 }
    6836              :             }
    6837              :           break;
    6838       541105 :         case INDIRECT_REF:
    6839       541105 :           if (!TREE_ADDRESSABLE (TREE_TYPE (*from_p)))
    6840              :             /* If we have code like
    6841              : 
    6842              :              *(const A*)(A*)&x
    6843              : 
    6844              :              where the type of "x" is a (possibly cv-qualified variant
    6845              :              of "A"), treat the entire expression as identical to "x".
    6846              :              This kind of code arises in C++ when an object is bound
    6847              :              to a const reference, and if "x" is a TARGET_EXPR we want
    6848              :              to take advantage of the optimization below.  But not if
    6849              :              the type is TREE_ADDRESSABLE; then C++17 says that the
    6850              :              TARGET_EXPR needs to be a temporary.  */
    6851      1065744 :             if (tree t
    6852       532872 :                 = gimple_fold_indirect_ref_rhs (TREE_OPERAND (*from_p, 0)))
    6853              :               {
    6854       130456 :                 bool volatile_p = TREE_THIS_VOLATILE (*from_p);
    6855       130456 :                 if (TREE_THIS_VOLATILE (t) != volatile_p)
    6856              :                   {
    6857            5 :                     if (DECL_P (t))
    6858            0 :                       t = build_simple_mem_ref_loc (EXPR_LOCATION (*from_p),
    6859              :                                                     build_fold_addr_expr (t));
    6860            5 :                     if (REFERENCE_CLASS_P (t))
    6861            5 :                       TREE_THIS_VOLATILE (t) = volatile_p;
    6862              :                   }
    6863       130456 :                 *from_p = t;
    6864       130456 :                 ret = GS_OK;
    6865       130456 :                 changed = true;
    6866              :               }
    6867              :           break;
    6868              : 
    6869       199449 :         case TARGET_EXPR:
    6870       199449 :           {
    6871              :             /* If we are initializing something from a TARGET_EXPR, strip the
    6872              :                TARGET_EXPR and initialize it directly, if possible.  This can't
    6873              :                be done if the initializer is void, since that implies that the
    6874              :                temporary is set in some non-trivial way.
    6875              : 
    6876              :                ??? What about code that pulls out the temp and uses it
    6877              :                elsewhere? I think that such code never uses the TARGET_EXPR as
    6878              :                an initializer.  If I'm wrong, we'll die because the temp won't
    6879              :                have any RTL.  In that case, I guess we'll need to replace
    6880              :                references somehow.  */
    6881       199449 :             tree init = TARGET_EXPR_INITIAL (*from_p);
    6882              : 
    6883       199449 :             if (init
    6884       198733 :                 && (TREE_CODE (*expr_p) != MODIFY_EXPR
    6885        51739 :                     || !TARGET_EXPR_NO_ELIDE (*from_p))
    6886       397964 :                 && !VOID_TYPE_P (TREE_TYPE (init)))
    6887              :               {
    6888       182844 :                 *from_p = init;
    6889       182844 :                 ret = GS_OK;
    6890       182844 :                 changed = true;
    6891              :               }
    6892              :           }
    6893              :           break;
    6894              : 
    6895       333566 :         case COMPOUND_EXPR:
    6896              :           /* Remove any COMPOUND_EXPR in the RHS so the following cases will be
    6897              :              caught.  */
    6898       333566 :           gimplify_compound_expr (from_p, pre_p, true);
    6899       333566 :           ret = GS_OK;
    6900       333566 :           changed = true;
    6901       333566 :           break;
    6902              : 
    6903      1372305 :         case CONSTRUCTOR:
    6904              :           /* If we already made some changes, let the front end have a
    6905              :              crack at this before we break it down.  */
    6906      1372305 :           if (ret != GS_UNHANDLED)
    6907              :             break;
    6908              : 
    6909              :           /* If we're initializing from a CONSTRUCTOR, break this into
    6910              :              individual MODIFY_EXPRs.  */
    6911      1351341 :           ret = gimplify_init_constructor (expr_p, pre_p, post_p, want_value,
    6912              :                                            false);
    6913      1351341 :           return ret;
    6914              : 
    6915       228564 :         case COND_EXPR:
    6916              :           /* If we're assigning to a non-register type, push the assignment
    6917              :              down into the branches.  This is mandatory for ADDRESSABLE types,
    6918              :              since we cannot generate temporaries for such, but it saves a
    6919              :              copy in other cases as well.
    6920              :              Also avoid an extra temporary and copy when assigning to
    6921              :              a register.  */
    6922       228564 :           if (!is_gimple_reg_type (TREE_TYPE (*from_p))
    6923       228564 :               || (is_gimple_reg (*to_p) && !gimplify_ctxp->allow_rhs_cond_expr))
    6924              :             {
    6925              :               /* This code should mirror the code in gimplify_cond_expr. */
    6926       146640 :               enum tree_code code = TREE_CODE (*expr_p);
    6927       146640 :               tree cond = *from_p;
    6928       146640 :               tree result = *to_p;
    6929              : 
    6930       146640 :               ret = gimplify_expr (&result, pre_p, post_p,
    6931              :                                    is_gimple_lvalue, fb_lvalue);
    6932       146640 :               if (ret != GS_ERROR)
    6933       146640 :                 ret = GS_OK;
    6934              : 
    6935              :               /* If we are going to write RESULT more than once, clear
    6936              :                  TREE_READONLY flag, otherwise we might incorrectly promote
    6937              :                  the variable to static const and initialize it at compile
    6938              :                  time in one of the branches.  */
    6939       146640 :               if (VAR_P (result)
    6940       145612 :                   && TREE_TYPE (TREE_OPERAND (cond, 1)) != void_type_node
    6941       283179 :                   && TREE_TYPE (TREE_OPERAND (cond, 2)) != void_type_node)
    6942       136418 :                 TREE_READONLY (result) = 0;
    6943       146640 :               if (TREE_TYPE (TREE_OPERAND (cond, 1)) != void_type_node)
    6944       137431 :                 TREE_OPERAND (cond, 1)
    6945       274862 :                   = build2 (code, void_type_node, result,
    6946       137431 :                             TREE_OPERAND (cond, 1));
    6947       146640 :               if (TREE_TYPE (TREE_OPERAND (cond, 2)) != void_type_node)
    6948       146519 :                 TREE_OPERAND (cond, 2)
    6949       293038 :                   = build2 (code, void_type_node, unshare_expr (result),
    6950       146519 :                             TREE_OPERAND (cond, 2));
    6951              : 
    6952       146640 :               TREE_TYPE (cond) = void_type_node;
    6953       146640 :               recalculate_side_effects (cond);
    6954              : 
    6955       146640 :               if (want_value)
    6956              :                 {
    6957           68 :                   gimplify_and_add (cond, pre_p);
    6958           68 :                   *expr_p = unshare_expr (result);
    6959              :                 }
    6960              :               else
    6961              :                 *expr_p = cond;
    6962       146640 :               return ret;
    6963              :             }
    6964              :           break;
    6965              : 
    6966     10302743 :         case CALL_EXPR:
    6967              :           /* For calls that return in memory, give *to_p as the CALL_EXPR's
    6968              :              return slot so that we don't generate a temporary.  */
    6969     10302743 :           if (!CALL_EXPR_RETURN_SLOT_OPT (*from_p)
    6970     10302743 :               && aggregate_value_p (*from_p, *from_p))
    6971              :             {
    6972       254192 :               bool use_target;
    6973              : 
    6974       254192 :               if (!(rhs_predicate_for (*to_p))(*from_p))
    6975              :                 /* If we need a temporary, *to_p isn't accurate.  */
    6976              :                 use_target = false;
    6977              :               /* It's OK to use the return slot directly unless it's an NRV. */
    6978       253371 :               else if (TREE_CODE (*to_p) == RESULT_DECL
    6979         6800 :                        && DECL_NAME (*to_p) == NULL_TREE
    6980       260026 :                        && needs_to_live_in_memory (*to_p))
    6981              :                 use_target = true;
    6982       246716 :               else if (is_gimple_reg_type (TREE_TYPE (*to_p))
    6983       246716 :                        || (DECL_P (*to_p) && DECL_REGISTER (*to_p)))
    6984              :                 /* Don't force regs into memory.  */
    6985              :                 use_target = false;
    6986       232062 :               else if (TREE_CODE (*expr_p) == INIT_EXPR)
    6987              :                 /* It's OK to use the target directly if it's being
    6988              :                    initialized. */
    6989              :                 use_target = true;
    6990        14807 :               else if (TREE_CODE (TYPE_SIZE_UNIT (TREE_TYPE (*to_p)))
    6991              :                        != INTEGER_CST)
    6992              :                 /* Always use the target and thus RSO for variable-sized types.
    6993              :                    GIMPLE cannot deal with a variable-sized assignment
    6994              :                    embedded in a call statement.  */
    6995              :                 use_target = true;
    6996        14779 :               else if (TREE_CODE (*to_p) != SSA_NAME
    6997        14779 :                       && (!is_gimple_variable (*to_p)
    6998        11394 :                           || needs_to_live_in_memory (*to_p)))
    6999              :                 /* Don't use the original target if it's already addressable;
    7000              :                    if its address escapes, and the called function uses the
    7001              :                    NRV optimization, a conforming program could see *to_p
    7002              :                    change before the called function returns; see c++/19317.
    7003              :                    When optimizing, the return_slot pass marks more functions
    7004              :                    as safe after we have escape info.  */
    7005              :                 use_target = false;
    7006              :               else
    7007              :                 use_target = true;
    7008              : 
    7009              :               if (use_target)
    7010              :                 {
    7011       226139 :                   CALL_EXPR_RETURN_SLOT_OPT (*from_p) = 1;
    7012       226139 :                   mark_addressable (*to_p);
    7013              :                 }
    7014              :             }
    7015              :           break;
    7016              : 
    7017         1566 :         case WITH_SIZE_EXPR:
    7018              :           /* Likewise for calls that return an aggregate of non-constant size,
    7019              :              since we would not be able to generate a temporary at all.  */
    7020         1566 :           if (TREE_CODE (TREE_OPERAND (*from_p, 0)) == CALL_EXPR)
    7021              :             {
    7022          352 :               *from_p = TREE_OPERAND (*from_p, 0);
    7023              :               /* We don't change ret in this case because the
    7024              :                  WITH_SIZE_EXPR might have been added in
    7025              :                  gimplify_modify_expr, so returning GS_OK would lead to an
    7026              :                  infinite loop.  */
    7027          352 :               changed = true;
    7028              :             }
    7029              :           break;
    7030              : 
    7031              :           /* If we're initializing from a container, push the initialization
    7032              :              inside it.  */
    7033         2021 :         case CLEANUP_POINT_EXPR:
    7034         2021 :         case BIND_EXPR:
    7035         2021 :         case STATEMENT_LIST:
    7036         2021 :           {
    7037         2021 :             tree wrap = *from_p;
    7038         2021 :             tree t;
    7039              : 
    7040         2021 :             ret = gimplify_expr (to_p, pre_p, post_p, is_gimple_min_lval,
    7041              :                                  fb_lvalue);
    7042         2021 :             if (ret != GS_ERROR)
    7043         2021 :               ret = GS_OK;
    7044              : 
    7045         2021 :             t = voidify_wrapper_expr (wrap, *expr_p);
    7046         2021 :             gcc_assert (t == *expr_p);
    7047              : 
    7048         2021 :             if (want_value)
    7049              :               {
    7050            6 :                 gimplify_and_add (wrap, pre_p);
    7051            6 :                 *expr_p = unshare_expr (*to_p);
    7052              :               }
    7053              :             else
    7054         2015 :               *expr_p = wrap;
    7055              :             return GS_OK;
    7056              :           }
    7057              : 
    7058      9878110 :         case NOP_EXPR:
    7059              :           /* Pull out compound literal expressions from a NOP_EXPR.
    7060              :              Those are created in the C FE to drop qualifiers during
    7061              :              lvalue conversion.  */
    7062      9878110 :           if ((TREE_CODE (TREE_OPERAND (*from_p, 0)) == COMPOUND_LITERAL_EXPR)
    7063      9878110 :               && tree_ssa_useless_type_conversion (*from_p))
    7064              :             {
    7065           13 :               *from_p = TREE_OPERAND (*from_p, 0);
    7066           13 :               ret = GS_OK;
    7067           13 :               changed = true;
    7068              :             }
    7069              :           break;
    7070              : 
    7071         4990 :         case COMPOUND_LITERAL_EXPR:
    7072         4990 :           {
    7073         4990 :             tree complit = TREE_OPERAND (*expr_p, 1);
    7074         4990 :             tree decl_s = COMPOUND_LITERAL_EXPR_DECL_EXPR (complit);
    7075         4990 :             tree decl = DECL_EXPR_DECL (decl_s);
    7076         4990 :             tree init = DECL_INITIAL (decl);
    7077              : 
    7078              :             /* struct T x = (struct T) { 0, 1, 2 } can be optimized
    7079              :                into struct T x = { 0, 1, 2 } if the address of the
    7080              :                compound literal has never been taken.  */
    7081         4990 :             if (!TREE_ADDRESSABLE (complit)
    7082         4990 :                 && !TREE_ADDRESSABLE (decl)
    7083         4990 :                 && init)
    7084              :               {
    7085         4990 :                 *expr_p = copy_node (*expr_p);
    7086         4990 :                 TREE_OPERAND (*expr_p, 1) = init;
    7087         4990 :                 return GS_OK;
    7088              :               }
    7089              :           }
    7090              : 
    7091              :         default:
    7092              :           break;
    7093              :         }
    7094              :     }
    7095              :   while (changed);
    7096              : 
    7097              :   return ret;
    7098              : }
    7099              : 
    7100              : 
    7101              : /* Return true if T looks like a valid GIMPLE statement.  */
    7102              : 
    7103              : static bool
    7104     19627757 : is_gimple_stmt (tree t)
    7105              : {
    7106     19627757 :   const enum tree_code code = TREE_CODE (t);
    7107              : 
    7108     19627757 :   switch (code)
    7109              :     {
    7110      1765959 :     case NOP_EXPR:
    7111              :       /* The only valid NOP_EXPR is the empty statement.  */
    7112      1765959 :       return IS_EMPTY_STMT (t);
    7113              : 
    7114            0 :     case BIND_EXPR:
    7115            0 :     case COND_EXPR:
    7116              :       /* These are only valid if they're void.  */
    7117            0 :       return TREE_TYPE (t) == NULL || VOID_TYPE_P (TREE_TYPE (t));
    7118              : 
    7119              :     case SWITCH_EXPR:
    7120              :     case GOTO_EXPR:
    7121              :     case RETURN_EXPR:
    7122              :     case LABEL_EXPR:
    7123              :     case CASE_LABEL_EXPR:
    7124              :     case TRY_CATCH_EXPR:
    7125              :     case TRY_FINALLY_EXPR:
    7126              :     case EH_FILTER_EXPR:
    7127              :     case CATCH_EXPR:
    7128              :     case ASM_EXPR:
    7129              :     case STATEMENT_LIST:
    7130              :     case OACC_PARALLEL:
    7131              :     case OACC_KERNELS:
    7132              :     case OACC_SERIAL:
    7133              :     case OACC_DATA:
    7134              :     case OACC_HOST_DATA:
    7135              :     case OACC_DECLARE:
    7136              :     case OACC_UPDATE:
    7137              :     case OACC_ENTER_DATA:
    7138              :     case OACC_EXIT_DATA:
    7139              :     case OACC_CACHE:
    7140              :     case OMP_PARALLEL:
    7141              :     case OMP_FOR:
    7142              :     case OMP_SIMD:
    7143              :     case OMP_DISTRIBUTE:
    7144              :     case OMP_LOOP:
    7145              :     case OMP_TILE:
    7146              :     case OMP_UNROLL:
    7147              :     case OACC_LOOP:
    7148              :     case OMP_SCAN:
    7149              :     case OMP_SCOPE:
    7150              :     case OMP_DISPATCH:
    7151              :     case OMP_SECTIONS:
    7152              :     case OMP_SECTION:
    7153              :     case OMP_STRUCTURED_BLOCK:
    7154              :     case OMP_SINGLE:
    7155              :     case OMP_MASTER:
    7156              :     case OMP_MASKED:
    7157              :     case OMP_TASKGROUP:
    7158              :     case OMP_ORDERED:
    7159              :     case OMP_CRITICAL:
    7160              :     case OMP_METADIRECTIVE:
    7161              :     case OMP_TASK:
    7162              :     case OMP_TARGET:
    7163              :     case OMP_TARGET_DATA:
    7164              :     case OMP_TARGET_UPDATE:
    7165              :     case OMP_TARGET_ENTER_DATA:
    7166              :     case OMP_TARGET_EXIT_DATA:
    7167              :     case OMP_TASKLOOP:
    7168              :     case OMP_TEAMS:
    7169              :       /* These are always void.  */
    7170              :       return true;
    7171              : 
    7172              :     case CALL_EXPR:
    7173              :     case MODIFY_EXPR:
    7174              :     case PREDICT_EXPR:
    7175              :       /* These are valid regardless of their type.  */
    7176              :       return true;
    7177              : 
    7178              :     default:
    7179              :       return false;
    7180              :     }
    7181              : }
    7182              : 
    7183              : 
    7184              : /* Promote partial stores to COMPLEX variables to total stores.  *EXPR_P is
    7185              :    a MODIFY_EXPR with a lhs of a REAL/IMAGPART_EXPR of a gimple register.
    7186              : 
    7187              :    IMPORTANT NOTE: This promotion is performed by introducing a load of the
    7188              :    other, unmodified part of the complex object just before the total store.
    7189              :    As a consequence, if the object is still uninitialized, an undefined value
    7190              :    will be loaded into a register, which may result in a spurious exception
    7191              :    if the register is floating-point and the value happens to be a signaling
    7192              :    NaN for example.  Then the fully-fledged complex operations lowering pass
    7193              :    followed by a DCE pass are necessary in order to fix things up.  */
    7194              : 
    7195              : static enum gimplify_status
    7196         1960 : gimplify_modify_expr_complex_part (tree *expr_p, gimple_seq *pre_p,
    7197              :                                    bool want_value)
    7198              : {
    7199         1960 :   enum tree_code code, ocode;
    7200         1960 :   tree lhs, rhs, new_rhs, other, realpart, imagpart;
    7201              : 
    7202         1960 :   lhs = TREE_OPERAND (*expr_p, 0);
    7203         1960 :   rhs = TREE_OPERAND (*expr_p, 1);
    7204         1960 :   code = TREE_CODE (lhs);
    7205         1960 :   lhs = TREE_OPERAND (lhs, 0);
    7206              : 
    7207         1960 :   ocode = code == REALPART_EXPR ? IMAGPART_EXPR : REALPART_EXPR;
    7208         1960 :   other = build1 (ocode, TREE_TYPE (rhs), lhs);
    7209         1960 :   suppress_warning (other);
    7210         1960 :   other = get_formal_tmp_var (other, pre_p);
    7211              : 
    7212         1960 :   realpart = code == REALPART_EXPR ? rhs : other;
    7213          969 :   imagpart = code == REALPART_EXPR ? other : rhs;
    7214              : 
    7215         1960 :   if (TREE_CONSTANT (realpart) && TREE_CONSTANT (imagpart))
    7216            0 :     new_rhs = build_complex (TREE_TYPE (lhs), realpart, imagpart);
    7217              :   else
    7218         1960 :     new_rhs = build2 (COMPLEX_EXPR, TREE_TYPE (lhs), realpart, imagpart);
    7219              : 
    7220         1960 :   gimplify_seq_add_stmt (pre_p, gimple_build_assign (lhs, new_rhs));
    7221         1960 :   *expr_p = (want_value) ? rhs : NULL_TREE;
    7222              : 
    7223         1960 :   return GS_ALL_DONE;
    7224              : }
    7225              : 
    7226              : /* Gimplify the MODIFY_EXPR node pointed to by EXPR_P.
    7227              : 
    7228              :       modify_expr
    7229              :               : varname '=' rhs
    7230              :               | '*' ID '=' rhs
    7231              : 
    7232              :     PRE_P points to the list where side effects that must happen before
    7233              :         *EXPR_P should be stored.
    7234              : 
    7235              :     POST_P points to the list where side effects that must happen after
    7236              :         *EXPR_P should be stored.
    7237              : 
    7238              :     WANT_VALUE is nonzero iff we want to use the value of this expression
    7239              :         in another expression.  */
    7240              : 
    7241              : static enum gimplify_status
    7242     51431604 : gimplify_modify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
    7243              :                       bool want_value)
    7244              : {
    7245     51431604 :   tree *from_p = &TREE_OPERAND (*expr_p, 1);
    7246     51431604 :   tree *to_p = &TREE_OPERAND (*expr_p, 0);
    7247     51431604 :   enum gimplify_status ret = GS_UNHANDLED;
    7248     51431604 :   gimple *assign;
    7249     51431604 :   location_t loc = EXPR_LOCATION (*expr_p);
    7250     51431604 :   gimple_stmt_iterator gsi;
    7251              : 
    7252     51431604 :   if (error_operand_p (*from_p) || error_operand_p (*to_p))
    7253              :     return GS_ERROR;
    7254              : 
    7255     51431576 :   gcc_assert (TREE_CODE (*expr_p) == MODIFY_EXPR
    7256              :               || TREE_CODE (*expr_p) == INIT_EXPR);
    7257              : 
    7258              :   /* Trying to simplify a clobber using normal logic doesn't work,
    7259              :      so handle it here.  */
    7260     51431576 :   if (TREE_CLOBBER_P (*from_p))
    7261              :     {
    7262       615086 :       ret = gimplify_expr (to_p, pre_p, post_p, is_gimple_lvalue, fb_lvalue);
    7263       615086 :       if (ret == GS_ERROR)
    7264              :         return ret;
    7265       615086 :       gcc_assert (!want_value);
    7266       615086 :       if (!VAR_P (*to_p) && TREE_CODE (*to_p) != MEM_REF)
    7267              :         {
    7268          208 :           tree addr = get_initialized_tmp_var (build_fold_addr_expr (*to_p),
    7269              :                                                pre_p, post_p);
    7270          208 :           *to_p = build_simple_mem_ref_loc (EXPR_LOCATION (*to_p), addr);
    7271              :         }
    7272       615086 :       gimplify_seq_add_stmt (pre_p, gimple_build_assign (*to_p, *from_p));
    7273       615086 :       *expr_p = NULL;
    7274       615086 :       return GS_ALL_DONE;
    7275              :     }
    7276              : 
    7277              :   /* Convert initialization from an empty variable-size CONSTRUCTOR to
    7278              :      memset.  */
    7279     50816490 :   if (TREE_TYPE (*from_p) != error_mark_node
    7280     50816490 :       && TYPE_SIZE_UNIT (TREE_TYPE (*from_p))
    7281     50816490 :       && !poly_int_tree_p (TYPE_SIZE_UNIT (TREE_TYPE (*from_p)))
    7282         1599 :       && TREE_CODE (*from_p) == CONSTRUCTOR
    7283     50816594 :       && CONSTRUCTOR_NELTS (*from_p) == 0)
    7284              :     {
    7285           84 :       maybe_with_size_expr (from_p);
    7286           84 :       gcc_assert (TREE_CODE (*from_p) == WITH_SIZE_EXPR);
    7287           84 :       return gimplify_modify_expr_to_memset (expr_p,
    7288           84 :                                              TREE_OPERAND (*from_p, 1),
    7289           84 :                                              want_value, pre_p);
    7290              :     }
    7291              : 
    7292              :   /* Insert pointer conversions required by the middle-end that are not
    7293              :      required by the frontend.  This fixes middle-end type checking for
    7294              :      for example gcc.dg/redecl-6.c.  */
    7295     50816406 :   if (POINTER_TYPE_P (TREE_TYPE (*to_p)))
    7296              :     {
    7297     13735193 :       STRIP_USELESS_TYPE_CONVERSION (*from_p);
    7298     13735193 :       if (!useless_type_conversion_p (TREE_TYPE (*to_p), TREE_TYPE (*from_p)))
    7299          754 :         *from_p = fold_convert_loc (loc, TREE_TYPE (*to_p), *from_p);
    7300              :     }
    7301              : 
    7302              :   /* See if any simplifications can be done based on what the RHS is.  */
    7303     50816406 :   ret = gimplify_modify_expr_rhs (expr_p, from_p, to_p, pre_p, post_p,
    7304              :                                   want_value);
    7305     50816406 :   if (ret != GS_UNHANDLED)
    7306              :     return ret;
    7307              : 
    7308              :   /* For empty types only gimplify the left hand side and right hand
    7309              :      side as statements and throw away the assignment.  Do this after
    7310              :      gimplify_modify_expr_rhs so we handle TARGET_EXPRs of addressable
    7311              :      types properly.  */
    7312     48731509 :   if (is_empty_type (TREE_TYPE (*from_p))
    7313        13353 :       && !want_value
    7314              :       /* Don't do this for calls that return addressable types, expand_call
    7315              :          relies on those having a lhs.  */
    7316     48744855 :       && !(TREE_ADDRESSABLE (TREE_TYPE (*from_p))
    7317        11287 :            && TREE_CODE (*from_p) == CALL_EXPR))
    7318              :     {
    7319         2059 :       suppress_warning (*from_p, OPT_Wunused_result);
    7320         2059 :       gimplify_stmt (from_p, pre_p);
    7321         2059 :       gimplify_stmt (to_p, pre_p);
    7322         2059 :       *expr_p = NULL_TREE;
    7323         2059 :       return GS_ALL_DONE;
    7324              :     }
    7325              : 
    7326              :   /* If the value being copied is of variable width, compute the length
    7327              :      of the copy into a WITH_SIZE_EXPR.   Note that we need to do this
    7328              :      before gimplifying any of the operands so that we can resolve any
    7329              :      PLACEHOLDER_EXPRs in the size.  Also note that the RTL expander uses
    7330              :      the size of the expression to be copied, not of the destination, so
    7331              :      that is what we must do here.  */
    7332     48729450 :   maybe_with_size_expr (from_p);
    7333              : 
    7334              :   /* As a special case, we have to temporarily allow for assignments
    7335              :      with a CALL_EXPR on the RHS.  Since in GIMPLE a function call is
    7336              :      a toplevel statement, when gimplifying the GENERIC expression
    7337              :      MODIFY_EXPR <a, CALL_EXPR <foo>>, we cannot create the tuple
    7338              :      GIMPLE_ASSIGN <a, GIMPLE_CALL <foo>>.
    7339              : 
    7340              :      Instead, we need to create the tuple GIMPLE_CALL <a, foo>.  To
    7341              :      prevent gimplify_expr from trying to create a new temporary for
    7342              :      foo's LHS, we tell it that it should only gimplify until it
    7343              :      reaches the CALL_EXPR.  On return from gimplify_expr, the newly
    7344              :      created GIMPLE_CALL <foo> will be the last statement in *PRE_P
    7345              :      and all we need to do here is set 'a' to be its LHS.  */
    7346              : 
    7347              :   /* Gimplify the RHS first for C++17 and bug 71104.  */
    7348     48729450 :   gimple_predicate initial_pred = initial_rhs_predicate_for (*to_p);
    7349     48729450 :   ret = gimplify_expr (from_p, pre_p, post_p, initial_pred, fb_rvalue);
    7350     48729450 :   if (ret == GS_ERROR)
    7351              :     return ret;
    7352              : 
    7353              :   /* Then gimplify the LHS.  */
    7354              :   /* If we gimplified the RHS to a CALL_EXPR and that call may return
    7355              :      twice we have to make sure to gimplify into non-SSA as otherwise
    7356              :      the abnormal edge added later will make those defs not dominate
    7357              :      their uses.
    7358              :      ???  Technically this applies only to the registers used in the
    7359              :      resulting non-register *TO_P.  */
    7360     48729300 :   bool saved_into_ssa = gimplify_ctxp->into_ssa;
    7361     48729300 :   if (saved_into_ssa
    7362     46331271 :       && TREE_CODE (*from_p) == CALL_EXPR
    7363     53981463 :       && call_expr_flags (*from_p) & ECF_RETURNS_TWICE)
    7364         1539 :     gimplify_ctxp->into_ssa = false;
    7365     48729300 :   ret = gimplify_expr (to_p, pre_p, post_p, is_gimple_lvalue, fb_lvalue);
    7366     48729300 :   gimplify_ctxp->into_ssa = saved_into_ssa;
    7367     48729300 :   if (ret == GS_ERROR)
    7368              :     return ret;
    7369              : 
    7370              :   /* Now that the LHS is gimplified, re-gimplify the RHS if our initial
    7371              :      guess for the predicate was wrong.  */
    7372     48729225 :   gimple_predicate final_pred = rhs_predicate_for (*to_p);
    7373     48729225 :   if (final_pred != initial_pred)
    7374              :     {
    7375      8332365 :       ret = gimplify_expr (from_p, pre_p, post_p, final_pred, fb_rvalue);
    7376      8332365 :       if (ret == GS_ERROR)
    7377              :         return ret;
    7378              :     }
    7379              : 
    7380              :   /* In case of va_arg internal fn wrapped in a WITH_SIZE_EXPR, add the type
    7381              :      size as argument to the call.  */
    7382     48729225 :   if (TREE_CODE (*from_p) == WITH_SIZE_EXPR)
    7383              :     {
    7384         1463 :       tree call = TREE_OPERAND (*from_p, 0);
    7385         1463 :       tree vlasize = TREE_OPERAND (*from_p, 1);
    7386              : 
    7387         1463 :       if (TREE_CODE (call) == CALL_EXPR
    7388         1463 :           && CALL_EXPR_IFN (call) == IFN_VA_ARG)
    7389              :         {
    7390           95 :           int nargs = call_expr_nargs (call);
    7391           95 :           tree type = TREE_TYPE (call);
    7392           95 :           tree ap = CALL_EXPR_ARG (call, 0);
    7393           95 :           tree tag = CALL_EXPR_ARG (call, 1);
    7394           95 :           tree aptag = CALL_EXPR_ARG (call, 2);
    7395           95 :           tree newcall = build_call_expr_internal_loc (EXPR_LOCATION (call),
    7396              :                                                        IFN_VA_ARG, type,
    7397              :                                                        nargs + 1, ap, tag,
    7398              :                                                        aptag, vlasize);
    7399           95 :           TREE_OPERAND (*from_p, 0) = newcall;
    7400              :         }
    7401              :     }
    7402              : 
    7403              :   /* Now see if the above changed *from_p to something we handle specially.  */
    7404     48729225 :   ret = gimplify_modify_expr_rhs (expr_p, from_p, to_p, pre_p, post_p,
    7405              :                                   want_value);
    7406     48729225 :   if (ret != GS_UNHANDLED)
    7407              :     return ret;
    7408              : 
    7409              :   /* If we've got a variable sized assignment between two lvalues (i.e. does
    7410              :      not involve a call), then we can make things a bit more straightforward
    7411              :      by converting the assignment to memcpy or memset.  */
    7412     48728256 :   if (TREE_CODE (*from_p) == WITH_SIZE_EXPR)
    7413              :     {
    7414         1111 :       tree from = TREE_OPERAND (*from_p, 0);
    7415         1111 :       tree size = TREE_OPERAND (*from_p, 1);
    7416              : 
    7417         1111 :       if (TREE_CODE (from) == CONSTRUCTOR)
    7418            0 :         return gimplify_modify_expr_to_memset (expr_p, size, want_value, pre_p);
    7419         1111 :       else if (is_gimple_addressable (from)
    7420         1111 :                && ADDR_SPACE_GENERIC_P (TYPE_ADDR_SPACE (TREE_TYPE (*to_p)))
    7421         2221 :                && ADDR_SPACE_GENERIC_P (TYPE_ADDR_SPACE (TREE_TYPE (from))))
    7422              :         {
    7423         1110 :           *from_p = from;
    7424         1110 :           return gimplify_modify_expr_to_memcpy (expr_p, size, want_value,
    7425         1110 :                                                  pre_p);
    7426              :         }
    7427              :     }
    7428              : 
    7429              :   /* Transform partial stores to non-addressable complex variables into
    7430              :      total stores.  This allows us to use real instead of virtual operands
    7431              :      for these variables, which improves optimization.  */
    7432     48727146 :   if ((TREE_CODE (*to_p) == REALPART_EXPR
    7433     48727146 :        || TREE_CODE (*to_p) == IMAGPART_EXPR)
    7434     48727146 :       && is_gimple_reg (TREE_OPERAND (*to_p, 0)))
    7435         1960 :     return gimplify_modify_expr_complex_part (expr_p, pre_p, want_value);
    7436              : 
    7437              :   /* Try to alleviate the effects of the gimplification creating artificial
    7438              :      temporaries (see for example is_gimple_reg_rhs) on the debug info, but
    7439              :      make sure not to create DECL_DEBUG_EXPR links across functions.  */
    7440     48725186 :   if (!gimplify_ctxp->into_ssa
    7441      2397099 :       && VAR_P (*from_p)
    7442       473228 :       && DECL_IGNORED_P (*from_p)
    7443       351830 :       && DECL_P (*to_p)
    7444       138747 :       && !DECL_IGNORED_P (*to_p)
    7445        38291 :       && decl_function_context (*to_p) == current_function_decl
    7446     48757895 :       && decl_function_context (*from_p) == current_function_decl)
    7447              :     {
    7448        32704 :       if (!DECL_NAME (*from_p) && DECL_NAME (*to_p))
    7449        27413 :         DECL_NAME (*from_p)
    7450        54826 :           = create_tmp_var_name (IDENTIFIER_POINTER (DECL_NAME (*to_p)));
    7451        32704 :       DECL_HAS_DEBUG_EXPR_P (*from_p) = 1;
    7452        32704 :       SET_DECL_DEBUG_EXPR (*from_p, *to_p);
    7453              :    }
    7454              : 
    7455     48725186 :   if (want_value && TREE_THIS_VOLATILE (*to_p))
    7456          522 :     *from_p = get_initialized_tmp_var (*from_p, pre_p, post_p);
    7457              : 
    7458     48725186 :   if (TREE_CODE (*from_p) == CALL_EXPR)
    7459              :     {
    7460              :       /* Since the RHS is a CALL_EXPR, we need to create a GIMPLE_CALL
    7461              :          instead of a GIMPLE_ASSIGN.  */
    7462      4929426 :       gcall *call_stmt;
    7463      4929426 :       if (CALL_EXPR_FN (*from_p) == NULL_TREE)
    7464              :         {
    7465              :           /* Gimplify internal functions created in the FEs.  */
    7466       347742 :           int nargs = call_expr_nargs (*from_p), i;
    7467       347742 :           enum internal_fn ifn = CALL_EXPR_IFN (*from_p);
    7468       347742 :           auto_vec<tree> vargs (nargs);
    7469              : 
    7470      1750158 :           for (i = 0; i < nargs; i++)
    7471              :             {
    7472      1054674 :               gimplify_arg (&CALL_EXPR_ARG (*from_p, i), pre_p,
    7473      1054674 :                             EXPR_LOCATION (*from_p));
    7474      1054674 :               vargs.quick_push (CALL_EXPR_ARG (*from_p, i));
    7475              :             }
    7476       347742 :           call_stmt = gimple_build_call_internal_vec (ifn, vargs);
    7477       347742 :           gimple_call_set_nothrow (call_stmt, TREE_NOTHROW (*from_p));
    7478       695484 :           gimple_set_location (call_stmt, EXPR_LOCATION (*expr_p));
    7479       347742 :         }
    7480              :       else
    7481              :         {
    7482      4581684 :           tree fnptrtype = TREE_TYPE (CALL_EXPR_FN (*from_p));
    7483      4581684 :           CALL_EXPR_FN (*from_p) = TREE_OPERAND (CALL_EXPR_FN (*from_p), 0);
    7484      4581684 :           STRIP_USELESS_TYPE_CONVERSION (CALL_EXPR_FN (*from_p));
    7485      4581684 :           tree fndecl = get_callee_fndecl (*from_p);
    7486      4581684 :           if (fndecl
    7487      4465740 :               && fndecl_built_in_p (fndecl, BUILT_IN_EXPECT)
    7488      4720462 :               && call_expr_nargs (*from_p) == 3)
    7489        69077 :             call_stmt = gimple_build_call_internal (IFN_BUILTIN_EXPECT, 3,
    7490        69077 :                                                     CALL_EXPR_ARG (*from_p, 0),
    7491        69077 :                                                     CALL_EXPR_ARG (*from_p, 1),
    7492        69077 :                                                     CALL_EXPR_ARG (*from_p, 2));
    7493              :           else
    7494              :             {
    7495      4512607 :               call_stmt = gimple_build_call_from_tree (*from_p, fnptrtype);
    7496              :             }
    7497              :         }
    7498      4929426 :       notice_special_calls (call_stmt);
    7499      4929426 :       if (!gimple_call_noreturn_p (call_stmt) || !should_remove_lhs_p (*to_p))
    7500      4929182 :         gimple_call_set_lhs (call_stmt, *to_p);
    7501          244 :       else if (TREE_CODE (*to_p) == SSA_NAME)
    7502              :         /* The above is somewhat premature, avoid ICEing later for a
    7503              :            SSA name w/o a definition.  We may have uses in the GIMPLE IL.
    7504              :            ???  This doesn't make it a default-def.  */
    7505          137 :         SSA_NAME_DEF_STMT (*to_p) = gimple_build_nop ();
    7506              : 
    7507              :       assign = call_stmt;
    7508              :     }
    7509              :   else
    7510              :     {
    7511     43795760 :       assign = gimple_build_assign (*to_p, *from_p);
    7512     43795760 :       gimple_set_location (assign, EXPR_LOCATION (*expr_p));
    7513     43795760 :       if (COMPARISON_CLASS_P (*from_p))
    7514      1253142 :         copy_warning (assign, *from_p);
    7515              :     }
    7516              : 
    7517     48725186 :   if (gimplify_ctxp->into_ssa && is_gimple_reg (*to_p))
    7518              :     {
    7519              :       /* We should have got an SSA name from the start.  */
    7520     36013382 :       gcc_assert (TREE_CODE (*to_p) == SSA_NAME
    7521              :                   || ! gimple_in_ssa_p (cfun));
    7522              :     }
    7523              : 
    7524     48725186 :   gimplify_seq_add_stmt (pre_p, assign);
    7525     48725186 :   gsi = gsi_last (*pre_p);
    7526     48725186 :   maybe_fold_stmt (&gsi);
    7527              : 
    7528     48725186 :   if (want_value)
    7529              :     {
    7530       698548 :       *expr_p = TREE_THIS_VOLATILE (*to_p) ? *from_p : unshare_expr (*to_p);
    7531       698548 :       return GS_OK;
    7532              :     }
    7533              :   else
    7534     48026638 :     *expr_p = NULL;
    7535              : 
    7536     48026638 :   return GS_ALL_DONE;
    7537              : }
    7538              : 
    7539              : /* Gimplify a comparison between two variable-sized objects.  Do this
    7540              :    with a call to BUILT_IN_MEMCMP.  */
    7541              : 
    7542              : static enum gimplify_status
    7543            0 : gimplify_variable_sized_compare (tree *expr_p)
    7544              : {
    7545            0 :   location_t loc = EXPR_LOCATION (*expr_p);
    7546            0 :   tree op0 = TREE_OPERAND (*expr_p, 0);
    7547            0 :   tree op1 = TREE_OPERAND (*expr_p, 1);
    7548            0 :   tree t, arg, dest, src, expr;
    7549              : 
    7550            0 :   arg = TYPE_SIZE_UNIT (TREE_TYPE (op0));
    7551            0 :   arg = unshare_expr (arg);
    7552            0 :   arg = SUBSTITUTE_PLACEHOLDER_IN_EXPR (arg, op0);
    7553            0 :   src = build_fold_addr_expr_loc (loc, op1);
    7554            0 :   dest = build_fold_addr_expr_loc (loc, op0);
    7555            0 :   t = builtin_decl_implicit (BUILT_IN_MEMCMP);
    7556            0 :   t = build_call_expr_loc (loc, t, 3, dest, src, arg);
    7557              : 
    7558            0 :   expr
    7559            0 :     = build2 (TREE_CODE (*expr_p), TREE_TYPE (*expr_p), t, integer_zero_node);
    7560            0 :   SET_EXPR_LOCATION (expr, loc);
    7561            0 :   *expr_p = expr;
    7562              : 
    7563            0 :   return GS_OK;
    7564              : }
    7565              : 
    7566              : /* Gimplify a comparison between two aggregate objects of integral scalar
    7567              :    mode as a comparison between the bitwise equivalent scalar values.  */
    7568              : 
    7569              : static enum gimplify_status
    7570           16 : gimplify_scalar_mode_aggregate_compare (tree *expr_p)
    7571              : {
    7572           16 :   const location_t loc = EXPR_LOCATION (*expr_p);
    7573           16 :   const enum tree_code code = TREE_CODE (*expr_p);
    7574           16 :   tree op0 = TREE_OPERAND (*expr_p, 0);
    7575           16 :   tree op1 = TREE_OPERAND (*expr_p, 1);
    7576           16 :   tree type = TREE_TYPE (op0);
    7577           16 :   tree scalar_type = lang_hooks.types.type_for_mode (TYPE_MODE (type), 1);
    7578              : 
    7579           16 :   op0 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, scalar_type, op0);
    7580           16 :   op1 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, scalar_type, op1);
    7581              : 
    7582              :   /* We need to perform ordering comparisons in memory order like memcmp and,
    7583              :      therefore, may need to byte-swap operands for little-endian targets.  */
    7584           16 :   if (code != EQ_EXPR && code != NE_EXPR)
    7585              :     {
    7586            0 :       gcc_assert (BYTES_BIG_ENDIAN == WORDS_BIG_ENDIAN);
    7587            0 :       gcc_assert (TREE_CODE (scalar_type) == INTEGER_TYPE);
    7588            0 :       tree fndecl;
    7589              : 
    7590            0 :       if (BYTES_BIG_ENDIAN)
    7591              :         fndecl = NULL_TREE;
    7592              :       else
    7593            0 :         switch (int_size_in_bytes (scalar_type))
    7594              :           {
    7595              :           case 1:
    7596              :             fndecl = NULL_TREE;
    7597              :             break;
    7598            0 :           case 2:
    7599            0 :             fndecl = builtin_decl_implicit (BUILT_IN_BSWAP16);
    7600              :             break;
    7601            0 :           case 4:
    7602            0 :             fndecl = builtin_decl_implicit (BUILT_IN_BSWAP32);
    7603              :             break;
    7604            0 :           case 8:
    7605            0 :             fndecl = builtin_decl_implicit (BUILT_IN_BSWAP64);
    7606              :             break;
    7607            0 :           case 16:
    7608            0 :             fndecl = builtin_decl_implicit (BUILT_IN_BSWAP128);
    7609              :             break;
    7610            0 :           default:
    7611            0 :             gcc_unreachable ();
    7612              :           }
    7613              : 
    7614            0 :       if (fndecl)
    7615              :         {
    7616            0 :           op0 = build_call_expr_loc (loc, fndecl, 1, op0);
    7617            0 :           op1 = build_call_expr_loc (loc, fndecl, 1, op1);
    7618              :         }
    7619              :     }
    7620              : 
    7621           16 :   *expr_p = fold_build2_loc (loc, code, TREE_TYPE (*expr_p), op0, op1);
    7622              : 
    7623           16 :   return GS_OK;
    7624              : }
    7625              : 
    7626              : /* Gimplify an expression sequence.  This function gimplifies each
    7627              :    expression and rewrites the original expression with the last
    7628              :    expression of the sequence in GIMPLE form.
    7629              : 
    7630              :    PRE_P points to the list where the side effects for all the
    7631              :        expressions in the sequence will be emitted.
    7632              : 
    7633              :    WANT_VALUE is true when the result of the last COMPOUND_EXPR is used.  */
    7634              : 
    7635              : static enum gimplify_status
    7636      1135626 : gimplify_compound_expr (tree *expr_p, gimple_seq *pre_p, bool want_value)
    7637              : {
    7638      1135626 :   tree t = *expr_p;
    7639              : 
    7640      1227378 :   do
    7641              :     {
    7642      1227378 :       tree *sub_p = &TREE_OPERAND (t, 0);
    7643              : 
    7644      1227378 :       if (TREE_CODE (*sub_p) == COMPOUND_EXPR)
    7645       215128 :         gimplify_compound_expr (sub_p, pre_p, false);
    7646              :       else
    7647      1012250 :         gimplify_stmt (sub_p, pre_p);
    7648              : 
    7649      1227378 :       t = TREE_OPERAND (t, 1);
    7650              :     }
    7651      1227378 :   while (TREE_CODE (t) == COMPOUND_EXPR);
    7652              : 
    7653      1135626 :   *expr_p = t;
    7654      1135626 :   if (want_value)
    7655              :     return GS_OK;
    7656              :   else
    7657              :     {
    7658       634416 :       gimplify_stmt (expr_p, pre_p);
    7659       634416 :       return GS_ALL_DONE;
    7660              :     }
    7661              : }
    7662              : 
    7663              : /* Gimplify a SAVE_EXPR node.  EXPR_P points to the expression to
    7664              :    gimplify.  After gimplification, EXPR_P will point to a new temporary
    7665              :    that holds the original value of the SAVE_EXPR node.
    7666              : 
    7667              :    PRE_P points to the list where side effects that must happen before
    7668              :    *EXPR_P should be stored.  */
    7669              : 
    7670              : static enum gimplify_status
    7671       444683 : gimplify_save_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
    7672              : {
    7673       444683 :   enum gimplify_status ret = GS_ALL_DONE;
    7674       444683 :   tree val;
    7675              : 
    7676       444683 :   gcc_assert (TREE_CODE (*expr_p) == SAVE_EXPR);
    7677       444683 :   val = TREE_OPERAND (*expr_p, 0);
    7678              : 
    7679       444683 :   if (val && TREE_TYPE (val) == error_mark_node)
    7680              :     return GS_ERROR;
    7681              : 
    7682              :   /* If the SAVE_EXPR has not been resolved, then evaluate it once.  */
    7683       444681 :   if (!SAVE_EXPR_RESOLVED_P (*expr_p))
    7684              :     {
    7685              :       /* The operand may be a void-valued expression.  It is
    7686              :          being executed only for its side-effects.  */
    7687       170981 :       if (TREE_TYPE (val) == void_type_node)
    7688              :         {
    7689          751 :           ret = gimplify_expr (&TREE_OPERAND (*expr_p, 0), pre_p, post_p,
    7690              :                                is_gimple_stmt, fb_none);
    7691          751 :           val = NULL;
    7692              :         }
    7693              :       else
    7694              :         /* The temporary may not be an SSA name as later abnormal and EH
    7695              :            control flow may invalidate use/def domination.  When in SSA
    7696              :            form then assume there are no such issues and SAVE_EXPRs only
    7697              :            appear via GENERIC foldings.  */
    7698       340460 :         val = get_initialized_tmp_var (val, pre_p, post_p,
    7699              :                                        gimple_in_ssa_p (cfun));
    7700              : 
    7701       170981 :       TREE_OPERAND (*expr_p, 0) = val;
    7702       170981 :       SAVE_EXPR_RESOLVED_P (*expr_p) = 1;
    7703              :     }
    7704              : 
    7705       444681 :   *expr_p = val;
    7706              : 
    7707       444681 :   return ret;
    7708              : }
    7709              : 
    7710              : /* Rewrite the ADDR_EXPR node pointed to by EXPR_P
    7711              : 
    7712              :       unary_expr
    7713              :               : ...
    7714              :               | '&' varname
    7715              :               ...
    7716              : 
    7717              :     PRE_P points to the list where side effects that must happen before
    7718              :         *EXPR_P should be stored.
    7719              : 
    7720              :     POST_P points to the list where side effects that must happen after
    7721              :         *EXPR_P should be stored.  */
    7722              : 
    7723              : static enum gimplify_status
    7724     34156475 : gimplify_addr_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
    7725              : {
    7726     34156475 :   tree expr = *expr_p;
    7727     34156475 :   tree op0 = TREE_OPERAND (expr, 0);
    7728     34156475 :   enum gimplify_status ret;
    7729     34156475 :   location_t loc = EXPR_LOCATION (*expr_p);
    7730              : 
    7731     34156475 :   switch (TREE_CODE (op0))
    7732              :     {
    7733       101154 :     case INDIRECT_REF:
    7734       101154 :     do_indirect_ref:
    7735              :       /* Check if we are dealing with an expression of the form '&*ptr'.
    7736              :          While the front end folds away '&*ptr' into 'ptr', these
    7737              :          expressions may be generated internally by the compiler (e.g.,
    7738              :          builtins like __builtin_va_end).  */
    7739              :       /* Caution: the silent array decomposition semantics we allow for
    7740              :          ADDR_EXPR means we can't always discard the pair.  */
    7741              :       /* Gimplification of the ADDR_EXPR operand may drop
    7742              :          cv-qualification conversions, so make sure we add them if
    7743              :          needed.  */
    7744       101154 :       {
    7745       101154 :         tree op00 = TREE_OPERAND (op0, 0);
    7746       101154 :         tree t_expr = TREE_TYPE (expr);
    7747       101154 :         tree t_op00 = TREE_TYPE (op00);
    7748              : 
    7749       101154 :         if (!useless_type_conversion_p (t_expr, t_op00))
    7750            0 :           op00 = fold_convert_loc (loc, TREE_TYPE (expr), op00);
    7751       101154 :         *expr_p = op00;
    7752       101154 :         ret = GS_OK;
    7753              :       }
    7754       101154 :       break;
    7755              : 
    7756            0 :     case VIEW_CONVERT_EXPR:
    7757              :       /* Take the address of our operand and then convert it to the type of
    7758              :          this ADDR_EXPR.
    7759              : 
    7760              :          ??? The interactions of VIEW_CONVERT_EXPR and aliasing is not at
    7761              :          all clear.  The impact of this transformation is even less clear.  */
    7762              : 
    7763              :       /* If the operand is a useless conversion, look through it.  Doing so
    7764              :          guarantees that the ADDR_EXPR and its operand will remain of the
    7765              :          same type.  */
    7766            0 :       if (tree_ssa_useless_type_conversion (TREE_OPERAND (op0, 0)))
    7767            0 :         op0 = TREE_OPERAND (op0, 0);
    7768              : 
    7769            0 :       *expr_p = fold_convert_loc (loc, TREE_TYPE (expr),
    7770              :                                   build_fold_addr_expr_loc (loc,
    7771            0 :                                                         TREE_OPERAND (op0, 0)));
    7772            0 :       ret = GS_OK;
    7773            0 :       break;
    7774              : 
    7775        66593 :     case MEM_REF:
    7776        66593 :       if (integer_zerop (TREE_OPERAND (op0, 1)))
    7777        38096 :         goto do_indirect_ref;
    7778              : 
    7779              :       /* fall through */
    7780              : 
    7781     34073238 :     default:
    7782              :       /* If we see a call to a declared builtin or see its address
    7783              :          being taken (we can unify those cases here) then we can mark
    7784              :          the builtin for implicit generation by GCC.  */
    7785     34073238 :       if (TREE_CODE (op0) == FUNCTION_DECL
    7786     16938186 :           && fndecl_built_in_p (op0, BUILT_IN_NORMAL)
    7787     38071106 :           && builtin_decl_declared_p (DECL_FUNCTION_CODE (op0)))
    7788      1310127 :         set_builtin_decl_implicit_p (DECL_FUNCTION_CODE (op0), true);
    7789              : 
    7790              :       /* We use fb_either here because the C frontend sometimes takes
    7791              :          the address of a call that returns a struct; see
    7792              :          gcc.dg/c99-array-lval-1.c.  The gimplifier will correctly make
    7793              :          the implied temporary explicit.  */
    7794              : 
    7795              :       /* Make the operand addressable.  */
    7796     34073238 :       ret = gimplify_expr (&TREE_OPERAND (expr, 0), pre_p, post_p,
    7797              :                            is_gimple_addressable, fb_either);
    7798     34073238 :       if (ret == GS_ERROR)
    7799              :         break;
    7800              : 
    7801              :       /* Then mark it.  Beware that it may not be possible to do so directly
    7802              :          if a temporary has been created by the gimplification.  */
    7803     34073234 :       prepare_gimple_addressable (&TREE_OPERAND (expr, 0), pre_p);
    7804              : 
    7805     34073234 :       op0 = TREE_OPERAND (expr, 0);
    7806              : 
    7807              :       /* For various reasons, the gimplification of the expression
    7808              :          may have made a new INDIRECT_REF.  */
    7809     34073234 :       if (INDIRECT_REF_P (op0)
    7810     34073234 :           || (TREE_CODE (op0) == MEM_REF
    7811        46414 :               && integer_zerop (TREE_OPERAND (op0, 1))))
    7812        17917 :         goto do_indirect_ref;
    7813              : 
    7814     34055317 :       mark_addressable (TREE_OPERAND (expr, 0));
    7815              : 
    7816              :       /* The FEs may end up building ADDR_EXPRs early on a decl with
    7817              :          an incomplete type.  Re-build ADDR_EXPRs in canonical form
    7818              :          here.  */
    7819     34055317 :       if (!types_compatible_p (TREE_TYPE (op0), TREE_TYPE (TREE_TYPE (expr))))
    7820       151034 :         *expr_p = build_fold_addr_expr (op0);
    7821              : 
    7822              :       /* Make sure TREE_CONSTANT and TREE_SIDE_EFFECTS are set properly.  */
    7823     34055317 :       if (TREE_CODE (*expr_p) == ADDR_EXPR)
    7824     34055317 :         recompute_tree_invariant_for_addr_expr (*expr_p);
    7825              : 
    7826              :       /* If we re-built the ADDR_EXPR add a conversion to the original type
    7827              :          if required.  */
    7828     34055317 :       if (!useless_type_conversion_p (TREE_TYPE (expr), TREE_TYPE (*expr_p)))
    7829            0 :         *expr_p = fold_convert (TREE_TYPE (expr), *expr_p);
    7830              : 
    7831              :       break;
    7832              :     }
    7833              : 
    7834     34156475 :   return ret;
    7835              : }
    7836              : 
    7837              : /* Return the number of times character C occurs in string S.  */
    7838              : 
    7839              : static int
    7840        95202 : num_occurrences (int c, const char *s)
    7841              : {
    7842        95202 :   int n = 0;
    7843       258140 :   while (*s)
    7844       162938 :     n += (*s++ == c);
    7845        95202 :   return n;
    7846              : }
    7847              : 
    7848              : /* A subroutine of gimplify_asm_expr.  Check that all operands have
    7849              :    the same number of alternatives.  Return -1 if this is violated.  Otherwise
    7850              :    return the number of alternatives.  */
    7851              : 
    7852              : static int
    7853       166534 : num_alternatives (const_tree link)
    7854              : {
    7855       166534 :   if (link == nullptr)
    7856              :     return 0;
    7857              : 
    7858        48906 :   const char *constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
    7859        48906 :   int num = num_occurrences (',', constraint);
    7860              : 
    7861        48906 :   if (num + 1 > MAX_RECOG_ALTERNATIVES)
    7862              :     return -1;
    7863              : 
    7864        95202 :   for (link = TREE_CHAIN (link); link; link = TREE_CHAIN (link))
    7865              :     {
    7866        46296 :       constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
    7867        92592 :       if (num_occurrences (',', constraint) != num)
    7868              :         return -1;
    7869              :     }
    7870        48906 :   return num + 1;
    7871              : }
    7872              : 
    7873              : /* Gimplify the operands of an ASM_EXPR.  Input operands should be a gimple
    7874              :    value; output operands should be a gimple lvalue.  */
    7875              : 
    7876              : static enum gimplify_status
    7877        83267 : gimplify_asm_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
    7878              : {
    7879        83267 :   tree expr;
    7880        83267 :   int noutputs;
    7881        83267 :   const char **oconstraints;
    7882        83267 :   int i;
    7883        83267 :   tree link;
    7884        83267 :   const char *constraint;
    7885        83267 :   bool allows_mem, allows_reg, is_inout;
    7886        83267 :   enum gimplify_status ret, tret;
    7887        83267 :   gasm *stmt;
    7888        83267 :   vec<tree, va_gc> *inputs;
    7889        83267 :   vec<tree, va_gc> *outputs;
    7890        83267 :   vec<tree, va_gc> *clobbers;
    7891        83267 :   vec<tree, va_gc> *labels;
    7892        83267 :   tree link_next;
    7893              : 
    7894        83267 :   expr = *expr_p;
    7895        83267 :   noutputs = list_length (ASM_OUTPUTS (expr));
    7896        83267 :   oconstraints = (const char **) alloca ((noutputs) * sizeof (const char *));
    7897              : 
    7898        83267 :   inputs = NULL;
    7899        83267 :   outputs = NULL;
    7900        83267 :   clobbers = NULL;
    7901        83267 :   labels = NULL;
    7902              : 
    7903        83267 :   int num_alternatives_out = num_alternatives (ASM_OUTPUTS (expr));
    7904        83267 :   int num_alternatives_in = num_alternatives (ASM_INPUTS (expr));
    7905        83267 :   if (num_alternatives_out == -1 || num_alternatives_in == -1
    7906        83267 :       || (num_alternatives_out > 0 && num_alternatives_in > 0
    7907        17324 :           && num_alternatives_out != num_alternatives_in))
    7908              :     {
    7909            0 :       error ("operand constraints for %<asm%> differ "
    7910              :              "in number of alternatives");
    7911            0 :       return GS_ERROR;
    7912              :     }
    7913        83267 :   int num_alternatives = MAX (num_alternatives_out, num_alternatives_in);
    7914              : 
    7915        83267 :   gimplify_reg_info reg_info (num_alternatives, noutputs);
    7916              : 
    7917        83267 :   link_next = NULL_TREE;
    7918       164056 :   for (link = ASM_CLOBBERS (expr); link; link = link_next)
    7919              :     {
    7920              :       /* The clobber entry could also be an error marker.  */
    7921        80789 :       if (TREE_CODE (TREE_VALUE (link)) == STRING_CST)
    7922              :         {
    7923        80766 :           const char *regname= TREE_STRING_POINTER (TREE_VALUE (link));
    7924        80766 :           int regno = decode_reg_name (regname);
    7925        80766 :           if (regno >= 0)
    7926        41267 :             reg_info.set_clobbered (regno);
    7927              :         }
    7928        80789 :       link_next = TREE_CHAIN (link);
    7929        80789 :       TREE_CHAIN (link) = NULL_TREE;
    7930        80789 :       vec_safe_push (clobbers, link);
    7931              :     }
    7932              : 
    7933        83267 :   ret = GS_ALL_DONE;
    7934        83267 :   link_next = NULL_TREE;
    7935       144681 :   for (i = 0, link = ASM_OUTPUTS (expr); link; ++i, link = link_next)
    7936              :     {
    7937        61447 :       bool ok;
    7938        61447 :       size_t constraint_len;
    7939              : 
    7940        61447 :       if (error_operand_p (TREE_VALUE (link)))
    7941              :         return GS_ERROR;
    7942        61426 :       link_next = TREE_CHAIN (link);
    7943              : 
    7944       122852 :       oconstraints[i]
    7945        61426 :         = constraint
    7946        61426 :         = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
    7947        61426 :       constraint_len = strlen (constraint);
    7948        61426 :       if (constraint_len == 0)
    7949            0 :         continue;
    7950              : 
    7951        61426 :       reg_info.operand = TREE_VALUE (link);
    7952        61426 :       ok = parse_output_constraint (&constraint, i, 0, 0, &allows_mem,
    7953              :                                     &allows_reg, &is_inout, &reg_info);
    7954        61426 :       if (!ok)
    7955              :         {
    7956           11 :           ret = GS_ERROR;
    7957           11 :           is_inout = false;
    7958              :         }
    7959              : 
    7960              :       /* If we can't make copies, we can only accept memory.
    7961              :          Similarly for VLAs.  */
    7962        61426 :       tree outtype = TREE_TYPE (TREE_VALUE (link));
    7963        61426 :       if (TREE_ADDRESSABLE (outtype)
    7964        61403 :           || !COMPLETE_TYPE_P (outtype)
    7965       122788 :           || !tree_fits_poly_uint64_p (TYPE_SIZE_UNIT (outtype)))
    7966              :         {
    7967          288 :           if (allows_mem)
    7968          276 :             allows_reg = 0;
    7969              :           else
    7970              :             {
    7971           12 :               error ("impossible constraint in %<asm%>");
    7972           12 :               error ("non-memory output %d must stay in memory", i);
    7973           12 :               return GS_ERROR;
    7974              :             }
    7975              :         }
    7976              : 
    7977        61414 :       if (!allows_reg && allows_mem)
    7978         2079 :         mark_addressable (TREE_VALUE (link));
    7979              : 
    7980        61414 :       tree orig = TREE_VALUE (link);
    7981       114098 :       tret = gimplify_expr (&TREE_VALUE (link), pre_p, post_p,
    7982              :                             is_inout ? is_gimple_min_lval : is_gimple_lvalue,
    7983              :                             fb_lvalue | fb_mayfail);
    7984        61414 :       if (tret == GS_ERROR)
    7985              :         {
    7986            0 :           if (orig != error_mark_node)
    7987            0 :             error ("invalid lvalue in %<asm%> output %d", i);
    7988              :           ret = tret;
    7989              :         }
    7990              : 
    7991              :       /* If the gimplified operand is a register we do not allow memory.  */
    7992        61414 :       if (allows_reg
    7993        59332 :           && allows_mem
    7994        67609 :           && (is_gimple_reg (TREE_VALUE (link))
    7995          267 :               || (handled_component_p (TREE_VALUE (link))
    7996           36 :                   && is_gimple_reg (TREE_OPERAND (TREE_VALUE (link), 0)))))
    7997         5929 :         allows_mem = 0;
    7998              : 
    7999              :       /* If the constraint does not allow memory make sure we gimplify
    8000              :          it to a register if it is not already but its base is.  This
    8001              :          happens for complex and vector components.  */
    8002        61414 :       if (!allows_mem)
    8003              :         {
    8004        59069 :           tree op = TREE_VALUE (link);
    8005        59069 :           if (! is_gimple_val (op)
    8006        21486 :               && is_gimple_reg_type (TREE_TYPE (op))
    8007        80554 :               && is_gimple_reg (get_base_address (op)))
    8008              :             {
    8009           19 :               tree tem = create_tmp_reg (TREE_TYPE (op));
    8010           19 :               tree ass;
    8011           19 :               if (is_inout)
    8012              :                 {
    8013            7 :                   ass = build2 (MODIFY_EXPR, TREE_TYPE (tem),
    8014              :                                 tem, unshare_expr (op));
    8015            7 :                   gimplify_and_add (ass, pre_p);
    8016              :                 }
    8017           19 :               ass = build2 (MODIFY_EXPR, TREE_TYPE (tem), op, tem);
    8018           19 :               gimplify_and_add (ass, post_p);
    8019              : 
    8020           19 :               TREE_VALUE (link) = tem;
    8021           19 :               tret = GS_OK;
    8022              :             }
    8023              :         }
    8024              : 
    8025        61414 :       vec_safe_push (outputs, link);
    8026        61414 :       TREE_CHAIN (link) = NULL_TREE;
    8027              : 
    8028        61414 :       if (is_inout)
    8029              :         {
    8030              :           /* An input/output operand.  To give the optimizers more
    8031              :              flexibility, split it into separate input and output
    8032              :              operands.  */
    8033         8730 :           tree input;
    8034              :           /* Buffer big enough to format a 32-bit UINT_MAX into.  */
    8035         8730 :           char buf[11];
    8036              : 
    8037              :           /* Turn the in/out constraint into an output constraint.  */
    8038         8730 :           char *p = xstrdup (constraint);
    8039         8730 :           p[0] = '=';
    8040         8730 :           TREE_PURPOSE (link) = unshare_expr (TREE_PURPOSE (link));
    8041         8730 :           TREE_VALUE (TREE_PURPOSE (link)) = build_string (constraint_len, p);
    8042              : 
    8043              :           /* And add a matching input constraint.  */
    8044         8730 :           if (allows_reg)
    8045              :             {
    8046         8229 :               sprintf (buf, "%u", i);
    8047              : 
    8048              :               /* If there are multiple alternatives in the constraint,
    8049              :                  handle each of them individually.  Those that allow register
    8050              :                  will be replaced with operand number, the others will stay
    8051              :                  unchanged.  */
    8052         8229 :               if (strchr (p, ',') != NULL)
    8053              :                 {
    8054          100 :                   size_t len = 0, buflen = strlen (buf);
    8055          100 :                   char *beg, *end, *str, *dst;
    8056              : 
    8057          100 :                   for (beg = p + 1;;)
    8058              :                     {
    8059          200 :                       end = strchr (beg, ',');
    8060          200 :                       if (end == NULL)
    8061          100 :                         end = strchr (beg, '\0');
    8062          200 :                       if ((size_t) (end - beg) < buflen)
    8063           28 :                         len += buflen + 1;
    8064              :                       else
    8065          172 :                         len += end - beg + 1;
    8066          200 :                       if (*end)
    8067          100 :                         beg = end + 1;
    8068              :                       else
    8069              :                         break;
    8070              :                     }
    8071              : 
    8072          100 :                   str = (char *) alloca (len);
    8073          100 :                   for (beg = p + 1, dst = str;;)
    8074              :                     {
    8075          200 :                       const char *tem;
    8076          200 :                       bool mem_p, reg_p, inout_p;
    8077              : 
    8078          200 :                       end = strchr (beg, ',');
    8079          200 :                       if (end)
    8080          100 :                         *end = '\0';
    8081          200 :                       beg[-1] = '=';
    8082          200 :                       tem = beg - 1;
    8083          200 :                       parse_output_constraint (&tem, i, 0, 0, &mem_p, &reg_p,
    8084              :                                                &inout_p, nullptr);
    8085          200 :                       if (dst != str)
    8086          100 :                         *dst++ = ',';
    8087          200 :                       if (reg_p)
    8088              :                         {
    8089          115 :                           memcpy (dst, buf, buflen);
    8090          115 :                           dst += buflen;
    8091              :                         }
    8092              :                       else
    8093              :                         {
    8094           85 :                           if (end)
    8095            0 :                             len = end - beg;
    8096              :                           else
    8097           85 :                             len = strlen (beg);
    8098           85 :                           memcpy (dst, beg, len);
    8099           85 :                           dst += len;
    8100              :                         }
    8101          200 :                       if (end)
    8102          100 :                         beg = end + 1;
    8103              :                       else
    8104              :                         break;
    8105          100 :                     }
    8106          100 :                   *dst = '\0';
    8107          100 :                   input = build_string (dst - str, str);
    8108              :                 }
    8109              :               else
    8110         8129 :                 input = build_string (strlen (buf), buf);
    8111              :             }
    8112              :           else
    8113          501 :             input = build_string (constraint_len - 1, constraint + 1);
    8114              : 
    8115         8730 :           free (p);
    8116              : 
    8117         8730 :           input = build_tree_list (build_tree_list (NULL_TREE, input),
    8118         8730 :                                    unshare_expr (TREE_VALUE (link)));
    8119         8730 :           ASM_INPUTS (expr) = chainon (ASM_INPUTS (expr), input);
    8120              :         }
    8121              :     }
    8122              : 
    8123              :   /* After all output operands have been gimplified, verify that each output
    8124              :      operand is used at most once in case of hard register constraints.  Thus,
    8125              :      error out in cases like
    8126              :        asm ("" : "={0}" (x), "={1}" (x));
    8127              :      or even for
    8128              :        asm ("" : "=r" (x), "={1}" (x));
    8129              : 
    8130              :      FIXME: Ideally we would also error out for cases like
    8131              :        int x;
    8132              :        asm ("" : "=r" (x), "=r" (x));
    8133              :      However, since code like that was previously accepted, erroring out now might
    8134              :      break existing code.  On the other hand, we already error out for register
    8135              :      asm like
    8136              :        register int x asm ("0");
    8137              :        asm ("" : "=r" (x), "=r" (x));
    8138              :      Thus, maybe it wouldn't be too bad to also error out in the former
    8139              :      non-register-asm case.
    8140              :   */
    8141       144644 :   for (unsigned i = 0; i < vec_safe_length (outputs); ++i)
    8142              :     {
    8143        61412 :       tree link = (*outputs)[i];
    8144        61412 :       tree op1 = TREE_VALUE (link);
    8145        61412 :       const char *constraint
    8146        61412 :         = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
    8147        61412 :       if (strchr (constraint, '{') != nullptr)
    8148        61552 :         for (unsigned j = 0; j < vec_safe_length (outputs); ++j)
    8149              :           {
    8150          142 :             if (i == j)
    8151           95 :               continue;
    8152           47 :             tree link2 = (*outputs)[j];
    8153           47 :             tree op2 = TREE_VALUE (link2);
    8154           47 :             if (op1 == op2)
    8155              :               {
    8156            2 :                 error ("multiple outputs to lvalue %qE", op2);
    8157            2 :                 return GS_ERROR;
    8158              :               }
    8159              :           }
    8160              :     }
    8161              : 
    8162        83232 :   link_next = NULL_TREE;
    8163        83232 :   int input_num = 0;
    8164       125655 :   for (link = ASM_INPUTS (expr); link; ++input_num, ++i, link = link_next)
    8165              :     {
    8166        42481 :       if (error_operand_p (TREE_VALUE (link)))
    8167              :         return GS_ERROR;
    8168        42432 :       link_next = TREE_CHAIN (link);
    8169        42432 :       constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
    8170        42432 :       reg_info.operand = TREE_VALUE (link);
    8171        42432 :       bool ok = parse_input_constraint (&constraint, input_num, 0, noutputs, 0,
    8172              :                                         oconstraints, &allows_mem, &allows_reg,
    8173              :                                         &reg_info);
    8174        42432 :       if (!ok)
    8175              :         {
    8176           30 :           ret = GS_ERROR;
    8177           30 :           is_inout = false;
    8178              :         }
    8179              : 
    8180              :       /* If we can't make copies, we can only accept memory.  */
    8181        42432 :       tree intype = TREE_TYPE (TREE_VALUE (link));
    8182        42432 :       if (TREE_ADDRESSABLE (intype)
    8183        42406 :           || !COMPLETE_TYPE_P (intype)
    8184        84783 :           || !tree_fits_poly_uint64_p (TYPE_SIZE_UNIT (intype)))
    8185              :         {
    8186          297 :           if (allows_mem)
    8187          288 :             allows_reg = 0;
    8188              :           else
    8189              :             {
    8190            9 :               error ("impossible constraint in %<asm%>");
    8191            9 :               error ("non-memory input %d must stay in memory", i);
    8192            9 :               return GS_ERROR;
    8193              :             }
    8194              :         }
    8195              : 
    8196              :       /* If the operand is a memory input, it should be an lvalue.  */
    8197        42423 :       if (!allows_reg && allows_mem)
    8198              :         {
    8199         2391 :           tree inputv = TREE_VALUE (link);
    8200         2391 :           STRIP_NOPS (inputv);
    8201         2391 :           if (TREE_CODE (inputv) == PREDECREMENT_EXPR
    8202              :               || TREE_CODE (inputv) == PREINCREMENT_EXPR
    8203              :               || TREE_CODE (inputv) == POSTDECREMENT_EXPR
    8204         2391 :               || TREE_CODE (inputv) == POSTINCREMENT_EXPR
    8205         2379 :               || TREE_CODE (inputv) == MODIFY_EXPR
    8206         4768 :               || VOID_TYPE_P (TREE_TYPE (inputv)))
    8207           38 :             TREE_VALUE (link) = error_mark_node;
    8208         2391 :           tret = gimplify_expr (&TREE_VALUE (link), pre_p, post_p,
    8209              :                                 is_gimple_lvalue, fb_lvalue | fb_mayfail);
    8210         2391 :           if (tret != GS_ERROR)
    8211              :             {
    8212              :               /* Unlike output operands, memory inputs are not guaranteed
    8213              :                  to be lvalues by the FE, and while the expressions are
    8214              :                  marked addressable there, if it is e.g. a statement
    8215              :                  expression, temporaries in it might not end up being
    8216              :                  addressable.  They might be already used in the IL and thus
    8217              :                  it is too late to make them addressable now though.  */
    8218         2345 :               tree x = TREE_VALUE (link);
    8219         2487 :               while (handled_component_p (x))
    8220          142 :                 x = TREE_OPERAND (x, 0);
    8221         2345 :               if (TREE_CODE (x) == MEM_REF
    8222         2345 :                   && TREE_CODE (TREE_OPERAND (x, 0)) == ADDR_EXPR)
    8223            0 :                 x = TREE_OPERAND (TREE_OPERAND (x, 0), 0);
    8224         2345 :               if ((VAR_P (x)
    8225              :                    || TREE_CODE (x) == PARM_DECL
    8226              :                    || TREE_CODE (x) == RESULT_DECL)
    8227         1488 :                   && !TREE_ADDRESSABLE (x)
    8228           23 :                   && is_gimple_reg (x))
    8229              :                 {
    8230           17 :                   warning_at (EXPR_LOC_OR_LOC (TREE_VALUE (link),
    8231           17 :                                                input_location), 0,
    8232              :                               "memory input %d is not directly addressable",
    8233              :                               i);
    8234           17 :                   prepare_gimple_addressable (&TREE_VALUE (link), pre_p);
    8235              :                 }
    8236              :             }
    8237         2391 :           mark_addressable (TREE_VALUE (link));
    8238         2391 :           if (tret == GS_ERROR)
    8239              :             {
    8240           46 :               if (inputv != error_mark_node)
    8241           46 :                 error_at (EXPR_LOC_OR_LOC (TREE_VALUE (link), input_location),
    8242              :                           "memory input %d is not directly addressable", i);
    8243              :               ret = tret;
    8244              :             }
    8245              :         }
    8246              :       else
    8247              :         {
    8248        40032 :           tret = gimplify_expr (&TREE_VALUE (link), pre_p, post_p,
    8249              :                                 is_gimple_asm_val, fb_rvalue);
    8250        40032 :           if (tret == GS_ERROR)
    8251           46 :             ret = tret;
    8252              :         }
    8253              : 
    8254        42423 :       TREE_CHAIN (link) = NULL_TREE;
    8255        42423 :       vec_safe_push (inputs, link);
    8256              :     }
    8257              : 
    8258        83174 :   link_next = NULL_TREE;
    8259        84131 :   for (link = ASM_LABELS (expr); link; link = link_next)
    8260              :     {
    8261          957 :       link_next = TREE_CHAIN (link);
    8262          957 :       TREE_CHAIN (link) = NULL_TREE;
    8263          957 :       vec_safe_push (labels, link);
    8264              :     }
    8265              : 
    8266              :   /* Do not add ASMs with errors to the gimple IL stream.  */
    8267        83174 :   if (ret != GS_ERROR)
    8268              :     {
    8269        83091 :       stmt = gimple_build_asm_vec (TREE_STRING_POINTER (ASM_STRING (expr)),
    8270              :                                    inputs, outputs, clobbers, labels);
    8271              : 
    8272              :       /* asm is volatile if it was marked by the user as volatile or
    8273              :          there are no outputs or this is an asm goto.  */
    8274        90610 :       gimple_asm_set_volatile (stmt,
    8275        83091 :                                ASM_VOLATILE_P (expr)
    8276         7533 :                                || noutputs == 0
    8277              :                                || labels);
    8278        83091 :       gimple_asm_set_basic (stmt, ASM_BASIC_P (expr));
    8279        83091 :       gimple_asm_set_inline (stmt, ASM_INLINE_P (expr));
    8280              : 
    8281        83091 :       gimplify_seq_add_stmt (pre_p, stmt);
    8282              :     }
    8283              : 
    8284              :   return ret;
    8285        83267 : }
    8286              : 
    8287              : /* Gimplify a CLEANUP_POINT_EXPR.  Currently this works by adding
    8288              :    GIMPLE_WITH_CLEANUP_EXPRs to the prequeue as we encounter cleanups while
    8289              :    gimplifying the body, and converting them to TRY_FINALLY_EXPRs when we
    8290              :    return to this function.
    8291              : 
    8292              :    FIXME should we complexify the prequeue handling instead?  Or use flags
    8293              :    for all the cleanups and let the optimizer tighten them up?  The current
    8294              :    code seems pretty fragile; it will break on a cleanup within any
    8295              :    non-conditional nesting.  But any such nesting would be broken, anyway;
    8296              :    we can't write a TRY_FINALLY_EXPR that starts inside a nesting construct
    8297              :    and continues out of it.  We can do that at the RTL level, though, so
    8298              :    having an optimizer to tighten up try/finally regions would be a Good
    8299              :    Thing.  */
    8300              : 
    8301              : static enum gimplify_status
    8302      5436768 : gimplify_cleanup_point_expr (tree *expr_p, gimple_seq *pre_p)
    8303              : {
    8304      5436768 :   gimple_stmt_iterator iter;
    8305      5436768 :   gimple_seq body_sequence = NULL;
    8306              : 
    8307      5436768 :   tree temp = voidify_wrapper_expr (*expr_p, NULL);
    8308              : 
    8309              :   /* We only care about the number of conditions between the innermost
    8310              :      CLEANUP_POINT_EXPR and the cleanup.  So save and reset the count and
    8311              :      any cleanups collected outside the CLEANUP_POINT_EXPR.  */
    8312      5436768 :   int old_conds = gimplify_ctxp->conditions;
    8313      5436768 :   gimple_seq old_cleanups = gimplify_ctxp->conditional_cleanups;
    8314      5436768 :   bool old_in_cleanup_point_expr = gimplify_ctxp->in_cleanup_point_expr;
    8315      5436768 :   gimplify_ctxp->conditions = 0;
    8316      5436768 :   gimplify_ctxp->conditional_cleanups = NULL;
    8317      5436768 :   gimplify_ctxp->in_cleanup_point_expr = true;
    8318              : 
    8319      5436768 :   gimplify_stmt (&TREE_OPERAND (*expr_p, 0), &body_sequence);
    8320              : 
    8321      5436768 :   gimplify_ctxp->conditions = old_conds;
    8322      5436768 :   gimplify_ctxp->conditional_cleanups = old_cleanups;
    8323      5436768 :   gimplify_ctxp->in_cleanup_point_expr = old_in_cleanup_point_expr;
    8324              : 
    8325     28183398 :   for (iter = gsi_start (body_sequence); !gsi_end_p (iter); )
    8326              :     {
    8327     17356720 :       gimple *wce = gsi_stmt (iter);
    8328              : 
    8329     17356720 :       if (gimple_code (wce) == GIMPLE_WITH_CLEANUP_EXPR)
    8330              :         {
    8331       650491 :           if (gsi_one_before_end_p (iter))
    8332              :             {
    8333              :               /* Note that gsi_insert_seq_before and gsi_remove do not
    8334              :                  scan operands, unlike some other sequence mutators.  */
    8335         3125 :               if (!gimple_wce_cleanup_eh_only (wce))
    8336         3125 :                 gsi_insert_seq_before_without_update (&iter,
    8337              :                                                       gimple_wce_cleanup (wce),
    8338              :                                                       GSI_SAME_STMT);
    8339         3125 :               gsi_remove (&iter, true);
    8340         3125 :               break;
    8341              :             }
    8342              :           else
    8343              :             {
    8344       647366 :               gtry *gtry;
    8345       647366 :               gimple_seq seq;
    8346       647366 :               enum gimple_try_flags kind;
    8347              : 
    8348       647366 :               if (gimple_wce_cleanup_eh_only (wce))
    8349              :                 kind = GIMPLE_TRY_CATCH;
    8350              :               else
    8351       630131 :                 kind = GIMPLE_TRY_FINALLY;
    8352       647366 :               seq = gsi_split_seq_after (iter);
    8353              : 
    8354       647366 :               gtry = gimple_build_try (seq, gimple_wce_cleanup (wce), kind);
    8355              :               /* Do not use gsi_replace here, as it may scan operands.
    8356              :                  We want to do a simple structural modification only.  */
    8357       647366 :               gsi_set_stmt (&iter, gtry);
    8358      1294732 :               iter = gsi_start (gtry->eval);
    8359              :             }
    8360              :         }
    8361              :       else
    8362     16706229 :         gsi_next (&iter);
    8363              :     }
    8364              : 
    8365      5436768 :   gimplify_seq_add_seq (pre_p, body_sequence);
    8366      5436768 :   if (temp)
    8367              :     {
    8368              :       *expr_p = temp;
    8369              :       return GS_OK;
    8370              :     }
    8371              :   else
    8372              :     {
    8373      5080520 :       *expr_p = NULL;
    8374      5080520 :       return GS_ALL_DONE;
    8375              :     }
    8376              : }
    8377              : 
    8378              : /* Insert a cleanup marker for gimplify_cleanup_point_expr.  CLEANUP
    8379              :    is the cleanup action required.  EH_ONLY is true if the cleanup should
    8380              :    only be executed if an exception is thrown, not on normal exit.
    8381              :    If FORCE_UNCOND is true perform the cleanup unconditionally;  this is
    8382              :    only valid for clobbers.  */
    8383              : 
    8384              : static void
    8385       655614 : gimple_push_cleanup (tree var, tree cleanup, bool eh_only, gimple_seq *pre_p,
    8386              :                      bool force_uncond = false)
    8387              : {
    8388       655614 :   gimple *wce;
    8389       655614 :   gimple_seq cleanup_stmts = NULL;
    8390              : 
    8391              :   /* Errors can result in improperly nested cleanups.  Which results in
    8392              :      confusion when trying to resolve the GIMPLE_WITH_CLEANUP_EXPR.  */
    8393       655614 :   if (seen_error ())
    8394         5123 :     return;
    8395              : 
    8396       650491 :   if (gimple_conditional_context ())
    8397              :     {
    8398              :       /* If we're in a conditional context, this is more complex.  We only
    8399              :          want to run the cleanup if we actually ran the initialization that
    8400              :          necessitates it, but we want to run it after the end of the
    8401              :          conditional context.  So we wrap the try/finally around the
    8402              :          condition and use a flag to determine whether or not to actually
    8403              :          run the destructor.  Thus
    8404              : 
    8405              :            test ? f(A()) : 0
    8406              : 
    8407              :          becomes (approximately)
    8408              : 
    8409              :            flag = 0;
    8410              :            try {
    8411              :              if (test) { A::A(temp); flag = 1; val = f(temp); }
    8412              :              else { val = 0; }
    8413              :            } finally {
    8414              :              if (flag) A::~A(temp);
    8415              :            }
    8416              :            val
    8417              :       */
    8418        15540 :       if (force_uncond)
    8419              :         {
    8420        14670 :           gimplify_stmt (&cleanup, &cleanup_stmts);
    8421        14670 :           wce = gimple_build_wce (cleanup_stmts);
    8422        14670 :           gimplify_seq_add_stmt (&gimplify_ctxp->conditional_cleanups, wce);
    8423              :         }
    8424              :       else
    8425              :         {
    8426          870 :           tree flag = create_tmp_var (boolean_type_node, "cleanup");
    8427          870 :           gassign *ffalse = gimple_build_assign (flag, boolean_false_node);
    8428          870 :           gassign *ftrue = gimple_build_assign (flag, boolean_true_node);
    8429              : 
    8430          870 :           cleanup = build3 (COND_EXPR, void_type_node, flag, cleanup, NULL);
    8431          870 :           gimplify_stmt (&cleanup, &cleanup_stmts);
    8432          870 :           wce = gimple_build_wce (cleanup_stmts);
    8433          870 :           gimple_wce_set_cleanup_eh_only (wce, eh_only);
    8434              : 
    8435          870 :           gimplify_seq_add_stmt (&gimplify_ctxp->conditional_cleanups, ffalse);
    8436          870 :           gimplify_seq_add_stmt (&gimplify_ctxp->conditional_cleanups, wce);
    8437          870 :           gimplify_seq_add_stmt (pre_p, ftrue);
    8438              : 
    8439              :           /* Because of this manipulation, and the EH edges that jump
    8440              :              threading cannot redirect, the temporary (VAR) will appear
    8441              :              to be used uninitialized.  Don't warn.  */
    8442          870 :           suppress_warning (var, OPT_Wuninitialized);
    8443              :         }
    8444              :     }
    8445              :   else
    8446              :     {
    8447       634951 :       gimplify_stmt (&cleanup, &cleanup_stmts);
    8448       634951 :       wce = gimple_build_wce (cleanup_stmts);
    8449       634951 :       gimple_wce_set_cleanup_eh_only (wce, eh_only);
    8450       634951 :       gimplify_seq_add_stmt (pre_p, wce);
    8451              :     }
    8452              : }
    8453              : 
    8454              : /* Gimplify a TARGET_EXPR which doesn't appear on the rhs of an INIT_EXPR.  */
    8455              : 
    8456              : static enum gimplify_status
    8457       868857 : gimplify_target_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
    8458              : {
    8459       868857 :   tree targ = *expr_p;
    8460       868857 :   tree temp = TARGET_EXPR_SLOT (targ);
    8461       868857 :   tree init = TARGET_EXPR_INITIAL (targ);
    8462       868857 :   enum gimplify_status ret;
    8463              : 
    8464       868857 :   bool unpoison_empty_seq = false;
    8465       868857 :   gimple_stmt_iterator unpoison_it;
    8466              : 
    8467       868857 :   if (init)
    8468              :     {
    8469       834588 :       gimple_seq init_pre_p = NULL;
    8470       834588 :       bool is_vla = false;
    8471              : 
    8472              :       /* TARGET_EXPR temps aren't part of the enclosing block, so add it
    8473              :          to the temps list.  Handle also variable length TARGET_EXPRs.  */
    8474       834588 :       if (!poly_int_tree_p (DECL_SIZE (temp)))
    8475              :         {
    8476           44 :           if (!TYPE_SIZES_GIMPLIFIED (TREE_TYPE (temp)))
    8477            1 :             gimplify_type_sizes (TREE_TYPE (temp), &init_pre_p);
    8478              :           /* FIXME: this is correct only when the size of the type does
    8479              :              not depend on expressions evaluated in init.  */
    8480           44 :           gimplify_vla_decl (temp, &init_pre_p);
    8481           44 :           is_vla = true;
    8482              :         }
    8483              :       else
    8484              :         {
    8485              :           /* Save location where we need to place unpoisoning.  It's possible
    8486              :              that a variable will be converted to needs_to_live_in_memory.  */
    8487       834544 :           unpoison_it = gsi_last (*pre_p);
    8488       834544 :           unpoison_empty_seq = gsi_end_p (unpoison_it);
    8489              : 
    8490       834544 :           gimple_add_tmp_var (temp);
    8491              :         }
    8492              : 
    8493       834588 :       if (var_needs_auto_init_p (temp) && VOID_TYPE_P (TREE_TYPE (init)))
    8494              :         {
    8495        60050 :           gimple_add_init_for_auto_var (temp, flag_auto_var_init, &init_pre_p);
    8496        60050 :           if ((flag_auto_var_init & ~AUTO_INIT_CXX26) == AUTO_INIT_PATTERN
    8497            0 :               && !is_gimple_reg (temp)
    8498        60050 :               && clear_padding_type_may_have_padding_p (TREE_TYPE (temp)))
    8499            0 :             gimple_add_padding_init_for_auto_var (temp, is_vla, &init_pre_p);
    8500              :         }
    8501              : 
    8502              :       /* If TARGET_EXPR_INITIAL is void, then the mere evaluation of the
    8503              :          expression is supposed to initialize the slot.  */
    8504       834588 :       if (VOID_TYPE_P (TREE_TYPE (init)))
    8505       294032 :         ret = gimplify_expr (&init, &init_pre_p, post_p, is_gimple_stmt,
    8506              :                              fb_none);
    8507              :       else
    8508              :         {
    8509       540556 :           tree init_expr = build2 (INIT_EXPR, void_type_node, temp, init);
    8510       540556 :           init = init_expr;
    8511       540556 :           ret = gimplify_expr (&init, &init_pre_p, post_p, is_gimple_stmt,
    8512              :                                fb_none);
    8513       540556 :           init = NULL;
    8514       540556 :           ggc_free (init_expr);
    8515              :         }
    8516       834588 :       if (ret == GS_ERROR)
    8517              :         {
    8518              :           /* PR c++/28266 Make sure this is expanded only once. */
    8519           10 :           TARGET_EXPR_INITIAL (targ) = NULL_TREE;
    8520           10 :           return GS_ERROR;
    8521              :         }
    8522              : 
    8523       834578 :       if (init)
    8524            0 :         gimplify_and_add (init, &init_pre_p);
    8525              : 
    8526              :       /* Add a clobber for the temporary going out of scope, like
    8527              :          gimplify_bind_expr.  But only if we did not promote the
    8528              :          temporary to static storage.  */
    8529       834578 :       if (gimplify_ctxp->in_cleanup_point_expr
    8530       690303 :           && !TREE_STATIC (temp)
    8531      1522731 :           && needs_to_live_in_memory (temp))
    8532              :         {
    8533       501802 :           if (flag_stack_reuse == SR_ALL)
    8534              :             {
    8535       501322 :               tree clobber = build_clobber (TREE_TYPE (temp),
    8536              :                                             CLOBBER_STORAGE_END);
    8537       501322 :               clobber = build2 (MODIFY_EXPR, TREE_TYPE (temp), temp, clobber);
    8538       501322 :               gimple_push_cleanup (temp, clobber, false, pre_p, true);
    8539              :             }
    8540       501802 :           if (asan_poisoned_variables
    8541          480 :               && DECL_ALIGN (temp) <= MAX_SUPPORTED_STACK_ALIGNMENT
    8542          480 :               && !TREE_STATIC (temp)
    8543          480 :               && dbg_cnt (asan_use_after_scope)
    8544       502282 :               && !gimplify_omp_ctxp)
    8545              :             {
    8546          472 :               tree asan_cleanup = build_asan_poison_call_expr (temp);
    8547          472 :               if (asan_cleanup)
    8548              :                 {
    8549          472 :                   if (unpoison_empty_seq)
    8550          217 :                     unpoison_it = gsi_start (*pre_p);
    8551              : 
    8552          472 :                   asan_poison_variable (temp, false, &unpoison_it,
    8553              :                                         unpoison_empty_seq);
    8554          472 :                   gimple_push_cleanup (temp, asan_cleanup, false, pre_p);
    8555              :                 }
    8556              :             }
    8557              :         }
    8558              : 
    8559       834578 :       gimple_seq_add_seq (pre_p, init_pre_p);
    8560              : 
    8561              :       /* If needed, push the cleanup for the temp.  */
    8562       834578 :       if (TARGET_EXPR_CLEANUP (targ))
    8563       153820 :         gimple_push_cleanup (temp, TARGET_EXPR_CLEANUP (targ),
    8564       153820 :                              CLEANUP_EH_ONLY (targ), pre_p);
    8565              : 
    8566              :       /* Only expand this once.  */
    8567       834578 :       TREE_OPERAND (targ, 3) = init;
    8568       834578 :       TARGET_EXPR_INITIAL (targ) = NULL_TREE;
    8569              :     }
    8570              :   else
    8571              :     /* We should have expanded this before.  */
    8572        34269 :     gcc_assert (DECL_SEEN_IN_BIND_EXPR_P (temp));
    8573              : 
    8574       868847 :   *expr_p = temp;
    8575       868847 :   return GS_OK;
    8576              : }
    8577              : 
    8578              : /* Gimplification of expression trees.  */
    8579              : 
    8580              : /* Gimplify an expression which appears at statement context.  The
    8581              :    corresponding GIMPLE statements are added to *SEQ_P.  If *SEQ_P is
    8582              :    NULL, a new sequence is allocated.
    8583              : 
    8584              :    Return true if we actually added a statement to the queue.  */
    8585              : 
    8586              : bool
    8587    104662482 : gimplify_stmt (tree *stmt_p, gimple_seq *seq_p)
    8588              : {
    8589    104662482 :   gimple_seq_node last;
    8590              : 
    8591    104662482 :   last = gimple_seq_last (*seq_p);
    8592    104662482 :   gimplify_expr (stmt_p, seq_p, NULL, is_gimple_stmt, fb_none);
    8593    104662482 :   return last != gimple_seq_last (*seq_p);
    8594              : }
    8595              : 
    8596              : /* Add FIRSTPRIVATE entries for DECL in the OpenMP the surrounding parallels
    8597              :    to CTX.  If entries already exist, force them to be some flavor of private.
    8598              :    If there is no enclosing parallel, do nothing.  */
    8599              : 
    8600              : void
    8601       126482 : omp_firstprivatize_variable (struct gimplify_omp_ctx *ctx, tree decl)
    8602              : {
    8603       126482 :   splay_tree_node n;
    8604              : 
    8605       126482 :   if (decl == NULL || !DECL_P (decl) || ctx->region_type == ORT_NONE)
    8606              :     return;
    8607              : 
    8608        37045 :   do
    8609              :     {
    8610        37045 :       n = splay_tree_lookup (ctx->variables, (splay_tree_key)decl);
    8611        37045 :       if (n != NULL)
    8612              :         {
    8613         4307 :           if (n->value & GOVD_SHARED)
    8614          183 :             n->value = GOVD_FIRSTPRIVATE | (n->value & GOVD_SEEN);
    8615         4124 :           else if (n->value & GOVD_MAP)
    8616           58 :             n->value |= GOVD_MAP_TO_ONLY;
    8617              :           else
    8618              :             return;
    8619              :         }
    8620        32738 :       else if ((ctx->region_type & ORT_TARGET) != 0)
    8621              :         {
    8622         7057 :           if (ctx->defaultmap[GDMK_SCALAR] & GOVD_FIRSTPRIVATE)
    8623         4288 :             omp_add_variable (ctx, decl, GOVD_FIRSTPRIVATE);
    8624              :           else
    8625         2769 :             omp_add_variable (ctx, decl, GOVD_MAP | GOVD_MAP_TO_ONLY);
    8626              :         }
    8627        25681 :       else if (ctx->region_type != ORT_WORKSHARE
    8628              :                && ctx->region_type != ORT_TASKGROUP
    8629        23852 :                && ctx->region_type != ORT_SIMD
    8630        23228 :                && ctx->region_type != ORT_ACC
    8631        23111 :                && !(ctx->region_type & ORT_TARGET_DATA))
    8632        14530 :         omp_add_variable (ctx, decl, GOVD_FIRSTPRIVATE);
    8633              : 
    8634        32979 :       ctx = ctx->outer_context;
    8635              :     }
    8636        32979 :   while (ctx);
    8637              : }
    8638              : 
    8639              : /* Similarly for each of the type sizes of TYPE.  */
    8640              : 
    8641              : static void
    8642        40317 : omp_firstprivatize_type_sizes (struct gimplify_omp_ctx *ctx, tree type)
    8643              : {
    8644        40317 :   if (type == NULL || type == error_mark_node)
    8645              :     return;
    8646        40313 :   type = TYPE_MAIN_VARIANT (type);
    8647              : 
    8648        40313 :   if (ctx->privatized_types->add (type))
    8649              :     return;
    8650              : 
    8651        31383 :   switch (TREE_CODE (type))
    8652              :     {
    8653        13632 :     case INTEGER_TYPE:
    8654        13632 :     case ENUMERAL_TYPE:
    8655        13632 :     case BOOLEAN_TYPE:
    8656        13632 :     case REAL_TYPE:
    8657        13632 :     case FIXED_POINT_TYPE:
    8658        13632 :       omp_firstprivatize_variable (ctx, TYPE_MIN_VALUE (type));
    8659        13632 :       omp_firstprivatize_variable (ctx, TYPE_MAX_VALUE (type));
    8660        13632 :       break;
    8661              : 
    8662         5430 :     case ARRAY_TYPE:
    8663         5430 :       omp_firstprivatize_type_sizes (ctx, TREE_TYPE (type));
    8664         5430 :       omp_firstprivatize_type_sizes (ctx, TYPE_DOMAIN (type));
    8665         5430 :       break;
    8666              : 
    8667         2510 :     case RECORD_TYPE:
    8668         2510 :     case UNION_TYPE:
    8669         2510 :     case QUAL_UNION_TYPE:
    8670         2510 :       {
    8671         2510 :         tree field;
    8672        26797 :         for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
    8673        24287 :           if (TREE_CODE (field) == FIELD_DECL)
    8674              :             {
    8675         9013 :               omp_firstprivatize_variable (ctx, DECL_FIELD_OFFSET (field));
    8676         9013 :               omp_firstprivatize_type_sizes (ctx, TREE_TYPE (field));
    8677              :             }
    8678              :       }
    8679              :       break;
    8680              : 
    8681         9161 :     case POINTER_TYPE:
    8682         9161 :     case REFERENCE_TYPE:
    8683         9161 :       omp_firstprivatize_type_sizes (ctx, TREE_TYPE (type));
    8684         9161 :       break;
    8685              : 
    8686              :     default:
    8687              :       break;
    8688              :     }
    8689              : 
    8690        31383 :   omp_firstprivatize_variable (ctx, TYPE_SIZE (type));
    8691        31383 :   omp_firstprivatize_variable (ctx, TYPE_SIZE_UNIT (type));
    8692        31383 :   lang_hooks.types.omp_firstprivatize_type_sizes (ctx, type);
    8693              : }
    8694              : 
    8695              : /* Add an entry for DECL in the OMP context CTX with FLAGS.  */
    8696              : 
    8697              : static void
    8698       719564 : omp_add_variable (struct gimplify_omp_ctx *ctx, tree decl, unsigned int flags)
    8699              : {
    8700       719564 :   splay_tree_node n;
    8701       719564 :   unsigned int nflags;
    8702       719564 :   tree t;
    8703              : 
    8704       719564 :   if (error_operand_p (decl) || ctx->region_type == ORT_NONE)
    8705              :     return;
    8706              : 
    8707              :   /* Never elide decls whose type has TREE_ADDRESSABLE set.  This means
    8708              :      there are constructors involved somewhere.  Exception is a shared clause,
    8709              :      there is nothing privatized in that case.  */
    8710       719416 :   if ((flags & GOVD_SHARED) == 0
    8711       719416 :       && (TREE_ADDRESSABLE (TREE_TYPE (decl))
    8712       658728 :           || TYPE_NEEDS_CONSTRUCTING (TREE_TYPE (decl))))
    8713         7806 :     flags |= GOVD_SEEN;
    8714              : 
    8715       719416 :   n = splay_tree_lookup (ctx->variables, (splay_tree_key)decl);
    8716       719416 :   if (n != NULL && (n->value & GOVD_DATA_SHARE_CLASS) != 0)
    8717              :     {
    8718              :       /* We shouldn't be re-adding the decl with the same data
    8719              :          sharing class.  */
    8720          828 :       gcc_assert ((n->value & GOVD_DATA_SHARE_CLASS & flags) == 0);
    8721          828 :       nflags = n->value | flags;
    8722              :       /* The only combination of data sharing classes we should see is
    8723              :          FIRSTPRIVATE and LASTPRIVATE.  However, OpenACC permits
    8724              :          reduction variables to be used in data sharing clauses.  */
    8725          828 :       gcc_assert ((ctx->region_type & ORT_ACC) != 0
    8726              :                   || ((nflags & GOVD_DATA_SHARE_CLASS)
    8727              :                       == (GOVD_FIRSTPRIVATE | GOVD_LASTPRIVATE))
    8728              :                   || (flags & GOVD_DATA_SHARE_CLASS) == 0);
    8729          828 :       n->value = nflags;
    8730          828 :       return;
    8731              :     }
    8732              : 
    8733              :   /* When adding a variable-sized variable, we have to handle all sorts
    8734              :      of additional bits of data: the pointer replacement variable, and
    8735              :      the parameters of the type.  */
    8736       718588 :   if (DECL_SIZE (decl) && !poly_int_tree_p (DECL_SIZE (decl)))
    8737              :     {
    8738              :       /* Add the pointer replacement variable as PRIVATE if the variable
    8739              :          replacement is private, else FIRSTPRIVATE since we'll need the
    8740              :          address of the original variable either for SHARED, or for the
    8741              :          copy into or out of the context.  */
    8742         1342 :       if (!(flags & GOVD_LOCAL) && ctx->region_type != ORT_TASKGROUP)
    8743              :         {
    8744         1249 :           if (flags & GOVD_MAP)
    8745              :             nflags = GOVD_MAP | GOVD_MAP_TO_ONLY | GOVD_EXPLICIT;
    8746          540 :           else if (flags & GOVD_PRIVATE)
    8747              :             nflags = GOVD_PRIVATE;
    8748          450 :           else if (((ctx->region_type & (ORT_TARGET | ORT_TARGET_DATA)) != 0
    8749           54 :                     && (flags & GOVD_FIRSTPRIVATE))
    8750          436 :                    || (ctx->region_type == ORT_TARGET_DATA
    8751            6 :                        && (flags & GOVD_DATA_SHARE_CLASS) == 0))
    8752              :             nflags = GOVD_PRIVATE | GOVD_EXPLICIT;
    8753              :           else
    8754         1249 :             nflags = GOVD_FIRSTPRIVATE;
    8755         1249 :           nflags |= flags & GOVD_SEEN;
    8756         1249 :           t = DECL_VALUE_EXPR (decl);
    8757         1249 :           gcc_assert (INDIRECT_REF_P (t));
    8758         1249 :           t = TREE_OPERAND (t, 0);
    8759         1249 :           gcc_assert (DECL_P (t));
    8760         1249 :           omp_add_variable (ctx, t, nflags);
    8761              :         }
    8762              : 
    8763              :       /* Add all of the variable and type parameters (which should have
    8764              :          been gimplified to a formal temporary) as FIRSTPRIVATE.  */
    8765         1342 :       omp_firstprivatize_variable (ctx, DECL_SIZE_UNIT (decl));
    8766         1342 :       omp_firstprivatize_variable (ctx, DECL_SIZE (decl));
    8767         1342 :       omp_firstprivatize_type_sizes (ctx, TREE_TYPE (decl));
    8768              : 
    8769              :       /* The variable-sized variable itself is never SHARED, only some form
    8770              :          of PRIVATE.  The sharing would take place via the pointer variable
    8771              :          which we remapped above.  */
    8772         1342 :       if (flags & GOVD_SHARED)
    8773          285 :         flags = GOVD_SHARED | GOVD_DEBUG_PRIVATE
    8774          285 :                 | (flags & (GOVD_SEEN | GOVD_EXPLICIT));
    8775              : 
    8776              :       /* We're going to make use of the TYPE_SIZE_UNIT at least in the
    8777              :          alloca statement we generate for the variable, so make sure it
    8778              :          is available.  This isn't automatically needed for the SHARED
    8779              :          case, since we won't be allocating local storage then.
    8780              :          For local variables TYPE_SIZE_UNIT might not be gimplified yet,
    8781              :          in this case omp_notice_variable will be called later
    8782              :          on when it is gimplified.  */
    8783         1057 :       else if (! (flags & (GOVD_LOCAL | GOVD_MAP))
    8784         1057 :                && DECL_P (TYPE_SIZE_UNIT (TREE_TYPE (decl))))
    8785          283 :         omp_notice_variable (ctx, TYPE_SIZE_UNIT (TREE_TYPE (decl)), true);
    8786              :     }
    8787       717246 :   else if ((flags & (GOVD_MAP | GOVD_LOCAL)) == 0
    8788       717246 :            && omp_privatize_by_reference (decl))
    8789              :     {
    8790         9941 :       omp_firstprivatize_type_sizes (ctx, TREE_TYPE (decl));
    8791              : 
    8792              :       /* Similar to the direct variable sized case above, we'll need the
    8793              :          size of references being privatized.  */
    8794         9941 :       if ((flags & GOVD_SHARED) == 0)
    8795              :         {
    8796         6221 :           t = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (decl)));
    8797         6221 :           if (t && DECL_P (t))
    8798         1398 :             omp_notice_variable (ctx, t, true);
    8799              :         }
    8800              :     }
    8801              : 
    8802       718588 :   if (n != NULL)
    8803         1870 :     n->value |= flags;
    8804              :   else
    8805       716718 :     splay_tree_insert (ctx->variables, (splay_tree_key)decl, flags);
    8806              : 
    8807              :   /* For reductions clauses in OpenACC loop directives, by default create a
    8808              :      copy clause on the enclosing parallel construct for carrying back the
    8809              :      results.  */
    8810       718588 :   if (ctx->region_type == ORT_ACC && (flags & GOVD_REDUCTION))
    8811              :     {
    8812         4716 :       struct gimplify_omp_ctx *outer_ctx = ctx->outer_context;
    8813         6454 :       while (outer_ctx)
    8814              :         {
    8815         5788 :           n = splay_tree_lookup (outer_ctx->variables, (splay_tree_key)decl);
    8816         5788 :           if (n != NULL)
    8817              :             {
    8818              :               /* Ignore local variables and explicitly declared clauses.  */
    8819         3990 :               if (n->value & (GOVD_LOCAL | GOVD_EXPLICIT))
    8820              :                 break;
    8821          443 :               else if (outer_ctx->region_type == ORT_ACC_KERNELS)
    8822              :                 {
    8823              :                   /* According to the OpenACC spec, such a reduction variable
    8824              :                      should already have a copy map on a kernels construct,
    8825              :                      verify that here.  */
    8826          143 :                   gcc_assert (!(n->value & GOVD_FIRSTPRIVATE)
    8827              :                               && (n->value & GOVD_MAP));
    8828              :                 }
    8829          300 :               else if (outer_ctx->region_type == ORT_ACC_PARALLEL)
    8830              :                 {
    8831              :                   /* Remove firstprivate and make it a copy map.  */
    8832          150 :                   n->value &= ~GOVD_FIRSTPRIVATE;
    8833          150 :                   n->value |= GOVD_MAP;
    8834              :                 }
    8835              :             }
    8836         1798 :           else if (outer_ctx->region_type == ORT_ACC_PARALLEL)
    8837              :             {
    8838          503 :               splay_tree_insert (outer_ctx->variables, (splay_tree_key)decl,
    8839              :                                  GOVD_MAP | GOVD_SEEN);
    8840          503 :               break;
    8841              :             }
    8842         1738 :           outer_ctx = outer_ctx->outer_context;
    8843              :         }
    8844              :     }
    8845              : }
    8846              : 
    8847              : /* Notice a threadprivate variable DECL used in OMP context CTX.
    8848              :    This just prints out diagnostics about threadprivate variable uses
    8849              :    in untied tasks.  If DECL2 is non-NULL, prevent this warning
    8850              :    on that variable.  */
    8851              : 
    8852              : static bool
    8853        14183 : omp_notice_threadprivate_variable (struct gimplify_omp_ctx *ctx, tree decl,
    8854              :                                    tree decl2)
    8855              : {
    8856        14183 :   splay_tree_node n;
    8857        14183 :   struct gimplify_omp_ctx *octx;
    8858              : 
    8859        28798 :   for (octx = ctx; octx; octx = octx->outer_context)
    8860        14615 :     if ((octx->region_type & ORT_TARGET) != 0
    8861        14615 :         || octx->order_concurrent)
    8862              :       {
    8863          112 :         n = splay_tree_lookup (octx->variables, (splay_tree_key)decl);
    8864          112 :         if (n == NULL)
    8865              :           {
    8866           20 :             if (octx->order_concurrent)
    8867              :               {
    8868           20 :                 error ("threadprivate variable %qE used in a region with"
    8869           20 :                        " %<order(concurrent)%> clause", DECL_NAME (decl));
    8870           20 :                 inform (octx->location, "enclosing region");
    8871              :               }
    8872              :             else
    8873              :               {
    8874            0 :                 error ("threadprivate variable %qE used in target region",
    8875            0 :                        DECL_NAME (decl));
    8876            0 :                 inform (octx->location, "enclosing target region");
    8877              :               }
    8878           20 :             splay_tree_insert (octx->variables, (splay_tree_key)decl, 0);
    8879              :           }
    8880          112 :         if (decl2)
    8881            0 :           splay_tree_insert (octx->variables, (splay_tree_key)decl2, 0);
    8882              :       }
    8883              : 
    8884        14183 :   if (ctx->region_type != ORT_UNTIED_TASK)
    8885              :     return false;
    8886           37 :   n = splay_tree_lookup (ctx->variables, (splay_tree_key)decl);
    8887           37 :   if (n == NULL)
    8888              :     {
    8889            6 :       error ("threadprivate variable %qE used in untied task",
    8890            6 :              DECL_NAME (decl));
    8891            6 :       inform (ctx->location, "enclosing task");
    8892            6 :       splay_tree_insert (ctx->variables, (splay_tree_key)decl, 0);
    8893              :     }
    8894           37 :   if (decl2)
    8895            4 :     splay_tree_insert (ctx->variables, (splay_tree_key)decl2, 0);
    8896              :   return false;
    8897              : }
    8898              : 
    8899              : /* Return true if global var DECL is device resident.  */
    8900              : 
    8901              : static bool
    8902          642 : device_resident_p (tree decl)
    8903              : {
    8904          642 :   tree attr = lookup_attribute ("oacc declare target", DECL_ATTRIBUTES (decl));
    8905              : 
    8906          642 :   if (!attr)
    8907              :     return false;
    8908              : 
    8909            0 :   for (tree t = TREE_VALUE (attr); t; t = TREE_PURPOSE (t))
    8910              :     {
    8911            0 :       tree c = TREE_VALUE (t);
    8912            0 :       if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DEVICE_RESIDENT)
    8913              :         return true;
    8914              :     }
    8915              : 
    8916              :   return false;
    8917              : }
    8918              : 
    8919              : /* Return true if DECL has an ACC DECLARE attribute.  */
    8920              : 
    8921              : static bool
    8922         9101 : is_oacc_declared (tree decl)
    8923              : {
    8924         9101 :   tree t = TREE_CODE (decl) == MEM_REF ? TREE_OPERAND (decl, 0) : decl;
    8925         9101 :   tree declared = lookup_attribute ("oacc declare target", DECL_ATTRIBUTES (t));
    8926         9101 :   return declared != NULL_TREE;
    8927              : }
    8928              : 
    8929              : /* Determine outer default flags for DECL mentioned in an OMP region
    8930              :    but not declared in an enclosing clause.
    8931              : 
    8932              :    ??? Some compiler-generated variables (like SAVE_EXPRs) could be
    8933              :    remapped firstprivate instead of shared.  To some extent this is
    8934              :    addressed in omp_firstprivatize_type_sizes, but not
    8935              :    effectively.  */
    8936              : 
    8937              : static unsigned
    8938        48106 : omp_default_clause (struct gimplify_omp_ctx *ctx, tree decl,
    8939              :                     bool in_code, unsigned flags)
    8940              : {
    8941        48106 :   enum omp_clause_default_kind default_kind = ctx->default_kind;
    8942        48106 :   enum omp_clause_default_kind kind;
    8943              : 
    8944        48106 :   kind = lang_hooks.decls.omp_predetermined_sharing (decl);
    8945        48106 :   if (ctx->region_type & ORT_TASK)
    8946              :     {
    8947         3996 :       tree detach_clause = omp_find_clause (ctx->clauses, OMP_CLAUSE_DETACH);
    8948              : 
    8949              :       /* The event-handle specified by a detach clause should always be firstprivate,
    8950              :          regardless of the current default.  */
    8951         4208 :       if (detach_clause && OMP_CLAUSE_DECL (detach_clause) == decl)
    8952              :         kind = OMP_CLAUSE_DEFAULT_FIRSTPRIVATE;
    8953              :     }
    8954        48106 :   if (kind != OMP_CLAUSE_DEFAULT_UNSPECIFIED)
    8955              :     default_kind = kind;
    8956        45302 :   else if (VAR_P (decl) && TREE_STATIC (decl) && DECL_IN_CONSTANT_POOL (decl))
    8957              :     default_kind = OMP_CLAUSE_DEFAULT_SHARED;
    8958              :   /* For C/C++ default({,first}private), variables with static storage duration
    8959              :      declared in a namespace or global scope and referenced in construct
    8960              :      must be explicitly specified, i.e. acts as default(none).  */
    8961        45301 :   else if ((default_kind == OMP_CLAUSE_DEFAULT_PRIVATE
    8962        45301 :             || default_kind == OMP_CLAUSE_DEFAULT_FIRSTPRIVATE)
    8963          379 :            && VAR_P (decl)
    8964          336 :            && is_global_var (decl)
    8965          184 :            && (DECL_FILE_SCOPE_P (decl)
    8966          104 :                || (DECL_CONTEXT (decl)
    8967          104 :                    && TREE_CODE (DECL_CONTEXT (decl)) == NAMESPACE_DECL))
    8968        45429 :            && !lang_GNU_Fortran ())
    8969              :     default_kind = OMP_CLAUSE_DEFAULT_NONE;
    8970              : 
    8971        47977 :   switch (default_kind)
    8972              :     {
    8973          267 :     case OMP_CLAUSE_DEFAULT_NONE:
    8974          267 :       {
    8975          267 :         const char *rtype;
    8976              : 
    8977          267 :         if (ctx->region_type & ORT_PARALLEL)
    8978              :           rtype = "parallel";
    8979          108 :         else if ((ctx->region_type & ORT_TASKLOOP) == ORT_TASKLOOP)
    8980              :           rtype = "taskloop";
    8981           72 :         else if (ctx->region_type & ORT_TASK)
    8982              :           rtype = "task";
    8983           36 :         else if (ctx->region_type & ORT_TEAMS)
    8984              :           rtype = "teams";
    8985              :         else
    8986            0 :           gcc_unreachable ();
    8987              : 
    8988          267 :         error ("%qE not specified in enclosing %qs",
    8989          267 :                DECL_NAME (lang_hooks.decls.omp_report_decl (decl)), rtype);
    8990          267 :         inform (ctx->location, "enclosing %qs", rtype);
    8991              :       }
    8992              :       /* FALLTHRU */
    8993        44439 :     case OMP_CLAUSE_DEFAULT_SHARED:
    8994        44439 :       flags |= GOVD_SHARED;
    8995        44439 :       break;
    8996          155 :     case OMP_CLAUSE_DEFAULT_PRIVATE:
    8997          155 :       flags |= GOVD_PRIVATE;
    8998          155 :       break;
    8999          288 :     case OMP_CLAUSE_DEFAULT_FIRSTPRIVATE:
    9000          288 :       flags |= GOVD_FIRSTPRIVATE;
    9001          288 :       break;
    9002         3224 :     case OMP_CLAUSE_DEFAULT_UNSPECIFIED:
    9003              :       /* decl will be either GOVD_FIRSTPRIVATE or GOVD_SHARED.  */
    9004         3224 :       gcc_assert ((ctx->region_type & ORT_TASK) != 0);
    9005         3224 :       if (struct gimplify_omp_ctx *octx = ctx->outer_context)
    9006              :         {
    9007         2166 :           omp_notice_variable (octx, decl, in_code);
    9008         5164 :           for (; octx; octx = octx->outer_context)
    9009              :             {
    9010         2942 :               splay_tree_node n2;
    9011              : 
    9012         2942 :               n2 = splay_tree_lookup (octx->variables, (splay_tree_key) decl);
    9013         2942 :               if ((octx->region_type & (ORT_TARGET_DATA | ORT_TARGET)) != 0
    9014            5 :                   && (n2 == NULL || (n2->value & GOVD_DATA_SHARE_CLASS) == 0))
    9015            5 :                 continue;
    9016         2937 :               if (n2 && (n2->value & GOVD_DATA_SHARE_CLASS) != GOVD_SHARED)
    9017              :                 {
    9018          898 :                   flags |= GOVD_FIRSTPRIVATE;
    9019          898 :                   goto found_outer;
    9020              :                 }
    9021         2039 :               if ((octx->region_type & (ORT_PARALLEL | ORT_TEAMS)) != 0)
    9022              :                 {
    9023         1212 :                   flags |= GOVD_SHARED;
    9024         1212 :                   goto found_outer;
    9025              :                 }
    9026              :             }
    9027              :         }
    9028              : 
    9029         1114 :       if (TREE_CODE (decl) == PARM_DECL
    9030         1114 :           || (!is_global_var (decl)
    9031          294 :               && DECL_CONTEXT (decl) == current_function_decl))
    9032          734 :         flags |= GOVD_FIRSTPRIVATE;
    9033              :       else
    9034          380 :         flags |= GOVD_SHARED;
    9035        48106 :     found_outer:
    9036              :       break;
    9037              : 
    9038            0 :     default:
    9039            0 :       gcc_unreachable ();
    9040              :     }
    9041              : 
    9042        48106 :   return flags;
    9043              : }
    9044              : 
    9045              : /* Return string name for types of OpenACC constructs from ORT_* values.  */
    9046              : 
    9047              : static const char *
    9048          570 : oacc_region_type_name (enum omp_region_type region_type)
    9049              : {
    9050          570 :   switch (region_type)
    9051              :     {
    9052              :     case ORT_ACC_DATA:
    9053              :       return "data";
    9054          140 :     case ORT_ACC_PARALLEL:
    9055          140 :       return "parallel";
    9056          140 :     case ORT_ACC_KERNELS:
    9057          140 :       return "kernels";
    9058          140 :     case ORT_ACC_SERIAL:
    9059          140 :       return "serial";
    9060            0 :     default:
    9061            0 :       gcc_unreachable ();
    9062              :     }
    9063              : }
    9064              : 
    9065              : /* Determine outer default flags for DECL mentioned in an OACC region
    9066              :    but not declared in an enclosing clause.  */
    9067              : 
    9068              : static unsigned
    9069         8839 : oacc_default_clause (struct gimplify_omp_ctx *ctx, tree decl, unsigned flags)
    9070              : {
    9071         8839 :   struct gimplify_omp_ctx *ctx_default = ctx;
    9072              :   /* If no 'default' clause appears on this compute construct...  */
    9073         8839 :   if (ctx_default->default_kind == OMP_CLAUSE_DEFAULT_SHARED)
    9074              :     {
    9075              :       /* ..., see if one appears on a lexically containing 'data'
    9076              :          construct.  */
    9077         9618 :       while ((ctx_default = ctx_default->outer_context))
    9078              :         {
    9079         1534 :           if (ctx_default->region_type == ORT_ACC_DATA
    9080         1534 :               && ctx_default->default_kind != OMP_CLAUSE_DEFAULT_SHARED)
    9081              :             break;
    9082              :         }
    9083              :       /* If not, reset.  */
    9084         8260 :       if (!ctx_default)
    9085         8663 :         ctx_default = ctx;
    9086              :     }
    9087              : 
    9088         8839 :   bool on_device = false;
    9089         8839 :   bool is_private = false;
    9090         8839 :   bool declared = is_oacc_declared (decl);
    9091         8839 :   tree type = TREE_TYPE (decl);
    9092              : 
    9093         8839 :   if (omp_privatize_by_reference (decl))
    9094          338 :     type = TREE_TYPE (type);
    9095              : 
    9096              :   /* For Fortran COMMON blocks, only used variables in those blocks are
    9097              :      transferred and remapped.  The block itself will have a private clause to
    9098              :      avoid transferring the data twice.
    9099              :      The hook evaluates to false by default.  For a variable in Fortran's COMMON
    9100              :      or EQUIVALENCE block, returns 'true' (as we have shared=false) - as only
    9101              :      the variables in such a COMMON/EQUIVALENCE block shall be privatized not
    9102              :      the whole block.  For C++ and Fortran, it can also be true under certain
    9103              :      other conditions, if DECL_HAS_VALUE_EXPR.  */
    9104         8839 :   if (RECORD_OR_UNION_TYPE_P (type))
    9105          983 :     is_private = lang_hooks.decls.omp_disregard_value_expr (decl, false);
    9106              : 
    9107         8839 :   if ((ctx->region_type & (ORT_ACC_PARALLEL | ORT_ACC_KERNELS)) != 0
    9108         8839 :       && is_global_var (decl)
    9109          642 :       && device_resident_p (decl)
    9110         8839 :       && !is_private)
    9111              :     {
    9112            0 :       on_device = true;
    9113            0 :       flags |= GOVD_MAP_TO_ONLY;
    9114              :     }
    9115              : 
    9116         8839 :   switch (ctx->region_type)
    9117              :     {
    9118         1460 :     case ORT_ACC_KERNELS:
    9119         1460 :       if (is_private)
    9120            0 :         flags |= GOVD_FIRSTPRIVATE;
    9121         1460 :       else if (AGGREGATE_TYPE_P (type))
    9122              :         {
    9123              :           /* Aggregates default to 'present_or_copy', or 'present'.  */
    9124          415 :           if (ctx_default->default_kind != OMP_CLAUSE_DEFAULT_PRESENT)
    9125          397 :             flags |= GOVD_MAP;
    9126              :           else
    9127           18 :             flags |= GOVD_MAP | GOVD_MAP_FORCE_PRESENT;
    9128              :         }
    9129              :       else
    9130              :         /* Scalars default to 'copy'.  */
    9131         1045 :         flags |= GOVD_MAP | GOVD_MAP_FORCE;
    9132              : 
    9133              :       break;
    9134              : 
    9135         7379 :     case ORT_ACC_PARALLEL:
    9136         7379 :     case ORT_ACC_SERIAL:
    9137         7379 :       if (is_private)
    9138            0 :         flags |= GOVD_FIRSTPRIVATE;
    9139         7379 :       else if (on_device || declared)
    9140           17 :         flags |= GOVD_MAP;
    9141         7362 :       else if (AGGREGATE_TYPE_P (type))
    9142              :         {
    9143              :           /* Aggregates default to 'present_or_copy', or 'present'.  */
    9144         3697 :           if (ctx_default->default_kind != OMP_CLAUSE_DEFAULT_PRESENT)
    9145         3431 :             flags |= GOVD_MAP;
    9146              :           else
    9147          266 :             flags |= GOVD_MAP | GOVD_MAP_FORCE_PRESENT;
    9148              :         }
    9149              :       else
    9150              :         /* Scalars default to 'firstprivate'.  */
    9151         3665 :         flags |= GOVD_FIRSTPRIVATE;
    9152              : 
    9153              :       break;
    9154              : 
    9155            0 :     default:
    9156            0 :       gcc_unreachable ();
    9157              :     }
    9158              : 
    9159         8839 :   if (DECL_ARTIFICIAL (decl))
    9160              :     ; /* We can get compiler-generated decls, and should not complain
    9161              :          about them.  */
    9162         8528 :   else if (ctx_default->default_kind == OMP_CLAUSE_DEFAULT_NONE)
    9163              :     {
    9164          420 :       error ("%qE not specified in enclosing OpenACC %qs construct",
    9165          210 :              DECL_NAME (lang_hooks.decls.omp_report_decl (decl)),
    9166              :              oacc_region_type_name (ctx->region_type));
    9167          210 :       if (ctx_default != ctx)
    9168          150 :         inform (ctx->location, "enclosing OpenACC %qs construct and",
    9169              :                 oacc_region_type_name (ctx->region_type));
    9170          210 :       inform (ctx_default->location,
    9171              :               "enclosing OpenACC %qs construct with %qs clause",
    9172              :               oacc_region_type_name (ctx_default->region_type),
    9173              :               "default(none)");
    9174              :     }
    9175         8318 :   else if (ctx_default->default_kind == OMP_CLAUSE_DEFAULT_PRESENT)
    9176              :     ; /* Handled above.  */
    9177              :   else
    9178         7785 :     gcc_checking_assert (ctx_default->default_kind == OMP_CLAUSE_DEFAULT_SHARED);
    9179              : 
    9180         8839 :   return flags;
    9181              : }
    9182              : 
    9183              : /* Record the fact that DECL was used within the OMP context CTX.
    9184              :    IN_CODE is true when real code uses DECL, and false when we should
    9185              :    merely emit default(none) errors.  Return true if DECL is going to
    9186              :    be remapped and thus DECL shouldn't be gimplified into its
    9187              :    DECL_VALUE_EXPR (if any).  */
    9188              : 
    9189              : static bool
    9190      3924829 : omp_notice_variable (struct gimplify_omp_ctx *ctx, tree decl, bool in_code)
    9191              : {
    9192      3924829 :   splay_tree_node n;
    9193      3924829 :   unsigned flags = in_code ? GOVD_SEEN : 0;
    9194      3924829 :   bool ret = false, shared;
    9195              : 
    9196      3924829 :   if (error_operand_p (decl))
    9197              :     return false;
    9198              : 
    9199      3924829 :   if (DECL_ARTIFICIAL (decl))
    9200              :     {
    9201      2240950 :       tree attr = lookup_attribute ("omp allocate var", DECL_ATTRIBUTES (decl));
    9202      2240950 :       if (attr)
    9203         1053 :         decl = TREE_VALUE (TREE_VALUE (attr));
    9204              :     }
    9205              : 
    9206      3924829 :   if (ctx->region_type == ORT_NONE)
    9207          248 :     return lang_hooks.decls.omp_disregard_value_expr (decl, false);
    9208              : 
    9209      3924581 :   if (is_global_var (decl))
    9210              :     {
    9211              :       /* Threadprivate variables are predetermined.  */
    9212       503005 :       if (DECL_THREAD_LOCAL_P (decl))
    9213        13565 :         return omp_notice_threadprivate_variable (ctx, decl, NULL_TREE);
    9214              : 
    9215       489440 :       if (DECL_HAS_VALUE_EXPR_P (decl))
    9216              :         {
    9217         4380 :           if (ctx->region_type & ORT_ACC)
    9218              :             /* For OpenACC, defer expansion of value to avoid transferring
    9219              :                privatized common block data instead of im-/explicitly
    9220              :                transferred variables which are in common blocks.  */
    9221              :             ;
    9222              :           else
    9223              :             {
    9224         2104 :               tree value = get_base_address (DECL_VALUE_EXPR (decl));
    9225              : 
    9226         2104 :               if (value && DECL_P (value) && DECL_THREAD_LOCAL_P (value))
    9227          618 :                 return omp_notice_threadprivate_variable (ctx, decl, value);
    9228              :             }
    9229              :         }
    9230              : 
    9231       488822 :       if (gimplify_omp_ctxp->outer_context == NULL
    9232       114646 :           && VAR_P (decl)
    9233       603468 :           && oacc_get_fn_attrib (current_function_decl))
    9234              :         {
    9235          765 :           location_t loc = DECL_SOURCE_LOCATION (decl);
    9236              : 
    9237          765 :           if (lookup_attribute ("omp declare target link",
    9238          765 :                                 DECL_ATTRIBUTES (decl)))
    9239              :             {
    9240           48 :               error_at (loc,
    9241              :                         "%qE with %<link%> clause used in %<routine%> function",
    9242           24 :                         DECL_NAME (decl));
    9243           24 :               return false;
    9244              :             }
    9245          741 :           else if (!lookup_attribute ("omp declare target",
    9246          741 :                                       DECL_ATTRIBUTES (decl)))
    9247              :             {
    9248          132 :               error_at (loc,
    9249              :                         "%qE requires a %<declare%> directive for use "
    9250           66 :                         "in a %<routine%> function", DECL_NAME (decl));
    9251           66 :               return false;
    9252              :             }
    9253              :         }
    9254              :     }
    9255              : 
    9256      3910308 :   n = splay_tree_lookup (ctx->variables, (splay_tree_key)decl);
    9257      3910308 :   if ((ctx->region_type & ORT_TARGET) != 0)
    9258              :     {
    9259       853560 :       if (n == NULL)
    9260              :         {
    9261        45088 :           unsigned nflags = flags;
    9262        45088 :           if ((ctx->region_type & ORT_ACC) == 0)
    9263              :             {
    9264        34134 :               bool is_declare_target = false;
    9265        34134 :               if (is_global_var (decl)
    9266        34134 :                   && varpool_node::get_create (decl)->offloadable)
    9267              :                 {
    9268         7054 :                   struct gimplify_omp_ctx *octx;
    9269         7054 :                   for (octx = ctx->outer_context;
    9270         7062 :                        octx; octx = octx->outer_context)
    9271              :                     {
    9272            9 :                       n = splay_tree_lookup (octx->variables,
    9273              :                                              (splay_tree_key)decl);
    9274            9 :                       if (n
    9275            9 :                           && (n->value & GOVD_DATA_SHARE_CLASS) != GOVD_SHARED
    9276            9 :                           && (n->value & GOVD_DATA_SHARE_CLASS) != 0)
    9277              :                         break;
    9278              :                     }
    9279         7054 :                   is_declare_target = octx == NULL;
    9280              :                 }
    9281         7054 :               if (!is_declare_target)
    9282              :                 {
    9283        27081 :                   int gdmk;
    9284        27081 :                   enum omp_clause_defaultmap_kind kind;
    9285        27081 :                   if (lang_hooks.decls.omp_allocatable_p (decl))
    9286              :                     gdmk = GDMK_ALLOCATABLE;
    9287        26732 :                   else if (lang_hooks.decls.omp_scalar_target_p (decl))
    9288              :                     gdmk = GDMK_SCALAR_TARGET;
    9289        26657 :                   else if (lang_hooks.decls.omp_scalar_p (decl, false))
    9290              :                     gdmk = GDMK_SCALAR;
    9291        11219 :                   else if (TREE_CODE (TREE_TYPE (decl)) == POINTER_TYPE
    9292        11219 :                            || (TREE_CODE (TREE_TYPE (decl)) == REFERENCE_TYPE
    9293         3108 :                                && (TREE_CODE (TREE_TYPE (TREE_TYPE (decl)))
    9294              :                                    == POINTER_TYPE)))
    9295              :                     gdmk = GDMK_POINTER;
    9296              :                   else
    9297              :                     gdmk = GDMK_AGGREGATE;
    9298        27081 :                   kind = lang_hooks.decls.omp_predetermined_mapping (decl);
    9299        27081 :                   if (kind != OMP_CLAUSE_DEFAULTMAP_CATEGORY_UNSPECIFIED)
    9300              :                     {
    9301         1014 :                       if (kind == OMP_CLAUSE_DEFAULTMAP_FIRSTPRIVATE)
    9302          101 :                         nflags |= GOVD_FIRSTPRIVATE;
    9303          913 :                       else if (kind == OMP_CLAUSE_DEFAULTMAP_TO)
    9304          913 :                         nflags |= GOVD_MAP | GOVD_MAP_TO_ONLY;
    9305              :                       else
    9306            0 :                         gcc_unreachable ();
    9307              :                     }
    9308        26067 :                   else if (ctx->defaultmap[gdmk] == 0)
    9309              :                     {
    9310           75 :                       tree d = lang_hooks.decls.omp_report_decl (decl);
    9311           75 :                       error ("%qE not specified in enclosing %<target%>",
    9312           75 :                              DECL_NAME (d));
    9313           75 :                       inform (ctx->location, "enclosing %<target%>");
    9314              :                     }
    9315        25992 :                   else if (ctx->defaultmap[gdmk]
    9316        25992 :                            & (GOVD_MAP_0LEN_ARRAY | GOVD_FIRSTPRIVATE))
    9317         9164 :                     nflags |= ctx->defaultmap[gdmk];
    9318        16828 :                   else if (ctx->defaultmap[gdmk] & GOVD_MAP_FORCE_PRESENT)
    9319              :                     {
    9320           42 :                       gcc_assert (ctx->defaultmap[gdmk] & GOVD_MAP);
    9321           42 :                       nflags |= ctx->defaultmap[gdmk] | GOVD_MAP_ALLOC_ONLY;
    9322              :                     }
    9323              :                   else
    9324              :                     {
    9325        16786 :                       gcc_assert (ctx->defaultmap[gdmk] & GOVD_MAP);
    9326        16786 :                       nflags |= ctx->defaultmap[gdmk] & ~GOVD_MAP;
    9327              :                     }
    9328              :                 }
    9329              :             }
    9330              : 
    9331        45088 :           struct gimplify_omp_ctx *octx = ctx->outer_context;
    9332        45088 :           if ((ctx->region_type & ORT_ACC) && octx)
    9333              :             {
    9334              :               /* Look in outer OpenACC contexts, to see if there's a
    9335              :                  data attribute for this variable.  */
    9336         3535 :               omp_notice_variable (octx, decl, in_code);
    9337              : 
    9338         8842 :               for (; octx; octx = octx->outer_context)
    9339              :                 {
    9340         3883 :                   if (!(octx->region_type & (ORT_TARGET_DATA | ORT_TARGET)))
    9341              :                     break;
    9342         3883 :                   splay_tree_node n2
    9343         3883 :                     = splay_tree_lookup (octx->variables,
    9344              :                                          (splay_tree_key) decl);
    9345         3883 :                   if (n2)
    9346              :                     {
    9347         2111 :                       if (octx->region_type == ORT_ACC_HOST_DATA)
    9348            4 :                         error ("variable %qE declared in enclosing "
    9349            4 :                                "%<host_data%> region", DECL_NAME (decl));
    9350         2111 :                       nflags |= GOVD_MAP;
    9351         2111 :                       if (octx->region_type == ORT_ACC_DATA
    9352         2107 :                           && (n2->value & GOVD_MAP_0LEN_ARRAY))
    9353          288 :                         nflags |= GOVD_MAP_0LEN_ARRAY;
    9354         2111 :                       goto found_outer;
    9355              :                     }
    9356              :                 }
    9357              :             }
    9358              : 
    9359        42977 :           if ((nflags & ~(GOVD_MAP_TO_ONLY | GOVD_MAP_FROM_ONLY
    9360              :                           | GOVD_MAP_ALLOC_ONLY)) == flags)
    9361              :             {
    9362        32757 :               tree type = TREE_TYPE (decl);
    9363        32757 :               location_t loc = DECL_SOURCE_LOCATION (decl);
    9364              : 
    9365        32757 :               if (gimplify_omp_ctxp->target_firstprivatize_array_bases
    9366        32757 :                   && omp_privatize_by_reference (decl))
    9367           28 :                 type = TREE_TYPE (type);
    9368              : 
    9369        32757 :               if (!verify_type_context (loc, TCTX_OMP_MAP_IMP_REF, type))
    9370              :                 /* Check if TYPE can appear in a target region.
    9371              :                    verify_type_context has already issued an error if it
    9372              :                    can't.  */
    9373            0 :                 nflags |= GOVD_MAP | GOVD_EXPLICIT;
    9374        32757 :               else if (!omp_mappable_type (type))
    9375              :                 {
    9376            8 :                   error ("%qD referenced in target region does not have "
    9377              :                          "a mappable type", decl);
    9378            8 :                   nflags |= GOVD_MAP | GOVD_EXPLICIT;
    9379              :                 }
    9380              :               else
    9381              :                 {
    9382        32749 :                   if ((ctx->region_type & ORT_ACC) != 0)
    9383         8839 :                     nflags = oacc_default_clause (ctx, decl, flags);
    9384              :                   else
    9385        23910 :                     nflags |= GOVD_MAP;
    9386              :                 }
    9387              :             }
    9388        10220 :         found_outer:
    9389        45088 :           omp_add_variable (ctx, decl, nflags);
    9390        45088 :           if (ctx->region_type & ORT_ACC)
    9391              :             /* For OpenACC, as remarked above, defer expansion.  */
    9392              :             shared = false;
    9393              :           else
    9394        34134 :             shared = (nflags & (GOVD_PRIVATE | GOVD_FIRSTPRIVATE)) == 0;
    9395        45088 :           ret = lang_hooks.decls.omp_disregard_value_expr (decl, shared);
    9396              :         }
    9397              :       else
    9398              :         {
    9399       808472 :           if (ctx->region_type & ORT_ACC)
    9400              :             /* For OpenACC, as remarked above, defer expansion.  */
    9401              :             shared = false;
    9402              :           else
    9403       512431 :             shared = ((n->value | flags)
    9404       512431 :                       & (GOVD_PRIVATE | GOVD_FIRSTPRIVATE)) == 0;
    9405       808472 :           ret = lang_hooks.decls.omp_disregard_value_expr (decl, shared);
    9406              :           /* If nothing changed, there's nothing left to do.  */
    9407       808472 :           if ((n->value & flags) == flags)
    9408              :             return ret;
    9409        16801 :           flags |= n->value;
    9410        16801 :           n->value = flags;
    9411              :         }
    9412        61889 :       goto do_outer;
    9413              :     }
    9414              : 
    9415      3056748 :   if (n == NULL)
    9416              :     {
    9417      1184620 :       if (ctx->region_type == ORT_WORKSHARE
    9418              :           || ctx->region_type == ORT_TASKGROUP
    9419       562490 :           || ctx->region_type == ORT_SIMD
    9420       288690 :           || ctx->region_type == ORT_ACC
    9421        92821 :           || (ctx->region_type & ORT_TARGET_DATA) != 0)
    9422      1136514 :         goto do_outer;
    9423              : 
    9424        48106 :       flags = omp_default_clause (ctx, decl, in_code, flags);
    9425              : 
    9426        48106 :       if ((flags & GOVD_PRIVATE)
    9427        48106 :           && lang_hooks.decls.omp_private_outer_ref (decl))
    9428            6 :         flags |= GOVD_PRIVATE_OUTER_REF;
    9429              : 
    9430        48106 :       omp_add_variable (ctx, decl, flags);
    9431              : 
    9432        48106 :       shared = (flags & GOVD_SHARED) != 0;
    9433        48106 :       ret = lang_hooks.decls.omp_disregard_value_expr (decl, shared);
    9434        48106 :       goto do_outer;
    9435              :     }
    9436              : 
    9437              :   /* Don't mark as GOVD_SEEN addressable temporaries seen only in simd
    9438              :      lb, b or incr expressions, those shouldn't be turned into simd arrays.  */
    9439      1872128 :   if (ctx->region_type == ORT_SIMD
    9440       151619 :       && ctx->in_for_exprs
    9441           70 :       && ((n->value & (GOVD_PRIVATE | GOVD_SEEN | GOVD_EXPLICIT))
    9442              :           == GOVD_PRIVATE))
    9443      1872128 :     flags &= ~GOVD_SEEN;
    9444              : 
    9445      1872128 :   if ((n->value & (GOVD_SEEN | GOVD_LOCAL)) == 0
    9446        32052 :       && (flags & (GOVD_SEEN | GOVD_LOCAL)) == GOVD_SEEN
    9447      1904141 :       && DECL_SIZE (decl))
    9448              :     {
    9449        32012 :       tree size;
    9450        32012 :       if (!poly_int_tree_p (DECL_SIZE (decl)))
    9451              :         {
    9452          150 :           splay_tree_node n2;
    9453          150 :           tree t = DECL_VALUE_EXPR (decl);
    9454          150 :           gcc_assert (INDIRECT_REF_P (t));
    9455          150 :           t = TREE_OPERAND (t, 0);
    9456          150 :           gcc_assert (DECL_P (t));
    9457          150 :           n2 = splay_tree_lookup (ctx->variables, (splay_tree_key) t);
    9458          150 :           n2->value |= GOVD_SEEN;
    9459              :         }
    9460        31862 :       else if (omp_privatize_by_reference (decl)
    9461         4193 :                && (size = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (decl))))
    9462        36018 :                && !poly_int_tree_p (size))
    9463              :         {
    9464         1337 :           splay_tree_node n2;
    9465         1337 :           gcc_assert (DECL_P (size));
    9466         1337 :           n2 = splay_tree_lookup (ctx->variables, (splay_tree_key) size);
    9467         1337 :           if (n2)
    9468          629 :             omp_notice_variable (ctx, size, true);
    9469              :         }
    9470              :     }
    9471              : 
    9472      1872128 :   if (ctx->region_type & ORT_ACC)
    9473              :     /* For OpenACC, as remarked above, defer expansion.  */
    9474              :     shared = false;
    9475              :   else
    9476      1689617 :     shared = ((flags | n->value) & GOVD_SHARED) != 0;
    9477      1872128 :   ret = lang_hooks.decls.omp_disregard_value_expr (decl, shared);
    9478              : 
    9479              :   /* If nothing changed, there's nothing left to do.  */
    9480      1872128 :   if ((n->value & flags) == flags)
    9481              :     return ret;
    9482        32013 :   flags |= n->value;
    9483        32013 :   n->value = flags;
    9484              : 
    9485      1278522 :  do_outer:
    9486              :   /* If the variable is private in the current context, then we don't
    9487              :      need to propagate anything to an outer context.  */
    9488      1278522 :   if ((flags & GOVD_PRIVATE) && !(flags & GOVD_PRIVATE_OUTER_REF))
    9489              :     return ret;
    9490      1268062 :   if ((flags & (GOVD_LINEAR | GOVD_LINEAR_LASTPRIVATE_NO_OUTER))
    9491              :       == (GOVD_LINEAR | GOVD_LINEAR_LASTPRIVATE_NO_OUTER))
    9492              :     return ret;
    9493      1268002 :   if ((flags & (GOVD_FIRSTPRIVATE | GOVD_LASTPRIVATE
    9494              :                 | GOVD_LINEAR_LASTPRIVATE_NO_OUTER))
    9495              :       == (GOVD_LASTPRIVATE | GOVD_LINEAR_LASTPRIVATE_NO_OUTER))
    9496              :     return ret;
    9497      1268002 :   if (ctx->outer_context
    9498      1268002 :       && omp_notice_variable (ctx->outer_context, decl, in_code))
    9499         2151 :     return true;
    9500              :   return ret;
    9501              : }
    9502              : 
    9503              : /* Verify that DECL is private within CTX.  If there's specific information
    9504              :    to the contrary in the innermost scope, generate an error.  */
    9505              : 
    9506              : static bool
    9507        50393 : omp_is_private (struct gimplify_omp_ctx *ctx, tree decl, int simd)
    9508              : {
    9509        96209 :   splay_tree_node n;
    9510              : 
    9511        96209 :   n = splay_tree_lookup (ctx->variables, (splay_tree_key)decl);
    9512        96209 :   if (n != NULL)
    9513              :     {
    9514        18013 :       if (n->value & GOVD_SHARED)
    9515              :         {
    9516          407 :           if (ctx == gimplify_omp_ctxp)
    9517              :             {
    9518            0 :               if (simd)
    9519            0 :                 error ("iteration variable %qE is predetermined linear",
    9520            0 :                        DECL_NAME (decl));
    9521              :               else
    9522            0 :                 error ("iteration variable %qE should be private",
    9523            0 :                        DECL_NAME (decl));
    9524            0 :               n->value = GOVD_PRIVATE;
    9525            0 :               return true;
    9526              :             }
    9527              :           else
    9528              :             return false;
    9529              :         }
    9530        17606 :       else if ((n->value & GOVD_EXPLICIT) != 0
    9531         9745 :                && (ctx == gimplify_omp_ctxp
    9532          340 :                    || (ctx->region_type == ORT_COMBINED_PARALLEL
    9533          220 :                        && gimplify_omp_ctxp->outer_context == ctx)))
    9534              :         {
    9535         9625 :           if ((n->value & GOVD_FIRSTPRIVATE) != 0)
    9536            4 :             error ("iteration variable %qE should not be firstprivate",
    9537            4 :                    DECL_NAME (decl));
    9538         9621 :           else if ((n->value & GOVD_REDUCTION) != 0)
    9539            8 :             error ("iteration variable %qE should not be reduction",
    9540            8 :                    DECL_NAME (decl));
    9541         9613 :           else if (simd != 1 && (n->value & GOVD_LINEAR) != 0)
    9542           58 :             error ("iteration variable %qE should not be linear",
    9543           58 :                    DECL_NAME (decl));
    9544              :         }
    9545        17606 :       return (ctx == gimplify_omp_ctxp
    9546        17606 :               || (ctx->region_type == ORT_COMBINED_PARALLEL
    9547         2619 :                   && gimplify_omp_ctxp->outer_context == ctx));
    9548              :     }
    9549              : 
    9550        78196 :   if (ctx->region_type != ORT_WORKSHARE
    9551              :       && ctx->region_type != ORT_TASKGROUP
    9552        49599 :       && ctx->region_type != ORT_SIMD
    9553        36244 :       && ctx->region_type != ORT_ACC)
    9554              :     return false;
    9555        54125 :   else if (ctx->outer_context)
    9556              :     return omp_is_private (ctx->outer_context, decl, simd);
    9557              :   return false;
    9558              : }
    9559              : 
    9560              : /* Return true if DECL is private within a parallel region
    9561              :    that binds to the current construct's context or in parallel
    9562              :    region's REDUCTION clause.  */
    9563              : 
    9564              : static bool
    9565        11625 : omp_check_private (struct gimplify_omp_ctx *ctx, tree decl, bool copyprivate)
    9566              : {
    9567        11697 :   splay_tree_node n;
    9568              : 
    9569        11697 :   do
    9570              :     {
    9571        11697 :       ctx = ctx->outer_context;
    9572        11697 :       if (ctx == NULL)
    9573              :         {
    9574         2786 :           if (is_global_var (decl))
    9575              :             return false;
    9576              : 
    9577              :           /* References might be private, but might be shared too,
    9578              :              when checking for copyprivate, assume they might be
    9579              :              private, otherwise assume they might be shared.  */
    9580         1292 :           if (copyprivate)
    9581              :             return true;
    9582              : 
    9583         1246 :           if (omp_privatize_by_reference (decl))
    9584              :             return false;
    9585              : 
    9586              :           /* Treat C++ privatized non-static data members outside
    9587              :              of the privatization the same.  */
    9588         1200 :           if (omp_member_access_dummy_var (decl))
    9589              :             return false;
    9590              : 
    9591         1144 :           return true;
    9592              :         }
    9593              : 
    9594         8911 :       n = splay_tree_lookup (ctx->variables, (splay_tree_key) decl);
    9595              : 
    9596         8911 :       if ((ctx->region_type & (ORT_TARGET | ORT_TARGET_DATA)) != 0
    9597         1291 :           && (n == NULL || (n->value & GOVD_DATA_SHARE_CLASS) == 0))
    9598              :         {
    9599          242 :           if ((ctx->region_type & ORT_TARGET_DATA) != 0
    9600          238 :               || n == NULL
    9601          230 :               || (n->value & GOVD_MAP) == 0)
    9602           12 :             continue;
    9603              :           return false;
    9604              :         }
    9605              : 
    9606         7620 :       if (n != NULL)
    9607              :         {
    9608         5402 :           if ((n->value & GOVD_LOCAL) != 0
    9609         5402 :               && omp_member_access_dummy_var (decl))
    9610              :             return false;
    9611         5363 :           return (n->value & GOVD_SHARED) == 0;
    9612              :         }
    9613              : 
    9614         3267 :       if (ctx->region_type == ORT_WORKSHARE
    9615              :           || ctx->region_type == ORT_TASKGROUP
    9616         3211 :           || ctx->region_type == ORT_SIMD
    9617         3207 :           || ctx->region_type == ORT_ACC)
    9618           60 :         continue;
    9619              : 
    9620              :       break;
    9621              :     }
    9622              :   while (1);
    9623              :   return false;
    9624              : }
    9625              : 
    9626              : /* Callback for walk_tree to find a DECL_EXPR for the given DECL.  */
    9627              : 
    9628              : static tree
    9629         3148 : find_decl_expr (tree *tp, int *walk_subtrees, void *data)
    9630              : {
    9631         3148 :   tree t = *tp;
    9632              : 
    9633              :   /* If this node has been visited, unmark it and keep looking.  */
    9634         3148 :   if (TREE_CODE (t) == DECL_EXPR && DECL_EXPR_DECL (t) == (tree) data)
    9635              :     return t;
    9636              : 
    9637         2724 :   if (IS_TYPE_OR_DECL_P (t))
    9638          452 :     *walk_subtrees = 0;
    9639              :   return NULL_TREE;
    9640              : }
    9641              : 
    9642              : 
    9643              : /* Gimplify the affinity clause but effectively ignore it.
    9644              :    Generate:
    9645              :      var = begin;
    9646              :      if ((step > 1) ? var <= end : var > end)
    9647              :        locatator_var_expr;  */
    9648              : 
    9649              : static void
    9650          510 : gimplify_omp_affinity (tree *list_p, gimple_seq *pre_p)
    9651              : {
    9652          510 :   tree last_iter = NULL_TREE;
    9653          510 :   tree last_bind = NULL_TREE;
    9654          510 :   tree label = NULL_TREE;
    9655          510 :   tree *last_body = NULL;
    9656         1287 :   for (tree c = *list_p; c; c = OMP_CLAUSE_CHAIN (c))
    9657          777 :     if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_AFFINITY)
    9658              :       {
    9659          777 :         tree t = OMP_CLAUSE_DECL (c);
    9660          777 :         if (OMP_ITERATOR_DECL_P (t))
    9661              :           {
    9662          389 :             if (TREE_VALUE (t) == null_pointer_node)
    9663          201 :               continue;
    9664          188 :             if (TREE_PURPOSE (t) != last_iter)
    9665              :               {
    9666          127 :                 if (last_bind)
    9667              :                   {
    9668            9 :                     append_to_statement_list (label, last_body);
    9669            9 :                     gimplify_and_add (last_bind, pre_p);
    9670            9 :                     last_bind = NULL_TREE;
    9671              :                   }
    9672          274 :                 for (tree it = TREE_PURPOSE (t); it; it = TREE_CHAIN (it))
    9673              :                   {
    9674          147 :                     if (gimplify_expr (&OMP_ITERATOR_BEGIN (it), pre_p, NULL,
    9675              :                                        is_gimple_val, fb_rvalue) == GS_ERROR
    9676          147 :                         || gimplify_expr (&OMP_ITERATOR_END (it), pre_p, NULL,
    9677              :                                           is_gimple_val, fb_rvalue) == GS_ERROR
    9678          147 :                         || gimplify_expr (&OMP_ITERATOR_STEP (it), pre_p, NULL,
    9679              :                                           is_gimple_val, fb_rvalue) == GS_ERROR
    9680          294 :                         || (gimplify_expr (&OMP_ITERATOR_ORIG_STEP (it), pre_p,
    9681              :                                            NULL, is_gimple_val, fb_rvalue)
    9682              :                             == GS_ERROR))
    9683            0 :                       return;
    9684              :                   }
    9685          127 :             last_iter = TREE_PURPOSE (t);
    9686          127 :             tree block = OMP_ITERATOR_BLOCK (TREE_PURPOSE (t));
    9687          127 :             last_bind = build3 (BIND_EXPR, void_type_node, BLOCK_VARS (block),
    9688              :                                 NULL, block);
    9689          127 :             last_body = &BIND_EXPR_BODY (last_bind);
    9690          127 :             tree cond = NULL_TREE;
    9691          127 :             location_t loc = OMP_CLAUSE_LOCATION (c);
    9692          274 :             for (tree it = TREE_PURPOSE (t); it; it = TREE_CHAIN (it))
    9693              :               {
    9694          147 :                 tree var = OMP_ITERATOR_VAR (it);
    9695          147 :                 tree begin = OMP_ITERATOR_BEGIN (it);
    9696          147 :                 tree end = OMP_ITERATOR_END (it);
    9697          147 :                 tree step = OMP_ITERATOR_STEP (it);
    9698          147 :                 loc = DECL_SOURCE_LOCATION (var);
    9699          147 :                 tree tem = build2_loc (loc, MODIFY_EXPR, void_type_node,
    9700              :                                        var, begin);
    9701          147 :                 append_to_statement_list_force (tem, last_body);
    9702              : 
    9703          147 :                 tree cond1 = fold_build2_loc (loc, GT_EXPR, boolean_type_node,
    9704          147 :                                step, build_zero_cst (TREE_TYPE (step)));
    9705          147 :                 tree cond2 = fold_build2_loc (loc, LE_EXPR, boolean_type_node,
    9706              :                                               var, end);
    9707          147 :                 tree cond3 = fold_build2_loc (loc, GT_EXPR, boolean_type_node,
    9708              :                                               var, end);
    9709          147 :                 cond1 = fold_build3_loc (loc, COND_EXPR, boolean_type_node,
    9710              :                                          cond1, cond2, cond3);
    9711          147 :                 if (cond)
    9712           20 :                   cond = fold_build2_loc (loc, TRUTH_AND_EXPR,
    9713              :                                           boolean_type_node, cond, cond1);
    9714              :                 else
    9715              :                   cond = cond1;
    9716              :               }
    9717          127 :             tree cont_label = create_artificial_label (loc);
    9718          127 :             label = build1 (LABEL_EXPR, void_type_node, cont_label);
    9719          127 :             tree tem = fold_build3_loc (loc, COND_EXPR, void_type_node, cond,
    9720              :                                         void_node,
    9721              :                                         build_and_jump (&cont_label));
    9722          127 :             append_to_statement_list_force (tem, last_body);
    9723              :               }
    9724          188 :             if (TREE_CODE (TREE_VALUE (t)) == COMPOUND_EXPR)
    9725              :               {
    9726            0 :                 append_to_statement_list (TREE_OPERAND (TREE_VALUE (t), 0),
    9727              :                                           last_body);
    9728            0 :                 TREE_VALUE (t) = TREE_OPERAND (TREE_VALUE (t), 1);
    9729              :               }
    9730          188 :             if (error_operand_p (TREE_VALUE (t)))
    9731              :               return;
    9732          188 :             append_to_statement_list_force (TREE_VALUE (t), last_body);
    9733          188 :             TREE_VALUE (t) = null_pointer_node;
    9734              :           }
    9735              :         else
    9736              :           {
    9737          388 :             if (last_bind)
    9738              :               {
    9739            9 :                 append_to_statement_list (label, last_body);
    9740            9 :                 gimplify_and_add (last_bind, pre_p);
    9741            9 :                 last_bind = NULL_TREE;
    9742              :               }
    9743          388 :             if (TREE_CODE (OMP_CLAUSE_DECL (c)) == COMPOUND_EXPR)
    9744              :               {
    9745            0 :                 gimplify_expr (&TREE_OPERAND (OMP_CLAUSE_DECL (c), 0), pre_p,
    9746              :                                NULL, is_gimple_val, fb_rvalue);
    9747            0 :                 OMP_CLAUSE_DECL (c) = TREE_OPERAND (OMP_CLAUSE_DECL (c), 1);
    9748              :               }
    9749          388 :             if (error_operand_p (OMP_CLAUSE_DECL (c)))
    9750              :               return;
    9751          388 :             if (gimplify_expr (&OMP_CLAUSE_DECL (c), pre_p, NULL,
    9752              :                                is_gimple_lvalue, fb_lvalue) == GS_ERROR)
    9753              :               return;
    9754          388 :             gimplify_and_add (OMP_CLAUSE_DECL (c), pre_p);
    9755              :           }
    9756              :       }
    9757          510 :   if (last_bind)
    9758              :     {
    9759          109 :       append_to_statement_list (label, last_body);
    9760          109 :       gimplify_and_add (last_bind, pre_p);
    9761              :     }
    9762              :   return;
    9763              : }
    9764              : 
    9765              : /* Returns a tree expression containing the total iteration count of the
    9766              :    OpenMP iterator IT.  */
    9767              : 
    9768              : static tree
    9769          391 : compute_omp_iterator_count (tree it, gimple_seq *pre_p)
    9770              : {
    9771          391 :   tree tcnt = size_one_node;
    9772          861 :   for (; it; it = TREE_CHAIN (it))
    9773              :     {
    9774          470 :       if (gimplify_expr (&OMP_ITERATOR_BEGIN (it), pre_p, NULL,
    9775              :                          is_gimple_val, fb_rvalue) == GS_ERROR
    9776          470 :           || gimplify_expr (&OMP_ITERATOR_END (it), pre_p, NULL,
    9777              :                             is_gimple_val, fb_rvalue) == GS_ERROR
    9778          470 :           || gimplify_expr (&OMP_ITERATOR_STEP (it), pre_p, NULL,
    9779              :                             is_gimple_val, fb_rvalue) == GS_ERROR
    9780          940 :           || (gimplify_expr (&OMP_ITERATOR_ORIG_STEP (it), pre_p, NULL,
    9781              :                              is_gimple_val, fb_rvalue) == GS_ERROR))
    9782            0 :         return NULL_TREE;
    9783          470 :       tree var = OMP_ITERATOR_VAR (it);
    9784          470 :       tree begin = OMP_ITERATOR_BEGIN (it);
    9785          470 :       tree end = OMP_ITERATOR_END (it);
    9786          470 :       tree step = OMP_ITERATOR_STEP (it);
    9787          470 :       tree orig_step = OMP_ITERATOR_ORIG_STEP (it);
    9788          470 :       tree type = TREE_TYPE (var);
    9789          470 :       tree stype = TREE_TYPE (step);
    9790          470 :       location_t loc = DECL_SOURCE_LOCATION (var);
    9791          470 :       tree endmbegin;
    9792              :       /* Compute count for this iterator as
    9793              :          orig_step > 0
    9794              :          ? (begin < end ? (end - begin + (step - 1)) / step : 0)
    9795              :          : (begin > end ? (end - begin + (step + 1)) / step : 0)
    9796              :          and compute product of those for the entire clause.  */
    9797          470 :       if (POINTER_TYPE_P (type))
    9798           42 :         endmbegin = fold_build2_loc (loc, POINTER_DIFF_EXPR, stype, end, begin);
    9799              :       else
    9800          428 :         endmbegin = fold_build2_loc (loc, MINUS_EXPR, type, end, begin);
    9801              :       /* Account for iteration stopping on the end value in Fortran rather
    9802              :          than before it.  */
    9803          470 :       tree stepm1 = step;
    9804          470 :       tree stepp1 = step;
    9805          470 :       if (!lang_GNU_Fortran ())
    9806              :         {
    9807          372 :           stepm1 = fold_build2_loc (loc, MINUS_EXPR, stype, step,
    9808              :                                     build_int_cst (stype, 1));
    9809          372 :           stepp1 = fold_build2_loc (loc, PLUS_EXPR, stype, step,
    9810              :                                     build_int_cst (stype, 1));
    9811              :         }
    9812          470 :       tree pos = fold_build2_loc (loc, PLUS_EXPR, stype,
    9813              :                                   unshare_expr (endmbegin), stepm1);
    9814          470 :       pos = fold_build2_loc (loc, TRUNC_DIV_EXPR, stype, pos, step);
    9815          470 :       tree neg = fold_build2_loc (loc, PLUS_EXPR, stype, endmbegin, stepp1);
    9816          470 :       if (TYPE_UNSIGNED (stype))
    9817              :         {
    9818           31 :           neg = fold_build1_loc (loc, NEGATE_EXPR, stype, neg);
    9819           31 :           step = fold_build1_loc (loc, NEGATE_EXPR, stype, step);
    9820              :         }
    9821          470 :       neg = fold_build2_loc (loc, TRUNC_DIV_EXPR, stype, neg, step);
    9822          470 :       step = NULL_TREE;
    9823          470 :       tree_code cmp_op = lang_GNU_Fortran () ? LE_EXPR : LT_EXPR;
    9824          470 :       tree cond = fold_build2_loc (loc, cmp_op, boolean_type_node, begin, end);
    9825          470 :       pos = fold_build3_loc (loc, COND_EXPR, stype, cond, pos,
    9826              :                              build_int_cst (stype, 0));
    9827          470 :       cond = fold_build2_loc (loc, cmp_op, boolean_type_node, end, begin);
    9828          470 :       neg = fold_build3_loc (loc, COND_EXPR, stype, cond, neg,
    9829              :                              build_int_cst (stype, 0));
    9830          470 :       tree osteptype = TREE_TYPE (orig_step);
    9831          470 :       cond = fold_build2_loc (loc, GT_EXPR, boolean_type_node, orig_step,
    9832              :                               build_int_cst (osteptype, 0));
    9833          470 :       tree cnt = fold_build3_loc (loc, COND_EXPR, stype, cond, pos, neg);
    9834          470 :       cnt = fold_convert_loc (loc, sizetype, cnt);
    9835          470 :       if (gimplify_expr (&cnt, pre_p, NULL, is_gimple_val,
    9836              :                          fb_rvalue) == GS_ERROR)
    9837              :         return NULL_TREE;
    9838          470 :       tcnt = size_binop_loc (loc, MULT_EXPR, tcnt, cnt);
    9839              :     }
    9840          391 :   if (gimplify_expr (&tcnt, pre_p, NULL, is_gimple_val, fb_rvalue) == GS_ERROR)
    9841              :     return NULL_TREE;
    9842              : 
    9843          391 :   return tcnt;
    9844              : }
    9845              : 
    9846              : /* Build loops iterating over the space defined by the OpenMP iterator IT.
    9847              :    Returns a pointer to the BIND_EXPR_BODY in the innermost loop body.
    9848              :    LAST_BIND is set to point to the BIND_EXPR containing the whole loop.  */
    9849              : 
    9850              : static tree *
    9851          391 : build_omp_iterator_loop (tree it, gimple_seq *pre_p, tree *last_bind)
    9852              : {
    9853          391 :   if (*last_bind)
    9854           31 :     gimplify_and_add (*last_bind, pre_p);
    9855          391 :   tree block = OMP_ITERATOR_BLOCK (it);
    9856          476 :   tree block_stmts = lang_GNU_Fortran () ? BLOCK_SUBBLOCKS (block) : NULL_TREE;
    9857          391 :   *last_bind = build3 (BIND_EXPR, void_type_node,
    9858          391 :                        BLOCK_VARS (block), NULL, block);
    9859          391 :   TREE_SIDE_EFFECTS (*last_bind) = 1;
    9860          391 :   tree *p = &BIND_EXPR_BODY (*last_bind);
    9861         1252 :   for (; it; it = TREE_CHAIN (it))
    9862              :     {
    9863          470 :       tree var = OMP_ITERATOR_VAR (it);
    9864          470 :       tree begin = OMP_ITERATOR_BEGIN (it);
    9865          470 :       tree end = OMP_ITERATOR_END (it);
    9866          470 :       tree step = OMP_ITERATOR_STEP (it);
    9867          470 :       tree orig_step = OMP_ITERATOR_ORIG_STEP (it);
    9868          470 :       block = OMP_ITERATOR_BLOCK (it);
    9869          470 :       tree type = TREE_TYPE (var);
    9870          470 :       location_t loc = DECL_SOURCE_LOCATION (var);
    9871              :       /* Emit:
    9872              :          var = begin;
    9873              :          goto cond_label;
    9874              :          beg_label:
    9875              :          ...
    9876              :          var = var + step;
    9877              :          cond_label:
    9878              :          if (orig_step > 0) {
    9879              :            if (var < end) goto beg_label;  // <= for Fortran
    9880              :          } else {
    9881              :            if (var > end) goto beg_label;  // >= for Fortran
    9882              :          }
    9883              :          for each iterator, with inner iterators added to
    9884              :          the ... above.  */
    9885          470 :       tree beg_label = create_artificial_label (loc);
    9886          470 :       tree cond_label = NULL_TREE;
    9887          470 :       tree tem = build2_loc (loc, MODIFY_EXPR, void_type_node, var, begin);
    9888          470 :       append_to_statement_list_force (tem, p);
    9889          470 :       tem = build_and_jump (&cond_label);
    9890          470 :       append_to_statement_list_force (tem, p);
    9891          470 :       tem = build1 (LABEL_EXPR, void_type_node, beg_label);
    9892          470 :       append_to_statement_list (tem, p);
    9893          470 :       tree bind = build3 (BIND_EXPR, void_type_node, NULL_TREE,
    9894              :                           NULL_TREE, NULL_TREE);
    9895          470 :       TREE_SIDE_EFFECTS (bind) = 1;
    9896          470 :       SET_EXPR_LOCATION (bind, loc);
    9897          470 :       append_to_statement_list_force (bind, p);
    9898          470 :       if (POINTER_TYPE_P (type))
    9899           42 :         tem = build2_loc (loc, POINTER_PLUS_EXPR, type,
    9900              :                           var, fold_convert_loc (loc, sizetype, step));
    9901              :       else
    9902          428 :         tem = build2_loc (loc, PLUS_EXPR, type, var, step);
    9903          470 :       tem = build2_loc (loc, MODIFY_EXPR, void_type_node, var, tem);
    9904          470 :       append_to_statement_list_force (tem, p);
    9905          470 :       tem = build1 (LABEL_EXPR, void_type_node, cond_label);
    9906          470 :       append_to_statement_list (tem, p);
    9907          842 :       tree cond = fold_build2_loc (loc, lang_GNU_Fortran () ? LE_EXPR : LT_EXPR,
    9908              :                                    boolean_type_node, var, end);
    9909          470 :       tree pos = fold_build3_loc (loc, COND_EXPR, void_type_node, cond,
    9910              :                                   build_and_jump (&beg_label), void_node);
    9911          842 :       cond = fold_build2_loc (loc, lang_GNU_Fortran () ? GE_EXPR : GT_EXPR,
    9912              :                               boolean_type_node, var, end);
    9913          470 :       tree neg = fold_build3_loc (loc, COND_EXPR, void_type_node, cond,
    9914              :                                   build_and_jump (&beg_label), void_node);
    9915          470 :       tree osteptype = TREE_TYPE (orig_step);
    9916          470 :       cond = fold_build2_loc (loc, GT_EXPR, boolean_type_node, orig_step,
    9917              :                               build_int_cst (osteptype, 0));
    9918          470 :       tem = fold_build3_loc (loc, COND_EXPR, void_type_node, cond, pos, neg);
    9919          470 :       append_to_statement_list_force (tem, p);
    9920          470 :       p = &BIND_EXPR_BODY (bind);
    9921              :       /* The Fortran front-end stashes statements into the BLOCK_SUBBLOCKS
    9922              :          of the last element of the first iterator.  These should go into the
    9923              :          body of the innermost loop.  */
    9924          470 :       if (!TREE_CHAIN (it))
    9925          391 :         append_to_statement_list_force (block_stmts, p);
    9926              :     }
    9927              : 
    9928          391 :   return p;
    9929              : }
    9930              : 
    9931              : 
    9932              : /* Callback for walk_tree to find a VAR_DECL (stored in DATA) in the
    9933              :    tree TP.  */
    9934              : 
    9935              : static tree
    9936         3900 : find_var_decl (tree *tp, int *, void *data)
    9937              : {
    9938         3900 :   if (*tp == (tree) data)
    9939          369 :     return *tp;
    9940              : 
    9941              :   return NULL_TREE;
    9942              : }
    9943              : 
    9944              : /* Returns an element-by-element copy of OMP iterator tree IT.  If
    9945              :    EXPAND is true, force the result to be an expanded iterator; otherwise
    9946              :    it makes the copy the same size as IT.  */
    9947              : static tree
    9948          337 : copy_omp_iterator (tree it, bool expand = false)
    9949              : {
    9950          337 :   int elem_count = TREE_VEC_LENGTH (it);
    9951          337 :   tree new_it =
    9952          337 :     (expand ? make_expanded_omp_iterator () : make_tree_vec (elem_count));
    9953         2359 :   for (int i = 0; i < TREE_VEC_LENGTH (it); i++)
    9954         2022 :     TREE_VEC_ELT (new_it, i) = TREE_VEC_ELT (it, i);
    9955          337 :   return new_it;
    9956              : }
    9957              : 
    9958              : /* Helper function for walk_tree in remap_omp_iterator_var.  */
    9959              : 
    9960              : static tree
    9961         1063 : remap_omp_iterator_var_1 (tree *tp, int *, void *data)
    9962              : {
    9963         1063 :   tree old_var = ((tree *) data)[0];
    9964         1063 :   tree new_var = ((tree *) data)[1];
    9965              : 
    9966         1063 :   if (*tp == old_var)
    9967          106 :     *tp = new_var;
    9968         1063 :   return NULL_TREE;
    9969              : }
    9970              : 
    9971              : /* Replace instances of OLD_VAR in TP with NEW_VAR.  */
    9972              : 
    9973              : static void
    9974          212 : remap_omp_iterator_var (tree *tp, tree old_var, tree new_var)
    9975              : {
    9976          212 :   tree vars[2] = { old_var, new_var };
    9977          212 :   walk_tree (tp, remap_omp_iterator_var_1, vars, NULL);
    9978          212 : }
    9979              : 
    9980              : /* Scan through all clauses using OpenMP iterators in LIST_P.  If any
    9981              :    clauses have iterators with variables that are not used by the clause
    9982              :    decl or size, issue a warning and replace the iterator with a copy with
    9983              :    the unused variables removed.  */
    9984              : 
    9985              : static void
    9986        27888 : remove_unused_omp_iterator_vars (tree *list_p)
    9987              : {
    9988        27888 :   auto_vec< vec<tree> > iter_vars;
    9989        27888 :   auto_vec<tree> new_iterators;
    9990              : 
    9991        81594 :   for (tree c = *list_p; c; c = OMP_CLAUSE_CHAIN (c))
    9992              :     {
    9993        53706 :       if (!OMP_CLAUSE_HAS_ITERATORS (c))
    9994        53642 :         continue;
    9995          288 :       auto_vec<tree> vars;
    9996          288 :       bool need_new_iterators = false;
    9997          734 :       for (tree it = OMP_CLAUSE_ITERATORS (c); it; it = TREE_CHAIN (it))
    9998              :         {
    9999          446 :           tree var = OMP_ITERATOR_VAR (it);
   10000          446 :           tree t = walk_tree (&OMP_CLAUSE_DECL (c), find_var_decl, var, NULL);
   10001          446 :           if (t == NULL_TREE)
   10002          143 :             t = walk_tree (&OMP_CLAUSE_SIZE (c), find_var_decl, var, NULL);
   10003          143 :           if (t == NULL_TREE)
   10004              :             need_new_iterators = true;
   10005              :           else
   10006          369 :             vars.safe_push (var);
   10007              :         }
   10008          288 :       if (!need_new_iterators)
   10009          211 :         continue;
   10010           77 :       if (vars.is_empty ())
   10011              :         {
   10012              :           /* No iteration variables are used in the clause - remove the
   10013              :              iterator from the clause.  */
   10014           13 :           OMP_CLAUSE_ITERATORS (c) = NULL_TREE;
   10015           13 :           continue;
   10016              :         }
   10017              : 
   10018              :       /* If a new iterator has been created for the current set of used
   10019              :          iterator variables, then use that as the iterator.  Otherwise,
   10020              :          create a new iterator for the current iterator variable set. */
   10021              :       unsigned i;
   10022          106 :       for (i = 0; i < iter_vars.length (); i++)
   10023              :         {
   10024          124 :           if (vars.length () != iter_vars[i].length ())
   10025            0 :             continue;
   10026              :           bool identical_p = true;
   10027          150 :           for (unsigned j = 0; j < vars.length () && identical_p; j++)
   10028           88 :             identical_p = vars[j] == iter_vars[i][j];
   10029              : 
   10030           62 :           if (identical_p)
   10031              :             break;
   10032              :         }
   10033           64 :       if (i < iter_vars.length ())
   10034           20 :         OMP_CLAUSE_ITERATORS (c) = new_iterators[i];
   10035              :       else
   10036              :         {
   10037           44 :           tree new_iters = NULL_TREE;
   10038           44 :           tree *new_iters_p = &new_iters;
   10039           44 :           tree new_vars = NULL_TREE;
   10040           44 :           tree *new_vars_p = &new_vars;
   10041           44 :           i = 0;
   10042          140 :           for (tree it = OMP_CLAUSE_ITERATORS (c); it && i < vars.length();
   10043           96 :                it = TREE_CHAIN (it))
   10044              :             {
   10045           96 :               tree var = OMP_ITERATOR_VAR (it);
   10046           96 :               if (var == vars[i])
   10047              :                 {
   10048           74 :                   *new_iters_p = copy_omp_iterator (it);
   10049           74 :                   *new_vars_p = build_decl (OMP_CLAUSE_LOCATION (c), VAR_DECL,
   10050           74 :                                             DECL_NAME (var), TREE_TYPE (var));
   10051           74 :                   DECL_ARTIFICIAL (*new_vars_p) = 1;
   10052           74 :                   DECL_CONTEXT (*new_vars_p) = DECL_CONTEXT (var);
   10053           74 :                   OMP_ITERATOR_VAR (*new_iters_p) = *new_vars_p;
   10054           74 :                   new_iters_p = &TREE_CHAIN (*new_iters_p);
   10055           74 :                   new_vars_p = &DECL_CHAIN (*new_vars_p);
   10056           74 :                   i++;
   10057              :                 }
   10058              :             }
   10059           44 :           tree old_block = OMP_ITERATOR_BLOCK (OMP_CLAUSE_ITERATORS (c));
   10060           44 :           tree new_block = make_node (BLOCK);
   10061           44 :           BLOCK_VARS (new_block) = new_vars;
   10062           44 :           if (BLOCK_SUBBLOCKS (old_block))
   10063              :             {
   10064            4 :               BLOCK_SUBBLOCKS (new_block) = BLOCK_SUBBLOCKS (old_block);
   10065            4 :               BLOCK_SUBBLOCKS (old_block) = NULL_TREE;
   10066              :             }
   10067           44 :           OMP_ITERATOR_BLOCK (new_iters) = new_block;
   10068           44 :           new_iterators.safe_push (new_iters);
   10069           44 :           iter_vars.safe_push (vars.copy ());
   10070           44 :           OMP_CLAUSE_ITERATORS (c) = new_iters;
   10071              :         }
   10072              : 
   10073              :       /* Remap clause to use the new variables.  */
   10074           64 :       i = 0;
   10075          170 :       for (tree it = OMP_CLAUSE_ITERATORS (c); it; it = TREE_CHAIN (it))
   10076              :         {
   10077          106 :           tree old_var = vars[i++];
   10078          106 :           tree new_var = OMP_ITERATOR_VAR (it);
   10079          106 :           remap_omp_iterator_var (&OMP_CLAUSE_DECL (c), old_var, new_var);
   10080          106 :           remap_omp_iterator_var (&OMP_CLAUSE_SIZE (c), old_var, new_var);
   10081              :         }
   10082          288 :     }
   10083              : 
   10084        27932 :   for (unsigned i = 0; i < iter_vars.length (); i++)
   10085           44 :     iter_vars[i].release ();
   10086        27888 : }
   10087              : 
   10088          272 : struct iterator_loop_info_t
   10089              : {
   10090              :   tree bind;
   10091              :   tree count;
   10092              :   tree index;
   10093              :   tree body_label;
   10094              :   auto_vec<tree> clauses;
   10095              : };
   10096              : 
   10097              : typedef hash_map<tree, iterator_loop_info_t> iterator_loop_info_map_t;
   10098              : 
   10099              : /* Builds a loop to expand any OpenMP iterators in the clauses in LIST_P,
   10100              :    reusing any previously built loops if they use the same set of iterators.
   10101              :    Generated Gimple statements are placed into LOOPS_SEQ_P.  The clause
   10102              :    iterators are updated with information on how and where to insert code into
   10103              :    the loop body.  */
   10104              : 
   10105              : static void
   10106        27888 : build_omp_iterators_loops (tree *list_p, gimple_seq *loops_seq_p)
   10107              : {
   10108        27888 :   iterator_loop_info_map_t loops;
   10109              : 
   10110        81594 :   for (tree c = *list_p; c; c = OMP_CLAUSE_CHAIN (c))
   10111              :     {
   10112        53706 :       if (!OMP_CLAUSE_HAS_ITERATORS (c))
   10113        53443 :         continue;
   10114              : 
   10115          275 :       bool built_p;
   10116          275 :       iterator_loop_info_t &loop
   10117          275 :         = loops.get_or_insert (OMP_CLAUSE_ITERATORS (c), &built_p);
   10118              : 
   10119          275 :       if (!built_p)
   10120              :         {
   10121          136 :           loop.count = compute_omp_iterator_count (OMP_CLAUSE_ITERATORS (c),
   10122              :                                                    loops_seq_p);
   10123          136 :           if (!loop.count)
   10124            0 :             continue;
   10125          136 :           if (integer_zerop (loop.count))
   10126            8 :             warning_at (OMP_CLAUSE_LOCATION (c), OPT_Wopenmp,
   10127              :                         "iteration count is zero");
   10128              : 
   10129          136 :           loop.bind = NULL_TREE;
   10130          136 :           tree *body = build_omp_iterator_loop (OMP_CLAUSE_ITERATORS (c),
   10131              :                                                 loops_seq_p, &loop.bind);
   10132              : 
   10133          136 :           loop.index = create_tmp_var (sizetype);
   10134          136 :           SET_EXPR_LOCATION (loop.bind, OMP_CLAUSE_LOCATION (c));
   10135              : 
   10136              :           /* BEFORE LOOP:  */
   10137              :           /* idx = -1;  */
   10138              :           /* This should be initialized to before the individual elements,
   10139              :              as idx is pre-incremented in the loop body.  */
   10140          136 :           gimple *assign = gimple_build_assign (loop.index, size_int (-1));
   10141          136 :           gimple_seq_add_stmt (loops_seq_p, assign);
   10142              : 
   10143              :           /* IN LOOP BODY:  */
   10144              :           /* Create a label so we can find this point later.  */
   10145          136 :           loop.body_label = create_artificial_label (OMP_CLAUSE_LOCATION (c));
   10146          136 :           tree tem = build1 (LABEL_EXPR, void_type_node, loop.body_label);
   10147          136 :           append_to_statement_list_force (tem, body);
   10148              : 
   10149              :           /* idx += 2;  */
   10150          136 :           tem = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
   10151              :                             void_type_node, loop.index,
   10152              :                             size_binop (PLUS_EXPR, loop.index, size_int (2)));
   10153          136 :           append_to_statement_list_force (tem, body);
   10154              :         }
   10155              : 
   10156              :       /* Create array to hold expanded values.  */
   10157          275 :       tree last_count_2 = size_binop (MULT_EXPR, loop.count, size_int (2));
   10158          275 :       tree arr_length = size_binop (PLUS_EXPR, last_count_2, size_int (1));
   10159          275 :       tree elems = NULL_TREE;
   10160          275 :       if (TREE_CONSTANT (arr_length))
   10161              :         {
   10162          263 :           tree type = build_array_type (ptr_type_node,
   10163              :                                         build_index_type (arr_length));
   10164          263 :           elems = create_tmp_var_raw (type, "omp_iter_data");
   10165          263 :           TREE_ADDRESSABLE (elems) = 1;
   10166          263 :           gimple_add_tmp_var (elems);
   10167              :         }
   10168              :       else
   10169              :         {
   10170              :           /* Handle dynamic sizes.  */
   10171           12 :           sorry ("dynamic iterator sizes not implemented yet");
   10172           12 :           OMP_CLAUSE_ITERATORS (c) = NULL_TREE;
   10173           12 :           continue;
   10174              :         }
   10175              : 
   10176              :       /* BEFORE LOOP:  */
   10177              :       /* elems[0] = count;  */
   10178          263 :       tree lhs = build4 (ARRAY_REF, ptr_type_node, elems, size_int (0),
   10179              :                          NULL_TREE, NULL_TREE);
   10180          263 :       tree tem = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
   10181              :                              void_type_node, lhs, loop.count);
   10182          263 :       gimplify_and_add (tem, loops_seq_p);
   10183              : 
   10184              :       /* Make a copy of the iterator with extra info at the end.  */
   10185          263 :       tree new_iterator = copy_omp_iterator (OMP_CLAUSE_ITERATORS (c), true);
   10186          263 :       OMP_ITERATOR_LABEL (new_iterator) = loop.body_label;
   10187          263 :       OMP_ITERATOR_INDEX (new_iterator) = loop.index;
   10188          263 :       OMP_ITERATOR_ELEMS (new_iterator) = elems;
   10189          263 :       TREE_CHAIN (new_iterator) = TREE_CHAIN (OMP_CLAUSE_ITERATORS (c));
   10190          263 :       OMP_CLAUSE_ITERATORS (c) = new_iterator;
   10191              : 
   10192          263 :       loop.clauses.safe_push (c);
   10193              :     }
   10194              : 
   10195              :   /* Now gimplify and add all the loops that were built.  */
   10196        28024 :   for (hash_map<tree, iterator_loop_info_t>::iterator it = loops.begin ();
   10197        55912 :        it != loops.end (); ++it)
   10198          136 :     gimplify_and_add ((*it).second.bind, loops_seq_p);
   10199        27888 : }
   10200              : 
   10201              : /* Helper function for enter_omp_iterator_loop_context.  */
   10202              : 
   10203              : static gimple_seq *
   10204         2954 : enter_omp_iterator_loop_context_1 (tree iterator, gimple_seq *loops_seq_p)
   10205              : {
   10206              :   /* Drill into the nested bind expressions to get to the loop body.  */
   10207         2954 :   for (gimple_stmt_iterator gsi = gsi_start (*loops_seq_p);
   10208        19650 :        !gsi_end_p (gsi); gsi_next (&gsi))
   10209              :     {
   10210        19042 :       gimple *stmt = gsi_stmt (gsi);
   10211              : 
   10212        19042 :       switch (gimple_code (stmt))
   10213              :         {
   10214         2256 :         case GIMPLE_BIND:
   10215         2256 :           {
   10216         2256 :             gbind *bind_stmt = as_a<gbind *> (stmt);
   10217         2256 :             gimple_push_bind_expr (bind_stmt);
   10218         2256 :             gimple_seq *bind_body_p = gimple_bind_body_ptr (bind_stmt);
   10219         2256 :             gimple_seq *seq =
   10220         2256 :               enter_omp_iterator_loop_context_1 (iterator, bind_body_p);
   10221         2256 :             if (seq)
   10222              :               return seq;
   10223          608 :             gimple_pop_bind_expr ();
   10224              :           }
   10225          608 :           break;
   10226            3 :         case GIMPLE_TRY:
   10227            3 :           {
   10228            3 :             gimple_seq *try_eval_p = gimple_try_eval_ptr (stmt);
   10229            3 :             gimple_seq *seq =
   10230            3 :               enter_omp_iterator_loop_context_1 (iterator, try_eval_p);
   10231            3 :             if (seq)
   10232              :               return seq;
   10233              :           }
   10234              :           break;
   10235         3018 :         case GIMPLE_LABEL:
   10236         3018 :           {
   10237         3018 :             glabel *label_stmt = as_a<glabel *> (stmt);
   10238         3018 :             tree label = gimple_label_label (label_stmt);
   10239         3018 :             if (label == OMP_ITERATOR_LABEL (iterator))
   10240              :               return loops_seq_p;
   10241              :           }
   10242              :           break;
   10243              :         default:
   10244              :           break;
   10245              :         }
   10246              :     }
   10247              : 
   10248              :   return NULL;
   10249              : }
   10250              : 
   10251              : /* Enter the Gimplification context in LOOPS_SEQ_P for the iterator loop
   10252              :    associated with OpenMP clause C.  Returns the gimple_seq for the loop body
   10253              :    if C has OpenMP iterators, or ALT_SEQ_P if not.  */
   10254              : 
   10255              : static gimple_seq *
   10256        65332 : enter_omp_iterator_loop_context (tree c, gimple_seq *loops_seq_p,
   10257              :                                  gimple_seq *alt_seq_p)
   10258              : {
   10259        65332 :   if (!OMP_CLAUSE_HAS_ITERATORS (c))
   10260              :     return alt_seq_p;
   10261              : 
   10262          695 :   push_gimplify_context ();
   10263              : 
   10264          695 :   gimple_seq *seq = enter_omp_iterator_loop_context_1 (OMP_CLAUSE_ITERATORS (c),
   10265              :                                                        loops_seq_p);
   10266          695 :   gcc_assert (seq);
   10267              :   return seq;
   10268              : }
   10269              : 
   10270              : /* Enter the Gimplification context in STMT for the iterator loop associated
   10271              :    with OpenMP clause C.  Returns the gimple_seq for the loop body if C has
   10272              :    OpenMP iterators, or ALT_SEQ_P if not.  */
   10273              : 
   10274              : gimple_seq *
   10275          434 : enter_omp_iterator_loop_context (tree c, gomp_target *stmt,
   10276              :                                  gimple_seq *alt_seq_p)
   10277              : {
   10278          434 :   gimple_seq *loops_seq_p = gimple_omp_target_iterator_loops_ptr (stmt);
   10279          434 :   return enter_omp_iterator_loop_context (c, loops_seq_p, alt_seq_p);
   10280              : }
   10281              : 
   10282              : /* Exit the Gimplification context for the OpenMP clause C.  */
   10283              : 
   10284              : void
   10285        65422 : exit_omp_iterator_loop_context (tree c)
   10286              : {
   10287        65422 :   if (!OMP_CLAUSE_HAS_ITERATORS (c))
   10288              :     return;
   10289         2343 :   while (!gimplify_ctxp->bind_expr_stack.is_empty ())
   10290         1648 :     gimple_pop_bind_expr ();
   10291          695 :   pop_gimplify_context (NULL);
   10292              : }
   10293              : 
   10294              : /* If *LIST_P contains any OpenMP depend clauses with iterators,
   10295              :    lower all the depend clauses by populating corresponding depend
   10296              :    array.  Returns 0 if there are no such depend clauses, or
   10297              :    2 if all depend clauses should be removed, 1 otherwise.  */
   10298              : 
   10299              : static int
   10300         1905 : gimplify_omp_depend (tree *list_p, gimple_seq *pre_p)
   10301              : {
   10302         1905 :   tree c;
   10303         1905 :   gimple *g;
   10304         1905 :   size_t n[5] = { 0, 0, 0, 0, 0 };
   10305         1905 :   bool unused[5];
   10306         1905 :   tree counts[5] = { NULL_TREE, NULL_TREE, NULL_TREE, NULL_TREE, NULL_TREE };
   10307         1905 :   tree last_iter = NULL_TREE, last_count = NULL_TREE;
   10308         1905 :   size_t i, j;
   10309         1905 :   location_t first_loc = UNKNOWN_LOCATION;
   10310              : 
   10311         6254 :   for (c = *list_p; c; c = OMP_CLAUSE_CHAIN (c))
   10312         4349 :     if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
   10313              :       {
   10314         2207 :         switch (OMP_CLAUSE_DEPEND_KIND (c))
   10315              :           {
   10316              :           case OMP_CLAUSE_DEPEND_IN:
   10317              :             i = 2;
   10318              :             break;
   10319              :           case OMP_CLAUSE_DEPEND_OUT:
   10320              :           case OMP_CLAUSE_DEPEND_INOUT:
   10321              :             i = 0;
   10322              :             break;
   10323              :           case OMP_CLAUSE_DEPEND_MUTEXINOUTSET:
   10324              :             i = 1;
   10325              :             break;
   10326              :           case OMP_CLAUSE_DEPEND_DEPOBJ:
   10327              :             i = 3;
   10328              :             break;
   10329              :           case OMP_CLAUSE_DEPEND_INOUTSET:
   10330              :             i = 4;
   10331              :             break;
   10332            0 :           default:
   10333            0 :             gcc_unreachable ();
   10334              :           }
   10335         2207 :         tree t = OMP_CLAUSE_DECL (c);
   10336         2207 :         if (first_loc == UNKNOWN_LOCATION)
   10337         1905 :           first_loc = OMP_CLAUSE_LOCATION (c);
   10338         2207 :         if (OMP_ITERATOR_DECL_P (t))
   10339              :           {
   10340          302 :             if (TREE_PURPOSE (t) != last_iter)
   10341              :               {
   10342          255 :                 tree tcnt = compute_omp_iterator_count (TREE_PURPOSE (t),
   10343              :                                                         pre_p);
   10344          255 :                 if (!tcnt)
   10345              :                   return 2;
   10346          255 :                 last_iter = TREE_PURPOSE (t);
   10347          255 :                 last_count = tcnt;
   10348              :               }
   10349          302 :             if (counts[i] == NULL_TREE)
   10350          248 :               counts[i] = last_count;
   10351              :             else
   10352           54 :               counts[i] = size_binop_loc (OMP_CLAUSE_LOCATION (c),
   10353              :                                           PLUS_EXPR, counts[i], last_count);
   10354              :           }
   10355              :         else
   10356         1905 :           n[i]++;
   10357              :       }
   10358        10620 :   for (i = 0; i < 5; i++)
   10359         8925 :     if (counts[i])
   10360              :       break;
   10361         1905 :   if (i == 5)
   10362              :     return 0;
   10363              : 
   10364          210 :   tree total = size_zero_node;
   10365         1260 :   for (i = 0; i < 5; i++)
   10366              :     {
   10367         1050 :       unused[i] = counts[i] == NULL_TREE && n[i] == 0;
   10368         1050 :       if (counts[i] == NULL_TREE)
   10369          802 :         counts[i] = size_zero_node;
   10370         1050 :       if (n[i])
   10371           44 :         counts[i] = size_binop (PLUS_EXPR, counts[i], size_int (n[i]));
   10372         1050 :       if (gimplify_expr (&counts[i], pre_p, NULL, is_gimple_val,
   10373              :                          fb_rvalue) == GS_ERROR)
   10374              :         return 2;
   10375         1050 :       total = size_binop (PLUS_EXPR, total, counts[i]);
   10376              :     }
   10377              : 
   10378          210 :   if (gimplify_expr (&total, pre_p, NULL, is_gimple_val, fb_rvalue)
   10379              :       == GS_ERROR)
   10380              :     return 2;
   10381          210 :   bool is_old = unused[1] && unused[3] && unused[4];
   10382          210 :   tree totalpx = size_binop (PLUS_EXPR, unshare_expr (total),
   10383              :                              size_int (is_old ? 1 : 4));
   10384          210 :   if (!unused[4])
   10385            3 :     totalpx = size_binop (PLUS_EXPR, totalpx,
   10386              :                           size_binop (MULT_EXPR, counts[4], size_int (2)));
   10387          210 :   tree type = build_array_type (ptr_type_node, build_index_type (totalpx));
   10388          210 :   tree array = create_tmp_var_raw (type);
   10389          210 :   TREE_ADDRESSABLE (array) = 1;
   10390          210 :   if (!poly_int_tree_p (totalpx))
   10391              :     {
   10392           83 :       if (!TYPE_SIZES_GIMPLIFIED (TREE_TYPE (array)))
   10393           83 :         gimplify_type_sizes (TREE_TYPE (array), pre_p);
   10394           83 :       if (gimplify_omp_ctxp)
   10395              :         {
   10396              :           struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
   10397              :           while (ctx
   10398           22 :                  && (ctx->region_type == ORT_WORKSHARE
   10399              :                      || ctx->region_type == ORT_TASKGROUP
   10400           22 :                      || ctx->region_type == ORT_SIMD
   10401           22 :                      || ctx->region_type == ORT_ACC))
   10402            0 :             ctx = ctx->outer_context;
   10403           22 :           if (ctx)
   10404           22 :             omp_add_variable (ctx, array, GOVD_LOCAL | GOVD_SEEN);
   10405              :         }
   10406           83 :       gimplify_vla_decl (array, pre_p);
   10407              :     }
   10408              :   else
   10409          127 :     gimple_add_tmp_var (array);
   10410          210 :   tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE,
   10411              :                    NULL_TREE);
   10412          210 :   tree tem;
   10413          210 :   if (!is_old)
   10414              :     {
   10415           31 :       tem = build2 (MODIFY_EXPR, void_type_node, r,
   10416              :                     build_int_cst (ptr_type_node, 0));
   10417           31 :       gimplify_and_add (tem, pre_p);
   10418           31 :       r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE,
   10419              :                   NULL_TREE);
   10420              :     }
   10421          210 :   tem = build2 (MODIFY_EXPR, void_type_node, r,
   10422              :                 fold_convert (ptr_type_node, total));
   10423          210 :   gimplify_and_add (tem, pre_p);
   10424          816 :   for (i = 1; i < (is_old ? 2 : 4); i++)
   10425              :     {
   10426          272 :       r = build4 (ARRAY_REF, ptr_type_node, array, size_int (i + !is_old),
   10427              :                   NULL_TREE, NULL_TREE);
   10428          272 :       tem = build2 (MODIFY_EXPR, void_type_node, r, counts[i - 1]);
   10429          272 :       gimplify_and_add (tem, pre_p);
   10430              :     }
   10431              : 
   10432              :   tree cnts[6];
   10433          740 :   for (j = 5; j; j--)
   10434          740 :     if (!unused[j - 1])
   10435              :       break;
   10436         1260 :   for (i = 0; i < 5; i++)
   10437              :     {
   10438         1050 :       if (i && (i >= j || unused[i - 1]))
   10439              :         {
   10440          790 :           cnts[i] = cnts[i - 1];
   10441          790 :           continue;
   10442              :         }
   10443          260 :       cnts[i] = create_tmp_var (sizetype);
   10444          260 :       if (i == 0)
   10445          241 :         g = gimple_build_assign (cnts[i], size_int (is_old ? 2 : 5));
   10446              :       else
   10447              :         {
   10448           50 :           tree t;
   10449           50 :           if (is_old)
   10450           38 :             t = size_binop (PLUS_EXPR, counts[0], size_int (2));
   10451              :           else
   10452           12 :             t = size_binop (PLUS_EXPR, cnts[i - 1], counts[i - 1]);
   10453           50 :           if (gimplify_expr (&t, pre_p, NULL, is_gimple_val, fb_rvalue)
   10454              :               == GS_ERROR)
   10455            0 :             return 2;
   10456           50 :           g = gimple_build_assign (cnts[i], t);
   10457              :         }
   10458          260 :       gimple_seq_add_stmt (pre_p, g);
   10459              :     }
   10460          210 :   if (unused[4])
   10461          207 :     cnts[5] = NULL_TREE;
   10462              :   else
   10463              :     {
   10464            3 :       tree t = size_binop (PLUS_EXPR, total, size_int (5));
   10465            3 :       cnts[5] = create_tmp_var (sizetype);
   10466            3 :       g = gimple_build_assign (cnts[i], t);
   10467            3 :       gimple_seq_add_stmt (pre_p, g);
   10468              :     }
   10469              : 
   10470          210 :   last_iter = NULL_TREE;
   10471          210 :   tree last_bind = NULL_TREE;
   10472          210 :   tree *last_body = NULL;
   10473          573 :   for (c = *list_p; c; c = OMP_CLAUSE_CHAIN (c))
   10474          363 :     if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
   10475              :       {
   10476          348 :         switch (OMP_CLAUSE_DEPEND_KIND (c))
   10477              :           {
   10478              :           case OMP_CLAUSE_DEPEND_IN:
   10479              :             i = 2;
   10480              :             break;
   10481              :           case OMP_CLAUSE_DEPEND_OUT:
   10482              :           case OMP_CLAUSE_DEPEND_INOUT:
   10483              :             i = 0;
   10484              :             break;
   10485              :           case OMP_CLAUSE_DEPEND_MUTEXINOUTSET:
   10486              :             i = 1;
   10487              :             break;
   10488              :           case OMP_CLAUSE_DEPEND_DEPOBJ:
   10489              :             i = 3;
   10490              :             break;
   10491              :           case OMP_CLAUSE_DEPEND_INOUTSET:
   10492              :             i = 4;
   10493              :             break;
   10494            0 :           default:
   10495            0 :             gcc_unreachable ();
   10496              :           }
   10497          348 :         tree t = OMP_CLAUSE_DECL (c);
   10498          348 :         if (OMP_ITERATOR_DECL_P (t))
   10499              :           {
   10500          302 :             if (TREE_PURPOSE (t) != last_iter)
   10501              :               {
   10502          255 :                 last_body = build_omp_iterator_loop (TREE_PURPOSE (t), pre_p,
   10503              :                                                      &last_bind);
   10504          255 :                 SET_EXPR_LOCATION (last_bind, OMP_CLAUSE_LOCATION (c));
   10505              :               }
   10506          302 :             last_iter = TREE_PURPOSE (t);
   10507          302 :             if (TREE_CODE (TREE_VALUE (t)) == COMPOUND_EXPR)
   10508              :               {
   10509            0 :                 append_to_statement_list (TREE_OPERAND (TREE_VALUE (t),
   10510              :                                           0), last_body);
   10511            0 :                 TREE_VALUE (t) = TREE_OPERAND (TREE_VALUE (t), 1);
   10512              :               }
   10513          302 :             if (error_operand_p (TREE_VALUE (t)))
   10514              :               return 2;
   10515          302 :             if (TREE_VALUE (t) != null_pointer_node)
   10516          296 :               TREE_VALUE (t) = build_fold_addr_expr (TREE_VALUE (t));
   10517          302 :             if (i == 4)
   10518              :               {
   10519            3 :                 r = build4 (ARRAY_REF, ptr_type_node, array, cnts[i],
   10520              :                             NULL_TREE, NULL_TREE);
   10521            3 :                 tree r2 = build4 (ARRAY_REF, ptr_type_node, array, cnts[5],
   10522              :                                   NULL_TREE, NULL_TREE);
   10523            3 :                 r2 = build_fold_addr_expr_with_type (r2, ptr_type_node);
   10524            3 :                 tem = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
   10525              :                                   void_type_node, r, r2);
   10526            3 :                 append_to_statement_list_force (tem, last_body);
   10527            3 :                 tem = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
   10528              :                                   void_type_node, cnts[i],
   10529              :                                   size_binop (PLUS_EXPR, cnts[i],
   10530              :                                               size_int (1)));
   10531            3 :                 append_to_statement_list_force (tem, last_body);
   10532            3 :                 i = 5;
   10533              :               }
   10534          302 :             r = build4 (ARRAY_REF, ptr_type_node, array, cnts[i],
   10535              :                         NULL_TREE, NULL_TREE);
   10536          604 :             tem = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
   10537          302 :                               void_type_node, r, TREE_VALUE (t));
   10538          302 :             append_to_statement_list_force (tem, last_body);
   10539          302 :             if (i == 5)
   10540              :               {
   10541            3 :                 r = build4 (ARRAY_REF, ptr_type_node, array,
   10542              :                             size_binop (PLUS_EXPR, cnts[i], size_int (1)),
   10543              :                             NULL_TREE, NULL_TREE);
   10544            3 :                 tem = build_int_cst (ptr_type_node, GOMP_DEPEND_INOUTSET);
   10545            3 :                 tem = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
   10546              :                                   void_type_node, r, tem);
   10547            3 :                 append_to_statement_list_force (tem, last_body);
   10548              :               }
   10549          302 :             tem = build2_loc (OMP_CLAUSE_LOCATION (c), MODIFY_EXPR,
   10550              :                               void_type_node, cnts[i],
   10551          302 :                               size_binop (PLUS_EXPR, cnts[i],
   10552              :                                           size_int (1 + (i == 5))));
   10553          302 :             append_to_statement_list_force (tem, last_body);
   10554          302 :             TREE_VALUE (t) = null_pointer_node;
   10555              :           }
   10556              :         else
   10557              :           {
   10558           46 :             if (last_bind)
   10559              :               {
   10560           18 :                 gimplify_and_add (last_bind, pre_p);
   10561           18 :                 last_bind = NULL_TREE;
   10562              :               }
   10563           46 :             if (TREE_CODE (OMP_CLAUSE_DECL (c)) == COMPOUND_EXPR)
   10564              :               {
   10565            0 :                 gimplify_expr (&TREE_OPERAND (OMP_CLAUSE_DECL (c), 0), pre_p,
   10566              :                                NULL, is_gimple_val, fb_rvalue);
   10567            0 :                 OMP_CLAUSE_DECL (c) = TREE_OPERAND (OMP_CLAUSE_DECL (c), 1);
   10568              :               }
   10569           46 :             if (error_operand_p (OMP_CLAUSE_DECL (c)))
   10570              :               return 2;
   10571           46 :             if (OMP_CLAUSE_DECL (c) != null_pointer_node)
   10572           46 :               OMP_CLAUSE_DECL (c) = build_fold_addr_expr (OMP_CLAUSE_DECL (c));
   10573           46 :             if (gimplify_expr (&OMP_CLAUSE_DECL (c), pre_p, NULL,
   10574              :                                is_gimple_val, fb_rvalue) == GS_ERROR)
   10575              :               return 2;
   10576           46 :             if (i == 4)
   10577              :               {
   10578            0 :                 r = build4 (ARRAY_REF, ptr_type_node, array, cnts[i],
   10579              :                             NULL_TREE, NULL_TREE);
   10580            0 :                 tree r2 = build4 (ARRAY_REF, ptr_type_node, array, cnts[5],
   10581              :                                   NULL_TREE, NULL_TREE);
   10582            0 :                 r2 = build_fold_addr_expr_with_type (r2, ptr_type_node);
   10583            0 :                 tem = build2 (MODIFY_EXPR, void_type_node, r, r2);
   10584            0 :                 gimplify_and_add (tem, pre_p);
   10585            0 :                 g = gimple_build_assign (cnts[i], size_binop (PLUS_EXPR,
   10586              :                                                               cnts[i],
   10587              :                                                               size_int (1)));
   10588            0 :                 gimple_seq_add_stmt (pre_p, g);
   10589            0 :                 i = 5;
   10590              :               }
   10591           46 :             r = build4 (ARRAY_REF, ptr_type_node, array, cnts[i],
   10592              :                         NULL_TREE, NULL_TREE);
   10593           46 :             tem = build2 (MODIFY_EXPR, void_type_node, r, OMP_CLAUSE_DECL (c));
   10594           46 :             gimplify_and_add (tem, pre_p);
   10595           46 :             if (i == 5)
   10596              :               {
   10597            0 :                 r = build4 (ARRAY_REF, ptr_type_node, array,
   10598              :                             size_binop (PLUS_EXPR, cnts[i], size_int (1)),
   10599              :                             NULL_TREE, NULL_TREE);
   10600            0 :                 tem = build_int_cst (ptr_type_node, GOMP_DEPEND_INOUTSET);
   10601            0 :                 tem = build2 (MODIFY_EXPR, void_type_node, r, tem);
   10602            0 :                 append_to_statement_list_force (tem, last_body);
   10603            0 :                 gimplify_and_add (tem, pre_p);
   10604              :               }
   10605           46 :             g = gimple_build_assign (cnts[i],
   10606           46 :                                      size_binop (PLUS_EXPR, cnts[i],
   10607              :                                                  size_int (1 + (i == 5))));
   10608           46 :             gimple_seq_add_stmt (pre_p, g);
   10609              :           }
   10610              :       }
   10611          210 :   if (last_bind)
   10612          206 :     gimplify_and_add (last_bind, pre_p);
   10613          210 :   tree cond = boolean_false_node;
   10614          210 :   if (is_old)
   10615              :     {
   10616          179 :       if (!unused[0])
   10617           92 :         cond = build2_loc (first_loc, NE_EXPR, boolean_type_node, cnts[0],
   10618              :                            size_binop_loc (first_loc, PLUS_EXPR, counts[0],
   10619              :                                            size_int (2)));
   10620          179 :       if (!unused[2])
   10621          125 :         cond = build2_loc (first_loc, TRUTH_OR_EXPR, boolean_type_node, cond,
   10622              :                            build2_loc (first_loc, NE_EXPR, boolean_type_node,
   10623              :                                        cnts[2],
   10624              :                                        size_binop_loc (first_loc, PLUS_EXPR,
   10625              :                                                        totalpx,
   10626              :                                                        size_int (1))));
   10627              :     }
   10628              :   else
   10629              :     {
   10630           31 :       tree prev = size_int (5);
   10631          186 :       for (i = 0; i < 5; i++)
   10632              :         {
   10633          155 :           if (unused[i])
   10634          112 :             continue;
   10635           43 :           prev = size_binop_loc (first_loc, PLUS_EXPR, counts[i], prev);
   10636           43 :           cond = build2_loc (first_loc, TRUTH_OR_EXPR, boolean_type_node, cond,
   10637              :                              build2_loc (first_loc, NE_EXPR, boolean_type_node,
   10638              :                                          cnts[i], unshare_expr (prev)));
   10639              :         }
   10640              :     }
   10641          210 :   tem = build3_loc (first_loc, COND_EXPR, void_type_node, cond,
   10642              :                     build_call_expr_loc (first_loc,
   10643              :                                          builtin_decl_explicit (BUILT_IN_TRAP),
   10644              :                                          0), void_node);
   10645          210 :   gimplify_and_add (tem, pre_p);
   10646          210 :   c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND);
   10647          210 :   OMP_CLAUSE_DEPEND_KIND (c) = OMP_CLAUSE_DEPEND_LAST;
   10648          210 :   OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array);
   10649          210 :   OMP_CLAUSE_CHAIN (c) = *list_p;
   10650          210 :   *list_p = c;
   10651          210 :   return 1;
   10652              : }
   10653              : 
   10654              : /* True if mapping node C maps, or unmaps, a (Fortran) array descriptor.  */
   10655              : 
   10656              : static bool
   10657       126298 : omp_map_clause_descriptor_p (tree c)
   10658              : {
   10659       126298 :   if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP)
   10660              :     return false;
   10661              : 
   10662       126294 :   if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_TO_PSET)
   10663              :     return true;
   10664              : 
   10665        83076 :   if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_RELEASE
   10666        75038 :        || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DELETE)
   10667        83558 :       && OMP_CLAUSE_RELEASE_DESCRIPTOR (c))
   10668         6135 :     return true;
   10669              : 
   10670              :   return false;
   10671              : }
   10672              : 
   10673              : /* For a set of mappings describing an array section pointed to by a struct
   10674              :    (or derived type, etc.) component, create an "alloc" or "release" node to
   10675              :    insert into a list following a GOMP_MAP_STRUCT node.  For some types of
   10676              :    mapping (e.g. Fortran arrays with descriptors), an additional mapping may
   10677              :    be created that is inserted into the list of mapping nodes attached to the
   10678              :    directive being processed -- not part of the sorted list of nodes after
   10679              :    GOMP_MAP_STRUCT.
   10680              : 
   10681              :    CODE is the code of the directive being processed.  GRP_START and GRP_END
   10682              :    are the first and last of two or three nodes representing this array section
   10683              :    mapping (e.g. a data movement node like GOMP_MAP_{TO,FROM}, optionally a
   10684              :    GOMP_MAP_TO_PSET, and finally a GOMP_MAP_ALWAYS_POINTER).  EXTRA_NODE is
   10685              :    filled with the additional node described above, if needed.
   10686              : 
   10687              :    This function does not add the new nodes to any lists itself.  It is the
   10688              :    responsibility of the caller to do that.  */
   10689              : 
   10690              : static tree
   10691         1729 : build_omp_struct_comp_nodes (enum tree_code code, tree grp_start, tree grp_end,
   10692              :                              tree *extra_node)
   10693              : {
   10694         1524 :   enum gomp_map_kind mkind
   10695         1729 :     = (code == OMP_TARGET_EXIT_DATA || code == OACC_EXIT_DATA)
   10696         1729 :       ? GOMP_MAP_RELEASE : GOMP_MAP_ALLOC;
   10697              : 
   10698         1729 :   gcc_assert (grp_start != grp_end);
   10699              : 
   10700         1729 :   tree c2 = build_omp_clause (OMP_CLAUSE_LOCATION (grp_end), OMP_CLAUSE_MAP);
   10701         1729 :   OMP_CLAUSE_SET_MAP_KIND (c2, mkind);
   10702         1729 :   OMP_CLAUSE_DECL (c2) = unshare_expr (OMP_CLAUSE_DECL (grp_end));
   10703         1729 :   OMP_CLAUSE_CHAIN (c2) = NULL_TREE;
   10704         1729 :   tree grp_mid = NULL_TREE;
   10705         1729 :   if (OMP_CLAUSE_CHAIN (grp_start) != grp_end)
   10706          197 :     grp_mid = OMP_CLAUSE_CHAIN (grp_start);
   10707              : 
   10708          197 :   if (grp_mid && omp_map_clause_descriptor_p (grp_mid))
   10709            0 :     OMP_CLAUSE_SIZE (c2) = OMP_CLAUSE_SIZE (grp_mid);
   10710              :   else
   10711         1729 :     OMP_CLAUSE_SIZE (c2) = TYPE_SIZE_UNIT (ptr_type_node);
   10712              : 
   10713         1729 :   if (grp_mid
   10714          197 :       && OMP_CLAUSE_CODE (grp_mid) == OMP_CLAUSE_MAP
   10715         1926 :       && OMP_CLAUSE_MAP_KIND (grp_mid) == GOMP_MAP_ALWAYS_POINTER)
   10716              :     {
   10717            0 :       tree c3
   10718            0 :         = build_omp_clause (OMP_CLAUSE_LOCATION (grp_end), OMP_CLAUSE_MAP);
   10719            0 :       OMP_CLAUSE_SET_MAP_KIND (c3, mkind);
   10720            0 :       OMP_CLAUSE_DECL (c3) = unshare_expr (OMP_CLAUSE_DECL (grp_mid));
   10721            0 :       OMP_CLAUSE_SIZE (c3) = TYPE_SIZE_UNIT (ptr_type_node);
   10722            0 :       OMP_CLAUSE_CHAIN (c3) = NULL_TREE;
   10723              : 
   10724            0 :       *extra_node = c3;
   10725              :     }
   10726              :   else
   10727         1729 :     *extra_node = NULL_TREE;
   10728              : 
   10729         1729 :   return c2;
   10730              : }
   10731              : 
   10732              : /* Strip ARRAY_REFS or an indirect ref off BASE, find the containing object,
   10733              :    and set *BITPOSP and *POFFSETP to the bit offset of the access.
   10734              :    If BASE_REF is non-NULL and the containing object is a reference, set
   10735              :    *BASE_REF to that reference before dereferencing the object.
   10736              :    If BASE_REF is NULL, check that the containing object is a COMPONENT_REF or
   10737              :    has array type, else return NULL.  */
   10738              : 
   10739              : static tree
   10740         7758 : extract_base_bit_offset (tree base, poly_int64 *bitposp,
   10741              :                          poly_offset_int *poffsetp,
   10742              :                          bool *variable_offset,
   10743              :                          tree iterator)
   10744              : {
   10745         7758 :   tree offset;
   10746         7758 :   poly_int64 bitsize, bitpos;
   10747         7758 :   machine_mode mode;
   10748         7758 :   int unsignedp, reversep, volatilep = 0;
   10749         7758 :   poly_offset_int poffset;
   10750              : 
   10751         7758 :   STRIP_NOPS (base);
   10752              : 
   10753         7758 :   if (iterator)
   10754              :     {
   10755              :       /* Replace any iterator variables with the lower bound of the iterator.
   10756              :          This will give us the nominal offset and bit position of the first
   10757              :          element, which is all we should need to lay out the mappings.  The
   10758              :          actual locations of the iterated mappings are elsewhere.  */
   10759              :       tree it;
   10760           72 :       for (it = iterator; it; it = TREE_CHAIN (it))
   10761           39 :         base = simplify_replace_tree (base, OMP_ITERATOR_VAR (it),
   10762           39 :                                       OMP_ITERATOR_BEGIN (it));
   10763              :     }
   10764              : 
   10765         7758 :   base = get_inner_reference (base, &bitsize, &bitpos, &offset, &mode,
   10766              :                               &unsignedp, &reversep, &volatilep);
   10767              : 
   10768         7758 :   STRIP_NOPS (base);
   10769              : 
   10770         7758 :   if (offset && poly_int_tree_p (offset))
   10771              :     {
   10772            0 :       poffset = wi::to_poly_offset (offset);
   10773            0 :       *variable_offset = false;
   10774              :     }
   10775              :   else
   10776              :     {
   10777         7758 :       poffset = 0;
   10778         7758 :       *variable_offset = (offset != NULL_TREE);
   10779              :     }
   10780              : 
   10781         7758 :   if (maybe_ne (bitpos, 0))
   10782         5632 :     poffset += bits_to_bytes_round_down (bitpos);
   10783              : 
   10784         7758 :   *bitposp = bitpos;
   10785         7758 :   *poffsetp = poffset;
   10786              : 
   10787         7758 :   return base;
   10788              : }
   10789              : 
   10790              : /* Used for topological sorting of mapping groups.  UNVISITED means we haven't
   10791              :    started processing the group yet.  The TEMPORARY mark is used when we first
   10792              :    encounter a group on a depth-first traversal, and the PERMANENT mark is used
   10793              :    when we have processed all the group's children (i.e. all the base pointers
   10794              :    referred to by the group's mapping nodes, recursively).  */
   10795              : 
   10796              : enum omp_tsort_mark {
   10797              :   UNVISITED,
   10798              :   TEMPORARY,
   10799              :   PERMANENT
   10800              : };
   10801              : 
   10802              : /* Hash for trees based on operand_equal_p.  Like tree_operand_hash
   10803              :    but ignores side effects in the equality comparisons.  */
   10804              : 
   10805              : struct tree_operand_hash_no_se : tree_operand_hash
   10806              : {
   10807              :   static inline bool equal (const value_type &,
   10808              :                             const compare_type &);
   10809              : };
   10810              : 
   10811              : inline bool
   10812       416537 : tree_operand_hash_no_se::equal (const value_type &t1,
   10813              :                                 const compare_type &t2)
   10814              : {
   10815       416537 :   return operand_equal_p (t1, t2, OEP_MATCH_SIDE_EFFECTS);
   10816              : }
   10817              : 
   10818              : /* A group of OMP_CLAUSE_MAP nodes that correspond to a single "map"
   10819              :    clause.  */
   10820              : 
   10821              : struct omp_mapping_group {
   10822              :   tree *grp_start;
   10823              :   tree grp_end;
   10824              :   omp_tsort_mark mark;
   10825              :   /* If we've removed the group but need to reindex, mark the group as
   10826              :      deleted.  */
   10827              :   bool deleted;
   10828              :   /* The group points to an already-created "GOMP_MAP_STRUCT
   10829              :      GOMP_MAP_ATTACH_DETACH" pair.  */
   10830              :   bool reprocess_struct;
   10831              :   /* The group should use "zero-length" allocations for pointers that are not
   10832              :      mapped "to" on the same directive.  */
   10833              :   bool fragile;
   10834              :   struct omp_mapping_group *sibling;
   10835              :   struct omp_mapping_group *next;
   10836              : };
   10837              : 
   10838              : DEBUG_FUNCTION void
   10839            0 : debug_mapping_group (omp_mapping_group *grp)
   10840              : {
   10841            0 :   tree tmp = OMP_CLAUSE_CHAIN (grp->grp_end);
   10842            0 :   OMP_CLAUSE_CHAIN (grp->grp_end) = NULL;
   10843            0 :   debug_generic_expr (*grp->grp_start);
   10844            0 :   OMP_CLAUSE_CHAIN (grp->grp_end) = tmp;
   10845            0 : }
   10846              : 
   10847              : /* Return the OpenMP "base pointer" of an expression EXPR, or NULL if there
   10848              :    isn't one.  */
   10849              : 
   10850              : static tree
   10851        36735 : omp_get_base_pointer (tree expr)
   10852              : {
   10853        36735 :   while (TREE_CODE (expr) == ARRAY_REF
   10854        44134 :          || TREE_CODE (expr) == COMPONENT_REF)
   10855         7399 :     expr = TREE_OPERAND (expr, 0);
   10856              : 
   10857        36735 :   if (INDIRECT_REF_P (expr)
   10858        36735 :       || (TREE_CODE (expr) == MEM_REF
   10859            1 :           && integer_zerop (TREE_OPERAND (expr, 1))))
   10860              :     {
   10861        10612 :       expr = TREE_OPERAND (expr, 0);
   10862        10649 :       while (TREE_CODE (expr) == COMPOUND_EXPR)
   10863           37 :         expr = TREE_OPERAND (expr, 1);
   10864        10612 :       if (TREE_CODE (expr) == POINTER_PLUS_EXPR)
   10865          841 :         expr = TREE_OPERAND (expr, 0);
   10866        10612 :       if (TREE_CODE (expr) == SAVE_EXPR)
   10867           31 :         expr = TREE_OPERAND (expr, 0);
   10868        10612 :       STRIP_NOPS (expr);
   10869        10612 :       return expr;
   10870              :     }
   10871              : 
   10872              :   return NULL_TREE;
   10873              : }
   10874              : 
   10875              : /* An attach or detach operation depends directly on the address being
   10876              :    attached/detached.  Return that address, or none if there are no
   10877              :    attachments/detachments.  */
   10878              : 
   10879              : static tree
   10880        17028 : omp_get_attachment (omp_mapping_group *grp)
   10881              : {
   10882        17028 :   tree node = *grp->grp_start;
   10883              : 
   10884        17028 :   switch (OMP_CLAUSE_MAP_KIND (node))
   10885              :     {
   10886        14057 :     case GOMP_MAP_TO:
   10887        14057 :     case GOMP_MAP_FROM:
   10888        14057 :     case GOMP_MAP_TOFROM:
   10889        14057 :     case GOMP_MAP_ALWAYS_FROM:
   10890        14057 :     case GOMP_MAP_ALWAYS_TO:
   10891        14057 :     case GOMP_MAP_ALWAYS_TOFROM:
   10892        14057 :     case GOMP_MAP_FORCE_FROM:
   10893        14057 :     case GOMP_MAP_FORCE_TO:
   10894        14057 :     case GOMP_MAP_FORCE_TOFROM:
   10895        14057 :     case GOMP_MAP_FORCE_PRESENT:
   10896        14057 :     case GOMP_MAP_PRESENT_ALLOC:
   10897        14057 :     case GOMP_MAP_PRESENT_FROM:
   10898        14057 :     case GOMP_MAP_PRESENT_TO:
   10899        14057 :     case GOMP_MAP_PRESENT_TOFROM:
   10900        14057 :     case GOMP_MAP_ALWAYS_PRESENT_FROM:
   10901        14057 :     case GOMP_MAP_ALWAYS_PRESENT_TO:
   10902        14057 :     case GOMP_MAP_ALWAYS_PRESENT_TOFROM:
   10903        14057 :     case GOMP_MAP_ALLOC:
   10904        14057 :     case GOMP_MAP_RELEASE:
   10905        14057 :     case GOMP_MAP_DELETE:
   10906        14057 :     case GOMP_MAP_FORCE_ALLOC:
   10907        14057 :       if (node == grp->grp_end)
   10908              :         return NULL_TREE;
   10909              : 
   10910         6659 :       node = OMP_CLAUSE_CHAIN (node);
   10911         6659 :       if (node && omp_map_clause_descriptor_p (node))
   10912              :         {
   10913         1038 :           gcc_assert (node != grp->grp_end);
   10914         1038 :           node = OMP_CLAUSE_CHAIN (node);
   10915              :         }
   10916         6659 :       if (node)
   10917         6659 :         switch (OMP_CLAUSE_MAP_KIND (node))
   10918              :           {
   10919              :           case GOMP_MAP_POINTER:
   10920              :           case GOMP_MAP_ALWAYS_POINTER:
   10921              :           case GOMP_MAP_FIRSTPRIVATE_POINTER:
   10922              :           case GOMP_MAP_FIRSTPRIVATE_REFERENCE:
   10923              :           case GOMP_MAP_POINTER_TO_ZERO_LENGTH_ARRAY_SECTION:
   10924              :           case GOMP_MAP_USES_ALLOCATORS:
   10925              :             return NULL_TREE;
   10926              : 
   10927         2559 :           case GOMP_MAP_ATTACH_DETACH:
   10928         2559 :           case GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION:
   10929         2559 :           case GOMP_MAP_DETACH:
   10930         2559 :             return OMP_CLAUSE_DECL (node);
   10931              : 
   10932            0 :           default:
   10933            0 :             internal_error ("unexpected mapping node");
   10934              :           }
   10935            0 :       return error_mark_node;
   10936              : 
   10937            0 :     case GOMP_MAP_TO_PSET:
   10938            0 :       gcc_assert (node != grp->grp_end);
   10939            0 :       node = OMP_CLAUSE_CHAIN (node);
   10940            0 :       if (OMP_CLAUSE_MAP_KIND (node) == GOMP_MAP_ATTACH
   10941            0 :           || OMP_CLAUSE_MAP_KIND (node) == GOMP_MAP_DETACH)
   10942            0 :         return OMP_CLAUSE_DECL (node);
   10943              :       else
   10944            0 :         internal_error ("unexpected mapping node");
   10945              :       return error_mark_node;
   10946              : 
   10947          541 :     case GOMP_MAP_ATTACH:
   10948          541 :     case GOMP_MAP_DETACH:
   10949          541 :       node = OMP_CLAUSE_CHAIN (node);
   10950          541 :       if (!node || *grp->grp_start == grp->grp_end)
   10951          541 :         return OMP_CLAUSE_DECL (*grp->grp_start);
   10952            0 :       if (OMP_CLAUSE_MAP_KIND (node) == GOMP_MAP_FIRSTPRIVATE_POINTER
   10953            0 :           || OMP_CLAUSE_MAP_KIND (node) == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
   10954            0 :         return OMP_CLAUSE_DECL (*grp->grp_start);
   10955              :       else
   10956            0 :         internal_error ("unexpected mapping node");
   10957              :       return error_mark_node;
   10958              : 
   10959              :     case GOMP_MAP_STRUCT:
   10960              :     case GOMP_MAP_STRUCT_UNORD:
   10961              :     case GOMP_MAP_FORCE_DEVICEPTR:
   10962              :     case GOMP_MAP_DEVICE_RESIDENT:
   10963              :     case GOMP_MAP_LINK:
   10964              :     case GOMP_MAP_IF_PRESENT:
   10965              :     case GOMP_MAP_FIRSTPRIVATE:
   10966              :     case GOMP_MAP_FIRSTPRIVATE_INT:
   10967              :     case GOMP_MAP_USE_DEVICE_PTR:
   10968              :     case GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION:
   10969              :     case GOMP_MAP_USES_ALLOCATORS:
   10970              :       return NULL_TREE;
   10971              : 
   10972            0 :     default:
   10973            0 :       internal_error ("unexpected mapping node");
   10974              :     }
   10975              : 
   10976              :   return error_mark_node;
   10977              : }
   10978              : 
   10979              : /* Given a pointer START_P to the start of a group of related (e.g. pointer)
   10980              :    mappings, return the chain pointer to the end of that group in the list.  */
   10981              : 
   10982              : static tree *
   10983       112006 : omp_group_last (tree *start_p)
   10984              : {
   10985       112006 :   tree c = *start_p, nc, *grp_last_p = start_p;
   10986              : 
   10987       112006 :   gcc_assert (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP);
   10988              : 
   10989       112006 :   nc = OMP_CLAUSE_CHAIN (c);
   10990              : 
   10991       195325 :   if (!nc || OMP_CLAUSE_CODE (nc) != OMP_CLAUSE_MAP)
   10992              :     return grp_last_p;
   10993              : 
   10994        71006 :   switch (OMP_CLAUSE_MAP_KIND (c))
   10995              :     {
   10996              :     default:
   10997              :       while (nc
   10998       114004 :              && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP
   10999       239467 :              && (OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
   11000       106826 :                  || OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_FIRSTPRIVATE_POINTER
   11001        96133 :                  || OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_ATTACH_DETACH
   11002        83306 :                  || OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_POINTER
   11003        57218 :                  || (OMP_CLAUSE_MAP_KIND (nc)
   11004              :                      == GOMP_MAP_POINTER_TO_ZERO_LENGTH_ARRAY_SECTION)
   11005        57202 :                  || (OMP_CLAUSE_MAP_KIND (nc)
   11006              :                      == GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION)
   11007        56835 :                  || OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_DETACH
   11008        56780 :                  || OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_ALWAYS_POINTER
   11009        54812 :                  || omp_map_clause_descriptor_p (nc)))
   11010              :         {
   11011        67115 :           tree nc2 = OMP_CLAUSE_CHAIN (nc);
   11012        67115 :           if (OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_DETACH)
   11013              :             {
   11014              :               /* In the specific case we're doing "exit data" on an array
   11015              :                  slice of a reference-to-pointer struct component, we will see
   11016              :                  DETACH followed by ATTACH_DETACH here.  We want to treat that
   11017              :                  as a single group. In other cases DETACH might represent a
   11018              :                  stand-alone "detach" clause, so we don't want to consider
   11019              :                  that part of the group.  */
   11020           55 :               if (nc2
   11021           16 :                   && OMP_CLAUSE_CODE (nc2) == OMP_CLAUSE_MAP
   11022           71 :                   && OMP_CLAUSE_MAP_KIND (nc2) == GOMP_MAP_ATTACH_DETACH)
   11023            0 :                 goto consume_two_nodes;
   11024              :               else
   11025              :                 break;
   11026              :             }
   11027        67060 :           if (nc2
   11028        49092 :               && OMP_CLAUSE_CODE (nc2) == OMP_CLAUSE_MAP
   11029        42579 :               && (OMP_CLAUSE_MAP_KIND (nc)
   11030              :                   == GOMP_MAP_POINTER_TO_ZERO_LENGTH_ARRAY_SECTION)
   11031        67076 :               && OMP_CLAUSE_MAP_KIND (nc2) == GOMP_MAP_ATTACH)
   11032              :             {
   11033           16 :             consume_two_nodes:
   11034           16 :               grp_last_p = &OMP_CLAUSE_CHAIN (nc);
   11035           16 :               c = nc2;
   11036           16 :               nc = OMP_CLAUSE_CHAIN (nc2);
   11037              :             }
   11038              :           else
   11039              :             {
   11040        67044 :               grp_last_p = &OMP_CLAUSE_CHAIN (c);
   11041        67044 :               c = nc;
   11042        67044 :               nc = nc2;
   11043              :             }
   11044              :         }
   11045              :       break;
   11046              : 
   11047          302 :     case GOMP_MAP_ATTACH:
   11048          302 :     case GOMP_MAP_DETACH:
   11049              :       /* This is a weird artifact of how directives are parsed: bare attach or
   11050              :          detach clauses get a subsequent (meaningless) FIRSTPRIVATE_POINTER or
   11051              :          FIRSTPRIVATE_REFERENCE node.  FIXME.  */
   11052          302 :       if (nc
   11053          302 :           && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP
   11054          302 :           && (OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
   11055          302 :               || OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_FIRSTPRIVATE_POINTER))
   11056            0 :         grp_last_p = &OMP_CLAUSE_CHAIN (c);
   11057              :       break;
   11058              : 
   11059           39 :     case GOMP_MAP_TO_PSET:
   11060           39 :       if (OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP
   11061           39 :           && (OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_ATTACH
   11062           12 :               || OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_DETACH))
   11063           39 :         grp_last_p = &OMP_CLAUSE_CHAIN (c);
   11064              :       break;
   11065              : 
   11066         5749 :     case GOMP_MAP_STRUCT:
   11067         5749 :     case GOMP_MAP_STRUCT_UNORD:
   11068         5749 :       {
   11069         5749 :         unsigned HOST_WIDE_INT num_mappings
   11070         5749 :           = tree_to_uhwi (OMP_CLAUSE_SIZE (c));
   11071         5749 :         if (OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_FIRSTPRIVATE_POINTER
   11072         5411 :             || OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
   11073        10678 :             || OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_ATTACH_DETACH)
   11074         1134 :           grp_last_p = &OMP_CLAUSE_CHAIN (*grp_last_p);
   11075        14447 :         for (unsigned i = 0; i < num_mappings; i++)
   11076         8698 :           grp_last_p = &OMP_CLAUSE_CHAIN (*grp_last_p);
   11077              :       }
   11078              :       break;
   11079              :     }
   11080              : 
   11081              :   return grp_last_p;
   11082              : }
   11083              : 
   11084              : /* Walk through LIST_P, and return a list of groups of mappings found (e.g.
   11085              :    OMP_CLAUSE_MAP with GOMP_MAP_{TO/FROM/TOFROM} followed by one or two
   11086              :    associated GOMP_MAP_POINTER mappings).  Return a vector of omp_mapping_group
   11087              :    if we have more than one such group, else return NULL.  */
   11088              : 
   11089              : static void
   11090        90236 : omp_gather_mapping_groups_1 (tree *list_p, vec<omp_mapping_group> *groups,
   11091              :                              tree gather_sentinel)
   11092              : {
   11093        90236 :   for (tree *cp = list_p;
   11094       270313 :        *cp && *cp != gather_sentinel;
   11095       180077 :        cp = &OMP_CLAUSE_CHAIN (*cp))
   11096              :     {
   11097       180077 :       if (OMP_CLAUSE_CODE (*cp) != OMP_CLAUSE_MAP)
   11098        99972 :         continue;
   11099              : 
   11100        80105 :       tree *grp_last_p = omp_group_last (cp);
   11101        80105 :       omp_mapping_group grp;
   11102              : 
   11103        80105 :       grp.grp_start = cp;
   11104        80105 :       grp.grp_end = *grp_last_p;
   11105        80105 :       grp.mark = UNVISITED;
   11106        80105 :       grp.sibling = NULL;
   11107        80105 :       grp.deleted = false;
   11108        80105 :       grp.reprocess_struct = false;
   11109        80105 :       grp.fragile = false;
   11110        80105 :       grp.next = NULL;
   11111        80105 :       groups->safe_push (grp);
   11112              : 
   11113        80105 :       cp = grp_last_p;
   11114              :     }
   11115        90236 : }
   11116              : 
   11117              : static vec<omp_mapping_group> *
   11118        89918 : omp_gather_mapping_groups (tree *list_p)
   11119              : {
   11120        89918 :   vec<omp_mapping_group> *groups = new vec<omp_mapping_group> ();
   11121              : 
   11122        89918 :   omp_gather_mapping_groups_1 (list_p, groups, NULL_TREE);
   11123              : 
   11124        89918 :   if (groups->length () > 0)
   11125              :     return groups;
   11126              :   else
   11127              :     {
   11128        44828 :       delete groups;
   11129        44828 :       return NULL;
   11130              :     }
   11131              : }
   11132              : 
   11133              : /* A pointer mapping group GRP may define a block of memory starting at some
   11134              :    base address, and maybe also define a firstprivate pointer or firstprivate
   11135              :    reference that points to that block.  The return value is a node containing
   11136              :    the former, and the *FIRSTPRIVATE pointer is set if we have the latter.
   11137              :    If we define several base pointers, i.e. for a GOMP_MAP_STRUCT mapping,
   11138              :    return the number of consecutive chained nodes in CHAINED.  */
   11139              : 
   11140              : static tree
   11141        80499 : omp_group_base (omp_mapping_group *grp, unsigned int *chained,
   11142              :                 tree *firstprivate)
   11143              : {
   11144        80499 :   tree node = *grp->grp_start;
   11145              : 
   11146        80499 :   *firstprivate = NULL_TREE;
   11147        80499 :   *chained = 1;
   11148              : 
   11149        80499 :   switch (OMP_CLAUSE_MAP_KIND (node))
   11150              :     {
   11151        76290 :     case GOMP_MAP_TO:
   11152        76290 :     case GOMP_MAP_FROM:
   11153        76290 :     case GOMP_MAP_TOFROM:
   11154        76290 :     case GOMP_MAP_ALWAYS_FROM:
   11155        76290 :     case GOMP_MAP_ALWAYS_TO:
   11156        76290 :     case GOMP_MAP_ALWAYS_TOFROM:
   11157        76290 :     case GOMP_MAP_FORCE_FROM:
   11158        76290 :     case GOMP_MAP_FORCE_TO:
   11159        76290 :     case GOMP_MAP_FORCE_TOFROM:
   11160        76290 :     case GOMP_MAP_FORCE_PRESENT:
   11161        76290 :     case GOMP_MAP_PRESENT_ALLOC:
   11162        76290 :     case GOMP_MAP_PRESENT_FROM:
   11163        76290 :     case GOMP_MAP_PRESENT_TO:
   11164        76290 :     case GOMP_MAP_PRESENT_TOFROM:
   11165        76290 :     case GOMP_MAP_ALWAYS_PRESENT_FROM:
   11166        76290 :     case GOMP_MAP_ALWAYS_PRESENT_TO:
   11167        76290 :     case GOMP_MAP_ALWAYS_PRESENT_TOFROM:
   11168        76290 :     case GOMP_MAP_ALLOC:
   11169        76290 :     case GOMP_MAP_RELEASE:
   11170        76290 :     case GOMP_MAP_DELETE:
   11171        76290 :     case GOMP_MAP_FORCE_ALLOC:
   11172        76290 :     case GOMP_MAP_IF_PRESENT:
   11173        76290 :       if (node == grp->grp_end)
   11174              :         return node;
   11175              : 
   11176        34894 :       node = OMP_CLAUSE_CHAIN (node);
   11177        34894 :       if (!node)
   11178            0 :         internal_error ("unexpected mapping node");
   11179        34894 :       if (omp_map_clause_descriptor_p (node))
   11180              :         {
   11181        10754 :           if (node == grp->grp_end)
   11182            0 :             return *grp->grp_start;
   11183        10754 :           node = OMP_CLAUSE_CHAIN (node);
   11184              :         }
   11185        34894 :       switch (OMP_CLAUSE_MAP_KIND (node))
   11186              :         {
   11187        23539 :         case GOMP_MAP_POINTER:
   11188        23539 :         case GOMP_MAP_FIRSTPRIVATE_POINTER:
   11189        23539 :         case GOMP_MAP_FIRSTPRIVATE_REFERENCE:
   11190        23539 :         case GOMP_MAP_POINTER_TO_ZERO_LENGTH_ARRAY_SECTION:
   11191        23539 :           *firstprivate = OMP_CLAUSE_DECL (node);
   11192        23539 :           return *grp->grp_start;
   11193              : 
   11194        11355 :         case GOMP_MAP_ALWAYS_POINTER:
   11195        11355 :         case GOMP_MAP_ATTACH_DETACH:
   11196        11355 :         case GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION:
   11197        11355 :         case GOMP_MAP_DETACH:
   11198        11355 :           return *grp->grp_start;
   11199              : 
   11200            0 :         default:
   11201            0 :           internal_error ("unexpected mapping node");
   11202              :         }
   11203              :       return error_mark_node;
   11204              : 
   11205           26 :     case GOMP_MAP_TO_PSET:
   11206           26 :       gcc_assert (node != grp->grp_end);
   11207           26 :       node = OMP_CLAUSE_CHAIN (node);
   11208           26 :       if (OMP_CLAUSE_MAP_KIND (node) == GOMP_MAP_ATTACH
   11209           26 :           || OMP_CLAUSE_MAP_KIND (node) == GOMP_MAP_DETACH)
   11210              :         return NULL_TREE;
   11211              :       else
   11212            0 :         internal_error ("unexpected mapping node");
   11213              :       return error_mark_node;
   11214              : 
   11215         1109 :     case GOMP_MAP_ATTACH:
   11216         1109 :     case GOMP_MAP_DETACH:
   11217         1109 :       node = OMP_CLAUSE_CHAIN (node);
   11218         1109 :       if (!node || *grp->grp_start == grp->grp_end)
   11219              :         return NULL_TREE;
   11220            0 :       if (OMP_CLAUSE_MAP_KIND (node) == GOMP_MAP_FIRSTPRIVATE_POINTER
   11221            0 :           || OMP_CLAUSE_MAP_KIND (node) == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
   11222              :         {
   11223              :           /* We're mapping the base pointer itself in a bare attach or detach
   11224              :              node.  This is a side effect of how parsing works, and the mapping
   11225              :              will be removed anyway (at least for enter/exit data directives).
   11226              :              We should ignore the mapping here.  FIXME.  */
   11227              :           return NULL_TREE;
   11228              :         }
   11229              :       else
   11230            0 :         internal_error ("unexpected mapping node");
   11231              :       return error_mark_node;
   11232              : 
   11233         2790 :     case GOMP_MAP_STRUCT:
   11234         2790 :     case GOMP_MAP_STRUCT_UNORD:
   11235         2790 :       {
   11236         2790 :         unsigned HOST_WIDE_INT num_mappings
   11237         2790 :           = tree_to_uhwi (OMP_CLAUSE_SIZE (node));
   11238         2790 :         node = OMP_CLAUSE_CHAIN (node);
   11239         2790 :         if (OMP_CLAUSE_MAP_KIND (node) == GOMP_MAP_FIRSTPRIVATE_POINTER
   11240         2790 :             || OMP_CLAUSE_MAP_KIND (node) == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
   11241              :           {
   11242          393 :             *firstprivate = OMP_CLAUSE_DECL (node);
   11243          393 :             node = OMP_CLAUSE_CHAIN (node);
   11244              :           }
   11245         2397 :         else if (OMP_CLAUSE_MAP_KIND (node) == GOMP_MAP_ATTACH_DETACH)
   11246          314 :           node = OMP_CLAUSE_CHAIN (node);
   11247         2790 :         *chained = num_mappings;
   11248         2790 :         return node;
   11249              :       }
   11250              : 
   11251              :     case GOMP_MAP_FORCE_DEVICEPTR:
   11252              :     case GOMP_MAP_DEVICE_RESIDENT:
   11253              :     case GOMP_MAP_LINK:
   11254              :     case GOMP_MAP_FIRSTPRIVATE:
   11255              :     case GOMP_MAP_FIRSTPRIVATE_INT:
   11256              :     case GOMP_MAP_USE_DEVICE_PTR:
   11257              :     case GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION:
   11258              :     case GOMP_MAP_USES_ALLOCATORS:
   11259              :       return NULL_TREE;
   11260              : 
   11261            0 :     case GOMP_MAP_FIRSTPRIVATE_POINTER:
   11262            0 :     case GOMP_MAP_FIRSTPRIVATE_REFERENCE:
   11263            0 :     case GOMP_MAP_POINTER:
   11264            0 :     case GOMP_MAP_ALWAYS_POINTER:
   11265            0 :     case GOMP_MAP_POINTER_TO_ZERO_LENGTH_ARRAY_SECTION:
   11266              :       /* These shouldn't appear by themselves.  */
   11267            0 :       if (!seen_error ())
   11268            0 :         internal_error ("unexpected pointer mapping node");
   11269            0 :       return error_mark_node;
   11270              : 
   11271            0 :     default:
   11272            0 :       gcc_unreachable ();
   11273              :     }
   11274              : 
   11275              :   return error_mark_node;
   11276              : }
   11277              : 
   11278              : /* Given a vector of omp_mapping_groups, build a hash table so we can look up
   11279              :    nodes by tree_operand_hash_no_se.  */
   11280              : 
   11281              : static void
   11282        45574 : omp_index_mapping_groups_1 (hash_map<tree_operand_hash_no_se,
   11283              :                                      omp_mapping_group *> *grpmap,
   11284              :                             vec<omp_mapping_group> *groups,
   11285              :                             tree reindex_sentinel)
   11286              : {
   11287        45574 :   omp_mapping_group *grp;
   11288        45574 :   unsigned int i;
   11289        45574 :   bool reindexing = reindex_sentinel != NULL_TREE, above_hwm = false;
   11290              : 
   11291       126527 :   FOR_EACH_VEC_ELT (*groups, i, grp)
   11292              :     {
   11293        80953 :       if (reindexing && *grp->grp_start == reindex_sentinel)
   11294        80953 :         above_hwm = true;
   11295              : 
   11296        80953 :       if (reindexing && !above_hwm)
   11297        57021 :         continue;
   11298              : 
   11299        80653 :       if (grp->reprocess_struct)
   11300          154 :         continue;
   11301              : 
   11302        80499 :       tree fpp;
   11303        80499 :       unsigned int chained;
   11304        80499 :       tree node = omp_group_base (grp, &chained, &fpp);
   11305              : 
   11306        80499 :       if (node == error_mark_node || (!node && !fpp))
   11307         1419 :         continue;
   11308              : 
   11309              :       for (unsigned j = 0;
   11310       159607 :            node && j < chained;
   11311        80527 :            node = OMP_CLAUSE_CHAIN (node), j++)
   11312              :         {
   11313        80527 :           tree decl = OMP_CLAUSE_DECL (node);
   11314              :           /* Sometimes we see zero-offset MEM_REF instead of INDIRECT_REF,
   11315              :              meaning node-hash lookups don't work.  This is a workaround for
   11316              :              that, but ideally we should just create the INDIRECT_REF at
   11317              :              source instead.  FIXME.  */
   11318        80527 :           if (TREE_CODE (decl) == MEM_REF
   11319        80527 :               && integer_zerop (TREE_OPERAND (decl, 1)))
   11320            0 :             decl = build_fold_indirect_ref (TREE_OPERAND (decl, 0));
   11321              : 
   11322        80527 :           omp_mapping_group **prev = grpmap->get (decl);
   11323              : 
   11324        80527 :           if (prev && *prev == grp)
   11325              :             /* Empty.  */;
   11326        80527 :           else if (prev)
   11327              :             {
   11328              :               /* Mapping the same thing twice is normally diagnosed as an error,
   11329              :                  but can happen under some circumstances, e.g. in pr99928-16.c,
   11330              :                  the directive:
   11331              : 
   11332              :                  #pragma omp target simd reduction(+:a[:3]) \
   11333              :                                          map(always, tofrom: a[:6])
   11334              :                  ...
   11335              : 
   11336              :                  will result in two "a[0]" mappings (of different sizes).  */
   11337              : 
   11338          326 :               grp->sibling = (*prev)->sibling;
   11339          326 :               (*prev)->sibling = grp;
   11340              :             }
   11341              :           else
   11342        80201 :             grpmap->put (decl, grp);
   11343              :         }
   11344              : 
   11345        79080 :       if (!fpp)
   11346        55148 :         continue;
   11347              : 
   11348        23932 :       omp_mapping_group **prev = grpmap->get (fpp);
   11349        23932 :       if (prev && *prev != grp)
   11350              :         {
   11351           12 :           grp->sibling = (*prev)->sibling;
   11352           12 :           (*prev)->sibling = grp;
   11353              :         }
   11354              :       else
   11355        23920 :         grpmap->put (fpp, grp);
   11356              :     }
   11357        45574 : }
   11358              : 
   11359              : static hash_map<tree_operand_hash_no_se, omp_mapping_group *> *
   11360        45090 : omp_index_mapping_groups (vec<omp_mapping_group> *groups)
   11361              : {
   11362        45090 :   hash_map<tree_operand_hash_no_se, omp_mapping_group *> *grpmap
   11363        45090 :     = new hash_map<tree_operand_hash_no_se, omp_mapping_group *>;
   11364              : 
   11365        45090 :   omp_index_mapping_groups_1 (grpmap, groups, NULL_TREE);
   11366              : 
   11367        45090 :   return grpmap;
   11368              : }
   11369              : 
   11370              : /* Rebuild group map from partially-processed clause list (during
   11371              :    omp_build_struct_sibling_lists).  We have already processed nodes up until
   11372              :    a high-water mark (HWM).  This is a bit tricky because the list is being
   11373              :    reordered as it is scanned, but we know:
   11374              : 
   11375              :    1. The list after HWM has not been touched yet, so we can reindex it safely.
   11376              : 
   11377              :    2. The list before and including HWM has been altered, but remains
   11378              :       well-formed throughout the sibling-list building operation.
   11379              : 
   11380              :    so, we can do the reindex operation in two parts, on the processed and
   11381              :    then the unprocessed halves of the list.  */
   11382              : 
   11383              : static hash_map<tree_operand_hash_no_se, omp_mapping_group *> *
   11384          318 : omp_reindex_mapping_groups (tree *list_p,
   11385              :                             vec<omp_mapping_group> *groups,
   11386              :                             vec<omp_mapping_group> *processed_groups,
   11387              :                             tree sentinel)
   11388              : {
   11389          318 :   hash_map<tree_operand_hash_no_se, omp_mapping_group *> *grpmap
   11390          318 :     = new hash_map<tree_operand_hash_no_se, omp_mapping_group *>;
   11391              : 
   11392          318 :   processed_groups->truncate (0);
   11393              : 
   11394          318 :   omp_gather_mapping_groups_1 (list_p, processed_groups, sentinel);
   11395          318 :   omp_index_mapping_groups_1 (grpmap, processed_groups, NULL_TREE);
   11396          318 :   if (sentinel)
   11397          166 :     omp_index_mapping_groups_1 (grpmap, groups, sentinel);
   11398              : 
   11399          318 :   return grpmap;
   11400              : }
   11401              : 
   11402              : /* Find the immediately-containing struct for a component ref (etc.)
   11403              :    expression EXPR.  */
   11404              : 
   11405              : static tree
   11406        48356 : omp_containing_struct (tree expr)
   11407              : {
   11408        48356 :   tree expr0 = expr;
   11409              : 
   11410        48356 :   STRIP_NOPS (expr);
   11411              : 
   11412              :   /* Note: don't strip NOPs unless we're also stripping off array refs or a
   11413              :      component ref.  */
   11414        48356 :   if (TREE_CODE (expr) != ARRAY_REF && TREE_CODE (expr) != COMPONENT_REF)
   11415              :     return expr0;
   11416              : 
   11417        27442 :   while (TREE_CODE (expr) == ARRAY_REF)
   11418         3727 :     expr = TREE_OPERAND (expr, 0);
   11419              : 
   11420        23715 :   if (TREE_CODE (expr) == COMPONENT_REF)
   11421        21213 :     expr = TREE_OPERAND (expr, 0);
   11422              : 
   11423              :   return expr;
   11424              : }
   11425              : 
   11426              : /* Return TRUE if DECL describes a component that is part of a whole structure
   11427              :    that is mapped elsewhere in GRPMAP.  *MAPPED_BY_GROUP is set to the group
   11428              :    that maps that structure, if present.  */
   11429              : 
   11430              : static bool
   11431        25939 : omp_mapped_by_containing_struct (hash_map<tree_operand_hash_no_se,
   11432              :                                           omp_mapping_group *> *grpmap,
   11433              :                                  tree decl,
   11434              :                                  omp_mapping_group **mapped_by_group)
   11435              : {
   11436        25939 :   tree wsdecl = NULL_TREE;
   11437              : 
   11438        25939 :   *mapped_by_group = NULL;
   11439              : 
   11440        70773 :   while (true)
   11441              :     {
   11442        48356 :       wsdecl = omp_containing_struct (decl);
   11443        48356 :       if (wsdecl == decl)
   11444              :         break;
   11445        23715 :       omp_mapping_group **wholestruct = grpmap->get (wsdecl);
   11446        23715 :       if (!wholestruct
   11447        21087 :           && TREE_CODE (wsdecl) == MEM_REF
   11448        23716 :           && integer_zerop (TREE_OPERAND (wsdecl, 1)))
   11449              :         {
   11450            1 :           tree deref = TREE_OPERAND (wsdecl, 0);
   11451            1 :           deref = build_fold_indirect_ref (deref);
   11452            1 :           wholestruct = grpmap->get (deref);
   11453              :         }
   11454        23715 :       if (wholestruct)
   11455              :         {
   11456              :           /* An intermediate descriptor should not match here because the
   11457              :              pointee is actually not mapped by this group -- it is just a
   11458              :              zero-length alloc.  */
   11459         2628 :           tree desc = OMP_CLAUSE_CHAIN (*(*wholestruct)->grp_start);
   11460         2628 :           if (desc != NULL_TREE && omp_map_clause_descriptor_p (desc))
   11461         1330 :             goto next;
   11462         1298 :           *mapped_by_group = *wholestruct;
   11463         1298 :           return true;
   11464              :         }
   11465        21087 :     next:
   11466        22417 :       decl = wsdecl;
   11467        22417 :     }
   11468              : 
   11469              :   return false;
   11470              : }
   11471              : 
   11472              : /* Helper function for omp_tsort_mapping_groups.  Returns TRUE on success, or
   11473              :    FALSE on error.  */
   11474              : 
   11475              : static bool
   11476        20380 : omp_tsort_mapping_groups_1 (omp_mapping_group ***outlist,
   11477              :                             vec<omp_mapping_group> *groups,
   11478              :                             hash_map<tree_operand_hash_no_se,
   11479              :                                      omp_mapping_group *> *grpmap,
   11480              :                             omp_mapping_group *grp)
   11481              : {
   11482        20380 :   if (grp->mark == PERMANENT)
   11483              :     return true;
   11484        17028 :   if (grp->mark == TEMPORARY)
   11485              :     {
   11486            0 :       fprintf (stderr, "when processing group:\n");
   11487            0 :       debug_mapping_group (grp);
   11488            0 :       internal_error ("base pointer cycle detected");
   11489              :       return false;
   11490              :     }
   11491        17028 :   grp->mark = TEMPORARY;
   11492              : 
   11493        17028 :   tree attaches_to = omp_get_attachment (grp);
   11494              : 
   11495        17028 :   if (attaches_to)
   11496              :     {
   11497         3100 :       omp_mapping_group **basep = grpmap->get (attaches_to);
   11498              : 
   11499         3100 :       if (basep && *basep != grp)
   11500              :         {
   11501         2924 :           for (omp_mapping_group *w = *basep; w; w = w->sibling)
   11502         1462 :             if (!omp_tsort_mapping_groups_1 (outlist, groups, grpmap, w))
   11503              :               return false;
   11504              :         }
   11505              :     }
   11506              : 
   11507        17028 :   tree decl = OMP_CLAUSE_DECL (*grp->grp_start);
   11508              : 
   11509        23251 :   while (decl)
   11510              :     {
   11511        23251 :       tree base = omp_get_base_pointer (decl);
   11512              : 
   11513        23251 :       if (!base)
   11514              :         break;
   11515              : 
   11516         9000 :       omp_mapping_group **innerp = grpmap->get (base);
   11517         9000 :       omp_mapping_group *wholestruct;
   11518              : 
   11519              :       /* We should treat whole-structure mappings as if all (pointer, in this
   11520              :          case) members are mapped as individual list items.  Check if we have
   11521              :          such a whole-structure mapping, if we don't have an explicit reference
   11522              :          to the pointer member itself.  */
   11523         9000 :       if (!innerp
   11524         4186 :           && TREE_CODE (base) == COMPONENT_REF
   11525        11246 :           && omp_mapped_by_containing_struct (grpmap, base, &wholestruct))
   11526              :         innerp = &wholestruct;
   11527              : 
   11528         9000 :       if (innerp && *innerp != grp)
   11529              :         {
   11530         5566 :           for (omp_mapping_group *w = *innerp; w; w = w->sibling)
   11531         2789 :             if (!omp_tsort_mapping_groups_1 (outlist, groups, grpmap, w))
   11532            0 :               return false;
   11533              :           break;
   11534              :         }
   11535              : 
   11536         6223 :       decl = base;
   11537              :     }
   11538              : 
   11539        17028 :   grp->mark = PERMANENT;
   11540              : 
   11541              :   /* Emit grp to output list.  */
   11542              : 
   11543        17028 :   **outlist = grp;
   11544        17028 :   *outlist = &grp->next;
   11545              : 
   11546        17028 :   return true;
   11547              : }
   11548              : 
   11549              : /* Topologically sort GROUPS, so that OMP 5.0-defined base pointers come
   11550              :    before mappings that use those pointers.  This is an implementation of the
   11551              :    depth-first search algorithm, described e.g. at:
   11552              : 
   11553              :      https://en.wikipedia.org/wiki/Topological_sorting
   11554              : */
   11555              : 
   11556              : static omp_mapping_group *
   11557         8655 : omp_tsort_mapping_groups (vec<omp_mapping_group> *groups,
   11558              :                           hash_map<tree_operand_hash_no_se, omp_mapping_group *>
   11559              :                             *grpmap,
   11560              :                           bool enter_exit_data)
   11561              : {
   11562         8655 :   omp_mapping_group *grp, *outlist = NULL, **cursor;
   11563         8655 :   unsigned int i;
   11564         8655 :   bool saw_runtime_implicit = false;
   11565              : 
   11566         8655 :   cursor = &outlist;
   11567              : 
   11568        25683 :   FOR_EACH_VEC_ELT (*groups, i, grp)
   11569              :     {
   11570        17028 :       if (grp->mark != PERMANENT)
   11571              :         {
   11572        16153 :           if (OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (*grp->grp_start))
   11573              :             {
   11574          683 :               saw_runtime_implicit = true;
   11575          683 :               continue;
   11576              :             }
   11577        15470 :           if (!omp_tsort_mapping_groups_1 (&cursor, groups, grpmap, grp))
   11578              :             return NULL;
   11579              :         }
   11580              :     }
   11581              : 
   11582         8655 :   if (!saw_runtime_implicit)
   11583         8373 :     return outlist;
   11584              : 
   11585         1634 :   FOR_EACH_VEC_ELT (*groups, i, grp)
   11586              :     {
   11587         1352 :       if (grp->mark != PERMANENT
   11588         1352 :           && OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (*grp->grp_start))
   11589              :         {
   11590              :           /* Clear the flag for enter/exit data because it is currently
   11591              :              meaningless for those operations in libgomp.  */
   11592          659 :           if (enter_exit_data)
   11593          446 :             OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (*grp->grp_start) = 0;
   11594              : 
   11595          659 :           if (!omp_tsort_mapping_groups_1 (&cursor, groups, grpmap, grp))
   11596              :             return NULL;
   11597              :         }
   11598              :     }
   11599              : 
   11600          282 :   return outlist;
   11601              : }
   11602              : 
   11603              : /* Split INLIST into three parts:
   11604              : 
   11605              :      - "present" alloc/to/from groups
   11606              :      - other to/from groups
   11607              :      - other alloc/release/delete groups
   11608              : 
   11609              :    These sub-lists are then concatenated together to form the final list.
   11610              :    Each sub-list retains the order of the original list.
   11611              :    Note that ATTACH nodes are later moved to the end of the list in
   11612              :    gimplify_adjust_omp_clauses, for target regions.  */
   11613              : 
   11614              : static omp_mapping_group *
   11615         8655 : omp_segregate_mapping_groups (omp_mapping_group *inlist)
   11616              : {
   11617         8655 :   omp_mapping_group *ard_groups = NULL, *tf_groups = NULL;
   11618         8655 :   omp_mapping_group *p_groups = NULL;
   11619         8655 :   omp_mapping_group **ard_tail = &ard_groups, **tf_tail = &tf_groups;
   11620         8655 :   omp_mapping_group **p_tail = &p_groups;
   11621              : 
   11622        25683 :   for (omp_mapping_group *w = inlist; w;)
   11623              :     {
   11624        17028 :       tree c = *w->grp_start;
   11625        17028 :       omp_mapping_group *next = w->next;
   11626              : 
   11627        17028 :       gcc_assert (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP);
   11628              : 
   11629        17028 :       switch (OMP_CLAUSE_MAP_KIND (c))
   11630              :         {
   11631         1939 :         case GOMP_MAP_ALLOC:
   11632         1939 :         case GOMP_MAP_RELEASE:
   11633         1939 :         case GOMP_MAP_DELETE:
   11634         1939 :           *ard_tail = w;
   11635         1939 :           w->next = NULL;
   11636         1939 :           ard_tail = &w->next;
   11637         1939 :           break;
   11638              : 
   11639              :         /* These map types are all semantically identical, so are moved into a
   11640              :            single group.  They will each be changed into GOMP_MAP_FORCE_PRESENT
   11641              :            in gimplify_adjust_omp_clauses.  */
   11642          151 :         case GOMP_MAP_PRESENT_ALLOC:
   11643          151 :         case GOMP_MAP_PRESENT_FROM:
   11644          151 :         case GOMP_MAP_PRESENT_TO:
   11645          151 :         case GOMP_MAP_PRESENT_TOFROM:
   11646          151 :           *p_tail = w;
   11647          151 :           w->next = NULL;
   11648          151 :           p_tail = &w->next;
   11649          151 :           break;
   11650              : 
   11651        14938 :         default:
   11652        14938 :           *tf_tail = w;
   11653        14938 :           w->next = NULL;
   11654        14938 :           tf_tail = &w->next;
   11655              :         }
   11656              : 
   11657        17028 :       w = next;
   11658              :     }
   11659              : 
   11660              :   /* Now splice the lists together...  */
   11661         8655 :   *tf_tail = ard_groups;
   11662         8655 :   *p_tail = tf_groups;
   11663              : 
   11664         8655 :   return p_groups;
   11665              : }
   11666              : 
   11667              : /* Given a list LIST_P containing groups of mappings given by GROUPS, reorder
   11668              :    those groups based on the output list of omp_tsort_mapping_groups --
   11669              :    singly-linked, threaded through each element's NEXT pointer starting at
   11670              :    HEAD.  Each list element appears exactly once in that linked list.
   11671              : 
   11672              :    Each element of GROUPS may correspond to one or several mapping nodes.
   11673              :    Node groups are kept together, and in the reordered list, the positions of
   11674              :    the original groups are reused for the positions of the reordered list.
   11675              :    Hence if we have e.g.
   11676              : 
   11677              :      {to ptr ptr} firstprivate {tofrom ptr} ...
   11678              :       ^             ^           ^
   11679              :       first group  non-"map"    second group
   11680              : 
   11681              :    and say the second group contains a base pointer for the first so must be
   11682              :    moved before it, the resulting list will contain:
   11683              : 
   11684              :      {tofrom ptr} firstprivate {to ptr ptr} ...
   11685              :       ^ prev. second group      ^ prev. first group
   11686              : */
   11687              : 
   11688              : static tree *
   11689         8655 : omp_reorder_mapping_groups (vec<omp_mapping_group> *groups,
   11690              :                             omp_mapping_group *head,
   11691              :                             tree *list_p)
   11692              : {
   11693         8655 :   omp_mapping_group *grp;
   11694         8655 :   unsigned int i;
   11695         8655 :   unsigned numgroups = groups->length ();
   11696         8655 :   auto_vec<tree> old_heads (numgroups);
   11697         8655 :   auto_vec<tree *> old_headps (numgroups);
   11698         8655 :   auto_vec<tree> new_heads (numgroups);
   11699         8655 :   auto_vec<tree> old_succs (numgroups);
   11700         8655 :   bool map_at_start = (list_p == (*groups)[0].grp_start);
   11701              : 
   11702         8655 :   tree *new_grp_tail = NULL;
   11703              : 
   11704              :   /* Stash the start & end nodes of each mapping group before we start
   11705              :      modifying the list.  */
   11706        25683 :   FOR_EACH_VEC_ELT (*groups, i, grp)
   11707              :     {
   11708        17028 :       old_headps.quick_push (grp->grp_start);
   11709        17028 :       old_heads.quick_push (*grp->grp_start);
   11710        17028 :       old_succs.quick_push (OMP_CLAUSE_CHAIN (grp->grp_end));
   11711              :     }
   11712              : 
   11713              :   /* And similarly, the heads of the groups in the order we want to rearrange
   11714              :      the list to.  */
   11715        25683 :   for (omp_mapping_group *w = head; w; w = w->next)
   11716        17028 :     new_heads.quick_push (*w->grp_start);
   11717              : 
   11718        25683 :   FOR_EACH_VEC_ELT (*groups, i, grp)
   11719              :     {
   11720        17028 :       gcc_assert (head);
   11721              : 
   11722        17028 :       if (new_grp_tail && old_succs[i - 1] == old_heads[i])
   11723              :         {
   11724              :           /* a {b c d} {e f g} h i j   (original)
   11725              :              -->
   11726              :              a {k l m} {e f g} h i j   (inserted new group on last iter)
   11727              :              -->
   11728              :              a {k l m} {n o p} h i j   (this time, chain last group to new one)
   11729              :                       ^new_grp_tail
   11730              :           */
   11731         7751 :           *new_grp_tail = new_heads[i];
   11732              :         }
   11733         9277 :       else if (new_grp_tail)
   11734              :         {
   11735              :           /* a {b c d} e {f g h} i j k   (original)
   11736              :              -->
   11737              :              a {l m n} e {f g h} i j k   (gap after last iter's group)
   11738              :              -->
   11739              :              a {l m n} e {o p q} h i j   (chain last group to old successor)
   11740              :                       ^new_grp_tail
   11741              :            */
   11742          622 :           *new_grp_tail = old_succs[i - 1];
   11743          622 :           *old_headps[i] = new_heads[i];
   11744              :         }
   11745              :       else
   11746              :         {
   11747              :           /* The first inserted group -- point to new group, and leave end
   11748              :              open.
   11749              :              a {b c d} e f
   11750              :              -->
   11751              :              a {g h i...
   11752              :           */
   11753         8655 :           *grp->grp_start = new_heads[i];
   11754              :         }
   11755              : 
   11756        17028 :       new_grp_tail = &OMP_CLAUSE_CHAIN (head->grp_end);
   11757              : 
   11758        17028 :       head = head->next;
   11759              :     }
   11760              : 
   11761         8655 :   if (new_grp_tail)
   11762         8655 :     *new_grp_tail = old_succs[numgroups - 1];
   11763              : 
   11764         8655 :   gcc_assert (!head);
   11765              : 
   11766        12163 :   return map_at_start ? (*groups)[0].grp_start : list_p;
   11767         8655 : }
   11768              : 
   11769              : /* DECL is supposed to have lastprivate semantics in the outer contexts
   11770              :    of combined/composite constructs, starting with OCTX.
   11771              :    Add needed lastprivate, shared or map clause if no data sharing or
   11772              :    mapping clause are present.  IMPLICIT_P is true if it is an implicit
   11773              :    clause (IV on simd), in which case the lastprivate will not be
   11774              :    copied to some constructs.  */
   11775              : 
   11776              : static void
   11777        14663 : omp_lastprivate_for_combined_outer_constructs (struct gimplify_omp_ctx *octx,
   11778              :                                                tree decl, bool implicit_p)
   11779              : {
   11780        14663 :   struct gimplify_omp_ctx *orig_octx = octx;
   11781        26997 :   for (; octx; octx = octx->outer_context)
   11782              :     {
   11783        25922 :       if ((octx->region_type == ORT_COMBINED_PARALLEL
   11784        20882 :            || (octx->region_type & ORT_COMBINED_TEAMS) == ORT_COMBINED_TEAMS)
   11785        25353 :           && splay_tree_lookup (octx->variables,
   11786              :                                 (splay_tree_key) decl) == NULL)
   11787              :         {
   11788         3891 :           omp_add_variable (octx, decl, GOVD_SHARED | GOVD_SEEN);
   11789         3891 :           continue;
   11790              :         }
   11791        18388 :       if ((octx->region_type & ORT_TASK) != 0
   11792          575 :           && octx->combined_loop
   11793        18669 :           && splay_tree_lookup (octx->variables,
   11794              :                                 (splay_tree_key) decl) == NULL)
   11795              :         {
   11796          248 :           omp_add_variable (octx, decl, GOVD_LASTPRIVATE | GOVD_SEEN);
   11797          248 :           continue;
   11798              :         }
   11799        22729 :       if (implicit_p
   11800        13632 :           && octx->region_type == ORT_WORKSHARE
   11801        10414 :           && octx->combined_loop
   11802        10309 :           && splay_tree_lookup (octx->variables,
   11803              :                                 (splay_tree_key) decl) == NULL
   11804        10309 :           && octx->outer_context
   11805         8916 :           && octx->outer_context->region_type == ORT_COMBINED_PARALLEL
   11806        22736 :           && splay_tree_lookup (octx->outer_context->variables,
   11807              :                                 (splay_tree_key) decl) == NULL)
   11808              :         {
   11809         4837 :           octx = octx->outer_context;
   11810         4837 :           omp_add_variable (octx, decl, GOVD_LASTPRIVATE | GOVD_SEEN);
   11811         4837 :           continue;
   11812              :         }
   11813         5569 :       if ((octx->region_type == ORT_WORKSHARE || octx->region_type == ORT_ACC)
   11814         7486 :           && octx->combined_loop
   11815         7299 :           && splay_tree_lookup (octx->variables,
   11816              :                                 (splay_tree_key) decl) == NULL
   11817        19084 :           && !omp_check_private (octx, decl, false))
   11818              :         {
   11819         3358 :           omp_add_variable (octx, decl, GOVD_LASTPRIVATE | GOVD_SEEN);
   11820         3358 :           continue;
   11821              :         }
   11822         9697 :       if (octx->region_type == ORT_COMBINED_TARGET)
   11823              :         {
   11824         2010 :           splay_tree_node n = splay_tree_lookup (octx->variables,
   11825              :                                                  (splay_tree_key) decl);
   11826         2010 :           if (n == NULL)
   11827              :             {
   11828         1988 :               omp_add_variable (octx, decl, GOVD_MAP | GOVD_SEEN);
   11829         1988 :               octx = octx->outer_context;
   11830              :             }
   11831           22 :           else if (!implicit_p
   11832           22 :                    && (n->value & GOVD_FIRSTPRIVATE_IMPLICIT))
   11833              :             {
   11834           12 :               n->value &= ~(GOVD_FIRSTPRIVATE
   11835              :                             | GOVD_FIRSTPRIVATE_IMPLICIT
   11836              :                             | GOVD_EXPLICIT);
   11837           12 :               omp_add_variable (octx, decl, GOVD_MAP | GOVD_SEEN);
   11838           12 :               octx = octx->outer_context;
   11839              :             }
   11840              :         }
   11841              :       break;
   11842              :     }
   11843        14663 :   if (octx && (implicit_p || octx != orig_octx))
   11844         4048 :     omp_notice_variable (octx, decl, true);
   11845        14663 : }
   11846              : 
   11847              : /* We might have indexed several groups for DECL, e.g. a "TO" mapping and also
   11848              :    a "FIRSTPRIVATE" mapping.  Return the one that isn't firstprivate, etc.  */
   11849              : 
   11850              : static omp_mapping_group *
   11851         6008 : omp_get_nonfirstprivate_group (hash_map<tree_operand_hash_no_se,
   11852              :                                         omp_mapping_group *> *grpmap,
   11853              :                                tree decl, bool allow_deleted = false)
   11854              : {
   11855         6008 :   omp_mapping_group **to_group_p = grpmap->get (decl);
   11856              : 
   11857         6008 :   if (!to_group_p)
   11858              :     return NULL;
   11859              : 
   11860         2475 :   omp_mapping_group *to_group = *to_group_p;
   11861              : 
   11862         3828 :   for (; to_group; to_group = to_group->sibling)
   11863              :     {
   11864         2512 :       tree grp_end = to_group->grp_end;
   11865         2512 :       switch (OMP_CLAUSE_MAP_KIND (grp_end))
   11866              :         {
   11867              :         case GOMP_MAP_FIRSTPRIVATE_POINTER:
   11868              :         case GOMP_MAP_FIRSTPRIVATE_REFERENCE:
   11869              :           break;
   11870              : 
   11871         1161 :         default:
   11872         1161 :           if (allow_deleted || !to_group->deleted)
   11873              :             return to_group;
   11874              :         }
   11875              :     }
   11876              : 
   11877              :   return NULL;
   11878              : }
   11879              : 
   11880              : /* Return TRUE if the directive (whose clauses are described by the hash table
   11881              :    of mapping groups, GRPMAP) maps DECL explicitly.  If TO_SPECIFICALLY is
   11882              :    true, only count TO mappings.  If ALLOW_DELETED is true, ignore the
   11883              :    "deleted" flag for groups.  If CONTAINED_IN_STRUCT is true, also return
   11884              :    TRUE if DECL is mapped as a member of a whole-struct mapping.  */
   11885              : 
   11886              : static bool
   11887         4450 : omp_directive_maps_explicitly (hash_map<tree_operand_hash_no_se,
   11888              :                                         omp_mapping_group *> *grpmap,
   11889              :                                tree decl, omp_mapping_group **base_group,
   11890              :                                bool to_specifically, bool allow_deleted,
   11891              :                                bool contained_in_struct)
   11892              : {
   11893         4450 :   omp_mapping_group *decl_group
   11894         4450 :     = omp_get_nonfirstprivate_group (grpmap, decl, allow_deleted);
   11895              : 
   11896         4450 :   *base_group = NULL;
   11897              : 
   11898         4450 :   if (decl_group)
   11899              :     {
   11900         1042 :       tree grp_first = *decl_group->grp_start;
   11901              :       /* We might be called during omp_build_struct_sibling_lists, when
   11902              :          GOMP_MAP_STRUCT might have been inserted at the start of the group.
   11903              :          Skip over that, and also possibly the node after it.  */
   11904         1042 :       if (OMP_CLAUSE_MAP_KIND (grp_first) == GOMP_MAP_STRUCT
   11905         1042 :           || OMP_CLAUSE_MAP_KIND (grp_first) == GOMP_MAP_STRUCT_UNORD)
   11906              :         {
   11907            6 :           grp_first = OMP_CLAUSE_CHAIN (grp_first);
   11908            6 :           if (OMP_CLAUSE_MAP_KIND (grp_first) == GOMP_MAP_FIRSTPRIVATE_POINTER
   11909            6 :               || (OMP_CLAUSE_MAP_KIND (grp_first)
   11910              :                   == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
   11911           12 :               || OMP_CLAUSE_MAP_KIND (grp_first) == GOMP_MAP_ATTACH_DETACH)
   11912            0 :             grp_first = OMP_CLAUSE_CHAIN (grp_first);
   11913              :         }
   11914         1042 :       enum gomp_map_kind first_kind = OMP_CLAUSE_MAP_KIND (grp_first);
   11915         1042 :       if (!to_specifically
   11916          592 :           || GOMP_MAP_COPY_TO_P (first_kind)
   11917          382 :           || first_kind == GOMP_MAP_ALLOC)
   11918              :         {
   11919          960 :           *base_group = decl_group;
   11920          960 :           return true;
   11921              :         }
   11922              :     }
   11923              : 
   11924         3490 :   if (contained_in_struct
   11925         3490 :       && omp_mapped_by_containing_struct (grpmap, decl, base_group))
   11926              :     return true;
   11927              : 
   11928              :   return false;
   11929              : }
   11930              : 
   11931              : /* If we have mappings INNER and OUTER, where INNER is a component access and
   11932              :    OUTER is a mapping of the whole containing struct, check that the mappings
   11933              :    are compatible.  We'll be deleting the inner mapping, so we need to make
   11934              :    sure the outer mapping does (at least) the same transfers to/from the device
   11935              :    as the inner mapping.  */
   11936              : 
   11937              : bool
   11938          172 : omp_check_mapping_compatibility (location_t loc,
   11939              :                                  omp_mapping_group *outer,
   11940              :                                  omp_mapping_group *inner)
   11941              : {
   11942          172 :   tree first_outer = *outer->grp_start, first_inner = *inner->grp_start;
   11943              : 
   11944          172 :   gcc_assert (OMP_CLAUSE_CODE (first_outer) == OMP_CLAUSE_MAP);
   11945          172 :   gcc_assert (OMP_CLAUSE_CODE (first_inner) == OMP_CLAUSE_MAP);
   11946              : 
   11947          172 :   enum gomp_map_kind outer_kind = OMP_CLAUSE_MAP_KIND (first_outer);
   11948          172 :   enum gomp_map_kind inner_kind = OMP_CLAUSE_MAP_KIND (first_inner);
   11949              : 
   11950          172 :   if (outer_kind == inner_kind)
   11951              :     return true;
   11952              : 
   11953           70 :   switch (outer_kind)
   11954              :     {
   11955            0 :     case GOMP_MAP_ALWAYS_TO:
   11956            0 :       if (inner_kind == GOMP_MAP_FORCE_PRESENT
   11957            0 :           || inner_kind == GOMP_MAP_ALLOC
   11958            0 :           || inner_kind == GOMP_MAP_TO)
   11959              :         return true;
   11960              :       break;
   11961              : 
   11962            0 :     case GOMP_MAP_ALWAYS_FROM:
   11963            0 :       if (inner_kind == GOMP_MAP_FORCE_PRESENT
   11964            0 :           || inner_kind == GOMP_MAP_RELEASE
   11965              :           || inner_kind == GOMP_MAP_FROM)
   11966              :         return true;
   11967              :       break;
   11968              : 
   11969           10 :     case GOMP_MAP_TO:
   11970           10 :       if (inner_kind == GOMP_MAP_FORCE_PRESENT
   11971           10 :           || inner_kind == GOMP_MAP_ALLOC)
   11972              :         return true;
   11973              :       break;
   11974              : 
   11975            8 :     case GOMP_MAP_FROM:
   11976            8 :       if (inner_kind == GOMP_MAP_RELEASE
   11977            8 :           || inner_kind == GOMP_MAP_FORCE_PRESENT)
   11978              :         return true;
   11979              :       break;
   11980              : 
   11981           32 :     case GOMP_MAP_ALWAYS_TOFROM:
   11982           32 :     case GOMP_MAP_TOFROM:
   11983           32 :       if (inner_kind == GOMP_MAP_FORCE_PRESENT
   11984           32 :           || inner_kind == GOMP_MAP_ALLOC
   11985              :           || inner_kind == GOMP_MAP_TO
   11986           24 :           || inner_kind == GOMP_MAP_FROM
   11987           12 :           || inner_kind == GOMP_MAP_TOFROM)
   11988              :         return true;
   11989              :       break;
   11990              : 
   11991           28 :     default:
   11992           28 :       ;
   11993              :     }
   11994              : 
   11995           84 :   error_at (loc, "data movement for component %qE is not compatible with "
   11996           28 :             "movement for struct %qE", OMP_CLAUSE_DECL (first_inner),
   11997           28 :             OMP_CLAUSE_DECL (first_outer));
   11998              : 
   11999           28 :   return false;
   12000              : }
   12001              : 
   12002              : /* This function handles several cases where clauses on a mapping directive
   12003              :    can interact with each other.
   12004              : 
   12005              :    If we have a FIRSTPRIVATE_POINTER node and we're also mapping the pointer
   12006              :    on the same directive, change the mapping of the first node to
   12007              :    ATTACH_DETACH.  We should have detected that this will happen already in
   12008              :    c-omp.cc:c_omp_adjust_map_clauses and marked the appropriate decl
   12009              :    as addressable.  (If we didn't, bail out.)
   12010              : 
   12011              :    If we have a FIRSTPRIVATE_REFERENCE (for a reference to pointer) and we're
   12012              :    mapping the base pointer also, we may need to change the mapping type to
   12013              :    ATTACH_DETACH and synthesize an alloc node for the reference itself.
   12014              : 
   12015              :    If we have an ATTACH_DETACH node, this is an array section with a pointer
   12016              :    base.  If we're mapping the base on the same directive too, we can drop its
   12017              :    mapping.  However, if we have a reference to pointer, make other appropriate
   12018              :    adjustments to the mapping nodes instead.
   12019              : 
   12020              :    If we have an ATTACH_DETACH node with a Fortran pointer-set (array
   12021              :    descriptor) mapping for a derived-type component, and we're also mapping the
   12022              :    whole of the derived-type variable on another clause, the pointer-set
   12023              :    mapping is removed.
   12024              : 
   12025              :    If we have a component access but we're also mapping the whole of the
   12026              :    containing struct, drop the former access.
   12027              : 
   12028              :    If the expression is a component access, and we're also mapping a base
   12029              :    pointer used in that component access in the same expression, change the
   12030              :    mapping type of the latter to ALLOC (ready for processing by
   12031              :    omp_build_struct_sibling_lists).  */
   12032              : 
   12033              : void
   12034         8655 : omp_resolve_clause_dependencies (enum tree_code code,
   12035              :                                  vec<omp_mapping_group> *groups,
   12036              :                                  hash_map<tree_operand_hash_no_se,
   12037              :                                           omp_mapping_group *> *grpmap)
   12038              : {
   12039         8655 :   int i;
   12040         8655 :   omp_mapping_group *grp;
   12041         8655 :   bool repair_chain = false;
   12042              : 
   12043        24622 :   FOR_EACH_VEC_ELT (*groups, i, grp)
   12044              :     {
   12045        15967 :       tree grp_end = grp->grp_end;
   12046        15967 :       tree decl = OMP_CLAUSE_DECL (grp_end);
   12047              : 
   12048        15967 :       gcc_assert (OMP_CLAUSE_CODE (grp_end) == OMP_CLAUSE_MAP);
   12049              : 
   12050        15967 :       switch (OMP_CLAUSE_MAP_KIND (grp_end))
   12051              :         {
   12052         1242 :         case GOMP_MAP_FIRSTPRIVATE_POINTER:
   12053         1242 :           {
   12054         1242 :             omp_mapping_group *to_group
   12055         1242 :               = omp_get_nonfirstprivate_group (grpmap, decl);
   12056              : 
   12057         1242 :             if (!to_group || to_group == grp)
   12058         1217 :               continue;
   12059              : 
   12060           25 :             tree grp_first = *to_group->grp_start;
   12061           25 :             enum gomp_map_kind first_kind = OMP_CLAUSE_MAP_KIND (grp_first);
   12062              : 
   12063           25 :             if ((GOMP_MAP_COPY_TO_P (first_kind)
   12064            7 :                  || first_kind == GOMP_MAP_ALLOC)
   12065           50 :                 && (OMP_CLAUSE_MAP_KIND (to_group->grp_end)
   12066              :                     != GOMP_MAP_FIRSTPRIVATE_POINTER))
   12067              :               {
   12068           25 :                 gcc_assert (TREE_ADDRESSABLE (OMP_CLAUSE_DECL (grp_end)));
   12069           25 :                 OMP_CLAUSE_SET_MAP_KIND (grp_end, GOMP_MAP_ATTACH_DETACH);
   12070              :               }
   12071              :           }
   12072              :           break;
   12073              : 
   12074          160 :         case GOMP_MAP_FIRSTPRIVATE_REFERENCE:
   12075          160 :           {
   12076          160 :             tree ptr = build_fold_indirect_ref (decl);
   12077              : 
   12078          160 :             omp_mapping_group *to_group
   12079          160 :               = omp_get_nonfirstprivate_group (grpmap, ptr);
   12080              : 
   12081          160 :             if (!to_group || to_group == grp)
   12082          156 :               continue;
   12083              : 
   12084            4 :             tree grp_first = *to_group->grp_start;
   12085            4 :             enum gomp_map_kind first_kind = OMP_CLAUSE_MAP_KIND (grp_first);
   12086              : 
   12087            4 :             if (GOMP_MAP_COPY_TO_P (first_kind)
   12088            4 :                 || first_kind == GOMP_MAP_ALLOC)
   12089              :               {
   12090            4 :                 OMP_CLAUSE_SET_MAP_KIND (grp_end, GOMP_MAP_ATTACH_DETACH);
   12091            4 :                 OMP_CLAUSE_DECL (grp_end) = ptr;
   12092            4 :                 if ((OMP_CLAUSE_CHAIN (*to_group->grp_start)
   12093            4 :                      == to_group->grp_end)
   12094            4 :                     && (OMP_CLAUSE_MAP_KIND (to_group->grp_end)
   12095              :                         == GOMP_MAP_FIRSTPRIVATE_REFERENCE))
   12096              :                   {
   12097            0 :                     gcc_assert (TREE_ADDRESSABLE
   12098              :                                   (OMP_CLAUSE_DECL (to_group->grp_end)));
   12099            0 :                     OMP_CLAUSE_SET_MAP_KIND (to_group->grp_end,
   12100              :                                              GOMP_MAP_ATTACH_DETACH);
   12101              : 
   12102            0 :                     location_t loc = OMP_CLAUSE_LOCATION (to_group->grp_end);
   12103            0 :                     tree alloc
   12104            0 :                       = build_omp_clause (loc, OMP_CLAUSE_MAP);
   12105            0 :                     OMP_CLAUSE_SET_MAP_KIND (alloc, GOMP_MAP_ALLOC);
   12106            0 :                     tree tmp = build_fold_addr_expr (OMP_CLAUSE_DECL
   12107              :                                                       (to_group->grp_end));
   12108            0 :                     tree char_ptr_type = build_pointer_type (char_type_node);
   12109            0 :                     OMP_CLAUSE_DECL (alloc)
   12110            0 :                       = build2 (MEM_REF, char_type_node,
   12111              :                                 tmp,
   12112              :                                 build_int_cst (char_ptr_type, 0));
   12113            0 :                     OMP_CLAUSE_SIZE (alloc) = TYPE_SIZE_UNIT (TREE_TYPE (tmp));
   12114              : 
   12115            0 :                     OMP_CLAUSE_CHAIN (alloc)
   12116            0 :                       = OMP_CLAUSE_CHAIN (*to_group->grp_start);
   12117            0 :                     OMP_CLAUSE_CHAIN (*to_group->grp_start) = alloc;
   12118              :                   }
   12119              :               }
   12120              :           }
   12121              :           break;
   12122              : 
   12123              :         case GOMP_MAP_ATTACH_DETACH:
   12124              :         case GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION:
   12125              :           {
   12126         2626 :             tree base_ptr, referenced_ptr_node = NULL_TREE;
   12127              : 
   12128         2626 :             while (TREE_CODE (decl) == ARRAY_REF)
   12129            0 :               decl = TREE_OPERAND (decl, 0);
   12130              : 
   12131         2626 :             if (TREE_CODE (decl) == INDIRECT_REF)
   12132           13 :               decl = TREE_OPERAND (decl, 0);
   12133              : 
   12134              :             /* Only component accesses.  */
   12135         2626 :             if (DECL_P (decl))
   12136          230 :               continue;
   12137              : 
   12138              :             /* We want the pointer itself when checking if the base pointer is
   12139              :                mapped elsewhere in the same directive -- if we have a
   12140              :                reference to the pointer, don't use that.  */
   12141              : 
   12142         2396 :             if (TREE_CODE (TREE_TYPE (decl)) == REFERENCE_TYPE
   12143         2396 :                 && TREE_CODE (TREE_TYPE (TREE_TYPE (decl))) == POINTER_TYPE)
   12144              :               {
   12145          394 :                 referenced_ptr_node = OMP_CLAUSE_CHAIN (*grp->grp_start);
   12146          394 :                 base_ptr = OMP_CLAUSE_DECL (referenced_ptr_node);
   12147              :               }
   12148              :             else
   12149              :               base_ptr = decl;
   12150              : 
   12151         1978 :             gomp_map_kind zlas_kind
   12152         2396 :               = (code == OACC_EXIT_DATA || code == OMP_TARGET_EXIT_DATA)
   12153         2396 :                 ? GOMP_MAP_DETACH : GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION;
   12154              : 
   12155         2396 :             if (TREE_CODE (TREE_TYPE (base_ptr)) == POINTER_TYPE)
   12156              :               {
   12157              :                 /* If we map the base TO, and we're doing an attachment, we can
   12158              :                    skip the TO mapping altogether and create an ALLOC mapping
   12159              :                    instead, since the attachment will overwrite the device
   12160              :                    pointer in that location immediately anyway.  Otherwise,
   12161              :                    change our mapping to
   12162              :                    GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION in case the
   12163              :                    attachment target has not been copied to the device already
   12164              :                    by some earlier directive.  */
   12165              : 
   12166         1972 :                 bool base_mapped_to = false;
   12167              : 
   12168         1972 :                 omp_mapping_group *base_group;
   12169              : 
   12170         1972 :                 if (omp_directive_maps_explicitly (grpmap, base_ptr,
   12171              :                                                    &base_group, false, true,
   12172              :                                                    false))
   12173              :                   {
   12174          450 :                     if (referenced_ptr_node)
   12175              :                       {
   12176          129 :                         base_mapped_to = true;
   12177          129 :                         if ((OMP_CLAUSE_MAP_KIND (base_group->grp_end)
   12178              :                              == GOMP_MAP_ATTACH_DETACH)
   12179          129 :                             && (OMP_CLAUSE_CHAIN (*base_group->grp_start)
   12180              :                                 == base_group->grp_end))
   12181              :                           {
   12182          258 :                             OMP_CLAUSE_CHAIN (*base_group->grp_start)
   12183          129 :                               = OMP_CLAUSE_CHAIN (base_group->grp_end);
   12184          129 :                             base_group->grp_end = *base_group->grp_start;
   12185          129 :                             repair_chain = true;
   12186              :                           }
   12187              :                       }
   12188              :                     else
   12189              :                       {
   12190          321 :                         base_group->deleted = true;
   12191          321 :                         OMP_CLAUSE_ATTACHMENT_MAPPING_ERASED (grp_end) = 1;
   12192              :                       }
   12193              :                   }
   12194              : 
   12195              :                 /* We're dealing with a reference to a pointer, and we are
   12196              :                    attaching both the reference and the pointer.  We know the
   12197              :                    reference itself is on the target, because we are going to
   12198              :                    create an ALLOC node for it in accumulate_sibling_list.  The
   12199              :                    pointer might be on the target already or it might not, but
   12200              :                    if it isn't then it's not an error, so use
   12201              :                    GOMP_MAP_ATTACH_ZLAS for it.  */
   12202         1972 :                 if (!base_mapped_to && referenced_ptr_node)
   12203           56 :                   OMP_CLAUSE_SET_MAP_KIND (referenced_ptr_node, zlas_kind);
   12204              : 
   12205         1972 :                 omp_mapping_group *struct_group;
   12206         1972 :                 tree desc;
   12207         1972 :                 if ((desc = OMP_CLAUSE_CHAIN (*grp->grp_start))
   12208         1972 :                     && omp_map_clause_descriptor_p (desc)
   12209         3050 :                     && omp_mapped_by_containing_struct (grpmap, decl,
   12210              :                                                         &struct_group))
   12211              :                   /* If we have a pointer set but we're mapping (or unmapping)
   12212              :                      the whole of the containing struct, we can remove the
   12213              :                      pointer set mapping.  */
   12214           15 :                   OMP_CLAUSE_CHAIN (*grp->grp_start) = OMP_CLAUSE_CHAIN (desc);
   12215              :               }
   12216          424 :             else if (TREE_CODE (TREE_TYPE (base_ptr)) == REFERENCE_TYPE
   12217          424 :                      && (TREE_CODE (TREE_TYPE (TREE_TYPE (base_ptr)))
   12218              :                          == ARRAY_TYPE)
   12219          562 :                      && OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION
   12220              :                           (*grp->grp_start))
   12221           92 :               OMP_CLAUSE_SET_MAP_KIND (grp->grp_end, zlas_kind);
   12222              :           }
   12223              :           break;
   12224              : 
   12225              :         case GOMP_MAP_ATTACH:
   12226              :           /* Ignore standalone attach here.  */
   12227              :           break;
   12228              : 
   12229        11872 :         default:
   12230        11872 :           {
   12231        11872 :             omp_mapping_group *struct_group;
   12232        11872 :             if (omp_mapped_by_containing_struct (grpmap, decl, &struct_group)
   12233        11872 :                 && *grp->grp_start == grp_end)
   12234              :               {
   12235           94 :                 omp_check_mapping_compatibility (OMP_CLAUSE_LOCATION (grp_end),
   12236              :                                                  struct_group, grp);
   12237              :                 /* Remove the whole of this mapping -- redundant.  */
   12238           94 :                 grp->deleted = true;
   12239              :               }
   12240              : 
   12241        11872 :             tree base = decl;
   12242        13484 :             while ((base = omp_get_base_pointer (base)))
   12243              :               {
   12244         1612 :                 omp_mapping_group *base_group;
   12245              : 
   12246         1612 :                 if (omp_directive_maps_explicitly (grpmap, base, &base_group,
   12247              :                                                    true, true, false))
   12248              :                   {
   12249          343 :                     tree grp_first = *base_group->grp_start;
   12250          343 :                     OMP_CLAUSE_SET_MAP_KIND (grp_first, GOMP_MAP_ALLOC);
   12251              :                   }
   12252              :               }
   12253              :           }
   12254              :         }
   12255              :     }
   12256              : 
   12257         8655 :   if (repair_chain)
   12258              :     {
   12259              :       /* Group start pointers may have become detached from the
   12260              :          OMP_CLAUSE_CHAIN of previous groups if elements were removed from the
   12261              :          end of those groups.  Fix that now.  */
   12262              :       tree *new_next = NULL;
   12263          716 :       FOR_EACH_VEC_ELT (*groups, i, grp)
   12264              :         {
   12265          587 :           if (new_next)
   12266          458 :             grp->grp_start = new_next;
   12267              : 
   12268          587 :           new_next = &OMP_CLAUSE_CHAIN (grp->grp_end);
   12269              :         }
   12270              :     }
   12271         8655 : }
   12272              : 
   12273              : /* Similar to omp_resolve_clause_dependencies, but for OpenACC.  The only
   12274              :    clause dependencies we handle for now are struct element mappings and
   12275              :    whole-struct mappings on the same directive, and duplicate clause
   12276              :    detection.  */
   12277              : 
   12278              : void
   12279         9462 : oacc_resolve_clause_dependencies (vec<omp_mapping_group> *groups,
   12280              :                                   hash_map<tree_operand_hash_no_se,
   12281              :                                            omp_mapping_group *> *grpmap)
   12282              : {
   12283         9462 :   int i;
   12284         9462 :   omp_mapping_group *grp;
   12285         9462 :   hash_set<tree_operand_hash> *seen_components = NULL;
   12286         9462 :   hash_set<tree_operand_hash> *shown_error = NULL;
   12287              : 
   12288        24643 :   FOR_EACH_VEC_ELT (*groups, i, grp)
   12289              :     {
   12290        15181 :       tree grp_end = grp->grp_end;
   12291        15181 :       tree decl = OMP_CLAUSE_DECL (grp_end);
   12292              : 
   12293        15181 :       gcc_assert (OMP_CLAUSE_CODE (grp_end) == OMP_CLAUSE_MAP);
   12294              : 
   12295        15181 :       if (DECL_P (grp_end))
   12296        14664 :         continue;
   12297              : 
   12298        15181 :       tree c = OMP_CLAUSE_DECL (*grp->grp_start);
   12299        16806 :       while (TREE_CODE (c) == ARRAY_REF)
   12300         1625 :         c = TREE_OPERAND (c, 0);
   12301        15181 :       if (TREE_CODE (c) != COMPONENT_REF)
   12302        14664 :         continue;
   12303          517 :       if (!seen_components)
   12304          474 :         seen_components = new hash_set<tree_operand_hash> ();
   12305          517 :       if (!shown_error)
   12306          474 :         shown_error = new hash_set<tree_operand_hash> ();
   12307          517 :       if (seen_components->contains (c)
   12308          517 :           && !shown_error->contains (c))
   12309              :         {
   12310           10 :           error_at (OMP_CLAUSE_LOCATION (grp_end),
   12311              :                     "%qE appears more than once in map clauses",
   12312            5 :                     OMP_CLAUSE_DECL (grp_end));
   12313            5 :           shown_error->add (c);
   12314              :         }
   12315              :       else
   12316          512 :         seen_components->add (c);
   12317              : 
   12318          517 :       omp_mapping_group *struct_group;
   12319          517 :       if (omp_mapped_by_containing_struct (grpmap, decl, &struct_group)
   12320          517 :           && *grp->grp_start == grp_end)
   12321              :         {
   12322           78 :           omp_check_mapping_compatibility (OMP_CLAUSE_LOCATION (grp_end),
   12323              :                                            struct_group, grp);
   12324              :           /* Remove the whole of this mapping -- redundant.  */
   12325           78 :           grp->deleted = true;
   12326              :         }
   12327              :     }
   12328              : 
   12329         9462 :   if (seen_components)
   12330          474 :     delete seen_components;
   12331         9462 :   if (shown_error)
   12332          474 :     delete shown_error;
   12333         9462 : }
   12334              : 
   12335              : /* Link node NEWNODE so it is pointed to by chain INSERT_AT.  NEWNODE's chain
   12336              :    is linked to the previous node pointed to by INSERT_AT.  */
   12337              : 
   12338              : static tree *
   12339         1146 : omp_siblist_insert_node_after (tree newnode, tree *insert_at)
   12340              : {
   12341         1146 :   OMP_CLAUSE_CHAIN (newnode) = *insert_at;
   12342         1146 :   *insert_at = newnode;
   12343         1146 :   return &OMP_CLAUSE_CHAIN (newnode);
   12344              : }
   12345              : 
   12346              : /* Move NODE (which is currently pointed to by the chain OLD_POS) so it is
   12347              :    pointed to by chain MOVE_AFTER instead.  */
   12348              : 
   12349              : static void
   12350         1323 : omp_siblist_move_node_after (tree node, tree *old_pos, tree *move_after)
   12351              : {
   12352         1323 :   gcc_assert (node == *old_pos);
   12353         1323 :   *old_pos = OMP_CLAUSE_CHAIN (node);
   12354         1323 :   OMP_CLAUSE_CHAIN (node) = *move_after;
   12355         1323 :   *move_after = node;
   12356         1323 : }
   12357              : 
   12358              : /* Move nodes from FIRST_PTR (pointed to by previous node's chain) to
   12359              :    LAST_NODE to after MOVE_AFTER chain.  Similar to below function, but no
   12360              :    new nodes are prepended to the list before splicing into the new position.
   12361              :    Return the position we should continue scanning the list at, or NULL to
   12362              :    stay where we were.  */
   12363              : 
   12364              : static tree *
   12365          254 : omp_siblist_move_nodes_after (tree *first_ptr, tree last_node,
   12366              :                               tree *move_after)
   12367              : {
   12368          254 :   if (first_ptr == move_after)
   12369              :     return NULL;
   12370              : 
   12371          243 :   tree tmp = *first_ptr;
   12372          243 :   *first_ptr = OMP_CLAUSE_CHAIN (last_node);
   12373          243 :   OMP_CLAUSE_CHAIN (last_node) = *move_after;
   12374          243 :   *move_after = tmp;
   12375              : 
   12376          243 :   return first_ptr;
   12377              : }
   12378              : 
   12379              : /* Concatenate two lists described by [FIRST_NEW, LAST_NEW_TAIL] and
   12380              :    [FIRST_PTR, LAST_NODE], and insert them in the OMP clause list after chain
   12381              :    pointer MOVE_AFTER.
   12382              : 
   12383              :    The latter list was previously part of the OMP clause list, and the former
   12384              :    (prepended) part is comprised of new nodes.
   12385              : 
   12386              :    We start with a list of nodes starting with a struct mapping node.  We
   12387              :    rearrange the list so that new nodes starting from FIRST_NEW and whose last
   12388              :    node's chain is LAST_NEW_TAIL comes directly after MOVE_AFTER, followed by
   12389              :    the group of mapping nodes we are currently processing (from the chain
   12390              :    FIRST_PTR to LAST_NODE).  The return value is the pointer to the next chain
   12391              :    we should continue processing from, or NULL to stay where we were.
   12392              : 
   12393              :    The transformation (in the case where MOVE_AFTER and FIRST_PTR are
   12394              :    different) is worked through below.  Here we are processing LAST_NODE, and
   12395              :    FIRST_PTR points at the preceding mapping clause:
   12396              : 
   12397              :   #. mapping node               chain
   12398              :   ---------------------------------------------------
   12399              :   A. struct_node                [->B]
   12400              :   B. comp_1                     [->C]
   12401              :   C. comp_2                     [->D (move_after)]
   12402              :   D. map_to_3                   [->E]
   12403              :   E. attach_3                   [->F (first_ptr)]
   12404              :   F. map_to_4                   [->G (continue_at)]
   12405              :   G. attach_4 (last_node)       [->H]
   12406              :   H. ...
   12407              : 
   12408              :      *last_new_tail = *first_ptr;
   12409              : 
   12410              :   I. new_node (first_new)       [->F (last_new_tail)]
   12411              : 
   12412              :      *first_ptr = OMP_CLAUSE_CHAIN (last_node)
   12413              : 
   12414              :   #. mapping node               chain
   12415              :   ----------------------------------------------------
   12416              :   A. struct_node                [->B]
   12417              :   B. comp_1                     [->C]
   12418              :   C. comp_2                     [->D (move_after)]
   12419              :   D. map_to_3                   [->E]
   12420              :   E. attach_3                   [->H (first_ptr)]
   12421              :   F. map_to_4                   [->G (continue_at)]
   12422              :   G. attach_4 (last_node)       [->H]
   12423              :   H. ...
   12424              : 
   12425              :   I. new_node (first_new)       [->F  (last_new_tail)]
   12426              : 
   12427              :      OMP_CLAUSE_CHAIN (last_node) = *move_after;
   12428              : 
   12429              :   #. mapping node               chain
   12430              :   ---------------------------------------------------
   12431              :   A. struct_node                [->B]
   12432              :   B. comp_1                     [->C]
   12433              :   C. comp_2                     [->D (move_after)]
   12434              :   D. map_to_3                   [->E]
   12435              :   E. attach_3                   [->H (continue_at)]
   12436              :   F. map_to_4                   [->G]
   12437              :   G. attach_4 (last_node)       [->D]
   12438              :   H. ...
   12439              : 
   12440              :   I. new_node (first_new)       [->F  (last_new_tail)]
   12441              : 
   12442              :      *move_after = first_new;
   12443              : 
   12444              :   #. mapping node               chain
   12445              :   ---------------------------------------------------
   12446              :   A. struct_node                [->B]
   12447              :   B. comp_1                     [->C]
   12448              :   C. comp_2                     [->I (move_after)]
   12449              :   D. map_to_3                   [->E]
   12450              :   E. attach_3                   [->H (continue_at)]
   12451              :   F. map_to_4                   [->G]
   12452              :   G. attach_4 (last_node)       [->D]
   12453              :   H. ...
   12454              :   I. new_node (first_new)       [->F (last_new_tail)]
   12455              : 
   12456              :   or, in order:
   12457              : 
   12458              :   #. mapping node               chain
   12459              :   ---------------------------------------------------
   12460              :   A. struct_node                [->B]
   12461              :   B. comp_1                     [->C]
   12462              :   C. comp_2                     [->I (move_after)]
   12463              :   I. new_node (first_new)       [->F (last_new_tail)]
   12464              :   F. map_to_4                   [->G]
   12465              :   G. attach_4 (last_node)       [->D]
   12466              :   D. map_to_3                   [->E]
   12467              :   E. attach_3                   [->H (continue_at)]
   12468              :   H. ...
   12469              : */
   12470              : 
   12471              : static tree *
   12472           71 : omp_siblist_move_concat_nodes_after (tree first_new, tree *last_new_tail,
   12473              :                                      tree *first_ptr, tree last_node,
   12474              :                                      tree *move_after)
   12475              : {
   12476           71 :   tree *continue_at = NULL;
   12477           71 :   *last_new_tail = *first_ptr;
   12478           71 :   if (first_ptr == move_after)
   12479           12 :     *move_after = first_new;
   12480              :   else
   12481              :     {
   12482           59 :       *first_ptr = OMP_CLAUSE_CHAIN (last_node);
   12483           59 :       continue_at = first_ptr;
   12484           59 :       OMP_CLAUSE_CHAIN (last_node) = *move_after;
   12485           59 :       *move_after = first_new;
   12486              :     }
   12487           71 :   return continue_at;
   12488              : }
   12489              : 
   12490              : static omp_addr_token *
   12491        11614 : omp_first_chained_access_token (vec<omp_addr_token *> &addr_tokens)
   12492              : {
   12493        11614 :   using namespace omp_addr_tokenizer;
   12494        11614 :   int idx = addr_tokens.length () - 1;
   12495        11614 :   gcc_assert (idx >= 0);
   12496        11614 :   if (addr_tokens[idx]->type != ACCESS_METHOD)
   12497              :     return addr_tokens[idx];
   12498        11645 :   while (idx > 0 && addr_tokens[idx - 1]->type == ACCESS_METHOD)
   12499              :     idx--;
   12500        11614 :   return addr_tokens[idx];
   12501              : }
   12502              : 
   12503              : /* Mapping struct members causes an additional set of nodes to be created,
   12504              :    starting with GOMP_MAP_STRUCT followed by a number of mappings equal to the
   12505              :    number of members being mapped, in order of ascending position (address or
   12506              :    bitwise).
   12507              : 
   12508              :    We scan through the list of mapping clauses, calling this function for each
   12509              :    struct member mapping we find, and build up the list of mappings after the
   12510              :    initial GOMP_MAP_STRUCT node.  For pointer members, these will be
   12511              :    newly-created ALLOC nodes.  For non-pointer members, the existing mapping is
   12512              :    moved into place in the sorted list.
   12513              : 
   12514              :      struct {
   12515              :        int *a;
   12516              :        int *b;
   12517              :        int c;
   12518              :        int *d;
   12519              :      };
   12520              : 
   12521              :      #pragma (acc|omp directive) copy(struct.a[0:n], struct.b[0:n], struct.c,
   12522              :                                       struct.d[0:n])
   12523              : 
   12524              :      GOMP_MAP_STRUCT (4)
   12525              :      [GOMP_MAP_FIRSTPRIVATE_REFERENCE -- for refs to structs]
   12526              :      GOMP_MAP_ALLOC  (struct.a)
   12527              :      GOMP_MAP_ALLOC  (struct.b)
   12528              :      GOMP_MAP_TO     (struct.c)
   12529              :      GOMP_MAP_ALLOC  (struct.d)
   12530              :      ...
   12531              : 
   12532              :    In the case where we are mapping references to pointers, or in Fortran if
   12533              :    we are mapping an array with a descriptor, additional nodes may be created
   12534              :    after the struct node list also.
   12535              : 
   12536              :    The return code is either a pointer to the next node to process (if the
   12537              :    list has been rearranged), else NULL to continue with the next node in the
   12538              :    original list.  */
   12539              : 
   12540              : static tree *
   12541         4781 : omp_accumulate_sibling_list (enum omp_region_type region_type,
   12542              :                              enum tree_code code,
   12543              :                              hash_map<tree_operand_hash, tree>
   12544              :                                *&struct_map_to_clause,
   12545              :                              hash_map<tree_operand_hash_no_se,
   12546              :                                       omp_mapping_group *> *group_map,
   12547              :                              tree *grp_start_p, tree grp_end,
   12548              :                              vec<omp_addr_token *> &addr_tokens, tree **inner,
   12549              :                              bool *fragile_p, bool reprocessing_struct,
   12550              :                              tree **added_tail)
   12551              : {
   12552         4781 :   using namespace omp_addr_tokenizer;
   12553         4781 :   poly_offset_int coffset;
   12554         4781 :   poly_int64 cbitpos;
   12555         4781 :   tree ocd = OMP_CLAUSE_DECL (grp_end);
   12556         4781 :   bool openmp = !(region_type & ORT_ACC);
   12557         4781 :   bool target = (region_type & ORT_TARGET) != 0;
   12558         4781 :   tree *continue_at = NULL;
   12559              : 
   12560         5094 :   while (TREE_CODE (ocd) == ARRAY_REF)
   12561          313 :     ocd = TREE_OPERAND (ocd, 0);
   12562              : 
   12563         4781 :   if (*fragile_p)
   12564              :     {
   12565          156 :       omp_mapping_group *to_group
   12566          156 :         = omp_get_nonfirstprivate_group (group_map, ocd, true);
   12567              : 
   12568          156 :       if (to_group)
   12569              :         return NULL;
   12570              :     }
   12571              : 
   12572         4693 :   omp_addr_token *last_token = omp_first_chained_access_token (addr_tokens);
   12573         4693 :   if (last_token->type == ACCESS_METHOD)
   12574              :     {
   12575         4693 :       switch (last_token->u.access_kind)
   12576              :         {
   12577          735 :         case ACCESS_REF:
   12578          735 :         case ACCESS_REF_TO_POINTER:
   12579          735 :         case ACCESS_REF_TO_POINTER_OFFSET:
   12580          735 :         case ACCESS_INDEXED_REF_TO_ARRAY:
   12581              :           /* We may see either a bare reference or a dereferenced
   12582              :              "convert_from_reference"-like one here.  Handle either way.  */
   12583          735 :           if (TREE_CODE (ocd) == INDIRECT_REF)
   12584           64 :             ocd = TREE_OPERAND (ocd, 0);
   12585          735 :           gcc_assert (TREE_CODE (TREE_TYPE (ocd)) == REFERENCE_TYPE);
   12586              :           break;
   12587              : 
   12588              :         default:
   12589              :           ;
   12590              :         }
   12591              :     }
   12592              : 
   12593         4693 :   bool variable_offset;
   12594         9386 :   tree iterators = OMP_CLAUSE_HAS_ITERATORS (grp_end)
   12595         4726 :                      ? OMP_CLAUSE_ITERATORS (grp_end) : NULL_TREE;
   12596         4693 :   tree base
   12597         4693 :     = extract_base_bit_offset (ocd, &cbitpos, &coffset, &variable_offset,
   12598         4693 :                                iterators);
   12599              : 
   12600         4693 :   int base_token;
   12601        23977 :   for (base_token = addr_tokens.length () - 1; base_token >= 0; base_token--)
   12602              :     {
   12603        19284 :       if (addr_tokens[base_token]->type == ARRAY_BASE
   12604        19284 :           || addr_tokens[base_token]->type == STRUCTURE_BASE)
   12605              :         break;
   12606              :     }
   12607              : 
   12608              :   /* The two expressions in the assertion below aren't quite the same: if we
   12609              :      have 'struct_base_decl access_indexed_array' for something like
   12610              :      "myvar[2].x" then base will be "myvar" and addr_tokens[base_token]->expr
   12611              :      will be "myvar[2]" -- the actual base of the structure.
   12612              :      The former interpretation leads to a strange situation where we get
   12613              :        struct(myvar) alloc(myvar[2].ptr1)
   12614              :      That is, the array of structures is kind of treated as one big structure
   12615              :      for the purposes of gathering sibling lists, etc.  */
   12616              :   /* gcc_assert (base == addr_tokens[base_token]->expr);  */
   12617              : 
   12618         4693 :   bool attach_detach = ((OMP_CLAUSE_MAP_KIND (grp_end)
   12619              :                          == GOMP_MAP_ATTACH_DETACH)
   12620         4693 :                         || (OMP_CLAUSE_MAP_KIND (grp_end)
   12621         4693 :                             == GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION));
   12622         4693 :   bool has_descriptor = false;
   12623         4693 :   if (OMP_CLAUSE_CHAIN (*grp_start_p) != grp_end)
   12624              :     {
   12625         3153 :       tree grp_mid = OMP_CLAUSE_CHAIN (*grp_start_p);
   12626         3153 :       if (grp_mid && omp_map_clause_descriptor_p (grp_mid))
   12627              :         has_descriptor = true;
   12628              :     }
   12629              : 
   12630         4693 :   if (!struct_map_to_clause || struct_map_to_clause->get (base) == NULL)
   12631              :     {
   12632         3071 :       enum gomp_map_kind str_kind = GOMP_MAP_STRUCT;
   12633              : 
   12634         3071 :       if (struct_map_to_clause == NULL)
   12635         2507 :         struct_map_to_clause = new hash_map<tree_operand_hash, tree>;
   12636              : 
   12637         3071 :       if (variable_offset)
   12638          276 :         str_kind = GOMP_MAP_STRUCT_UNORD;
   12639              : 
   12640         3071 :       tree l = build_omp_clause (OMP_CLAUSE_LOCATION (grp_end), OMP_CLAUSE_MAP);
   12641              : 
   12642         3071 :       OMP_CLAUSE_SET_MAP_KIND (l, str_kind);
   12643         3071 :       OMP_CLAUSE_DECL (l) = unshare_expr (base);
   12644         3071 :       OMP_CLAUSE_SIZE (l) = size_int (1);
   12645              : 
   12646         3071 :       struct_map_to_clause->put (base, l);
   12647              : 
   12648              :       /* On first iterating through the clause list, we insert the struct node
   12649              :          just before the component access node that triggers the initial
   12650              :          omp_accumulate_sibling_list call for a particular sibling list (and
   12651              :          it then forms the first entry in that list).  When reprocessing
   12652              :          struct bases that are themselves component accesses, we insert the
   12653              :          struct node on an off-side list to avoid inserting the new
   12654              :          GOMP_MAP_STRUCT into the middle of the old one.  */
   12655         3071 :       tree *insert_node_pos = reprocessing_struct ? *added_tail : grp_start_p;
   12656              : 
   12657         3071 :       if (has_descriptor)
   12658              :         {
   12659          759 :           tree desc = OMP_CLAUSE_CHAIN (*grp_start_p);
   12660          759 :           if (code == OMP_TARGET_EXIT_DATA || code == OACC_EXIT_DATA)
   12661          187 :             OMP_CLAUSE_SET_MAP_KIND (desc, GOMP_MAP_RELEASE);
   12662          759 :           tree sc = *insert_node_pos;
   12663          759 :           OMP_CLAUSE_CHAIN (l) = desc;
   12664          759 :           OMP_CLAUSE_CHAIN (*grp_start_p) = OMP_CLAUSE_CHAIN (desc);
   12665          759 :           OMP_CLAUSE_CHAIN (desc) = sc;
   12666          759 :           *insert_node_pos = l;
   12667              :         }
   12668         2312 :       else if (attach_detach)
   12669              :         {
   12670         1396 :           tree extra_node;
   12671         1396 :           tree alloc_node
   12672         1396 :             = build_omp_struct_comp_nodes (code, *grp_start_p, grp_end,
   12673              :                                            &extra_node);
   12674         1396 :           tree *tail;
   12675         1396 :           OMP_CLAUSE_CHAIN (l) = alloc_node;
   12676              : 
   12677         1396 :           if (extra_node)
   12678              :             {
   12679            0 :               OMP_CLAUSE_CHAIN (extra_node) = *insert_node_pos;
   12680            0 :               OMP_CLAUSE_CHAIN (alloc_node) = extra_node;
   12681            0 :               tail = &OMP_CLAUSE_CHAIN (extra_node);
   12682              :             }
   12683              :           else
   12684              :             {
   12685         1396 :               OMP_CLAUSE_CHAIN (alloc_node) = *insert_node_pos;
   12686         1396 :               tail = &OMP_CLAUSE_CHAIN (alloc_node);
   12687              :             }
   12688              : 
   12689              :           /* For OpenMP semantics, we don't want to implicitly allocate
   12690              :              space for the pointer here for non-compute regions (e.g. "enter
   12691              :              data").  A FRAGILE_P node is only being created so that
   12692              :              omp-low.cc is able to rewrite the struct properly.
   12693              :              For references (to pointers), we want to actually allocate the
   12694              :              space for the reference itself in the sorted list following the
   12695              :              struct node.
   12696              :              For pointers, we want to allocate space if we had an explicit
   12697              :              mapping of the attachment point, but not otherwise.  */
   12698         1396 :           if (*fragile_p
   12699         1396 :               || (openmp
   12700              :                   && !target
   12701              :                   && attach_detach
   12702          234 :                   && TREE_CODE (TREE_TYPE (ocd)) == POINTER_TYPE
   12703           88 :                   && !OMP_CLAUSE_ATTACHMENT_MAPPING_ERASED (grp_end)))
   12704              :             {
   12705          133 :               if (!lang_GNU_Fortran ())
   12706              :                /* In Fortran, pointers are dereferenced automatically, but may
   12707              :                   be unassociated.  So we still want to allocate space for the
   12708              :                   pointer (as the base for an attach operation that should be
   12709              :                   present in the same directive's clause list also).  */
   12710          103 :                 OMP_CLAUSE_SIZE (alloc_node) = size_zero_node;
   12711          133 :               OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (alloc_node) = 1;
   12712              :             }
   12713              : 
   12714         1396 :           *insert_node_pos = l;
   12715              : 
   12716         1396 :           if (reprocessing_struct)
   12717              :             {
   12718              :               /* When reprocessing a struct node group used as the base of a
   12719              :                  subcomponent access, if we have a reference-to-pointer base,
   12720              :                  we will see:
   12721              :                    struct(**ptr) attach(*ptr)
   12722              :                  whereas for a non-reprocess-struct group, we see, e.g.:
   12723              :                    tofrom(**ptr) attach(*ptr) attach(ptr)
   12724              :                  and we create the "alloc" for the second "attach", i.e.
   12725              :                  for the reference itself.  When reprocessing a struct group we
   12726              :                  thus change the pointer attachment into a reference attachment
   12727              :                  by stripping the indirection.  (The attachment of the
   12728              :                  referenced pointer must happen elsewhere, either on the same
   12729              :                  directive, or otherwise.)  */
   12730          180 :               tree adecl = OMP_CLAUSE_DECL (alloc_node);
   12731              : 
   12732          180 :               if ((TREE_CODE (adecl) == INDIRECT_REF
   12733          148 :                    || (TREE_CODE (adecl) == MEM_REF
   12734            0 :                        && integer_zerop (TREE_OPERAND (adecl, 1))))
   12735           32 :                   && (TREE_CODE (TREE_TYPE (TREE_OPERAND (adecl, 0)))
   12736              :                       == REFERENCE_TYPE)
   12737          212 :                   && (TREE_CODE (TREE_TYPE (TREE_TYPE
   12738              :                         (TREE_OPERAND (adecl, 0)))) == POINTER_TYPE))
   12739           32 :                 OMP_CLAUSE_DECL (alloc_node) = TREE_OPERAND (adecl, 0);
   12740              : 
   12741          180 :               *added_tail = tail;
   12742              :             }
   12743              :         }
   12744              :       else
   12745              :         {
   12746          916 :           gcc_assert (*grp_start_p == grp_end);
   12747          916 :           if (reprocessing_struct)
   12748              :             {
   12749              :               /* If we don't have an attach/detach node, this is a
   12750              :                  "target data" directive or similar, not an offload region.
   12751              :                  Synthesize an "alloc" node using just the initiating
   12752              :                  GOMP_MAP_STRUCT decl.  */
   12753           16 :               gomp_map_kind k = (code == OMP_TARGET_EXIT_DATA
   12754           32 :                                  || code == OACC_EXIT_DATA)
   12755           32 :                                 ? GOMP_MAP_RELEASE : GOMP_MAP_ALLOC;
   12756           32 :               tree alloc_node
   12757           32 :                 = build_omp_clause (OMP_CLAUSE_LOCATION (grp_end),
   12758              :                                     OMP_CLAUSE_MAP);
   12759           32 :               OMP_CLAUSE_SET_MAP_KIND (alloc_node, k);
   12760           32 :               OMP_CLAUSE_DECL (alloc_node) = unshare_expr (last_token->expr);
   12761           64 :               OMP_CLAUSE_SIZE (alloc_node)
   12762           32 :                 = TYPE_SIZE_UNIT (TREE_TYPE (OMP_CLAUSE_DECL (alloc_node)));
   12763              : 
   12764           32 :               OMP_CLAUSE_CHAIN (alloc_node) = OMP_CLAUSE_CHAIN (l);
   12765           32 :               OMP_CLAUSE_CHAIN (l) = alloc_node;
   12766           32 :               *insert_node_pos = l;
   12767           32 :               *added_tail = &OMP_CLAUSE_CHAIN (alloc_node);
   12768              :             }
   12769              :           else
   12770          884 :             grp_start_p = omp_siblist_insert_node_after (l, insert_node_pos);
   12771              :         }
   12772              : 
   12773         3071 :       unsigned last_access = base_token + 1;
   12774              : 
   12775         3071 :       while (last_access + 1 < addr_tokens.length ()
   12776         3406 :              && addr_tokens[last_access + 1]->type == ACCESS_METHOD)
   12777              :         last_access++;
   12778              : 
   12779         3071 :       if ((region_type & ORT_TARGET)
   12780         3071 :           && addr_tokens[base_token + 1]->type == ACCESS_METHOD)
   12781              :         {
   12782         1719 :           bool base_ref = false;
   12783         1719 :           access_method_kinds access_kind
   12784         1719 :             = addr_tokens[last_access]->u.access_kind;
   12785              : 
   12786         1719 :           switch (access_kind)
   12787              :             {
   12788              :             case ACCESS_DIRECT:
   12789              :             case ACCESS_INDEXED_ARRAY:
   12790         1071 :               return NULL;
   12791              : 
   12792          403 :             case ACCESS_REF:
   12793          403 :             case ACCESS_REF_TO_POINTER:
   12794          403 :             case ACCESS_REF_TO_POINTER_OFFSET:
   12795          403 :             case ACCESS_INDEXED_REF_TO_ARRAY:
   12796          403 :               base_ref = true;
   12797          403 :               break;
   12798              : 
   12799          866 :             default:
   12800          866 :               ;
   12801              :             }
   12802          866 :           tree c2 = build_omp_clause (OMP_CLAUSE_LOCATION (grp_end),
   12803              :                                       OMP_CLAUSE_MAP);
   12804          866 :           enum gomp_map_kind mkind;
   12805          866 :           omp_mapping_group *decl_group;
   12806          866 :           tree use_base;
   12807          866 :           switch (access_kind)
   12808              :             {
   12809          463 :             case ACCESS_POINTER:
   12810          463 :             case ACCESS_POINTER_OFFSET:
   12811          463 :               use_base = addr_tokens[last_access]->expr;
   12812          463 :               break;
   12813          198 :             case ACCESS_REF_TO_POINTER:
   12814          198 :             case ACCESS_REF_TO_POINTER_OFFSET:
   12815          198 :               use_base
   12816          198 :                 = build_fold_indirect_ref (addr_tokens[last_access]->expr);
   12817          198 :               break;
   12818          205 :             default:
   12819          205 :               use_base = addr_tokens[base_token]->expr;
   12820              :             }
   12821          866 :           bool mapped_to_p
   12822          866 :             = omp_directive_maps_explicitly (group_map, use_base, &decl_group,
   12823              :                                              true, false, true);
   12824          866 :           if (addr_tokens[base_token]->type == STRUCTURE_BASE
   12825          866 :               && DECL_P (addr_tokens[last_access]->expr)
   12826         1372 :               && !mapped_to_p)
   12827          444 :             mkind = base_ref ? GOMP_MAP_FIRSTPRIVATE_REFERENCE
   12828              :                              : GOMP_MAP_FIRSTPRIVATE_POINTER;
   12829              :           else
   12830              :             mkind = GOMP_MAP_ATTACH_DETACH;
   12831              : 
   12832          866 :           OMP_CLAUSE_SET_MAP_KIND (c2, mkind);
   12833              :           /* If we have a reference to pointer base, we want to attach the
   12834              :              pointer here, not the reference.  The reference attachment happens
   12835              :              elsewhere.  */
   12836          866 :           bool ref_to_ptr
   12837          866 :             = (access_kind == ACCESS_REF_TO_POINTER
   12838          866 :                || access_kind == ACCESS_REF_TO_POINTER_OFFSET);
   12839          866 :           tree sdecl = addr_tokens[last_access]->expr;
   12840          866 :           tree sdecl_ptr = ref_to_ptr ? build_fold_indirect_ref (sdecl)
   12841              :                                       : sdecl;
   12842              :           /* For the FIRSTPRIVATE_REFERENCE after the struct node, we
   12843              :              want to use the reference itself for the decl, but we
   12844              :              still want to use the pointer to calculate the bias.  */
   12845          866 :           OMP_CLAUSE_DECL (c2) = (mkind == GOMP_MAP_ATTACH_DETACH)
   12846          866 :                                  ? sdecl_ptr : sdecl;
   12847          866 :           sdecl = sdecl_ptr;
   12848          866 :           tree baddr = build_fold_addr_expr (base);
   12849          866 :           baddr = fold_convert_loc (OMP_CLAUSE_LOCATION (grp_end),
   12850              :                                     ptrdiff_type_node, baddr);
   12851          866 :           tree decladdr = fold_convert_loc (OMP_CLAUSE_LOCATION (grp_end),
   12852              :                                             ptrdiff_type_node, sdecl);
   12853          866 :           OMP_CLAUSE_SIZE (c2)
   12854          866 :             = fold_build2_loc (OMP_CLAUSE_LOCATION (grp_end), MINUS_EXPR,
   12855              :                                ptrdiff_type_node, baddr, decladdr);
   12856              :           /* Insert after struct node.  */
   12857          866 :           OMP_CLAUSE_CHAIN (c2) = OMP_CLAUSE_CHAIN (l);
   12858          866 :           OMP_CLAUSE_CHAIN (l) = c2;
   12859              : 
   12860          866 :           if (addr_tokens[base_token]->type == STRUCTURE_BASE
   12861          866 :               && (addr_tokens[base_token]->u.structure_base_kind
   12862              :                   == BASE_COMPONENT_EXPR)
   12863          286 :               && mkind == GOMP_MAP_ATTACH_DETACH
   12864         1152 :               && addr_tokens[last_access]->u.access_kind != ACCESS_REF)
   12865              :             {
   12866          218 :               *inner = insert_node_pos;
   12867          218 :               if (openmp)
   12868          166 :                 *fragile_p = true;
   12869              :               return NULL;
   12870              :             }
   12871              :         }
   12872              : 
   12873         2000 :       if (addr_tokens[base_token]->type == STRUCTURE_BASE
   12874         2000 :           && (addr_tokens[base_token]->u.structure_base_kind
   12875              :               == BASE_COMPONENT_EXPR)
   12876         2467 :           && addr_tokens[last_access]->u.access_kind == ACCESS_REF)
   12877          100 :         *inner = insert_node_pos;
   12878              : 
   12879              :       return NULL;
   12880              :     }
   12881         1622 :   else if (struct_map_to_clause)
   12882              :     {
   12883         1622 :       tree *osc = struct_map_to_clause->get (base);
   12884         1622 :       tree *sc = NULL, *scp = NULL;
   12885              : 
   12886         1622 :       unsigned HOST_WIDE_INT i, elems = tree_to_uhwi (OMP_CLAUSE_SIZE (*osc));
   12887         1622 :       sc = &OMP_CLAUSE_CHAIN (*osc);
   12888              :       /* The struct mapping might be immediately followed by a
   12889              :          FIRSTPRIVATE_POINTER,  FIRSTPRIVATE_REFERENCE or an ATTACH_DETACH --
   12890              :          if it's an indirect access or a reference, or if the structure base
   12891              :          is not a decl.  The FIRSTPRIVATE_* nodes are removed in omp-low.cc
   12892              :          after they have been processed there, and ATTACH_DETACH nodes are
   12893              :          recomputed and moved out of the GOMP_MAP_STRUCT construct once
   12894              :          sibling list building is complete.  */
   12895         1622 :       if (OMP_CLAUSE_MAP_KIND (*sc) == GOMP_MAP_FIRSTPRIVATE_POINTER
   12896         1577 :           || OMP_CLAUSE_MAP_KIND (*sc) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
   12897         3154 :           || OMP_CLAUSE_MAP_KIND (*sc) == GOMP_MAP_ATTACH_DETACH)
   12898          201 :         sc = &OMP_CLAUSE_CHAIN (*sc);
   12899         4112 :       for (i = 0; i < elems; i++, sc = &OMP_CLAUSE_CHAIN (*sc))
   12900         3065 :         if (attach_detach && sc == grp_start_p)
   12901              :           break;
   12902         3065 :         else if (TREE_CODE (OMP_CLAUSE_DECL (*sc)) != COMPONENT_REF
   12903          182 :                  && TREE_CODE (OMP_CLAUSE_DECL (*sc)) != INDIRECT_REF
   12904         3247 :                  && TREE_CODE (OMP_CLAUSE_DECL (*sc)) != ARRAY_REF)
   12905              :           break;
   12906              :         else
   12907              :           {
   12908         3065 :             tree sc_decl = OMP_CLAUSE_DECL (*sc);
   12909         3065 :             poly_offset_int offset;
   12910         3065 :             poly_int64 bitpos;
   12911              : 
   12912         3065 :             if (TREE_CODE (sc_decl) == ARRAY_REF)
   12913              :               {
   12914          366 :                 while (TREE_CODE (sc_decl) == ARRAY_REF)
   12915          184 :                   sc_decl = TREE_OPERAND (sc_decl, 0);
   12916          182 :                 if (TREE_CODE (sc_decl) != COMPONENT_REF
   12917          182 :                     || TREE_CODE (TREE_TYPE (sc_decl)) != ARRAY_TYPE)
   12918              :                   break;
   12919              :               }
   12920         2883 :             else if (INDIRECT_REF_P (sc_decl)
   12921            0 :                      && TREE_CODE (TREE_OPERAND (sc_decl, 0)) == COMPONENT_REF
   12922         2883 :                      && (TREE_CODE (TREE_TYPE (TREE_OPERAND (sc_decl, 0)))
   12923              :                          == REFERENCE_TYPE))
   12924            0 :               sc_decl = TREE_OPERAND (sc_decl, 0);
   12925              : 
   12926         3065 :             bool variable_offset2;
   12927         3065 :             tree iterators2 = OMP_CLAUSE_HAS_ITERATORS (*sc)
   12928         3065 :                                 ? OMP_CLAUSE_ITERATORS (*sc) : NULL_TREE;
   12929              : 
   12930         3065 :             tree base2 = extract_base_bit_offset (sc_decl, &bitpos, &offset,
   12931              :                                                   &variable_offset2,
   12932              :                                                   iterators2);
   12933         3065 :             if (!base2 || !operand_equal_p (base2, base, 0))
   12934              :               break;
   12935         3065 :             if (scp)
   12936          469 :               continue;
   12937         2845 :             if (variable_offset2)
   12938              :               {
   12939          342 :                 OMP_CLAUSE_SET_MAP_KIND (*osc, GOMP_MAP_STRUCT_UNORD);
   12940              : 
   12941          342 :                 if (has_descriptor)
   12942              :                   {
   12943              :                     /* Sort mapped components by offset. This is needed for
   12944              :                        libgomp to handle Fortran derived-type allocatable
   12945              :                        components transparently.  */
   12946              : 
   12947          323 :                     poly_int64 bitsize;
   12948          323 :                     tree offset, coffset;
   12949          323 :                     machine_mode mode;
   12950          323 :                     int unsignedp, reversep, volatilep;
   12951          323 :                     tree inner_ref1
   12952          323 :                       = get_inner_reference (sc_decl, &bitsize, &bitpos,
   12953              :                                              &offset, &mode, &unsignedp,
   12954              :                                              &reversep, &volatilep);
   12955          323 :                     tree osc_decl = ocd;
   12956          323 :                     STRIP_NOPS (osc_decl);
   12957          323 :                     tree inner_ref2
   12958          323 :                       = get_inner_reference (osc_decl, &bitsize, &bitpos,
   12959              :                                              &coffset, &mode, &unsignedp,
   12960              :                                              &reversep, &volatilep);
   12961          323 :                     gcc_assert (operand_equal_p (inner_ref1, inner_ref2, 0));
   12962          323 :                     tree offset_diff
   12963          323 :                       = fold_binary_to_constant (MINUS_EXPR, size_type_node,
   12964              :                                                  coffset, offset);
   12965          572 :                     if (offset_diff == NULL_TREE
   12966          323 :                         || TREE_INT_CST_ELT (offset_diff, 0) > 0)
   12967          249 :                       continue;
   12968              :                     else
   12969              :                       break;
   12970              :                   }
   12971              :               }
   12972         2503 :             else if ((region_type & ORT_ACC) != 0)
   12973              :               {
   12974              :                 /* For OpenACC, allow (ignore) duplicate struct accesses in
   12975              :                    the middle of a mapping clause, e.g. "mystruct->foo" in:
   12976              :                    copy(mystruct->foo->bar) copy(mystruct->foo->qux).  */
   12977          223 :                 if (reprocessing_struct
   12978            8 :                     && known_eq (coffset, offset)
   12979          223 :                     && known_eq (cbitpos, bitpos))
   12980           16 :                   return NULL;
   12981              :               }
   12982         2280 :             else if (known_eq (coffset, offset)
   12983         2280 :                      && known_eq (cbitpos, bitpos))
   12984              :               {
   12985              :                 /* Having two struct members at the same offset doesn't work,
   12986              :                    so make sure we don't.  (We're allowed to ignore this.
   12987              :                    Should we report the error?)  */
   12988              :                 /*error_at (OMP_CLAUSE_LOCATION (grp_end),
   12989              :                           "duplicate struct member %qE in map clauses",
   12990              :                           OMP_CLAUSE_DECL (grp_end));*/
   12991              :                 return NULL;
   12992              :               }
   12993         2506 :             if (maybe_lt (coffset, offset)
   12994         4235 :                 || (known_eq (coffset, offset)
   12995           20 :                     && maybe_lt (cbitpos, bitpos)))
   12996              :               {
   12997          777 :                 if (attach_detach)
   12998              :                   scp = sc;
   12999              :                 else
   13000              :                   break;
   13001              :               }
   13002              :           }
   13003              : 
   13004         1606 :       OMP_CLAUSE_SIZE (*osc)
   13005         1606 :         = size_binop (PLUS_EXPR, OMP_CLAUSE_SIZE (*osc), size_one_node);
   13006              : 
   13007         1606 :       if (reprocessing_struct)
   13008              :         {
   13009              :           /* If we're reprocessing a struct node, we don't want to do most of
   13010              :              the list manipulation below.  We only need to handle the (pointer
   13011              :              or reference) attach/detach case.  */
   13012            8 :           tree extra_node, alloc_node;
   13013            8 :           if (has_descriptor)
   13014            0 :             gcc_unreachable ();
   13015            8 :           else if (attach_detach)
   13016            8 :             alloc_node = build_omp_struct_comp_nodes (code, *grp_start_p,
   13017              :                                                       grp_end, &extra_node);
   13018              :           else
   13019              :             {
   13020              :               /* If we don't have an attach/detach node, this is a
   13021              :                  "target data" directive or similar, not an offload region.
   13022              :                  Synthesize an "alloc" node using just the initiating
   13023              :                  GOMP_MAP_STRUCT decl.  */
   13024            0 :               gomp_map_kind k = (code == OMP_TARGET_EXIT_DATA
   13025            0 :                                  || code == OACC_EXIT_DATA)
   13026            0 :                                 ? GOMP_MAP_RELEASE : GOMP_MAP_ALLOC;
   13027            0 :               alloc_node
   13028            0 :                 = build_omp_clause (OMP_CLAUSE_LOCATION (grp_end),
   13029              :                                     OMP_CLAUSE_MAP);
   13030            0 :               OMP_CLAUSE_SET_MAP_KIND (alloc_node, k);
   13031            0 :               OMP_CLAUSE_DECL (alloc_node) = unshare_expr (last_token->expr);
   13032            0 :               OMP_CLAUSE_SIZE (alloc_node)
   13033            0 :                 = TYPE_SIZE_UNIT (TREE_TYPE (OMP_CLAUSE_DECL (alloc_node)));
   13034              :             }
   13035              : 
   13036            8 :           if (scp)
   13037            0 :             omp_siblist_insert_node_after (alloc_node, scp);
   13038              :           else
   13039              :             {
   13040            8 :               tree *new_end = omp_siblist_insert_node_after (alloc_node, sc);
   13041            8 :               if (sc == *added_tail)
   13042            8 :                 *added_tail = new_end;
   13043              :             }
   13044              : 
   13045            8 :           return NULL;
   13046              :         }
   13047              : 
   13048         1598 :       if (has_descriptor)
   13049              :         {
   13050          538 :           tree desc = OMP_CLAUSE_CHAIN (*grp_start_p);
   13051          538 :           if (code == OMP_TARGET_EXIT_DATA
   13052          538 :               || code == OACC_EXIT_DATA)
   13053          105 :             OMP_CLAUSE_SET_MAP_KIND (desc, GOMP_MAP_RELEASE);
   13054         1076 :           omp_siblist_move_node_after (desc,
   13055          538 :                                        &OMP_CLAUSE_CHAIN (*grp_start_p),
   13056              :                                        scp ? scp : sc);
   13057              :         }
   13058         1060 :       else if (attach_detach)
   13059              :         {
   13060          325 :           tree cl = NULL_TREE, extra_node;
   13061          325 :           tree alloc_node = build_omp_struct_comp_nodes (code, *grp_start_p,
   13062              :                                                          grp_end, &extra_node);
   13063          325 :           tree *tail_chain = NULL;
   13064              : 
   13065          325 :           if (*fragile_p
   13066          325 :               || (openmp
   13067              :                   && !target
   13068              :                   && attach_detach
   13069           62 :                   && TREE_CODE (TREE_TYPE (ocd)) == POINTER_TYPE
   13070           23 :                   && !OMP_CLAUSE_ATTACHMENT_MAPPING_ERASED (grp_end)))
   13071              :             {
   13072            6 :               if (!lang_GNU_Fortran ())
   13073            6 :                 OMP_CLAUSE_SIZE (alloc_node) = size_zero_node;
   13074            6 :               OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (alloc_node) = 1;
   13075              :             }
   13076              : 
   13077              :           /* Here, we have:
   13078              : 
   13079              :              grp_end : the last (or only) node in this group.
   13080              :              grp_start_p : pointer to the first node in a pointer mapping group
   13081              :                            up to and including GRP_END.
   13082              :              sc : pointer to the chain for the end of the struct component
   13083              :                   list.
   13084              :              scp : pointer to the chain for the sorted position at which we
   13085              :                    should insert in the middle of the struct component list
   13086              :                    (else NULL to insert at end).
   13087              :              alloc_node : the "alloc" node for the structure (pointer-type)
   13088              :                           component. We insert at SCP (if present), else SC
   13089              :                           (the end of the struct component list).
   13090              :              extra_node : a newly-synthesized node for an additional indirect
   13091              :                           pointer mapping or a Fortran pointer set, if needed.
   13092              :              cl : first node to prepend before grp_start_p.
   13093              :              tail_chain : pointer to chain of last prepended node.
   13094              : 
   13095              :              The general idea is we move the nodes for this struct mapping
   13096              :              together: the alloc node goes into the sorted list directly after
   13097              :              the struct mapping, and any extra nodes (together with the nodes
   13098              :              mapping arrays pointed to by struct components) get moved after
   13099              :              that list.  When SCP is NULL, we insert the nodes at SC, i.e. at
   13100              :              the end of the struct component mapping list.  It's important that
   13101              :              the alloc_node comes first in that case because it's part of the
   13102              :              sorted component mapping list (but subsequent nodes are not!).  */
   13103              : 
   13104          325 :           if (scp)
   13105          254 :             omp_siblist_insert_node_after (alloc_node, scp);
   13106              : 
   13107              :           /* Make [cl,tail_chain] a list of the alloc node (if we haven't
   13108              :              already inserted it) and the extra_node (if it is present).  The
   13109              :              list can be empty if we added alloc_node above and there is no
   13110              :              extra node.  */
   13111          254 :           if (scp && extra_node)
   13112              :             {
   13113            0 :               cl = extra_node;
   13114            0 :               tail_chain = &OMP_CLAUSE_CHAIN (extra_node);
   13115              :             }
   13116          325 :           else if (extra_node)
   13117              :             {
   13118            0 :               OMP_CLAUSE_CHAIN (alloc_node) = extra_node;
   13119            0 :               cl = alloc_node;
   13120            0 :               tail_chain = &OMP_CLAUSE_CHAIN (extra_node);
   13121              :             }
   13122          325 :           else if (!scp)
   13123              :             {
   13124           71 :               cl = alloc_node;
   13125           71 :               tail_chain = &OMP_CLAUSE_CHAIN (alloc_node);
   13126              :             }
   13127              : 
   13128           71 :           continue_at
   13129           71 :             = cl ? omp_siblist_move_concat_nodes_after (cl, tail_chain,
   13130              :                                                         grp_start_p, grp_end,
   13131              :                                                         sc)
   13132          254 :                  : omp_siblist_move_nodes_after (grp_start_p, grp_end, sc);
   13133              :         }
   13134          735 :       else if (*sc != grp_end)
   13135              :         {
   13136          695 :           gcc_assert (*grp_start_p == grp_end);
   13137              : 
   13138              :           /* We are moving the current node back to a previous struct node:
   13139              :              the node that used to point to the current node will now point to
   13140              :              the next node.  */
   13141          695 :           continue_at = grp_start_p;
   13142              :           /* In the non-pointer case, the mapping clause itself is moved into
   13143              :              the correct position in the struct component list, which in this
   13144              :              case is just SC.  */
   13145          695 :           omp_siblist_move_node_after (*grp_start_p, grp_start_p, sc);
   13146              :         }
   13147              :     }
   13148              :   return continue_at;
   13149              : }
   13150              : 
   13151              : /* Scan through GROUPS, and create sorted structure sibling lists without
   13152              :    gimplifying.  */
   13153              : 
   13154              : static bool
   13155        18117 : omp_build_struct_sibling_lists (enum tree_code code,
   13156              :                                 enum omp_region_type region_type,
   13157              :                                 vec<omp_mapping_group> *groups,
   13158              :                                 hash_map<tree_operand_hash_no_se,
   13159              :                                          omp_mapping_group *> **grpmap,
   13160              :                                 tree *list_p)
   13161              : {
   13162        18117 :   using namespace omp_addr_tokenizer;
   13163        18117 :   unsigned i;
   13164        18117 :   omp_mapping_group *grp;
   13165        18117 :   hash_map<tree_operand_hash, tree> *struct_map_to_clause = NULL;
   13166        18117 :   bool success = true;
   13167        18117 :   tree *new_next = NULL;
   13168        36234 :   tree *tail = &OMP_CLAUSE_CHAIN ((*groups)[groups->length () - 1].grp_end);
   13169        18117 :   tree added_nodes = NULL_TREE;
   13170        18117 :   tree *added_tail = &added_nodes;
   13171        18117 :   auto_vec<omp_mapping_group> pre_hwm_groups;
   13172              : 
   13173        67700 :   FOR_EACH_VEC_ELT (*groups, i, grp)
   13174              :     {
   13175        31466 :       tree c = grp->grp_end;
   13176        31466 :       tree decl = OMP_CLAUSE_DECL (c);
   13177        31466 :       tree grp_end = grp->grp_end;
   13178        31466 :       auto_vec<omp_addr_token *> addr_tokens;
   13179        31466 :       tree sentinel = OMP_CLAUSE_CHAIN (grp_end);
   13180              : 
   13181        31466 :       if (new_next && !grp->reprocess_struct)
   13182          625 :         grp->grp_start = new_next;
   13183              : 
   13184        31466 :       new_next = NULL;
   13185              : 
   13186        31466 :       tree *grp_start_p = grp->grp_start;
   13187              : 
   13188        31466 :       if (DECL_P (decl))
   13189        21276 :         continue;
   13190              : 
   13191              :       /* Skip groups we marked for deletion in
   13192              :          {omp,oacc}_resolve_clause_dependencies.  */
   13193        10190 :       if (grp->deleted)
   13194          462 :         continue;
   13195              : 
   13196         9728 :       if (OMP_CLAUSE_CHAIN (*grp_start_p)
   13197         9728 :           && OMP_CLAUSE_CHAIN (*grp_start_p) != grp_end)
   13198              :         {
   13199              :           /* Don't process an array descriptor that isn't inside a derived type
   13200              :              as a struct (the GOMP_MAP_POINTER following will have the form
   13201              :              "var.data", but such mappings are handled specially).  */
   13202         6140 :           tree grpmid = OMP_CLAUSE_CHAIN (*grp_start_p);
   13203         6140 :           if (omp_map_clause_descriptor_p (grpmid)
   13204        10291 :               && DECL_P (OMP_CLAUSE_DECL (grpmid)))
   13205         2807 :             continue;
   13206              :         }
   13207              : 
   13208         7965 :       tree expr = decl;
   13209              : 
   13210         7965 :       while (TREE_CODE (expr) == ARRAY_REF)
   13211         1044 :         expr = TREE_OPERAND (expr, 0);
   13212              : 
   13213         6921 :       if (!omp_parse_expr (addr_tokens, expr))
   13214            0 :         continue;
   13215              : 
   13216         6921 :       omp_addr_token *last_token
   13217         6921 :         = omp_first_chained_access_token (addr_tokens);
   13218              : 
   13219              :       /* A mapping of a reference to a pointer member that doesn't specify an
   13220              :          array section, etc., like this:
   13221              :            *mystruct.ref_to_ptr
   13222              :          should not be processed by the struct sibling-list handling code --
   13223              :          it just transfers the referenced pointer.
   13224              : 
   13225              :          In contrast, the quite similar-looking construct:
   13226              :            *mystruct.ptr
   13227              :          which is equivalent to e.g.
   13228              :            mystruct.ptr[0]
   13229              :          *does* trigger sibling-list processing.
   13230              : 
   13231              :          An exception for the former case is for "fragile" groups where the
   13232              :          reference itself is not handled otherwise; this is subject to special
   13233              :          handling in omp_accumulate_sibling_list also.  */
   13234              : 
   13235         6921 :       if (TREE_CODE (TREE_TYPE (decl)) == POINTER_TYPE
   13236         3402 :           && last_token->type == ACCESS_METHOD
   13237         3402 :           && last_token->u.access_kind == ACCESS_REF
   13238         7203 :           && !grp->fragile)
   13239          217 :         continue;
   13240              : 
   13241         6704 :       tree d = decl;
   13242         6704 :       if (TREE_CODE (d) == ARRAY_REF)
   13243              :         {
   13244         2036 :           while (TREE_CODE (d) == ARRAY_REF)
   13245         1029 :             d = TREE_OPERAND (d, 0);
   13246         1007 :           if (TREE_CODE (d) == COMPONENT_REF
   13247         1007 :               && TREE_CODE (TREE_TYPE (d)) == ARRAY_TYPE)
   13248              :             decl = d;
   13249              :         }
   13250         6704 :       if (d == decl
   13251         6010 :           && INDIRECT_REF_P (decl)
   13252          738 :           && TREE_CODE (TREE_OPERAND (decl, 0)) == COMPONENT_REF
   13253          105 :           && (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0)))
   13254              :               == REFERENCE_TYPE)
   13255         6801 :           && (OMP_CLAUSE_MAP_KIND (c)
   13256              :               != GOMP_MAP_POINTER_TO_ZERO_LENGTH_ARRAY_SECTION))
   13257           97 :         decl = TREE_OPERAND (decl, 0);
   13258              : 
   13259         6704 :       STRIP_NOPS (decl);
   13260              : 
   13261         6704 :       if (TREE_CODE (decl) != COMPONENT_REF)
   13262         1368 :         continue;
   13263              : 
   13264              :       /* If we're mapping the whole struct in another node, skip adding this
   13265              :          node to a sibling list.  */
   13266         5336 :       omp_mapping_group *wholestruct;
   13267         5336 :       if (omp_mapped_by_containing_struct (*grpmap, OMP_CLAUSE_DECL (c),
   13268              :                                            &wholestruct))
   13269              :         {
   13270          192 :           if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH_DETACH)
   13271          161 :             OMP_CLAUSE_MAP_SIZE_NEEDS_ADJUSTMENT (c) = 0;
   13272          192 :           continue;
   13273              :         }
   13274              : 
   13275         5144 :       if (OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_TO_PSET
   13276         5144 :           && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ATTACH
   13277         5039 :           && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_DETACH
   13278              :           && code != OACC_UPDATE
   13279        10116 :           && code != OMP_TARGET_UPDATE)
   13280              :         {
   13281         4781 :           if (error_operand_p (decl))
   13282              :             {
   13283            0 :               success = false;
   13284            0 :               goto error_out;
   13285              :             }
   13286              : 
   13287         4781 :           tree stype = TREE_TYPE (decl);
   13288         4781 :           if (TREE_CODE (stype) == REFERENCE_TYPE)
   13289          768 :             stype = TREE_TYPE (stype);
   13290         4781 :           if (TYPE_SIZE_UNIT (stype) == NULL
   13291         4781 :               || TREE_CODE (TYPE_SIZE_UNIT (stype)) != INTEGER_CST)
   13292              :             {
   13293            0 :               error_at (OMP_CLAUSE_LOCATION (c),
   13294              :                         "mapping field %qE of variable length "
   13295            0 :                         "structure", OMP_CLAUSE_DECL (c));
   13296            0 :               success = false;
   13297            0 :               goto error_out;
   13298              :             }
   13299              : 
   13300         4781 :           tree *inner = NULL;
   13301         4781 :           bool fragile_p = grp->fragile;
   13302              : 
   13303         4781 :           new_next
   13304         4781 :             = omp_accumulate_sibling_list (region_type, code,
   13305              :                                            struct_map_to_clause, *grpmap,
   13306              :                                            grp_start_p, grp_end, addr_tokens,
   13307              :                                            &inner, &fragile_p,
   13308              :                                            grp->reprocess_struct, &added_tail);
   13309              : 
   13310         4781 :           if (inner)
   13311              :             {
   13312          318 :               omp_mapping_group newgrp;
   13313          318 :               newgrp.grp_start = inner;
   13314          318 :               if (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (*inner))
   13315              :                   == GOMP_MAP_ATTACH_DETACH)
   13316          286 :                 newgrp.grp_end = OMP_CLAUSE_CHAIN (*inner);
   13317              :               else
   13318              :                 newgrp.grp_end = *inner;
   13319          318 :               newgrp.mark = UNVISITED;
   13320          318 :               newgrp.sibling = NULL;
   13321          318 :               newgrp.deleted = false;
   13322          318 :               newgrp.reprocess_struct = true;
   13323          318 :               newgrp.fragile = fragile_p;
   13324          318 :               newgrp.next = NULL;
   13325          318 :               groups->safe_push (newgrp);
   13326              : 
   13327              :               /* !!! Growing GROUPS might invalidate the pointers in the group
   13328              :                  map.  Rebuild it here.  This is a bit inefficient, but
   13329              :                  shouldn't happen very often.  */
   13330          636 :               delete (*grpmap);
   13331          318 :               *grpmap
   13332          318 :                 = omp_reindex_mapping_groups (list_p, groups, &pre_hwm_groups,
   13333              :                                               sentinel);
   13334              :             }
   13335              :         }
   13336        31466 :     }
   13337              : 
   13338              :   /* Delete groups marked for deletion above.  At this point the order of the
   13339              :      groups may no longer correspond to the order of the underlying list,
   13340              :      which complicates this a little.  First clear out OMP_CLAUSE_DECL for
   13341              :      deleted nodes...  */
   13342              : 
   13343        49583 :   FOR_EACH_VEC_ELT (*groups, i, grp)
   13344        31466 :     if (grp->deleted)
   13345          462 :       for (tree d = *grp->grp_start;
   13346          924 :            d != OMP_CLAUSE_CHAIN (grp->grp_end);
   13347          462 :            d = OMP_CLAUSE_CHAIN (d))
   13348          462 :         OMP_CLAUSE_DECL (d) = NULL_TREE;
   13349              : 
   13350              :   /* ...then sweep through the list removing the now-empty nodes.  */
   13351              : 
   13352              :   tail = list_p;
   13353        96593 :   while (*tail)
   13354              :     {
   13355        78476 :       if (OMP_CLAUSE_CODE (*tail) == OMP_CLAUSE_MAP
   13356        78476 :           && OMP_CLAUSE_DECL (*tail) == NULL_TREE)
   13357          462 :         *tail = OMP_CLAUSE_CHAIN (*tail);
   13358              :       else
   13359        78014 :         tail = &OMP_CLAUSE_CHAIN (*tail);
   13360              :     }
   13361              : 
   13362              :   /* Tack on the struct nodes added during nested struct reprocessing.  */
   13363        18117 :   if (added_nodes)
   13364              :     {
   13365          192 :       *tail = added_nodes;
   13366          192 :       tail = added_tail;
   13367              :     }
   13368              : 
   13369              :   /* Find each attach node whose bias needs to be adjusted and move it to the
   13370              :      group containing its pointee, right after the struct node, so that it can
   13371              :      be picked up by the adjustment code further down in this function.  */
   13372        18117 :   bool attach_bias_needs_adjustment;
   13373        18117 :   attach_bias_needs_adjustment = false;
   13374        67700 :   FOR_EACH_VEC_ELT_REVERSE (*groups, i, grp)
   13375              :     {
   13376        31466 :       tree c = *grp->grp_start;
   13377        30527 :       if (c != NULL && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
   13378        30512 :           && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_STRUCT
   13379        27430 :               || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_STRUCT_UNORD)
   13380         3364 :           && OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c)) == GOMP_MAP_TO_PSET
   13381          560 :           && OMP_CLAUSE_MAP_KIND (grp->grp_end) == GOMP_MAP_ATTACH_DETACH
   13382        32026 :           && OMP_CLAUSE_MAP_SIZE_NEEDS_ADJUSTMENT (grp->grp_end))
   13383              :         {
   13384           90 :           OMP_CLAUSE_MAP_SIZE_NEEDS_ADJUSTMENT (grp->grp_end) = 0;
   13385           90 :           attach_bias_needs_adjustment = true;
   13386           90 :           tree *cp;
   13387           90 :           for (cp = &OMP_CLAUSE_CHAIN (c); cp != NULL;
   13388          206 :                cp = &OMP_CLAUSE_CHAIN (*cp))
   13389          296 :             if (*cp == grp->grp_end)
   13390              :               {
   13391              :                 c = *cp;
   13392              :                 break;
   13393              :               }
   13394              : 
   13395           90 :           tree base = OMP_CLAUSE_DECL (c);
   13396           90 :           gcc_assert (TREE_CODE (base) == NOP_EXPR);
   13397           90 :           base = build_fold_indirect_ref (base);
   13398           90 :           tree *struct_node = struct_map_to_clause->get (base);
   13399           90 :           omp_siblist_move_node_after (c, cp, &OMP_CLAUSE_CHAIN (*struct_node));
   13400              :         }
   13401              :     }
   13402              : 
   13403              :   /* Now we have finished building the struct sibling lists, reprocess
   13404              :      newly-added "attach" nodes: we need the address of the first
   13405              :      mapped element of each struct sibling list for the bias of the attach
   13406              :      operation -- not necessarily the base address of the whole struct.  */
   13407        18117 :   if (struct_map_to_clause)
   13408         3071 :     for (hash_map<tree_operand_hash, tree>::iterator iter
   13409         2507 :            = struct_map_to_clause->begin ();
   13410         8085 :          iter != struct_map_to_clause->end ();
   13411         3071 :          ++iter)
   13412              :       {
   13413         3071 :         tree struct_node = (*iter).second;
   13414         3071 :         gcc_assert (OMP_CLAUSE_CODE (struct_node) == OMP_CLAUSE_MAP);
   13415         3071 :         tree attach = OMP_CLAUSE_CHAIN (struct_node);
   13416              : 
   13417         3071 :         if (OMP_CLAUSE_CODE (attach) != OMP_CLAUSE_MAP
   13418         3071 :             || OMP_CLAUSE_MAP_KIND (attach) != GOMP_MAP_ATTACH_DETACH)
   13419         2559 :           continue;
   13420              : 
   13421          512 :         OMP_CLAUSE_SET_MAP_KIND (attach, GOMP_MAP_ATTACH);
   13422              : 
   13423              :         /* Sanity check: the standalone attach node will not work if we have
   13424              :            an "enter data" operation (because for those, variables need to be
   13425              :            mapped separately and attach nodes must be grouped together with the
   13426              :            base they attach to).  We should only have created the
   13427              :            ATTACH_DETACH node either after GOMP_MAP_STRUCT for a target region
   13428              :            or for an intermediate descriptor that needs adjustment -- so this
   13429              :            should never be true.  */
   13430          512 :         gcc_assert ((region_type & ORT_TARGET) != 0
   13431              :                     || attach_bias_needs_adjustment);
   13432              : 
   13433              :         /* This is the first sorted node in the struct sibling list.  Use it
   13434              :            to recalculate the correct bias to use.
   13435              :            (&first_node - attach_decl).
   13436              :            For GOMP_MAP_STRUCT_UNORD, we need e.g. the
   13437              :            min(min(min(first,second),third),fourth) element, because the
   13438              :            elements aren't in any particular order.  */
   13439          512 :         tree lowest_addr;
   13440          512 :         if (OMP_CLAUSE_MAP_KIND (struct_node) == GOMP_MAP_STRUCT_UNORD)
   13441              :           {
   13442           94 :             tree first_node = OMP_CLAUSE_CHAIN (attach);
   13443           94 :             unsigned HOST_WIDE_INT num_mappings
   13444           94 :               = tree_to_uhwi (OMP_CLAUSE_SIZE (struct_node));
   13445           94 :             lowest_addr = OMP_CLAUSE_DECL (first_node);
   13446           94 :             lowest_addr = build_fold_addr_expr (lowest_addr);
   13447           94 :             lowest_addr = fold_convert (pointer_sized_int_node, lowest_addr);
   13448           94 :             tree next_node = OMP_CLAUSE_CHAIN (first_node);
   13449          185 :             while (num_mappings > 1)
   13450              :               {
   13451           91 :                 tree tmp = OMP_CLAUSE_DECL (next_node);
   13452           91 :                 tmp = build_fold_addr_expr (tmp);
   13453           91 :                 tmp = fold_convert (pointer_sized_int_node, tmp);
   13454           91 :                 lowest_addr = fold_build2 (MIN_EXPR, pointer_sized_int_node,
   13455              :                                            lowest_addr, tmp);
   13456           91 :                 next_node = OMP_CLAUSE_CHAIN (next_node);
   13457           91 :                 num_mappings--;
   13458              :               }
   13459           94 :             lowest_addr = fold_convert (ptrdiff_type_node, lowest_addr);
   13460              :           }
   13461              :         else
   13462              :           {
   13463          418 :             tree first_node = OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (attach));
   13464          418 :             first_node = build_fold_addr_expr (first_node);
   13465          418 :             lowest_addr = fold_convert (ptrdiff_type_node, first_node);
   13466              :           }
   13467          512 :         tree attach_decl = OMP_CLAUSE_DECL (attach);
   13468          512 :         attach_decl = fold_convert (ptrdiff_type_node, attach_decl);
   13469          512 :         OMP_CLAUSE_SIZE (attach)
   13470          512 :           = fold_build2 (MINUS_EXPR, ptrdiff_type_node, lowest_addr,
   13471              :                          attach_decl);
   13472              : 
   13473              :         /* Remove GOMP_MAP_ATTACH node from after struct node.  */
   13474          512 :         OMP_CLAUSE_CHAIN (struct_node) = OMP_CLAUSE_CHAIN (attach);
   13475              :         /* ...and re-insert it at the end of our clause list.  */
   13476          512 :         *tail = attach;
   13477          512 :         OMP_CLAUSE_CHAIN (attach) = NULL_TREE;
   13478          512 :         tail = &OMP_CLAUSE_CHAIN (attach);
   13479              :       }
   13480              : 
   13481        15610 : error_out:
   13482        18117 :   if (struct_map_to_clause)
   13483         2507 :     delete struct_map_to_clause;
   13484              : 
   13485        18117 :   return success;
   13486        18117 : }
   13487              : 
   13488              : struct instantiate_mapper_info
   13489              : {
   13490              :   tree *mapper_clauses_p;
   13491              :   struct gimplify_omp_ctx *omp_ctx;
   13492              :   gimple_seq *pre_p;
   13493              : };
   13494              : 
   13495              : /* Helper function for omp_instantiate_mapper.  */
   13496              : 
   13497              : static tree
   13498         1592 : remap_mapper_decl_1 (tree *tp, int *walk_subtrees, void *data)
   13499              : {
   13500         1592 :   copy_body_data *id = (copy_body_data *) data;
   13501              : 
   13502         1592 :   if (DECL_P (*tp))
   13503              :     {
   13504          724 :       tree replacement = remap_decl (*tp, id);
   13505          724 :       if (*tp != replacement)
   13506              :         {
   13507          369 :           *tp = unshare_expr (replacement);
   13508          369 :           *walk_subtrees = 0;
   13509              :         }
   13510              :     }
   13511              : 
   13512         1592 :   return NULL_TREE;
   13513              : }
   13514              : 
   13515              : /* A copy_decl implementation (for use with tree-inline.cc functions) that
   13516              :    only transform decls or SSA names that are part of a map we already
   13517              :    prepared.  */
   13518              : 
   13519              : static tree
   13520          126 : omp_mapper_copy_decl (tree var, copy_body_data *cb)
   13521              : {
   13522          126 :   tree *repl = cb->decl_map->get (var);
   13523              : 
   13524          126 :   if (repl)
   13525            0 :     return *repl;
   13526              : 
   13527          126 :   return var;
   13528              : }
   13529              : 
   13530              : static tree *
   13531           90 : omp_instantiate_mapper (gimple_seq *pre_p,
   13532              :                         hash_map<omp_name_type<tree>, tree> *implicit_mappers,
   13533              :                         tree mapperfn, tree expr, enum gomp_map_kind outer_kind,
   13534              :                         tree *mapper_clauses_p)
   13535              : {
   13536           90 :   tree mapper_name = NULL_TREE;
   13537           90 :   tree mapper = lang_hooks.decls.omp_extract_mapper_directive (mapperfn);
   13538           90 :   gcc_assert (TREE_CODE (mapper) == OMP_DECLARE_MAPPER);
   13539              : 
   13540           90 :   tree clause = OMP_DECLARE_MAPPER_CLAUSES (mapper);
   13541           90 :   tree dummy_var = OMP_DECLARE_MAPPER_DECL (mapper);
   13542              : 
   13543              :   /* The "extraction map" is used to map the mapper variable in the "declare
   13544              :      mapper" directive, and also any temporary variables that have been created
   13545              :      as part of expanding the mapper function's body (which are expanded as a
   13546              :      "bind" expression in the pre_p sequence).  */
   13547           90 :   hash_map<tree, tree> extraction_map;
   13548              : 
   13549           90 :   extraction_map.put (dummy_var, expr);
   13550           90 :   extraction_map.put (expr, expr);
   13551              : 
   13552              :   /* This copy_body_data is only used to remap the decls in the
   13553              :      OMP_DECLARE_MAPPER tree node expansion itself.  All relevant decls should
   13554              :      already be in the current function.  */
   13555           90 :   copy_body_data id;
   13556           90 :   memset (&id, 0, sizeof (id));
   13557           90 :   id.src_fn = current_function_decl;
   13558           90 :   id.dst_fn = current_function_decl;
   13559           90 :   id.src_cfun = cfun;
   13560           90 :   id.decl_map = &extraction_map;
   13561           90 :   id.copy_decl = omp_mapper_copy_decl;
   13562           90 :   id.transform_call_graph_edges = CB_CGE_DUPLICATE; // ???
   13563           90 :   id.transform_new_cfg = true; // ???
   13564              : 
   13565          282 :   for (; clause; clause = OMP_CLAUSE_CHAIN (clause))
   13566              :     {
   13567          192 :       enum gomp_map_kind map_kind = OMP_CLAUSE_MAP_KIND (clause);
   13568          192 :       tree *nested_mapper_p = NULL;
   13569              : 
   13570          192 :       if (map_kind == GOMP_MAP_PUSH_MAPPER_NAME)
   13571              :         {
   13572            0 :           mapper_name = OMP_CLAUSE_DECL (clause);
   13573           25 :           continue;
   13574              :         }
   13575          192 :       else if (map_kind == GOMP_MAP_POP_MAPPER_NAME)
   13576              :         {
   13577            0 :           mapper_name = NULL_TREE;
   13578            0 :           continue;
   13579              :         }
   13580              : 
   13581          192 :       if (OMP_CLAUSE_HAS_ITERATORS (clause))
   13582              :         {
   13583            2 :           sorry_at (OMP_CLAUSE_LOCATION (clause),
   13584              :                     "user-defined mapper that uses a %<map%> clause "
   13585              :                     "with %<iterator%>");
   13586            2 :           continue;
   13587              :         }
   13588              : 
   13589          190 :       tree decl = OMP_CLAUSE_DECL (clause);
   13590          190 :       tree unshared, type;
   13591          190 :       bool nonunit_array_with_mapper = false;
   13592              : 
   13593          190 :       if (TREE_CODE (decl) == OMP_ARRAY_SECTION)
   13594              :         {
   13595           62 :           location_t loc = OMP_CLAUSE_LOCATION (clause);
   13596           62 :           tree tmp = lang_hooks.decls.omp_map_array_section (loc, decl);
   13597           62 :           if (tmp == decl)
   13598              :             {
   13599           48 :               unshared = unshare_expr (clause);
   13600           48 :               nonunit_array_with_mapper = true;
   13601           48 :               type = TREE_TYPE (TREE_TYPE (decl));
   13602              :             }
   13603              :           else
   13604              :             {
   13605           14 :               unshared = build_omp_clause (OMP_CLAUSE_LOCATION (clause),
   13606           14 :                                            OMP_CLAUSE_CODE (clause));
   13607           14 :               OMP_CLAUSE_DECL (unshared) = tmp;
   13608           14 :               OMP_CLAUSE_SIZE (unshared)
   13609           28 :                 = DECL_P (tmp) ? DECL_SIZE_UNIT (tmp)
   13610           14 :                                : TYPE_SIZE_UNIT (TREE_TYPE (tmp));
   13611           14 :               type = TREE_TYPE (tmp);
   13612              :             }
   13613              :         }
   13614              :       else
   13615              :         {
   13616          128 :           unshared = unshare_expr (clause);
   13617          128 :           type = TREE_TYPE (decl);
   13618              :         }
   13619              : 
   13620          190 :       walk_tree (&unshared, remap_mapper_decl_1, &id, NULL);
   13621              : 
   13622          190 :       if (OMP_CLAUSE_MAP_KIND (unshared) == GOMP_MAP_UNSET)
   13623           28 :         OMP_CLAUSE_SET_MAP_KIND (unshared, outer_kind);
   13624              : 
   13625          190 :       decl = OMP_CLAUSE_DECL (unshared);
   13626          190 :       type = TYPE_MAIN_VARIANT (type);
   13627              : 
   13628          190 :       nested_mapper_p = implicit_mappers->get ({ mapper_name, type });
   13629              : 
   13630          190 :       if (nested_mapper_p && *nested_mapper_p != mapperfn)
   13631              :         {
   13632           23 :           if (nonunit_array_with_mapper)
   13633              :             {
   13634            8 :               sorry ("user-defined mapper with non-unit length array section");
   13635            8 :               continue;
   13636              :             }
   13637              : 
   13638           15 :           if (map_kind == GOMP_MAP_UNSET)
   13639            0 :             map_kind = outer_kind;
   13640              : 
   13641           15 :           mapper_clauses_p
   13642           15 :             = omp_instantiate_mapper (pre_p, implicit_mappers,
   13643              :                                       *nested_mapper_p, decl, map_kind,
   13644              :                                       mapper_clauses_p);
   13645           15 :           continue;
   13646              :         }
   13647              : 
   13648          167 :       *mapper_clauses_p = unshared;
   13649          167 :       mapper_clauses_p = &OMP_CLAUSE_CHAIN (unshared);
   13650              :     }
   13651              : 
   13652           90 :   return mapper_clauses_p;
   13653           90 : }
   13654              : 
   13655              : static int
   13656       138968 : omp_instantiate_implicit_mappers (splay_tree_node n, void *data)
   13657              : {
   13658       138968 :   tree decl = (tree) n->key;
   13659       138968 :   instantiate_mapper_info *im_info = (instantiate_mapper_info *) data;
   13660       138968 :   gimplify_omp_ctx *ctx = im_info->omp_ctx;
   13661       138968 :   tree *mapper_p = NULL;
   13662       138968 :   tree type = TREE_TYPE (decl);
   13663       138968 :   bool ref_p = false;
   13664       138968 :   unsigned flags = n->value;
   13665              : 
   13666       138968 :   if (flags & (GOVD_EXPLICIT | GOVD_LOCAL))
   13667              :     return 0;
   13668        23762 :   if ((flags & GOVD_SEEN) == 0)
   13669              :     return 0;
   13670              :   /* If we already have clauses pertaining to a struct variable, then we don't
   13671              :      want to implicitly invoke a user-defined mapper.  */
   13672        20202 :   if ((flags & GOVD_EXPLICIT) != 0 && AGGREGATE_TYPE_P (TREE_TYPE (decl)))
   13673              :     return 0;
   13674              : 
   13675        20202 :   if (TREE_CODE (type) == REFERENCE_TYPE)
   13676              :     {
   13677          893 :       ref_p = true;
   13678          893 :       type = TREE_TYPE (type);
   13679              :     }
   13680              : 
   13681        20202 :   type = TYPE_MAIN_VARIANT (type);
   13682              : 
   13683        20202 :   if (DECL_P (decl) && type && AGGREGATE_TYPE_P (type))
   13684              :     {
   13685         5330 :       gcc_assert (ctx);
   13686         5330 :       mapper_p = ctx->implicit_mappers->get ({ NULL_TREE, type });
   13687              :     }
   13688              : 
   13689         5330 :   if (mapper_p)
   13690              :     {
   13691              :       /* If we have a reference, map the pointed-to object rather than the
   13692              :          reference itself.  */
   13693           75 :       if (ref_p)
   13694            2 :         decl = build_fold_indirect_ref (decl);
   13695              : 
   13696           75 :       im_info->mapper_clauses_p
   13697           75 :         = omp_instantiate_mapper (im_info->pre_p, ctx->implicit_mappers,
   13698              :                                   *mapper_p, decl, GOMP_MAP_TOFROM,
   13699              :                                   im_info->mapper_clauses_p);
   13700              :       /* Make sure we don't map the same variable implicitly in
   13701              :          gimplify_adjust_omp_clauses_1 also.  */
   13702           75 :       n->value |= GOVD_EXPLICIT;
   13703              :     }
   13704              : 
   13705              :   return 0;
   13706              : }
   13707              : 
   13708              : /* Scan the OMP clauses in *LIST_P, installing mappings into a new
   13709              :    and previous omp contexts.  */
   13710              : 
   13711              : static void
   13712       131912 : gimplify_scan_omp_clauses (tree *list_p, gimple_seq *pre_p,
   13713              :                            enum omp_region_type region_type,
   13714              :                            enum tree_code code,
   13715              :                            gimple_seq *loops_seq_p = NULL)
   13716              : {
   13717       131912 :   using namespace omp_addr_tokenizer;
   13718       131912 :   struct gimplify_omp_ctx *ctx, *outer_ctx;
   13719       131912 :   tree c;
   13720       131912 :   tree *orig_list_p = list_p;
   13721       131912 :   int handled_depend_iterators = -1;
   13722       131912 :   int nowait = -1;
   13723              : 
   13724       131912 :   ctx = new_omp_context (region_type);
   13725       131912 :   ctx->code = code;
   13726       131912 :   outer_ctx = ctx->outer_context;
   13727       131912 :   if (code == OMP_TARGET)
   13728              :     {
   13729        13631 :       if (!lang_GNU_Fortran ())
   13730        11219 :         ctx->defaultmap[GDMK_POINTER] = GOVD_MAP | GOVD_MAP_0LEN_ARRAY;
   13731        13631 :       ctx->defaultmap[GDMK_SCALAR] = GOVD_FIRSTPRIVATE;
   13732        27262 :       ctx->defaultmap[GDMK_SCALAR_TARGET] = (lang_GNU_Fortran ()
   13733        13631 :                                              ? GOVD_MAP : GOVD_FIRSTPRIVATE);
   13734              :     }
   13735       131912 :   if (!lang_GNU_Fortran ())
   13736       100383 :     switch (code)
   13737              :       {
   13738        18722 :       case OMP_TARGET:
   13739        18722 :       case OMP_TARGET_DATA:
   13740        18722 :       case OMP_TARGET_ENTER_DATA:
   13741        18722 :       case OMP_TARGET_EXIT_DATA:
   13742        18722 :       case OACC_DECLARE:
   13743        18722 :       case OACC_HOST_DATA:
   13744        18722 :       case OACC_PARALLEL:
   13745        18722 :       case OACC_KERNELS:
   13746        18722 :         ctx->target_firstprivatize_array_bases = true;
   13747              :       default:
   13748              :         break;
   13749              :       }
   13750              : 
   13751       131912 :   vec<omp_mapping_group> *groups = NULL;
   13752       131912 :   hash_map<tree_operand_hash_no_se, omp_mapping_group *> *grpmap = NULL;
   13753       131912 :   unsigned grpnum = 0;
   13754       131912 :   tree *grp_start_p = NULL, grp_end = NULL_TREE;
   13755              : 
   13756       131912 :   if (code == OMP_TARGET
   13757       131912 :       || code == OMP_TARGET_DATA
   13758       131912 :       || code == OMP_TARGET_ENTER_DATA
   13759              :       || code == OMP_TARGET_EXIT_DATA
   13760              :       || code == OACC_DATA
   13761              :       || code == OACC_KERNELS
   13762              :       || code == OACC_PARALLEL
   13763              :       || code == OACC_SERIAL
   13764              :       || code == OACC_ENTER_DATA
   13765              :       || code == OACC_EXIT_DATA
   13766              :       || code == OACC_UPDATE
   13767              :       || code == OACC_DECLARE)
   13768              :     {
   13769        34065 :       if (!(region_type & ORT_ACC))
   13770        17553 :         *list_p = omp_remove_duplicate_maps (*list_p, false);
   13771        34065 :       groups = omp_gather_mapping_groups (list_p);
   13772              : 
   13773        34065 :       if (groups)
   13774        18318 :         grpmap = omp_index_mapping_groups (groups);
   13775              :     }
   13776              : 
   13777       339231 :   while ((c = *list_p) != NULL)
   13778              :     {
   13779       207319 :       bool remove = false;
   13780       207319 :       bool notice_outer = true;
   13781       207319 :       bool map_descriptor;
   13782       207319 :       const char *check_non_private = NULL;
   13783       207319 :       unsigned int flags;
   13784       207319 :       tree decl;
   13785       207319 :       auto_vec<omp_addr_token *, 10> addr_tokens;
   13786       207319 :       tree op = NULL_TREE;
   13787       207319 :       location_t loc = OMP_CLAUSE_LOCATION (c);
   13788              : 
   13789       244960 :       if (grp_end && c == OMP_CLAUSE_CHAIN (grp_end))
   13790              :         {
   13791              :           grp_start_p = NULL;
   13792              :           grp_end = NULL_TREE;
   13793              :         }
   13794              : 
   13795       207319 :       if (code == OMP_TARGET
   13796              :           || code == OMP_TARGET_DATA
   13797              :           || code == OMP_TARGET_ENTER_DATA
   13798       173992 :           || code == OMP_TARGET_EXIT_DATA)
   13799              :         /* Do some target-specific type checks for map operands.  */
   13800        35760 :         switch (OMP_CLAUSE_CODE (c))
   13801              :           {
   13802        25878 :           case OMP_CLAUSE_MAP:
   13803        25878 :             op = OMP_CLAUSE_OPERAND (c, 0);
   13804        25878 :             verify_type_context (loc, TCTX_OMP_MAP, TREE_TYPE (op));
   13805        25878 :             break;
   13806          186 :           case OMP_CLAUSE_PRIVATE:
   13807          186 :             op = OMP_CLAUSE_OPERAND (c, 0);
   13808          186 :             verify_type_context (loc, TCTX_OMP_PRIVATE, TREE_TYPE (op));
   13809          186 :             break;
   13810         1516 :           case OMP_CLAUSE_FIRSTPRIVATE:
   13811         1516 :             op = OMP_CLAUSE_OPERAND (c, 0);
   13812         1516 :             verify_type_context (loc, TCTX_OMP_FIRSTPRIVATE, TREE_TYPE (op));
   13813         1516 :             break;
   13814         2923 :           case OMP_CLAUSE_IS_DEVICE_PTR:
   13815         2923 :           case OMP_CLAUSE_USE_DEVICE_ADDR:
   13816         2923 :           case OMP_CLAUSE_USE_DEVICE_PTR:
   13817         2923 :           case OMP_CLAUSE_HAS_DEVICE_ADDR:
   13818         2923 :             op = OMP_CLAUSE_OPERAND (c, 0);
   13819         2923 :             verify_type_context (loc, TCTX_OMP_DEVICE_ADDR, TREE_TYPE (op));
   13820         2923 :             break;
   13821              :           default:
   13822              :             break;
   13823              :           }
   13824              : 
   13825       207319 :       switch (OMP_CLAUSE_CODE (c))
   13826              :         {
   13827        12309 :         case OMP_CLAUSE_PRIVATE:
   13828        12309 :           flags = GOVD_PRIVATE | GOVD_EXPLICIT;
   13829        12309 :           if (lang_hooks.decls.omp_private_outer_ref (OMP_CLAUSE_DECL (c)))
   13830              :             {
   13831          167 :               flags |= GOVD_PRIVATE_OUTER_REF;
   13832          167 :               OMP_CLAUSE_PRIVATE_OUTER_REF (c) = 1;
   13833              :             }
   13834              :           else
   13835              :             notice_outer = false;
   13836        12309 :           goto do_add;
   13837         5571 :         case OMP_CLAUSE_SHARED:
   13838         5571 :           flags = GOVD_SHARED | GOVD_EXPLICIT;
   13839         5571 :           goto do_add;
   13840         8229 :         case OMP_CLAUSE_FIRSTPRIVATE:
   13841         8229 :           flags = GOVD_FIRSTPRIVATE | GOVD_EXPLICIT;
   13842         8229 :           check_non_private = "firstprivate";
   13843         8229 :           if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
   13844              :             {
   13845          380 :               gcc_assert (code == OMP_TARGET);
   13846              :               flags |= GOVD_FIRSTPRIVATE_IMPLICIT;
   13847              :             }
   13848         8229 :           goto do_add;
   13849         7338 :         case OMP_CLAUSE_LASTPRIVATE:
   13850         7338 :           if (OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
   13851          496 :             switch (code)
   13852              :               {
   13853           25 :               case OMP_DISTRIBUTE:
   13854           25 :                 error_at (OMP_CLAUSE_LOCATION (c),
   13855              :                           "conditional %<lastprivate%> clause on "
   13856              :                           "%qs construct", "distribute");
   13857           25 :                 OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c) = 0;
   13858           25 :                 break;
   13859           13 :               case OMP_TASKLOOP:
   13860           13 :                 error_at (OMP_CLAUSE_LOCATION (c),
   13861              :                           "conditional %<lastprivate%> clause on "
   13862              :                           "%qs construct", "taskloop");
   13863           13 :                 OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c) = 0;
   13864           13 :                 break;
   13865              :               default:
   13866              :                 break;
   13867              :               }
   13868         7338 :           flags = GOVD_LASTPRIVATE | GOVD_SEEN | GOVD_EXPLICIT;
   13869         7338 :           if (code != OMP_LOOP)
   13870         6987 :             check_non_private = "lastprivate";
   13871         7338 :           decl = OMP_CLAUSE_DECL (c);
   13872         7338 :           if (error_operand_p (decl))
   13873            0 :             goto do_add;
   13874         7338 :           if (OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c)
   13875         7338 :               && !lang_hooks.decls.omp_scalar_p (decl, true))
   13876              :             {
   13877            5 :               error_at (OMP_CLAUSE_LOCATION (c),
   13878              :                         "non-scalar variable %qD in conditional "
   13879              :                         "%<lastprivate%> clause", decl);
   13880            5 :               OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c) = 0;
   13881              :             }
   13882         7338 :           if (OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
   13883          453 :             flags |= GOVD_LASTPRIVATE_CONDITIONAL;
   13884         7338 :           omp_lastprivate_for_combined_outer_constructs (outer_ctx, decl,
   13885              :                                                          false);
   13886         7338 :           goto do_add;
   13887        15402 :         case OMP_CLAUSE_REDUCTION:
   13888        15402 :           if (OMP_CLAUSE_REDUCTION_TASK (c))
   13889              :             {
   13890          612 :               if (region_type == ORT_WORKSHARE || code == OMP_SCOPE)
   13891              :                 {
   13892          417 :                   if (nowait == -1)
   13893          302 :                     nowait = omp_find_clause (*list_p,
   13894          302 :                                               OMP_CLAUSE_NOWAIT) != NULL_TREE;
   13895          417 :                   if (nowait
   13896           15 :                       && (outer_ctx == NULL
   13897            0 :                           || outer_ctx->region_type != ORT_COMBINED_PARALLEL))
   13898              :                     {
   13899           15 :                       error_at (OMP_CLAUSE_LOCATION (c),
   13900              :                                 "%<task%> reduction modifier on a construct "
   13901              :                                 "with a %<nowait%> clause");
   13902           15 :                       OMP_CLAUSE_REDUCTION_TASK (c) = 0;
   13903              :                     }
   13904              :                 }
   13905          195 :               else if ((region_type & ORT_PARALLEL) != ORT_PARALLEL)
   13906              :                 {
   13907           40 :                   error_at (OMP_CLAUSE_LOCATION (c),
   13908              :                             "invalid %<task%> reduction modifier on construct "
   13909              :                             "other than %<parallel%>, %qs, %<sections%> or "
   13910           20 :                             "%<scope%>", lang_GNU_Fortran () ? "do" : "for");
   13911           20 :                   OMP_CLAUSE_REDUCTION_TASK (c) = 0;
   13912              :                 }
   13913              :             }
   13914        15402 :           if (OMP_CLAUSE_REDUCTION_INSCAN (c))
   13915          851 :             switch (code)
   13916              :               {
   13917            4 :               case OMP_SECTIONS:
   13918            4 :                 error_at (OMP_CLAUSE_LOCATION (c),
   13919              :                           "%<inscan%> %<reduction%> clause on "
   13920              :                           "%qs construct", "sections");
   13921            4 :                 OMP_CLAUSE_REDUCTION_INSCAN (c) = 0;
   13922            4 :                 break;
   13923            4 :               case OMP_PARALLEL:
   13924            4 :                 error_at (OMP_CLAUSE_LOCATION (c),
   13925              :                           "%<inscan%> %<reduction%> clause on "
   13926              :                           "%qs construct", "parallel");
   13927            4 :                 OMP_CLAUSE_REDUCTION_INSCAN (c) = 0;
   13928            4 :                 break;
   13929            4 :               case OMP_TEAMS:
   13930            4 :                 error_at (OMP_CLAUSE_LOCATION (c),
   13931              :                           "%<inscan%> %<reduction%> clause on "
   13932              :                           "%qs construct", "teams");
   13933            4 :                 OMP_CLAUSE_REDUCTION_INSCAN (c) = 0;
   13934            4 :                 break;
   13935            4 :               case OMP_TASKLOOP:
   13936            4 :                 error_at (OMP_CLAUSE_LOCATION (c),
   13937              :                           "%<inscan%> %<reduction%> clause on "
   13938              :                           "%qs construct", "taskloop");
   13939            4 :                 OMP_CLAUSE_REDUCTION_INSCAN (c) = 0;
   13940            4 :                 break;
   13941            4 :               case OMP_SCOPE:
   13942            4 :                 error_at (OMP_CLAUSE_LOCATION (c),
   13943              :                           "%<inscan%> %<reduction%> clause on "
   13944              :                           "%qs construct", "scope");
   13945            4 :                 OMP_CLAUSE_REDUCTION_INSCAN (c) = 0;
   13946            4 :                 break;
   13947              :               default:
   13948              :                 break;
   13949              :               }
   13950              :           /* FALLTHRU */
   13951        18031 :         case OMP_CLAUSE_IN_REDUCTION:
   13952        18031 :         case OMP_CLAUSE_TASK_REDUCTION:
   13953        18031 :           flags = GOVD_REDUCTION | GOVD_SEEN | GOVD_EXPLICIT;
   13954              :           /* OpenACC permits reductions on private variables.  */
   13955        18031 :           if (!(region_type & ORT_ACC)
   13956              :               /* taskgroup is actually not a worksharing region.  */
   13957        12512 :               && code != OMP_TASKGROUP)
   13958        11982 :             check_non_private = omp_clause_code_name[OMP_CLAUSE_CODE (c)];
   13959        18031 :           decl = OMP_CLAUSE_DECL (c);
   13960        18031 :           if (TREE_CODE (decl) == MEM_REF)
   13961              :             {
   13962         2614 :               tree type = TREE_TYPE (decl);
   13963         2614 :               bool saved_into_ssa = gimplify_ctxp->into_ssa;
   13964         2614 :               gimplify_ctxp->into_ssa = false;
   13965         2614 :               if (gimplify_expr (&TYPE_MAX_VALUE (TYPE_DOMAIN (type)), pre_p,
   13966              :                                  NULL, is_gimple_val, fb_rvalue, false)
   13967              :                   == GS_ERROR)
   13968              :                 {
   13969            0 :                   gimplify_ctxp->into_ssa = saved_into_ssa;
   13970            0 :                   remove = true;
   13971            0 :                   break;
   13972              :                 }
   13973         2614 :               gimplify_ctxp->into_ssa = saved_into_ssa;
   13974         2614 :               tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
   13975         2614 :               if (DECL_P (v))
   13976              :                 {
   13977          571 :                   omp_firstprivatize_variable (ctx, v);
   13978          571 :                   omp_notice_variable (ctx, v, true);
   13979              :                 }
   13980         2614 :               decl = TREE_OPERAND (decl, 0);
   13981         2614 :               if (TREE_CODE (decl) == POINTER_PLUS_EXPR)
   13982              :                 {
   13983          477 :                   gimplify_ctxp->into_ssa = false;
   13984          477 :                   if (gimplify_expr (&TREE_OPERAND (decl, 1), pre_p,
   13985              :                                      NULL, is_gimple_val, fb_rvalue, false)
   13986              :                       == GS_ERROR)
   13987              :                     {
   13988            0 :                       gimplify_ctxp->into_ssa = saved_into_ssa;
   13989            0 :                       remove = true;
   13990            0 :                       break;
   13991              :                     }
   13992          477 :                   gimplify_ctxp->into_ssa = saved_into_ssa;
   13993          477 :                   v = TREE_OPERAND (decl, 1);
   13994          477 :                   if (DECL_P (v))
   13995              :                     {
   13996          477 :                       omp_firstprivatize_variable (ctx, v);
   13997          477 :                       omp_notice_variable (ctx, v, true);
   13998              :                     }
   13999          477 :                   decl = TREE_OPERAND (decl, 0);
   14000              :                 }
   14001         2614 :               if (TREE_CODE (decl) == ADDR_EXPR
   14002         1220 :                   || TREE_CODE (decl) == INDIRECT_REF)
   14003         1510 :                 decl = TREE_OPERAND (decl, 0);
   14004              :             }
   14005        18031 :           goto do_add_decl;
   14006         2698 :         case OMP_CLAUSE_LINEAR:
   14007         2698 :           if (gimplify_expr (&OMP_CLAUSE_LINEAR_STEP (c), pre_p, NULL,
   14008              :                              is_gimple_val, fb_rvalue) == GS_ERROR)
   14009              :             {
   14010              :               remove = true;
   14011              :               break;
   14012              :             }
   14013              :           else
   14014              :             {
   14015         2698 :               if (code == OMP_SIMD
   14016         2698 :                   && !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
   14017              :                 {
   14018         1154 :                   struct gimplify_omp_ctx *octx = outer_ctx;
   14019         1154 :                   if (octx
   14020          741 :                       && octx->region_type == ORT_WORKSHARE
   14021          500 :                       && octx->combined_loop
   14022          500 :                       && !octx->distribute)
   14023              :                     {
   14024          492 :                       if (octx->outer_context
   14025          430 :                           && (octx->outer_context->region_type
   14026              :                               == ORT_COMBINED_PARALLEL))
   14027          393 :                         octx = octx->outer_context->outer_context;
   14028              :                       else
   14029              :                         octx = octx->outer_context;
   14030              :                     }
   14031          905 :                   if (octx
   14032          316 :                       && octx->region_type == ORT_WORKSHARE
   14033           16 :                       && octx->combined_loop
   14034           16 :                       && octx->distribute)
   14035              :                     {
   14036           16 :                       error_at (OMP_CLAUSE_LOCATION (c),
   14037              :                                 "%<linear%> clause for variable other than "
   14038              :                                 "loop iterator specified on construct "
   14039              :                                 "combined with %<distribute%>");
   14040           16 :                       remove = true;
   14041           16 :                       break;
   14042              :                     }
   14043              :                 }
   14044              :               /* For combined #pragma omp parallel for simd, need to put
   14045              :                  lastprivate and perhaps firstprivate too on the
   14046              :                  parallel.  Similarly for #pragma omp for simd.  */
   14047              :               struct gimplify_omp_ctx *octx = outer_ctx;
   14048              :               bool taskloop_seen = false;
   14049              :               decl = NULL_TREE;
   14050         3498 :               do
   14051              :                 {
   14052         3498 :                   if (OMP_CLAUSE_LINEAR_NO_COPYIN (c)
   14053         3498 :                       && OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
   14054              :                     break;
   14055         3438 :                   decl = OMP_CLAUSE_DECL (c);
   14056         3438 :                   if (error_operand_p (decl))
   14057              :                     {
   14058              :                       decl = NULL_TREE;
   14059              :                       break;
   14060              :                     }
   14061         3438 :                   flags = GOVD_SEEN;
   14062         3438 :                   if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c))
   14063         1672 :                     flags |= GOVD_FIRSTPRIVATE;
   14064         3438 :                   if (!OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
   14065         3438 :                     flags |= GOVD_LASTPRIVATE;
   14066         3438 :                   if (octx
   14067         2548 :                       && octx->region_type == ORT_WORKSHARE
   14068         1233 :                       && octx->combined_loop)
   14069              :                     {
   14070         1227 :                       if (octx->outer_context
   14071         1055 :                           && (octx->outer_context->region_type
   14072              :                               == ORT_COMBINED_PARALLEL))
   14073              :                         octx = octx->outer_context;
   14074          488 :                       else if (omp_check_private (octx, decl, false))
   14075              :                         break;
   14076              :                     }
   14077              :                   else if (octx
   14078         1321 :                            && (octx->region_type & ORT_TASK) != 0
   14079          307 :                            && octx->combined_loop)
   14080              :                     taskloop_seen = true;
   14081              :                   else if (octx
   14082         1018 :                            && octx->region_type == ORT_COMBINED_PARALLEL
   14083          301 :                            && ((ctx->region_type == ORT_WORKSHARE
   14084          201 :                                 && octx == outer_ctx)
   14085          100 :                                || taskloop_seen))
   14086              :                     flags = GOVD_SEEN | GOVD_SHARED;
   14087              :                   else if (octx
   14088          717 :                            && ((octx->region_type & ORT_COMBINED_TEAMS)
   14089              :                                == ORT_COMBINED_TEAMS))
   14090              :                     flags = GOVD_SEEN | GOVD_SHARED;
   14091          552 :                   else if (octx
   14092          552 :                            && octx->region_type == ORT_COMBINED_TARGET)
   14093              :                     {
   14094          201 :                       if (flags & GOVD_LASTPRIVATE)
   14095          201 :                         flags = GOVD_SEEN | GOVD_MAP;
   14096              :                     }
   14097              :                   else
   14098              :                     break;
   14099         2105 :                   splay_tree_node on
   14100         2105 :                     = splay_tree_lookup (octx->variables,
   14101              :                                          (splay_tree_key) decl);
   14102         2105 :                   if (on && (on->value & GOVD_DATA_SHARE_CLASS) != 0)
   14103              :                     {
   14104              :                       octx = NULL;
   14105              :                       break;
   14106              :                     }
   14107         2101 :                   omp_add_variable (octx, decl, flags);
   14108         2101 :                   if (octx->outer_context == NULL)
   14109              :                     break;
   14110              :                   octx = octx->outer_context;
   14111              :                 }
   14112              :               while (1);
   14113         2682 :               if (octx
   14114         2682 :                   && decl
   14115         2682 :                   && (!OMP_CLAUSE_LINEAR_NO_COPYIN (c)
   14116          745 :                       || !OMP_CLAUSE_LINEAR_NO_COPYOUT (c)))
   14117         1728 :                 omp_notice_variable (octx, decl, true);
   14118              :             }
   14119         2682 :           flags = GOVD_LINEAR | GOVD_EXPLICIT;
   14120         2682 :           if (OMP_CLAUSE_LINEAR_NO_COPYIN (c)
   14121         2682 :               && OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
   14122              :             {
   14123              :               notice_outer = false;
   14124              :               flags |= GOVD_LINEAR_LASTPRIVATE_NO_OUTER;
   14125              :             }
   14126         2682 :           goto do_add;
   14127              : 
   14128        51463 :         case OMP_CLAUSE_MAP:
   14129        51463 :           if (!grp_start_p)
   14130              :             {
   14131        31243 :               grp_start_p = list_p;
   14132        31243 :               grp_end = (*groups)[grpnum].grp_end;
   14133        31243 :               grpnum++;
   14134              :             }
   14135        51463 :           decl = OMP_CLAUSE_DECL (c);
   14136              : 
   14137        51463 :           if (error_operand_p (decl))
   14138              :             {
   14139              :               remove = true;
   14140              :               break;
   14141              :             }
   14142              : 
   14143        51463 :           if (!omp_parse_expr (addr_tokens, decl))
   14144              :             {
   14145              :               remove = true;
   14146              :               break;
   14147              :             }
   14148              : 
   14149        51463 :           if (remove)
   14150              :             break;
   14151        51463 :           if (DECL_P (decl) && outer_ctx && (region_type & ORT_ACC))
   14152              :             {
   14153              :               struct gimplify_omp_ctx *octx;
   14154         1167 :               for (octx = outer_ctx; octx; octx = octx->outer_context)
   14155              :                 {
   14156         1167 :                   if (octx->region_type != ORT_ACC_HOST_DATA)
   14157              :                     break;
   14158           12 :                   splay_tree_node n2
   14159           12 :                     = splay_tree_lookup (octx->variables,
   14160              :                                          (splay_tree_key) decl);
   14161           12 :                   if (n2)
   14162            4 :                     error_at (OMP_CLAUSE_LOCATION (c), "variable %qE "
   14163              :                               "declared in enclosing %<host_data%> region",
   14164            4 :                               DECL_NAME (decl));
   14165              :                 }
   14166              :             }
   14167              : 
   14168        51463 :           map_descriptor = false;
   14169              : 
   14170              :           /* This condition checks if we're mapping an array descriptor that
   14171              :              isn't inside a derived type -- these have special handling, and
   14172              :              are not handled as structs in omp_build_struct_sibling_lists.
   14173              :              See that function for further details.  */
   14174        51463 :           if (*grp_start_p != grp_end
   14175        34262 :               && OMP_CLAUSE_CHAIN (*grp_start_p)
   14176        85725 :               && OMP_CLAUSE_CHAIN (*grp_start_p) != grp_end)
   14177              :             {
   14178        17096 :               tree grp_mid = OMP_CLAUSE_CHAIN (*grp_start_p);
   14179        17096 :               if (omp_map_clause_descriptor_p (grp_mid)
   14180        32365 :                   && DECL_P (OMP_CLAUSE_DECL (grp_mid)))
   14181              :                 map_descriptor = true;
   14182              :             }
   14183        34367 :           else if (OMP_CLAUSE_CODE (grp_end) == OMP_CLAUSE_MAP
   14184        34367 :                    && (OMP_CLAUSE_MAP_KIND (grp_end) == GOMP_MAP_RELEASE
   14185        33623 :                        || OMP_CLAUSE_MAP_KIND (grp_end) == GOMP_MAP_DELETE)
   14186        35289 :                    && OMP_CLAUSE_RELEASE_DESCRIPTOR (grp_end))
   14187              :             map_descriptor = true;
   14188              : 
   14189              :           /* Adding the decl for a struct access: we haven't created
   14190              :              GOMP_MAP_STRUCT nodes yet, so this statement needs to predict
   14191              :              whether they will be created in gimplify_adjust_omp_clauses.
   14192              :              NOTE: Technically we should probably look through DECL_VALUE_EXPR
   14193              :              here because something that looks like a DECL_P may actually be a
   14194              :              struct access, e.g. variables in a lambda closure
   14195              :              (__closure->__foo) or class members (this->foo). Currently in both
   14196              :              those cases we map the whole of the containing object (directly in
   14197              :              the C++ FE) though, so struct nodes are not created.  */
   14198        51463 :           if (c == grp_end
   14199        31243 :               && addr_tokens[0]->type == STRUCTURE_BASE
   14200         8431 :               && addr_tokens[0]->u.structure_base_kind == BASE_DECL
   14201        59894 :               && !map_descriptor)
   14202              :             {
   14203         5627 :               gcc_assert (addr_tokens[1]->type == ACCESS_METHOD);
   14204              :               /* If we got to this struct via a chain of pointers, maybe we
   14205              :                  want to map it implicitly instead.  */
   14206         5627 :               if (omp_access_chain_p (addr_tokens, 1))
   14207              :                 break;
   14208         5466 :               omp_mapping_group *wholestruct;
   14209         5466 :               if (!(region_type & ORT_ACC)
   14210         9657 :                   && omp_mapped_by_containing_struct (grpmap,
   14211         4191 :                                                       OMP_CLAUSE_DECL (c),
   14212              :                                                       &wholestruct))
   14213              :                 break;
   14214         5215 :               decl = addr_tokens[1]->expr;
   14215         5215 :               if (splay_tree_lookup (ctx->variables, (splay_tree_key) decl))
   14216              :                 break;
   14217              :               /* Standalone attach or detach clauses for a struct element
   14218              :                  should not inhibit implicit mapping of the whole struct.  */
   14219         2998 :               if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
   14220         2998 :                   || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
   14221              :                 break;
   14222         2848 :               flags = GOVD_MAP | GOVD_EXPLICIT;
   14223              : 
   14224         2848 :               gcc_assert (addr_tokens[1]->u.access_kind != ACCESS_DIRECT
   14225              :                           || TREE_ADDRESSABLE (decl));
   14226         2848 :               goto do_add_decl;
   14227              :             }
   14228              : 
   14229        45836 :           if (!DECL_P (decl))
   14230              :             {
   14231        21149 :               tree d = decl, *pd;
   14232        21149 :               if (TREE_CODE (d) == ARRAY_REF)
   14233              :                 {
   14234         5128 :                   while (TREE_CODE (d) == ARRAY_REF)
   14235         2621 :                     d = TREE_OPERAND (d, 0);
   14236         2507 :                   if (TREE_CODE (d) == COMPONENT_REF
   14237         2507 :                       && TREE_CODE (TREE_TYPE (d)) == ARRAY_TYPE)
   14238              :                     decl = d;
   14239              :                 }
   14240        21149 :               pd = &OMP_CLAUSE_DECL (c);
   14241        21149 :               if (d == decl
   14242        18688 :                   && TREE_CODE (decl) == INDIRECT_REF
   14243        13960 :                   && TREE_CODE (TREE_OPERAND (decl, 0)) == COMPONENT_REF
   14244         1084 :                   && (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0)))
   14245              :                       == REFERENCE_TYPE)
   14246        21814 :                   && (OMP_CLAUSE_MAP_KIND (c)
   14247              :                       != GOMP_MAP_POINTER_TO_ZERO_LENGTH_ARRAY_SECTION))
   14248              :                 {
   14249          661 :                   pd = &TREE_OPERAND (decl, 0);
   14250          661 :                   decl = TREE_OPERAND (decl, 0);
   14251              :                 }
   14252              : 
   14253        21149 :               if (addr_tokens[0]->type == STRUCTURE_BASE
   14254        11454 :                   && addr_tokens[0]->u.structure_base_kind == BASE_DECL
   14255        11453 :                   && addr_tokens[1]->type == ACCESS_METHOD
   14256        11453 :                   && (addr_tokens[1]->u.access_kind == ACCESS_POINTER
   14257        10955 :                       || (addr_tokens[1]->u.access_kind
   14258              :                           == ACCESS_POINTER_OFFSET))
   14259        21729 :                   && GOMP_MAP_ALWAYS_P (OMP_CLAUSE_MAP_KIND (c)))
   14260              :                 {
   14261            0 :                   tree base = addr_tokens[1]->expr;
   14262            0 :                   splay_tree_node n
   14263            0 :                     = splay_tree_lookup (ctx->variables,
   14264              :                                          (splay_tree_key) base);
   14265            0 :                   n->value |= GOVD_SEEN;
   14266              :                 }
   14267              : 
   14268        21149 :               if (code == OMP_TARGET && OMP_CLAUSE_MAP_IN_REDUCTION (c))
   14269              :                 {
   14270              :                   /* Don't gimplify *pd fully at this point, as the base
   14271              :                      will need to be adjusted during omp lowering.  */
   14272           88 :                   auto_vec<tree, 10> expr_stack;
   14273           88 :                   tree *p = pd;
   14274           88 :                   while (handled_component_p (*p)
   14275              :                          || TREE_CODE (*p) == INDIRECT_REF
   14276              :                          || TREE_CODE (*p) == ADDR_EXPR
   14277              :                          || TREE_CODE (*p) == MEM_REF
   14278          224 :                          || TREE_CODE (*p) == NON_LVALUE_EXPR)
   14279              :                     {
   14280          136 :                       expr_stack.safe_push (*p);
   14281          136 :                       p = &TREE_OPERAND (*p, 0);
   14282              :                     }
   14283          312 :                   for (int i = expr_stack.length () - 1; i >= 0; i--)
   14284              :                     {
   14285          136 :                       tree t = expr_stack[i];
   14286          136 :                       if (TREE_CODE (t) == ARRAY_REF
   14287          136 :                           || TREE_CODE (t) == ARRAY_RANGE_REF)
   14288              :                         {
   14289           56 :                           if (TREE_OPERAND (t, 2) == NULL_TREE)
   14290              :                             {
   14291           56 :                               tree low = unshare_expr (array_ref_low_bound (t));
   14292           56 :                               if (!is_gimple_min_invariant (low))
   14293              :                                 {
   14294            0 :                                   TREE_OPERAND (t, 2) = low;
   14295            0 :                                   if (gimplify_expr (&TREE_OPERAND (t, 2),
   14296              :                                                      pre_p, NULL,
   14297              :                                                      is_gimple_reg,
   14298              :                                                      fb_rvalue) == GS_ERROR)
   14299           56 :                                     remove = true;
   14300              :                                 }
   14301              :                             }
   14302            0 :                           else if (gimplify_expr (&TREE_OPERAND (t, 2), pre_p,
   14303              :                                                   NULL, is_gimple_reg,
   14304              :                                                   fb_rvalue) == GS_ERROR)
   14305           56 :                             remove = true;
   14306           56 :                           if (TREE_OPERAND (t, 3) == NULL_TREE)
   14307              :                             {
   14308           56 :                               tree elmt_size = array_ref_element_size (t);
   14309           56 :                               if (!is_gimple_min_invariant (elmt_size))
   14310              :                                 {
   14311            0 :                                   elmt_size = unshare_expr (elmt_size);
   14312            0 :                                   tree elmt_type
   14313            0 :                                     = TREE_TYPE (TREE_TYPE (TREE_OPERAND (t,
   14314              :                                                                           0)));
   14315            0 :                                   tree factor
   14316            0 :                                     = size_int (TYPE_ALIGN_UNIT (elmt_type));
   14317            0 :                                   elmt_size
   14318            0 :                                     = size_binop (EXACT_DIV_EXPR, elmt_size,
   14319              :                                                   factor);
   14320            0 :                                   TREE_OPERAND (t, 3) = elmt_size;
   14321            0 :                                   if (gimplify_expr (&TREE_OPERAND (t, 3),
   14322              :                                                      pre_p, NULL,
   14323              :                                                      is_gimple_reg,
   14324              :                                                      fb_rvalue) == GS_ERROR)
   14325            0 :                                     remove = true;
   14326              :                                 }
   14327              :                             }
   14328            0 :                           else if (gimplify_expr (&TREE_OPERAND (t, 3), pre_p,
   14329              :                                                   NULL, is_gimple_reg,
   14330              :                                                   fb_rvalue) == GS_ERROR)
   14331            0 :                             remove = true;
   14332              :                         }
   14333           80 :                       else if (TREE_CODE (t) == COMPONENT_REF)
   14334              :                         {
   14335            0 :                           if (TREE_OPERAND (t, 2) == NULL_TREE)
   14336              :                             {
   14337            0 :                               tree offset = component_ref_field_offset (t);
   14338            0 :                               if (!is_gimple_min_invariant (offset))
   14339              :                                 {
   14340            0 :                                   offset = unshare_expr (offset);
   14341            0 :                                   tree field = TREE_OPERAND (t, 1);
   14342            0 :                                   tree factor
   14343            0 :                                     = size_int (DECL_OFFSET_ALIGN (field)
   14344              :                                                 / BITS_PER_UNIT);
   14345            0 :                                   offset = size_binop (EXACT_DIV_EXPR, offset,
   14346              :                                                        factor);
   14347            0 :                                   TREE_OPERAND (t, 2) = offset;
   14348            0 :                                   if (gimplify_expr (&TREE_OPERAND (t, 2),
   14349              :                                                      pre_p, NULL,
   14350              :                                                      is_gimple_reg,
   14351              :                                                      fb_rvalue) == GS_ERROR)
   14352            0 :                                     remove = true;
   14353              :                                 }
   14354              :                             }
   14355            0 :                           else if (gimplify_expr (&TREE_OPERAND (t, 2), pre_p,
   14356              :                                                   NULL, is_gimple_reg,
   14357              :                                                   fb_rvalue) == GS_ERROR)
   14358            0 :                             remove = true;
   14359              :                         }
   14360              :                     }
   14361          224 :                   for (; expr_stack.length () > 0; )
   14362              :                     {
   14363          136 :                       tree t = expr_stack.pop ();
   14364              : 
   14365          136 :                       if (TREE_CODE (t) == ARRAY_REF
   14366          136 :                           || TREE_CODE (t) == ARRAY_RANGE_REF)
   14367              :                         {
   14368           56 :                           if (!is_gimple_min_invariant (TREE_OPERAND (t, 1))
   14369           56 :                               && gimplify_expr (&TREE_OPERAND (t, 1), pre_p,
   14370              :                                                 NULL, is_gimple_val,
   14371              :                                                 fb_rvalue) == GS_ERROR)
   14372              :                             remove = true;
   14373              :                         }
   14374              :                     }
   14375           88 :                 }
   14376              :               break;
   14377              :             }
   14378              : 
   14379        24687 :           if ((code == OMP_TARGET
   14380              :                || code == OMP_TARGET_DATA
   14381              :                || code == OMP_TARGET_ENTER_DATA
   14382        14229 :                || code == OMP_TARGET_EXIT_DATA)
   14383        25329 :               && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH_DETACH)
   14384              :             {
   14385              :               /* If we have attach/detach but the decl we have is a pointer to
   14386              :                  pointer, we're probably mapping the "base level" array
   14387              :                  implicitly.  Make sure we don't add the decl as if we mapped
   14388              :                  it explicitly.  That is,
   14389              : 
   14390              :                    int **arr;
   14391              :                    [...]
   14392              :                    #pragma omp target map(arr[a][b:c])
   14393              : 
   14394              :                  should *not* map "arr" explicitly.  That way we get a
   14395              :                  zero-length "alloc" mapping for it, and assuming it's been
   14396              :                  mapped by some previous directive, etc., things work as they
   14397              :                  should.  */
   14398              : 
   14399          217 :               tree basetype = TREE_TYPE (addr_tokens[0]->expr);
   14400              : 
   14401          217 :               if (TREE_CODE (basetype) == REFERENCE_TYPE)
   14402           34 :                 basetype = TREE_TYPE (basetype);
   14403              : 
   14404          217 :               if (code == OMP_TARGET
   14405           54 :                   && addr_tokens[0]->type == ARRAY_BASE
   14406           54 :                   && addr_tokens[0]->u.structure_base_kind == BASE_DECL
   14407           54 :                   && TREE_CODE (basetype) == POINTER_TYPE
   14408          271 :                   && TREE_CODE (TREE_TYPE (basetype)) == POINTER_TYPE)
   14409              :                 break;
   14410              :             }
   14411              : 
   14412        24665 :           flags = GOVD_MAP | GOVD_EXPLICIT;
   14413        24665 :           if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ALWAYS_TO
   14414        24528 :               || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ALWAYS_TOFROM
   14415        24020 :               || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ALWAYS_PRESENT_TO
   14416        48655 :               || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ALWAYS_PRESENT_TOFROM)
   14417              :             flags |= GOVD_MAP_ALWAYS_TO;
   14418              : 
   14419        24665 :           goto do_add;
   14420              : 
   14421          510 :         case OMP_CLAUSE_AFFINITY:
   14422          510 :           gimplify_omp_affinity (list_p, pre_p);
   14423          510 :           remove = true;
   14424          510 :           break;
   14425            8 :         case OMP_CLAUSE_DOACROSS:
   14426            8 :           if (OMP_CLAUSE_DOACROSS_KIND (c) == OMP_CLAUSE_DOACROSS_SINK)
   14427              :             {
   14428            4 :               tree deps = OMP_CLAUSE_DECL (c);
   14429            8 :               while (deps && TREE_CODE (deps) == TREE_LIST)
   14430              :                 {
   14431            4 :                   if (TREE_CODE (TREE_PURPOSE (deps)) == TRUNC_DIV_EXPR
   14432            4 :                       && DECL_P (TREE_OPERAND (TREE_PURPOSE (deps), 1)))
   14433            0 :                     gimplify_expr (&TREE_OPERAND (TREE_PURPOSE (deps), 1),
   14434              :                                    pre_p, NULL, is_gimple_val, fb_rvalue);
   14435            4 :                   deps = TREE_CHAIN (deps);
   14436              :                 }
   14437              :             }
   14438              :           else
   14439            4 :             gcc_assert (OMP_CLAUSE_DOACROSS_KIND (c)
   14440              :                         == OMP_CLAUSE_DOACROSS_SOURCE);
   14441              :           break;
   14442         2207 :         case OMP_CLAUSE_DEPEND:
   14443         2207 :           if (handled_depend_iterators == -1)
   14444         1905 :             handled_depend_iterators = gimplify_omp_depend (list_p, pre_p);
   14445         2207 :           if (handled_depend_iterators)
   14446              :             {
   14447          348 :               if (handled_depend_iterators == 2)
   14448            0 :                 remove = true;
   14449              :               break;
   14450              :             }
   14451         1859 :           if (TREE_CODE (OMP_CLAUSE_DECL (c)) == COMPOUND_EXPR)
   14452              :             {
   14453            0 :               gimplify_expr (&TREE_OPERAND (OMP_CLAUSE_DECL (c), 0), pre_p,
   14454              :                              NULL, is_gimple_val, fb_rvalue);
   14455            0 :               OMP_CLAUSE_DECL (c) = TREE_OPERAND (OMP_CLAUSE_DECL (c), 1);
   14456              :             }
   14457         1859 :           if (error_operand_p (OMP_CLAUSE_DECL (c)))
   14458              :             {
   14459              :               remove = true;
   14460              :               break;
   14461              :             }
   14462         1859 :           if (OMP_CLAUSE_DECL (c) != null_pointer_node)
   14463              :             {
   14464         1828 :               OMP_CLAUSE_DECL (c) = build_fold_addr_expr (OMP_CLAUSE_DECL (c));
   14465         1828 :               if (gimplify_expr (&OMP_CLAUSE_DECL (c), pre_p, NULL,
   14466              :                                  is_gimple_val, fb_rvalue) == GS_ERROR)
   14467              :                 {
   14468              :                   remove = true;
   14469              :                   break;
   14470              :                 }
   14471              :             }
   14472         1859 :           if (code == OMP_TASK)
   14473         1401 :             ctx->has_depend = true;
   14474              :           break;
   14475              : 
   14476         8383 :         case OMP_CLAUSE_TO:
   14477         8383 :         case OMP_CLAUSE_FROM:
   14478         8383 :         case OMP_CLAUSE__CACHE_:
   14479         8383 :           decl = OMP_CLAUSE_DECL (c);
   14480         8383 :           if (error_operand_p (decl))
   14481              :             {
   14482              :               remove = true;
   14483              :               break;
   14484              :             }
   14485         8383 :           if (OMP_CLAUSE_SIZE (c) == NULL_TREE)
   14486         7682 :             OMP_CLAUSE_SIZE (c) = DECL_P (decl) ? DECL_SIZE_UNIT (decl)
   14487          800 :                                   : TYPE_SIZE_UNIT (TREE_TYPE (decl));
   14488         8383 :           gimple_seq *seq_p;
   14489         8383 :           seq_p = enter_omp_iterator_loop_context (c, loops_seq_p, pre_p);
   14490         8383 :           if (gimplify_expr (&OMP_CLAUSE_SIZE (c), seq_p, NULL,
   14491              :                              is_gimple_val, fb_rvalue) == GS_ERROR)
   14492              :             {
   14493            0 :               remove = true;
   14494            0 :               exit_omp_iterator_loop_context (c);
   14495            0 :               break;
   14496              :             }
   14497         8383 :           if (!DECL_P (decl))
   14498              :             {
   14499         2180 :               if (gimplify_expr (&OMP_CLAUSE_DECL (c), seq_p, NULL,
   14500              :                                  is_gimple_lvalue, fb_lvalue) == GS_ERROR)
   14501            0 :                 remove = true;
   14502         2180 :               exit_omp_iterator_loop_context (c);
   14503         2180 :               break;
   14504              :             }
   14505         6203 :           exit_omp_iterator_loop_context (c);
   14506         6203 :           goto do_notice;
   14507              : 
   14508          135 :         case OMP_CLAUSE__MAPPER_BINDING_:
   14509          135 :           {
   14510          135 :             tree name = OMP_CLAUSE__MAPPER_BINDING__ID (c);
   14511          135 :             tree var = OMP_CLAUSE__MAPPER_BINDING__DECL (c);
   14512          135 :             tree type = TYPE_MAIN_VARIANT (TREE_TYPE (var));
   14513          135 :             tree fndecl = OMP_CLAUSE__MAPPER_BINDING__MAPPER (c);
   14514          135 :             ctx->implicit_mappers->put ({ name, type }, fndecl);
   14515          135 :             remove = true;
   14516          135 :             break;
   14517              :           }
   14518              : 
   14519         2118 :         case OMP_CLAUSE_USE_DEVICE_PTR:
   14520         2118 :         case OMP_CLAUSE_USE_DEVICE_ADDR:
   14521         2118 :           flags = GOVD_EXPLICIT;
   14522         2118 :           goto do_add;
   14523              : 
   14524          558 :         case OMP_CLAUSE_HAS_DEVICE_ADDR:
   14525          558 :           decl = OMP_CLAUSE_DECL (c);
   14526          558 :           while (TREE_CODE (decl) == INDIRECT_REF
   14527          607 :                  || TREE_CODE (decl) == ARRAY_REF)
   14528           49 :             decl = TREE_OPERAND (decl, 0);
   14529          558 :           flags = GOVD_EXPLICIT;
   14530          558 :           goto do_add_decl;
   14531              : 
   14532          500 :         case OMP_CLAUSE_IS_DEVICE_PTR:
   14533          500 :           flags = GOVD_FIRSTPRIVATE | GOVD_EXPLICIT;
   14534          500 :           goto do_add;
   14535              : 
   14536        63602 :         do_add:
   14537        63602 :           decl = OMP_CLAUSE_DECL (c);
   14538        85039 :         do_add_decl:
   14539        85039 :           if (error_operand_p (decl))
   14540              :             {
   14541              :               remove = true;
   14542              :               break;
   14543              :             }
   14544        85031 :           if (DECL_NAME (decl) == NULL_TREE && (flags & GOVD_SHARED) == 0)
   14545              :             {
   14546         1888 :               tree t = omp_member_access_dummy_var (decl);
   14547         1888 :               if (t)
   14548              :                 {
   14549          668 :                   tree v = DECL_VALUE_EXPR (decl);
   14550          668 :                   DECL_NAME (decl) = DECL_NAME (TREE_OPERAND (v, 1));
   14551          668 :                   if (outer_ctx)
   14552          140 :                     omp_notice_variable (outer_ctx, t, true);
   14553              :                 }
   14554              :             }
   14555        85031 :           if (code == OACC_DATA
   14556         2402 :               && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
   14557        87433 :               && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER)
   14558          307 :             flags |= GOVD_MAP_0LEN_ARRAY;
   14559        85031 :           omp_add_variable (ctx, decl, flags);
   14560        85031 :           if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
   14561        69629 :                || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
   14562        67530 :                || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
   14563        87660 :               && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
   14564              :             {
   14565         2138 :               struct gimplify_omp_ctx *pctx
   14566         2198 :                 = code == OMP_TARGET ? outer_ctx : ctx;
   14567         2198 :               if (pctx)
   14568         2168 :                 omp_add_variable (pctx, OMP_CLAUSE_REDUCTION_PLACEHOLDER (c),
   14569              :                                   GOVD_LOCAL | GOVD_SEEN);
   14570         2168 :               if (pctx
   14571         2168 :                   && OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c)
   14572          632 :                   && walk_tree (&OMP_CLAUSE_REDUCTION_INIT (c),
   14573              :                                 find_decl_expr,
   14574              :                                 OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c),
   14575              :                                 NULL) == NULL_TREE)
   14576          208 :                 omp_add_variable (pctx,
   14577          208 :                                   OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c),
   14578              :                                   GOVD_LOCAL | GOVD_SEEN);
   14579         2198 :               gimplify_omp_ctxp = pctx;
   14580         2198 :               push_gimplify_context ();
   14581              : 
   14582         2198 :               OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
   14583         2198 :               OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
   14584              : 
   14585         2198 :               gimplify_and_add (OMP_CLAUSE_REDUCTION_INIT (c),
   14586         2198 :                                 &OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c));
   14587         2198 :               pop_gimplify_context
   14588         2198 :                 (gimple_seq_first_stmt (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c)));
   14589         2198 :               push_gimplify_context ();
   14590         4396 :               gimplify_and_add (OMP_CLAUSE_REDUCTION_MERGE (c),
   14591         2198 :                                 &OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
   14592         2198 :               pop_gimplify_context
   14593         2198 :                 (gimple_seq_first_stmt (OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c)));
   14594         2198 :               OMP_CLAUSE_REDUCTION_INIT (c) = NULL_TREE;
   14595         2198 :               OMP_CLAUSE_REDUCTION_MERGE (c) = NULL_TREE;
   14596              : 
   14597         2198 :               gimplify_omp_ctxp = outer_ctx;
   14598              :             }
   14599        82833 :           else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
   14600        82833 :                    && OMP_CLAUSE_LASTPRIVATE_STMT (c))
   14601              :             {
   14602          303 :               gimplify_omp_ctxp = ctx;
   14603          303 :               push_gimplify_context ();
   14604          303 :               if (TREE_CODE (OMP_CLAUSE_LASTPRIVATE_STMT (c)) != BIND_EXPR)
   14605              :                 {
   14606          303 :                   tree bind = build3 (BIND_EXPR, void_type_node, NULL,
   14607              :                                       NULL, NULL);
   14608          303 :                   TREE_SIDE_EFFECTS (bind) = 1;
   14609          303 :                   BIND_EXPR_BODY (bind) = OMP_CLAUSE_LASTPRIVATE_STMT (c);
   14610          303 :                   OMP_CLAUSE_LASTPRIVATE_STMT (c) = bind;
   14611              :                 }
   14612          606 :               gimplify_and_add (OMP_CLAUSE_LASTPRIVATE_STMT (c),
   14613          303 :                                 &OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c));
   14614          303 :               pop_gimplify_context
   14615          303 :                 (gimple_seq_first_stmt (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c)));
   14616          303 :               OMP_CLAUSE_LASTPRIVATE_STMT (c) = NULL_TREE;
   14617              : 
   14618          303 :               gimplify_omp_ctxp = outer_ctx;
   14619              :             }
   14620        82530 :           else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
   14621        82530 :                    && OMP_CLAUSE_LINEAR_STMT (c))
   14622              :             {
   14623           60 :               gimplify_omp_ctxp = ctx;
   14624           60 :               push_gimplify_context ();
   14625           60 :               if (TREE_CODE (OMP_CLAUSE_LINEAR_STMT (c)) != BIND_EXPR)
   14626              :                 {
   14627           60 :                   tree bind = build3 (BIND_EXPR, void_type_node, NULL,
   14628              :                                       NULL, NULL);
   14629           60 :                   TREE_SIDE_EFFECTS (bind) = 1;
   14630           60 :                   BIND_EXPR_BODY (bind) = OMP_CLAUSE_LINEAR_STMT (c);
   14631           60 :                   OMP_CLAUSE_LINEAR_STMT (c) = bind;
   14632              :                 }
   14633          120 :               gimplify_and_add (OMP_CLAUSE_LINEAR_STMT (c),
   14634           60 :                                 &OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c));
   14635           60 :               pop_gimplify_context
   14636           60 :                 (gimple_seq_first_stmt (OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c)));
   14637           60 :               OMP_CLAUSE_LINEAR_STMT (c) = NULL_TREE;
   14638              : 
   14639           60 :               gimplify_omp_ctxp = outer_ctx;
   14640              :             }
   14641        85031 :           if (notice_outer)
   14642        72829 :             goto do_notice;
   14643              :           break;
   14644              : 
   14645          913 :         case OMP_CLAUSE_COPYIN:
   14646          913 :         case OMP_CLAUSE_COPYPRIVATE:
   14647          913 :           decl = OMP_CLAUSE_DECL (c);
   14648          913 :           if (error_operand_p (decl))
   14649              :             {
   14650              :               remove = true;
   14651              :               break;
   14652              :             }
   14653          913 :           if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_COPYPRIVATE
   14654              :               && !remove
   14655          913 :               && !omp_check_private (ctx, decl, true))
   14656              :             {
   14657           35 :               remove = true;
   14658           35 :               if (is_global_var (decl))
   14659              :                 {
   14660           30 :                   if (DECL_THREAD_LOCAL_P (decl))
   14661              :                     remove = false;
   14662           10 :                   else if (DECL_HAS_VALUE_EXPR_P (decl))
   14663              :                     {
   14664            2 :                       tree value = get_base_address (DECL_VALUE_EXPR (decl));
   14665              : 
   14666            2 :                       if (value
   14667            2 :                           && DECL_P (value)
   14668            4 :                           && DECL_THREAD_LOCAL_P (value))
   14669              :                         remove = false;
   14670              :                     }
   14671              :                 }
   14672              :               if (remove)
   14673           13 :                 error_at (OMP_CLAUSE_LOCATION (c),
   14674              :                           "copyprivate variable %qE is not threadprivate"
   14675           13 :                           " or private in outer context", DECL_NAME (decl));
   14676              :             }
   14677        79945 :         do_notice:
   14678        79945 :           if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
   14679        64543 :                || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE
   14680        56314 :                || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE)
   14681        30969 :               && outer_ctx
   14682        17698 :               && ((region_type & ORT_TASKLOOP) == ORT_TASKLOOP
   14683        16359 :                    || (region_type == ORT_WORKSHARE
   14684         3911 :                        && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
   14685         1113 :                        && (OMP_CLAUSE_REDUCTION_INSCAN (c)
   14686          930 :                            || code == OMP_LOOP)))
   14687        81898 :               && (outer_ctx->region_type == ORT_COMBINED_PARALLEL
   14688          857 :                   || (code == OMP_LOOP
   14689          138 :                       && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
   14690          138 :                       && ((outer_ctx->region_type & ORT_COMBINED_TEAMS)
   14691              :                           == ORT_COMBINED_TEAMS))))
   14692              :             {
   14693         1217 :               splay_tree_node on
   14694         1217 :                 = splay_tree_lookup (outer_ctx->variables,
   14695              :                                      (splay_tree_key)decl);
   14696         1217 :               if (on == NULL || (on->value & GOVD_DATA_SHARE_CLASS) == 0)
   14697              :                 {
   14698          899 :                   if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
   14699          713 :                       && TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF
   14700         1011 :                       && (TREE_CODE (TREE_TYPE (decl)) == POINTER_TYPE
   14701           56 :                           || (TREE_CODE (TREE_TYPE (decl)) == REFERENCE_TYPE
   14702            0 :                               && (TREE_CODE (TREE_TYPE (TREE_TYPE (decl)))
   14703              :                                   == POINTER_TYPE))))
   14704           56 :                     omp_firstprivatize_variable (outer_ctx, decl);
   14705              :                   else
   14706              :                     {
   14707          843 :                       omp_add_variable (outer_ctx, decl,
   14708              :                                         GOVD_SEEN | GOVD_SHARED);
   14709          843 :                       if (outer_ctx->outer_context)
   14710          235 :                         omp_notice_variable (outer_ctx->outer_context, decl,
   14711              :                                              true);
   14712              :                     }
   14713              :                 }
   14714              :             }
   14715        79337 :           if (outer_ctx)
   14716        29780 :             omp_notice_variable (outer_ctx, decl, true);
   14717        79945 :           if (check_non_private
   14718        27198 :               && (region_type == ORT_WORKSHARE || code == OMP_SCOPE)
   14719         4806 :               && (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
   14720         1691 :                   || decl == OMP_CLAUSE_DECL (c)
   14721          224 :                   || (TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF
   14722          224 :                       && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c), 0))
   14723              :                           == ADDR_EXPR
   14724          103 :                           || (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c), 0))
   14725              :                               == POINTER_PLUS_EXPR
   14726           22 :                               && (TREE_CODE (TREE_OPERAND (TREE_OPERAND
   14727              :                                                 (OMP_CLAUSE_DECL (c), 0), 0))
   14728              :                                   == ADDR_EXPR)))))
   14729        84663 :               && omp_check_private (ctx, decl, false))
   14730              :             {
   14731           58 :               error ("%s variable %qE is private in outer context",
   14732           29 :                      check_non_private, DECL_NAME (decl));
   14733           29 :               remove = true;
   14734              :             }
   14735              :           break;
   14736              : 
   14737          190 :         case OMP_CLAUSE_DETACH:
   14738          190 :           flags = GOVD_FIRSTPRIVATE | GOVD_SEEN;
   14739          190 :           goto do_add;
   14740              : 
   14741         4124 :         case OMP_CLAUSE_IF:
   14742         4124 :           if (OMP_CLAUSE_IF_MODIFIER (c) != ERROR_MARK
   14743         4124 :               && OMP_CLAUSE_IF_MODIFIER (c) != code)
   14744              :             {
   14745              :               const char *p[2];
   14746          168 :               for (int i = 0; i < 2; i++)
   14747          112 :                 switch (i ? OMP_CLAUSE_IF_MODIFIER (c) : code)
   14748              :                   {
   14749            8 :                   case VOID_CST: p[i] = "cancel"; break;
   14750           24 :                   case OMP_PARALLEL: p[i] = "parallel"; break;
   14751            4 :                   case OMP_SIMD: p[i] = "simd"; break;
   14752           12 :                   case OMP_TASK: p[i] = "task"; break;
   14753           12 :                   case OMP_TASKLOOP: p[i] = "taskloop"; break;
   14754            8 :                   case OMP_TARGET_DATA: p[i] = "target data"; break;
   14755           12 :                   case OMP_TARGET: p[i] = "target"; break;
   14756           12 :                   case OMP_TARGET_UPDATE: p[i] = "target update"; break;
   14757            8 :                   case OMP_TARGET_ENTER_DATA:
   14758            8 :                     p[i] = "target enter data"; break;
   14759           12 :                   case OMP_TARGET_EXIT_DATA: p[i] = "target exit data"; break;
   14760            0 :                   default: gcc_unreachable ();
   14761              :                   }
   14762           56 :               error_at (OMP_CLAUSE_LOCATION (c),
   14763              :                         "expected %qs %<if%> clause modifier rather than %qs",
   14764              :                         p[0], p[1]);
   14765           56 :               remove = true;
   14766              :             }
   14767              :           /* Fall through.  */
   14768              : 
   14769         4834 :         case OMP_CLAUSE_SELF:
   14770         4834 :         case OMP_CLAUSE_FINAL:
   14771         4834 :           OMP_CLAUSE_OPERAND (c, 0)
   14772         9668 :             = gimple_boolify (OMP_CLAUSE_OPERAND (c, 0));
   14773              :           /* Fall through.  */
   14774              : 
   14775         5904 :         case OMP_CLAUSE_NUM_TEAMS:
   14776         5904 :           if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_TEAMS
   14777         1070 :               && OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c)
   14778         6180 :               && !is_gimple_min_invariant (OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c)))
   14779              :             {
   14780          217 :               if (error_operand_p (OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c)))
   14781              :                 {
   14782              :                   remove = true;
   14783              :                   break;
   14784              :                 }
   14785          217 :               OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c)
   14786          434 :                 = get_initialized_tmp_var (OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c),
   14787              :                                            pre_p, NULL, true);
   14788              :             }
   14789              :           /* Fall through.  */
   14790              : 
   14791        22214 :         case OMP_CLAUSE_SCHEDULE:
   14792        22214 :         case OMP_CLAUSE_NUM_THREADS:
   14793        22214 :         case OMP_CLAUSE_THREAD_LIMIT:
   14794        22214 :         case OMP_CLAUSE_DIST_SCHEDULE:
   14795        22214 :         case OMP_CLAUSE_DEVICE:
   14796        22214 :           if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEVICE
   14797        22214 :               && OMP_CLAUSE_DEVICE_ANCESTOR (c))
   14798              :             {
   14799          125 :               if (code != OMP_TARGET)
   14800              :                 {
   14801           20 :                   error_at (OMP_CLAUSE_LOCATION (c),
   14802              :                             "%<device%> clause with %<ancestor%> is only "
   14803              :                             "allowed on %<target%> construct");
   14804           20 :                   remove = true;
   14805           20 :                   break;
   14806              :                 }
   14807              : 
   14808          105 :               tree clauses = *orig_list_p;
   14809          330 :               for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
   14810          236 :                 if (OMP_CLAUSE_CODE (clauses) != OMP_CLAUSE_DEVICE
   14811              :                     && OMP_CLAUSE_CODE (clauses) != OMP_CLAUSE_FIRSTPRIVATE
   14812              :                     && OMP_CLAUSE_CODE (clauses) != OMP_CLAUSE_PRIVATE
   14813              :                     && OMP_CLAUSE_CODE (clauses) != OMP_CLAUSE_DEFAULTMAP
   14814              :                     && OMP_CLAUSE_CODE (clauses) != OMP_CLAUSE_MAP
   14815              :                    )
   14816              :                   {
   14817           11 :                     error_at (OMP_CLAUSE_LOCATION (c),
   14818              :                               "with %<ancestor%>, only the %<device%>, "
   14819              :                               "%<firstprivate%>, %<private%>, %<defaultmap%>, "
   14820              :                               "and %<map%> clauses may appear on the "
   14821              :                               "construct");
   14822           11 :                     remove = true;
   14823           11 :                     break;
   14824              :                   }
   14825              :             }
   14826        22089 :           else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEVICE
   14827        22089 :                    && code == OMP_DISPATCH)
   14828              :             {
   14829          272 :               bool saved_into_ssa = gimplify_ctxp->into_ssa;
   14830          272 :               gimplify_ctxp->into_ssa = false;
   14831          272 :               if (gimplify_expr (&OMP_CLAUSE_DEVICE_ID (c), pre_p, NULL,
   14832              :                                  is_gimple_val, fb_rvalue)
   14833              :                   == GS_ERROR)
   14834              :                 remove = true;
   14835          272 :               else if (DECL_P (OMP_CLAUSE_DEVICE_ID (c)))
   14836           55 :                 omp_add_variable (ctx, OMP_CLAUSE_DEVICE_ID (c),
   14837              :                                   GOVD_SHARED | GOVD_SEEN);
   14838          272 :               gimplify_ctxp->into_ssa = saved_into_ssa;
   14839          272 :               break;
   14840              :             }
   14841              :           /* Fall through.  */
   14842              : 
   14843        31668 :         case OMP_CLAUSE_PRIORITY:
   14844        31668 :         case OMP_CLAUSE_GRAINSIZE:
   14845        31668 :         case OMP_CLAUSE_NUM_TASKS:
   14846        31668 :         case OMP_CLAUSE_FILTER:
   14847        31668 :         case OMP_CLAUSE_HINT:
   14848        31668 :         case OMP_CLAUSE_ASYNC:
   14849        31668 :         case OMP_CLAUSE_WAIT:
   14850        31668 :         case OMP_CLAUSE_NUM_GANGS:
   14851        31668 :         case OMP_CLAUSE_NUM_WORKERS:
   14852        31668 :         case OMP_CLAUSE_VECTOR_LENGTH:
   14853        31668 :         case OMP_CLAUSE_WORKER:
   14854        31668 :         case OMP_CLAUSE_VECTOR:
   14855        31668 :           if (OMP_CLAUSE_OPERAND (c, 0)
   14856        31668 :               && TREE_CODE (OMP_CLAUSE_OPERAND (c, 0)) == TREE_LIST)
   14857              :             {
   14858         2541 :               for (tree t = OMP_CLAUSE_OPERAND (c, 0); t; t = TREE_CHAIN (t))
   14859         1314 :                 if (!is_gimple_min_invariant (TREE_VALUE (t)))
   14860              :                   {
   14861          207 :                     if (error_operand_p (TREE_VALUE (t)))
   14862              :                       {
   14863              :                         remove = true;
   14864              :                         break;
   14865              :                       }
   14866          414 :                     TREE_VALUE (t)
   14867          207 :                       = get_initialized_tmp_var (TREE_VALUE (t), pre_p,
   14868              :                                                  NULL, true);
   14869              :                   }
   14870              :             }
   14871        30441 :           else if (OMP_CLAUSE_OPERAND (c, 0)
   14872        30441 :                    && !is_gimple_min_invariant (OMP_CLAUSE_OPERAND (c, 0)))
   14873              :             {
   14874         7662 :               if (error_operand_p (OMP_CLAUSE_OPERAND (c, 0)))
   14875              :                 {
   14876              :                   remove = true;
   14877              :                   break;
   14878              :                 }
   14879              :               /* All these clauses care about value, not a particular decl,
   14880              :                  so try to force it into a SSA_NAME or fresh temporary.  */
   14881         7655 :               OMP_CLAUSE_OPERAND (c, 0)
   14882        15310 :                 = get_initialized_tmp_var (OMP_CLAUSE_OPERAND (c, 0),
   14883              :                                            pre_p, NULL, true);
   14884              :             }
   14885              :           break;
   14886              : 
   14887         2332 :         case OMP_CLAUSE_GANG:
   14888         2332 :           if (gimplify_expr (&OMP_CLAUSE_OPERAND (c, 0), pre_p, NULL,
   14889              :                              is_gimple_val, fb_rvalue) == GS_ERROR)
   14890            0 :             remove = true;
   14891         2332 :           if (gimplify_expr (&OMP_CLAUSE_OPERAND (c, 1), pre_p, NULL,
   14892              :                              is_gimple_val, fb_rvalue) == GS_ERROR)
   14893            0 :             remove = true;
   14894              :           break;
   14895              : 
   14896        13163 :         case OMP_CLAUSE_NOWAIT:
   14897        13163 :           nowait = 1;
   14898        13163 :           break;
   14899              : 
   14900          116 :         case OMP_CLAUSE_USES_ALLOCATORS:
   14901          116 :           if (TREE_CODE (OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c))
   14902              :               != INTEGER_CST)
   14903              :             {
   14904          112 :               decl = OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (c);
   14905          112 :               omp_add_variable (ctx, decl,
   14906              :                                 GOVD_SEEN | GOVD_USES_ALLOCATORS_ALLOCATOR);
   14907              :             }
   14908              :           else
   14909              :             remove = true;
   14910              :           break;
   14911              : 
   14912              :         case OMP_CLAUSE_ORDERED:
   14913              :         case OMP_CLAUSE_UNTIED:
   14914              :         case OMP_CLAUSE_COLLAPSE:
   14915              :         case OMP_CLAUSE_TILE:
   14916              :         case OMP_CLAUSE_AUTO:
   14917              :         case OMP_CLAUSE_SEQ:
   14918              :         case OMP_CLAUSE_INDEPENDENT:
   14919              :         case OMP_CLAUSE_MERGEABLE:
   14920              :         case OMP_CLAUSE_MESSAGE:
   14921              :         case OMP_CLAUSE_PROC_BIND:
   14922              :         case OMP_CLAUSE_SAFELEN:
   14923              :         case OMP_CLAUSE_NOGROUP:
   14924              :         case OMP_CLAUSE_THREADS:
   14925              :         case OMP_CLAUSE_SIMD:
   14926              :         case OMP_CLAUSE_BIND:
   14927              :         case OMP_CLAUSE_IF_PRESENT:
   14928              :         case OMP_CLAUSE_FINALIZE:
   14929              :         case OMP_CLAUSE_INTEROP:
   14930              :         case OMP_CLAUSE_INIT:
   14931              :         case OMP_CLAUSE_USE:
   14932              :         case OMP_CLAUSE_DESTROY:
   14933              :         case OMP_CLAUSE_DEVICE_TYPE:
   14934              :           break;
   14935              : 
   14936          672 :         case OMP_CLAUSE_SIMDLEN:
   14937          672 :           if (OMP_CLAUSE_SIMDLEN_TYPE (c))
   14938              :             {
   14939           30 :               remove = true;
   14940           30 :               sorry_at (OMP_CLAUSE_LOCATION (c),
   14941              :                         "%<simdlen%> clause with %<scaled%> modifier");
   14942              :             }
   14943              :           break;
   14944              : 
   14945           85 :         case OMP_CLAUSE_DYN_GROUPPRIVATE:
   14946           85 :           remove = true;
   14947           85 :           sorry_at (OMP_CLAUSE_LOCATION (c),"%<dyn_groupprivate%> clause");
   14948           85 :           break;
   14949              : 
   14950         3991 :         case OMP_CLAUSE_ORDER:
   14951         3991 :           ctx->order_concurrent = true;
   14952         3991 :           break;
   14953              : 
   14954         1032 :         case OMP_CLAUSE_DEFAULTMAP:
   14955         1032 :           enum gimplify_defaultmap_kind gdmkmin, gdmkmax;
   14956         1032 :           switch (OMP_CLAUSE_DEFAULTMAP_CATEGORY (c))
   14957              :             {
   14958              :             case OMP_CLAUSE_DEFAULTMAP_CATEGORY_UNSPECIFIED:
   14959              :             case OMP_CLAUSE_DEFAULTMAP_CATEGORY_ALL:
   14960              :               gdmkmin = GDMK_SCALAR;
   14961              :               gdmkmax = GDMK_POINTER;
   14962              :               break;
   14963              :             case OMP_CLAUSE_DEFAULTMAP_CATEGORY_SCALAR:
   14964              :               gdmkmin = GDMK_SCALAR;
   14965              :               gdmkmax = GDMK_SCALAR_TARGET;
   14966              :               break;
   14967              :             case OMP_CLAUSE_DEFAULTMAP_CATEGORY_AGGREGATE:
   14968              :               gdmkmin = gdmkmax = GDMK_AGGREGATE;
   14969              :               break;
   14970              :             case OMP_CLAUSE_DEFAULTMAP_CATEGORY_ALLOCATABLE:
   14971              :               gdmkmin = gdmkmax = GDMK_ALLOCATABLE;
   14972              :               break;
   14973              :             case OMP_CLAUSE_DEFAULTMAP_CATEGORY_POINTER:
   14974              :               gdmkmin = gdmkmax = GDMK_POINTER;
   14975              :               break;
   14976            0 :             default:
   14977            0 :               gcc_unreachable ();
   14978              :             }
   14979         4570 :           for (int gdmk = gdmkmin; gdmk <= gdmkmax; gdmk++)
   14980         3538 :             switch (OMP_CLAUSE_DEFAULTMAP_BEHAVIOR (c))
   14981              :               {
   14982           91 :               case OMP_CLAUSE_DEFAULTMAP_ALLOC:
   14983           91 :                 ctx->defaultmap[gdmk] = GOVD_MAP | GOVD_MAP_ALLOC_ONLY;
   14984           91 :                 break;
   14985           96 :               case OMP_CLAUSE_DEFAULTMAP_TO:
   14986           96 :                 ctx->defaultmap[gdmk] = GOVD_MAP | GOVD_MAP_TO_ONLY;
   14987           96 :                 break;
   14988           28 :               case OMP_CLAUSE_DEFAULTMAP_FROM:
   14989           28 :                 ctx->defaultmap[gdmk] = GOVD_MAP | GOVD_MAP_FROM_ONLY;
   14990           28 :                 break;
   14991          681 :               case OMP_CLAUSE_DEFAULTMAP_TOFROM:
   14992          681 :                 ctx->defaultmap[gdmk] = GOVD_MAP;
   14993          681 :                 break;
   14994          394 :               case OMP_CLAUSE_DEFAULTMAP_FIRSTPRIVATE:
   14995          394 :                 ctx->defaultmap[gdmk] = GOVD_FIRSTPRIVATE;
   14996          394 :                 break;
   14997         2147 :               case OMP_CLAUSE_DEFAULTMAP_NONE:
   14998         2147 :                 ctx->defaultmap[gdmk] = 0;
   14999         2147 :                 break;
   15000           45 :               case OMP_CLAUSE_DEFAULTMAP_PRESENT:
   15001           45 :                 ctx->defaultmap[gdmk] = GOVD_MAP | GOVD_MAP_FORCE_PRESENT;
   15002           45 :                 break;
   15003           56 :               case OMP_CLAUSE_DEFAULTMAP_DEFAULT:
   15004           56 :                 switch (gdmk)
   15005              :                   {
   15006           11 :                   case GDMK_SCALAR:
   15007           11 :                     ctx->defaultmap[gdmk] = GOVD_FIRSTPRIVATE;
   15008           11 :                     break;
   15009           11 :                   case GDMK_SCALAR_TARGET:
   15010           11 :                     ctx->defaultmap[gdmk] = (lang_GNU_Fortran ()
   15011           11 :                                              ? GOVD_MAP : GOVD_FIRSTPRIVATE);
   15012           11 :                     break;
   15013           14 :                   case GDMK_AGGREGATE:
   15014           14 :                   case GDMK_ALLOCATABLE:
   15015           14 :                     ctx->defaultmap[gdmk] = GOVD_MAP;
   15016           14 :                     break;
   15017           20 :                   case GDMK_POINTER:
   15018           20 :                     ctx->defaultmap[gdmk] = GOVD_MAP;
   15019           20 :                     if (!lang_GNU_Fortran ())
   15020           12 :                       ctx->defaultmap[gdmk] |= GOVD_MAP_0LEN_ARRAY;
   15021              :                     break;
   15022              :                   default:
   15023              :                     gcc_unreachable ();
   15024              :                   }
   15025              :                 break;
   15026            0 :               default:
   15027            0 :                 gcc_unreachable ();
   15028              :               }
   15029              :           break;
   15030              : 
   15031          824 :         case OMP_CLAUSE_ALIGNED:
   15032          824 :           decl = OMP_CLAUSE_DECL (c);
   15033          824 :           if (error_operand_p (decl))
   15034              :             {
   15035              :               remove = true;
   15036              :               break;
   15037              :             }
   15038          824 :           if (gimplify_expr (&OMP_CLAUSE_ALIGNED_ALIGNMENT (c), pre_p, NULL,
   15039              :                              is_gimple_val, fb_rvalue) == GS_ERROR)
   15040              :             {
   15041              :               remove = true;
   15042              :               break;
   15043              :             }
   15044          824 :           if (!is_global_var (decl)
   15045          824 :               && TREE_CODE (TREE_TYPE (decl)) == POINTER_TYPE)
   15046          592 :             omp_add_variable (ctx, decl, GOVD_ALIGNED);
   15047              :           break;
   15048              : 
   15049          489 :         case OMP_CLAUSE_NONTEMPORAL:
   15050          489 :           decl = OMP_CLAUSE_DECL (c);
   15051          489 :           if (error_operand_p (decl))
   15052              :             {
   15053              :               remove = true;
   15054              :               break;
   15055              :             }
   15056          489 :           omp_add_variable (ctx, decl, GOVD_NONTEMPORAL);
   15057          489 :           break;
   15058              : 
   15059         3691 :         case OMP_CLAUSE_ALLOCATE:
   15060         3691 :           decl = OMP_CLAUSE_ALLOCATE_ALLOCATOR (c);
   15061         3691 :           if (decl
   15062         1722 :               && TREE_CODE (decl) == INTEGER_CST
   15063         3720 :               && wi::eq_p (wi::to_widest (decl), GOMP_OMP_PREDEF_ALLOC_THREAD)
   15064         3720 :               && (code == OMP_TARGET || code == OMP_TASK || code == OMP_TASKLOOP))
   15065           35 :             warning_at (OMP_CLAUSE_LOCATION (c), OPT_Wopenmp,
   15066              :                         "allocator with access trait set to %<thread%> "
   15067              :                         "results in undefined behavior for %qs directive",
   15068              :                         code == OMP_TARGET ? "target"
   15069              :                                            : (code == OMP_TASK
   15070           15 :                                               ? "task" : "taskloop"));
   15071         3691 :           decl = OMP_CLAUSE_DECL (c);
   15072         3691 :           if (error_operand_p (decl))
   15073              :             {
   15074              :               remove = true;
   15075              :               break;
   15076              :             }
   15077         3691 :           if ((omp_requires_mask & OMP_REQUIRES_DYNAMIC_ALLOCATORS) == 0
   15078         3679 :               && OMP_CLAUSE_ALLOCATE_ALLOCATOR (c)
   15079         5413 :               && TREE_CODE (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c)) != INTEGER_CST)
   15080              :             {
   15081          521 :               tree allocator = OMP_CLAUSE_ALLOCATE_ALLOCATOR (c);
   15082          521 :               tree clauses = NULL_TREE;
   15083              : 
   15084              :               /* Get clause list of the nearest enclosing target construct.  */
   15085          521 :               if (ctx->code == OMP_TARGET)
   15086           14 :                 clauses = *orig_list_p;
   15087              :               else
   15088              :                 {
   15089          507 :                   struct gimplify_omp_ctx *tctx = ctx->outer_context;
   15090          826 :                   while (tctx && tctx->code != OMP_TARGET)
   15091          319 :                     tctx = tctx->outer_context;
   15092          507 :                   if (tctx)
   15093           18 :                     clauses = tctx->clauses;
   15094              :                 }
   15095              : 
   15096           32 :               if (clauses)
   15097              :                 {
   15098           30 :                   tree uc;
   15099           30 :                   if (TREE_CODE (allocator) == MEM_REF
   15100           30 :                       || TREE_CODE (allocator) == INDIRECT_REF)
   15101            0 :                     allocator = TREE_OPERAND (allocator, 0);
   15102          132 :                   for (uc = clauses; uc; uc = OMP_CLAUSE_CHAIN (uc))
   15103          123 :                     if (OMP_CLAUSE_CODE (uc) == OMP_CLAUSE_USES_ALLOCATORS)
   15104              :                       {
   15105           29 :                         tree uc_allocator
   15106           29 :                           = OMP_CLAUSE_USES_ALLOCATORS_ALLOCATOR (uc);
   15107           29 :                         if (operand_equal_p (allocator, uc_allocator))
   15108              :                           break;
   15109              :                       }
   15110           30 :                   if (uc == NULL_TREE)
   15111              :                     {
   15112            9 :                       error_at (OMP_CLAUSE_LOCATION (c), "allocator %qE "
   15113              :                                 "requires %<uses_allocators(%E)%> clause in "
   15114              :                                 "target region", allocator, allocator);
   15115            9 :                       remove = true;
   15116            9 :                       break;
   15117              :                     }
   15118              :                 }
   15119              :             }
   15120         3682 :           if (gimplify_expr (&OMP_CLAUSE_ALLOCATE_ALLOCATOR (c), pre_p, NULL,
   15121              :                              is_gimple_val, fb_rvalue) == GS_ERROR)
   15122              :             {
   15123              :               remove = true;
   15124              :               break;
   15125              :             }
   15126         3682 :           else if (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) == NULL_TREE
   15127         3682 :                    || (TREE_CODE (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c))
   15128              :                        == INTEGER_CST))
   15129              :             ;
   15130          512 :           else if (code == OMP_TASKLOOP
   15131          512 :                    || !DECL_P (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c)))
   15132           66 :             OMP_CLAUSE_ALLOCATE_ALLOCATOR (c)
   15133          132 :               = get_initialized_tmp_var (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c),
   15134              :                                          pre_p, NULL, false);
   15135              :           break;
   15136              : 
   15137         4369 :         case OMP_CLAUSE_DEFAULT:
   15138         4369 :           ctx->default_kind = OMP_CLAUSE_DEFAULT_KIND (c);
   15139         4369 :           break;
   15140              : 
   15141          652 :         case OMP_CLAUSE_INCLUSIVE:
   15142          652 :         case OMP_CLAUSE_EXCLUSIVE:
   15143          652 :           decl = OMP_CLAUSE_DECL (c);
   15144          652 :           {
   15145          652 :             splay_tree_node n = splay_tree_lookup (outer_ctx->variables,
   15146              :                                                    (splay_tree_key) decl);
   15147          652 :             if (n == NULL || (n->value & GOVD_REDUCTION) == 0)
   15148              :               {
   15149            5 :                 error_at (OMP_CLAUSE_LOCATION (c),
   15150              :                           "%qD specified in %qs clause but not in %<inscan%> "
   15151              :                           "%<reduction%> clause on the containing construct",
   15152            5 :                           decl, omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
   15153            5 :                 remove = true;
   15154              :               }
   15155              :             else
   15156              :               {
   15157          647 :                 n->value |= GOVD_REDUCTION_INSCAN;
   15158          647 :                 if (outer_ctx->region_type == ORT_SIMD
   15159          528 :                     && outer_ctx->outer_context
   15160          115 :                     && outer_ctx->outer_context->region_type == ORT_WORKSHARE)
   15161              :                   {
   15162          115 :                     n = splay_tree_lookup (outer_ctx->outer_context->variables,
   15163              :                                            (splay_tree_key) decl);
   15164          115 :                     if (n && (n->value & GOVD_REDUCTION) != 0)
   15165          115 :                       n->value |= GOVD_REDUCTION_INSCAN;
   15166              :                   }
   15167              :               }
   15168              :           }
   15169              :           break;
   15170              : 
   15171          103 :         case OMP_CLAUSE_NOVARIANTS:
   15172          103 :           OMP_CLAUSE_NOVARIANTS_EXPR (c)
   15173          103 :             = gimple_boolify (OMP_CLAUSE_NOVARIANTS_EXPR (c));
   15174          103 :           break;
   15175          115 :         case OMP_CLAUSE_NOCONTEXT:
   15176          115 :           OMP_CLAUSE_NOCONTEXT_EXPR (c)
   15177          115 :             = gimple_boolify (OMP_CLAUSE_NOCONTEXT_EXPR (c));
   15178          115 :           break;
   15179            0 :         case OMP_CLAUSE_NOHOST:
   15180            0 :         default:
   15181            0 :           gcc_unreachable ();
   15182              :         }
   15183              : 
   15184       207319 :       if (code == OACC_DATA
   15185         5123 :           && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
   15186       212130 :           && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
   15187         4504 :               || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_REFERENCE))
   15188              :         remove = true;
   15189       207012 :       if (remove)
   15190         1245 :         *list_p = OMP_CLAUSE_CHAIN (c);
   15191              :       else
   15192       206074 :         list_p = &OMP_CLAUSE_CHAIN (c);
   15193       207319 :     }
   15194              : 
   15195       131912 :   if (groups)
   15196              :     {
   15197        36636 :       delete grpmap;
   15198        18318 :       delete groups;
   15199              :     }
   15200              : 
   15201       131912 :   ctx->clauses = *orig_list_p;
   15202       131912 :   gimplify_omp_ctxp = ctx;
   15203       131912 : }
   15204              : 
   15205              : /* Return true if DECL is a candidate for shared to firstprivate
   15206              :    optimization.  We only consider non-addressable scalars, not
   15207              :    too big, and not references.  */
   15208              : 
   15209              : static bool
   15210       407956 : omp_shared_to_firstprivate_optimizable_decl_p (tree decl)
   15211              : {
   15212       407956 :   if (TREE_ADDRESSABLE (decl))
   15213              :     return false;
   15214       357905 :   tree type = TREE_TYPE (decl);
   15215       357905 :   if (!is_gimple_reg_type (type)
   15216       333079 :       || TREE_CODE (type) == REFERENCE_TYPE
   15217       686666 :       || TREE_ADDRESSABLE (type))
   15218              :     return false;
   15219              :   /* Don't optimize too large decls, as each thread/task will have
   15220              :      its own.  */
   15221       328761 :   HOST_WIDE_INT len = int_size_in_bytes (type);
   15222       328761 :   if (len == -1 || len > 4 * POINTER_SIZE / BITS_PER_UNIT)
   15223              :     return false;
   15224       328744 :   if (omp_privatize_by_reference (decl))
   15225         1324 :     return false;
   15226              :   return true;
   15227              : }
   15228              : 
   15229              : /* Helper function of omp_find_stores_op and gimplify_adjust_omp_clauses*.
   15230              :    For omp_shared_to_firstprivate_optimizable_decl_p decl mark it as
   15231              :    GOVD_WRITTEN in outer contexts.  */
   15232              : 
   15233              : static void
   15234       305767 : omp_mark_stores (struct gimplify_omp_ctx *ctx, tree decl)
   15235              : {
   15236       468223 :   for (; ctx; ctx = ctx->outer_context)
   15237              :     {
   15238       451110 :       splay_tree_node n = splay_tree_lookup (ctx->variables,
   15239              :                                              (splay_tree_key) decl);
   15240       451110 :       if (n == NULL)
   15241       158869 :         continue;
   15242       292241 :       else if (n->value & GOVD_SHARED)
   15243              :         {
   15244         9172 :           n->value |= GOVD_WRITTEN;
   15245         9172 :           return;
   15246              :         }
   15247       283069 :       else if (n->value & GOVD_DATA_SHARE_CLASS)
   15248              :         return;
   15249              :     }
   15250              : }
   15251              : 
   15252              : /* Helper callback for walk_gimple_seq to discover possible stores
   15253              :    to omp_shared_to_firstprivate_optimizable_decl_p decls and set
   15254              :    GOVD_WRITTEN if they are GOVD_SHARED in some outer context
   15255              :    for those.  */
   15256              : 
   15257              : static tree
   15258      1137293 : omp_find_stores_op (tree *tp, int *walk_subtrees, void *data)
   15259              : {
   15260      1137293 :   struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
   15261              : 
   15262      1137293 :   *walk_subtrees = 0;
   15263      1137293 :   if (!wi->is_lhs)
   15264              :     return NULL_TREE;
   15265              : 
   15266       324398 :   tree op = *tp;
   15267       404477 :   do
   15268              :     {
   15269       404477 :       if (handled_component_p (op))
   15270        80079 :         op = TREE_OPERAND (op, 0);
   15271       324398 :       else if ((TREE_CODE (op) == MEM_REF || TREE_CODE (op) == TARGET_MEM_REF)
   15272       324398 :                && TREE_CODE (TREE_OPERAND (op, 0)) == ADDR_EXPR)
   15273            0 :         op = TREE_OPERAND (TREE_OPERAND (op, 0), 0);
   15274              :       else
   15275              :         break;
   15276              :     }
   15277              :   while (1);
   15278       324398 :   if (!DECL_P (op) || !omp_shared_to_firstprivate_optimizable_decl_p (op))
   15279              :     return NULL_TREE;
   15280              : 
   15281       265262 :   omp_mark_stores (gimplify_omp_ctxp, op);
   15282       265262 :   return NULL_TREE;
   15283              : }
   15284              : 
   15285              : /* Helper callback for walk_gimple_seq to discover possible stores
   15286              :    to omp_shared_to_firstprivate_optimizable_decl_p decls and set
   15287              :    GOVD_WRITTEN if they are GOVD_SHARED in some outer context
   15288              :    for those.  */
   15289              : 
   15290              : static tree
   15291       632775 : omp_find_stores_stmt (gimple_stmt_iterator *gsi_p,
   15292              :                       bool *handled_ops_p,
   15293              :                       struct walk_stmt_info *wi)
   15294              : {
   15295       632775 :   gimple *stmt = gsi_stmt (*gsi_p);
   15296       632775 :   switch (gimple_code (stmt))
   15297              :     {
   15298              :     /* Don't recurse on OpenMP constructs for which
   15299              :        gimplify_adjust_omp_clauses already handled the bodies,
   15300              :        except handle gimple_omp_for_pre_body.  */
   15301        26882 :     case GIMPLE_OMP_FOR:
   15302        26882 :       *handled_ops_p = true;
   15303        26882 :       if (gimple_omp_for_pre_body (stmt))
   15304         1784 :         walk_gimple_seq (gimple_omp_for_pre_body (stmt),
   15305              :                          omp_find_stores_stmt, omp_find_stores_op, wi);
   15306              :       break;
   15307         8680 :     case GIMPLE_OMP_PARALLEL:
   15308         8680 :     case GIMPLE_OMP_TASK:
   15309         8680 :     case GIMPLE_OMP_SECTIONS:
   15310         8680 :     case GIMPLE_OMP_SINGLE:
   15311         8680 :     case GIMPLE_OMP_SCOPE:
   15312         8680 :     case GIMPLE_OMP_TARGET:
   15313         8680 :     case GIMPLE_OMP_TEAMS:
   15314         8680 :     case GIMPLE_OMP_CRITICAL:
   15315         8680 :       *handled_ops_p = true;
   15316         8680 :       break;
   15317              :     default:
   15318              :       break;
   15319              :     }
   15320       632775 :   return NULL_TREE;
   15321              : }
   15322              : 
   15323              : struct gimplify_adjust_omp_clauses_data
   15324              : {
   15325              :   tree *list_p;
   15326              :   gimple_seq *pre_p;
   15327              : };
   15328              : 
   15329              : /* For all variables that were not actually used within the context,
   15330              :    remove PRIVATE, SHARED, and FIRSTPRIVATE clauses.  */
   15331              : 
   15332              : static int
   15333       680385 : gimplify_adjust_omp_clauses_1 (splay_tree_node n, void *data)
   15334              : {
   15335       680385 :   tree *list_p = ((struct gimplify_adjust_omp_clauses_data *) data)->list_p;
   15336       680385 :   gimple_seq *pre_p
   15337              :     = ((struct gimplify_adjust_omp_clauses_data *) data)->pre_p;
   15338       680385 :   tree decl = (tree) n->key;
   15339       680385 :   unsigned flags = n->value;
   15340       680385 :   enum omp_clause_code code;
   15341       680385 :   tree clause;
   15342       680385 :   bool private_debug;
   15343              : 
   15344       680385 :   if (gimplify_omp_ctxp->region_type == ORT_COMBINED_PARALLEL
   15345       133073 :       && (flags & GOVD_LASTPRIVATE_CONDITIONAL) != 0)
   15346              :     flags = GOVD_SHARED | GOVD_SEEN | GOVD_WRITTEN;
   15347       680260 :   if (flags & (GOVD_EXPLICIT | GOVD_LOCAL))
   15348              :     return 0;
   15349       179340 :   if ((flags & GOVD_SEEN) == 0)
   15350              :     return 0;
   15351       158191 :   if (flags & GOVD_DEBUG_PRIVATE)
   15352              :     {
   15353          261 :       gcc_assert ((flags & GOVD_DATA_SHARE_CLASS) == GOVD_SHARED);
   15354              :       private_debug = true;
   15355              :     }
   15356       157930 :   else if (flags & GOVD_MAP)
   15357              :     private_debug = false;
   15358              :   else
   15359       139796 :     private_debug
   15360       139796 :       = lang_hooks.decls.omp_private_debug_clause (decl,
   15361       139796 :                                                    !!(flags & GOVD_SHARED));
   15362       139796 :   if (private_debug)
   15363              :     code = OMP_CLAUSE_PRIVATE;
   15364       157808 :   else if (flags & GOVD_MAP)
   15365              :     {
   15366        18134 :       code = OMP_CLAUSE_MAP;
   15367        18134 :       if ((gimplify_omp_ctxp->region_type & ORT_ACC) == 0
   15368        18134 :           && TYPE_ATOMIC (strip_array_types (TREE_TYPE (decl))))
   15369              :         {
   15370            2 :           error ("%<_Atomic%> %qD in implicit %<map%> clause", decl);
   15371            2 :           return 0;
   15372              :         }
   15373        18132 :       if (VAR_P (decl)
   15374        16190 :           && DECL_IN_CONSTANT_POOL (decl)
   15375        18133 :           && !lookup_attribute ("omp declare target",
   15376            1 :                                 DECL_ATTRIBUTES (decl)))
   15377              :         {
   15378            1 :           tree id = get_identifier ("omp declare target");
   15379            1 :           DECL_ATTRIBUTES (decl)
   15380            1 :             = tree_cons (id, NULL_TREE, DECL_ATTRIBUTES (decl));
   15381            1 :           varpool_node *node = varpool_node::get (decl);
   15382            1 :           if (node)
   15383              :             {
   15384            1 :               node->offloadable = 1;
   15385            1 :               if (ENABLE_OFFLOADING)
   15386              :                 g->have_offload = true;
   15387              :             }
   15388              :         }
   15389              :     }
   15390       139674 :   else if (flags & GOVD_SHARED)
   15391              :     {
   15392        49272 :       if (is_global_var (decl))
   15393              :         {
   15394        16164 :           struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp->outer_context;
   15395        25361 :           while (ctx != NULL)
   15396              :             {
   15397        17614 :               splay_tree_node on
   15398        17614 :                 = splay_tree_lookup (ctx->variables, (splay_tree_key) decl);
   15399        17614 :               if (on && (on->value & (GOVD_FIRSTPRIVATE | GOVD_LASTPRIVATE
   15400              :                                       | GOVD_PRIVATE | GOVD_REDUCTION
   15401              :                                       | GOVD_LINEAR | GOVD_MAP)) != 0)
   15402              :                 break;
   15403         9197 :               ctx = ctx->outer_context;
   15404              :             }
   15405        16164 :           if (ctx == NULL)
   15406              :             return 0;
   15407              :         }
   15408        41525 :       code = OMP_CLAUSE_SHARED;
   15409              :       /* Don't optimize shared into firstprivate for read-only vars
   15410              :          on tasks with depend clause, we shouldn't try to copy them
   15411              :          until the dependencies are satisfied.  */
   15412        41525 :       if (gimplify_omp_ctxp->has_depend)
   15413          350 :         flags |= GOVD_WRITTEN;
   15414              :     }
   15415        90402 :   else if (flags & GOVD_PRIVATE)
   15416              :     code = OMP_CLAUSE_PRIVATE;
   15417        31721 :   else if (flags & GOVD_FIRSTPRIVATE)
   15418              :     {
   15419        22181 :       code = OMP_CLAUSE_FIRSTPRIVATE;
   15420        22181 :       if ((gimplify_omp_ctxp->region_type & ORT_TARGET)
   15421        14069 :           && (gimplify_omp_ctxp->region_type & ORT_ACC) == 0
   15422        32569 :           && TYPE_ATOMIC (strip_array_types (TREE_TYPE (decl))))
   15423              :         {
   15424            1 :           error ("%<_Atomic%> %qD in implicit %<firstprivate%> clause on "
   15425              :                  "%<target%> construct", decl);
   15426            1 :           return 0;
   15427              :         }
   15428              :     }
   15429         9540 :   else if (flags & GOVD_LASTPRIVATE)
   15430              :     code = OMP_CLAUSE_LASTPRIVATE;
   15431          353 :   else if (flags & (GOVD_ALIGNED | GOVD_NONTEMPORAL))
   15432              :     return 0;
   15433          230 :   else if (flags & GOVD_CONDTEMP)
   15434              :     {
   15435          118 :       code = OMP_CLAUSE__CONDTEMP_;
   15436          118 :       gimple_add_tmp_var (decl);
   15437              :     }
   15438          112 :   else if (flags & GOVD_USES_ALLOCATORS_ALLOCATOR)
   15439              :     return 0;
   15440              :   else
   15441            0 :     gcc_unreachable ();
   15442              : 
   15443       141019 :   if (((flags & GOVD_LASTPRIVATE)
   15444       140329 :        || (code == OMP_CLAUSE_SHARED && (flags & GOVD_WRITTEN)))
   15445       156314 :       && omp_shared_to_firstprivate_optimizable_decl_p (decl))
   15446        15281 :     omp_mark_stores (gimplify_omp_ctxp->outer_context, decl);
   15447              : 
   15448       150206 :   tree chain = *list_p;
   15449       150206 :   clause = build_omp_clause (input_location, code);
   15450       150206 :   OMP_CLAUSE_DECL (clause) = decl;
   15451       150206 :   OMP_CLAUSE_CHAIN (clause) = chain;
   15452       150206 :   if (private_debug)
   15453          383 :     OMP_CLAUSE_PRIVATE_DEBUG (clause) = 1;
   15454       149823 :   else if (code == OMP_CLAUSE_PRIVATE && (flags & GOVD_PRIVATE_OUTER_REF))
   15455            6 :     OMP_CLAUSE_PRIVATE_OUTER_REF (clause) = 1;
   15456       149817 :   else if (code == OMP_CLAUSE_SHARED
   15457        41525 :            && (flags & GOVD_WRITTEN) == 0
   15458       185234 :            && omp_shared_to_firstprivate_optimizable_decl_p (decl))
   15459        21127 :     OMP_CLAUSE_SHARED_READONLY (clause) = 1;
   15460       128690 :   else if (code == OMP_CLAUSE_FIRSTPRIVATE && (flags & GOVD_EXPLICIT) == 0)
   15461        22180 :     OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (clause) = 1;
   15462       106510 :   else if (code == OMP_CLAUSE_MAP && (flags & GOVD_MAP_0LEN_ARRAY) != 0)
   15463              :     {
   15464          630 :       tree nc = build_omp_clause (input_location, OMP_CLAUSE_MAP);
   15465          630 :       OMP_CLAUSE_DECL (nc) = decl;
   15466          630 :       if (TREE_CODE (TREE_TYPE (decl)) == REFERENCE_TYPE
   15467          630 :           && TREE_CODE (TREE_TYPE (TREE_TYPE (decl))) == POINTER_TYPE)
   15468           15 :         OMP_CLAUSE_DECL (clause)
   15469           30 :           = build_fold_indirect_ref_loc (input_location, decl);
   15470          630 :       OMP_CLAUSE_DECL (clause)
   15471          630 :         = build2 (MEM_REF, char_type_node, OMP_CLAUSE_DECL (clause),
   15472              :                   build_int_cst (build_pointer_type (char_type_node), 0));
   15473          630 :       OMP_CLAUSE_SIZE (clause) = size_zero_node;
   15474          630 :       OMP_CLAUSE_SIZE (nc) = size_zero_node;
   15475          630 :       OMP_CLAUSE_SET_MAP_KIND (clause, GOMP_MAP_ALLOC);
   15476          630 :       OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (clause) = 1;
   15477          630 :       tree dtype = TREE_TYPE (decl);
   15478          630 :       if (TREE_CODE (dtype) == REFERENCE_TYPE)
   15479           15 :         dtype = TREE_TYPE (dtype);
   15480              :       /* FIRSTPRIVATE_POINTER doesn't work well if we have a
   15481              :          multiply-indirected pointer.  If we have a reference to a pointer to
   15482              :          a pointer, it's possible that this should really be
   15483              :          GOMP_MAP_FIRSTPRIVATE_REFERENCE -- but that also doesn't work at the
   15484              :          moment, so stick with this.  (See PR113279 and testcases
   15485              :          baseptrs-{4,6}.C:ref2ptrptr_offset_decl_member_slice).  */
   15486          630 :       if (TREE_CODE (dtype) == POINTER_TYPE
   15487          630 :           && TREE_CODE (TREE_TYPE (dtype)) == POINTER_TYPE)
   15488           19 :         OMP_CLAUSE_SET_MAP_KIND (nc, GOMP_MAP_POINTER);
   15489              :       else
   15490          611 :         OMP_CLAUSE_SET_MAP_KIND (nc, GOMP_MAP_FIRSTPRIVATE_POINTER);
   15491          630 :       OMP_CLAUSE_CHAIN (nc) = chain;
   15492          630 :       OMP_CLAUSE_CHAIN (clause) = nc;
   15493          630 :       struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
   15494          630 :       gimplify_omp_ctxp = ctx->outer_context;
   15495          630 :       gimplify_expr (&TREE_OPERAND (OMP_CLAUSE_DECL (clause), 0),
   15496              :                      pre_p, NULL, is_gimple_val, fb_rvalue);
   15497          630 :       gimplify_omp_ctxp = ctx;
   15498          630 :     }
   15499        17502 :   else if (code == OMP_CLAUSE_MAP)
   15500              :     {
   15501        17502 :       int kind;
   15502              :       /* Not all combinations of these GOVD_MAP flags are actually valid.  */
   15503        17502 :       switch (flags & (GOVD_MAP_TO_ONLY
   15504              :                        | GOVD_MAP_FORCE
   15505              :                        | GOVD_MAP_FORCE_PRESENT
   15506              :                        | GOVD_MAP_ALLOC_ONLY
   15507              :                        | GOVD_MAP_FROM_ONLY))
   15508              :         {
   15509              :         case 0:
   15510              :           kind = GOMP_MAP_TOFROM;
   15511              :           break;
   15512         1042 :         case GOVD_MAP_FORCE:
   15513         1042 :           kind = GOMP_MAP_TOFROM | GOMP_MAP_FLAG_FORCE;
   15514         1042 :           break;
   15515          933 :         case GOVD_MAP_TO_ONLY:
   15516          933 :           kind = GOMP_MAP_TO;
   15517          933 :           break;
   15518           16 :         case GOVD_MAP_FROM_ONLY:
   15519           16 :           kind = GOMP_MAP_FROM;
   15520           16 :           break;
   15521           37 :         case GOVD_MAP_ALLOC_ONLY:
   15522           37 :           kind = GOMP_MAP_ALLOC;
   15523           37 :           break;
   15524            3 :         case GOVD_MAP_TO_ONLY | GOVD_MAP_FORCE:
   15525            3 :           kind = GOMP_MAP_TO | GOMP_MAP_FLAG_FORCE;
   15526            3 :           break;
   15527              :         case GOVD_MAP_FORCE_PRESENT:
   15528          326 :           kind = GOMP_MAP_FORCE_PRESENT;
   15529              :           break;
   15530              :         case GOVD_MAP_FORCE_PRESENT | GOVD_MAP_ALLOC_ONLY:
   15531          326 :           kind = GOMP_MAP_FORCE_PRESENT;
   15532              :           break;
   15533            0 :         default:
   15534            0 :           gcc_unreachable ();
   15535              :         }
   15536        17502 :       OMP_CLAUSE_SET_MAP_KIND (clause, kind);
   15537              :       /* Setting of the implicit flag for the runtime is currently disabled for
   15538              :          OpenACC.  */
   15539        17502 :       if ((gimplify_omp_ctxp->region_type & ORT_ACC) == 0)
   15540         9968 :         OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (clause) = 1;
   15541        17502 :       if (DECL_SIZE (decl)
   15542        17502 :           && !poly_int_tree_p (DECL_SIZE (decl)))
   15543              :         {
   15544          537 :           tree decl2 = DECL_VALUE_EXPR (decl);
   15545          537 :           gcc_assert (INDIRECT_REF_P (decl2));
   15546          537 :           decl2 = TREE_OPERAND (decl2, 0);
   15547          537 :           gcc_assert (DECL_P (decl2));
   15548          537 :           tree mem = build_simple_mem_ref (decl2);
   15549          537 :           OMP_CLAUSE_DECL (clause) = mem;
   15550          537 :           OMP_CLAUSE_SIZE (clause) = TYPE_SIZE_UNIT (TREE_TYPE (decl));
   15551          537 :           if (gimplify_omp_ctxp->outer_context)
   15552              :             {
   15553          446 :               struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp->outer_context;
   15554          446 :               omp_notice_variable (ctx, decl2, true);
   15555          446 :               omp_notice_variable (ctx, OMP_CLAUSE_SIZE (clause), true);
   15556              :             }
   15557          537 :           tree nc = build_omp_clause (OMP_CLAUSE_LOCATION (clause),
   15558              :                                       OMP_CLAUSE_MAP);
   15559          537 :           OMP_CLAUSE_DECL (nc) = decl;
   15560          537 :           OMP_CLAUSE_SIZE (nc) = size_zero_node;
   15561          537 :           if (gimplify_omp_ctxp->target_firstprivatize_array_bases)
   15562          537 :             OMP_CLAUSE_SET_MAP_KIND (nc, GOMP_MAP_FIRSTPRIVATE_POINTER);
   15563              :           else
   15564            0 :             OMP_CLAUSE_SET_MAP_KIND (nc, GOMP_MAP_POINTER);
   15565          537 :           OMP_CLAUSE_CHAIN (nc) = OMP_CLAUSE_CHAIN (clause);
   15566          537 :           OMP_CLAUSE_CHAIN (clause) = nc;
   15567              :         }
   15568        16965 :       else if (gimplify_omp_ctxp->target_firstprivatize_array_bases
   15569        16965 :                && omp_privatize_by_reference (decl))
   15570              :         {
   15571           28 :           OMP_CLAUSE_DECL (clause) = build_simple_mem_ref (decl);
   15572           28 :           OMP_CLAUSE_SIZE (clause)
   15573           28 :             = unshare_expr (TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (decl))));
   15574           28 :           struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
   15575           28 :           gimplify_omp_ctxp = ctx->outer_context;
   15576           28 :           gimplify_expr (&OMP_CLAUSE_SIZE (clause),
   15577              :                          pre_p, NULL, is_gimple_val, fb_rvalue);
   15578           28 :           gimplify_omp_ctxp = ctx;
   15579           28 :           tree nc = build_omp_clause (OMP_CLAUSE_LOCATION (clause),
   15580              :                                       OMP_CLAUSE_MAP);
   15581           28 :           OMP_CLAUSE_DECL (nc) = decl;
   15582           28 :           OMP_CLAUSE_SIZE (nc) = size_zero_node;
   15583           28 :           OMP_CLAUSE_SET_MAP_KIND (nc, GOMP_MAP_FIRSTPRIVATE_REFERENCE);
   15584           28 :           OMP_CLAUSE_CHAIN (nc) = OMP_CLAUSE_CHAIN (clause);
   15585           28 :           OMP_CLAUSE_CHAIN (clause) = nc;
   15586              :         }
   15587              :       else
   15588        16937 :         OMP_CLAUSE_SIZE (clause) = DECL_SIZE_UNIT (decl);
   15589              :     }
   15590       150206 :   if (code == OMP_CLAUSE_FIRSTPRIVATE && (flags & GOVD_LASTPRIVATE) != 0)
   15591              :     {
   15592          690 :       tree nc = build_omp_clause (input_location, OMP_CLAUSE_LASTPRIVATE);
   15593          690 :       OMP_CLAUSE_DECL (nc) = decl;
   15594          690 :       OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (nc) = 1;
   15595          690 :       OMP_CLAUSE_CHAIN (nc) = chain;
   15596          690 :       OMP_CLAUSE_CHAIN (clause) = nc;
   15597          690 :       struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
   15598          690 :       gimplify_omp_ctxp = ctx->outer_context;
   15599          690 :       lang_hooks.decls.omp_finish_clause (nc, pre_p,
   15600          690 :                                           (ctx->region_type & ORT_ACC) != 0);
   15601          690 :       gimplify_omp_ctxp = ctx;
   15602              :     }
   15603       150206 :   *list_p = clause;
   15604       150206 :   struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
   15605       150206 :   gimplify_omp_ctxp = ctx->outer_context;
   15606              :   /* Don't call omp_finish_clause on implicitly added OMP_CLAUSE_PRIVATE
   15607              :      in simd.  Those are only added for the local vars inside of simd body
   15608              :      and they don't need to be e.g. default constructible.  */
   15609       150206 :   if (code != OMP_CLAUSE_PRIVATE || ctx->region_type != ORT_SIMD)
   15610       142154 :     lang_hooks.decls.omp_finish_clause (clause, pre_p,
   15611       142154 :                                         (ctx->region_type & ORT_ACC) != 0);
   15612       150206 :   if (gimplify_omp_ctxp)
   15613       184555 :     for (; clause != chain; clause = OMP_CLAUSE_CHAIN (clause))
   15614        93690 :       if (OMP_CLAUSE_CODE (clause) == OMP_CLAUSE_MAP
   15615        93690 :           && DECL_P (OMP_CLAUSE_SIZE (clause)))
   15616         1241 :         omp_notice_variable (gimplify_omp_ctxp, OMP_CLAUSE_SIZE (clause),
   15617              :                              true);
   15618       150206 :   gimplify_omp_ctxp = ctx;
   15619       150206 :   return 0;
   15620              : }
   15621              : 
   15622              : static void
   15623       130448 : gimplify_adjust_omp_clauses (gimple_seq *pre_p, gimple_seq body, tree *list_p,
   15624              :                              enum tree_code code,
   15625              :                              gimple_seq *loops_seq_p = NULL)
   15626              : {
   15627       130448 :   struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
   15628       130448 :   tree *orig_list_p = list_p;
   15629       130448 :   tree c, decl;
   15630       130448 :   bool has_inscan_reductions = false;
   15631              : 
   15632       130448 :   if (body)
   15633              :     {
   15634              :       struct gimplify_omp_ctx *octx;
   15635       226762 :       for (octx = ctx; octx; octx = octx->outer_context)
   15636       175138 :         if ((octx->region_type & (ORT_PARALLEL | ORT_TASK | ORT_TEAMS)) != 0)
   15637              :           break;
   15638       112708 :       if (octx)
   15639              :         {
   15640        61084 :           struct walk_stmt_info wi;
   15641        61084 :           memset (&wi, 0, sizeof (wi));
   15642        61084 :           walk_gimple_seq (body, omp_find_stores_stmt,
   15643              :                            omp_find_stores_op, &wi);
   15644              :         }
   15645              :     }
   15646              : 
   15647       130448 :   if (ctx->add_safelen1)
   15648              :     {
   15649              :       /* If there are VLAs in the body of simd loop, prevent
   15650              :          vectorization.  */
   15651            2 :       gcc_assert (ctx->region_type == ORT_SIMD);
   15652            2 :       c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_SAFELEN);
   15653            2 :       OMP_CLAUSE_SAFELEN_EXPR (c) = integer_one_node;
   15654            2 :       OMP_CLAUSE_CHAIN (c) = *list_p;
   15655            2 :       *list_p = c;
   15656            2 :       list_p = &OMP_CLAUSE_CHAIN (c);
   15657              :     }
   15658              : 
   15659       130448 :   if (ctx->region_type == ORT_WORKSHARE
   15660        40051 :       && ctx->outer_context
   15661        28104 :       && ctx->outer_context->region_type == ORT_COMBINED_PARALLEL)
   15662              :     {
   15663        25066 :       for (c = ctx->outer_context->clauses; c; c = OMP_CLAUSE_CHAIN (c))
   15664        12652 :         if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
   15665        12652 :             && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
   15666              :           {
   15667          125 :             decl = OMP_CLAUSE_DECL (c);
   15668          125 :             splay_tree_node n
   15669          125 :               = splay_tree_lookup (ctx->outer_context->variables,
   15670              :                                    (splay_tree_key) decl);
   15671          125 :             gcc_checking_assert (!splay_tree_lookup (ctx->variables,
   15672              :                                                      (splay_tree_key) decl));
   15673          125 :             omp_add_variable (ctx, decl, n->value);
   15674          125 :             tree c2 = copy_node (c);
   15675          125 :             OMP_CLAUSE_CHAIN (c2) = *list_p;
   15676          125 :             *list_p = c2;
   15677          125 :             if ((n->value & GOVD_FIRSTPRIVATE) == 0)
   15678          103 :               continue;
   15679           22 :             c2 = build_omp_clause (OMP_CLAUSE_LOCATION (c),
   15680              :                                    OMP_CLAUSE_FIRSTPRIVATE);
   15681           22 :             OMP_CLAUSE_DECL (c2) = decl;
   15682           22 :             OMP_CLAUSE_CHAIN (c2) = *list_p;
   15683           22 :             *list_p = c2;
   15684              :           }
   15685              :     }
   15686              : 
   15687       130448 :   if (code == OMP_TARGET
   15688       130448 :       || code == OMP_TARGET_DATA
   15689       130448 :       || code == OMP_TARGET_ENTER_DATA
   15690       113940 :       || code == OMP_TARGET_EXIT_DATA)
   15691              :     {
   15692        17299 :       tree mapper_clauses = NULL_TREE;
   15693        17299 :       instantiate_mapper_info im_info;
   15694              : 
   15695        17299 :       im_info.mapper_clauses_p = &mapper_clauses;
   15696        17299 :       im_info.omp_ctx = ctx;
   15697        17299 :       im_info.pre_p = pre_p;
   15698              : 
   15699        17299 :       splay_tree_foreach (ctx->variables,
   15700              :                           omp_instantiate_implicit_mappers,
   15701              :                           (void *) &im_info);
   15702              : 
   15703        17299 :       if (mapper_clauses)
   15704              :         {
   15705           53 :           mapper_clauses
   15706           53 :             = lang_hooks.decls.omp_finish_mapper_clauses (mapper_clauses);
   15707              : 
   15708              :           /* Stick the implicitly-expanded mapper clauses at the end of the
   15709              :              clause list.  */
   15710           53 :           tree *tail = list_p;
   15711          163 :           while (*tail)
   15712          110 :             tail = &OMP_CLAUSE_CHAIN (*tail);
   15713           53 :           *tail = mapper_clauses;
   15714              :         }
   15715              : 
   15716        17299 :       vec<omp_mapping_group> *groups;
   15717        17299 :       groups = omp_gather_mapping_groups (list_p);
   15718        17299 :       hash_map<tree_operand_hash_no_se, omp_mapping_group *> *grpmap = NULL;
   15719              : 
   15720        17299 :       if (groups)
   15721              :         {
   15722         8655 :           grpmap = omp_index_mapping_groups (groups);
   15723              : 
   15724         8655 :           omp_resolve_clause_dependencies (code, groups, grpmap);
   15725         8655 :           omp_build_struct_sibling_lists (code, ctx->region_type, groups,
   15726              :                                           &grpmap, list_p);
   15727              : 
   15728         8655 :           omp_mapping_group *outlist = NULL;
   15729              : 
   15730        17310 :           delete grpmap;
   15731         8655 :           delete groups;
   15732              : 
   15733              :           /* Rebuild now we have struct sibling lists.  */
   15734         8655 :           groups = omp_gather_mapping_groups (list_p);
   15735         8655 :           grpmap = omp_index_mapping_groups (groups);
   15736              : 
   15737         8655 :           bool enter_exit = (code == OMP_TARGET_ENTER_DATA
   15738         8655 :                              || code == OMP_TARGET_EXIT_DATA);
   15739              : 
   15740         8655 :           outlist = omp_tsort_mapping_groups (groups, grpmap, enter_exit);
   15741         8655 :           outlist = omp_segregate_mapping_groups (outlist);
   15742         8655 :           list_p = omp_reorder_mapping_groups (groups, outlist, list_p);
   15743              : 
   15744         8655 :           delete grpmap;
   15745         8655 :           delete groups;
   15746              :         }
   15747        17299 :     }
   15748       113149 :   else if (ctx->region_type & ORT_ACC)
   15749              :     {
   15750        29899 :       vec<omp_mapping_group> *groups;
   15751        29899 :       groups = omp_gather_mapping_groups (list_p);
   15752        29899 :       if (groups)
   15753              :         {
   15754         9462 :           hash_map<tree_operand_hash_no_se, omp_mapping_group *> *grpmap;
   15755         9462 :           grpmap = omp_index_mapping_groups (groups);
   15756              : 
   15757         9462 :           oacc_resolve_clause_dependencies (groups, grpmap);
   15758         9462 :           omp_build_struct_sibling_lists (code, ctx->region_type, groups,
   15759              :                                           &grpmap, list_p);
   15760              : 
   15761         9462 :           delete groups;
   15762        18924 :           delete grpmap;
   15763              :         }
   15764              :     }
   15765              : 
   15766       130448 :   tree attach_list = NULL_TREE;
   15767       130448 :   tree *attach_tail = &attach_list;
   15768              : 
   15769       130448 :   tree *grp_start_p = NULL, grp_end = NULL_TREE;
   15770              : 
   15771       381224 :   while ((c = *list_p) != NULL)
   15772              :     {
   15773       250776 :       splay_tree_node n;
   15774       250776 :       bool remove = false;
   15775       250776 :       bool move_attach = false;
   15776              : 
   15777       293526 :       if (grp_end && c == OMP_CLAUSE_CHAIN (grp_end))
   15778              :         grp_end = NULL_TREE;
   15779              : 
   15780       250776 :       switch (OMP_CLAUSE_CODE (c))
   15781              :         {
   15782         8245 :         case OMP_CLAUSE_FIRSTPRIVATE:
   15783         8245 :           if ((ctx->region_type & ORT_TARGET)
   15784         2122 :               && (ctx->region_type & ORT_ACC) == 0
   15785         9761 :               && TYPE_ATOMIC (strip_array_types
   15786              :                                         (TREE_TYPE (OMP_CLAUSE_DECL (c)))))
   15787              :             {
   15788            4 :               error_at (OMP_CLAUSE_LOCATION (c),
   15789              :                         "%<_Atomic%> %qD in %<firstprivate%> clause on "
   15790            2 :                         "%<target%> construct", OMP_CLAUSE_DECL (c));
   15791            2 :               remove = true;
   15792            2 :               break;
   15793              :             }
   15794         8243 :           if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c))
   15795              :             {
   15796          380 :               decl = OMP_CLAUSE_DECL (c);
   15797          380 :               n = splay_tree_lookup (ctx->variables, (splay_tree_key) decl);
   15798          380 :               if ((n->value & GOVD_MAP) != 0)
   15799              :                 {
   15800              :                   remove = true;
   15801              :                   break;
   15802              :                 }
   15803          368 :               OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT_TARGET (c) = 0;
   15804          368 :               OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c) = 0;
   15805              :             }
   15806              :           /* FALLTHRU */
   15807        37273 :         case OMP_CLAUSE_PRIVATE:
   15808        37273 :         case OMP_CLAUSE_SHARED:
   15809        37273 :         case OMP_CLAUSE_LINEAR:
   15810        37273 :           decl = OMP_CLAUSE_DECL (c);
   15811        37273 :           n = splay_tree_lookup (ctx->variables, (splay_tree_key) decl);
   15812        37273 :           remove = !(n->value & GOVD_SEEN);
   15813        37273 :           if ((n->value & GOVD_LASTPRIVATE_CONDITIONAL) != 0
   15814           48 :               && code == OMP_PARALLEL
   15815        37295 :               && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
   15816              :             remove = true;
   15817        37251 :           if (! remove)
   15818              :             {
   15819        32640 :               bool shared = OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED;
   15820        32640 :               if ((n->value & GOVD_DEBUG_PRIVATE)
   15821        32640 :                   || lang_hooks.decls.omp_private_debug_clause (decl, shared))
   15822              :                 {
   15823           87 :                   gcc_assert ((n->value & GOVD_DEBUG_PRIVATE) == 0
   15824              :                               || ((n->value & GOVD_DATA_SHARE_CLASS)
   15825              :                                   == GOVD_SHARED));
   15826           87 :                   OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_PRIVATE);
   15827           87 :                   OMP_CLAUSE_PRIVATE_DEBUG (c) = 1;
   15828              :                 }
   15829        32640 :               if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
   15830         4194 :                   && ctx->has_depend
   15831        33198 :                   && DECL_P (decl))
   15832          558 :                 n->value |= GOVD_WRITTEN;
   15833        32640 :               if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
   15834         4194 :                   && (n->value & GOVD_WRITTEN) == 0
   15835         3075 :                   && DECL_P (decl)
   15836        35715 :                   && omp_shared_to_firstprivate_optimizable_decl_p (decl))
   15837          526 :                 OMP_CLAUSE_SHARED_READONLY (c) = 1;
   15838        32114 :               else if (DECL_P (decl)
   15839        32114 :                        && ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
   15840         3668 :                             && (n->value & GOVD_WRITTEN) != 0)
   15841        30995 :                            || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
   15842         9146 :                                && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c)))
   15843        39184 :                        && omp_shared_to_firstprivate_optimizable_decl_p (decl))
   15844         5766 :                 omp_mark_stores (gimplify_omp_ctxp->outer_context, decl);
   15845              :             }
   15846              :           else
   15847         4633 :             n->value &= ~GOVD_EXPLICIT;
   15848              :           break;
   15849              : 
   15850        13015 :         case OMP_CLAUSE_LASTPRIVATE:
   15851              :           /* Make sure OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE is set to
   15852              :              accurately reflect the presence of a FIRSTPRIVATE clause.  */
   15853        13015 :           decl = OMP_CLAUSE_DECL (c);
   15854        13015 :           n = splay_tree_lookup (ctx->variables, (splay_tree_key) decl);
   15855        13015 :           OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)
   15856        13015 :             = (n->value & GOVD_FIRSTPRIVATE) != 0;
   15857        13015 :           if (code == OMP_DISTRIBUTE
   15858        13015 :               && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
   15859              :             {
   15860            4 :               remove = true;
   15861            4 :               error_at (OMP_CLAUSE_LOCATION (c),
   15862              :                         "same variable used in %<firstprivate%> and "
   15863              :                         "%<lastprivate%> clauses on %<distribute%> "
   15864              :                         "construct");
   15865              :             }
   15866        13015 :           if (!remove
   15867        13011 :               && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
   15868        13011 :               && DECL_P (decl)
   15869        13011 :               && omp_shared_to_firstprivate_optimizable_decl_p (decl))
   15870        11416 :             omp_mark_stores (gimplify_omp_ctxp->outer_context, decl);
   15871        13015 :           if (OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c) && code == OMP_PARALLEL)
   15872              :             remove = true;
   15873              :           break;
   15874              : 
   15875          824 :         case OMP_CLAUSE_ALIGNED:
   15876          824 :           decl = OMP_CLAUSE_DECL (c);
   15877          824 :           if (!is_global_var (decl))
   15878              :             {
   15879          738 :               n = splay_tree_lookup (ctx->variables, (splay_tree_key) decl);
   15880          738 :               remove = n == NULL || !(n->value & GOVD_SEEN);
   15881           88 :               if (!remove && TREE_CODE (TREE_TYPE (decl)) == POINTER_TYPE)
   15882              :                 {
   15883           88 :                   struct gimplify_omp_ctx *octx;
   15884           88 :                   if (n != NULL
   15885           88 :                       && (n->value & (GOVD_DATA_SHARE_CLASS
   15886              :                                       & ~GOVD_FIRSTPRIVATE)))
   15887              :                     remove = true;
   15888              :                   else
   15889          100 :                     for (octx = ctx->outer_context; octx;
   15890           12 :                          octx = octx->outer_context)
   15891              :                       {
   15892           24 :                         n = splay_tree_lookup (octx->variables,
   15893              :                                                (splay_tree_key) decl);
   15894           24 :                         if (n == NULL)
   15895           12 :                           continue;
   15896           12 :                         if (n->value & GOVD_LOCAL)
   15897              :                           break;
   15898              :                         /* We have to avoid assigning a shared variable
   15899              :                            to itself when trying to add
   15900              :                            __builtin_assume_aligned.  */
   15901           12 :                         if (n->value & GOVD_SHARED)
   15902              :                           {
   15903              :                             remove = true;
   15904              :                             break;
   15905              :                           }
   15906              :                       }
   15907              :                 }
   15908              :             }
   15909           86 :           else if (TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
   15910              :             {
   15911           86 :               n = splay_tree_lookup (ctx->variables, (splay_tree_key) decl);
   15912           86 :               if (n != NULL && (n->value & GOVD_DATA_SHARE_CLASS) != 0)
   15913              :                 remove = true;
   15914              :             }
   15915              :           break;
   15916              : 
   15917          553 :         case OMP_CLAUSE_HAS_DEVICE_ADDR:
   15918          553 :           decl = OMP_CLAUSE_DECL (c);
   15919          553 :           while (INDIRECT_REF_P (decl)
   15920          602 :                  || TREE_CODE (decl) == ARRAY_REF)
   15921           49 :             decl = TREE_OPERAND (decl, 0);
   15922          553 :           n = splay_tree_lookup (ctx->variables, (splay_tree_key) decl);
   15923          553 :           remove = n == NULL || !(n->value & GOVD_SEEN);
   15924              :           break;
   15925              : 
   15926          900 :         case OMP_CLAUSE_IS_DEVICE_PTR:
   15927          900 :         case OMP_CLAUSE_NONTEMPORAL:
   15928          900 :           decl = OMP_CLAUSE_DECL (c);
   15929          900 :           n = splay_tree_lookup (ctx->variables, (splay_tree_key) decl);
   15930          900 :           remove = n == NULL || !(n->value & GOVD_SEEN);
   15931              :           break;
   15932              : 
   15933        56940 :         case OMP_CLAUSE_MAP:
   15934        56940 :           if (OMP_CLAUSE_MAP_GIMPLE_ONLY (c))
   15935              :             {
   15936           90 :               remove = true;
   15937           90 :               goto end_adjust_omp_map_clause;
   15938              :             }
   15939        56850 :           decl = OMP_CLAUSE_DECL (c);
   15940        56850 :           if (!grp_end)
   15941              :             {
   15942        31901 :               grp_start_p = list_p;
   15943        31901 :               grp_end = *omp_group_last (grp_start_p);
   15944              :             }
   15945        56850 :           switch (OMP_CLAUSE_MAP_KIND (c))
   15946              :             {
   15947          183 :             case GOMP_MAP_PRESENT_ALLOC:
   15948          183 :             case GOMP_MAP_PRESENT_TO:
   15949          183 :             case GOMP_MAP_PRESENT_FROM:
   15950          183 :             case GOMP_MAP_PRESENT_TOFROM:
   15951          183 :               OMP_CLAUSE_SET_MAP_KIND (c, GOMP_MAP_FORCE_PRESENT);
   15952          183 :               break;
   15953              :             default:
   15954              :               break;
   15955              :             }
   15956        56850 :           switch (code)
   15957              :             {
   15958         4563 :             case OACC_DATA:
   15959         4563 :               if (TREE_CODE (TREE_TYPE (decl)) != ARRAY_TYPE)
   15960              :                 break;
   15961              :               /* Fallthrough.  */
   15962        19317 :             case OACC_HOST_DATA:
   15963        19317 :             case OACC_ENTER_DATA:
   15964        19317 :             case OACC_EXIT_DATA:
   15965        19317 :             case OMP_TARGET_DATA:
   15966        19317 :             case OMP_TARGET_ENTER_DATA:
   15967        19317 :             case OMP_TARGET_EXIT_DATA:
   15968        19317 :               if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
   15969        19317 :                   || (OMP_CLAUSE_MAP_KIND (c)
   15970              :                       == GOMP_MAP_FIRSTPRIVATE_REFERENCE))
   15971              :                 /* For target {,enter ,exit }data only the array slice is
   15972              :                    mapped, but not the pointer to it.  */
   15973              :                 remove = true;
   15974        19317 :               if (code == OMP_TARGET_EXIT_DATA
   15975        19317 :                   && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ALWAYS_POINTER
   15976         2805 :                       || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER))
   15977              :                 remove = true;
   15978              :               break;
   15979              :             case OMP_TARGET:
   15980              :               break;
   15981              :             default:
   15982              :               break;
   15983              :             }
   15984        19315 :           if (remove)
   15985              :             break;
   15986        56515 :           if (OMP_CLAUSE_SIZE (c) == NULL_TREE)
   15987              :             {
   15988              :               /* Sanity check: attach/detach map kinds use the size as a bias,
   15989              :                  and it's never right to use the decl size for such
   15990              :                  mappings.  */
   15991        17840 :               gcc_assert (OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ATTACH
   15992              :                           && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_DETACH
   15993              :                           && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FORCE_DETACH
   15994              :                           && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ATTACH_DETACH
   15995              :                           && (OMP_CLAUSE_MAP_KIND (c)
   15996              :                               != GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION));
   15997        21899 :               OMP_CLAUSE_SIZE (c) = DECL_P (decl) ? DECL_SIZE_UNIT (decl)
   15998         4059 :                                     : TYPE_SIZE_UNIT (TREE_TYPE (decl));
   15999              :             }
   16000        56515 :           gimplify_omp_ctxp = ctx->outer_context;
   16001        56515 :           gimple_seq *seq_p;
   16002        56515 :           seq_p = enter_omp_iterator_loop_context (c, loops_seq_p, pre_p);
   16003        56515 :           if (gimplify_expr (&OMP_CLAUSE_SIZE (c), seq_p, NULL,
   16004              :                                   is_gimple_val, fb_rvalue) == GS_ERROR)
   16005              :             {
   16006            0 :               gimplify_omp_ctxp = ctx;
   16007            0 :               remove = true;
   16008            0 :               goto end_adjust_omp_map_clause;
   16009              :             }
   16010        56515 :           else if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
   16011        53594 :                     || (OMP_CLAUSE_MAP_KIND (c)
   16012              :                         == GOMP_MAP_FIRSTPRIVATE_REFERENCE)
   16013        53181 :                     || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH_DETACH)
   16014        63158 :                    && TREE_CODE (OMP_CLAUSE_SIZE (c)) != INTEGER_CST)
   16015              :             {
   16016          912 :               OMP_CLAUSE_SIZE (c)
   16017          912 :                 = get_initialized_tmp_var (OMP_CLAUSE_SIZE (c), seq_p, NULL,
   16018              :                                            false);
   16019          912 :               if ((ctx->region_type & ORT_TARGET) != 0)
   16020          703 :                 omp_add_variable (ctx, OMP_CLAUSE_SIZE (c),
   16021              :                                   GOVD_FIRSTPRIVATE | GOVD_SEEN);
   16022              :             }
   16023        56515 :           gimplify_omp_ctxp = ctx;
   16024              :           /* Data clauses associated with reductions must be
   16025              :              compatible with present_or_copy.  Warn and adjust the clause
   16026              :              if that is not the case.  */
   16027        56515 :           if (ctx->region_type == ORT_ACC_PARALLEL
   16028        47536 :               || ctx->region_type == ORT_ACC_SERIAL)
   16029              :             {
   16030         9535 :               tree t = DECL_P (decl) ? decl : TREE_OPERAND (decl, 0);
   16031         9535 :               n = NULL;
   16032              : 
   16033         9535 :               if (DECL_P (t))
   16034         7770 :                 n = splay_tree_lookup (ctx->variables, (splay_tree_key) t);
   16035              : 
   16036         7770 :               if (n && (n->value & GOVD_REDUCTION))
   16037              :                 {
   16038          809 :                   enum gomp_map_kind kind = OMP_CLAUSE_MAP_KIND (c);
   16039              : 
   16040          809 :                   OMP_CLAUSE_MAP_IN_REDUCTION (c) = 1;
   16041          809 :                   if ((kind & GOMP_MAP_TOFROM) != GOMP_MAP_TOFROM
   16042          159 :                       && kind != GOMP_MAP_FORCE_PRESENT
   16043          159 :                       && kind != GOMP_MAP_POINTER)
   16044              :                     {
   16045          120 :                       warning_at (OMP_CLAUSE_LOCATION (c), 0,
   16046              :                                   "incompatible data clause with reduction "
   16047              :                                   "on %qE; promoting to %<present_or_copy%>",
   16048          120 :                                   DECL_NAME (t));
   16049          120 :                       OMP_CLAUSE_SET_MAP_KIND (c, GOMP_MAP_TOFROM);
   16050              :                     }
   16051              :                 }
   16052              :             }
   16053        56515 :           if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_STRUCT
   16054        53720 :                || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_STRUCT_UNORD)
   16055        56791 :               && (code == OMP_TARGET_EXIT_DATA || code == OACC_EXIT_DATA))
   16056              :             {
   16057          536 :               remove = true;
   16058          536 :               goto end_adjust_omp_map_clause;
   16059              :             }
   16060              :           /* If we have a DECL_VALUE_EXPR (e.g. this is a class member and/or
   16061              :              a variable captured in a lambda closure), look through that now
   16062              :              before the DECL_P check below.  (A code other than COMPONENT_REF,
   16063              :              i.e. INDIRECT_REF, will be a VLA/variable-length array
   16064              :              section.  A global var may be a variable in a common block.  We
   16065              :              don't want to do this here for either of those.)  */
   16066        55979 :           if ((ctx->region_type & ORT_ACC) == 0
   16067        29772 :               && DECL_P (decl)
   16068        12781 :               && !is_global_var (decl)
   16069        11369 :               && DECL_HAS_VALUE_EXPR_P (decl)
   16070        56169 :               && TREE_CODE (DECL_VALUE_EXPR (decl)) == COMPONENT_REF)
   16071            0 :             decl = OMP_CLAUSE_DECL (c) = DECL_VALUE_EXPR (decl);
   16072        55979 :           if (TREE_CODE (decl) == TARGET_EXPR)
   16073              :             {
   16074            0 :               if (gimplify_expr (&OMP_CLAUSE_DECL (c), seq_p, NULL,
   16075              :                                  is_gimple_lvalue, fb_lvalue) == GS_ERROR)
   16076        26340 :                 remove = true;
   16077              :             }
   16078        55979 :           else if (!DECL_P (decl))
   16079              :             {
   16080        29639 :               if ((ctx->region_type & ORT_TARGET) != 0
   16081        29639 :                   && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER)
   16082              :                 {
   16083           24 :                   if (INDIRECT_REF_P (decl)
   16084            0 :                       && TREE_CODE (TREE_OPERAND (decl, 0)) == COMPONENT_REF
   16085           24 :                       && (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0)))
   16086              :                           == REFERENCE_TYPE))
   16087            0 :                     decl = TREE_OPERAND (decl, 0);
   16088           24 :                   if (TREE_CODE (decl) == COMPONENT_REF)
   16089              :                     {
   16090            0 :                       while (TREE_CODE (decl) == COMPONENT_REF)
   16091            0 :                         decl = TREE_OPERAND (decl, 0);
   16092            0 :                       if (DECL_P (decl))
   16093              :                         {
   16094            0 :                           n = splay_tree_lookup (ctx->variables,
   16095              :                                                  (splay_tree_key) decl);
   16096            0 :                           if (!(n->value & GOVD_SEEN))
   16097        29639 :                             remove = true;
   16098              :                         }
   16099              :                     }
   16100              :                 }
   16101              : 
   16102        29639 :               tree d = decl, *pd;
   16103        29639 :               if (TREE_CODE (d) == ARRAY_REF)
   16104              :                 {
   16105         5836 :                   while (TREE_CODE (d) == ARRAY_REF)
   16106         2978 :                     d = TREE_OPERAND (d, 0);
   16107         2858 :                   if (TREE_CODE (d) == COMPONENT_REF
   16108         2858 :                       && TREE_CODE (TREE_TYPE (d)) == ARRAY_TYPE)
   16109              :                     decl = d;
   16110              :                 }
   16111        29639 :               pd = &OMP_CLAUSE_DECL (c);
   16112        29639 :               if (d == decl
   16113        27140 :                   && TREE_CODE (decl) == INDIRECT_REF
   16114        15476 :                   && TREE_CODE (TREE_OPERAND (decl, 0)) == COMPONENT_REF
   16115         1485 :                   && (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl, 0)))
   16116              :                       == REFERENCE_TYPE)
   16117        30453 :                   && (OMP_CLAUSE_MAP_KIND (c)
   16118              :                       != GOMP_MAP_POINTER_TO_ZERO_LENGTH_ARRAY_SECTION))
   16119              :                 {
   16120          810 :                   pd = &TREE_OPERAND (decl, 0);
   16121          810 :                   decl = TREE_OPERAND (decl, 0);
   16122              :                 }
   16123              : 
   16124        29639 :               if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH_DETACH)
   16125         3067 :                 switch (code)
   16126              :                   {
   16127          427 :                   case OACC_ENTER_DATA:
   16128          427 :                   case OACC_EXIT_DATA:
   16129          427 :                     if (TREE_CODE (TREE_TYPE (OMP_CLAUSE_DECL (c)))
   16130              :                         == ARRAY_TYPE)
   16131              :                       remove = true;
   16132          417 :                     else if (code == OACC_ENTER_DATA)
   16133          263 :                       goto change_to_attach;
   16134              :                     /* Fallthrough.  */
   16135          605 :                   case OMP_TARGET_EXIT_DATA:
   16136          605 :                     OMP_CLAUSE_SET_MAP_KIND (c, GOMP_MAP_DETACH);
   16137          605 :                     OMP_CLAUSE_MAP_SIZE_NEEDS_ADJUSTMENT (c) = 0;
   16138          605 :                     break;
   16139           71 :                   case OACC_UPDATE:
   16140              :                     /* An "attach/detach" operation on an update directive
   16141              :                        should behave as a GOMP_MAP_ALWAYS_POINTER.  Note that
   16142              :                        both GOMP_MAP_ATTACH_DETACH and GOMP_MAP_ALWAYS_POINTER
   16143              :                        kinds depend on the previous mapping (for non-TARGET
   16144              :                        regions).  */
   16145           71 :                     OMP_CLAUSE_SET_MAP_KIND (c, GOMP_MAP_ALWAYS_POINTER);
   16146           71 :                     break;
   16147         2391 :                   default:
   16148         2391 :                   change_to_attach:
   16149         2391 :                     gcc_assert (!OMP_CLAUSE_MAP_SIZE_NEEDS_ADJUSTMENT (c));
   16150         2391 :                     OMP_CLAUSE_SET_MAP_KIND (c, GOMP_MAP_ATTACH);
   16151         2391 :                     if ((ctx->region_type & ORT_TARGET) != 0)
   16152         2113 :                       move_attach = true;
   16153              :                   }
   16154        26572 :               else if ((ctx->region_type & ORT_TARGET) != 0
   16155        26572 :                        && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
   16156        11858 :                            || (OMP_CLAUSE_MAP_KIND (c)
   16157              :                                == GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION)))
   16158              :                 move_attach = true;
   16159              : 
   16160              :               /* If we have e.g. map(struct: *var), don't gimplify the
   16161              :                  argument since omp-low.cc wants to see the decl itself.  */
   16162        29639 :               if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_STRUCT)
   16163         1136 :                 goto end_adjust_omp_map_clause;
   16164              : 
   16165              :               /* We've already partly gimplified this in
   16166              :                  gimplify_scan_omp_clauses.  Don't do any more.  */
   16167        28503 :               if (code == OMP_TARGET && OMP_CLAUSE_MAP_IN_REDUCTION (c))
   16168           88 :                 goto end_adjust_omp_map_clause;
   16169              : 
   16170        28415 :               gimplify_omp_ctxp = ctx->outer_context;
   16171        28415 :               if (gimplify_expr (pd, seq_p, NULL, is_gimple_lvalue,
   16172              :                                  fb_lvalue) == GS_ERROR)
   16173            0 :                 remove = true;
   16174        28415 :               gimplify_omp_ctxp = ctx;
   16175        28415 :               goto end_adjust_omp_map_clause;
   16176              :             }
   16177              : 
   16178        26340 :          if ((code == OMP_TARGET
   16179              :               || code == OMP_TARGET_DATA
   16180              :               || code == OMP_TARGET_ENTER_DATA
   16181        14453 :               || code == OMP_TARGET_EXIT_DATA)
   16182        26972 :              && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH_DETACH)
   16183              :            {
   16184          242 :              bool firstprivatize = false;
   16185              : 
   16186          247 :              for (struct gimplify_omp_ctx *octx = ctx->outer_context; octx;
   16187            5 :                   octx = octx->outer_context)
   16188              :                {
   16189            6 :                  splay_tree_node n
   16190           12 :                    = splay_tree_lookup (octx->variables,
   16191            6 :                                         (splay_tree_key) OMP_CLAUSE_DECL (c));
   16192              :                  /* If this is contained in an outer OpenMP region as a
   16193              :                     firstprivate value, remove the attach/detach.  */
   16194            6 :                  if (n && (n->value & GOVD_FIRSTPRIVATE))
   16195              :                    {
   16196              :                      firstprivatize = true;
   16197              :                      break;
   16198              :                    }
   16199              :                }
   16200              : 
   16201          242 :              enum gomp_map_kind map_kind;
   16202          242 :              if (firstprivatize)
   16203              :                map_kind = GOMP_MAP_FIRSTPRIVATE_POINTER;
   16204          241 :              else if (code == OMP_TARGET_EXIT_DATA)
   16205              :                map_kind = GOMP_MAP_DETACH;
   16206              :              else
   16207          190 :                map_kind = GOMP_MAP_ATTACH;
   16208          242 :              OMP_CLAUSE_SET_MAP_KIND (c, map_kind);
   16209              :            }
   16210        26098 :          else if ((ctx->region_type & ORT_ACC) != 0
   16211        26098 :                   && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH_DETACH)
   16212              :            {
   16213            0 :              enum gomp_map_kind map_kind = (code == OACC_EXIT_DATA
   16214            0 :                                             ? GOMP_MAP_DETACH
   16215              :                                             : GOMP_MAP_ATTACH);
   16216            0 :              OMP_CLAUSE_SET_MAP_KIND (c, map_kind);
   16217              :            }
   16218              : 
   16219        26340 :           n = splay_tree_lookup (ctx->variables, (splay_tree_key) decl);
   16220        26340 :           if ((ctx->region_type & ORT_TARGET) != 0
   16221        16375 :               && !(n->value & GOVD_SEEN)
   16222         1523 :               && GOMP_MAP_ALWAYS_P (OMP_CLAUSE_MAP_KIND (c)) == 0
   16223        27616 :               && (!is_global_var (decl)
   16224           75 :                   || !lookup_attribute ("omp declare target link",
   16225           75 :                                         DECL_ATTRIBUTES (decl))))
   16226              :             {
   16227         1264 :               remove = true;
   16228              :               /* For struct element mapping, if struct is never referenced
   16229              :                  in target block and none of the mapping has always modifier,
   16230              :                  remove all the struct element mappings, which immediately
   16231              :                  follow the GOMP_MAP_STRUCT map clause.  */
   16232         1264 :               if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_STRUCT
   16233         1264 :                   || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_STRUCT_UNORD)
   16234              :                 {
   16235           30 :                   HOST_WIDE_INT cnt = tree_to_shwi (OMP_CLAUSE_SIZE (c));
   16236           80 :                   while (cnt--)
   16237           50 :                     OMP_CLAUSE_CHAIN (c)
   16238           50 :                       = OMP_CLAUSE_CHAIN (OMP_CLAUSE_CHAIN (c));
   16239              :                 }
   16240              :             }
   16241        25076 :           else if (DECL_SIZE (decl)
   16242        25052 :                    && !poly_int_tree_p (DECL_SIZE (decl))
   16243          199 :                    && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_POINTER
   16244          199 :                    && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER
   16245        25127 :                    && (OMP_CLAUSE_MAP_KIND (c)
   16246              :                        != GOMP_MAP_FIRSTPRIVATE_REFERENCE))
   16247              :             {
   16248              :               /* For GOMP_MAP_FORCE_DEVICEPTR, we'll never enter here, because
   16249              :                  for these, TREE_CODE (DECL_SIZE (decl)) will always be
   16250              :                  INTEGER_CST.  */
   16251           51 :               gcc_assert (OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FORCE_DEVICEPTR);
   16252              : 
   16253           51 :               tree decl2 = DECL_VALUE_EXPR (decl);
   16254           51 :               gcc_assert (INDIRECT_REF_P (decl2));
   16255           51 :               decl2 = TREE_OPERAND (decl2, 0);
   16256           51 :               gcc_assert (DECL_P (decl2));
   16257           51 :               tree mem = build_simple_mem_ref (decl2);
   16258           51 :               OMP_CLAUSE_DECL (c) = mem;
   16259           51 :               OMP_CLAUSE_SIZE (c) = TYPE_SIZE_UNIT (TREE_TYPE (decl));
   16260           51 :               if (ctx->outer_context)
   16261              :                 {
   16262           15 :                   omp_notice_variable (ctx->outer_context, decl2, true);
   16263           30 :                   omp_notice_variable (ctx->outer_context,
   16264           15 :                                        OMP_CLAUSE_SIZE (c), true);
   16265              :                 }
   16266           51 :               if (((ctx->region_type & ORT_TARGET) != 0
   16267           24 :                    || !ctx->target_firstprivatize_array_bases)
   16268           34 :                   && ((n->value & GOVD_SEEN) == 0
   16269           30 :                       || (n->value & (GOVD_PRIVATE | GOVD_FIRSTPRIVATE)) == 0))
   16270              :                 {
   16271           34 :                   tree nc = build_omp_clause (OMP_CLAUSE_LOCATION (c),
   16272              :                                               OMP_CLAUSE_MAP);
   16273           34 :                   OMP_CLAUSE_DECL (nc) = decl;
   16274           34 :                   OMP_CLAUSE_SIZE (nc) = size_zero_node;
   16275           34 :                   if (ctx->target_firstprivatize_array_bases)
   16276           27 :                     OMP_CLAUSE_SET_MAP_KIND (nc,
   16277              :                                              GOMP_MAP_FIRSTPRIVATE_POINTER);
   16278              :                   else
   16279            7 :                     OMP_CLAUSE_SET_MAP_KIND (nc, GOMP_MAP_POINTER);
   16280           34 :                   OMP_CLAUSE_CHAIN (nc) = OMP_CLAUSE_CHAIN (c);
   16281           34 :                   OMP_CLAUSE_CHAIN (c) = nc;
   16282           34 :                   c = nc;
   16283              :                 }
   16284              :             }
   16285              :           else
   16286              :             {
   16287        25025 :               if (OMP_CLAUSE_SIZE (c) == NULL_TREE)
   16288            0 :                 OMP_CLAUSE_SIZE (c) = DECL_SIZE_UNIT (decl);
   16289        25025 :               gcc_assert ((n->value & GOVD_SEEN) == 0
   16290              :                           || ((n->value & (GOVD_PRIVATE | GOVD_FIRSTPRIVATE))
   16291              :                               == 0));
   16292              :             }
   16293              : 
   16294              :           /* If we have a target region, we can push all the attaches to the
   16295              :              end of the list (we may have standalone "attach" operations
   16296              :              synthesized for GOMP_MAP_STRUCT nodes that must be processed after
   16297              :              the attachment point AND the pointed-to block have been mapped).
   16298              :              If we have something else, e.g. "enter data", we need to keep
   16299              :              "attach" nodes together with the previous node they attach to so
   16300              :              that separate "exit data" operations work properly (see
   16301              :              libgomp/target.c).  */
   16302        26340 :           if ((ctx->region_type & ORT_TARGET) != 0
   16303        26340 :               && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
   16304        16259 :                   || (OMP_CLAUSE_MAP_KIND (c)
   16305              :                       == GOMP_MAP_ATTACH_ZERO_LENGTH_ARRAY_SECTION)))
   16306              :             move_attach = true;
   16307              : 
   16308        56605 : end_adjust_omp_map_clause:
   16309        56605 :           exit_omp_iterator_loop_context (c);
   16310        56605 :           break;
   16311              : 
   16312         8383 :         case OMP_CLAUSE_TO:
   16313         8383 :         case OMP_CLAUSE_FROM:
   16314         8383 :         case OMP_CLAUSE__CACHE_:
   16315         8383 :           decl = OMP_CLAUSE_DECL (c);
   16316         8383 :           if (!DECL_P (decl))
   16317              :             break;
   16318         6203 :           if (DECL_SIZE (decl)
   16319         6203 :               && !poly_int_tree_p (DECL_SIZE (decl)))
   16320              :             {
   16321            4 :               tree decl2 = DECL_VALUE_EXPR (decl);
   16322            4 :               gcc_assert (INDIRECT_REF_P (decl2));
   16323            4 :               decl2 = TREE_OPERAND (decl2, 0);
   16324            4 :               gcc_assert (DECL_P (decl2));
   16325            4 :               tree mem = build_simple_mem_ref (decl2);
   16326            4 :               OMP_CLAUSE_DECL (c) = mem;
   16327            4 :               OMP_CLAUSE_SIZE (c) = TYPE_SIZE_UNIT (TREE_TYPE (decl));
   16328            4 :               if (ctx->outer_context)
   16329              :                 {
   16330            4 :                   omp_notice_variable (ctx->outer_context, decl2, true);
   16331            4 :                   omp_notice_variable (ctx->outer_context,
   16332            4 :                                        OMP_CLAUSE_SIZE (c), true);
   16333              :                 }
   16334              :             }
   16335         6199 :           else if (OMP_CLAUSE_SIZE (c) == NULL_TREE)
   16336            0 :             OMP_CLAUSE_SIZE (c) = DECL_SIZE_UNIT (decl);
   16337              :           break;
   16338              : 
   16339        15397 :         case OMP_CLAUSE_REDUCTION:
   16340        15397 :           if (OMP_CLAUSE_REDUCTION_INSCAN (c))
   16341              :             {
   16342          831 :               decl = OMP_CLAUSE_DECL (c);
   16343          831 :               n = splay_tree_lookup (ctx->variables, (splay_tree_key) decl);
   16344          831 :               if ((n->value & GOVD_REDUCTION_INSCAN) == 0)
   16345              :                 {
   16346           69 :                   remove = true;
   16347           69 :                   error_at (OMP_CLAUSE_LOCATION (c),
   16348              :                             "%qD specified in %<inscan%> %<reduction%> clause "
   16349              :                             "but not in %<scan%> directive clause", decl);
   16350           69 :                   break;
   16351              :                 }
   16352              :               has_inscan_reductions = true;
   16353              :             }
   16354              :           /* FALLTHRU */
   16355        17957 :         case OMP_CLAUSE_IN_REDUCTION:
   16356        17957 :         case OMP_CLAUSE_TASK_REDUCTION:
   16357        17957 :           decl = OMP_CLAUSE_DECL (c);
   16358              :           /* OpenACC reductions need a present_or_copy data clause.
   16359              :              Add one if necessary.  Emit error when the reduction is private.  */
   16360        17957 :           if (ctx->region_type == ORT_ACC_PARALLEL
   16361        17425 :               || ctx->region_type == ORT_ACC_SERIAL)
   16362              :             {
   16363          803 :               n = splay_tree_lookup (ctx->variables, (splay_tree_key) decl);
   16364          803 :               if (n->value & (GOVD_PRIVATE | GOVD_FIRSTPRIVATE))
   16365              :                 {
   16366           13 :                   remove = true;
   16367           13 :                   error_at (OMP_CLAUSE_LOCATION (c), "invalid private "
   16368           13 :                             "reduction on %qE", DECL_NAME (decl));
   16369              :                 }
   16370          790 :               else if ((n->value & GOVD_MAP) == 0)
   16371              :                 {
   16372          524 :                   tree next = OMP_CLAUSE_CHAIN (c);
   16373          524 :                   tree nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_MAP);
   16374          524 :                   OMP_CLAUSE_SET_MAP_KIND (nc, GOMP_MAP_TOFROM);
   16375          524 :                   OMP_CLAUSE_DECL (nc) = decl;
   16376          524 :                   OMP_CLAUSE_CHAIN (c) = nc;
   16377          524 :                   lang_hooks.decls.omp_finish_clause (nc, pre_p,
   16378          524 :                                                       (ctx->region_type
   16379              :                                                        & ORT_ACC) != 0);
   16380          550 :                   while (1)
   16381              :                     {
   16382          537 :                       OMP_CLAUSE_MAP_IN_REDUCTION (nc) = 1;
   16383          537 :                       if (OMP_CLAUSE_CHAIN (nc) == NULL)
   16384              :                         break;
   16385           13 :                       nc = OMP_CLAUSE_CHAIN (nc);
   16386              :                     }
   16387          524 :                   OMP_CLAUSE_CHAIN (nc) = next;
   16388          524 :                   n->value |= GOVD_MAP;
   16389              :                 }
   16390              :             }
   16391        17957 :           if (DECL_P (decl)
   16392        17957 :               && omp_shared_to_firstprivate_optimizable_decl_p (decl))
   16393         8042 :             omp_mark_stores (gimplify_omp_ctxp->outer_context, decl);
   16394              :           break;
   16395              : 
   16396         3682 :         case OMP_CLAUSE_ALLOCATE:
   16397         3682 :           decl = OMP_CLAUSE_DECL (c);
   16398         3682 :           n = splay_tree_lookup (ctx->variables, (splay_tree_key) decl);
   16399         3682 :           if (n != NULL && !(n->value & GOVD_SEEN))
   16400              :             {
   16401         1180 :               if ((n->value & (GOVD_PRIVATE | GOVD_FIRSTPRIVATE | GOVD_LINEAR))
   16402              :                   != 0
   16403         1180 :                   && (n->value & (GOVD_REDUCTION | GOVD_LASTPRIVATE)) == 0)
   16404              :                 remove = true;
   16405              :             }
   16406              :           if (!remove
   16407         2502 :               && OMP_CLAUSE_ALLOCATE_ALLOCATOR (c)
   16408         1286 :               && TREE_CODE (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c)) != INTEGER_CST
   16409          492 :               && ((ctx->region_type & (ORT_PARALLEL | ORT_TARGET)) != 0
   16410          347 :                   || (ctx->region_type & ORT_TASKLOOP) == ORT_TASK
   16411          287 :                   || (ctx->region_type & ORT_HOST_TEAMS) == ORT_HOST_TEAMS))
   16412              :             {
   16413          223 :               tree allocator = OMP_CLAUSE_ALLOCATE_ALLOCATOR (c);
   16414          223 :               n = splay_tree_lookup (ctx->variables, (splay_tree_key) allocator);
   16415          223 :               if (n == NULL)
   16416              :                 {
   16417           92 :                   enum omp_clause_default_kind default_kind
   16418              :                     = ctx->default_kind;
   16419           92 :                   ctx->default_kind = OMP_CLAUSE_DEFAULT_FIRSTPRIVATE;
   16420           92 :                   omp_notice_variable (ctx, OMP_CLAUSE_ALLOCATE_ALLOCATOR (c),
   16421              :                                        true);
   16422           92 :                   ctx->default_kind = default_kind;
   16423              :                 }
   16424              :               else
   16425          131 :                 omp_notice_variable (ctx, OMP_CLAUSE_ALLOCATE_ALLOCATOR (c),
   16426              :                                      true);
   16427              :             }
   16428              :           break;
   16429              : 
   16430              :         case OMP_CLAUSE_COPYIN:
   16431              :         case OMP_CLAUSE_COPYPRIVATE:
   16432              :         case OMP_CLAUSE_IF:
   16433              :         case OMP_CLAUSE_SELF:
   16434              :         case OMP_CLAUSE_NUM_THREADS:
   16435              :         case OMP_CLAUSE_NUM_TEAMS:
   16436              :         case OMP_CLAUSE_THREAD_LIMIT:
   16437              :         case OMP_CLAUSE_DIST_SCHEDULE:
   16438              :         case OMP_CLAUSE_DEVICE:
   16439              :         case OMP_CLAUSE_SCHEDULE:
   16440              :         case OMP_CLAUSE_NOWAIT:
   16441              :         case OMP_CLAUSE_ORDERED:
   16442              :         case OMP_CLAUSE_DEFAULT:
   16443              :         case OMP_CLAUSE_UNTIED:
   16444              :         case OMP_CLAUSE_COLLAPSE:
   16445              :         case OMP_CLAUSE_FINAL:
   16446              :         case OMP_CLAUSE_MERGEABLE:
   16447              :         case OMP_CLAUSE_PROC_BIND:
   16448              :         case OMP_CLAUSE_SAFELEN:
   16449              :         case OMP_CLAUSE_SIMDLEN:
   16450              :         case OMP_CLAUSE_DEPEND:
   16451              :         case OMP_CLAUSE_DOACROSS:
   16452              :         case OMP_CLAUSE_PRIORITY:
   16453              :         case OMP_CLAUSE_GRAINSIZE:
   16454              :         case OMP_CLAUSE_NUM_TASKS:
   16455              :         case OMP_CLAUSE_NOGROUP:
   16456              :         case OMP_CLAUSE_THREADS:
   16457              :         case OMP_CLAUSE_SIMD:
   16458              :         case OMP_CLAUSE_FILTER:
   16459              :         case OMP_CLAUSE_HINT:
   16460              :         case OMP_CLAUSE_DEFAULTMAP:
   16461              :         case OMP_CLAUSE_ORDER:
   16462              :         case OMP_CLAUSE_BIND:
   16463              :         case OMP_CLAUSE_DETACH:
   16464              :         case OMP_CLAUSE_USE_DEVICE_PTR:
   16465              :         case OMP_CLAUSE_USE_DEVICE_ADDR:
   16466              :         case OMP_CLAUSE_ASYNC:
   16467              :         case OMP_CLAUSE_WAIT:
   16468              :         case OMP_CLAUSE_INDEPENDENT:
   16469              :         case OMP_CLAUSE_NUM_GANGS:
   16470              :         case OMP_CLAUSE_NUM_WORKERS:
   16471              :         case OMP_CLAUSE_VECTOR_LENGTH:
   16472              :         case OMP_CLAUSE_GANG:
   16473              :         case OMP_CLAUSE_WORKER:
   16474              :         case OMP_CLAUSE_VECTOR:
   16475              :         case OMP_CLAUSE_AUTO:
   16476              :         case OMP_CLAUSE_SEQ:
   16477              :         case OMP_CLAUSE_TILE:
   16478              :         case OMP_CLAUSE_IF_PRESENT:
   16479              :         case OMP_CLAUSE_FINALIZE:
   16480              :         case OMP_CLAUSE_INCLUSIVE:
   16481              :         case OMP_CLAUSE_EXCLUSIVE:
   16482              :         case OMP_CLAUSE_USES_ALLOCATORS:
   16483              :         case OMP_CLAUSE_DEVICE_TYPE:
   16484              :         case OMP_CLAUSE_MESSAGE:
   16485              :           break;
   16486              : 
   16487            0 :         case OMP_CLAUSE_NOHOST:
   16488            0 :         default:
   16489            0 :           gcc_unreachable ();
   16490              :         }
   16491              : 
   16492       129220 :       if (remove)
   16493         9962 :         *list_p = OMP_CLAUSE_CHAIN (c);
   16494       240814 :       else if (move_attach)
   16495              :         {
   16496              :           /* Remove attach node from here, separate out into its own list.  */
   16497         2360 :           *attach_tail = c;
   16498         2360 :           *list_p = OMP_CLAUSE_CHAIN (c);
   16499         2360 :           OMP_CLAUSE_CHAIN (c) = NULL_TREE;
   16500         2360 :           attach_tail = &OMP_CLAUSE_CHAIN (c);
   16501              :         }
   16502              :       else
   16503       238454 :         list_p = &OMP_CLAUSE_CHAIN (c);
   16504              :     }
   16505              : 
   16506              :   /* Splice attach nodes at the end of the list.  */
   16507       130448 :   if (attach_list)
   16508              :     {
   16509         1127 :       *list_p = attach_list;
   16510         1127 :       list_p = attach_tail;
   16511              :     }
   16512              : 
   16513              :   /* Add in any implicit data sharing.  */
   16514       130448 :   struct gimplify_adjust_omp_clauses_data data;
   16515       130448 :   if ((gimplify_omp_ctxp->region_type & ORT_ACC) == 0)
   16516              :     {
   16517              :       /* OpenMP.  Implicit clauses are added at the start of the clause list,
   16518              :          but after any non-map clauses.  */
   16519              :       tree *implicit_add_list_p = orig_list_p;
   16520       255197 :       while (*implicit_add_list_p
   16521       255197 :              && OMP_CLAUSE_CODE (*implicit_add_list_p) != OMP_CLAUSE_MAP)
   16522       154648 :         implicit_add_list_p = &OMP_CLAUSE_CHAIN (*implicit_add_list_p);
   16523       100549 :       data.list_p = implicit_add_list_p;
   16524              :     }
   16525              :   else
   16526              :     /* OpenACC.  */
   16527        29899 :     data.list_p = list_p;
   16528       130448 :   data.pre_p = pre_p;
   16529       130448 :   splay_tree_foreach (ctx->variables, gimplify_adjust_omp_clauses_1, &data);
   16530              : 
   16531       130448 :   if (has_inscan_reductions)
   16532         2499 :     for (c = *orig_list_p; c; c = OMP_CLAUSE_CHAIN (c))
   16533         1870 :       if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
   16534         1870 :           && !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
   16535              :         {
   16536            5 :           error_at (OMP_CLAUSE_LOCATION (c),
   16537              :                     "%<inscan%> %<reduction%> clause used together with "
   16538              :                     "%<linear%> clause for a variable other than loop "
   16539              :                     "iterator");
   16540            5 :           break;
   16541              :         }
   16542              : 
   16543       130448 :   gimplify_omp_ctxp = ctx->outer_context;
   16544       130448 :   delete_omp_context (ctx);
   16545       130448 : }
   16546              : 
   16547              : /* Try to evaluate a novariants clause. Return 1 if true, 0 if false or absent,
   16548              :  * -1 if run-time evaluation is needed. */
   16549              : 
   16550              : int
   16551       257656 : omp_has_novariants (void)
   16552              : {
   16553       257656 :   struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
   16554       257656 :   if (ctx != NULL && ctx->code == OMP_DISPATCH && !ctx->in_call_args)
   16555              :     {
   16556         3290 :       tree c = omp_find_clause (ctx->clauses, OMP_CLAUSE_NOVARIANTS);
   16557         3290 :       if (c != NULL_TREE)
   16558              :         {
   16559          299 :           if (integer_nonzerop (OMP_CLAUSE_NOVARIANTS_EXPR (c)))
   16560              :             return 1;
   16561          171 :           else if (integer_zerop (OMP_CLAUSE_NOVARIANTS_EXPR (c)))
   16562              :             return 0;
   16563              :           else
   16564           38 :             return -1;
   16565              :         }
   16566              :       return 0;
   16567              :     }
   16568              :   return 0;
   16569              : }
   16570              : 
   16571              : /* Try to evaluate a nocontext clause. Return 1 if true, 0 if false or absent,
   16572              :  * -1 if run-time evaluation is needed. */
   16573              : 
   16574              : static int
   16575         1903 : omp_has_nocontext (void)
   16576              : {
   16577         1903 :   struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
   16578         1903 :   if (ctx != NULL && ctx->code == OMP_DISPATCH)
   16579              :     {
   16580         1903 :       tree c = omp_find_clause (ctx->clauses, OMP_CLAUSE_NOCONTEXT);
   16581         1903 :       if (c != NULL_TREE)
   16582              :         {
   16583          256 :           if (integer_nonzerop (OMP_CLAUSE_NOCONTEXT_EXPR (c)))
   16584              :             return 1;
   16585           94 :           else if (integer_zerop (OMP_CLAUSE_NOCONTEXT_EXPR (c)))
   16586              :             return 0;
   16587              :           else
   16588           44 :             return -1;
   16589              :         }
   16590              :       return 0;
   16591              :     }
   16592              :   return 0;
   16593              : }
   16594              : 
   16595              : /* Collect a list of traits for enclosing constructs in the current
   16596              :    OpenMP context.  The list is in the same format as the trait selector
   16597              :    list of construct trait sets built by the front ends.
   16598              : 
   16599              :    Per the OpenMP specification, the construct trait set includes constructs
   16600              :    up to an enclosing "target" construct.  If there is no "target" construct,
   16601              :    then additional things may be added to the construct trait set (simd for
   16602              :    simd clones, additional constructs associated with "declare variant",
   16603              :    the target trait for "declare target"); those are not handled here.
   16604              :    In particular simd clones are not known during gimplification so
   16605              :    matching/scoring of context selectors that might involve them needs
   16606              :    to be deferred to the omp_device_lower pass.  */
   16607              : 
   16608              : tree
   16609         2248 : omp_get_construct_context (void)
   16610              : {
   16611         2248 :   tree result = NULL_TREE;
   16612         4247 :   for (struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp; ctx;)
   16613              :     {
   16614         2121 :       if (((ctx->region_type & (ORT_TARGET | ORT_IMPLICIT_TARGET | ORT_ACC))
   16615              :                 == ORT_TARGET)
   16616          122 :                && ctx->code == OMP_TARGET)
   16617              :         {
   16618          122 :           result = make_trait_selector (OMP_TRAIT_CONSTRUCT_TARGET,
   16619              :                                         NULL_TREE, NULL_TREE, result);
   16620              :           /* We're not interested in any outer constructs.  */
   16621          122 :           break;
   16622              :         }
   16623         1999 :       else if ((ctx->region_type & ORT_PARALLEL) && ctx->code == OMP_PARALLEL)
   16624          216 :         result = make_trait_selector (OMP_TRAIT_CONSTRUCT_PARALLEL,
   16625              :                                       NULL_TREE, NULL_TREE, result);
   16626         1783 :       else if ((ctx->region_type & ORT_TEAMS) && ctx->code == OMP_TEAMS)
   16627           75 :         result = make_trait_selector (OMP_TRAIT_CONSTRUCT_TEAMS,
   16628              :                                       NULL_TREE, NULL_TREE, result);
   16629         1708 :       else if (ctx->region_type == ORT_WORKSHARE && ctx->code == OMP_FOR)
   16630          164 :         result = make_trait_selector (OMP_TRAIT_CONSTRUCT_FOR,
   16631              :                                       NULL_TREE, NULL_TREE, result);
   16632         1544 :       else if (ctx->code == OMP_DISPATCH && omp_has_nocontext () != 1)
   16633         1279 :         result = make_trait_selector (OMP_TRAIT_CONSTRUCT_DISPATCH,
   16634              :                                       NULL_TREE, NULL_TREE, result);
   16635          265 :       else if (ctx->region_type == ORT_SIMD
   16636           58 :                && ctx->code == OMP_SIMD
   16637          323 :                && !omp_find_clause (ctx->clauses, OMP_CLAUSE_BIND))
   16638              :         {
   16639           58 :           tree props = NULL_TREE;
   16640           58 :           tree *last = &props;
   16641          120 :           for (tree c = ctx->clauses; c; c = OMP_CLAUSE_CHAIN (c))
   16642           62 :             if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SIMDLEN
   16643           42 :                 || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_INBRANCH
   16644          104 :                 || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOTINBRANCH)
   16645              :               {
   16646           20 :                 *last = unshare_expr (c);
   16647           20 :                 last = &(OMP_CLAUSE_CHAIN (c));
   16648              :               }
   16649           58 :           result = make_trait_selector (OMP_TRAIT_CONSTRUCT_SIMD,
   16650              :                                         NULL_TREE, props, result);
   16651              :         }
   16652          207 :       else if (ctx->region_type == ORT_WORKSHARE
   16653            5 :                && ctx->code == OMP_LOOP
   16654            0 :                && ctx->outer_context
   16655            0 :                && ctx->outer_context->region_type == ORT_COMBINED_PARALLEL
   16656            0 :                && ctx->outer_context->outer_context
   16657            0 :                && ctx->outer_context->outer_context->code == OMP_LOOP
   16658            0 :                && ctx->outer_context->outer_context->distribute)
   16659         1999 :         ctx = ctx->outer_context->outer_context;
   16660         1999 :       ctx = ctx->outer_context;
   16661              :     }
   16662              : 
   16663         2248 :   return result;
   16664              : }
   16665              : 
   16666              : /* Gimplify OACC_CACHE.  */
   16667              : 
   16668              : static void
   16669          717 : gimplify_oacc_cache (tree *expr_p, gimple_seq *pre_p)
   16670              : {
   16671          717 :   tree expr = *expr_p;
   16672              : 
   16673          717 :   gimplify_scan_omp_clauses (&OACC_CACHE_CLAUSES (expr), pre_p, ORT_ACC,
   16674              :                              OACC_CACHE);
   16675          717 :   gimplify_adjust_omp_clauses (pre_p, NULL, &OACC_CACHE_CLAUSES (expr),
   16676              :                                OACC_CACHE);
   16677              : 
   16678              :   /* TODO: Do something sensible with this information.  */
   16679              : 
   16680          717 :   *expr_p = NULL_TREE;
   16681          717 : }
   16682              : 
   16683              : /* Helper function of gimplify_oacc_declare.  The helper's purpose is to,
   16684              :    if required, translate 'kind' in CLAUSE into an 'entry' kind and 'exit'
   16685              :    kind.  The entry kind will replace the one in CLAUSE, while the exit
   16686              :    kind will be used in a new omp_clause and returned to the caller.  */
   16687              : 
   16688              : static tree
   16689          190 : gimplify_oacc_declare_1 (tree clause)
   16690              : {
   16691          190 :   HOST_WIDE_INT kind, new_op;
   16692          190 :   bool ret = false;
   16693          190 :   tree c = NULL;
   16694              : 
   16695          190 :   kind = OMP_CLAUSE_MAP_KIND (clause);
   16696              : 
   16697          190 :   switch (kind)
   16698              :     {
   16699              :       case GOMP_MAP_ALLOC:
   16700              :         new_op = GOMP_MAP_RELEASE;
   16701              :         ret = true;
   16702              :         break;
   16703              : 
   16704           29 :       case GOMP_MAP_FROM:
   16705           29 :         OMP_CLAUSE_SET_MAP_KIND (clause, GOMP_MAP_FORCE_ALLOC);
   16706           29 :         new_op = GOMP_MAP_FROM;
   16707           29 :         ret = true;
   16708           29 :         break;
   16709              : 
   16710           40 :       case GOMP_MAP_TOFROM:
   16711           40 :         OMP_CLAUSE_SET_MAP_KIND (clause, GOMP_MAP_TO);
   16712           40 :         new_op = GOMP_MAP_FROM;
   16713           40 :         ret = true;
   16714           40 :         break;
   16715              : 
   16716              :       case GOMP_MAP_DEVICE_RESIDENT:
   16717              :       case GOMP_MAP_FORCE_DEVICEPTR:
   16718              :       case GOMP_MAP_FORCE_PRESENT:
   16719              :       case GOMP_MAP_LINK:
   16720              :       case GOMP_MAP_POINTER:
   16721              :       case GOMP_MAP_TO:
   16722              :         break;
   16723              : 
   16724            0 :       default:
   16725            0 :         gcc_unreachable ();
   16726           69 :         break;
   16727              :     }
   16728              : 
   16729           69 :   if (ret)
   16730              :     {
   16731          116 :       c = build_omp_clause (OMP_CLAUSE_LOCATION (clause), OMP_CLAUSE_MAP);
   16732          116 :       OMP_CLAUSE_SET_MAP_KIND (c, new_op);
   16733          116 :       OMP_CLAUSE_DECL (c) = OMP_CLAUSE_DECL (clause);
   16734              :     }
   16735              : 
   16736          190 :   return c;
   16737              : }
   16738              : 
   16739              : /* Gimplify OACC_DECLARE.  */
   16740              : 
   16741              : static void
   16742          254 : gimplify_oacc_declare (tree *expr_p, gimple_seq *pre_p)
   16743              : {
   16744          254 :   tree expr = *expr_p;
   16745          254 :   gomp_target *stmt;
   16746          254 :   tree clauses, t, decl;
   16747              : 
   16748          254 :   clauses = OACC_DECLARE_CLAUSES (expr);
   16749              : 
   16750          254 :   gimplify_scan_omp_clauses (&clauses, pre_p, ORT_TARGET_DATA, OACC_DECLARE);
   16751          254 :   gimplify_adjust_omp_clauses (pre_p, NULL, &clauses, OACC_DECLARE);
   16752              : 
   16753          516 :   for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
   16754              :     {
   16755          262 :       decl = OMP_CLAUSE_DECL (t);
   16756              : 
   16757          262 :       if (TREE_CODE (decl) == MEM_REF)
   16758            8 :         decl = TREE_OPERAND (decl, 0);
   16759              : 
   16760          262 :       if (VAR_P (decl) && !is_oacc_declared (decl))
   16761              :         {
   16762          262 :           tree attr = get_identifier ("oacc declare target");
   16763          262 :           DECL_ATTRIBUTES (decl) = tree_cons (attr, NULL_TREE,
   16764          262 :                                               DECL_ATTRIBUTES (decl));
   16765              :         }
   16766              : 
   16767          262 :       if (VAR_P (decl)
   16768          262 :           && !is_global_var (decl)
   16769          452 :           && DECL_CONTEXT (decl) == current_function_decl)
   16770              :         {
   16771          190 :           tree c = gimplify_oacc_declare_1 (t);
   16772          190 :           if (c)
   16773              :             {
   16774          116 :               if (oacc_declare_returns == NULL)
   16775           40 :                 oacc_declare_returns = new hash_map<tree, tree>;
   16776              : 
   16777          116 :               oacc_declare_returns->put (decl, c);
   16778              :             }
   16779              :         }
   16780              : 
   16781          262 :       if (gimplify_omp_ctxp)
   16782           72 :         omp_add_variable (gimplify_omp_ctxp, decl, GOVD_SEEN);
   16783              :     }
   16784              : 
   16785          254 :   stmt = gimple_build_omp_target (NULL, GF_OMP_TARGET_KIND_OACC_DECLARE,
   16786              :                                   clauses);
   16787              : 
   16788          254 :   gimplify_seq_add_stmt (pre_p, stmt);
   16789              : 
   16790          254 :   *expr_p = NULL_TREE;
   16791          254 : }
   16792              : 
   16793              : /* Gimplify the contents of an OMP_PARALLEL statement.  This involves
   16794              :    gimplification of the body, as well as scanning the body for used
   16795              :    variables.  We need to do this scan now, because variable-sized
   16796              :    decls will be decomposed during gimplification.  */
   16797              : 
   16798              : static void
   16799        18464 : gimplify_omp_parallel (tree *expr_p, gimple_seq *pre_p)
   16800              : {
   16801        18464 :   tree expr = *expr_p;
   16802        18464 :   gimple *g;
   16803        18464 :   gimple_seq body = NULL;
   16804              : 
   16805        36928 :   gimplify_scan_omp_clauses (&OMP_PARALLEL_CLAUSES (expr), pre_p,
   16806        18464 :                              OMP_PARALLEL_COMBINED (expr)
   16807              :                              ? ORT_COMBINED_PARALLEL
   16808              :                              : ORT_PARALLEL, OMP_PARALLEL);
   16809              : 
   16810        18464 :   push_gimplify_context ();
   16811              : 
   16812        18464 :   g = gimplify_and_return_first (OMP_PARALLEL_BODY (expr), &body);
   16813        18464 :   if (gimple_code (g) == GIMPLE_BIND)
   16814        18464 :     pop_gimplify_context (g);
   16815              :   else
   16816            0 :     pop_gimplify_context (NULL);
   16817              : 
   16818        18464 :   gimplify_adjust_omp_clauses (pre_p, body, &OMP_PARALLEL_CLAUSES (expr),
   16819              :                                OMP_PARALLEL);
   16820              : 
   16821        36928 :   g = gimple_build_omp_parallel (body,
   16822        18464 :                                  OMP_PARALLEL_CLAUSES (expr),
   16823              :                                  NULL_TREE, NULL_TREE);
   16824        18464 :   if (OMP_PARALLEL_COMBINED (expr))
   16825        12601 :     gimple_omp_set_subcode (g, GF_OMP_PARALLEL_COMBINED);
   16826        18464 :   gimplify_seq_add_stmt (pre_p, g);
   16827        18464 :   *expr_p = NULL_TREE;
   16828        18464 : }
   16829              : 
   16830              : /* Gimplify the contents of an OMP_TASK statement.  This involves
   16831              :    gimplification of the body, as well as scanning the body for used
   16832              :    variables.  We need to do this scan now, because variable-sized
   16833              :    decls will be decomposed during gimplification.  */
   16834              : 
   16835              : static void
   16836         4132 : gimplify_omp_task (tree *expr_p, gimple_seq *pre_p)
   16837              : {
   16838         4132 :   tree expr = *expr_p;
   16839         4132 :   gimple *g;
   16840         4132 :   gimple_seq body = NULL;
   16841         4132 :   bool nowait = false;
   16842         4132 :   bool has_depend = false;
   16843              : 
   16844         4132 :   if (OMP_TASK_BODY (expr) == NULL_TREE)
   16845              :     {
   16846          242 :       for (tree c = OMP_TASK_CLAUSES (expr); c; c = OMP_CLAUSE_CHAIN (c))
   16847          153 :         if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
   16848              :           {
   16849          111 :             has_depend = true;
   16850          111 :             if (OMP_CLAUSE_DEPEND_KIND (c) == OMP_CLAUSE_DEPEND_MUTEXINOUTSET)
   16851              :               {
   16852            9 :                 error_at (OMP_CLAUSE_LOCATION (c),
   16853              :                           "%<mutexinoutset%> kind in %<depend%> clause on a "
   16854              :                           "%<taskwait%> construct");
   16855            9 :                 break;
   16856              :               }
   16857              :           }
   16858           42 :         else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOWAIT)
   16859           42 :           nowait = true;
   16860           98 :       if (nowait && !has_depend)
   16861              :         {
   16862            5 :           error_at (EXPR_LOCATION (expr),
   16863              :                     "%<taskwait%> construct with %<nowait%> clause but no "
   16864              :                     "%<depend%> clauses");
   16865            5 :           *expr_p = NULL_TREE;
   16866            5 :           return;
   16867              :         }
   16868              :     }
   16869              : 
   16870        12381 :   gimplify_scan_omp_clauses (&OMP_TASK_CLAUSES (expr), pre_p,
   16871         4127 :                              omp_find_clause (OMP_TASK_CLAUSES (expr),
   16872              :                                               OMP_CLAUSE_UNTIED)
   16873              :                              ? ORT_UNTIED_TASK : ORT_TASK, OMP_TASK);
   16874              : 
   16875         4127 :   if (OMP_TASK_BODY (expr))
   16876              :     {
   16877         4034 :       push_gimplify_context ();
   16878              : 
   16879         4034 :       g = gimplify_and_return_first (OMP_TASK_BODY (expr), &body);
   16880         4034 :       if (gimple_code (g) == GIMPLE_BIND)
   16881         4034 :         pop_gimplify_context (g);
   16882              :       else
   16883            0 :         pop_gimplify_context (NULL);
   16884              :     }
   16885              : 
   16886         4127 :   gimplify_adjust_omp_clauses (pre_p, body, &OMP_TASK_CLAUSES (expr),
   16887              :                                OMP_TASK);
   16888              : 
   16889         8254 :   g = gimple_build_omp_task (body,
   16890         4127 :                              OMP_TASK_CLAUSES (expr),
   16891              :                              NULL_TREE, NULL_TREE,
   16892              :                              NULL_TREE, NULL_TREE, NULL_TREE);
   16893         4127 :   if (OMP_TASK_BODY (expr) == NULL_TREE)
   16894           93 :     gimple_omp_task_set_taskwait_p (g, true);
   16895         4127 :   gimplify_seq_add_stmt (pre_p, g);
   16896         4127 :   *expr_p = NULL_TREE;
   16897              : }
   16898              : 
   16899              : /* Helper function for gimplify_omp_for.  If *TP is not a gimple constant,
   16900              :    force it into a temporary initialized in PRE_P and add firstprivate clause
   16901              :    to ORIG_FOR_STMT.  */
   16902              : 
   16903              : static void
   16904         4399 : gimplify_omp_taskloop_expr (tree type, tree *tp, gimple_seq *pre_p,
   16905              :                             tree orig_for_stmt)
   16906              : {
   16907         4399 :   if (*tp == NULL || is_gimple_constant (*tp))
   16908              :     return;
   16909              : 
   16910          779 :   if (TREE_CODE (*tp) == SAVE_EXPR)
   16911          135 :     gimplify_save_expr (tp, pre_p, NULL);
   16912              :   else
   16913          644 :     *tp = get_initialized_tmp_var (*tp, pre_p, NULL, false);
   16914              :   /* Reference to pointer conversion is considered useless,
   16915              :      but is significant for firstprivate clause.  Force it
   16916              :      here.  */
   16917          779 :   if (type
   16918          687 :       && TREE_CODE (type) == POINTER_TYPE
   16919          869 :       && TREE_CODE (TREE_TYPE (*tp)) == REFERENCE_TYPE)
   16920              :     {
   16921            2 :       tree v = create_tmp_var (TYPE_MAIN_VARIANT (type));
   16922            2 :       tree m = build2 (INIT_EXPR, TREE_TYPE (v), v, *tp);
   16923            2 :       gimplify_and_add (m, pre_p);
   16924            2 :       *tp = v;
   16925              :     }
   16926              : 
   16927          779 :   tree c = build_omp_clause (input_location, OMP_CLAUSE_FIRSTPRIVATE);
   16928          779 :   OMP_CLAUSE_DECL (c) = *tp;
   16929          779 :   OMP_CLAUSE_CHAIN (c) = OMP_FOR_CLAUSES (orig_for_stmt);
   16930          779 :   OMP_FOR_CLAUSES (orig_for_stmt) = c;
   16931              : }
   16932              : 
   16933              : /* Helper function of gimplify_omp_for, find OMP_ORDERED with
   16934              :    null OMP_ORDERED_BODY inside of OMP_FOR's body.  */
   16935              : 
   16936              : static tree
   16937        16496 : find_standalone_omp_ordered (tree *tp, int *walk_subtrees, void *)
   16938              : {
   16939        16496 :   switch (TREE_CODE (*tp))
   16940              :     {
   16941          901 :     case OMP_ORDERED:
   16942          901 :       if (OMP_ORDERED_BODY (*tp) == NULL_TREE)
   16943          531 :         return *tp;
   16944              :       break;
   16945           23 :     case OMP_SIMD:
   16946           23 :     case OMP_PARALLEL:
   16947           23 :     case OMP_TARGET:
   16948           23 :       *walk_subtrees = 0;
   16949           23 :       break;
   16950              :     default:
   16951              :       break;
   16952              :     }
   16953              :   return NULL_TREE;
   16954              : }
   16955              : 
   16956              : /* Gimplify standalone loop transforming directive which has the
   16957              :    transformations applied already.  So, all that is needed is gimplify
   16958              :    the remaining loops as normal loops.  */
   16959              : 
   16960              : static enum gimplify_status
   16961         2266 : gimplify_omp_loop_xform (tree *expr_p, gimple_seq *pre_p)
   16962              : {
   16963         2266 :   tree for_stmt = *expr_p;
   16964              : 
   16965         2266 :   if (OMP_FOR_PRE_BODY (for_stmt))
   16966          759 :     gimplify_and_add (OMP_FOR_PRE_BODY (for_stmt), pre_p);
   16967              : 
   16968         2266 :   gimple_seq pre_body = NULL, post_body = NULL;
   16969         5207 :   for (int i = 0; i < TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt)); i++)
   16970              :     {
   16971         2941 :       if (TREE_VEC_ELT (OMP_FOR_INIT (for_stmt), i) == NULL_TREE)
   16972         1707 :         continue;
   16973         1285 :       tree iters = NULL_TREE;
   16974         1285 :       if (i == 0
   16975          828 :           && TREE_CODE (for_stmt) == OMP_UNROLL
   16976         1775 :           && !omp_find_clause (OMP_FOR_CLAUSES (for_stmt), OMP_CLAUSE_PARTIAL))
   16977              :         {
   16978          313 :           if (omp_find_clause (OMP_FOR_CLAUSES (for_stmt), OMP_CLAUSE_FULL))
   16979          270 :             iters = omp_loop_number_of_iterations (for_stmt, 0, NULL);
   16980              :           else
   16981           43 :             iters = build_int_cst (integer_type_node, 8);
   16982              :         }
   16983         1285 :       tree t = TREE_VEC_ELT (OMP_FOR_INIT (for_stmt), i);
   16984         1285 :       gcc_assert (TREE_CODE (t) == MODIFY_EXPR);
   16985         1285 :       tree decl = TREE_OPERAND (t, 0);
   16986         1285 :       gcc_assert (DECL_P (decl));
   16987         1285 :       gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (decl))
   16988              :                   || POINTER_TYPE_P (TREE_TYPE (decl)));
   16989         1285 :       if (DECL_ARTIFICIAL (decl)
   16990          876 :           && TREE_PRIVATE (t)
   16991          867 :           && gimplify_omp_ctxp
   16992         1615 :           && gimplify_omp_ctxp->region_type != ORT_NONE)
   16993              :         {
   16994              :           struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
   16995          504 :           do
   16996              :             {
   16997          504 :               splay_tree_node n
   16998          504 :                 = splay_tree_lookup (ctx->variables, (splay_tree_key) decl);
   16999          504 :               if (n != NULL)
   17000              :                 break;
   17001          346 :               else if (ctx->region_type != ORT_WORKSHARE
   17002              :                        && ctx->region_type != ORT_TASKGROUP
   17003           62 :                        && ctx->region_type != ORT_SIMD
   17004           38 :                        && ctx->region_type != ORT_ACC
   17005           38 :                        && !(ctx->region_type & ORT_TARGET_DATA))
   17006              :                 {
   17007           38 :                   omp_add_variable (ctx, decl, GOVD_PRIVATE);
   17008           38 :                   break;
   17009              :                 }
   17010          308 :               ctx = ctx->outer_context;
   17011              :             }
   17012          308 :           while (ctx);
   17013              :         }
   17014         1285 :       if (TREE_CODE (TREE_OPERAND (t, 1)) == TREE_VEC)
   17015              :         {
   17016           23 :           gcc_assert (seen_error ());
   17017           23 :           continue;
   17018              :         }
   17019         1262 :       gimplify_expr (&TREE_OPERAND (t, 1), pre_p, NULL, is_gimple_val,
   17020              :                      fb_rvalue);
   17021         1262 :       gimplify_and_add (t, &pre_body);
   17022         1262 :       t = TREE_VEC_ELT (OMP_FOR_COND (for_stmt), i);
   17023         1262 :       gcc_assert (TREE_OPERAND (t, 0) == decl);
   17024         1262 :       if (TREE_CODE (TREE_OPERAND (t, 1)) == TREE_VEC)
   17025              :         {
   17026           28 :           gcc_assert (seen_error ());
   17027           28 :           continue;
   17028              :         }
   17029         1234 :       gimplify_expr (&TREE_OPERAND (t, 1), pre_p, NULL, is_gimple_val,
   17030              :                      fb_rvalue);
   17031         1234 :       tree l1 = create_artificial_label (UNKNOWN_LOCATION);
   17032         1234 :       tree l2 = create_artificial_label (UNKNOWN_LOCATION);
   17033         1234 :       tree l3 = create_artificial_label (UNKNOWN_LOCATION);
   17034         1234 :       gimplify_seq_add_stmt (&pre_body, gimple_build_goto (l2));
   17035         1234 :       gimplify_seq_add_stmt (&pre_body, gimple_build_label (l1));
   17036         1234 :       gimple_seq this_post_body = NULL;
   17037         1234 :       t = TREE_VEC_ELT (OMP_FOR_INCR (for_stmt), i);
   17038         1234 :       if (TREE_CODE (t) == MODIFY_EXPR)
   17039              :         {
   17040          977 :           t = TREE_OPERAND (t, 1);
   17041          977 :           if (TREE_CODE (t) == PLUS_EXPR
   17042          977 :               && TREE_OPERAND (t, 1) == decl)
   17043              :             {
   17044            0 :               TREE_OPERAND (t, 1) = TREE_OPERAND (t, 0);
   17045            0 :               TREE_OPERAND (t, 0) = decl;
   17046              :             }
   17047          977 :           gimplify_expr (&TREE_OPERAND (t, 1), pre_p, NULL, is_gimple_val,
   17048              :                          fb_rvalue);
   17049              :         }
   17050         1234 :       gimplify_and_add (TREE_VEC_ELT (OMP_FOR_INCR (for_stmt), i),
   17051              :                         &this_post_body);
   17052         1234 :       gimplify_seq_add_stmt (&this_post_body, gimple_build_label (l2));
   17053         1234 :       t = TREE_VEC_ELT (OMP_FOR_COND (for_stmt), i);
   17054         1234 :       gcond *cond = NULL;
   17055         1234 :       tree d = decl;
   17056         1234 :       gimplify_expr (&d, &this_post_body, NULL, is_gimple_val, fb_rvalue);
   17057         1234 :       if (iters && tree_fits_uhwi_p (iters))
   17058              :         {
   17059          301 :           unsigned HOST_WIDE_INT niters = tree_to_uhwi (iters);
   17060          301 :           if ((unsigned HOST_WIDE_INT) (int) niters == niters
   17061          301 :               && (int) niters > 0)
   17062              :             {
   17063          301 :               t = build2 (TREE_CODE (t), boolean_type_node, d,
   17064          301 :                           TREE_OPERAND (t, 1));
   17065          301 :               t = build3 (ANNOTATE_EXPR, TREE_TYPE (t), t,
   17066              :                           build_int_cst (integer_type_node,
   17067              :                                          annot_expr_unroll_kind),
   17068          301 :                           build_int_cst (integer_type_node, niters));
   17069          301 :               gimplify_expr (&t, &this_post_body, NULL, is_gimple_val,
   17070              :                              fb_rvalue);
   17071          301 :               cond = gimple_build_cond (NE_EXPR, t, boolean_false_node,
   17072              :                                         l1, l3);
   17073              :             }
   17074              :         }
   17075          301 :       if (cond == NULL)
   17076          933 :         cond = gimple_build_cond (TREE_CODE (t), d, TREE_OPERAND (t, 1),
   17077              :                                   l1, l3);
   17078         1234 :       gimplify_seq_add_stmt (&this_post_body, cond);
   17079         1234 :       gimplify_seq_add_stmt (&this_post_body, gimple_build_label (l3));
   17080         1234 :       gimplify_seq_add_seq (&this_post_body, post_body);
   17081         1234 :       post_body = this_post_body;
   17082              :     }
   17083         2266 :   gimplify_seq_add_seq (pre_p, pre_body);
   17084         2266 :   gimplify_and_add (OMP_FOR_BODY (for_stmt), pre_p);
   17085         2266 :   gimplify_seq_add_seq (pre_p, post_body);
   17086              : 
   17087         2266 :   *expr_p = NULL_TREE;
   17088         2266 :   return GS_ALL_DONE;
   17089              : }
   17090              : 
   17091              : /* Gimplify the gross structure of an OMP_FOR statement.  */
   17092              : 
   17093              : static enum gimplify_status
   17094        58427 : gimplify_omp_for (tree *expr_p, gimple_seq *pre_p)
   17095              : {
   17096        58427 :   tree for_stmt, orig_for_stmt, inner_for_stmt = NULL_TREE, decl, var, t;
   17097        58427 :   enum gimplify_status ret = GS_ALL_DONE;
   17098        58427 :   enum gimplify_status tret;
   17099        58427 :   gomp_for *gfor;
   17100        58427 :   gimple_seq for_body, for_pre_body;
   17101        58427 :   int i;
   17102        58427 :   bitmap has_decl_expr = NULL;
   17103        58427 :   enum omp_region_type ort = ORT_WORKSHARE;
   17104        58427 :   bool openacc = TREE_CODE (*expr_p) == OACC_LOOP;
   17105              : 
   17106        58427 :   orig_for_stmt = for_stmt = *expr_p;
   17107              : 
   17108        58427 :   bool loop_p = (omp_find_clause (OMP_FOR_CLAUSES (for_stmt), OMP_CLAUSE_BIND)
   17109        58427 :                  != NULL_TREE);
   17110        58498 :   while (OMP_FOR_INIT (for_stmt) == NULL_TREE)
   17111              :     {
   17112        17821 :       tree *data[4] = { NULL, NULL, NULL, NULL };
   17113        17821 :       gcc_assert (TREE_CODE (for_stmt) != OACC_LOOP);
   17114        17821 :       inner_for_stmt = walk_tree (&OMP_FOR_BODY (for_stmt),
   17115              :                                   find_combined_omp_for, data, NULL);
   17116        17821 :       if (inner_for_stmt == NULL_TREE)
   17117              :         {
   17118           36 :           gcc_assert (seen_error ());
   17119           36 :           *expr_p = NULL_TREE;
   17120         3748 :           return GS_ERROR;
   17121              :         }
   17122        17785 :       gcc_assert (inner_for_stmt == *data[3]);
   17123        17785 :       omp_maybe_apply_loop_xforms (data[3],
   17124        17785 :                                    data[2]
   17125         4289 :                                    ? OMP_FOR_CLAUSES (*data[2])
   17126        13496 :                                    : TREE_CODE (for_stmt) == OMP_FOR
   17127        13496 :                                    ? OMP_FOR_CLAUSES (for_stmt)
   17128              :                                    : NULL_TREE);
   17129        17785 :       if (inner_for_stmt != *data[3])
   17130           71 :         continue;
   17131        17714 :       if (data[2] && OMP_FOR_PRE_BODY (*data[2]))
   17132              :         {
   17133            2 :           append_to_statement_list_force (OMP_FOR_PRE_BODY (*data[2]),
   17134              :                                           &OMP_FOR_PRE_BODY (for_stmt));
   17135            2 :           OMP_FOR_PRE_BODY (*data[2]) = NULL_TREE;
   17136              :         }
   17137        17714 :       if (OMP_FOR_PRE_BODY (inner_for_stmt))
   17138              :         {
   17139         7231 :           append_to_statement_list_force (OMP_FOR_PRE_BODY (inner_for_stmt),
   17140              :                                           &OMP_FOR_PRE_BODY (for_stmt));
   17141         7231 :           OMP_FOR_PRE_BODY (inner_for_stmt) = NULL_TREE;
   17142              :         }
   17143              : 
   17144        17714 :       if (data[0])
   17145              :         {
   17146              :           /* We have some statements or variable declarations in between
   17147              :              the composite construct directives.  Move them around the
   17148              :              inner_for_stmt.  */
   17149         3712 :           data[0] = expr_p;
   17150        14848 :           for (i = 0; i < 3; i++)
   17151        11136 :             if (data[i])
   17152              :               {
   17153         7034 :                 tree t = *data[i];
   17154         7034 :                 if (i < 2 && data[i + 1] == &OMP_BODY (t))
   17155         1005 :                   data[i + 1] = data[i];
   17156         7034 :                 *data[i] = OMP_BODY (t);
   17157         7034 :                 tree body = build3 (BIND_EXPR, void_type_node, NULL_TREE,
   17158              :                                     NULL_TREE, make_node (BLOCK));
   17159         7034 :                 OMP_BODY (t) = body;
   17160         7034 :                 append_to_statement_list_force (inner_for_stmt,
   17161              :                                                 &BIND_EXPR_BODY (body));
   17162         7034 :                 *data[3] = t;
   17163         7034 :                 data[3] = tsi_stmt_ptr (tsi_start (BIND_EXPR_BODY (body)));
   17164         7034 :                 gcc_assert (*data[3] == inner_for_stmt);
   17165              :               }
   17166              :           return GS_OK;
   17167              :         }
   17168              : 
   17169        36886 :       for (i = 0; i < TREE_VEC_LENGTH (OMP_FOR_INIT (inner_for_stmt)); i++)
   17170        22884 :         if (!loop_p
   17171        21994 :             && OMP_FOR_ORIG_DECLS (inner_for_stmt)
   17172        11134 :             && TREE_CODE (TREE_VEC_ELT (OMP_FOR_ORIG_DECLS (inner_for_stmt),
   17173              :                                         i)) == TREE_LIST
   17174        22952 :             && TREE_PURPOSE (TREE_VEC_ELT (OMP_FOR_ORIG_DECLS (inner_for_stmt),
   17175              :                                            i)))
   17176              :           {
   17177           40 :             tree orig = TREE_VEC_ELT (OMP_FOR_ORIG_DECLS (inner_for_stmt), i);
   17178              :             /* Class iterators aren't allowed on OMP_SIMD, so the only
   17179              :                case we need to solve is distribute parallel for.  They are
   17180              :                allowed on the loop construct, but that is already handled
   17181              :                in gimplify_omp_loop.  */
   17182           40 :             gcc_assert (TREE_CODE (inner_for_stmt) == OMP_FOR
   17183              :                         && TREE_CODE (for_stmt) == OMP_DISTRIBUTE
   17184              :                         && data[1]);
   17185           40 :             tree orig_decl = TREE_PURPOSE (orig);
   17186           40 :             tree last = TREE_VALUE (orig);
   17187           40 :             tree *pc;
   17188           40 :             for (pc = &OMP_FOR_CLAUSES (inner_for_stmt);
   17189           74 :                  *pc; pc = &OMP_CLAUSE_CHAIN (*pc))
   17190           42 :               if ((OMP_CLAUSE_CODE (*pc) == OMP_CLAUSE_PRIVATE
   17191           35 :                    || OMP_CLAUSE_CODE (*pc) == OMP_CLAUSE_LASTPRIVATE)
   17192           43 :                   && OMP_CLAUSE_DECL (*pc) == orig_decl)
   17193              :                 break;
   17194           40 :             if (*pc == NULL_TREE)
   17195              :               {
   17196           32 :                 tree *spc;
   17197           32 :                 for (spc = &OMP_PARALLEL_CLAUSES (*data[1]);
   17198           98 :                      *spc; spc = &OMP_CLAUSE_CHAIN (*spc))
   17199           67 :                   if (OMP_CLAUSE_CODE (*spc) == OMP_CLAUSE_PRIVATE
   17200           67 :                       && OMP_CLAUSE_DECL (*spc) == orig_decl)
   17201              :                     break;
   17202           32 :                 if (*spc)
   17203              :                   {
   17204            1 :                     tree c = *spc;
   17205            1 :                     *spc = OMP_CLAUSE_CHAIN (c);
   17206            1 :                     OMP_CLAUSE_CHAIN (c) = NULL_TREE;
   17207            1 :                     *pc = c;
   17208              :                   }
   17209              :               }
   17210           40 :             if (*pc == NULL_TREE)
   17211              :               ;
   17212            9 :             else if (OMP_CLAUSE_CODE (*pc) == OMP_CLAUSE_PRIVATE)
   17213              :               {
   17214              :                 /* private clause will appear only on inner_for_stmt.
   17215              :                    Change it into firstprivate, and add private clause
   17216              :                    on for_stmt.  */
   17217            8 :                 tree c = copy_node (*pc);
   17218            8 :                 OMP_CLAUSE_CHAIN (c) = OMP_FOR_CLAUSES (for_stmt);
   17219            8 :                 OMP_FOR_CLAUSES (for_stmt) = c;
   17220            8 :                 OMP_CLAUSE_CODE (*pc) = OMP_CLAUSE_FIRSTPRIVATE;
   17221            8 :                 lang_hooks.decls.omp_finish_clause (*pc, pre_p, openacc);
   17222              :               }
   17223              :             else
   17224              :               {
   17225              :                 /* lastprivate clause will appear on both inner_for_stmt
   17226              :                    and for_stmt.  Add firstprivate clause to
   17227              :                    inner_for_stmt.  */
   17228            1 :                 tree c = build_omp_clause (OMP_CLAUSE_LOCATION (*pc),
   17229              :                                            OMP_CLAUSE_FIRSTPRIVATE);
   17230            1 :                 OMP_CLAUSE_DECL (c) = OMP_CLAUSE_DECL (*pc);
   17231            1 :                 OMP_CLAUSE_CHAIN (c) = *pc;
   17232            1 :                 *pc = c;
   17233            1 :                 lang_hooks.decls.omp_finish_clause (*pc, pre_p, openacc);
   17234              :               }
   17235           40 :             tree c = build_omp_clause (UNKNOWN_LOCATION,
   17236              :                                        OMP_CLAUSE_FIRSTPRIVATE);
   17237           40 :             OMP_CLAUSE_DECL (c) = last;
   17238           40 :             OMP_CLAUSE_CHAIN (c) = OMP_PARALLEL_CLAUSES (*data[1]);
   17239           40 :             OMP_PARALLEL_CLAUSES (*data[1]) = c;
   17240           40 :             c = build_omp_clause (UNKNOWN_LOCATION,
   17241           40 :                                   *pc ? OMP_CLAUSE_SHARED
   17242              :                                       : OMP_CLAUSE_FIRSTPRIVATE);
   17243           40 :             OMP_CLAUSE_DECL (c) = orig_decl;
   17244           40 :             OMP_CLAUSE_CHAIN (c) = OMP_PARALLEL_CLAUSES (*data[1]);
   17245           40 :             OMP_PARALLEL_CLAUSES (*data[1]) = c;
   17246              :           }
   17247              :       /* Similarly, take care of C++ range for temporaries, those should
   17248              :          be firstprivate on OMP_PARALLEL if any.  */
   17249        14002 :       if (data[1])
   17250        17601 :         for (i = 0; i < TREE_VEC_LENGTH (OMP_FOR_INIT (inner_for_stmt)); i++)
   17251        11157 :           if (OMP_FOR_ORIG_DECLS (inner_for_stmt)
   17252         5763 :               && TREE_CODE (TREE_VEC_ELT (OMP_FOR_ORIG_DECLS (inner_for_stmt),
   17253              :                                           i)) == TREE_LIST
   17254        11233 :               && TREE_CHAIN (TREE_VEC_ELT (OMP_FOR_ORIG_DECLS (inner_for_stmt),
   17255              :                                            i)))
   17256              :             {
   17257           49 :               tree orig
   17258           49 :                 = TREE_VEC_ELT (OMP_FOR_ORIG_DECLS (inner_for_stmt), i);
   17259           49 :               tree v = TREE_CHAIN (orig);
   17260           49 :               tree c = build_omp_clause (UNKNOWN_LOCATION,
   17261              :                                          OMP_CLAUSE_FIRSTPRIVATE);
   17262              :               /* First add firstprivate clause for the __for_end artificial
   17263              :                  decl.  */
   17264           49 :               OMP_CLAUSE_DECL (c) = TREE_VEC_ELT (v, 1);
   17265           49 :               if (TREE_CODE (TREE_TYPE (OMP_CLAUSE_DECL (c)))
   17266              :                   == REFERENCE_TYPE)
   17267            0 :                 OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c) = 1;
   17268           49 :               OMP_CLAUSE_CHAIN (c) = OMP_PARALLEL_CLAUSES (*data[1]);
   17269           49 :               OMP_PARALLEL_CLAUSES (*data[1]) = c;
   17270           49 :               if (TREE_VEC_ELT (v, 0))
   17271              :                 {
   17272              :                   /* And now the same for __for_range artificial decl if it
   17273              :                      exists.  */
   17274           49 :                   c = build_omp_clause (UNKNOWN_LOCATION,
   17275              :                                         OMP_CLAUSE_FIRSTPRIVATE);
   17276           49 :                   OMP_CLAUSE_DECL (c) = TREE_VEC_ELT (v, 0);
   17277           49 :                   if (TREE_CODE (TREE_TYPE (OMP_CLAUSE_DECL (c)))
   17278              :                       == REFERENCE_TYPE)
   17279           49 :                     OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c) = 1;
   17280           49 :                   OMP_CLAUSE_CHAIN (c) = OMP_PARALLEL_CLAUSES (*data[1]);
   17281           49 :                   OMP_PARALLEL_CLAUSES (*data[1]) = c;
   17282              :                 }
   17283              :             }
   17284        14002 :       break;
   17285              :     }
   17286        54679 :   if (OMP_FOR_INIT (for_stmt) != NULL_TREE)
   17287              :     {
   17288        40677 :       omp_maybe_apply_loop_xforms (expr_p, NULL_TREE);
   17289        40677 :       if (*expr_p != for_stmt)
   17290              :         return GS_OK;
   17291              :     }
   17292              : 
   17293        53797 :   switch (TREE_CODE (for_stmt))
   17294              :     {
   17295        18312 :     case OMP_FOR:
   17296        30632 :       if (OMP_FOR_NON_RECTANGULAR (inner_for_stmt ? inner_for_stmt : for_stmt))
   17297              :         {
   17298          458 :           if (omp_find_clause (OMP_FOR_CLAUSES (for_stmt),
   17299              :                                OMP_CLAUSE_SCHEDULE))
   17300           47 :             error_at (EXPR_LOCATION (for_stmt),
   17301              :                       "%qs clause may not appear on non-rectangular %qs",
   17302           27 :                       "schedule", lang_GNU_Fortran () ? "do" : "for");
   17303          458 :           if (omp_find_clause (OMP_FOR_CLAUSES (for_stmt), OMP_CLAUSE_ORDERED))
   17304           34 :             error_at (EXPR_LOCATION (for_stmt),
   17305              :                       "%qs clause may not appear on non-rectangular %qs",
   17306           18 :                       "ordered", lang_GNU_Fortran () ? "do" : "for");
   17307              :         }
   17308              :       break;
   17309         8308 :     case OMP_DISTRIBUTE:
   17310         9255 :       if (OMP_FOR_NON_RECTANGULAR (inner_for_stmt ? inner_for_stmt : for_stmt)
   17311         8308 :           && omp_find_clause (OMP_FOR_CLAUSES (for_stmt),
   17312              :                               OMP_CLAUSE_DIST_SCHEDULE))
   17313           29 :         error_at (EXPR_LOCATION (for_stmt),
   17314              :                   "%qs clause may not appear on non-rectangular %qs",
   17315              :                   "dist_schedule", "distribute");
   17316              :       break;
   17317              :     case OACC_LOOP:
   17318              :       ort = ORT_ACC;
   17319              :       break;
   17320         1586 :     case OMP_TASKLOOP:
   17321         2523 :       if (OMP_FOR_NON_RECTANGULAR (inner_for_stmt ? inner_for_stmt : for_stmt))
   17322              :         {
   17323           36 :           if (omp_find_clause (OMP_FOR_CLAUSES (for_stmt),
   17324              :                                OMP_CLAUSE_GRAINSIZE))
   17325           11 :             error_at (EXPR_LOCATION (for_stmt),
   17326              :                       "%qs clause may not appear on non-rectangular %qs",
   17327              :                       "grainsize", "taskloop");
   17328           36 :           if (omp_find_clause (OMP_FOR_CLAUSES (for_stmt),
   17329              :                                OMP_CLAUSE_NUM_TASKS))
   17330            6 :             error_at (EXPR_LOCATION (for_stmt),
   17331              :                       "%qs clause may not appear on non-rectangular %qs",
   17332              :                       "num_tasks", "taskloop");
   17333              :         }
   17334         1586 :       if (omp_find_clause (OMP_FOR_CLAUSES (for_stmt), OMP_CLAUSE_UNTIED))
   17335              :         ort = ORT_UNTIED_TASKLOOP;
   17336              :       else
   17337              :         ort = ORT_TASKLOOP;
   17338              :       break;
   17339        10773 :     case OMP_SIMD:
   17340        10773 :       ort = ORT_SIMD;
   17341        10773 :       break;
   17342         2266 :     case OMP_TILE:
   17343         2266 :     case OMP_UNROLL:
   17344         2266 :       gcc_assert (inner_for_stmt == NULL_TREE);
   17345         2266 :       return gimplify_omp_loop_xform (expr_p, pre_p);
   17346            0 :     default:
   17347            0 :       gcc_unreachable ();
   17348              :     }
   17349              : 
   17350              :   /* Set OMP_CLAUSE_LINEAR_NO_COPYIN flag on explicit linear
   17351              :      clause for the IV.  */
   17352        10820 :   if (ort == ORT_SIMD && TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt)) == 1)
   17353              :     {
   17354         7945 :       t = TREE_VEC_ELT (OMP_FOR_INIT (for_stmt), 0);
   17355         7945 :       gcc_assert (TREE_CODE (t) == MODIFY_EXPR);
   17356         7945 :       decl = TREE_OPERAND (t, 0);
   17357        18805 :       for (tree c = OMP_FOR_CLAUSES (for_stmt); c; c = OMP_CLAUSE_CHAIN (c))
   17358        12004 :         if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
   17359        12004 :             && OMP_CLAUSE_DECL (c) == decl)
   17360              :           {
   17361         1144 :             OMP_CLAUSE_LINEAR_NO_COPYIN (c) = 1;
   17362         1144 :             break;
   17363              :           }
   17364              :     }
   17365              : 
   17366        51531 :   if (TREE_CODE (for_stmt) != OMP_TASKLOOP)
   17367        51870 :     gimplify_scan_omp_clauses (&OMP_FOR_CLAUSES (for_stmt), pre_p, ort,
   17368         1925 :                                loop_p && TREE_CODE (for_stmt) != OMP_SIMD
   17369              :                                ? OMP_LOOP : TREE_CODE (for_stmt));
   17370              : 
   17371        51531 :   if (TREE_CODE (for_stmt) == OMP_DISTRIBUTE)
   17372         8308 :     gimplify_omp_ctxp->distribute = true;
   17373              : 
   17374              :   /* Handle OMP_FOR_INIT.  */
   17375        51531 :   for_pre_body = NULL;
   17376        51531 :   if ((ort == ORT_SIMD
   17377        40758 :        || (inner_for_stmt && TREE_CODE (inner_for_stmt) == OMP_SIMD))
   17378        62308 :       && OMP_FOR_PRE_BODY (for_stmt))
   17379              :     {
   17380         6869 :       has_decl_expr = BITMAP_ALLOC (NULL);
   17381         6869 :       if (TREE_CODE (OMP_FOR_PRE_BODY (for_stmt)) == DECL_EXPR
   17382         6869 :           && VAR_P (DECL_EXPR_DECL (OMP_FOR_PRE_BODY (for_stmt))))
   17383              :         {
   17384           18 :           t = OMP_FOR_PRE_BODY (for_stmt);
   17385           18 :           bitmap_set_bit (has_decl_expr, DECL_UID (DECL_EXPR_DECL (t)));
   17386              :         }
   17387         6851 :       else if (TREE_CODE (OMP_FOR_PRE_BODY (for_stmt)) == STATEMENT_LIST)
   17388              :         {
   17389         6849 :           tree_stmt_iterator si;
   17390        11405 :           for (si = tsi_start (OMP_FOR_PRE_BODY (for_stmt)); !tsi_end_p (si);
   17391         4556 :                tsi_next (&si))
   17392              :             {
   17393         4556 :               t = tsi_stmt (si);
   17394         4556 :               if (TREE_CODE (t) == DECL_EXPR
   17395         4556 :                   && VAR_P (DECL_EXPR_DECL (t)))
   17396         4482 :                 bitmap_set_bit (has_decl_expr, DECL_UID (DECL_EXPR_DECL (t)));
   17397              :             }
   17398              :         }
   17399              :     }
   17400        51531 :   if (OMP_FOR_PRE_BODY (for_stmt))
   17401              :     {
   17402        21162 :       if (TREE_CODE (for_stmt) != OMP_TASKLOOP || gimplify_omp_ctxp)
   17403        20437 :         gimplify_and_add (OMP_FOR_PRE_BODY (for_stmt), &for_pre_body);
   17404              :       else
   17405              :         {
   17406          725 :           struct gimplify_omp_ctx ctx;
   17407          725 :           memset (&ctx, 0, sizeof (ctx));
   17408          725 :           ctx.region_type = ORT_NONE;
   17409          725 :           gimplify_omp_ctxp = &ctx;
   17410          725 :           gimplify_and_add (OMP_FOR_PRE_BODY (for_stmt), &for_pre_body);
   17411          725 :           gimplify_omp_ctxp = NULL;
   17412              :         }
   17413              :     }
   17414        51531 :   OMP_FOR_PRE_BODY (for_stmt) = NULL_TREE;
   17415              : 
   17416        51531 :   if (OMP_FOR_INIT (for_stmt) == NULL_TREE)
   17417        14002 :     for_stmt = inner_for_stmt;
   17418              : 
   17419              :   /* For taskloop, need to gimplify the start, end and step before the
   17420              :      taskloop, outside of the taskloop omp context.  */
   17421        51531 :   if (TREE_CODE (orig_for_stmt) == OMP_TASKLOOP)
   17422              :     {
   17423         3458 :       for (i = 0; i < TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt)); i++)
   17424              :         {
   17425         1872 :           t = TREE_VEC_ELT (OMP_FOR_INIT (for_stmt), i);
   17426         1872 :           gimple_seq *for_pre_p = (gimple_seq_empty_p (for_pre_body)
   17427         1872 :                                    ? pre_p : &for_pre_body);
   17428         1872 :               tree type = TREE_TYPE (TREE_OPERAND (t, 0));
   17429         1872 :           if (TREE_CODE (TREE_OPERAND (t, 1)) == TREE_VEC)
   17430              :             {
   17431           34 :               tree v = TREE_OPERAND (t, 1);
   17432           34 :               gimplify_omp_taskloop_expr (type, &TREE_VEC_ELT (v, 1),
   17433              :                                           for_pre_p, orig_for_stmt);
   17434           34 :               gimplify_omp_taskloop_expr (type, &TREE_VEC_ELT (v, 2),
   17435              :                                           for_pre_p, orig_for_stmt);
   17436              :             }
   17437              :           else
   17438         1838 :             gimplify_omp_taskloop_expr (type, &TREE_OPERAND (t, 1), for_pre_p,
   17439              :                                         orig_for_stmt);
   17440              : 
   17441              :           /* Handle OMP_FOR_COND.  */
   17442         1872 :           t = TREE_VEC_ELT (OMP_FOR_COND (for_stmt), i);
   17443         1872 :           if (TREE_CODE (TREE_OPERAND (t, 1)) == TREE_VEC)
   17444              :             {
   17445           31 :               tree v = TREE_OPERAND (t, 1);
   17446           31 :               gimplify_omp_taskloop_expr (type, &TREE_VEC_ELT (v, 1),
   17447              :                                           for_pre_p, orig_for_stmt);
   17448           31 :               gimplify_omp_taskloop_expr (type, &TREE_VEC_ELT (v, 2),
   17449              :                                           for_pre_p, orig_for_stmt);
   17450              :             }
   17451              :           else
   17452         1841 :             gimplify_omp_taskloop_expr (type, &TREE_OPERAND (t, 1), for_pre_p,
   17453              :                                         orig_for_stmt);
   17454              : 
   17455              :           /* Handle OMP_FOR_INCR.  */
   17456         1872 :           t = TREE_VEC_ELT (OMP_FOR_INCR (for_stmt), i);
   17457         1872 :           if (TREE_CODE (t) == MODIFY_EXPR)
   17458              :             {
   17459          590 :               decl = TREE_OPERAND (t, 0);
   17460          590 :               t = TREE_OPERAND (t, 1);
   17461          590 :               tree *tp = &TREE_OPERAND (t, 1);
   17462          590 :               if (TREE_CODE (t) == PLUS_EXPR && *tp == decl)
   17463           22 :                 tp = &TREE_OPERAND (t, 0);
   17464              : 
   17465          590 :               gimplify_omp_taskloop_expr (NULL_TREE, tp, for_pre_p,
   17466              :                                           orig_for_stmt);
   17467              :             }
   17468              :         }
   17469              : 
   17470         1586 :       gimplify_scan_omp_clauses (&OMP_FOR_CLAUSES (orig_for_stmt), pre_p, ort,
   17471              :                                  OMP_TASKLOOP);
   17472              :     }
   17473              : 
   17474        51531 :   if (orig_for_stmt != for_stmt)
   17475        14002 :     gimplify_omp_ctxp->combined_loop = true;
   17476              : 
   17477        51531 :   for_body = NULL;
   17478        51531 :   gcc_assert (TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt))
   17479              :               == TREE_VEC_LENGTH (OMP_FOR_COND (for_stmt)));
   17480        51531 :   gcc_assert (TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt))
   17481              :               == TREE_VEC_LENGTH (OMP_FOR_INCR (for_stmt)));
   17482              : 
   17483        51531 :   tree c = omp_find_clause (OMP_FOR_CLAUSES (for_stmt), OMP_CLAUSE_ORDERED);
   17484        51531 :   bool is_doacross = false;
   17485        51531 :   if (c && walk_tree_without_duplicates (&OMP_FOR_BODY (for_stmt),
   17486              :                                          find_standalone_omp_ordered, NULL))
   17487              :     {
   17488          531 :       OMP_CLAUSE_ORDERED_DOACROSS (c) = 1;
   17489          531 :       is_doacross = true;
   17490          531 :       int len = TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt));
   17491          531 :       gimplify_omp_ctxp->loop_iter_var.create (len * 2);
   17492         2200 :       for (tree *pc = &OMP_FOR_CLAUSES (for_stmt); *pc; )
   17493         1669 :         if (OMP_CLAUSE_CODE (*pc) == OMP_CLAUSE_LINEAR)
   17494              :           {
   17495           10 :             error_at (OMP_CLAUSE_LOCATION (*pc),
   17496              :                       "%<linear%> clause may not be specified together "
   17497              :                       "with %<ordered%> clause if stand-alone %<ordered%> "
   17498              :                       "construct is nested in it");
   17499           10 :             *pc = OMP_CLAUSE_CHAIN (*pc);
   17500              :           }
   17501              :         else
   17502         1659 :           pc = &OMP_CLAUSE_CHAIN (*pc);
   17503              :     }
   17504        51531 :   int collapse = 1, tile = 0;
   17505        51531 :   c = omp_find_clause (OMP_FOR_CLAUSES (for_stmt), OMP_CLAUSE_COLLAPSE);
   17506        51531 :   if (c)
   17507        13727 :     collapse = tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (c));
   17508        51531 :   c = omp_find_clause (OMP_FOR_CLAUSES (for_stmt), OMP_CLAUSE_TILE);
   17509        51531 :   if (c)
   17510          384 :     tile = list_length (OMP_CLAUSE_TILE_LIST (c));
   17511        51531 :   c = omp_find_clause (OMP_FOR_CLAUSES (for_stmt), OMP_CLAUSE_ALLOCATE);
   17512        51531 :   hash_set<tree> *allocate_uids = NULL;
   17513        51531 :   if (c)
   17514              :     {
   17515          275 :       allocate_uids = new hash_set<tree>;
   17516         1682 :       for (; c; c = OMP_CLAUSE_CHAIN (c))
   17517         1132 :         if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_ALLOCATE)
   17518          478 :           allocate_uids->add (OMP_CLAUSE_DECL (c));
   17519              :     }
   17520       124808 :   for (i = 0; i < TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt)); i++)
   17521              :     {
   17522        73277 :       t = TREE_VEC_ELT (OMP_FOR_INIT (for_stmt), i);
   17523        73277 :       gcc_assert (TREE_CODE (t) == MODIFY_EXPR);
   17524        73277 :       decl = TREE_OPERAND (t, 0);
   17525        73277 :       gcc_assert (DECL_P (decl));
   17526        73277 :       gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (decl))
   17527              :                   || POINTER_TYPE_P (TREE_TYPE (decl)));
   17528        73277 :       if (is_doacross)
   17529              :         {
   17530         1654 :           if (TREE_CODE (for_stmt) == OMP_FOR && OMP_FOR_ORIG_DECLS (for_stmt))
   17531              :             {
   17532          960 :               tree orig_decl = TREE_VEC_ELT (OMP_FOR_ORIG_DECLS (for_stmt), i);
   17533          960 :               if (TREE_CODE (orig_decl) == TREE_LIST)
   17534              :                 {
   17535           15 :                   orig_decl = TREE_PURPOSE (orig_decl);
   17536           15 :                   if (!orig_decl)
   17537            0 :                     orig_decl = decl;
   17538              :                 }
   17539          960 :               gimplify_omp_ctxp->loop_iter_var.quick_push (orig_decl);
   17540              :             }
   17541              :           else
   17542          694 :             gimplify_omp_ctxp->loop_iter_var.quick_push (decl);
   17543         1654 :           gimplify_omp_ctxp->loop_iter_var.quick_push (decl);
   17544              :         }
   17545              : 
   17546        73277 :       if (for_stmt == orig_for_stmt)
   17547              :         {
   17548        50393 :           tree orig_decl = decl;
   17549        50393 :           if (OMP_FOR_ORIG_DECLS (for_stmt))
   17550              :             {
   17551        24058 :               tree orig_decl = TREE_VEC_ELT (OMP_FOR_ORIG_DECLS (for_stmt), i);
   17552        24058 :               if (TREE_CODE (orig_decl) == TREE_LIST)
   17553              :                 {
   17554              :                   orig_decl = TREE_PURPOSE (orig_decl);
   17555              :                   if (!orig_decl)
   17556              :                     orig_decl = decl;
   17557              :                 }
   17558              :             }
   17559        50393 :           if (is_global_var (orig_decl) && DECL_THREAD_LOCAL_P (orig_decl))
   17560           36 :             error_at (EXPR_LOCATION (for_stmt),
   17561              :                       "threadprivate iteration variable %qD", orig_decl);
   17562              :         }
   17563              : 
   17564              :       /* Make sure the iteration variable is private.  */
   17565        73277 :       tree c = NULL_TREE;
   17566        73277 :       tree c2 = NULL_TREE;
   17567        73277 :       if (orig_for_stmt != for_stmt)
   17568              :         {
   17569              :           /* Preserve this information until we gimplify the inner simd.  */
   17570        22884 :           if (has_decl_expr
   17571        22884 :               && bitmap_bit_p (has_decl_expr, DECL_UID (decl)))
   17572         3421 :             TREE_PRIVATE (t) = 1;
   17573              :         }
   17574        50393 :       else if (ort == ORT_SIMD)
   17575              :         {
   17576        15846 :           splay_tree_node n = splay_tree_lookup (gimplify_omp_ctxp->variables,
   17577              :                                                  (splay_tree_key) decl);
   17578        15846 :           omp_is_private (gimplify_omp_ctxp, decl,
   17579        15846 :                           1 + (TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt))
   17580              :                                != 1));
   17581        15846 :           if (n != NULL && (n->value & GOVD_DATA_SHARE_CLASS) != 0)
   17582              :             {
   17583         2548 :               omp_notice_variable (gimplify_omp_ctxp, decl, true);
   17584         2548 :               if (n->value & GOVD_LASTPRIVATE_CONDITIONAL)
   17585            0 :                 for (tree c3 = omp_find_clause (OMP_FOR_CLAUSES (for_stmt),
   17586              :                                                 OMP_CLAUSE_LASTPRIVATE);
   17587            0 :                      c3; c3 = omp_find_clause (OMP_CLAUSE_CHAIN (c3),
   17588              :                                                OMP_CLAUSE_LASTPRIVATE))
   17589            0 :                   if (OMP_CLAUSE_DECL (c3) == decl)
   17590              :                     {
   17591            0 :                       warning_at (OMP_CLAUSE_LOCATION (c3), OPT_Wopenmp,
   17592              :                                   "conditional %<lastprivate%> on loop "
   17593              :                                   "iterator %qD ignored", decl);
   17594            0 :                       OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c3) = 0;
   17595            0 :                       n->value &= ~GOVD_LASTPRIVATE_CONDITIONAL;
   17596              :                     }
   17597              :             }
   17598        13298 :           else if (TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt)) == 1 && !loop_p)
   17599              :             {
   17600         5721 :               c = build_omp_clause (input_location, OMP_CLAUSE_LINEAR);
   17601         5721 :               OMP_CLAUSE_LINEAR_NO_COPYIN (c) = 1;
   17602         5721 :               unsigned int flags = GOVD_LINEAR | GOVD_EXPLICIT | GOVD_SEEN;
   17603         5721 :               if ((has_decl_expr
   17604         1385 :                    && bitmap_bit_p (has_decl_expr, DECL_UID (decl)))
   17605         6405 :                   || TREE_PRIVATE (t))
   17606              :                 {
   17607         2365 :                   OMP_CLAUSE_LINEAR_NO_COPYOUT (c) = 1;
   17608         2365 :                   flags |= GOVD_LINEAR_LASTPRIVATE_NO_OUTER;
   17609              :                 }
   17610         5721 :               struct gimplify_omp_ctx *outer
   17611         5721 :                 = gimplify_omp_ctxp->outer_context;
   17612         5721 :               if (outer && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
   17613              :                 {
   17614         2433 :                   if (outer->region_type == ORT_WORKSHARE
   17615         2193 :                       && outer->combined_loop)
   17616              :                     {
   17617         2142 :                       n = splay_tree_lookup (outer->variables,
   17618              :                                              (splay_tree_key)decl);
   17619         2142 :                       if (n != NULL && (n->value & GOVD_LOCAL) != 0)
   17620              :                         {
   17621            0 :                           OMP_CLAUSE_LINEAR_NO_COPYOUT (c) = 1;
   17622            0 :                           flags |= GOVD_LINEAR_LASTPRIVATE_NO_OUTER;
   17623              :                         }
   17624              :                       else
   17625              :                         {
   17626         2142 :                           struct gimplify_omp_ctx *octx = outer->outer_context;
   17627         2142 :                           if (octx
   17628         1714 :                               && octx->region_type == ORT_COMBINED_PARALLEL
   17629         1476 :                               && octx->outer_context
   17630         1209 :                               && (octx->outer_context->region_type
   17631              :                                   == ORT_WORKSHARE)
   17632         1079 :                               && octx->outer_context->combined_loop)
   17633              :                             {
   17634         1079 :                               octx = octx->outer_context;
   17635         1079 :                               n = splay_tree_lookup (octx->variables,
   17636              :                                                      (splay_tree_key)decl);
   17637         1079 :                               if (n != NULL && (n->value & GOVD_LOCAL) != 0)
   17638              :                                 {
   17639            0 :                                   OMP_CLAUSE_LINEAR_NO_COPYOUT (c) = 1;
   17640            0 :                                   flags |= GOVD_LINEAR_LASTPRIVATE_NO_OUTER;
   17641              :                                 }
   17642              :                             }
   17643              :                         }
   17644              :                     }
   17645              :                 }
   17646              : 
   17647         5721 :               OMP_CLAUSE_DECL (c) = decl;
   17648         5721 :               OMP_CLAUSE_CHAIN (c) = OMP_FOR_CLAUSES (for_stmt);
   17649         5721 :               OMP_FOR_CLAUSES (for_stmt) = c;
   17650         5721 :               omp_add_variable (gimplify_omp_ctxp, decl, flags);
   17651         5721 :               if (outer && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
   17652         2433 :                 omp_lastprivate_for_combined_outer_constructs (outer, decl,
   17653              :                                                                true);
   17654              :             }
   17655              :           else
   17656              :             {
   17657         7577 :               bool lastprivate
   17658              :                 = (!has_decl_expr
   17659         7577 :                    || !bitmap_bit_p (has_decl_expr, DECL_UID (decl)));
   17660         7577 :               if (TREE_PRIVATE (t))
   17661         1787 :                 lastprivate = false;
   17662         7577 :               if (loop_p && OMP_FOR_ORIG_DECLS (for_stmt))
   17663              :                 {
   17664          510 :                   tree elt = TREE_VEC_ELT (OMP_FOR_ORIG_DECLS (for_stmt), i);
   17665          510 :                   if (TREE_CODE (elt) == TREE_LIST && TREE_PURPOSE (elt))
   17666              :                     lastprivate = false;
   17667              :                 }
   17668              : 
   17669         7577 :               struct gimplify_omp_ctx *outer
   17670         7577 :                 = gimplify_omp_ctxp->outer_context;
   17671         7577 :               if (outer && lastprivate)
   17672         4892 :                 omp_lastprivate_for_combined_outer_constructs (outer, decl,
   17673              :                                                                true);
   17674              : 
   17675         9584 :               c = build_omp_clause (input_location,
   17676              :                                     lastprivate ? OMP_CLAUSE_LASTPRIVATE
   17677              :                                                 : OMP_CLAUSE_PRIVATE);
   17678         7577 :               OMP_CLAUSE_DECL (c) = decl;
   17679         7577 :               OMP_CLAUSE_CHAIN (c) = OMP_FOR_CLAUSES (for_stmt);
   17680         7577 :               OMP_FOR_CLAUSES (for_stmt) = c;
   17681         7577 :               omp_add_variable (gimplify_omp_ctxp, decl,
   17682              :                                 (lastprivate ? GOVD_LASTPRIVATE : GOVD_PRIVATE)
   17683              :                                 | GOVD_EXPLICIT | GOVD_SEEN);
   17684         7577 :               c = NULL_TREE;
   17685              :             }
   17686              :         }
   17687        34547 :       else if (omp_is_private (gimplify_omp_ctxp, decl, 0))
   17688              :         {
   17689         8740 :           omp_notice_variable (gimplify_omp_ctxp, decl, true);
   17690         8740 :           splay_tree_node n = splay_tree_lookup (gimplify_omp_ctxp->variables,
   17691              :                                                  (splay_tree_key) decl);
   17692         8740 :           if (n && (n->value & GOVD_LASTPRIVATE_CONDITIONAL))
   17693           80 :             for (tree c3 = omp_find_clause (OMP_FOR_CLAUSES (for_stmt),
   17694              :                                             OMP_CLAUSE_LASTPRIVATE);
   17695           80 :                  c3; c3 = omp_find_clause (OMP_CLAUSE_CHAIN (c3),
   17696              :                                            OMP_CLAUSE_LASTPRIVATE))
   17697           40 :               if (OMP_CLAUSE_DECL (c3) == decl)
   17698              :                 {
   17699           40 :                   warning_at (OMP_CLAUSE_LOCATION (c3), OPT_Wopenmp,
   17700              :                               "conditional %<lastprivate%> on loop "
   17701              :                               "iterator %qD ignored", decl);
   17702           40 :                   OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c3) = 0;
   17703           40 :                   n->value &= ~GOVD_LASTPRIVATE_CONDITIONAL;
   17704              :                 }
   17705              :         }
   17706              :       else
   17707        25807 :         omp_add_variable (gimplify_omp_ctxp, decl, GOVD_PRIVATE | GOVD_SEEN);
   17708              : 
   17709              :       /* If DECL is not a gimple register, create a temporary variable to act
   17710              :          as an iteration counter.  This is valid, since DECL cannot be
   17711              :          modified in the body of the loop.  Similarly for any iteration vars
   17712              :          in simd with collapse > 1 where the iterator vars must be
   17713              :          lastprivate.  And similarly for vars mentioned in allocate clauses.  */
   17714        73277 :       if (orig_for_stmt != for_stmt)
   17715        22884 :         var = decl;
   17716        50393 :       else if (!is_gimple_reg (decl)
   17717        47125 :                || (ort == ORT_SIMD
   17718        14245 :                    && TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt)) > 1)
   17719        90448 :                || (allocate_uids && allocate_uids->contains (decl)))
   17720              :         {
   17721        10370 :           struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
   17722              :           /* Make sure omp_add_variable is not called on it prematurely.
   17723              :              We call it ourselves a few lines later.  */
   17724        10370 :           gimplify_omp_ctxp = NULL;
   17725        10370 :           var = create_tmp_var (TREE_TYPE (decl), get_name (decl));
   17726        10370 :           gimplify_omp_ctxp = ctx;
   17727        10370 :           TREE_OPERAND (t, 0) = var;
   17728              : 
   17729        10370 :           gimplify_seq_add_stmt (&for_body, gimple_build_assign (decl, var));
   17730              : 
   17731        10370 :           if (ort == ORT_SIMD
   17732        10370 :               && TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt)) == 1)
   17733              :             {
   17734          770 :               c2 = build_omp_clause (input_location, OMP_CLAUSE_LINEAR);
   17735          770 :               OMP_CLAUSE_LINEAR_NO_COPYIN (c2) = 1;
   17736          770 :               OMP_CLAUSE_LINEAR_NO_COPYOUT (c2) = 1;
   17737          770 :               OMP_CLAUSE_DECL (c2) = var;
   17738          770 :               OMP_CLAUSE_CHAIN (c2) = OMP_FOR_CLAUSES (for_stmt);
   17739          770 :               OMP_FOR_CLAUSES (for_stmt) = c2;
   17740          770 :               omp_add_variable (gimplify_omp_ctxp, var,
   17741              :                                 GOVD_LINEAR | GOVD_EXPLICIT | GOVD_SEEN);
   17742          770 :               if (c == NULL_TREE)
   17743              :                 {
   17744          518 :                   c = c2;
   17745          518 :                   c2 = NULL_TREE;
   17746              :                 }
   17747              :             }
   17748              :           else
   17749         9600 :             omp_add_variable (gimplify_omp_ctxp, var,
   17750              :                               GOVD_PRIVATE | GOVD_SEEN);
   17751              :         }
   17752              :       else
   17753              :         var = decl;
   17754              : 
   17755        73277 :       gimplify_omp_ctxp->in_for_exprs = true;
   17756        73277 :       if (TREE_CODE (TREE_OPERAND (t, 1)) == TREE_VEC)
   17757              :         {
   17758          719 :           tree lb = TREE_OPERAND (t, 1);
   17759          719 :           tret = gimplify_expr (&TREE_VEC_ELT (lb, 1), &for_pre_body, NULL,
   17760              :                                 is_gimple_val, fb_rvalue, false);
   17761          719 :           ret = MIN (ret, tret);
   17762          719 :           tret = gimplify_expr (&TREE_VEC_ELT (lb, 2), &for_pre_body, NULL,
   17763              :                                 is_gimple_val, fb_rvalue, false);
   17764              :         }
   17765              :       else
   17766        72558 :         tret = gimplify_expr (&TREE_OPERAND (t, 1), &for_pre_body, NULL,
   17767              :                               is_gimple_val, fb_rvalue, false);
   17768        73277 :       gimplify_omp_ctxp->in_for_exprs = false;
   17769        73277 :       ret = MIN (ret, tret);
   17770        73277 :       if (ret == GS_ERROR)
   17771              :         return ret;
   17772              : 
   17773              :       /* Handle OMP_FOR_COND.  */
   17774        73277 :       t = TREE_VEC_ELT (OMP_FOR_COND (for_stmt), i);
   17775        73277 :       gcc_assert (COMPARISON_CLASS_P (t));
   17776        73277 :       gcc_assert (TREE_OPERAND (t, 0) == decl);
   17777              : 
   17778        73277 :       gimplify_omp_ctxp->in_for_exprs = true;
   17779        73277 :       if (TREE_CODE (TREE_OPERAND (t, 1)) == TREE_VEC)
   17780              :         {
   17781          599 :           tree ub = TREE_OPERAND (t, 1);
   17782          599 :           tret = gimplify_expr (&TREE_VEC_ELT (ub, 1), &for_pre_body, NULL,
   17783              :                                 is_gimple_val, fb_rvalue, false);
   17784          599 :           ret = MIN (ret, tret);
   17785          599 :           tret = gimplify_expr (&TREE_VEC_ELT (ub, 2), &for_pre_body, NULL,
   17786              :                                 is_gimple_val, fb_rvalue, false);
   17787              :         }
   17788              :       else
   17789        72678 :         tret = gimplify_expr (&TREE_OPERAND (t, 1), &for_pre_body, NULL,
   17790              :                               is_gimple_val, fb_rvalue, false);
   17791        73277 :       gimplify_omp_ctxp->in_for_exprs = false;
   17792        73277 :       ret = MIN (ret, tret);
   17793              : 
   17794              :       /* Handle OMP_FOR_INCR.  */
   17795        73277 :       t = TREE_VEC_ELT (OMP_FOR_INCR (for_stmt), i);
   17796        73277 :       switch (TREE_CODE (t))
   17797              :         {
   17798        35913 :         case PREINCREMENT_EXPR:
   17799        35913 :         case POSTINCREMENT_EXPR:
   17800        35913 :           {
   17801        35913 :             tree decl = TREE_OPERAND (t, 0);
   17802              :             /* c_omp_for_incr_canonicalize_ptr() should have been
   17803              :                called to massage things appropriately.  */
   17804        35913 :             gcc_assert (!POINTER_TYPE_P (TREE_TYPE (decl)));
   17805              : 
   17806        35913 :             if (orig_for_stmt != for_stmt)
   17807              :               break;
   17808        25490 :             t = build_int_cst (TREE_TYPE (decl), 1);
   17809        25490 :             if (c)
   17810         4554 :               OMP_CLAUSE_LINEAR_STEP (c) = t;
   17811        25490 :             t = build2 (PLUS_EXPR, TREE_TYPE (decl), var, t);
   17812        25490 :             t = build2 (MODIFY_EXPR, TREE_TYPE (var), var, t);
   17813        25490 :             TREE_VEC_ELT (OMP_FOR_INCR (for_stmt), i) = t;
   17814        25490 :             break;
   17815              :           }
   17816              : 
   17817         3830 :         case PREDECREMENT_EXPR:
   17818         3830 :         case POSTDECREMENT_EXPR:
   17819              :           /* c_omp_for_incr_canonicalize_ptr() should have been
   17820              :              called to massage things appropriately.  */
   17821         3830 :           gcc_assert (!POINTER_TYPE_P (TREE_TYPE (decl)));
   17822         3830 :           if (orig_for_stmt != for_stmt)
   17823              :             break;
   17824         1992 :           t = build_int_cst (TREE_TYPE (decl), -1);
   17825         1992 :           if (c)
   17826          112 :             OMP_CLAUSE_LINEAR_STEP (c) = t;
   17827         1992 :           t = build2 (PLUS_EXPR, TREE_TYPE (decl), var, t);
   17828         1992 :           t = build2 (MODIFY_EXPR, TREE_TYPE (var), var, t);
   17829         1992 :           TREE_VEC_ELT (OMP_FOR_INCR (for_stmt), i) = t;
   17830         1992 :           break;
   17831              : 
   17832        33534 :         case MODIFY_EXPR:
   17833        33534 :           gcc_assert (TREE_OPERAND (t, 0) == decl);
   17834        33534 :           TREE_OPERAND (t, 0) = var;
   17835              : 
   17836        33534 :           t = TREE_OPERAND (t, 1);
   17837        33534 :           switch (TREE_CODE (t))
   17838              :             {
   17839        26602 :             case PLUS_EXPR:
   17840        26602 :               if (TREE_OPERAND (t, 1) == decl)
   17841              :                 {
   17842          215 :                   TREE_OPERAND (t, 1) = TREE_OPERAND (t, 0);
   17843          215 :                   TREE_OPERAND (t, 0) = var;
   17844          215 :                   break;
   17845              :                 }
   17846              : 
   17847              :               /* Fallthru.  */
   17848        33319 :             case MINUS_EXPR:
   17849        33319 :             case POINTER_PLUS_EXPR:
   17850        33319 :               gcc_assert (TREE_OPERAND (t, 0) == decl);
   17851        33319 :               TREE_OPERAND (t, 0) = var;
   17852        33319 :               break;
   17853            0 :             default:
   17854            0 :               gcc_unreachable ();
   17855              :             }
   17856              : 
   17857        33534 :           gimplify_omp_ctxp->in_for_exprs = true;
   17858        33534 :           tret = gimplify_expr (&TREE_OPERAND (t, 1), &for_pre_body, NULL,
   17859              :                                 is_gimple_val, fb_rvalue, false);
   17860        33534 :           ret = MIN (ret, tret);
   17861        33534 :           if (c)
   17862              :             {
   17863         1573 :               tree step = TREE_OPERAND (t, 1);
   17864         1573 :               tree stept = TREE_TYPE (decl);
   17865         1573 :               if (POINTER_TYPE_P (stept))
   17866          280 :                 stept = sizetype;
   17867         1573 :               step = fold_convert (stept, step);
   17868         1573 :               if (TREE_CODE (t) == MINUS_EXPR)
   17869          295 :                 step = fold_build1 (NEGATE_EXPR, stept, step);
   17870         1573 :               OMP_CLAUSE_LINEAR_STEP (c) = step;
   17871         1573 :               if (step != TREE_OPERAND (t, 1))
   17872              :                 {
   17873          295 :                   tret = gimplify_expr (&OMP_CLAUSE_LINEAR_STEP (c),
   17874              :                                         &for_pre_body, NULL,
   17875              :                                         is_gimple_val, fb_rvalue, false);
   17876          295 :                   ret = MIN (ret, tret);
   17877              :                 }
   17878              :             }
   17879        33534 :           gimplify_omp_ctxp->in_for_exprs = false;
   17880        33534 :           break;
   17881              : 
   17882            0 :         default:
   17883            0 :           gcc_unreachable ();
   17884              :         }
   17885              : 
   17886        73277 :       if (c2)
   17887              :         {
   17888          252 :           gcc_assert (c);
   17889          252 :           OMP_CLAUSE_LINEAR_STEP (c2) = OMP_CLAUSE_LINEAR_STEP (c);
   17890              :         }
   17891              : 
   17892        73277 :       if ((var != decl || collapse > 1 || tile) && orig_for_stmt == for_stmt)
   17893              :         {
   17894        86700 :           for (c = OMP_FOR_CLAUSES (for_stmt); c ; c = OMP_CLAUSE_CHAIN (c))
   17895        65137 :             if (((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
   17896        16218 :                   && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) == NULL)
   17897        56128 :                  || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
   17898         2159 :                      && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c)
   17899         1373 :                      && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c) == NULL))
   17900        75497 :                 && OMP_CLAUSE_DECL (c) == decl)
   17901              :               {
   17902         7535 :                 if (is_doacross && (collapse == 1 || i >= collapse))
   17903              :                   t = var;
   17904              :                 else
   17905              :                   {
   17906         7462 :                     t = TREE_VEC_ELT (OMP_FOR_INCR (for_stmt), i);
   17907         7462 :                     gcc_assert (TREE_CODE (t) == MODIFY_EXPR);
   17908         7462 :                     gcc_assert (TREE_OPERAND (t, 0) == var);
   17909         7462 :                     t = TREE_OPERAND (t, 1);
   17910         7462 :                     gcc_assert (TREE_CODE (t) == PLUS_EXPR
   17911              :                                 || TREE_CODE (t) == MINUS_EXPR
   17912              :                                 || TREE_CODE (t) == POINTER_PLUS_EXPR);
   17913         7462 :                     gcc_assert (TREE_OPERAND (t, 0) == var);
   17914        14822 :                     t = build2 (TREE_CODE (t), TREE_TYPE (decl),
   17915              :                                 is_doacross ? var : decl,
   17916         7462 :                                 TREE_OPERAND (t, 1));
   17917              :                   }
   17918         7535 :                 gimple_seq *seq;
   17919         7535 :                 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE)
   17920         6939 :                   seq = &OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c);
   17921              :                 else
   17922          596 :                   seq = &OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c);
   17923         7535 :                 push_gimplify_context ();
   17924         7535 :                 gimplify_assign (decl, t, seq);
   17925         7535 :                 gimple *bind = NULL;
   17926         7535 :                 if (gimplify_ctxp->temps)
   17927              :                   {
   17928         2072 :                     bind = gimple_build_bind (NULL_TREE, *seq, NULL_TREE);
   17929         2072 :                     *seq = NULL;
   17930         2072 :                     gimplify_seq_add_stmt (seq, bind);
   17931              :                   }
   17932         7535 :                 pop_gimplify_context (bind);
   17933              :               }
   17934              :         }
   17935        73277 :       if (OMP_FOR_NON_RECTANGULAR (for_stmt) && var != decl)
   17936         2609 :         for (int j = i + 1; j < TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt)); j++)
   17937              :           {
   17938         1197 :             t = TREE_VEC_ELT (OMP_FOR_INIT (for_stmt), j);
   17939         1197 :             gcc_assert (TREE_CODE (t) == MODIFY_EXPR);
   17940         1197 :             if (TREE_CODE (TREE_OPERAND (t, 1)) == TREE_VEC
   17941         1197 :                 && TREE_VEC_ELT (TREE_OPERAND (t, 1), 0) == decl)
   17942          336 :               TREE_VEC_ELT (TREE_OPERAND (t, 1), 0) = var;
   17943         1197 :             t = TREE_VEC_ELT (OMP_FOR_COND (for_stmt), j);
   17944         1197 :             gcc_assert (COMPARISON_CLASS_P (t));
   17945         1197 :             if (TREE_CODE (TREE_OPERAND (t, 1)) == TREE_VEC
   17946         1197 :                 && TREE_VEC_ELT (TREE_OPERAND (t, 1), 0) == decl)
   17947          343 :               TREE_VEC_ELT (TREE_OPERAND (t, 1), 0) = var;
   17948              :           }
   17949              :     }
   17950              : 
   17951        51531 :   BITMAP_FREE (has_decl_expr);
   17952        51806 :   delete allocate_uids;
   17953              : 
   17954        51531 :   if (TREE_CODE (orig_for_stmt) == OMP_TASKLOOP
   17955        49945 :       || (loop_p && orig_for_stmt == for_stmt))
   17956              :     {
   17957         2648 :       push_gimplify_context ();
   17958         2648 :       if (TREE_CODE (OMP_FOR_BODY (orig_for_stmt)) != BIND_EXPR)
   17959              :         {
   17960         1897 :           OMP_FOR_BODY (orig_for_stmt)
   17961         1897 :             = build3 (BIND_EXPR, void_type_node, NULL,
   17962         1897 :                       OMP_FOR_BODY (orig_for_stmt), NULL);
   17963         1897 :           TREE_SIDE_EFFECTS (OMP_FOR_BODY (orig_for_stmt)) = 1;
   17964              :         }
   17965              :     }
   17966              : 
   17967        51531 :   gimple *g = gimplify_and_return_first (OMP_FOR_BODY (orig_for_stmt),
   17968              :                                          &for_body);
   17969              : 
   17970        51531 :   if (TREE_CODE (orig_for_stmt) == OMP_TASKLOOP
   17971        49945 :       || (loop_p && orig_for_stmt == for_stmt))
   17972              :     {
   17973         2648 :       if (gimple_code (g) == GIMPLE_BIND)
   17974         2648 :         pop_gimplify_context (g);
   17975              :       else
   17976            0 :         pop_gimplify_context (NULL);
   17977              :     }
   17978              : 
   17979        51531 :   if (orig_for_stmt != for_stmt)
   17980        36886 :     for (i = 0; i < TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt)); i++)
   17981              :       {
   17982        22884 :         t = TREE_VEC_ELT (OMP_FOR_INIT (for_stmt), i);
   17983        22884 :         decl = TREE_OPERAND (t, 0);
   17984        22884 :         struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
   17985        22884 :         if (TREE_CODE (orig_for_stmt) == OMP_TASKLOOP)
   17986          772 :           gimplify_omp_ctxp = ctx->outer_context;
   17987        22884 :         var = create_tmp_var (TREE_TYPE (decl), get_name (decl));
   17988        22884 :         gimplify_omp_ctxp = ctx;
   17989        22884 :         omp_add_variable (gimplify_omp_ctxp, var, GOVD_PRIVATE | GOVD_SEEN);
   17990        22884 :         TREE_OPERAND (t, 0) = var;
   17991        22884 :         t = TREE_VEC_ELT (OMP_FOR_INCR (for_stmt), i);
   17992        22884 :         TREE_OPERAND (t, 1) = copy_node (TREE_OPERAND (t, 1));
   17993        22884 :         TREE_OPERAND (TREE_OPERAND (t, 1), 0) = var;
   17994        22884 :         if (OMP_FOR_NON_RECTANGULAR (for_stmt))
   17995          791 :           for (int j = i + 1;
   17996          791 :                j < TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt)); j++)
   17997              :             {
   17998          377 :               t = TREE_VEC_ELT (OMP_FOR_INIT (for_stmt), j);
   17999          377 :               gcc_assert (TREE_CODE (t) == MODIFY_EXPR);
   18000          377 :               if (TREE_CODE (TREE_OPERAND (t, 1)) == TREE_VEC
   18001          377 :                   && TREE_VEC_ELT (TREE_OPERAND (t, 1), 0) == decl)
   18002              :                 {
   18003          133 :                   TREE_OPERAND (t, 1) = copy_node (TREE_OPERAND (t, 1));
   18004          133 :                   TREE_VEC_ELT (TREE_OPERAND (t, 1), 0) = var;
   18005              :                 }
   18006          377 :               t = TREE_VEC_ELT (OMP_FOR_COND (for_stmt), j);
   18007          377 :               gcc_assert (COMPARISON_CLASS_P (t));
   18008          377 :               if (TREE_CODE (TREE_OPERAND (t, 1)) == TREE_VEC
   18009          377 :                   && TREE_VEC_ELT (TREE_OPERAND (t, 1), 0) == decl)
   18010              :                 {
   18011           67 :                   TREE_OPERAND (t, 1) = copy_node (TREE_OPERAND (t, 1));
   18012           67 :                   TREE_VEC_ELT (TREE_OPERAND (t, 1), 0) = var;
   18013              :                 }
   18014              :           }
   18015              :       }
   18016              : 
   18017        51531 :   gimplify_adjust_omp_clauses (pre_p, for_body,
   18018              :                                &OMP_FOR_CLAUSES (orig_for_stmt),
   18019        51531 :                                TREE_CODE (orig_for_stmt));
   18020              : 
   18021        51531 :   int kind;
   18022        51531 :   switch (TREE_CODE (orig_for_stmt))
   18023              :     {
   18024              :     case OMP_FOR: kind = GF_OMP_FOR_KIND_FOR; break;
   18025        10773 :     case OMP_SIMD: kind = GF_OMP_FOR_KIND_SIMD; break;
   18026         8308 :     case OMP_DISTRIBUTE: kind = GF_OMP_FOR_KIND_DISTRIBUTE; break;
   18027         1586 :     case OMP_TASKLOOP: kind = GF_OMP_FOR_KIND_TASKLOOP; break;
   18028        12552 :     case OACC_LOOP: kind = GF_OMP_FOR_KIND_OACC_LOOP; break;
   18029            0 :     default:
   18030            0 :       gcc_unreachable ();
   18031              :     }
   18032        51531 :   if (loop_p && kind == GF_OMP_FOR_KIND_SIMD)
   18033              :     {
   18034         1062 :       gimplify_seq_add_seq (pre_p, for_pre_body);
   18035         1062 :       for_pre_body = NULL;
   18036              :     }
   18037        51531 :   gfor = gimple_build_omp_for (for_body, kind, OMP_FOR_CLAUSES (orig_for_stmt),
   18038        51531 :                                TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt)),
   18039              :                                for_pre_body);
   18040        51531 :   if (orig_for_stmt != for_stmt)
   18041        14002 :     gimple_omp_for_set_combined_p (gfor, true);
   18042        51531 :   if (gimplify_omp_ctxp
   18043        42768 :       && (gimplify_omp_ctxp->combined_loop
   18044        35210 :           || (gimplify_omp_ctxp->region_type == ORT_COMBINED_PARALLEL
   18045        12364 :               && gimplify_omp_ctxp->outer_context
   18046         7912 :               && gimplify_omp_ctxp->outer_context->combined_loop)))
   18047              :     {
   18048        14002 :       gimple_omp_for_set_combined_into_p (gfor, true);
   18049        14002 :       if (gimplify_omp_ctxp->combined_loop)
   18050         7558 :         gcc_assert (TREE_CODE (orig_for_stmt) == OMP_SIMD);
   18051              :       else
   18052         6444 :         gcc_assert (TREE_CODE (orig_for_stmt) == OMP_FOR);
   18053              :     }
   18054              : 
   18055       124808 :   for (i = 0; i < TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt)); i++)
   18056              :     {
   18057        73277 :       t = TREE_VEC_ELT (OMP_FOR_INIT (for_stmt), i);
   18058        73277 :       gimple_omp_for_set_index (gfor, i, TREE_OPERAND (t, 0));
   18059        73277 :       gimple_omp_for_set_initial (gfor, i, TREE_OPERAND (t, 1));
   18060        73277 :       t = TREE_VEC_ELT (OMP_FOR_COND (for_stmt), i);
   18061        73277 :       gimple_omp_for_set_cond (gfor, i, TREE_CODE (t));
   18062        73277 :       gimple_omp_for_set_final (gfor, i, TREE_OPERAND (t, 1));
   18063        73277 :       t = TREE_VEC_ELT (OMP_FOR_INCR (for_stmt), i);
   18064        73277 :       gimple_omp_for_set_incr (gfor, i, TREE_OPERAND (t, 1));
   18065              :     }
   18066              : 
   18067              :   /* OMP_TASKLOOP is gimplified as two GIMPLE_OMP_FOR taskloop
   18068              :      constructs with GIMPLE_OMP_TASK sandwiched in between them.
   18069              :      The outer taskloop stands for computing the number of iterations,
   18070              :      counts for collapsed loops and holding taskloop specific clauses.
   18071              :      The task construct stands for the effect of data sharing on the
   18072              :      explicit task it creates and the inner taskloop stands for expansion
   18073              :      of the static loop inside of the explicit task construct.  */
   18074        51531 :   if (TREE_CODE (orig_for_stmt) == OMP_TASKLOOP)
   18075              :     {
   18076         1586 :       tree *gfor_clauses_ptr = gimple_omp_for_clauses_ptr (gfor);
   18077         1586 :       tree task_clauses = NULL_TREE;
   18078         1586 :       tree c = *gfor_clauses_ptr;
   18079         1586 :       tree *gtask_clauses_ptr = &task_clauses;
   18080         1586 :       tree outer_for_clauses = NULL_TREE;
   18081         1586 :       tree *gforo_clauses_ptr = &outer_for_clauses;
   18082         1586 :       bitmap lastprivate_uids = NULL;
   18083         1586 :       if (omp_find_clause (c, OMP_CLAUSE_ALLOCATE))
   18084              :         {
   18085           36 :           c = omp_find_clause (c, OMP_CLAUSE_LASTPRIVATE);
   18086           36 :           if (c)
   18087              :             {
   18088           18 :               lastprivate_uids = BITMAP_ALLOC (NULL);
   18089           54 :               for (; c; c = omp_find_clause (OMP_CLAUSE_CHAIN (c),
   18090              :                                              OMP_CLAUSE_LASTPRIVATE))
   18091           18 :                 bitmap_set_bit (lastprivate_uids,
   18092           18 :                                 DECL_UID (OMP_CLAUSE_DECL (c)));
   18093              :             }
   18094           36 :           c = *gfor_clauses_ptr;
   18095              :         }
   18096        12254 :       for (; c; c = OMP_CLAUSE_CHAIN (c))
   18097        10668 :         switch (OMP_CLAUSE_CODE (c))
   18098              :           {
   18099              :           /* These clauses are allowed on task, move them there.  */
   18100         6230 :           case OMP_CLAUSE_SHARED:
   18101         6230 :           case OMP_CLAUSE_FIRSTPRIVATE:
   18102         6230 :           case OMP_CLAUSE_DEFAULT:
   18103         6230 :           case OMP_CLAUSE_IF:
   18104         6230 :           case OMP_CLAUSE_UNTIED:
   18105         6230 :           case OMP_CLAUSE_FINAL:
   18106         6230 :           case OMP_CLAUSE_MERGEABLE:
   18107         6230 :           case OMP_CLAUSE_PRIORITY:
   18108         6230 :           case OMP_CLAUSE_REDUCTION:
   18109         6230 :           case OMP_CLAUSE_IN_REDUCTION:
   18110         6230 :             *gtask_clauses_ptr = c;
   18111         6230 :             gtask_clauses_ptr = &OMP_CLAUSE_CHAIN (c);
   18112         6230 :             break;
   18113         2068 :           case OMP_CLAUSE_PRIVATE:
   18114         2068 :             if (OMP_CLAUSE_PRIVATE_TASKLOOP_IV (c))
   18115              :               {
   18116              :                 /* We want private on outer for and firstprivate
   18117              :                    on task.  */
   18118           26 :                 *gtask_clauses_ptr
   18119           26 :                   = build_omp_clause (OMP_CLAUSE_LOCATION (c),
   18120              :                                       OMP_CLAUSE_FIRSTPRIVATE);
   18121           26 :                 OMP_CLAUSE_DECL (*gtask_clauses_ptr) = OMP_CLAUSE_DECL (c);
   18122           26 :                 lang_hooks.decls.omp_finish_clause (*gtask_clauses_ptr, NULL,
   18123              :                                                     openacc);
   18124           26 :                 gtask_clauses_ptr = &OMP_CLAUSE_CHAIN (*gtask_clauses_ptr);
   18125           26 :                 *gforo_clauses_ptr = c;
   18126           26 :                 gforo_clauses_ptr = &OMP_CLAUSE_CHAIN (c);
   18127              :               }
   18128              :             else
   18129              :               {
   18130         2042 :                 *gtask_clauses_ptr = c;
   18131         2042 :                 gtask_clauses_ptr = &OMP_CLAUSE_CHAIN (c);
   18132              :               }
   18133              :             break;
   18134              :           /* These clauses go into outer taskloop clauses.  */
   18135          533 :           case OMP_CLAUSE_GRAINSIZE:
   18136          533 :           case OMP_CLAUSE_NUM_TASKS:
   18137          533 :           case OMP_CLAUSE_NOGROUP:
   18138          533 :             *gforo_clauses_ptr = c;
   18139          533 :             gforo_clauses_ptr = &OMP_CLAUSE_CHAIN (c);
   18140          533 :             break;
   18141              :           /* Collapse clause we duplicate on both taskloops.  */
   18142          584 :           case OMP_CLAUSE_COLLAPSE:
   18143          584 :             *gfor_clauses_ptr = c;
   18144          584 :             gfor_clauses_ptr = &OMP_CLAUSE_CHAIN (c);
   18145          584 :             *gforo_clauses_ptr = copy_node (c);
   18146          584 :             gforo_clauses_ptr = &OMP_CLAUSE_CHAIN (*gforo_clauses_ptr);
   18147          584 :             break;
   18148              :           /* For lastprivate, keep the clause on inner taskloop, and add
   18149              :              a shared clause on task.  If the same decl is also firstprivate,
   18150              :              add also firstprivate clause on the inner taskloop.  */
   18151         1191 :           case OMP_CLAUSE_LASTPRIVATE:
   18152         1191 :             if (OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c))
   18153              :               {
   18154              :                 /* For taskloop C++ lastprivate IVs, we want:
   18155              :                    1) private on outer taskloop
   18156              :                    2) firstprivate and shared on task
   18157              :                    3) lastprivate on inner taskloop  */
   18158           38 :                 *gtask_clauses_ptr
   18159           38 :                   = build_omp_clause (OMP_CLAUSE_LOCATION (c),
   18160              :                                       OMP_CLAUSE_FIRSTPRIVATE);
   18161           38 :                 OMP_CLAUSE_DECL (*gtask_clauses_ptr) = OMP_CLAUSE_DECL (c);
   18162           38 :                 lang_hooks.decls.omp_finish_clause (*gtask_clauses_ptr, NULL,
   18163              :                                                     openacc);
   18164           38 :                 gtask_clauses_ptr = &OMP_CLAUSE_CHAIN (*gtask_clauses_ptr);
   18165           38 :                 OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c) = 1;
   18166           38 :                 *gforo_clauses_ptr = build_omp_clause (OMP_CLAUSE_LOCATION (c),
   18167              :                                                        OMP_CLAUSE_PRIVATE);
   18168           38 :                 OMP_CLAUSE_DECL (*gforo_clauses_ptr) = OMP_CLAUSE_DECL (c);
   18169           38 :                 OMP_CLAUSE_PRIVATE_TASKLOOP_IV (*gforo_clauses_ptr) = 1;
   18170           38 :                 TREE_TYPE (*gforo_clauses_ptr) = TREE_TYPE (c);
   18171           38 :                 gforo_clauses_ptr = &OMP_CLAUSE_CHAIN (*gforo_clauses_ptr);
   18172              :               }
   18173         1191 :             *gfor_clauses_ptr = c;
   18174         1191 :             gfor_clauses_ptr = &OMP_CLAUSE_CHAIN (c);
   18175         1191 :             *gtask_clauses_ptr
   18176         1191 :               = build_omp_clause (OMP_CLAUSE_LOCATION (c), OMP_CLAUSE_SHARED);
   18177         1191 :             OMP_CLAUSE_DECL (*gtask_clauses_ptr) = OMP_CLAUSE_DECL (c);
   18178         1191 :             if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
   18179          312 :               OMP_CLAUSE_SHARED_FIRSTPRIVATE (*gtask_clauses_ptr) = 1;
   18180         1191 :             gtask_clauses_ptr
   18181         1191 :               = &OMP_CLAUSE_CHAIN (*gtask_clauses_ptr);
   18182         1191 :             break;
   18183              :           /* Allocate clause we duplicate on task and inner taskloop
   18184              :              if the decl is lastprivate, otherwise just put on task.  */
   18185           62 :           case OMP_CLAUSE_ALLOCATE:
   18186           62 :             if (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c)
   18187           62 :                 && DECL_P (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c)))
   18188              :               {
   18189              :                 /* Additionally, put firstprivate clause on task
   18190              :                    for the allocator if it is not constant.  */
   18191           34 :                 *gtask_clauses_ptr
   18192           34 :                   = build_omp_clause (OMP_CLAUSE_LOCATION (c),
   18193              :                                       OMP_CLAUSE_FIRSTPRIVATE);
   18194           34 :                 OMP_CLAUSE_DECL (*gtask_clauses_ptr)
   18195           34 :                   = OMP_CLAUSE_ALLOCATE_ALLOCATOR (c);
   18196           34 :                 gtask_clauses_ptr = &OMP_CLAUSE_CHAIN (*gtask_clauses_ptr);
   18197              :               }
   18198           62 :             if (lastprivate_uids
   18199           97 :                 && bitmap_bit_p (lastprivate_uids,
   18200           35 :                                  DECL_UID (OMP_CLAUSE_DECL (c))))
   18201              :               {
   18202           17 :                 *gfor_clauses_ptr = c;
   18203           17 :                 gfor_clauses_ptr = &OMP_CLAUSE_CHAIN (c);
   18204           17 :                 *gtask_clauses_ptr = copy_node (c);
   18205           17 :                 gtask_clauses_ptr = &OMP_CLAUSE_CHAIN (*gtask_clauses_ptr);
   18206              :               }
   18207              :             else
   18208              :               {
   18209           45 :                 *gtask_clauses_ptr = c;
   18210           45 :                 gtask_clauses_ptr = &OMP_CLAUSE_CHAIN (c);
   18211              :               }
   18212              :             break;
   18213            0 :           default:
   18214            0 :             gcc_unreachable ();
   18215              :           }
   18216         1586 :       *gfor_clauses_ptr = NULL_TREE;
   18217         1586 :       *gtask_clauses_ptr = NULL_TREE;
   18218         1586 :       *gforo_clauses_ptr = NULL_TREE;
   18219         1586 :       BITMAP_FREE (lastprivate_uids);
   18220         1586 :       gimple_set_location (gfor, input_location);
   18221         1586 :       g = gimple_build_bind (NULL_TREE, gfor, NULL_TREE);
   18222         1586 :       g = gimple_build_omp_task (g, task_clauses, NULL_TREE, NULL_TREE,
   18223              :                                  NULL_TREE, NULL_TREE, NULL_TREE);
   18224         1586 :       gimple_set_location (g, input_location);
   18225         1586 :       gimple_omp_task_set_taskloop_p (g, true);
   18226         1586 :       g = gimple_build_bind (NULL_TREE, g, NULL_TREE);
   18227         1586 :       gomp_for *gforo
   18228         1586 :         = gimple_build_omp_for (g, GF_OMP_FOR_KIND_TASKLOOP, outer_for_clauses,
   18229              :                                 gimple_omp_for_collapse (gfor),
   18230              :                                 gimple_omp_for_pre_body (gfor));
   18231         1586 :       gimple_omp_for_set_pre_body (gfor, NULL);
   18232         1586 :       gimple_omp_for_set_combined_p (gforo, true);
   18233         1586 :       gimple_omp_for_set_combined_into_p (gfor, true);
   18234         3458 :       for (i = 0; i < (int) gimple_omp_for_collapse (gfor); i++)
   18235              :         {
   18236         1872 :           tree type = TREE_TYPE (gimple_omp_for_index (gfor, i));
   18237         1872 :           tree v = create_tmp_var (type);
   18238         1872 :           gimple_omp_for_set_index (gforo, i, v);
   18239         1872 :           t = unshare_expr (gimple_omp_for_initial (gfor, i));
   18240         1872 :           gimple_omp_for_set_initial (gforo, i, t);
   18241         1872 :           gimple_omp_for_set_cond (gforo, i,
   18242              :                                    gimple_omp_for_cond (gfor, i));
   18243         1872 :           t = unshare_expr (gimple_omp_for_final (gfor, i));
   18244         1872 :           gimple_omp_for_set_final (gforo, i, t);
   18245         1872 :           t = unshare_expr (gimple_omp_for_incr (gfor, i));
   18246         1872 :           gcc_assert (TREE_OPERAND (t, 0) == gimple_omp_for_index (gfor, i));
   18247         1872 :           TREE_OPERAND (t, 0) = v;
   18248         1872 :           gimple_omp_for_set_incr (gforo, i, t);
   18249         1872 :           t = build_omp_clause (input_location, OMP_CLAUSE_PRIVATE);
   18250         1872 :           OMP_CLAUSE_DECL (t) = v;
   18251         1872 :           OMP_CLAUSE_CHAIN (t) = gimple_omp_for_clauses (gforo);
   18252         1872 :           gimple_omp_for_set_clauses (gforo, t);
   18253         1872 :           if (OMP_FOR_NON_RECTANGULAR (for_stmt))
   18254              :             {
   18255           90 :               tree *p1 = NULL, *p2 = NULL;
   18256           90 :               t = gimple_omp_for_initial (gforo, i);
   18257           90 :               if (TREE_CODE (t) == TREE_VEC)
   18258           34 :                 p1 = &TREE_VEC_ELT (t, 0);
   18259           90 :               t = gimple_omp_for_final (gforo, i);
   18260           90 :               if (TREE_CODE (t) == TREE_VEC)
   18261              :                 {
   18262           31 :                   if (p1)
   18263           23 :                     p2 = &TREE_VEC_ELT (t, 0);
   18264              :                   else
   18265            8 :                     p1 = &TREE_VEC_ELT (t, 0);
   18266              :                 }
   18267           90 :               if (p1)
   18268              :                 {
   18269              :                   int j;
   18270           58 :                   for (j = 0; j < i; j++)
   18271           58 :                     if (*p1 == gimple_omp_for_index (gfor, j))
   18272              :                       {
   18273           42 :                         *p1 = gimple_omp_for_index (gforo, j);
   18274           42 :                         if (p2)
   18275           23 :                           *p2 = *p1;
   18276              :                         break;
   18277              :                       }
   18278           42 :                   gcc_assert (j < i);
   18279              :                 }
   18280              :             }
   18281              :         }
   18282         1586 :       gimplify_seq_add_stmt (pre_p, gforo);
   18283              :     }
   18284              :   else
   18285        49945 :     gimplify_seq_add_stmt (pre_p, gfor);
   18286              : 
   18287        51531 :   if (TREE_CODE (orig_for_stmt) == OMP_FOR)
   18288              :     {
   18289        18312 :       struct gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
   18290        18312 :       unsigned lastprivate_conditional = 0;
   18291        18312 :       while (ctx
   18292        18348 :              && (ctx->region_type == ORT_TARGET_DATA
   18293        14262 :                  || ctx->region_type == ORT_TASKGROUP))
   18294           36 :         ctx = ctx->outer_context;
   18295        18312 :       if (ctx && (ctx->region_type & ORT_PARALLEL) != 0)
   18296        14002 :         for (tree c = gimple_omp_for_clauses (gfor);
   18297        63424 :              c; c = OMP_CLAUSE_CHAIN (c))
   18298        49422 :           if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
   18299        49422 :               && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
   18300          158 :             ++lastprivate_conditional;
   18301        14002 :       if (lastprivate_conditional)
   18302              :         {
   18303          118 :           struct omp_for_data fd;
   18304          118 :           omp_extract_for_data (gfor, &fd, NULL);
   18305          118 :           tree type = build_array_type_nelts (unsigned_type_for (fd.iter_type),
   18306          118 :                                               lastprivate_conditional);
   18307          118 :           tree var = create_tmp_var_raw (type);
   18308          118 :           tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__CONDTEMP_);
   18309          118 :           OMP_CLAUSE_DECL (c) = var;
   18310          118 :           OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (gfor);
   18311          118 :           gimple_omp_for_set_clauses (gfor, c);
   18312          118 :           omp_add_variable (ctx, var, GOVD_CONDTEMP | GOVD_SEEN);
   18313              :         }
   18314              :     }
   18315        33219 :   else if (TREE_CODE (orig_for_stmt) == OMP_SIMD)
   18316              :     {
   18317        10773 :       unsigned lastprivate_conditional = 0;
   18318        49437 :       for (tree c = gimple_omp_for_clauses (gfor); c; c = OMP_CLAUSE_CHAIN (c))
   18319        38664 :         if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
   18320        38664 :             && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
   18321          119 :           ++lastprivate_conditional;
   18322        10773 :       if (lastprivate_conditional)
   18323              :         {
   18324           91 :           struct omp_for_data fd;
   18325           91 :           omp_extract_for_data (gfor, &fd, NULL);
   18326           91 :           tree type = unsigned_type_for (fd.iter_type);
   18327          301 :           while (lastprivate_conditional--)
   18328              :             {
   18329          119 :               tree c = build_omp_clause (UNKNOWN_LOCATION,
   18330              :                                          OMP_CLAUSE__CONDTEMP_);
   18331          119 :               OMP_CLAUSE_DECL (c) = create_tmp_var (type);
   18332          119 :               OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (gfor);
   18333          119 :               gimple_omp_for_set_clauses (gfor, c);
   18334              :             }
   18335              :         }
   18336              :     }
   18337              : 
   18338        51531 :   if (ret != GS_ALL_DONE)
   18339              :     return GS_ERROR;
   18340        51531 :   *expr_p = NULL_TREE;
   18341        51531 :   return GS_ALL_DONE;
   18342              : }
   18343              : 
   18344              : /* Helper for gimplify_omp_loop, called through walk_tree.  */
   18345              : 
   18346              : static tree
   18347          294 : note_no_context_vars (tree *tp, int *, void *data)
   18348              : {
   18349          294 :   if (VAR_P (*tp)
   18350           56 :       && DECL_CONTEXT (*tp) == NULL_TREE
   18351          309 :       && !is_global_var (*tp))
   18352              :     {
   18353           15 :       vec<tree> *d = (vec<tree> *) data;
   18354           15 :       d->safe_push (*tp);
   18355           15 :       DECL_CONTEXT (*tp) = current_function_decl;
   18356              :     }
   18357          294 :   return NULL_TREE;
   18358              : }
   18359              : 
   18360              : /* Gimplify the gross structure of an OMP_LOOP statement.  */
   18361              : 
   18362              : static enum gimplify_status
   18363         1062 : gimplify_omp_loop (tree *expr_p, gimple_seq *pre_p)
   18364              : {
   18365         1062 :   tree for_stmt = *expr_p;
   18366         1062 :   tree clauses = OMP_FOR_CLAUSES (for_stmt);
   18367         1062 :   struct gimplify_omp_ctx *octx = gimplify_omp_ctxp;
   18368         1062 :   enum omp_clause_bind_kind kind = OMP_CLAUSE_BIND_THREAD;
   18369         1062 :   int i;
   18370              : 
   18371         1062 :   omp_maybe_apply_loop_xforms (expr_p, NULL_TREE);
   18372         1062 :   if (*expr_p != for_stmt)
   18373              :     return GS_OK;
   18374              : 
   18375              :   /* If order is not present, the behavior is as if order(concurrent)
   18376              :      appeared.  */
   18377         1062 :   tree order = omp_find_clause (clauses, OMP_CLAUSE_ORDER);
   18378         1062 :   if (order == NULL_TREE)
   18379              :     {
   18380          805 :       order = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_ORDER);
   18381          805 :       OMP_CLAUSE_CHAIN (order) = clauses;
   18382          805 :       OMP_FOR_CLAUSES (for_stmt) = clauses = order;
   18383              :     }
   18384              : 
   18385         1062 :   tree bind = omp_find_clause (clauses, OMP_CLAUSE_BIND);
   18386         1062 :   if (bind == NULL_TREE)
   18387              :     {
   18388          580 :       if (!flag_openmp) /* flag_openmp_simd */
   18389              :         ;
   18390          563 :       else if (octx && (octx->region_type & ORT_TEAMS) != 0)
   18391              :         kind = OMP_CLAUSE_BIND_TEAMS;
   18392          329 :       else if (octx && (octx->region_type & ORT_PARALLEL) != 0)
   18393              :         kind = OMP_CLAUSE_BIND_PARALLEL;
   18394              :       else
   18395              :         {
   18396          188 :           for (; octx; octx = octx->outer_context)
   18397              :             {
   18398           80 :               if ((octx->region_type & ORT_ACC) != 0
   18399           80 :                   || octx->region_type == ORT_NONE
   18400           80 :                   || octx->region_type == ORT_IMPLICIT_TARGET)
   18401            0 :                 continue;
   18402              :               break;
   18403              :             }
   18404          188 :           if (octx == NULL && !in_omp_construct)
   18405            4 :             error_at (EXPR_LOCATION (for_stmt),
   18406              :                       "%<bind%> clause not specified on a %<loop%> "
   18407              :                       "construct not nested inside another OpenMP construct");
   18408              :         }
   18409          580 :       bind = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_BIND);
   18410          580 :       OMP_CLAUSE_CHAIN (bind) = clauses;
   18411          580 :       OMP_CLAUSE_BIND_KIND (bind) = kind;
   18412          580 :       OMP_FOR_CLAUSES (for_stmt) = bind;
   18413              :     }
   18414              :   else
   18415          482 :     switch (OMP_CLAUSE_BIND_KIND (bind))
   18416              :       {
   18417              :       case OMP_CLAUSE_BIND_THREAD:
   18418              :         break;
   18419          166 :       case OMP_CLAUSE_BIND_PARALLEL:
   18420          166 :         if (!flag_openmp) /* flag_openmp_simd */
   18421              :           {
   18422            0 :             OMP_CLAUSE_BIND_KIND (bind) = OMP_CLAUSE_BIND_THREAD;
   18423            0 :             break;
   18424              :           }
   18425          240 :         for (; octx; octx = octx->outer_context)
   18426           78 :           if (octx->region_type == ORT_SIMD
   18427           78 :               && omp_find_clause (octx->clauses, OMP_CLAUSE_BIND) == NULL_TREE)
   18428              :             {
   18429            4 :               error_at (EXPR_LOCATION (for_stmt),
   18430              :                         "%<bind(parallel)%> on a %<loop%> construct nested "
   18431              :                         "inside %<simd%> construct");
   18432            4 :               OMP_CLAUSE_BIND_KIND (bind) = OMP_CLAUSE_BIND_THREAD;
   18433            4 :               break;
   18434              :             }
   18435              :         kind = OMP_CLAUSE_BIND_PARALLEL;
   18436              :         break;
   18437          162 :       case OMP_CLAUSE_BIND_TEAMS:
   18438          162 :         if (!flag_openmp) /* flag_openmp_simd */
   18439              :           {
   18440            0 :             OMP_CLAUSE_BIND_KIND (bind) = OMP_CLAUSE_BIND_THREAD;
   18441            0 :             break;
   18442              :           }
   18443          162 :         if ((octx
   18444          128 :              && octx->region_type != ORT_IMPLICIT_TARGET
   18445          111 :              && octx->region_type != ORT_NONE
   18446          111 :              && (octx->region_type & ORT_TEAMS) == 0)
   18447          110 :             || in_omp_construct)
   18448              :           {
   18449           64 :             error_at (EXPR_LOCATION (for_stmt),
   18450              :                       "%<bind(teams)%> on a %<loop%> region not strictly "
   18451              :                       "nested inside of a %<teams%> region");
   18452           64 :             OMP_CLAUSE_BIND_KIND (bind) = OMP_CLAUSE_BIND_THREAD;
   18453           64 :             break;
   18454              :           }
   18455              :         kind = OMP_CLAUSE_BIND_TEAMS;
   18456              :         break;
   18457            0 :       default:
   18458            0 :         gcc_unreachable ();
   18459              :       }
   18460              : 
   18461         4299 :   for (tree *pc = &OMP_FOR_CLAUSES (for_stmt); *pc; )
   18462         3237 :     switch (OMP_CLAUSE_CODE (*pc))
   18463              :       {
   18464          344 :       case OMP_CLAUSE_REDUCTION:
   18465          344 :         if (OMP_CLAUSE_REDUCTION_INSCAN (*pc))
   18466              :           {
   18467            4 :             error_at (OMP_CLAUSE_LOCATION (*pc),
   18468              :                       "%<inscan%> %<reduction%> clause on "
   18469              :                       "%qs construct", "loop");
   18470            4 :             OMP_CLAUSE_REDUCTION_INSCAN (*pc) = 0;
   18471              :           }
   18472          344 :         if (OMP_CLAUSE_REDUCTION_TASK (*pc))
   18473              :           {
   18474            8 :             error_at (OMP_CLAUSE_LOCATION (*pc),
   18475              :                       "invalid %<task%> reduction modifier on construct "
   18476              :                       "other than %<parallel%>, %qs or %<sections%>",
   18477            4 :                       lang_GNU_Fortran () ? "do" : "for");
   18478            4 :             OMP_CLAUSE_REDUCTION_TASK (*pc) = 0;
   18479              :           }
   18480          344 :         pc = &OMP_CLAUSE_CHAIN (*pc);
   18481          344 :         break;
   18482              :       case OMP_CLAUSE_LASTPRIVATE:
   18483          293 :         for (i = 0; i < TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt)); i++)
   18484              :           {
   18485          289 :             tree t = TREE_VEC_ELT (OMP_FOR_INIT (for_stmt), i);
   18486          289 :             gcc_assert (TREE_CODE (t) == MODIFY_EXPR);
   18487          289 :             if (OMP_CLAUSE_DECL (*pc) == TREE_OPERAND (t, 0))
   18488              :               break;
   18489           26 :             if (OMP_FOR_ORIG_DECLS (for_stmt)
   18490           25 :                 && TREE_CODE (TREE_VEC_ELT (OMP_FOR_ORIG_DECLS (for_stmt),
   18491              :                                             i)) == TREE_LIST
   18492           48 :                 && TREE_PURPOSE (TREE_VEC_ELT (OMP_FOR_ORIG_DECLS (for_stmt),
   18493              :                                                i)))
   18494              :               {
   18495           22 :                 tree orig = TREE_VEC_ELT (OMP_FOR_ORIG_DECLS (for_stmt), i);
   18496           22 :                 if (OMP_CLAUSE_DECL (*pc) == TREE_PURPOSE (orig))
   18497              :                   break;
   18498              :               }
   18499              :           }
   18500          289 :         if (i == TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt)))
   18501              :           {
   18502            4 :             error_at (OMP_CLAUSE_LOCATION (*pc),
   18503              :                       "%<lastprivate%> clause on a %<loop%> construct refers "
   18504              :                       "to a variable %qD which is not the loop iterator",
   18505            4 :                       OMP_CLAUSE_DECL (*pc));
   18506            4 :             *pc = OMP_CLAUSE_CHAIN (*pc);
   18507            4 :             break;
   18508              :           }
   18509          285 :         pc = &OMP_CLAUSE_CHAIN (*pc);
   18510          285 :         break;
   18511         2604 :       default:
   18512         2604 :         pc = &OMP_CLAUSE_CHAIN (*pc);
   18513         2604 :         break;
   18514              :     }
   18515              : 
   18516         1062 :   TREE_SET_CODE (for_stmt, OMP_SIMD);
   18517              : 
   18518         1062 :   int last;
   18519         1062 :   switch (kind)
   18520              :     {
   18521              :     case OMP_CLAUSE_BIND_THREAD: last = 0; break;
   18522          415 :     case OMP_CLAUSE_BIND_PARALLEL: last = 1; break;
   18523          224 :     case OMP_CLAUSE_BIND_TEAMS: last = 2; break;
   18524              :     }
   18525         1925 :   for (int pass = 1; pass <= last; pass++)
   18526              :     {
   18527          863 :       if (pass == 2)
   18528              :         {
   18529          224 :           tree bind = build3 (BIND_EXPR, void_type_node, NULL, NULL,
   18530              :                               make_node (BLOCK));
   18531          224 :           append_to_statement_list (*expr_p, &BIND_EXPR_BODY (bind));
   18532          224 :           *expr_p = make_node (OMP_PARALLEL);
   18533          224 :           TREE_TYPE (*expr_p) = void_type_node;
   18534          224 :           OMP_PARALLEL_BODY (*expr_p) = bind;
   18535          224 :           OMP_PARALLEL_COMBINED (*expr_p) = 1;
   18536          224 :           SET_EXPR_LOCATION (*expr_p, EXPR_LOCATION (for_stmt));
   18537          224 :           tree *pc = &OMP_PARALLEL_CLAUSES (*expr_p);
   18538          684 :           for (i = 0; i < TREE_VEC_LENGTH (OMP_FOR_INIT (for_stmt)); i++)
   18539          236 :             if (OMP_FOR_ORIG_DECLS (for_stmt)
   18540          236 :                 && (TREE_CODE (TREE_VEC_ELT (OMP_FOR_ORIG_DECLS (for_stmt), i))
   18541              :                     == TREE_LIST))
   18542              :               {
   18543           16 :                 tree elt = TREE_VEC_ELT (OMP_FOR_ORIG_DECLS (for_stmt), i);
   18544           16 :                 if (TREE_PURPOSE (elt) && TREE_VALUE (elt))
   18545              :                   {
   18546           13 :                     *pc = build_omp_clause (UNKNOWN_LOCATION,
   18547              :                                             OMP_CLAUSE_FIRSTPRIVATE);
   18548           13 :                     OMP_CLAUSE_DECL (*pc) = TREE_VALUE (elt);
   18549           13 :                     pc = &OMP_CLAUSE_CHAIN (*pc);
   18550              :                   }
   18551              :               }
   18552              :         }
   18553          863 :       tree t = make_node (pass == 2 ? OMP_DISTRIBUTE : OMP_FOR);
   18554          863 :       tree *pc = &OMP_FOR_CLAUSES (t);
   18555          863 :       TREE_TYPE (t) = void_type_node;
   18556          863 :       OMP_FOR_BODY (t) = *expr_p;
   18557          863 :       SET_EXPR_LOCATION (t, EXPR_LOCATION (for_stmt));
   18558         3995 :       for (tree c = OMP_FOR_CLAUSES (for_stmt); c; c = OMP_CLAUSE_CHAIN (c))
   18559         3132 :         switch (OMP_CLAUSE_CODE (c))
   18560              :           {
   18561         2010 :           case OMP_CLAUSE_BIND:
   18562         2010 :           case OMP_CLAUSE_ORDER:
   18563         2010 :           case OMP_CLAUSE_COLLAPSE:
   18564         2010 :             *pc = copy_node (c);
   18565         2010 :             pc = &OMP_CLAUSE_CHAIN (*pc);
   18566         2010 :             break;
   18567              :           case OMP_CLAUSE_PRIVATE:
   18568              :           case OMP_CLAUSE_FIRSTPRIVATE:
   18569              :             /* Only needed on innermost.  */
   18570              :             break;
   18571          351 :           case OMP_CLAUSE_LASTPRIVATE:
   18572          351 :             if (OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c) && pass != last)
   18573              :               {
   18574            7 :                 *pc = build_omp_clause (OMP_CLAUSE_LOCATION (c),
   18575              :                                         OMP_CLAUSE_FIRSTPRIVATE);
   18576            7 :                 OMP_CLAUSE_DECL (*pc) = OMP_CLAUSE_DECL (c);
   18577            7 :                 lang_hooks.decls.omp_finish_clause (*pc, NULL, false);
   18578            7 :                 pc = &OMP_CLAUSE_CHAIN (*pc);
   18579              :               }
   18580          351 :             *pc = copy_node (c);
   18581          351 :             OMP_CLAUSE_LASTPRIVATE_STMT (*pc) = NULL_TREE;
   18582          351 :             TREE_TYPE (*pc) = unshare_expr (TREE_TYPE (c));
   18583          351 :             if (OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c))
   18584              :               {
   18585           22 :                 if (pass != last)
   18586            7 :                   OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (*pc) = 1;
   18587              :                 else
   18588           15 :                   lang_hooks.decls.omp_finish_clause (*pc, NULL, false);
   18589           22 :                 OMP_CLAUSE_LASTPRIVATE_LOOP_IV (*pc) = 0;
   18590              :               }
   18591          351 :             pc = &OMP_CLAUSE_CHAIN (*pc);
   18592          351 :             break;
   18593          463 :           case OMP_CLAUSE_REDUCTION:
   18594          463 :             *pc = copy_node (c);
   18595          463 :             OMP_CLAUSE_DECL (*pc) = unshare_expr (OMP_CLAUSE_DECL (c));
   18596          463 :             TREE_TYPE (*pc) = unshare_expr (TREE_TYPE (c));
   18597          463 :             if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (*pc))
   18598              :               {
   18599           15 :                 auto_vec<tree> no_context_vars;
   18600           15 :                 int walk_subtrees = 0;
   18601           15 :                 note_no_context_vars (&OMP_CLAUSE_REDUCTION_PLACEHOLDER (c),
   18602              :                                       &walk_subtrees, &no_context_vars);
   18603           15 :                 if (tree p = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c))
   18604            0 :                   note_no_context_vars (&p, &walk_subtrees, &no_context_vars);
   18605           15 :                 walk_tree_without_duplicates (&OMP_CLAUSE_REDUCTION_INIT (c),
   18606              :                                               note_no_context_vars,
   18607              :                                               &no_context_vars);
   18608           15 :                 walk_tree_without_duplicates (&OMP_CLAUSE_REDUCTION_MERGE (c),
   18609              :                                               note_no_context_vars,
   18610              :                                               &no_context_vars);
   18611              : 
   18612           15 :                 OMP_CLAUSE_REDUCTION_PLACEHOLDER (*pc)
   18613           15 :                   = copy_node (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c));
   18614           15 :                 if (OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (*pc))
   18615            0 :                   OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (*pc)
   18616            0 :                     = copy_node (OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c));
   18617              : 
   18618           15 :                 hash_map<tree, tree> decl_map;
   18619           15 :                 decl_map.put (OMP_CLAUSE_DECL (c), OMP_CLAUSE_DECL (c));
   18620           15 :                 decl_map.put (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c),
   18621           15 :                               OMP_CLAUSE_REDUCTION_PLACEHOLDER (*pc));
   18622           15 :                 if (OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (*pc))
   18623            0 :                   decl_map.put (OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c),
   18624            0 :                                 OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (*pc));
   18625              : 
   18626           15 :                 copy_body_data id;
   18627           15 :                 memset (&id, 0, sizeof (id));
   18628           15 :                 id.src_fn = current_function_decl;
   18629           15 :                 id.dst_fn = current_function_decl;
   18630           15 :                 id.src_cfun = cfun;
   18631           15 :                 id.decl_map = &decl_map;
   18632           15 :                 id.copy_decl = copy_decl_no_change;
   18633           15 :                 id.transform_call_graph_edges = CB_CGE_DUPLICATE;
   18634           15 :                 id.transform_new_cfg = true;
   18635           15 :                 id.transform_return_to_modify = false;
   18636           15 :                 id.eh_lp_nr = 0;
   18637           15 :                 walk_tree (&OMP_CLAUSE_REDUCTION_INIT (*pc), copy_tree_body_r,
   18638              :                            &id, NULL);
   18639           15 :                 walk_tree (&OMP_CLAUSE_REDUCTION_MERGE (*pc), copy_tree_body_r,
   18640              :                            &id, NULL);
   18641              : 
   18642           60 :                 for (tree d : no_context_vars)
   18643              :                   {
   18644           15 :                     DECL_CONTEXT (d) = NULL_TREE;
   18645           15 :                     DECL_CONTEXT (*decl_map.get (d)) = NULL_TREE;
   18646              :                   }
   18647           15 :               }
   18648              :             else
   18649              :               {
   18650          448 :                 OMP_CLAUSE_REDUCTION_INIT (*pc)
   18651          448 :                   = unshare_expr (OMP_CLAUSE_REDUCTION_INIT (c));
   18652          448 :                 OMP_CLAUSE_REDUCTION_MERGE (*pc)
   18653          896 :                   = unshare_expr (OMP_CLAUSE_REDUCTION_MERGE (c));
   18654              :               }
   18655          463 :             pc = &OMP_CLAUSE_CHAIN (*pc);
   18656          463 :             break;
   18657            0 :           default:
   18658            0 :             gcc_unreachable ();
   18659              :           }
   18660          863 :       *pc = NULL_TREE;
   18661          863 :       *expr_p = t;
   18662              :     }
   18663         1062 :   return gimplify_expr (expr_p, pre_p, NULL, is_gimple_stmt, fb_none);
   18664              : }
   18665              : 
   18666              : 
   18667              : /* Helper function of optimize_target_teams, find OMP_TEAMS inside
   18668              :    of OMP_TARGET's body.  */
   18669              : 
   18670              : static tree
   18671        99356 : find_omp_teams (tree *tp, int *walk_subtrees, void *)
   18672              : {
   18673        99356 :   *walk_subtrees = 0;
   18674        99356 :   switch (TREE_CODE (*tp))
   18675              :     {
   18676              :     case OMP_TEAMS:
   18677              :       return *tp;
   18678        27418 :     case BIND_EXPR:
   18679        27418 :     case STATEMENT_LIST:
   18680        27418 :       *walk_subtrees = 1;
   18681        27418 :       break;
   18682              :     default:
   18683              :       break;
   18684              :     }
   18685              :   return NULL_TREE;
   18686              : }
   18687              : 
   18688              : /* Helper function of optimize_target_teams, determine if the expression
   18689              :    can be computed safely before the target construct on the host.  */
   18690              : 
   18691              : static tree
   18692         1186 : computable_teams_clause (tree *tp, int *walk_subtrees, void *)
   18693              : {
   18694         1706 :   splay_tree_node n;
   18695              : 
   18696         1706 :   if (TYPE_P (*tp))
   18697              :     {
   18698            0 :       *walk_subtrees = 0;
   18699            0 :       return NULL_TREE;
   18700              :     }
   18701         1706 :   switch (TREE_CODE (*tp))
   18702              :     {
   18703          858 :     case VAR_DECL:
   18704          858 :     case PARM_DECL:
   18705          858 :     case RESULT_DECL:
   18706          858 :       *walk_subtrees = 0;
   18707          858 :       if (error_operand_p (*tp)
   18708          858 :           || !INTEGRAL_TYPE_P (TREE_TYPE (*tp))
   18709          858 :           || DECL_HAS_VALUE_EXPR_P (*tp)
   18710          858 :           || DECL_THREAD_LOCAL_P (*tp)
   18711          858 :           || TREE_SIDE_EFFECTS (*tp)
   18712         1716 :           || TREE_THIS_VOLATILE (*tp))
   18713            0 :         return *tp;
   18714          858 :       if (is_global_var (*tp)
   18715          858 :           && (lookup_attribute ("omp declare target", DECL_ATTRIBUTES (*tp))
   18716            0 :               || lookup_attribute ("omp declare target link",
   18717            0 :                                    DECL_ATTRIBUTES (*tp))))
   18718           16 :         return *tp;
   18719          842 :       if (VAR_P (*tp)
   18720          536 :           && !DECL_SEEN_IN_BIND_EXPR_P (*tp)
   18721            0 :           && !is_global_var (*tp)
   18722          842 :           && decl_function_context (*tp) == current_function_decl)
   18723            0 :         return *tp;
   18724         1684 :       n = splay_tree_lookup (gimplify_omp_ctxp->variables,
   18725          842 :                              (splay_tree_key) *tp);
   18726          842 :       if (n == NULL)
   18727              :         {
   18728          210 :           if (gimplify_omp_ctxp->defaultmap[GDMK_SCALAR] & GOVD_FIRSTPRIVATE)
   18729              :             return NULL_TREE;
   18730            0 :           return *tp;
   18731              :         }
   18732          632 :       else if (n->value & GOVD_LOCAL)
   18733            0 :         return *tp;
   18734          632 :       else if (n->value & GOVD_FIRSTPRIVATE)
   18735              :         return NULL_TREE;
   18736          112 :       else if ((n->value & (GOVD_MAP | GOVD_MAP_ALWAYS_TO))
   18737              :                == (GOVD_MAP | GOVD_MAP_ALWAYS_TO))
   18738              :         return NULL_TREE;
   18739           96 :       return *tp;
   18740           76 :     case INTEGER_CST:
   18741           76 :       if (!INTEGRAL_TYPE_P (TREE_TYPE (*tp)))
   18742            0 :         return *tp;
   18743              :       return NULL_TREE;
   18744          520 :     case TARGET_EXPR:
   18745          520 :       if (TARGET_EXPR_INITIAL (*tp)
   18746          520 :           || TREE_CODE (TARGET_EXPR_SLOT (*tp)) != VAR_DECL)
   18747              :         return *tp;
   18748          520 :       return computable_teams_clause (&TARGET_EXPR_SLOT (*tp),
   18749          520 :                                       walk_subtrees, NULL);
   18750              :     /* Allow some reasonable subset of integral arithmetics.  */
   18751          193 :     case PLUS_EXPR:
   18752          193 :     case MINUS_EXPR:
   18753          193 :     case MULT_EXPR:
   18754          193 :     case TRUNC_DIV_EXPR:
   18755          193 :     case CEIL_DIV_EXPR:
   18756          193 :     case FLOOR_DIV_EXPR:
   18757          193 :     case ROUND_DIV_EXPR:
   18758          193 :     case TRUNC_MOD_EXPR:
   18759          193 :     case CEIL_MOD_EXPR:
   18760          193 :     case FLOOR_MOD_EXPR:
   18761          193 :     case ROUND_MOD_EXPR:
   18762          193 :     case RDIV_EXPR:
   18763          193 :     case EXACT_DIV_EXPR:
   18764          193 :     case MIN_EXPR:
   18765          193 :     case MAX_EXPR:
   18766          193 :     case LSHIFT_EXPR:
   18767          193 :     case RSHIFT_EXPR:
   18768          193 :     case BIT_IOR_EXPR:
   18769          193 :     case BIT_XOR_EXPR:
   18770          193 :     case BIT_AND_EXPR:
   18771          193 :     case NEGATE_EXPR:
   18772          193 :     case ABS_EXPR:
   18773          193 :     case BIT_NOT_EXPR:
   18774          193 :     case NON_LVALUE_EXPR:
   18775          193 :     CASE_CONVERT:
   18776          193 :       if (!INTEGRAL_TYPE_P (TREE_TYPE (*tp)))
   18777            0 :         return *tp;
   18778              :       return NULL_TREE;
   18779              :     /* And disallow anything else, except for comparisons.  */
   18780           59 :     default:
   18781           59 :       if (COMPARISON_CLASS_P (*tp))
   18782            0 :         return NULL_TREE;
   18783              :       return *tp;
   18784              :     }
   18785              : }
   18786              : 
   18787              : /* Try to determine if the num_teams and/or thread_limit expressions
   18788              :    can have their values determined already before entering the
   18789              :    target construct.
   18790              :    INTEGER_CSTs trivially are,
   18791              :    integral decls that are firstprivate (explicitly or implicitly)
   18792              :    or explicitly map(always, to:) or map(always, tofrom:) on the target
   18793              :    region too, and expressions involving simple arithmetics on those
   18794              :    too, function calls are not ok, dereferencing something neither etc.
   18795              :    Add NUM_TEAMS and THREAD_LIMIT clauses to the OMP_CLAUSES of
   18796              :    EXPR based on what we find:
   18797              :    0 stands for clause not specified at all, use implementation default
   18798              :    -1 stands for value that can't be determined easily before entering
   18799              :       the target construct.
   18800              :    -2 means that no explicit teams construct was specified
   18801              :    If teams construct is not present at all, use 1 for num_teams
   18802              :    and 0 for thread_limit (only one team is involved, and the thread
   18803              :    limit is implementation defined.  */
   18804              : 
   18805              : static void
   18806        13631 : optimize_target_teams (tree target, gimple_seq *pre_p)
   18807              : {
   18808              :   /* If the NUM_TEAMS clause is on TARGET, assume that the front end already
   18809              :      took care of the target-consistent clauses.  Note that user code cannot
   18810              :      specify 'num_teams' on 'target', only on 'teams'.  */
   18811        13631 :   if (omp_find_clause (OMP_TARGET_CLAUSES (target), OMP_CLAUSE_NUM_TEAMS))
   18812          114 :     return;
   18813              : 
   18814        13517 :   tree body = OMP_BODY (target);
   18815        13517 :   tree teams = walk_tree (&body, find_omp_teams, NULL, NULL);
   18816        13517 :   tree num_teams_lower = NULL_TREE;
   18817        13517 :   tree num_teams_upper = integer_zero_node;
   18818        13517 :   tree thread_limit = integer_zero_node;
   18819        13517 :   location_t num_teams_loc = EXPR_LOCATION (target);
   18820        13517 :   location_t thread_limit_loc = EXPR_LOCATION (target);
   18821        13517 :   tree c, *p, expr;
   18822        13517 :   struct gimplify_omp_ctx *target_ctx = gimplify_omp_ctxp;
   18823              : 
   18824        13517 :   if (teams == NULL_TREE)
   18825         7472 :     num_teams_upper = build_int_cst (integer_type_node, -2);
   18826              :   else
   18827        10129 :     for (c = OMP_TEAMS_CLAUSES (teams); c; c = OMP_CLAUSE_CHAIN (c))
   18828              :       {
   18829         4084 :         if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_TEAMS)
   18830              :           {
   18831          532 :             p = &num_teams_upper;
   18832          532 :             num_teams_loc = OMP_CLAUSE_LOCATION (c);
   18833          532 :             if (OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c))
   18834              :               {
   18835          136 :                 expr = OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c);
   18836          136 :                 if (TREE_CODE (expr) == INTEGER_CST)
   18837           18 :                   num_teams_lower = expr;
   18838          118 :                 else if (walk_tree (&expr, computable_teams_clause,
   18839              :                                     NULL, NULL))
   18840           12 :                   num_teams_lower = integer_minus_one_node;
   18841              :                 else
   18842              :                   {
   18843          106 :                     num_teams_lower = expr;
   18844          106 :                     gimplify_omp_ctxp = gimplify_omp_ctxp->outer_context;
   18845          106 :                     if (gimplify_expr (&num_teams_lower, pre_p, NULL,
   18846              :                                        is_gimple_val, fb_rvalue, false)
   18847              :                         == GS_ERROR)
   18848              :                       {
   18849            0 :                         gimplify_omp_ctxp = target_ctx;
   18850            0 :                         num_teams_lower = integer_minus_one_node;
   18851              :                       }
   18852              :                     else
   18853              :                       {
   18854          106 :                         gimplify_omp_ctxp = target_ctx;
   18855          106 :                         if (!DECL_P (expr) && TREE_CODE (expr) != TARGET_EXPR)
   18856           28 :                           OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c)
   18857           56 :                             = num_teams_lower;
   18858              :                       }
   18859              :                   }
   18860              :               }
   18861              :           }
   18862         3552 :         else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_THREAD_LIMIT)
   18863              :           {
   18864          419 :             p = &thread_limit;
   18865          419 :             thread_limit_loc = OMP_CLAUSE_LOCATION (c);
   18866              :           }
   18867              :         else
   18868         3133 :           continue;
   18869          951 :         expr = OMP_CLAUSE_OPERAND (c, 0);
   18870          951 :         if (TREE_CODE (expr) == INTEGER_CST)
   18871              :           {
   18872          152 :             *p = expr;
   18873          152 :             continue;
   18874              :           }
   18875          799 :         if (walk_tree (&expr, computable_teams_clause, NULL, NULL))
   18876              :           {
   18877          159 :             *p = integer_minus_one_node;
   18878          159 :             continue;
   18879              :           }
   18880          640 :         *p = expr;
   18881          640 :         gimplify_omp_ctxp = gimplify_omp_ctxp->outer_context;
   18882          640 :         if (gimplify_expr (p, pre_p, NULL, is_gimple_val, fb_rvalue, false)
   18883              :             == GS_ERROR)
   18884              :           {
   18885            0 :             gimplify_omp_ctxp = target_ctx;
   18886            0 :             *p = integer_minus_one_node;
   18887            0 :             continue;
   18888              :           }
   18889          640 :         gimplify_omp_ctxp = target_ctx;
   18890          640 :         if (!DECL_P (expr) && TREE_CODE (expr) != TARGET_EXPR)
   18891           48 :           OMP_CLAUSE_OPERAND (c, 0) = *p;
   18892              :       }
   18893        13517 :   if (!omp_find_clause (OMP_TARGET_CLAUSES (target), OMP_CLAUSE_THREAD_LIMIT))
   18894              :     {
   18895        13263 :       c = build_omp_clause (thread_limit_loc, OMP_CLAUSE_THREAD_LIMIT);
   18896        13263 :       OMP_CLAUSE_THREAD_LIMIT_EXPR (c) = thread_limit;
   18897        13263 :       OMP_CLAUSE_CHAIN (c) = OMP_TARGET_CLAUSES (target);
   18898        13263 :       OMP_TARGET_CLAUSES (target) = c;
   18899              :     }
   18900        13517 :   c = build_omp_clause (num_teams_loc, OMP_CLAUSE_NUM_TEAMS);
   18901        13517 :   OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (c) = num_teams_upper;
   18902        13517 :   OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c) = num_teams_lower;
   18903        13517 :   OMP_CLAUSE_CHAIN (c) = OMP_TARGET_CLAUSES (target);
   18904        13517 :   OMP_TARGET_CLAUSES (target) = c;
   18905              : }
   18906              : 
   18907              : /* Gimplify the gross structure of several OMP constructs.  */
   18908              : 
   18909              : static void
   18910        40168 : gimplify_omp_workshare (tree *expr_p, gimple_seq *pre_p)
   18911              : {
   18912        40168 :   tree expr = *expr_p;
   18913        40168 :   gimple *stmt;
   18914        40168 :   gimple_seq body = NULL;
   18915        40168 :   enum omp_region_type ort;
   18916              : 
   18917        40168 :   switch (TREE_CODE (expr))
   18918              :     {
   18919              :     case OMP_SECTIONS:
   18920              :     case OMP_SINGLE:
   18921              :       ort = ORT_WORKSHARE;
   18922              :       break;
   18923          242 :     case OMP_SCOPE:
   18924          242 :       ort = ORT_TASKGROUP;
   18925          242 :       break;
   18926        13631 :     case OMP_TARGET:
   18927        13631 :       ort = OMP_TARGET_COMBINED (expr) ? ORT_COMBINED_TARGET : ORT_TARGET;
   18928              :       break;
   18929         2543 :     case OACC_KERNELS:
   18930         2543 :       ort = ORT_ACC_KERNELS;
   18931         2543 :       break;
   18932         7901 :     case OACC_PARALLEL:
   18933         7901 :       ort = ORT_ACC_PARALLEL;
   18934         7901 :       break;
   18935         1040 :     case OACC_SERIAL:
   18936         1040 :       ort = ORT_ACC_SERIAL;
   18937         1040 :       break;
   18938         1950 :     case OACC_DATA:
   18939         1950 :       ort = ORT_ACC_DATA;
   18940         1950 :       break;
   18941         1875 :     case OMP_TARGET_DATA:
   18942         1875 :       ort = ORT_TARGET_DATA;
   18943         1875 :       break;
   18944         8935 :     case OMP_TEAMS:
   18945         8935 :       ort = OMP_TEAMS_COMBINED (expr) ? ORT_COMBINED_TEAMS : ORT_TEAMS;
   18946         8935 :       if (gimplify_omp_ctxp == NULL
   18947         6200 :           || gimplify_omp_ctxp->region_type == ORT_IMPLICIT_TARGET)
   18948         2735 :         ort = (enum omp_region_type) (ort | ORT_HOST_TEAMS);
   18949              :       break;
   18950          118 :     case OACC_HOST_DATA:
   18951          118 :       ort = ORT_ACC_HOST_DATA;
   18952          118 :       break;
   18953            0 :     default:
   18954            0 :       gcc_unreachable ();
   18955              :     }
   18956              : 
   18957        40168 :   gimple_seq iterator_loops_seq = NULL;
   18958        40168 :   if (TREE_CODE (expr) == OMP_TARGET || TREE_CODE (expr) == OMP_TARGET_DATA)
   18959              :     {
   18960        15506 :       remove_unused_omp_iterator_vars (&OMP_CLAUSES (expr));
   18961        15506 :       build_omp_iterators_loops (&OMP_CLAUSES (expr), &iterator_loops_seq);
   18962              :     }
   18963              : 
   18964        40168 :   bool save_in_omp_construct = in_omp_construct;
   18965        40168 :   if ((ort & ORT_ACC) == 0)
   18966        26616 :     in_omp_construct = false;
   18967        40168 :   gimplify_scan_omp_clauses (&OMP_CLAUSES (expr), pre_p, ort,
   18968        40168 :                              TREE_CODE (expr), &iterator_loops_seq);
   18969        40168 :   if (TREE_CODE (expr) == OMP_TARGET)
   18970        13631 :     optimize_target_teams (expr, pre_p);
   18971        40168 :   if ((ort & (ORT_TARGET | ORT_TARGET_DATA)) != 0
   18972        11110 :       || (ort & ORT_HOST_TEAMS) == ORT_HOST_TEAMS)
   18973              :     {
   18974        31793 :       push_gimplify_context ();
   18975        31793 :       gimple *g = gimplify_and_return_first (OMP_BODY (expr), &body);
   18976        31793 :       if (gimple_code (g) == GIMPLE_BIND)
   18977        31793 :         pop_gimplify_context (g);
   18978              :       else
   18979            0 :         pop_gimplify_context (NULL);
   18980        31793 :       if ((ort & ORT_TARGET_DATA) != 0)
   18981              :         {
   18982         3943 :           enum built_in_function end_ix;
   18983         3943 :           switch (TREE_CODE (expr))
   18984              :             {
   18985              :             case OACC_DATA:
   18986              :             case OACC_HOST_DATA:
   18987              :               end_ix = BUILT_IN_GOACC_DATA_END;
   18988              :               break;
   18989         1875 :             case OMP_TARGET_DATA:
   18990         1875 :               end_ix = BUILT_IN_GOMP_TARGET_END_DATA;
   18991         1875 :               break;
   18992            0 :             default:
   18993            0 :               gcc_unreachable ();
   18994              :             }
   18995         3943 :           tree fn = builtin_decl_explicit (end_ix);
   18996         3943 :           g = gimple_build_call (fn, 0);
   18997         3943 :           gimple_seq cleanup = NULL;
   18998         3943 :           gimple_seq_add_stmt (&cleanup, g);
   18999         3943 :           g = gimple_build_try (body, cleanup, GIMPLE_TRY_FINALLY);
   19000         3943 :           body = NULL;
   19001         3943 :           gimple_seq_add_stmt (&body, g);
   19002              :         }
   19003              :     }
   19004              :   else
   19005         8375 :     gimplify_and_add (OMP_BODY (expr), &body);
   19006        40168 :   gimplify_adjust_omp_clauses (pre_p, body, &OMP_CLAUSES (expr),
   19007        40168 :                                TREE_CODE (expr), &iterator_loops_seq);
   19008        40168 :   in_omp_construct = save_in_omp_construct;
   19009              : 
   19010        40168 :   switch (TREE_CODE (expr))
   19011              :     {
   19012         1950 :     case OACC_DATA:
   19013         3900 :       stmt = gimple_build_omp_target (body, GF_OMP_TARGET_KIND_OACC_DATA,
   19014         1950 :                                       OMP_CLAUSES (expr));
   19015         1950 :       break;
   19016          118 :     case OACC_HOST_DATA:
   19017          118 :       if (omp_find_clause (OMP_CLAUSES (expr), OMP_CLAUSE_IF_PRESENT))
   19018              :         {
   19019          128 :           for (tree c = OMP_CLAUSES (expr); c; c = OMP_CLAUSE_CHAIN (c))
   19020           95 :             if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_PTR)
   19021           57 :               OMP_CLAUSE_USE_DEVICE_PTR_IF_PRESENT (c) = 1;
   19022              :         }
   19023              : 
   19024          236 :       stmt = gimple_build_omp_target (body, GF_OMP_TARGET_KIND_OACC_HOST_DATA,
   19025          118 :                                       OMP_CLAUSES (expr));
   19026          118 :       break;
   19027         2543 :     case OACC_KERNELS:
   19028         5086 :       stmt = gimple_build_omp_target (body, GF_OMP_TARGET_KIND_OACC_KERNELS,
   19029         2543 :                                       OMP_CLAUSES (expr));
   19030         2543 :       break;
   19031         7901 :     case OACC_PARALLEL:
   19032        15802 :       stmt = gimple_build_omp_target (body, GF_OMP_TARGET_KIND_OACC_PARALLEL,
   19033         7901 :                                       OMP_CLAUSES (expr));
   19034         7901 :       break;
   19035         1040 :     case OACC_SERIAL:
   19036         2080 :       stmt = gimple_build_omp_target (body, GF_OMP_TARGET_KIND_OACC_SERIAL,
   19037         1040 :                                       OMP_CLAUSES (expr));
   19038         1040 :       break;
   19039          634 :     case OMP_SECTIONS:
   19040          634 :       stmt = gimple_build_omp_sections (body, OMP_CLAUSES (expr));
   19041          634 :       break;
   19042         1299 :     case OMP_SINGLE:
   19043         1299 :       stmt = gimple_build_omp_single (body, OMP_CLAUSES (expr));
   19044         1299 :       break;
   19045          242 :     case OMP_SCOPE:
   19046          242 :       stmt = gimple_build_omp_scope (body, OMP_CLAUSES (expr));
   19047          242 :       break;
   19048        13631 :     case OMP_TARGET:
   19049              :       /* Handle 'device_type(nohost)'.  */
   19050        13631 :       if (tree c = omp_find_clause (OMP_CLAUSES (expr), OMP_CLAUSE_DEVICE_TYPE))
   19051           20 :         if (OMP_CLAUSE_DEVICE_TYPE_KIND (c) == OMP_CLAUSE_DEVICE_TYPE_NOHOST)
   19052              :           {
   19053            9 :             location_t loc = gimple_location (body);
   19054            9 :             tree fn = builtin_decl_explicit (BUILT_IN_OMP_IS_INITIAL_DEVICE);
   19055            9 :             fn = build_call_expr_loc (loc, fn, 0);
   19056            9 :             tree err = builtin_decl_explicit (BUILT_IN_GOMP_ERROR);
   19057            9 :             const char *str = "Executing device-type 'nohost' target region "
   19058              :                               "on the host";
   19059            9 :             tree msg = build_string_literal (strlen (str) + 1, str);
   19060            9 :             tree msglen = build_int_cst (size_type_node, -2);
   19061            9 :             err = build_call_expr_loc (loc, err, 2, msg, msglen);
   19062            9 :             tree l1 = create_artificial_label (UNKNOWN_LOCATION);
   19063            9 :             tree l2 = create_artificial_label (UNKNOWN_LOCATION);
   19064            9 :             tree l3 = create_artificial_label (UNKNOWN_LOCATION);
   19065              : 
   19066            9 :             gimple_seq new_body = NULL;
   19067              : 
   19068            9 :             gimplify_expr (&fn, &new_body, NULL, is_gimple_val, fb_rvalue);
   19069            9 :             gcond *cond = gimple_build_cond (NE_EXPR, fn,
   19070            9 :                                              build_int_cst (TREE_TYPE (fn), 0),
   19071              :                                              l1, l2);
   19072            9 :             gimplify_seq_add_stmt (&new_body, cond);
   19073            9 :             gimplify_seq_add_stmt (&new_body, gimple_build_label (l1));
   19074            9 :             gimplify_and_add (err, &new_body);
   19075            9 :             gimplify_seq_add_stmt (&new_body, gimple_build_goto (l3));
   19076            9 :             gimplify_seq_add_stmt (&new_body, gimple_build_label (l2));
   19077            9 :             gimple_seq_add_seq (&new_body, body);
   19078            9 :             gimplify_seq_add_stmt (&new_body, gimple_build_label (l3));
   19079            9 :             body = gimple_build_bind (NULL_TREE, new_body, make_node (BLOCK));
   19080            9 :             gimple_set_location (body, loc);
   19081              :           }
   19082        27262 :       stmt = gimple_build_omp_target (body, GF_OMP_TARGET_KIND_REGION,
   19083        13631 :                                       OMP_CLAUSES (expr), iterator_loops_seq);
   19084        13631 :       break;
   19085         1875 :     case OMP_TARGET_DATA:
   19086              :       /* Put use_device_{ptr,addr} clauses last, as map clauses are supposed
   19087              :          to be evaluated before the use_device_{ptr,addr} clauses if they
   19088              :          refer to the same variables.  */
   19089         1875 :       {
   19090         1875 :         tree use_device_clauses;
   19091         1875 :         tree *pc, *uc = &use_device_clauses;
   19092         9974 :         for (pc = &OMP_CLAUSES (expr); *pc; )
   19093         8099 :           if (OMP_CLAUSE_CODE (*pc) == OMP_CLAUSE_USE_DEVICE_PTR
   19094         8099 :               || OMP_CLAUSE_CODE (*pc) == OMP_CLAUSE_USE_DEVICE_ADDR)
   19095              :             {
   19096         1959 :               *uc = *pc;
   19097         1959 :               *pc = OMP_CLAUSE_CHAIN (*pc);
   19098         1959 :               uc = &OMP_CLAUSE_CHAIN (*uc);
   19099              :             }
   19100              :           else
   19101         6140 :             pc = &OMP_CLAUSE_CHAIN (*pc);
   19102         1875 :         *uc = NULL_TREE;
   19103         1875 :         *pc = use_device_clauses;
   19104         1875 :         stmt = gimple_build_omp_target (body, GF_OMP_TARGET_KIND_DATA,
   19105         1875 :                                         OMP_CLAUSES (expr), iterator_loops_seq);
   19106              :       }
   19107         1875 :       break;
   19108         8935 :     case OMP_TEAMS:
   19109         8935 :       stmt = gimple_build_omp_teams (body, OMP_CLAUSES (expr));
   19110         8935 :       if ((ort & ORT_HOST_TEAMS) == ORT_HOST_TEAMS)
   19111         2735 :         gimple_omp_teams_set_host (as_a <gomp_teams *> (stmt), true);
   19112              :       break;
   19113            0 :     default:
   19114            0 :       gcc_unreachable ();
   19115              :     }
   19116              : 
   19117        40168 :   gimplify_seq_add_stmt (pre_p, stmt);
   19118        40168 :   *expr_p = NULL_TREE;
   19119        40168 : }
   19120              : 
   19121              : /* Gimplify the gross structure of OpenACC enter/exit data, update, and OpenMP
   19122              :    target update constructs.  */
   19123              : 
   19124              : static void
   19125        12382 : gimplify_omp_target_update (tree *expr_p, gimple_seq *pre_p)
   19126              : {
   19127        12382 :   tree expr = *expr_p;
   19128        12382 :   int kind;
   19129        12382 :   gomp_target *stmt;
   19130        12382 :   enum omp_region_type ort = ORT_WORKSHARE;
   19131              : 
   19132        12382 :   switch (TREE_CODE (expr))
   19133              :     {
   19134              :     case OACC_ENTER_DATA:
   19135              :       kind = GF_OMP_TARGET_KIND_OACC_ENTER_DATA;
   19136              :       ort = ORT_ACC;
   19137              :       break;
   19138              :     case OACC_EXIT_DATA:
   19139              :       kind = GF_OMP_TARGET_KIND_OACC_EXIT_DATA;
   19140              :       ort = ORT_ACC;
   19141              :       break;
   19142              :     case OACC_UPDATE:
   19143              :       kind = GF_OMP_TARGET_KIND_OACC_UPDATE;
   19144              :       ort = ORT_ACC;
   19145              :       break;
   19146              :     case OMP_TARGET_UPDATE:
   19147              :       kind = GF_OMP_TARGET_KIND_UPDATE;
   19148              :       break;
   19149              :     case OMP_TARGET_ENTER_DATA:
   19150              :       kind = GF_OMP_TARGET_KIND_ENTER_DATA;
   19151              :       break;
   19152              :     case OMP_TARGET_EXIT_DATA:
   19153              :       kind = GF_OMP_TARGET_KIND_EXIT_DATA;
   19154              :       break;
   19155            0 :     default:
   19156            0 :       gcc_unreachable ();
   19157              :     }
   19158              : 
   19159        12382 :   gimple_seq iterator_loops_seq = NULL;
   19160        12382 :   remove_unused_omp_iterator_vars (&OMP_STANDALONE_CLAUSES (expr));
   19161        12382 :   build_omp_iterators_loops (&OMP_STANDALONE_CLAUSES (expr),
   19162              :                              &iterator_loops_seq);
   19163              : 
   19164        12382 :   gimplify_scan_omp_clauses (&OMP_STANDALONE_CLAUSES (expr), pre_p,
   19165        12382 :                              ort, TREE_CODE (expr), &iterator_loops_seq);
   19166        12382 :   gimplify_adjust_omp_clauses (pre_p, NULL, &OMP_STANDALONE_CLAUSES (expr),
   19167        12382 :                                TREE_CODE (expr), &iterator_loops_seq);
   19168        12382 :   if (TREE_CODE (expr) == OACC_UPDATE
   19169        12382 :       && omp_find_clause (OMP_STANDALONE_CLAUSES (expr),
   19170              :                           OMP_CLAUSE_IF_PRESENT))
   19171              :     {
   19172              :       /* The runtime uses GOMP_MAP_{TO,FROM} to denote the if_present
   19173              :          clause.  */
   19174          123 :       for (tree c = OMP_STANDALONE_CLAUSES (expr); c; c = OMP_CLAUSE_CHAIN (c))
   19175           97 :         if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP)
   19176           47 :           switch (OMP_CLAUSE_MAP_KIND (c))
   19177              :             {
   19178           14 :             case GOMP_MAP_FORCE_TO:
   19179           14 :               OMP_CLAUSE_SET_MAP_KIND (c, GOMP_MAP_TO);
   19180           14 :               break;
   19181           24 :             case GOMP_MAP_FORCE_FROM:
   19182           24 :               OMP_CLAUSE_SET_MAP_KIND (c, GOMP_MAP_FROM);
   19183           24 :               break;
   19184              :             default:
   19185              :               break;
   19186              :             }
   19187              :     }
   19188        12356 :   else if (TREE_CODE (expr) == OACC_EXIT_DATA
   19189        12356 :            && omp_find_clause (OMP_STANDALONE_CLAUSES (expr),
   19190              :                                OMP_CLAUSE_FINALIZE))
   19191              :     {
   19192              :       /* Use GOMP_MAP_DELETE/GOMP_MAP_FORCE_FROM to denote "finalize"
   19193              :          semantics.  */
   19194           75 :       bool have_clause = false;
   19195          275 :       for (tree c = OMP_STANDALONE_CLAUSES (expr); c; c = OMP_CLAUSE_CHAIN (c))
   19196          200 :         if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP)
   19197          117 :           switch (OMP_CLAUSE_MAP_KIND (c))
   19198              :             {
   19199           45 :             case GOMP_MAP_FROM:
   19200           45 :               OMP_CLAUSE_SET_MAP_KIND (c, GOMP_MAP_FORCE_FROM);
   19201           45 :               have_clause = true;
   19202           45 :               break;
   19203           47 :             case GOMP_MAP_RELEASE:
   19204           47 :               OMP_CLAUSE_SET_MAP_KIND (c, GOMP_MAP_DELETE);
   19205           47 :               have_clause = true;
   19206           47 :               break;
   19207              :             case GOMP_MAP_TO_PSET:
   19208              :               /* Fortran arrays with descriptors must map that descriptor when
   19209              :                  doing standalone "attach" operations (in OpenACC).  In that
   19210              :                  case GOMP_MAP_TO_PSET appears by itself with no preceding
   19211              :                  clause (see trans-openmp.cc:gfc_trans_omp_clauses).  */
   19212              :               break;
   19213            2 :             case GOMP_MAP_POINTER:
   19214              :               /* TODO PR92929: we may see these here, but they'll always follow
   19215              :                  one of the clauses above, and will be handled by libgomp as
   19216              :                  one group, so no handling required here.  */
   19217            2 :               gcc_assert (have_clause);
   19218              :               break;
   19219           22 :             case GOMP_MAP_DETACH:
   19220           22 :               OMP_CLAUSE_SET_MAP_KIND (c, GOMP_MAP_FORCE_DETACH);
   19221           22 :               have_clause = false;
   19222           22 :               break;
   19223              :             case GOMP_MAP_STRUCT:
   19224              :             case GOMP_MAP_STRUCT_UNORD:
   19225          200 :               have_clause = false;
   19226              :               break;
   19227            0 :             default:
   19228            0 :               gcc_unreachable ();
   19229              :             }
   19230              :     }
   19231        12382 :   stmt = gimple_build_omp_target (NULL, kind, OMP_STANDALONE_CLAUSES (expr),
   19232              :                                   iterator_loops_seq);
   19233              : 
   19234        12382 :   gimplify_seq_add_stmt (pre_p, stmt);
   19235        12382 :   *expr_p = NULL_TREE;
   19236        12382 : }
   19237              : 
   19238              : /* A subroutine of gimplify_omp_atomic.  The front end is supposed to have
   19239              :    stabilized the lhs of the atomic operation as *ADDR.  Return true if
   19240              :    EXPR is this stabilized form.  */
   19241              : 
   19242              : static bool
   19243        36975 : goa_lhs_expr_p (tree expr, tree addr)
   19244              : {
   19245              :   /* Also include casts to other type variants.  The C front end is fond
   19246              :      of adding these for e.g. volatile variables.  This is like
   19247              :      STRIP_TYPE_NOPS but includes the main variant lookup.  */
   19248        36975 :   STRIP_USELESS_TYPE_CONVERSION (expr);
   19249              : 
   19250        36975 :   if (INDIRECT_REF_P (expr))
   19251              :     {
   19252         5044 :       expr = TREE_OPERAND (expr, 0);
   19253         5044 :       while (expr != addr
   19254           92 :              && (CONVERT_EXPR_P (expr)
   19255           92 :                  || TREE_CODE (expr) == NON_LVALUE_EXPR)
   19256            0 :              && TREE_CODE (expr) == TREE_CODE (addr)
   19257         5044 :              && types_compatible_p (TREE_TYPE (expr), TREE_TYPE (addr)))
   19258              :         {
   19259            0 :           expr = TREE_OPERAND (expr, 0);
   19260            0 :           addr = TREE_OPERAND (addr, 0);
   19261              :         }
   19262         5044 :       if (expr == addr)
   19263              :         return true;
   19264           92 :       return (TREE_CODE (addr) == ADDR_EXPR
   19265           62 :               && TREE_CODE (expr) == ADDR_EXPR
   19266           92 :               && TREE_OPERAND (addr, 0) == TREE_OPERAND (expr, 0));
   19267              :     }
   19268        31931 :   if (TREE_CODE (addr) == ADDR_EXPR && expr == TREE_OPERAND (addr, 0))
   19269              :     return true;
   19270              :   return false;
   19271              : }
   19272              : 
   19273              : /* Walk *EXPR_P and replace appearances of *LHS_ADDR with LHS_VAR.  If an
   19274              :    expression does not involve the lhs, evaluate it into a temporary.
   19275              :    Return 1 if the lhs appeared as a subexpression, 0 if it did not,
   19276              :    or -1 if an error was encountered.  */
   19277              : 
   19278              : static int
   19279        36975 : goa_stabilize_expr (tree *expr_p, gimple_seq *pre_p, tree lhs_addr,
   19280              :                     tree lhs_var, tree &target_expr, bool rhs, int depth)
   19281              : {
   19282        36975 :   tree expr = *expr_p;
   19283        36975 :   int saw_lhs = 0;
   19284              : 
   19285        36975 :   if (goa_lhs_expr_p (expr, lhs_addr))
   19286              :     {
   19287         9239 :       if (pre_p)
   19288         8580 :         *expr_p = lhs_var;
   19289              :       return 1;
   19290              :     }
   19291        27736 :   if (is_gimple_val (expr))
   19292              :     return 0;
   19293              : 
   19294              :   /* Maximum depth of lhs in expression is for the
   19295              :      __builtin_clear_padding (...), __builtin_clear_padding (...),
   19296              :      __builtin_memcmp (&TARGET_EXPR <lhs, >, ...) == 0 ? ... : lhs;  */
   19297        17335 :   if (++depth > 7)
   19298           16 :     goto finish;
   19299              : 
   19300        17319 :   switch (TREE_CODE_CLASS (TREE_CODE (expr)))
   19301              :     {
   19302         9705 :     case tcc_binary:
   19303         9705 :     case tcc_comparison:
   19304         9705 :       saw_lhs |= goa_stabilize_expr (&TREE_OPERAND (expr, 1), pre_p, lhs_addr,
   19305              :                                      lhs_var, target_expr, true, depth);
   19306              :       /* FALLTHRU */
   19307        11748 :     case tcc_unary:
   19308        11748 :       saw_lhs |= goa_stabilize_expr (&TREE_OPERAND (expr, 0), pre_p, lhs_addr,
   19309              :                                      lhs_var, target_expr, true, depth);
   19310        11748 :       break;
   19311         2983 :     case tcc_expression:
   19312         2983 :       switch (TREE_CODE (expr))
   19313              :         {
   19314          836 :         case TRUTH_ANDIF_EXPR:
   19315          836 :         case TRUTH_ORIF_EXPR:
   19316          836 :         case TRUTH_AND_EXPR:
   19317          836 :         case TRUTH_OR_EXPR:
   19318          836 :         case TRUTH_XOR_EXPR:
   19319          836 :         case BIT_INSERT_EXPR:
   19320          836 :           saw_lhs |= goa_stabilize_expr (&TREE_OPERAND (expr, 1), pre_p,
   19321              :                                          lhs_addr, lhs_var, target_expr, true,
   19322              :                                          depth);
   19323              :           /* FALLTHRU */
   19324          888 :         case TRUTH_NOT_EXPR:
   19325          888 :           saw_lhs |= goa_stabilize_expr (&TREE_OPERAND (expr, 0), pre_p,
   19326              :                                          lhs_addr, lhs_var, target_expr, true,
   19327              :                                          depth);
   19328          888 :           break;
   19329          550 :         case MODIFY_EXPR:
   19330          550 :           if (pre_p && !goa_stabilize_expr (expr_p, NULL, lhs_addr, lhs_var,
   19331              :                                             target_expr, true, depth))
   19332              :             break;
   19333          544 :           saw_lhs |= goa_stabilize_expr (&TREE_OPERAND (expr, 1), pre_p,
   19334              :                                          lhs_addr, lhs_var, target_expr, true,
   19335              :                                          depth);
   19336          544 :           saw_lhs |= goa_stabilize_expr (&TREE_OPERAND (expr, 0), pre_p,
   19337              :                                          lhs_addr, lhs_var, target_expr, false,
   19338              :                                          depth);
   19339          544 :           break;
   19340              :           /* FALLTHRU */
   19341          164 :         case ADDR_EXPR:
   19342          164 :           if (pre_p && !goa_stabilize_expr (expr_p, NULL, lhs_addr, lhs_var,
   19343              :                                             target_expr, true, depth))
   19344              :             break;
   19345          138 :           saw_lhs |= goa_stabilize_expr (&TREE_OPERAND (expr, 0), pre_p,
   19346              :                                          lhs_addr, lhs_var, target_expr, false,
   19347              :                                          depth);
   19348          138 :           break;
   19349              :         case COMPOUND_EXPR:
   19350              :           /* Break out any preevaluations from cp_build_modify_expr.  */
   19351           76 :           for (; TREE_CODE (expr) == COMPOUND_EXPR;
   19352           38 :                expr = TREE_OPERAND (expr, 1))
   19353              :             {
   19354              :               /* Special-case __builtin_clear_padding call before
   19355              :                  __builtin_memcmp.  */
   19356           38 :               if (TREE_CODE (TREE_OPERAND (expr, 0)) == CALL_EXPR)
   19357              :                 {
   19358            2 :                   tree fndecl = get_callee_fndecl (TREE_OPERAND (expr, 0));
   19359            2 :                   if (fndecl
   19360            2 :                       && fndecl_built_in_p (fndecl, BUILT_IN_CLEAR_PADDING)
   19361            0 :                       && VOID_TYPE_P (TREE_TYPE (TREE_OPERAND (expr, 0)))
   19362            2 :                       && (!pre_p
   19363            0 :                           || goa_stabilize_expr (&TREE_OPERAND (expr, 0), NULL,
   19364              :                                                  lhs_addr, lhs_var,
   19365              :                                                  target_expr, true, depth)))
   19366              :                     {
   19367            0 :                       if (pre_p)
   19368            0 :                         *expr_p = expr;
   19369            0 :                       saw_lhs = goa_stabilize_expr (&TREE_OPERAND (expr, 0),
   19370              :                                                     pre_p, lhs_addr, lhs_var,
   19371              :                                                     target_expr, true, depth);
   19372            0 :                       saw_lhs |= goa_stabilize_expr (&TREE_OPERAND (expr, 1),
   19373              :                                                      pre_p, lhs_addr, lhs_var,
   19374              :                                                      target_expr, rhs, depth);
   19375            0 :                       return saw_lhs;
   19376              :                     }
   19377              :                 }
   19378              : 
   19379           38 :               if (pre_p)
   19380           35 :                 gimplify_stmt (&TREE_OPERAND (expr, 0), pre_p);
   19381              :             }
   19382           38 :           if (!pre_p)
   19383            3 :             return goa_stabilize_expr (&expr, pre_p, lhs_addr, lhs_var,
   19384            3 :                                        target_expr, rhs, depth);
   19385           35 :           *expr_p = expr;
   19386           35 :           return goa_stabilize_expr (expr_p, pre_p, lhs_addr, lhs_var,
   19387           35 :                                      target_expr, rhs, depth);
   19388          442 :         case COND_EXPR:
   19389          442 :           if (!goa_stabilize_expr (&TREE_OPERAND (expr, 0), NULL, lhs_addr,
   19390              :                                    lhs_var, target_expr, true, depth))
   19391              :             break;
   19392          429 :           saw_lhs |= goa_stabilize_expr (&TREE_OPERAND (expr, 0), pre_p,
   19393              :                                          lhs_addr, lhs_var, target_expr, true,
   19394              :                                          depth);
   19395          429 :           saw_lhs |= goa_stabilize_expr (&TREE_OPERAND (expr, 1), pre_p,
   19396              :                                          lhs_addr, lhs_var, target_expr, true,
   19397              :                                          depth);
   19398          429 :           saw_lhs |= goa_stabilize_expr (&TREE_OPERAND (expr, 2), pre_p,
   19399              :                                          lhs_addr, lhs_var, target_expr, true,
   19400              :                                          depth);
   19401          429 :           break;
   19402          880 :         case TARGET_EXPR:
   19403          880 :           if (TARGET_EXPR_INITIAL (expr))
   19404              :             {
   19405          880 :               if (pre_p && !goa_stabilize_expr (expr_p, NULL, lhs_addr,
   19406              :                                                 lhs_var, target_expr, true,
   19407              :                                                 depth))
   19408              :                 break;
   19409          702 :               if (expr == target_expr)
   19410              :                 saw_lhs = 1;
   19411              :               else
   19412              :                 {
   19413          702 :                   saw_lhs = goa_stabilize_expr (&TARGET_EXPR_INITIAL (expr),
   19414              :                                                 pre_p, lhs_addr, lhs_var,
   19415              :                                                 target_expr, true, depth);
   19416          702 :                   if (saw_lhs && target_expr == NULL_TREE && pre_p)
   19417           26 :                     target_expr = expr;
   19418              :                 }
   19419              :             }
   19420              :           break;
   19421              :         default:
   19422              :           break;
   19423              :         }
   19424              :       break;
   19425          565 :     case tcc_reference:
   19426          565 :       if (TREE_CODE (expr) == BIT_FIELD_REF
   19427          499 :           || TREE_CODE (expr) == VIEW_CONVERT_EXPR)
   19428          362 :         saw_lhs |= goa_stabilize_expr (&TREE_OPERAND (expr, 0), pre_p,
   19429              :                                        lhs_addr, lhs_var, target_expr, true,
   19430              :                                        depth);
   19431              :       break;
   19432           58 :     case tcc_vl_exp:
   19433           58 :       if (TREE_CODE (expr) == CALL_EXPR)
   19434              :         {
   19435           58 :           if (tree fndecl = get_callee_fndecl (expr))
   19436           58 :             if (fndecl_built_in_p (fndecl, BUILT_IN_CLEAR_PADDING,
   19437              :                                            BUILT_IN_MEMCMP))
   19438              :               {
   19439           56 :                 int nargs = call_expr_nargs (expr);
   19440          224 :                 for (int i = 0; i < nargs; i++)
   19441          168 :                   saw_lhs |= goa_stabilize_expr (&CALL_EXPR_ARG (expr, i),
   19442              :                                                  pre_p, lhs_addr, lhs_var,
   19443              :                                                  target_expr, true, depth);
   19444              :               }
   19445              :         }
   19446              :       break;
   19447              :     default:
   19448              :       break;
   19449              :     }
   19450              : 
   19451        17297 :  finish:
   19452        17297 :   if (saw_lhs == 0 && pre_p)
   19453              :     {
   19454         3492 :       enum gimplify_status gs;
   19455         3492 :       if (TREE_CODE (expr) == CALL_EXPR && VOID_TYPE_P (TREE_TYPE (expr)))
   19456              :         {
   19457            0 :           gimplify_stmt (&expr, pre_p);
   19458            0 :           return saw_lhs;
   19459              :         }
   19460         3492 :       else if (rhs)
   19461         3314 :         gs = gimplify_expr (expr_p, pre_p, NULL, is_gimple_val, fb_rvalue);
   19462              :       else
   19463          178 :         gs = gimplify_expr (expr_p, pre_p, NULL, is_gimple_lvalue, fb_lvalue);
   19464         3492 :       if (gs != GS_ALL_DONE)
   19465        36975 :         saw_lhs = -1;
   19466              :     }
   19467              : 
   19468              :   return saw_lhs;
   19469              : }
   19470              : 
   19471              : /* Gimplify an OMP_ATOMIC statement.  */
   19472              : 
   19473              : static enum gimplify_status
   19474        10330 : gimplify_omp_atomic (tree *expr_p, gimple_seq *pre_p)
   19475              : {
   19476        10330 :   tree addr = TREE_OPERAND (*expr_p, 0);
   19477        10330 :   tree rhs = TREE_CODE (*expr_p) == OMP_ATOMIC_READ
   19478        10330 :              ? NULL : TREE_OPERAND (*expr_p, 1);
   19479        10330 :   tree type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
   19480        10330 :   tree tmp_load;
   19481        10330 :   gomp_atomic_load *loadstmt;
   19482        10330 :   gomp_atomic_store *storestmt;
   19483        10330 :   tree target_expr = NULL_TREE;
   19484              : 
   19485        10330 :   tmp_load = create_tmp_reg (type);
   19486        10330 :   if (rhs
   19487        10330 :       && goa_stabilize_expr (&rhs, pre_p, addr, tmp_load, target_expr,
   19488              :                              true, 0) < 0)
   19489              :     return GS_ERROR;
   19490              : 
   19491        10330 :   if (gimplify_expr (&addr, pre_p, NULL, is_gimple_val, fb_rvalue)
   19492              :       != GS_ALL_DONE)
   19493              :     return GS_ERROR;
   19494              : 
   19495        10330 :   loadstmt = gimple_build_omp_atomic_load (tmp_load, addr,
   19496        10330 :                                            OMP_ATOMIC_MEMORY_ORDER (*expr_p));
   19497        10330 :   gimplify_seq_add_stmt (pre_p, loadstmt);
   19498        10330 :   if (rhs)
   19499              :     {
   19500              :       /* BIT_INSERT_EXPR is not valid for non-integral bitfield
   19501              :          representatives.  Use BIT_FIELD_REF on the lhs instead.  */
   19502         9133 :       tree rhsarg = rhs;
   19503         9133 :       if (TREE_CODE (rhs) == COND_EXPR)
   19504          429 :         rhsarg = TREE_OPERAND (rhs, 1);
   19505         9133 :       if (TREE_CODE (rhsarg) == BIT_INSERT_EXPR
   19506         9133 :           && !INTEGRAL_TYPE_P (TREE_TYPE (tmp_load)))
   19507              :         {
   19508           32 :           tree bitpos = TREE_OPERAND (rhsarg, 2);
   19509           32 :           tree op1 = TREE_OPERAND (rhsarg, 1);
   19510           32 :           tree bitsize;
   19511           32 :           tree tmp_store = tmp_load;
   19512           32 :           if (TREE_CODE (*expr_p) == OMP_ATOMIC_CAPTURE_OLD)
   19513           12 :             tmp_store = get_initialized_tmp_var (tmp_load, pre_p);
   19514           32 :           if (INTEGRAL_TYPE_P (TREE_TYPE (op1)))
   19515           32 :             bitsize = bitsize_int (TYPE_PRECISION (TREE_TYPE (op1)));
   19516              :           else
   19517            0 :             bitsize = TYPE_SIZE (TREE_TYPE (op1));
   19518           32 :           gcc_assert (TREE_OPERAND (rhsarg, 0) == tmp_load);
   19519           32 :           tree t = build2_loc (EXPR_LOCATION (rhsarg),
   19520              :                                MODIFY_EXPR, void_type_node,
   19521           32 :                                build3_loc (EXPR_LOCATION (rhsarg),
   19522           32 :                                            BIT_FIELD_REF, TREE_TYPE (op1),
   19523              :                                            tmp_store, bitsize, bitpos), op1);
   19524           32 :           if (TREE_CODE (rhs) == COND_EXPR)
   19525           16 :             t = build3_loc (EXPR_LOCATION (rhs), COND_EXPR, void_type_node,
   19526           16 :                             TREE_OPERAND (rhs, 0), t, void_node);
   19527           32 :           gimplify_and_add (t, pre_p);
   19528           32 :           rhs = tmp_store;
   19529              :         }
   19530         9133 :       bool save_allow_rhs_cond_expr = gimplify_ctxp->allow_rhs_cond_expr;
   19531         9133 :       if (TREE_CODE (rhs) == COND_EXPR)
   19532          413 :         gimplify_ctxp->allow_rhs_cond_expr = true;
   19533         9133 :       enum gimplify_status gs = gimplify_expr (&rhs, pre_p, NULL,
   19534              :                                                is_gimple_val, fb_rvalue);
   19535         9133 :       gimplify_ctxp->allow_rhs_cond_expr = save_allow_rhs_cond_expr;
   19536         9133 :       if (gs != GS_ALL_DONE)
   19537              :         return GS_ERROR;
   19538              :     }
   19539              : 
   19540        10330 :   if (TREE_CODE (*expr_p) == OMP_ATOMIC_READ)
   19541         1197 :     rhs = tmp_load;
   19542        10330 :   storestmt
   19543        10330 :     = gimple_build_omp_atomic_store (rhs, OMP_ATOMIC_MEMORY_ORDER (*expr_p));
   19544        10330 :   if (TREE_CODE (*expr_p) != OMP_ATOMIC_READ && OMP_ATOMIC_WEAK (*expr_p))
   19545              :     {
   19546           38 :       gimple_omp_atomic_set_weak (loadstmt);
   19547           38 :       gimple_omp_atomic_set_weak (storestmt);
   19548              :     }
   19549        10330 :   gimplify_seq_add_stmt (pre_p, storestmt);
   19550        10330 :   switch (TREE_CODE (*expr_p))
   19551              :     {
   19552         2085 :     case OMP_ATOMIC_READ:
   19553         2085 :     case OMP_ATOMIC_CAPTURE_OLD:
   19554         2085 :       *expr_p = tmp_load;
   19555         2085 :       gimple_omp_atomic_set_need_value (loadstmt);
   19556         2085 :       break;
   19557          868 :     case OMP_ATOMIC_CAPTURE_NEW:
   19558          868 :       *expr_p = rhs;
   19559          868 :       gimple_omp_atomic_set_need_value (storestmt);
   19560          868 :       break;
   19561         7377 :     default:
   19562         7377 :       *expr_p = NULL;
   19563         7377 :       break;
   19564              :     }
   19565              : 
   19566              :   return GS_ALL_DONE;
   19567              : }
   19568              : 
   19569              : /* Gimplify a TRANSACTION_EXPR.  This involves gimplification of the
   19570              :    body, and adding some EH bits.  */
   19571              : 
   19572              : static enum gimplify_status
   19573          478 : gimplify_transaction (tree *expr_p, gimple_seq *pre_p)
   19574              : {
   19575          478 :   tree expr = *expr_p, temp, tbody = TRANSACTION_EXPR_BODY (expr);
   19576          478 :   gimple *body_stmt;
   19577          478 :   gtransaction *trans_stmt;
   19578          478 :   gimple_seq body = NULL;
   19579          478 :   int subcode = 0;
   19580              : 
   19581              :   /* Wrap the transaction body in a BIND_EXPR so we have a context
   19582              :      where to put decls for OMP.  */
   19583          478 :   if (TREE_CODE (tbody) != BIND_EXPR)
   19584              :     {
   19585          439 :       tree bind = build3 (BIND_EXPR, void_type_node, NULL, tbody, NULL);
   19586          439 :       TREE_SIDE_EFFECTS (bind) = 1;
   19587          439 :       SET_EXPR_LOCATION (bind, EXPR_LOCATION (tbody));
   19588          439 :       TRANSACTION_EXPR_BODY (expr) = bind;
   19589              :     }
   19590              : 
   19591          478 :   push_gimplify_context ();
   19592          478 :   temp = voidify_wrapper_expr (*expr_p, NULL);
   19593              : 
   19594          478 :   body_stmt = gimplify_and_return_first (TRANSACTION_EXPR_BODY (expr), &body);
   19595          478 :   pop_gimplify_context (body_stmt);
   19596              : 
   19597          478 :   trans_stmt = gimple_build_transaction (body);
   19598          478 :   if (TRANSACTION_EXPR_OUTER (expr))
   19599              :     subcode = GTMA_IS_OUTER;
   19600          447 :   else if (TRANSACTION_EXPR_RELAXED (expr))
   19601           81 :     subcode = GTMA_IS_RELAXED;
   19602          478 :   gimple_transaction_set_subcode (trans_stmt, subcode);
   19603              : 
   19604          478 :   gimplify_seq_add_stmt (pre_p, trans_stmt);
   19605              : 
   19606          478 :   if (temp)
   19607              :     {
   19608           76 :       *expr_p = temp;
   19609           76 :       return GS_OK;
   19610              :     }
   19611              : 
   19612              :   *expr_p = NULL_TREE;
   19613              :   return GS_ALL_DONE;
   19614              : }
   19615              : 
   19616              : /* Gimplify an OMP_ORDERED construct.  EXPR is the tree version.  BODY
   19617              :    is the OMP_BODY of the original EXPR (which has already been
   19618              :    gimplified so it's not present in the EXPR).
   19619              : 
   19620              :    Return the gimplified GIMPLE_OMP_ORDERED tuple.  */
   19621              : 
   19622              : static gimple *
   19623         1954 : gimplify_omp_ordered (tree expr, gimple_seq body)
   19624              : {
   19625         1954 :   tree c, decls;
   19626         1954 :   int failures = 0;
   19627         1954 :   unsigned int i;
   19628         1954 :   tree source_c = NULL_TREE;
   19629         1954 :   tree sink_c = NULL_TREE;
   19630              : 
   19631         1954 :   if (gimplify_omp_ctxp)
   19632              :     {
   19633         3291 :       for (c = OMP_ORDERED_CLAUSES (expr); c; c = OMP_CLAUSE_CHAIN (c))
   19634         1515 :         if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS
   19635         1515 :             && gimplify_omp_ctxp->loop_iter_var.is_empty ())
   19636              :           {
   19637           74 :             error_at (OMP_CLAUSE_LOCATION (c),
   19638              :                       "%<ordered%> construct with %qs clause must be "
   19639              :                       "closely nested inside a loop with %<ordered%> clause",
   19640           74 :                       OMP_CLAUSE_DOACROSS_DEPEND (c) ? "depend" : "doacross");
   19641           74 :             failures++;
   19642              :           }
   19643         1441 :         else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS
   19644         1441 :                  && OMP_CLAUSE_DOACROSS_KIND (c) == OMP_CLAUSE_DOACROSS_SINK)
   19645              :           {
   19646          642 :             bool fail = false;
   19647          642 :             sink_c = c;
   19648          642 :             if (OMP_CLAUSE_DECL (c) == NULL_TREE)
   19649           72 :               continue;  /* omp_cur_iteration - 1 */
   19650          570 :             for (decls = OMP_CLAUSE_DECL (c), i = 0;
   19651         3201 :                  decls && TREE_CODE (decls) == TREE_LIST;
   19652         2631 :                  decls = TREE_CHAIN (decls), ++i)
   19653         2631 :               if (i >= gimplify_omp_ctxp->loop_iter_var.length () / 2)
   19654            4 :                 continue;
   19655         5254 :               else if (TREE_VALUE (decls)
   19656         2627 :                        != gimplify_omp_ctxp->loop_iter_var[2 * i])
   19657              :                 {
   19658            8 :                   error_at (OMP_CLAUSE_LOCATION (c),
   19659              :                             "variable %qE is not an iteration "
   19660              :                             "of outermost loop %d, expected %qE",
   19661            8 :                             TREE_VALUE (decls), i + 1,
   19662            8 :                             gimplify_omp_ctxp->loop_iter_var[2 * i]);
   19663            8 :                   fail = true;
   19664            8 :                   failures++;
   19665              :                 }
   19666              :               else
   19667         5238 :                 TREE_VALUE (decls)
   19668         2619 :                   = gimplify_omp_ctxp->loop_iter_var[2 * i + 1];
   19669         1136 :             if (!fail && i != gimplify_omp_ctxp->loop_iter_var.length () / 2)
   19670              :               {
   19671           16 :                 error_at (OMP_CLAUSE_LOCATION (c),
   19672              :                           "number of variables in %qs clause with "
   19673              :                           "%<sink%> modifier does not match number of "
   19674              :                           "iteration variables",
   19675           16 :                           OMP_CLAUSE_DOACROSS_DEPEND (c)
   19676              :                           ? "depend" : "doacross");
   19677           16 :                 failures++;
   19678              :               }
   19679              :           }
   19680          799 :         else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DOACROSS
   19681          799 :                  && OMP_CLAUSE_DOACROSS_KIND (c) == OMP_CLAUSE_DOACROSS_SOURCE)
   19682              :           {
   19683          475 :             if (source_c)
   19684              :               {
   19685            4 :                 error_at (OMP_CLAUSE_LOCATION (c),
   19686              :                           "more than one %qs clause with %<source%> "
   19687              :                           "modifier on an %<ordered%> construct",
   19688            4 :                           OMP_CLAUSE_DOACROSS_DEPEND (source_c)
   19689              :                           ? "depend" : "doacross");
   19690            4 :                 failures++;
   19691              :               }
   19692              :             else
   19693              :               source_c = c;
   19694              :           }
   19695              :     }
   19696         1954 :   if (source_c && sink_c)
   19697              :     {
   19698            4 :       error_at (OMP_CLAUSE_LOCATION (source_c),
   19699              :                 "%qs clause with %<source%> modifier specified "
   19700              :                 "together with %qs clauses with %<sink%> modifier "
   19701              :                 "on the same construct",
   19702            4 :                 OMP_CLAUSE_DOACROSS_DEPEND (source_c) ? "depend" : "doacross",
   19703            4 :                 OMP_CLAUSE_DOACROSS_DEPEND (sink_c) ? "depend" : "doacross");
   19704            4 :       failures++;
   19705              :     }
   19706              : 
   19707         1954 :   if (failures)
   19708          102 :     return gimple_build_nop ();
   19709         1852 :   return gimple_build_omp_ordered (body, OMP_ORDERED_CLAUSES (expr));
   19710              : }
   19711              : 
   19712              : /* Gimplify an OMP_INTEROP statement.  */
   19713              : 
   19714              : static enum gimplify_status
   19715          612 : gimplify_omp_interop (tree *expr_p, gimple_seq *pre_p)
   19716              : {
   19717          612 :   tree expr = *expr_p;
   19718              : 
   19719          612 :   gimplify_scan_omp_clauses (&OMP_INTEROP_CLAUSES (expr), pre_p, ORT_TASK,
   19720              :                              OMP_INTEROP);
   19721          612 :   gimple *stmt = gimple_build_omp_interop (OMP_INTEROP_CLAUSES (expr));
   19722          612 :   gimplify_seq_add_stmt (pre_p, stmt);
   19723          612 :   *expr_p = NULL_TREE;
   19724          612 :   return GS_ALL_DONE;
   19725              : }
   19726              : 
   19727              : /* Callback for walk_tree to find an IFN_GOMP_DISPATCH.  */
   19728              : 
   19729              : static tree
   19730         3100 : find_ifn_gomp_dispatch (tree *tp, int *, void *modify)
   19731              : {
   19732         3100 :   tree t = *tp;
   19733              : 
   19734         3100 :   if (TREE_CODE (t) == CALL_EXPR && CALL_EXPR_IFN (t) == IFN_GOMP_DISPATCH)
   19735          852 :     return *(tree *) modify ? *(tree *) modify : *tp;
   19736              : 
   19737         2248 :   if (TREE_CODE (t) == MODIFY_EXPR)
   19738          513 :     *(tree *) modify = *tp;
   19739              : 
   19740              :   return NULL_TREE;
   19741              : }
   19742              : 
   19743              : /* Gimplify an OMP_DISPATCH construct.  */
   19744              : 
   19745              : static enum gimplify_status
   19746          852 : gimplify_omp_dispatch (tree *expr_p, gimple_seq *pre_p)
   19747              : {
   19748          852 :   tree expr = *expr_p;
   19749          852 :   gimple_seq body = NULL;
   19750              : 
   19751          852 :   gimplify_scan_omp_clauses (&OMP_DISPATCH_CLAUSES (expr), pre_p, ORT_DISPATCH,
   19752              :                              OMP_DISPATCH);
   19753          852 :   push_gimplify_context ();
   19754              : 
   19755              :   // If device clause, adjust ICV
   19756          852 :   tree device
   19757          852 :     = omp_find_clause (OMP_DISPATCH_CLAUSES (expr), OMP_CLAUSE_DEVICE);
   19758              :   // If no device clause exists but an interop clause with a single list
   19759              :   // item, use it to obtain the device number.
   19760          852 :   if (device)
   19761          272 :     device = OMP_CLAUSE_DEVICE_ID (device);
   19762              :   else
   19763              :     {
   19764          580 :       tree first_interop_obj
   19765          580 :         = omp_find_clause (OMP_DISPATCH_CLAUSES (expr), OMP_CLAUSE_INTEROP);
   19766          580 :       if (first_interop_obj)
   19767           96 :         for (tree c = TREE_CHAIN (first_interop_obj); c; c = TREE_CHAIN (c))
   19768            8 :           if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_INTEROP)
   19769              :             {
   19770              :               first_interop_obj = NULL_TREE;
   19771              :               break;
   19772              :             }
   19773           95 :       if (first_interop_obj)
   19774              :         {
   19775           88 :           device = create_tmp_var (integer_type_node);
   19776           88 :           tree c = build_omp_clause (OMP_CLAUSE_LOCATION (first_interop_obj),
   19777              :                                      OMP_CLAUSE_DEVICE);
   19778           88 :           OMP_CLAUSE_DEVICE_ID (c) = device;
   19779           88 :           TREE_CHAIN (c) = TREE_CHAIN (first_interop_obj);
   19780           88 :           TREE_CHAIN (first_interop_obj) = c;
   19781           88 :           first_interop_obj = OMP_CLAUSE_DECL (first_interop_obj);
   19782              :           /* device = omp_get_interop_int (obj, omp_ipr_device_num, NULL);  */
   19783           88 :           tree fn = builtin_decl_explicit (BUILT_IN_OMP_GET_INTEROP_INT);
   19784           88 :           fn = build_call_expr (fn, 3, first_interop_obj,
   19785              :                                 build_int_cst (integer_type_node, -5),
   19786              :                                 null_pointer_node);
   19787           88 :           gimplify_assign (device, fold_convert (integer_type_node, fn), &body);
   19788              :         }
   19789              :     }
   19790          852 :   tree saved_device_icv = NULL_TREE;
   19791          852 :   if (device
   19792          852 :       && (TREE_CODE (device) != INTEGER_CST
   19793          852 :           || !wi::eq_p (wi::to_wide (device), -1 /* omp_initial_device */)))
   19794              :     {
   19795              :       // Save current default-device-var ICV
   19796          360 :       saved_device_icv = create_tmp_var (integer_type_node);
   19797          360 :       tree fn = builtin_decl_explicit (BUILT_IN_OMP_GET_DEFAULT_DEVICE);
   19798          360 :       gcall *call = gimple_build_call (fn, 0);
   19799          360 :       gimple_call_set_lhs (call, saved_device_icv);
   19800          360 :       gimplify_seq_add_stmt (&body, call);
   19801              : 
   19802              :       // Set default device
   19803          360 :       fn = builtin_decl_explicit (BUILT_IN_OMP_SET_DEFAULT_DEVICE);
   19804          360 :       call = gimple_build_call (fn, 1, device);
   19805          360 :       gimplify_seq_add_stmt (&body, call);
   19806              :     }
   19807              : 
   19808              :   // If the novariants and nocontext clauses are not compile-time constants,
   19809              :   // we need to generate code for all possible cases:
   19810              :   //   if (novariants) // implies nocontext
   19811              :   //       base()
   19812              :   //   else if (nocontext)
   19813              :   //       variant1()
   19814              :   //   else
   19815              :   //       variant2()
   19816          852 :   tree *dispatch_body_p = &OMP_DISPATCH_BODY (expr);
   19817          852 :   if (TREE_CODE (*dispatch_body_p) == BIND_EXPR)
   19818           28 :     dispatch_body_p = &BIND_EXPR_BODY (*dispatch_body_p);
   19819          852 :   tree dispatch_body = *dispatch_body_p;
   19820              : 
   19821              :   // Look for IFN_GOMP_DISPATCH and extract the base function call
   19822          852 :   tree base_call_expr = NULL_TREE;
   19823          852 :   if (TREE_CODE (dispatch_body) == STATEMENT_LIST)
   19824          243 :     for (tree_stmt_iterator tsi = tsi_start (dispatch_body); !tsi_end_p (tsi);
   19825          187 :          tsi_next (&tsi))
   19826              :       {
   19827          243 :         tree modify = NULL_TREE;
   19828          243 :         tree stmt = tsi_stmt (tsi);
   19829          243 :         base_call_expr
   19830          243 :           = walk_tree (&stmt, find_ifn_gomp_dispatch, &modify, NULL);
   19831          243 :         if (base_call_expr != NULL_TREE)
   19832              :           break;
   19833              :       }
   19834              :   else
   19835              :     {
   19836          796 :       tree modify = NULL_TREE;
   19837          796 :       base_call_expr
   19838          796 :         = walk_tree (dispatch_body_p, find_ifn_gomp_dispatch, &modify, NULL);
   19839              :     }
   19840          852 :   gcc_assert (base_call_expr != NULL_TREE);
   19841              : 
   19842          852 :   tree dst = NULL_TREE;
   19843          852 :   if (TREE_CODE (base_call_expr) == MODIFY_EXPR)
   19844              :     {
   19845          290 :       dst = TREE_OPERAND (base_call_expr, 0);
   19846          290 :       base_call_expr = TREE_OPERAND (base_call_expr, 1);
   19847              :     }
   19848              : 
   19849          880 :   while (TREE_CODE (base_call_expr) == FLOAT_EXPR
   19850              :          || TREE_CODE (base_call_expr) == CONVERT_EXPR
   19851              :          || TREE_CODE (base_call_expr) == COMPLEX_EXPR
   19852              :          || TREE_CODE (base_call_expr) == INDIRECT_REF
   19853          880 :          || TREE_CODE (base_call_expr) == NOP_EXPR)
   19854           28 :     base_call_expr = TREE_OPERAND (base_call_expr, 0);
   19855              : 
   19856          852 :   gcc_assert (CALL_EXPR_IFN (base_call_expr) == IFN_GOMP_DISPATCH);
   19857          852 :   base_call_expr = CALL_EXPR_ARG (base_call_expr, 0);
   19858              : 
   19859          852 :   tree base_fndecl = get_callee_fndecl (base_call_expr);
   19860          852 :   if (base_fndecl != NULL_TREE)
   19861              :     {
   19862          844 :       if (DECL_VIRTUAL_P (base_fndecl))
   19863              :         {
   19864            6 :           error_at (
   19865            3 :             EXPR_LOCATION (base_call_expr),
   19866              :             "%qD is a virtual function but only a direct call is allowed "
   19867              :             "in a dispatch construct",
   19868            3 :             DECL_NAME (base_fndecl));
   19869              :         }
   19870              : 
   19871              :       /* We are not actually going to expand the variant call or use
   19872              :          the result of omp_get_dynamic candidates here; only check that
   19873              :          it does not trivially resolve to a call to the base function
   19874              :          so that we can avoid some extra work in building code that's
   19875              :          not needed in that case.  */
   19876          844 :       tree construct_context = omp_get_construct_context ();
   19877          844 :       vec<struct omp_variant> all_candidates
   19878          844 :         = omp_declare_variant_candidates (base_fndecl, construct_context);
   19879          844 :       gcc_assert (!all_candidates.is_empty ());
   19880          844 :       vec<struct omp_variant> candidates
   19881          844 :         = omp_get_dynamic_candidates (all_candidates, construct_context);
   19882          844 :       tree variant_fndecl
   19883         1676 :         = (candidates.length () == 1 ? candidates[0].alternative : NULL_TREE);
   19884              : 
   19885          844 :       if (base_fndecl != variant_fndecl
   19886          844 :           && (omp_has_novariants () == -1 || omp_has_nocontext () == -1))
   19887              :         {
   19888           22 :           tree novariants_clause = NULL_TREE, nocontext_clause = NULL_TREE,
   19889           22 :                novariants_cond = NULL_TREE, nocontext_cond = NULL_TREE;
   19890           59 :           for (tree c = OMP_DISPATCH_CLAUSES (expr); c; c = TREE_CHAIN (c))
   19891              :             {
   19892           37 :               if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOVARIANTS
   19893           37 :                   && !integer_zerop (OMP_CLAUSE_NOVARIANTS_EXPR (c)))
   19894              :                 {
   19895           16 :                   gcc_assert (novariants_cond == NULL_TREE);
   19896           16 :                   novariants_clause = c;
   19897           16 :                   novariants_cond = OMP_CLAUSE_NOVARIANTS_EXPR (c);
   19898              :                 }
   19899           21 :               else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOCONTEXT
   19900           21 :                        && !integer_zerop (OMP_CLAUSE_NOCONTEXT_EXPR (c)))
   19901              :                 {
   19902           16 :                   gcc_assert (nocontext_cond == NULL_TREE);
   19903           16 :                   nocontext_clause = c;
   19904           16 :                   nocontext_cond = OMP_CLAUSE_NOCONTEXT_EXPR (c);
   19905              :                 }
   19906              :             }
   19907           22 :           gcc_assert (novariants_cond != NULL_TREE
   19908              :                       || nocontext_cond != NULL_TREE);
   19909              : 
   19910           22 :           enum gimplify_status ret
   19911           22 :             = gimplify_expr (&novariants_cond, &body, NULL, is_gimple_val,
   19912              :                              fb_rvalue);
   19913           22 :           if (ret == GS_ERROR || ret == GS_UNHANDLED)
   19914            0 :             return ret;
   19915           22 :           ret = gimplify_expr (&nocontext_cond, &body, NULL, is_gimple_val,
   19916              :                                fb_rvalue);
   19917           22 :           if (ret == GS_ERROR || ret == GS_UNHANDLED)
   19918              :             return ret;
   19919              : 
   19920           22 :           tree end_label = create_artificial_label (UNKNOWN_LOCATION);
   19921              : 
   19922           22 :           if (novariants_cond != NULL_TREE)
   19923              :             {
   19924           16 :               tree base_label = create_artificial_label (UNKNOWN_LOCATION);
   19925           16 :               tree cond_label = create_artificial_label (UNKNOWN_LOCATION);
   19926           16 :               gcond *novariants_cond_stmt
   19927           16 :                 = gimple_build_cond_from_tree (novariants_cond, base_label,
   19928              :                                                cond_label);
   19929           16 :               gimplify_seq_add_stmt (&body, novariants_cond_stmt);
   19930              : 
   19931           16 :               gimplify_seq_add_stmt (&body, gimple_build_label (base_label));
   19932           16 :               tree base_call_expr2 = copy_node (base_call_expr);
   19933           16 :               base_call_expr2
   19934           16 :                 = build_call_expr_internal_loc (EXPR_LOCATION (base_call_expr2),
   19935              :                                                 IFN_GOMP_DISPATCH,
   19936           16 :                                                 TREE_TYPE (base_call_expr2), 1,
   19937              :                                                 base_call_expr2);
   19938           16 :               if (TREE_CODE (dispatch_body) == MODIFY_EXPR)
   19939              :                 {
   19940           16 :                   base_call_expr2 = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst,
   19941              :                                             base_call_expr2);
   19942              :                 }
   19943           16 :               OMP_CLAUSE_NOVARIANTS_EXPR (novariants_clause)
   19944           16 :                 = boolean_true_node;
   19945           16 :               gimplify_and_add (base_call_expr2, &body);
   19946           16 :               gimplify_seq_add_stmt (&body, gimple_build_goto (end_label));
   19947              : 
   19948           16 :               OMP_CLAUSE_NOVARIANTS_EXPR (novariants_clause)
   19949           16 :                 = boolean_false_node;
   19950           16 :               gimplify_seq_add_stmt (&body, gimple_build_label (cond_label));
   19951              :             }
   19952              : 
   19953           22 :           if (nocontext_cond != NULL_TREE)
   19954              :             {
   19955           16 :               tree variant1_label = create_artificial_label (UNKNOWN_LOCATION);
   19956           16 :               tree variant2_label = create_artificial_label (UNKNOWN_LOCATION);
   19957           16 :               gcond *nocontext_cond_stmt
   19958           16 :                 = gimple_build_cond_from_tree (nocontext_cond, variant1_label,
   19959              :                                                variant2_label);
   19960           16 :               gimplify_seq_add_stmt (&body, nocontext_cond_stmt);
   19961              : 
   19962           32 :               gimplify_seq_add_stmt (&body,
   19963           16 :                                      gimple_build_label (variant1_label));
   19964           16 :               tree variant_call_expr = copy_node (base_call_expr);
   19965           32 :               variant_call_expr = build_call_expr_internal_loc (
   19966           16 :                 EXPR_LOCATION (variant_call_expr), IFN_GOMP_DISPATCH,
   19967           16 :                 TREE_TYPE (variant_call_expr), 1, variant_call_expr);
   19968           16 :               if (TREE_CODE (dispatch_body) == MODIFY_EXPR)
   19969              :                 {
   19970           16 :                   variant_call_expr = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst,
   19971              :                                               variant_call_expr);
   19972              :                 }
   19973           16 :               OMP_CLAUSE_NOCONTEXT_EXPR (nocontext_clause) = boolean_true_node;
   19974           16 :               gimplify_and_add (variant_call_expr, &body);
   19975           16 :               gimplify_seq_add_stmt (&body, gimple_build_goto (end_label));
   19976           16 :               OMP_CLAUSE_NOCONTEXT_EXPR (nocontext_clause) = boolean_false_node;
   19977           32 :               gimplify_seq_add_stmt (&body,
   19978           16 :                                      gimple_build_label (variant2_label));
   19979              :             }
   19980              : 
   19981           22 :           tree variant_call_expr = base_call_expr;
   19982           22 :           variant_call_expr
   19983           22 :             = build_call_expr_internal_loc (EXPR_LOCATION (variant_call_expr),
   19984              :                                             IFN_GOMP_DISPATCH,
   19985           22 :                                             TREE_TYPE (variant_call_expr), 1,
   19986              :                                             variant_call_expr);
   19987           22 :           if (TREE_CODE (dispatch_body) == MODIFY_EXPR)
   19988              :             {
   19989           22 :               variant_call_expr
   19990           22 :                 = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, variant_call_expr);
   19991              :             }
   19992           22 :           gimplify_and_add (variant_call_expr, &body);
   19993           22 :           gimplify_seq_add_stmt (&body, gimple_build_label (end_label));
   19994              :         }
   19995              :       else
   19996          822 :         gimplify_and_add (OMP_DISPATCH_BODY (expr), &body);
   19997              :     }
   19998              :   else
   19999            8 :     gimplify_and_add (OMP_DISPATCH_BODY (expr), &body);
   20000              : 
   20001              :   // Restore default-device-var ICV
   20002          852 :   if (saved_device_icv != NULL_TREE)
   20003              :     {
   20004          360 :       tree fn = builtin_decl_explicit (BUILT_IN_OMP_SET_DEFAULT_DEVICE);
   20005          360 :       gcall *call = gimple_build_call (fn, 1, saved_device_icv);
   20006          360 :       gimplify_seq_add_stmt (&body, call);
   20007              :     }
   20008              : 
   20009              :   // Wrap dispatch body into a bind
   20010          852 :   gimple *bind = gimple_build_bind (NULL_TREE, body, NULL_TREE);
   20011          852 :   pop_gimplify_context (bind);
   20012              : 
   20013              :   // Manually tear down context created by gimplify_scan_omp_clauses to avoid a
   20014              :   // call to gimplify_adjust_omp_clauses
   20015          852 :   gimplify_omp_ctx *ctx = gimplify_omp_ctxp;
   20016          852 :   if (ctx != NULL)
   20017              :     {
   20018          852 :       gcc_assert (ctx->code == OMP_DISPATCH);
   20019          852 :       gimplify_omp_ctxp = ctx->outer_context;
   20020          852 :       delete_omp_context (ctx);
   20021              :     }
   20022              : 
   20023              :   // Remove nowait as it has no effect on dispatch (OpenMP 5.2), device as it
   20024              :   // has been handled above, and depend as the front end handled it by inserting
   20025              :   // taskwait.
   20026          852 :   tree *dispatch_clauses_ptr = &OMP_DISPATCH_CLAUSES (expr);
   20027         1445 :   for (tree c = *dispatch_clauses_ptr; c; c = *dispatch_clauses_ptr)
   20028              :     {
   20029          991 :       if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NOWAIT
   20030              :           || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
   20031              :           || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEVICE)
   20032              :         {
   20033          398 :           *dispatch_clauses_ptr = OMP_CLAUSE_CHAIN (c);
   20034          398 :           break;
   20035              :         }
   20036              :       else
   20037          593 :         dispatch_clauses_ptr = &OMP_CLAUSE_CHAIN (c);
   20038              :     }
   20039              : 
   20040          852 :   gimple *stmt = gimple_build_omp_dispatch (bind, OMP_DISPATCH_CLAUSES (expr));
   20041          852 :   gimplify_seq_add_stmt (pre_p, stmt);
   20042          852 :   *expr_p = NULL_TREE;
   20043          852 :   return GS_ALL_DONE;
   20044              : }
   20045              : 
   20046              : /* Expand a metadirective that has been resolved at gimplification time
   20047              :    into the candidate directive variants in CANDIDATES.  */
   20048              : 
   20049              : static enum gimplify_status
   20050          176 : expand_omp_metadirective (vec<struct omp_variant> &candidates,
   20051              :                           gimple_seq *pre_p)
   20052              : {
   20053          176 :   auto_vec<tree> selectors;
   20054          176 :   auto_vec<tree> directive_labels;
   20055          176 :   auto_vec<gimple_seq> directive_bodies;
   20056          176 :   tree body_label = NULL_TREE;
   20057          176 :   tree end_label = create_artificial_label (UNKNOWN_LOCATION);
   20058              : 
   20059              :   /* Construct bodies for each candidate.  */
   20060          608 :   for (unsigned i = 0; i < candidates.length(); i++)
   20061              :     {
   20062          256 :       struct omp_variant &candidate = candidates[i];
   20063          256 :       gimple_seq body = NULL;
   20064              : 
   20065          256 :       selectors.safe_push (omp_dynamic_cond (candidate.selector,
   20066              :                                              find_supercontext ()));
   20067          256 :       directive_labels.safe_push (create_artificial_label (UNKNOWN_LOCATION));
   20068              : 
   20069          256 :       gimplify_seq_add_stmt (&body,
   20070          256 :                              gimple_build_label (directive_labels.last ()));
   20071          256 :       if (candidate.alternative != NULL_TREE)
   20072          256 :         gimplify_stmt (&candidate.alternative, &body);
   20073          256 :       if (candidate.body != NULL_TREE)
   20074              :         {
   20075           36 :           if (body_label != NULL_TREE)
   20076            0 :             gimplify_seq_add_stmt (&body, gimple_build_goto (body_label));
   20077              :           else
   20078              :             {
   20079           36 :               body_label = create_artificial_label (UNKNOWN_LOCATION);
   20080           36 :               gimplify_seq_add_stmt (&body, gimple_build_label (body_label));
   20081           36 :               gimplify_stmt (&candidate.body, &body);
   20082              :             }
   20083              :         }
   20084              : 
   20085          256 :       directive_bodies.safe_push (body);
   20086              :     }
   20087              : 
   20088          176 :   auto_vec<tree> cond_labels;
   20089              : 
   20090          176 :   cond_labels.safe_push (NULL_TREE);
   20091          388 :   for (unsigned i = 1; i < candidates.length () - 1; i++)
   20092           18 :     cond_labels.safe_push (create_artificial_label (UNKNOWN_LOCATION));
   20093          176 :   if (candidates.length () > 1)
   20094           62 :     cond_labels.safe_push (directive_labels.last ());
   20095              : 
   20096              :   /* Generate conditionals to test each dynamic selector in turn, executing
   20097              :      the directive candidate if successful.  */
   20098          512 :   for (unsigned i = 0; i < candidates.length () - 1; i++)
   20099              :     {
   20100           80 :       if (i != 0)
   20101           18 :         gimplify_seq_add_stmt (pre_p, gimple_build_label (cond_labels [i]));
   20102              : 
   20103           80 :       enum gimplify_status ret = gimplify_expr (&selectors[i], pre_p, NULL,
   20104              :                                                 is_gimple_val, fb_rvalue);
   20105           80 :       if (ret == GS_ERROR || ret == GS_UNHANDLED)
   20106              :         return ret;
   20107              : 
   20108           80 :       gcond *cond_stmt
   20109           80 :         = gimple_build_cond_from_tree (selectors[i], directive_labels[i],
   20110           80 :                                        cond_labels[i + 1]);
   20111              : 
   20112           80 :       gimplify_seq_add_stmt (pre_p, cond_stmt);
   20113           80 :       gimplify_seq_add_seq (pre_p, directive_bodies[i]);
   20114           80 :       gimplify_seq_add_stmt (pre_p, gimple_build_goto (end_label));
   20115              :     }
   20116              : 
   20117          176 :   gimplify_seq_add_seq (pre_p, directive_bodies.last ());
   20118          176 :   gimplify_seq_add_stmt (pre_p, gimple_build_label (end_label));
   20119              : 
   20120          176 :   return GS_ALL_DONE;
   20121          176 : }
   20122              : 
   20123              : /* Expand a variant construct that requires late resolution in the ompdevlow
   20124              :    pass.  It's a bit easier to do this in tree form and then gimplify that,
   20125              :    than to emit gimple.  The output is going to look something like:
   20126              : 
   20127              :      switch_var = OMP_NEXT_VARIANT (0, state);
   20128              :      loop_label:
   20129              :      switch (switch_var)
   20130              :        {
   20131              :        case 1:
   20132              :          if (dynamic_selector_predicate_1)
   20133              :            {
   20134              :              alternative_1;
   20135              :              goto end_label;
   20136              :            }
   20137              :          else
   20138              :            {
   20139              :              switch_var = OMP_NEXT_VARIANT (1, state);
   20140              :              goto loop_label;
   20141              :            }
   20142              :         case 2:
   20143              :           ...
   20144              :         }
   20145              :       end_label:
   20146              : 
   20147              :   OMP_NEXT_VARIANT is a magic cookie that is replaced with the switch variable
   20148              :   index of the next variant to try, after late resolution.  */
   20149              : 
   20150              : static tree
   20151           16 : expand_late_variant_directive (vec<struct omp_variant> all_candidates,
   20152              :                                tree construct_context)
   20153              : {
   20154           16 :   tree body_label = NULL_TREE;
   20155           16 :   tree standalone_body = NULL_TREE;
   20156           16 :   tree loop_label = create_artificial_label (UNKNOWN_LOCATION);
   20157           16 :   tree end_label = create_artificial_label (UNKNOWN_LOCATION);
   20158           32 :   tree selectors = make_tree_vec (all_candidates.length ());
   20159           16 :   tree switch_body = NULL_TREE;
   20160           16 :   tree switch_var = create_tmp_var (integer_type_node, "variant");
   20161           16 :   tree state = tree_cons (NULL_TREE, construct_context, selectors);
   20162              : 
   20163          108 :   for (unsigned int i = 0; i < all_candidates.length (); i++)
   20164              :     {
   20165           76 :       tree selector = all_candidates[i].selector;
   20166           76 :       tree alternative = all_candidates[i].alternative;
   20167           76 :       tree body = all_candidates[i].body;
   20168           76 :       TREE_VEC_ELT (selectors, i) = selector;
   20169              : 
   20170              :       /* Case label.  Numbering is 1-based.  */
   20171           76 :       tree case_val = build_int_cst (integer_type_node, i + 1);
   20172           76 :       tree case_label
   20173           76 :         = build_case_label (case_val, NULL_TREE,
   20174              :                             create_artificial_label (UNKNOWN_LOCATION));
   20175           76 :       append_to_statement_list (case_label, &switch_body);
   20176              : 
   20177              :       /* The actual body of the variant.  */
   20178           76 :       tree variant_body = NULL_TREE;
   20179           76 :       append_to_statement_list (alternative, &variant_body);
   20180              : 
   20181           76 :       if (body != NULL_TREE)
   20182              :         {
   20183            0 :           if (standalone_body == NULL)
   20184              :             {
   20185            0 :               standalone_body = body;
   20186            0 :               body_label = create_artificial_label (UNKNOWN_LOCATION);
   20187              :             }
   20188            0 :           append_to_statement_list (build1 (GOTO_EXPR, void_type_node,
   20189              :                                             body_label),
   20190              :                                     &variant_body);
   20191              :         }
   20192              :       else
   20193           76 :         append_to_statement_list (build1 (GOTO_EXPR, void_type_node,
   20194              :                                           end_label),
   20195              :                                   &variant_body);
   20196              : 
   20197              :       /* If this is a dynamic selector, wrap variant_body with a conditional.
   20198              :          If the predicate doesn't match, the else clause sets switch_var and
   20199              :          jumps to loop_var to try again.  */
   20200           76 :       tree dynamic_selector = omp_dynamic_cond (selector, find_supercontext ());
   20201           76 :       if (dynamic_selector)
   20202              :         {
   20203           20 :           tree else_stmt = NULL_TREE;
   20204           20 :           tree next = build2 (OMP_NEXT_VARIANT, integer_type_node,
   20205              :                               case_val, state);
   20206           20 :           append_to_statement_list (build2 (MODIFY_EXPR, integer_type_node,
   20207              :                                             switch_var, next),
   20208              :                                     &else_stmt);
   20209           20 :           append_to_statement_list (build1 (GOTO_EXPR, void_type_node,
   20210              :                                             loop_label),
   20211              :                                     &else_stmt);
   20212           20 :           variant_body = build3 (COND_EXPR, void_type_node, dynamic_selector,
   20213              :                                  variant_body, else_stmt);
   20214              :         }
   20215           76 :       append_to_statement_list (variant_body, &switch_body);
   20216              :     }
   20217              : 
   20218              :   /* Put it all together.  */
   20219           16 :   tree result = NULL_TREE;
   20220           16 :   tree first = build2 (OMP_NEXT_VARIANT, integer_type_node, integer_zero_node,
   20221              :                        state);
   20222           16 :   append_to_statement_list (build2 (MODIFY_EXPR, integer_type_node,
   20223              :                                     switch_var, first),
   20224              :                             &result);
   20225           16 :   append_to_statement_list (build1 (LABEL_EXPR, void_type_node, loop_label),
   20226              :                             &result);
   20227           16 :   append_to_statement_list (build2 (SWITCH_EXPR, integer_type_node,
   20228              :                                     switch_var, switch_body),
   20229              :                             &result);
   20230           16 :   if (standalone_body)
   20231              :     {
   20232            0 :       append_to_statement_list (build1 (LABEL_EXPR, void_type_node,
   20233              :                                         body_label),
   20234              :                                 &result);
   20235            0 :       append_to_statement_list (standalone_body, &result);
   20236              :     }
   20237           16 :   append_to_statement_list (build1 (LABEL_EXPR, void_type_node, end_label),
   20238              :                             &result);
   20239           16 :   cgraph_node::get (cfun->decl)->has_omp_variant_constructs = 1;
   20240           16 :   return result;
   20241              : }
   20242              : 
   20243              : 
   20244              : /* Gimplify an OMP_METADIRECTIVE construct.   EXPR is the tree version.
   20245              :    The metadirective will be resolved at this point if possible, otherwise
   20246              :    a GIMPLE_OMP_VARIANT_CONSTRUCT is created.  */
   20247              : 
   20248              : static enum gimplify_status
   20249          180 : gimplify_omp_metadirective (tree *expr_p, gimple_seq *pre_p, gimple_seq *,
   20250              :                             bool (*) (tree), fallback_t)
   20251              : {
   20252              :   /* Try to resolve the metadirective.  */
   20253          180 :   tree construct_context = omp_get_construct_context ();
   20254          180 :   vec<struct omp_variant> all_candidates
   20255          180 :     = omp_metadirective_candidates (*expr_p, construct_context);
   20256          180 :   vec<struct omp_variant> candidates
   20257          180 :     = omp_get_dynamic_candidates (all_candidates, construct_context);
   20258          180 :   if (!candidates.is_empty ())
   20259          176 :     return expand_omp_metadirective (candidates, pre_p);
   20260              : 
   20261              :   /* The metadirective cannot be resolved yet.  Turn it into a loop with
   20262              :      a nested switch statement, using OMP_NEXT_VARIANT to set the control
   20263              :      variable for the switch.  */
   20264            4 :   *expr_p = expand_late_variant_directive (all_candidates, construct_context);
   20265            4 :   return GS_OK;
   20266              : }
   20267              : 
   20268              : /* Gimplify an OMP_DECLARE_MAPPER node (by just removing it).  */
   20269              : 
   20270              : static enum gimplify_status
   20271            0 : gimplify_omp_declare_mapper (tree *expr_p)
   20272              : {
   20273            0 :   *expr_p = NULL_TREE;
   20274            0 :   return GS_ALL_DONE;
   20275              : }
   20276              : 
   20277              : /* Convert the GENERIC expression tree *EXPR_P to GIMPLE.  If the
   20278              :    expression produces a value to be used as an operand inside a GIMPLE
   20279              :    statement, the value will be stored back in *EXPR_P.  This value will
   20280              :    be a tree of class tcc_declaration, tcc_constant, tcc_reference or
   20281              :    an SSA_NAME.  The corresponding sequence of GIMPLE statements is
   20282              :    emitted in PRE_P and POST_P.
   20283              : 
   20284              :    Additionally, this process may overwrite parts of the input
   20285              :    expression during gimplification.  Ideally, it should be
   20286              :    possible to do non-destructive gimplification.
   20287              : 
   20288              :    EXPR_P points to the GENERIC expression to convert to GIMPLE.  If
   20289              :       the expression needs to evaluate to a value to be used as
   20290              :       an operand in a GIMPLE statement, this value will be stored in
   20291              :       *EXPR_P on exit.  This happens when the caller specifies one
   20292              :       of fb_lvalue or fb_rvalue fallback flags.
   20293              : 
   20294              :    PRE_P will contain the sequence of GIMPLE statements corresponding
   20295              :        to the evaluation of EXPR and all the side-effects that must
   20296              :        be executed before the main expression.  On exit, the last
   20297              :        statement of PRE_P is the core statement being gimplified.  For
   20298              :        instance, when gimplifying 'if (++a)' the last statement in
   20299              :        PRE_P will be 'if (t.1)' where t.1 is the result of
   20300              :        pre-incrementing 'a'.
   20301              : 
   20302              :    POST_P will contain the sequence of GIMPLE statements corresponding
   20303              :        to the evaluation of all the side-effects that must be executed
   20304              :        after the main expression.  If this is NULL, the post
   20305              :        side-effects are stored at the end of PRE_P.
   20306              : 
   20307              :        The reason why the output is split in two is to handle post
   20308              :        side-effects explicitly.  In some cases, an expression may have
   20309              :        inner and outer post side-effects which need to be emitted in
   20310              :        an order different from the one given by the recursive
   20311              :        traversal.  For instance, for the expression (*p--)++ the post
   20312              :        side-effects of '--' must actually occur *after* the post
   20313              :        side-effects of '++'.  However, gimplification will first visit
   20314              :        the inner expression, so if a separate POST sequence was not
   20315              :        used, the resulting sequence would be:
   20316              : 
   20317              :             1   t.1 = *p
   20318              :             2   p = p - 1
   20319              :             3   t.2 = t.1 + 1
   20320              :             4   *p = t.2
   20321              : 
   20322              :        However, the post-decrement operation in line #2 must not be
   20323              :        evaluated until after the store to *p at line #4, so the
   20324              :        correct sequence should be:
   20325              : 
   20326              :             1   t.1 = *p
   20327              :             2   t.2 = t.1 + 1
   20328              :             3   *p = t.2
   20329              :             4   p = p - 1
   20330              : 
   20331              :        So, by specifying a separate post queue, it is possible
   20332              :        to emit the post side-effects in the correct order.
   20333              :        If POST_P is NULL, an internal queue will be used.  Before
   20334              :        returning to the caller, the sequence POST_P is appended to
   20335              :        the main output sequence PRE_P.
   20336              : 
   20337              :    GIMPLE_TEST_F points to a function that takes a tree T and
   20338              :        returns nonzero if T is in the GIMPLE form requested by the
   20339              :        caller.  The GIMPLE predicates are in gimple.cc.
   20340              : 
   20341              :    FALLBACK tells the function what sort of a temporary we want if
   20342              :        gimplification cannot produce an expression that complies with
   20343              :        GIMPLE_TEST_F.
   20344              : 
   20345              :        fb_none means that no temporary should be generated
   20346              :        fb_rvalue means that an rvalue is OK to generate
   20347              :        fb_lvalue means that an lvalue is OK to generate
   20348              :        fb_either means that either is OK, but an lvalue is preferable.
   20349              :        fb_mayfail means that gimplification may fail (in which case
   20350              :        GS_ERROR will be returned)
   20351              : 
   20352              :    The return value is either GS_ERROR or GS_ALL_DONE, since this
   20353              :    function iterates until EXPR is completely gimplified or an error
   20354              :    occurs.  */
   20355              : 
   20356              : enum gimplify_status
   20357    531689389 : gimplify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
   20358              :                bool (*gimple_test_f) (tree), fallback_t fallback)
   20359              : {
   20360    531689389 :   tree tmp;
   20361    531689389 :   gimple_seq internal_pre = NULL;
   20362    531689389 :   gimple_seq internal_post = NULL;
   20363    531689389 :   tree save_expr;
   20364    531689389 :   bool is_statement;
   20365    531689389 :   location_t saved_location;
   20366    531689389 :   enum gimplify_status ret;
   20367    531689389 :   gimple_stmt_iterator pre_last_gsi, post_last_gsi;
   20368    531689389 :   tree label;
   20369              : 
   20370    531689389 :   save_expr = *expr_p;
   20371    531689389 :   if (save_expr == NULL_TREE)
   20372              :     return GS_ALL_DONE;
   20373              : 
   20374              :   /* If we are gimplifying a top-level statement, PRE_P must be valid.  */
   20375    478212754 :   is_statement = gimple_test_f == is_gimple_stmt;
   20376    478212754 :   if (is_statement)
   20377    103596731 :     gcc_assert (pre_p);
   20378              : 
   20379              :   /* Consistency checks.  */
   20380    478212754 :   if (gimple_test_f == is_gimple_reg)
   20381      6466102 :     gcc_assert (fallback & (fb_rvalue | fb_lvalue));
   20382    471746652 :   else if (gimple_test_f == is_gimple_val
   20383    347866495 :            || gimple_test_f == is_gimple_call_addr
   20384    331020771 :            || gimple_test_f == is_gimple_condexpr_for_cond
   20385    325319244 :            || gimple_test_f == is_gimple_mem_rhs
   20386    325174593 :            || gimple_test_f == is_gimple_mem_rhs_or_call
   20387    313583904 :            || gimple_test_f == is_gimple_reg_rhs
   20388    312974861 :            || gimple_test_f == is_gimple_reg_rhs_or_call
   20389    239399757 :            || gimple_test_f == is_gimple_asm_val
   20390    239359703 :            || gimple_test_f == is_gimple_mem_ref_addr)
   20391    248178775 :     gcc_assert (fallback & fb_rvalue);
   20392    223567877 :   else if (gimple_test_f == is_gimple_min_lval
   20393    192534276 :            || gimple_test_f == is_gimple_lvalue)
   20394     85897908 :     gcc_assert (fallback & fb_lvalue);
   20395    137669969 :   else if (gimple_test_f == is_gimple_addressable)
   20396     34073238 :     gcc_assert (fallback & fb_either);
   20397    103596731 :   else if (gimple_test_f == is_gimple_stmt)
   20398    103596731 :     gcc_assert (fallback == fb_none);
   20399              :   else
   20400              :     {
   20401              :       /* We should have recognized the GIMPLE_TEST_F predicate to
   20402              :          know what kind of fallback to use in case a temporary is
   20403              :          needed to hold the value or address of *EXPR_P.  */
   20404            0 :       gcc_unreachable ();
   20405              :     }
   20406              : 
   20407              :   /* We used to check the predicate here and return immediately if it
   20408              :      succeeds.  This is wrong; the design is for gimplification to be
   20409              :      idempotent, and for the predicates to only test for valid forms, not
   20410              :      whether they are fully simplified.  */
   20411    478212754 :   if (pre_p == NULL)
   20412            0 :     pre_p = &internal_pre;
   20413              : 
   20414    478212754 :   if (post_p == NULL)
   20415    192490432 :     post_p = &internal_post;
   20416              : 
   20417              :   /* Remember the last statements added to PRE_P and POST_P.  Every
   20418              :      new statement added by the gimplification helpers needs to be
   20419              :      annotated with location information.  To centralize the
   20420              :      responsibility, we remember the last statement that had been
   20421              :      added to both queues before gimplifying *EXPR_P.  If
   20422              :      gimplification produces new statements in PRE_P and POST_P, those
   20423              :      statements will be annotated with the same location information
   20424              :      as *EXPR_P.  */
   20425    478212754 :   pre_last_gsi = gsi_last (*pre_p);
   20426    478212754 :   post_last_gsi = gsi_last (*post_p);
   20427              : 
   20428    478212754 :   saved_location = input_location;
   20429    478212754 :   if (save_expr != error_mark_node
   20430    478212754 :       && EXPR_HAS_LOCATION (*expr_p))
   20431    179331290 :     input_location = EXPR_LOCATION (*expr_p);
   20432              : 
   20433              :   /* Loop over the specific gimplifiers until the toplevel node
   20434              :      remains the same.  */
   20435    497092917 :   do
   20436              :     {
   20437              :       /* Strip away as many useless type conversions as possible
   20438              :          at the toplevel.  */
   20439    497092917 :       STRIP_USELESS_TYPE_CONVERSION (*expr_p);
   20440              : 
   20441              :       /* Remember the expr.  */
   20442    497092917 :       save_expr = *expr_p;
   20443              : 
   20444              :       /* Die, die, die, my darling.  */
   20445    497092917 :       if (error_operand_p (save_expr))
   20446              :         {
   20447              :           ret = GS_ERROR;
   20448              :           break;
   20449              :         }
   20450              : 
   20451              :       /* Do any language-specific gimplification.  */
   20452    497089201 :       ret = ((enum gimplify_status)
   20453    497089201 :              lang_hooks.gimplify_expr (expr_p, pre_p, post_p));
   20454    497089201 :       if (ret == GS_OK)
   20455              :         {
   20456     29723677 :           if (*expr_p == NULL_TREE)
   20457              :             break;
   20458     29723677 :           if (*expr_p != save_expr)
   20459      4969416 :             continue;
   20460              :         }
   20461    467365524 :       else if (ret != GS_UNHANDLED)
   20462              :         break;
   20463              : 
   20464              :       /* Make sure that all the cases set 'ret' appropriately.  */
   20465    491516964 :       ret = GS_UNHANDLED;
   20466    491516964 :       switch (TREE_CODE (*expr_p))
   20467              :         {
   20468              :           /* First deal with the special cases.  */
   20469              : 
   20470      1148204 :         case POSTINCREMENT_EXPR:
   20471      1148204 :         case POSTDECREMENT_EXPR:
   20472      1148204 :         case PREINCREMENT_EXPR:
   20473      1148204 :         case PREDECREMENT_EXPR:
   20474      2296408 :           ret = gimplify_self_mod_expr (expr_p, pre_p, post_p,
   20475              :                                         fallback != fb_none,
   20476      1148204 :                                         TREE_TYPE (*expr_p));
   20477      1148204 :           break;
   20478              : 
   20479       606652 :         case VIEW_CONVERT_EXPR:
   20480       606652 :           if ((fallback & fb_rvalue)
   20481       606538 :               && is_gimple_reg_type (TREE_TYPE (*expr_p))
   20482      1036100 :               && is_gimple_reg_type (TREE_TYPE (TREE_OPERAND (*expr_p, 0))))
   20483              :             {
   20484       417561 :               ret = gimplify_expr (&TREE_OPERAND (*expr_p, 0), pre_p,
   20485              :                                    post_p, is_gimple_val, fb_rvalue);
   20486       417561 :               recalculate_side_effects (*expr_p);
   20487       417561 :               break;
   20488              :             }
   20489              :           /* Fallthru.  */
   20490              : 
   20491     30970513 :         case ARRAY_REF:
   20492     30970513 :         case ARRAY_RANGE_REF:
   20493     30970513 :         case REALPART_EXPR:
   20494     30970513 :         case IMAGPART_EXPR:
   20495     30970513 :         case COMPONENT_REF:
   20496     30970513 :           ret = gimplify_compound_lval (expr_p, pre_p, post_p,
   20497              :                                         fallback ? fallback : fb_rvalue);
   20498     30970513 :           break;
   20499              : 
   20500      6326952 :         case COND_EXPR:
   20501      6326952 :           ret = gimplify_cond_expr (expr_p, pre_p, fallback);
   20502              : 
   20503              :           /* C99 code may assign to an array in a structure value of a
   20504              :              conditional expression, and this has undefined behavior
   20505              :              only on execution, so create a temporary if an lvalue is
   20506              :              required.  */
   20507      6326952 :           if (fallback == fb_lvalue)
   20508              :             {
   20509            7 :               *expr_p = get_initialized_tmp_var (*expr_p, pre_p, post_p, false);
   20510            7 :               mark_addressable (*expr_p);
   20511            7 :               ret = GS_OK;
   20512              :             }
   20513              :           break;
   20514              : 
   20515     17413258 :         case CALL_EXPR:
   20516     17413258 :           ret = gimplify_call_expr (expr_p, pre_p, fallback);
   20517              : 
   20518              :           /* C99 code may assign to an array in a structure returned
   20519              :              from a function, and this has undefined behavior only on
   20520              :              execution, so create a temporary if an lvalue is
   20521              :              required.  */
   20522     17413258 :           if (fallback == fb_lvalue)
   20523              :             {
   20524        17251 :               *expr_p = get_initialized_tmp_var (*expr_p, pre_p, post_p, false);
   20525        17251 :               mark_addressable (*expr_p);
   20526        17251 :               ret = GS_OK;
   20527              :             }
   20528              :           break;
   20529              : 
   20530            0 :         case TREE_LIST:
   20531            0 :           gcc_unreachable ();
   20532              : 
   20533            0 :         case OMP_ARRAY_SECTION:
   20534            0 :           gcc_unreachable ();
   20535              : 
   20536       505769 :         case COMPOUND_EXPR:
   20537       505769 :           ret = gimplify_compound_expr (expr_p, pre_p, fallback != fb_none);
   20538       505769 :           break;
   20539              : 
   20540        37437 :         case COMPOUND_LITERAL_EXPR:
   20541        37437 :           ret = gimplify_compound_literal_expr (expr_p, pre_p,
   20542              :                                                 gimple_test_f, fallback);
   20543        37437 :           break;
   20544              : 
   20545     51431604 :         case MODIFY_EXPR:
   20546     51431604 :         case INIT_EXPR:
   20547     51431604 :           ret = gimplify_modify_expr (expr_p, pre_p, post_p,
   20548              :                                       fallback != fb_none);
   20549     51431604 :           break;
   20550              : 
   20551       115565 :         case TRUTH_ANDIF_EXPR:
   20552       115565 :         case TRUTH_ORIF_EXPR:
   20553       115565 :           {
   20554              :             /* Preserve the original type of the expression and the
   20555              :                source location of the outer expression.  */
   20556       115565 :             tree org_type = TREE_TYPE (*expr_p);
   20557       115565 :             *expr_p = gimple_boolify (*expr_p);
   20558       115565 :             *expr_p = build3_loc (input_location, COND_EXPR,
   20559              :                                   org_type, *expr_p,
   20560              :                                   fold_convert_loc
   20561              :                                     (input_location,
   20562              :                                      org_type, boolean_true_node),
   20563              :                                   fold_convert_loc
   20564              :                                     (input_location,
   20565              :                                      org_type, boolean_false_node));
   20566       115565 :             ret = GS_OK;
   20567       115565 :             break;
   20568              :           }
   20569              : 
   20570       238136 :         case TRUTH_NOT_EXPR:
   20571       238136 :           {
   20572       238136 :             tree type = TREE_TYPE (*expr_p);
   20573              :             /* The parsers are careful to generate TRUTH_NOT_EXPR
   20574              :                only with operands that are always zero or one.
   20575              :                We do not fold here but handle the only interesting case
   20576              :                manually, as fold may re-introduce the TRUTH_NOT_EXPR.  */
   20577       238136 :             *expr_p = gimple_boolify (*expr_p);
   20578       238136 :             if (TYPE_PRECISION (TREE_TYPE (*expr_p)) == 1)
   20579       238136 :               *expr_p = build1_loc (input_location, BIT_NOT_EXPR,
   20580       238136 :                                     TREE_TYPE (*expr_p),
   20581       238136 :                                     TREE_OPERAND (*expr_p, 0));
   20582              :             else
   20583            0 :               *expr_p = build2_loc (input_location, BIT_XOR_EXPR,
   20584            0 :                                     TREE_TYPE (*expr_p),
   20585            0 :                                     TREE_OPERAND (*expr_p, 0),
   20586            0 :                                     build_int_cst (TREE_TYPE (*expr_p), 1));
   20587       238136 :             if (!useless_type_conversion_p (type, TREE_TYPE (*expr_p)))
   20588         6538 :               *expr_p = fold_convert_loc (input_location, type, *expr_p);
   20589              :             ret = GS_OK;
   20590              :             break;
   20591              :           }
   20592              : 
   20593     34156475 :         case ADDR_EXPR:
   20594     34156475 :           ret = gimplify_addr_expr (expr_p, pre_p, post_p);
   20595     34156475 :           break;
   20596              : 
   20597         5783 :         case ANNOTATE_EXPR:
   20598         5783 :           {
   20599         5783 :             tree cond = TREE_OPERAND (*expr_p, 0);
   20600         5783 :             tree kind = TREE_OPERAND (*expr_p, 1);
   20601         5783 :             tree data = TREE_OPERAND (*expr_p, 2);
   20602         5783 :             tree type = TREE_TYPE (cond);
   20603         5783 :             if (!INTEGRAL_TYPE_P (type))
   20604              :               {
   20605            0 :                 *expr_p = cond;
   20606            0 :                 ret = GS_OK;
   20607            0 :                 break;
   20608              :               }
   20609         5783 :             tree tmp = create_tmp_var (type);
   20610         5783 :             gimplify_arg (&cond, pre_p, EXPR_LOCATION (*expr_p));
   20611         5783 :             gcall *call
   20612         5783 :               = gimple_build_call_internal (IFN_ANNOTATE, 3, cond, kind, data);
   20613         5783 :             gimple_call_set_lhs (call, tmp);
   20614         5783 :             gimplify_seq_add_stmt (pre_p, call);
   20615         5783 :             *expr_p = tmp;
   20616         5783 :             ret = GS_ALL_DONE;
   20617         5783 :             break;
   20618              :           }
   20619              : 
   20620        51217 :         case VA_ARG_EXPR:
   20621        51217 :           ret = gimplify_va_arg_expr (expr_p, pre_p, post_p);
   20622        51217 :           break;
   20623              : 
   20624     17829711 :         CASE_CONVERT:
   20625     17829711 :           if (IS_EMPTY_STMT (*expr_p))
   20626              :             {
   20627              :               ret = GS_ALL_DONE;
   20628              :               break;
   20629              :             }
   20630              : 
   20631     16063762 :           if (VOID_TYPE_P (TREE_TYPE (*expr_p))
   20632     16063762 :               || fallback == fb_none)
   20633              :             {
   20634              :               /* Just strip a conversion to void (or in void context) and
   20635              :                  try again.  */
   20636      2736680 :               *expr_p = TREE_OPERAND (*expr_p, 0);
   20637      2736680 :               ret = GS_OK;
   20638      2736680 :               break;
   20639              :             }
   20640              : 
   20641     13327082 :           ret = gimplify_conversion (expr_p);
   20642     13327082 :           if (ret == GS_ERROR)
   20643              :             break;
   20644     13327082 :           if (*expr_p != save_expr)
   20645              :             break;
   20646              :           /* FALLTHRU */
   20647              : 
   20648     13455622 :         case FIX_TRUNC_EXPR:
   20649              :           /* unary_expr: ... | '(' cast ')' val | ...  */
   20650     13455622 :           ret = gimplify_expr (&TREE_OPERAND (*expr_p, 0), pre_p, post_p,
   20651              :                                is_gimple_val, fb_rvalue);
   20652     13455622 :           recalculate_side_effects (*expr_p);
   20653     13455622 :           break;
   20654              : 
   20655      6396278 :         case INDIRECT_REF:
   20656      6396278 :           {
   20657      6396278 :             bool volatilep = TREE_THIS_VOLATILE (*expr_p);
   20658      6396278 :             bool notrap = TREE_THIS_NOTRAP (*expr_p);
   20659      6396278 :             tree saved_ptr_type = TREE_TYPE (TREE_OPERAND (*expr_p, 0));
   20660              : 
   20661      6396278 :             *expr_p = fold_indirect_ref_loc (input_location, *expr_p);
   20662      6396278 :             if (*expr_p != save_expr)
   20663              :               {
   20664              :                 ret = GS_OK;
   20665              :                 break;
   20666              :               }
   20667              : 
   20668      6377217 :             ret = gimplify_expr (&TREE_OPERAND (*expr_p, 0), pre_p, post_p,
   20669              :                                  is_gimple_reg, fb_rvalue);
   20670      6377217 :             if (ret == GS_ERROR)
   20671              :               break;
   20672              : 
   20673      6377216 :             recalculate_side_effects (*expr_p);
   20674     12754432 :             *expr_p = fold_build2_loc (input_location, MEM_REF,
   20675      6377216 :                                        TREE_TYPE (*expr_p),
   20676      6377216 :                                        TREE_OPERAND (*expr_p, 0),
   20677              :                                        build_int_cst (saved_ptr_type, 0));
   20678      6377216 :             TREE_THIS_VOLATILE (*expr_p) = volatilep;
   20679      6377216 :             TREE_THIS_NOTRAP (*expr_p) = notrap;
   20680      6377216 :             ret = GS_OK;
   20681      6377216 :             break;
   20682              :           }
   20683              : 
   20684              :         /* We arrive here through the various re-gimplifcation paths.  */
   20685     16249916 :         case MEM_REF:
   20686              :           /* First try re-folding the whole thing.  */
   20687     16249916 :           tmp = fold_binary (MEM_REF, TREE_TYPE (*expr_p),
   20688              :                              TREE_OPERAND (*expr_p, 0),
   20689              :                              TREE_OPERAND (*expr_p, 1));
   20690     16249916 :           if (tmp)
   20691              :             {
   20692         6584 :               REF_REVERSE_STORAGE_ORDER (tmp)
   20693         3292 :                 = REF_REVERSE_STORAGE_ORDER (*expr_p);
   20694         3292 :               *expr_p = tmp;
   20695         3292 :               recalculate_side_effects (*expr_p);
   20696         3292 :               ret = GS_OK;
   20697         3292 :               break;
   20698              :             }
   20699              :           /* Avoid re-gimplifying the address operand if it is already
   20700              :              in suitable form.  Re-gimplifying would mark the address
   20701              :              operand addressable.  Always gimplify when not in SSA form
   20702              :              as we still may have to gimplify decls with value-exprs.  */
   20703     16246624 :           if (!gimplify_ctxp || !gimple_in_ssa_p (cfun)
   20704     17268038 :               || !is_gimple_mem_ref_addr (TREE_OPERAND (*expr_p, 0)))
   20705              :             {
   20706     15280402 :               ret = gimplify_expr (&TREE_OPERAND (*expr_p, 0), pre_p, post_p,
   20707              :                                    is_gimple_mem_ref_addr, fb_rvalue);
   20708     15280402 :               if (ret == GS_ERROR)
   20709              :                 break;
   20710              :             }
   20711     16246624 :           recalculate_side_effects (*expr_p);
   20712     16246624 :           ret = GS_ALL_DONE;
   20713     16246624 :           break;
   20714              : 
   20715              :         /* Constants need not be gimplified.  */
   20716     45364873 :         case INTEGER_CST:
   20717     45364873 :         case REAL_CST:
   20718     45364873 :         case FIXED_CST:
   20719     45364873 :         case STRING_CST:
   20720     45364873 :         case COMPLEX_CST:
   20721     45364873 :         case VECTOR_CST:
   20722              :           /* Drop the overflow flag on constants, we do not want
   20723              :              that in the GIMPLE IL.  */
   20724     45364873 :           if (TREE_OVERFLOW_P (*expr_p))
   20725          887 :             *expr_p = drop_tree_overflow (*expr_p);
   20726              :           ret = GS_ALL_DONE;
   20727              :           break;
   20728              : 
   20729       118680 :         case CONST_DECL:
   20730              :           /* If we require an lvalue, such as for ADDR_EXPR, retain the
   20731              :              CONST_DECL node.  Otherwise the decl is replaceable by its
   20732              :              value.  */
   20733              :           /* ??? Should be == fb_lvalue, but ADDR_EXPR passes fb_either.  */
   20734       118680 :           if (fallback & fb_lvalue)
   20735              :             ret = GS_ALL_DONE;
   20736              :           else
   20737              :             {
   20738         3310 :               *expr_p = DECL_INITIAL (*expr_p);
   20739         3310 :               ret = GS_OK;
   20740              :             }
   20741              :           break;
   20742              : 
   20743      6653752 :         case DECL_EXPR:
   20744      6653752 :           ret = gimplify_decl_expr (expr_p, pre_p);
   20745      6653752 :           break;
   20746              : 
   20747      6095400 :         case BIND_EXPR:
   20748      6095400 :           ret = gimplify_bind_expr (expr_p, pre_p);
   20749      6095400 :           break;
   20750              : 
   20751       210185 :         case LOOP_EXPR:
   20752       210185 :           ret = gimplify_loop_expr (expr_p, pre_p);
   20753       210185 :           break;
   20754              : 
   20755        50807 :         case SWITCH_EXPR:
   20756        50807 :           ret = gimplify_switch_expr (expr_p, pre_p);
   20757        50807 :           break;
   20758              : 
   20759         3411 :         case EXIT_EXPR:
   20760         3411 :           ret = gimplify_exit_expr (expr_p);
   20761         3411 :           break;
   20762              : 
   20763      1127324 :         case GOTO_EXPR:
   20764              :           /* If the target is not LABEL, then it is a computed jump
   20765              :              and the target needs to be gimplified.  */
   20766      1127324 :           if (TREE_CODE (GOTO_DESTINATION (*expr_p)) != LABEL_DECL)
   20767              :             {
   20768         1251 :               ret = gimplify_expr (&GOTO_DESTINATION (*expr_p), pre_p,
   20769              :                                    NULL, is_gimple_val, fb_rvalue);
   20770         1251 :               if (ret == GS_ERROR)
   20771              :                 break;
   20772              :             }
   20773      2254646 :           gimplify_seq_add_stmt (pre_p,
   20774      1127323 :                           gimple_build_goto (GOTO_DESTINATION (*expr_p)));
   20775      1127323 :           ret = GS_ALL_DONE;
   20776      1127323 :           break;
   20777              : 
   20778       104759 :         case PREDICT_EXPR:
   20779       419036 :           gimplify_seq_add_stmt (pre_p,
   20780       104759 :                         gimple_build_predict (PREDICT_EXPR_PREDICTOR (*expr_p),
   20781       104759 :                                               PREDICT_EXPR_OUTCOME (*expr_p)));
   20782       104759 :           ret = GS_ALL_DONE;
   20783       104759 :           break;
   20784              : 
   20785      2628015 :         case LABEL_EXPR:
   20786      2628015 :           ret = gimplify_label_expr (expr_p, pre_p);
   20787      2628015 :           label = LABEL_EXPR_LABEL (*expr_p);
   20788      2628015 :           gcc_assert (decl_function_context (label) == current_function_decl);
   20789              : 
   20790              :           /* If the label is used in a goto statement, or address of the label
   20791              :              is taken, we need to unpoison all variables that were seen so far.
   20792              :              Doing so would prevent us from reporting a false positives.  */
   20793      2628015 :           if (asan_poisoned_variables
   20794         5309 :               && asan_used_labels != NULL
   20795         1325 :               && asan_used_labels->contains (label)
   20796      2628927 :               && !gimplify_omp_ctxp)
   20797          904 :             asan_poison_variables (asan_poisoned_variables, false, pre_p);
   20798              :           break;
   20799              : 
   20800      1045941 :         case CASE_LABEL_EXPR:
   20801      1045941 :           ret = gimplify_case_label_expr (expr_p, pre_p);
   20802              : 
   20803      1045941 :           if (gimplify_ctxp->live_switch_vars)
   20804      1045754 :             asan_poison_variables (gimplify_ctxp->live_switch_vars, false,
   20805              :                                    pre_p);
   20806              :           break;
   20807              : 
   20808      2298186 :         case RETURN_EXPR:
   20809      2298186 :           ret = gimplify_return_expr (*expr_p, pre_p);
   20810      2298186 :           break;
   20811              : 
   20812       983379 :         case CONSTRUCTOR:
   20813              :           /* Don't reduce this in place; let gimplify_init_constructor work its
   20814              :              magic.  Buf if we're just elaborating this for side effects, just
   20815              :              gimplify any element that has side-effects.  */
   20816       983379 :           if (fallback == fb_none)
   20817              :             {
   20818          411 :               unsigned HOST_WIDE_INT ix;
   20819          411 :               tree val;
   20820          411 :               tree temp = NULL_TREE;
   20821          428 :               FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (*expr_p), ix, val)
   20822           17 :                 if (TREE_SIDE_EFFECTS (val))
   20823            4 :                   append_to_statement_list (val, &temp);
   20824              : 
   20825          411 :               *expr_p = temp;
   20826          411 :               ret = temp ? GS_OK : GS_ALL_DONE;
   20827              :             }
   20828              :           /* C99 code may assign to an array in a constructed
   20829              :              structure or union, and this has undefined behavior only
   20830              :              on execution, so create a temporary if an lvalue is
   20831              :              required.  */
   20832       982968 :           else if (fallback == fb_lvalue)
   20833              :             {
   20834            3 :               *expr_p = get_initialized_tmp_var (*expr_p, pre_p, post_p, false);
   20835            3 :               mark_addressable (*expr_p);
   20836            3 :               ret = GS_OK;
   20837              :             }
   20838              :           else
   20839              :             ret = GS_ALL_DONE;
   20840              :           break;
   20841              : 
   20842              :           /* The following are special cases that are not handled by the
   20843              :              original GIMPLE grammar.  */
   20844              : 
   20845              :           /* SAVE_EXPR nodes are converted into a GIMPLE identifier and
   20846              :              eliminated.  */
   20847       444548 :         case SAVE_EXPR:
   20848       444548 :           ret = gimplify_save_expr (expr_p, pre_p, post_p);
   20849       444548 :           break;
   20850              : 
   20851       357995 :         case BIT_FIELD_REF:
   20852       357995 :           ret = gimplify_expr (&TREE_OPERAND (*expr_p, 0), pre_p,
   20853              :                                post_p, is_gimple_lvalue, fb_either);
   20854       357995 :           recalculate_side_effects (*expr_p);
   20855       357995 :           break;
   20856              : 
   20857         2058 :         case TARGET_MEM_REF:
   20858         2058 :           {
   20859         2058 :             enum gimplify_status r0 = GS_ALL_DONE, r1 = GS_ALL_DONE;
   20860              : 
   20861         2058 :             if (TMR_BASE (*expr_p))
   20862         2058 :               r0 = gimplify_expr (&TMR_BASE (*expr_p), pre_p,
   20863              :                                   post_p, is_gimple_mem_ref_addr, fb_either);
   20864         2058 :             if (TMR_INDEX (*expr_p))
   20865         1221 :               r1 = gimplify_expr (&TMR_INDEX (*expr_p), pre_p,
   20866              :                                   post_p, is_gimple_val, fb_rvalue);
   20867         2058 :             if (TMR_INDEX2 (*expr_p))
   20868           72 :               r1 = gimplify_expr (&TMR_INDEX2 (*expr_p), pre_p,
   20869              :                                   post_p, is_gimple_val, fb_rvalue);
   20870              :             /* TMR_STEP and TMR_OFFSET are always integer constants.  */
   20871         2058 :             ret = MIN (r0, r1);
   20872              :           }
   20873              :           break;
   20874              : 
   20875            0 :         case NON_LVALUE_EXPR:
   20876              :           /* This should have been stripped above.  */
   20877            0 :           gcc_unreachable ();
   20878              : 
   20879        83267 :         case ASM_EXPR:
   20880        83267 :           ret = gimplify_asm_expr (expr_p, pre_p, post_p);
   20881        83267 :           break;
   20882              : 
   20883       511784 :         case TRY_FINALLY_EXPR:
   20884       511784 :         case TRY_CATCH_EXPR:
   20885       511784 :           {
   20886       511784 :             gimple_seq eval, cleanup;
   20887       511784 :             gtry *try_;
   20888              : 
   20889              :             /* Calls to destructors are generated automatically in FINALLY/CATCH
   20890              :                block. They should have location as UNKNOWN_LOCATION. However,
   20891              :                gimplify_call_expr will reset these call stmts to input_location
   20892              :                if it finds stmt's location is unknown. To prevent resetting for
   20893              :                destructors, we set the input_location to unknown.
   20894              :                Note that this only affects the destructor calls in FINALLY/CATCH
   20895              :                block, and will automatically reset to its original value by the
   20896              :                end of gimplify_expr.  */
   20897       511784 :             input_location = UNKNOWN_LOCATION;
   20898       511784 :             eval = cleanup = NULL;
   20899       511784 :             gimplify_and_add (TREE_OPERAND (*expr_p, 0), &eval);
   20900       511784 :             bool save_in_handler_expr = gimplify_ctxp->in_handler_expr;
   20901       511784 :             if (TREE_CODE (*expr_p) == TRY_FINALLY_EXPR
   20902       511784 :                 && TREE_CODE (TREE_OPERAND (*expr_p, 1)) == EH_ELSE_EXPR)
   20903              :               {
   20904          282 :                 gimple_seq n = NULL, e = NULL;
   20905          282 :                 gimplify_ctxp->in_handler_expr = true;
   20906          282 :                 gimplify_and_add (TREE_OPERAND (TREE_OPERAND (*expr_p, 1),
   20907              :                                                 0), &n);
   20908          282 :                 gimplify_and_add (TREE_OPERAND (TREE_OPERAND (*expr_p, 1),
   20909              :                                                 1), &e);
   20910          282 :                 if (!gimple_seq_empty_p (n) || !gimple_seq_empty_p (e))
   20911              :                   {
   20912          276 :                     geh_else *stmt = gimple_build_eh_else (n, e);
   20913          276 :                     gimple_seq_add_stmt (&cleanup, stmt);
   20914              :                   }
   20915              :               }
   20916              :             else
   20917              :               {
   20918       511502 :                 gimplify_ctxp->in_handler_expr = true;
   20919       511502 :                 gimplify_and_add (TREE_OPERAND (*expr_p, 1), &cleanup);
   20920              :               }
   20921       511784 :             gimplify_ctxp->in_handler_expr = save_in_handler_expr;
   20922              :             /* Don't create bogus GIMPLE_TRY with empty cleanup.  */
   20923       511784 :             if (gimple_seq_empty_p (cleanup))
   20924              :               {
   20925        27222 :                 gimple_seq_add_seq (pre_p, eval);
   20926        27222 :                 ret = GS_ALL_DONE;
   20927        27222 :                 break;
   20928              :               }
   20929       484562 :             try_ = gimple_build_try (eval, cleanup,
   20930       484562 :                                      TREE_CODE (*expr_p) == TRY_FINALLY_EXPR
   20931              :                                      ? GIMPLE_TRY_FINALLY
   20932              :                                      : GIMPLE_TRY_CATCH);
   20933       484562 :             if (EXPR_HAS_LOCATION (save_expr))
   20934       885400 :               gimple_set_location (try_, EXPR_LOCATION (save_expr));
   20935        41862 :             else if (LOCATION_LOCUS (saved_location) != UNKNOWN_LOCATION)
   20936        30860 :               gimple_set_location (try_, saved_location);
   20937       484562 :             if (TREE_CODE (*expr_p) == TRY_CATCH_EXPR)
   20938       172967 :               gimple_try_set_catch_is_cleanup (try_,
   20939       172967 :                                                TRY_CATCH_IS_CLEANUP (*expr_p));
   20940       484562 :             gimplify_seq_add_stmt (pre_p, try_);
   20941       484562 :             ret = GS_ALL_DONE;
   20942       484562 :             break;
   20943              :           }
   20944              : 
   20945      5436768 :         case CLEANUP_POINT_EXPR:
   20946      5436768 :           ret = gimplify_cleanup_point_expr (expr_p, pre_p);
   20947      5436768 :           break;
   20948              : 
   20949       868857 :         case TARGET_EXPR:
   20950       868857 :           ret = gimplify_target_expr (expr_p, pre_p, post_p);
   20951       868857 :           break;
   20952              : 
   20953        43030 :         case CATCH_EXPR:
   20954        43030 :           {
   20955        43030 :             gimple *c;
   20956        43030 :             gimple_seq handler = NULL;
   20957        43030 :             gimplify_and_add (CATCH_BODY (*expr_p), &handler);
   20958        43030 :             c = gimple_build_catch (CATCH_TYPES (*expr_p), handler);
   20959        43030 :             gimplify_seq_add_stmt (pre_p, c);
   20960        43030 :             ret = GS_ALL_DONE;
   20961        43030 :             break;
   20962              :           }
   20963              : 
   20964         4491 :         case EH_FILTER_EXPR:
   20965         4491 :           {
   20966         4491 :             gimple *ehf;
   20967         4491 :             gimple_seq failure = NULL;
   20968              : 
   20969         4491 :             gimplify_and_add (EH_FILTER_FAILURE (*expr_p), &failure);
   20970         4491 :             ehf = gimple_build_eh_filter (EH_FILTER_TYPES (*expr_p), failure);
   20971         4491 :             copy_warning (ehf, *expr_p);
   20972         4491 :             gimplify_seq_add_stmt (pre_p, ehf);
   20973         4491 :             ret = GS_ALL_DONE;
   20974         4491 :             break;
   20975              :           }
   20976              : 
   20977        52399 :         case OBJ_TYPE_REF:
   20978        52399 :           {
   20979        52399 :             enum gimplify_status r0, r1;
   20980        52399 :             r0 = gimplify_expr (&OBJ_TYPE_REF_OBJECT (*expr_p), pre_p,
   20981              :                                 post_p, is_gimple_val, fb_rvalue);
   20982        52399 :             r1 = gimplify_expr (&OBJ_TYPE_REF_EXPR (*expr_p), pre_p,
   20983              :                                 post_p, is_gimple_val, fb_rvalue);
   20984        52399 :             TREE_SIDE_EFFECTS (*expr_p) = 0;
   20985        52399 :             ret = MIN (r0, r1);
   20986              :           }
   20987              :           break;
   20988              : 
   20989        41122 :         case LABEL_DECL:
   20990              :           /* We get here when taking the address of a label.  We mark
   20991              :              the label as "forced"; meaning it can never be removed and
   20992              :              it is a potential target for any computed goto.  */
   20993        41122 :           FORCED_LABEL (*expr_p) = 1;
   20994        41122 :           ret = GS_ALL_DONE;
   20995        41122 :           break;
   20996              : 
   20997      8689877 :         case STATEMENT_LIST:
   20998      8689877 :           ret = gimplify_statement_list (expr_p, pre_p);
   20999      8689877 :           break;
   21000              : 
   21001         1745 :         case WITH_SIZE_EXPR:
   21002         1745 :           {
   21003         2027 :             gimplify_expr (&TREE_OPERAND (*expr_p, 0), pre_p,
   21004              :                            post_p == &internal_post ? NULL : post_p,
   21005              :                            gimple_test_f, fallback);
   21006         1745 :             gimplify_expr (&TREE_OPERAND (*expr_p, 1), pre_p, post_p,
   21007              :                            is_gimple_val, fb_rvalue);
   21008         1745 :             ret = GS_ALL_DONE;
   21009              :           }
   21010         1745 :           break;
   21011              : 
   21012    111751578 :         case VAR_DECL:
   21013    111751578 :         case PARM_DECL:
   21014    111751578 :           ret = gimplify_var_or_parm_decl (expr_p);
   21015    111751578 :           break;
   21016              : 
   21017       346905 :         case RESULT_DECL:
   21018              :           /* When within an OMP context, notice uses of variables.  */
   21019       346905 :           if (gimplify_omp_ctxp)
   21020         1412 :             omp_notice_variable (gimplify_omp_ctxp, *expr_p, true);
   21021              :           /* Handlers can refer to the function result; if that has been
   21022              :              moved, we need to track it.  */
   21023       346905 :           if (gimplify_ctxp->in_handler_expr && gimplify_ctxp->return_temp)
   21024           68 :             *expr_p = gimplify_ctxp->return_temp;
   21025              :           ret = GS_ALL_DONE;
   21026              :           break;
   21027              : 
   21028            0 :         case DEBUG_EXPR_DECL:
   21029            0 :           gcc_unreachable ();
   21030              : 
   21031      2486415 :         case DEBUG_BEGIN_STMT:
   21032      4972830 :           gimplify_seq_add_stmt (pre_p,
   21033              :                                  gimple_build_debug_begin_stmt
   21034      2486415 :                                  (TREE_BLOCK (*expr_p),
   21035      2486415 :                                   EXPR_LOCATION (*expr_p)));
   21036      2486415 :           ret = GS_ALL_DONE;
   21037      2486415 :           *expr_p = NULL;
   21038      2486415 :           break;
   21039              : 
   21040              :         case SSA_NAME:
   21041              :           /* Allow callbacks into the gimplifier during optimization.  */
   21042              :           ret = GS_ALL_DONE;
   21043              :           break;
   21044              : 
   21045        18464 :         case OMP_PARALLEL:
   21046        18464 :           gimplify_omp_parallel (expr_p, pre_p);
   21047        18464 :           ret = GS_ALL_DONE;
   21048        18464 :           break;
   21049              : 
   21050         4132 :         case OMP_TASK:
   21051         4132 :           gimplify_omp_task (expr_p, pre_p);
   21052         4132 :           ret = GS_ALL_DONE;
   21053         4132 :           break;
   21054              : 
   21055        10803 :         case OMP_SIMD:
   21056        10803 :           {
   21057              :             /* Temporarily disable into_ssa, as scan_omp_simd
   21058              :                which calls copy_gimple_seq_and_replace_locals can't deal
   21059              :                with SSA_NAMEs defined outside of the body properly.  */
   21060        10803 :             bool saved_into_ssa = gimplify_ctxp->into_ssa;
   21061        10803 :             gimplify_ctxp->into_ssa = false;
   21062        10803 :             ret = gimplify_omp_for (expr_p, pre_p);
   21063        10803 :             gimplify_ctxp->into_ssa = saved_into_ssa;
   21064        10803 :             break;
   21065              :           }
   21066              : 
   21067        47624 :         case OMP_FOR:
   21068        47624 :         case OMP_DISTRIBUTE:
   21069        47624 :         case OMP_TASKLOOP:
   21070        47624 :         case OMP_TILE:
   21071        47624 :         case OMP_UNROLL:
   21072        47624 :         case OACC_LOOP:
   21073        47624 :           ret = gimplify_omp_for (expr_p, pre_p);
   21074        47624 :           break;
   21075              : 
   21076         1062 :         case OMP_LOOP:
   21077         1062 :           ret = gimplify_omp_loop (expr_p, pre_p);
   21078         1062 :           break;
   21079              : 
   21080          717 :         case OACC_CACHE:
   21081          717 :           gimplify_oacc_cache (expr_p, pre_p);
   21082          717 :           ret = GS_ALL_DONE;
   21083          717 :           break;
   21084              : 
   21085          254 :         case OACC_DECLARE:
   21086          254 :           gimplify_oacc_declare (expr_p, pre_p);
   21087          254 :           ret = GS_ALL_DONE;
   21088          254 :           break;
   21089              : 
   21090        40168 :         case OACC_HOST_DATA:
   21091        40168 :         case OACC_DATA:
   21092        40168 :         case OACC_KERNELS:
   21093        40168 :         case OACC_PARALLEL:
   21094        40168 :         case OACC_SERIAL:
   21095        40168 :         case OMP_SCOPE:
   21096        40168 :         case OMP_SECTIONS:
   21097        40168 :         case OMP_SINGLE:
   21098        40168 :         case OMP_TARGET:
   21099        40168 :         case OMP_TARGET_DATA:
   21100        40168 :         case OMP_TEAMS:
   21101        40168 :           gimplify_omp_workshare (expr_p, pre_p);
   21102        40168 :           ret = GS_ALL_DONE;
   21103        40168 :           break;
   21104              : 
   21105        12382 :         case OACC_ENTER_DATA:
   21106        12382 :         case OACC_EXIT_DATA:
   21107        12382 :         case OACC_UPDATE:
   21108        12382 :         case OMP_TARGET_UPDATE:
   21109        12382 :         case OMP_TARGET_ENTER_DATA:
   21110        12382 :         case OMP_TARGET_EXIT_DATA:
   21111        12382 :           gimplify_omp_target_update (expr_p, pre_p);
   21112        12382 :           ret = GS_ALL_DONE;
   21113        12382 :           break;
   21114              : 
   21115         7092 :         case OMP_SECTION:
   21116         7092 :         case OMP_STRUCTURED_BLOCK:
   21117         7092 :         case OMP_MASTER:
   21118         7092 :         case OMP_MASKED:
   21119         7092 :         case OMP_ORDERED:
   21120         7092 :         case OMP_CRITICAL:
   21121         7092 :         case OMP_SCAN:
   21122         7092 :           {
   21123         7092 :             gimple_seq body = NULL;
   21124         7092 :             gimple *g;
   21125         7092 :             bool saved_in_omp_construct = in_omp_construct;
   21126              : 
   21127         7092 :             in_omp_construct = true;
   21128         7092 :             gimplify_and_add (OMP_BODY (*expr_p), &body);
   21129         7092 :             in_omp_construct = saved_in_omp_construct;
   21130         7092 :             switch (TREE_CODE (*expr_p))
   21131              :               {
   21132         1279 :               case OMP_SECTION:
   21133         1279 :                 g = gimple_build_omp_section (body);
   21134         1279 :                 break;
   21135          791 :               case OMP_STRUCTURED_BLOCK:
   21136          791 :                 g = gimple_build_omp_structured_block (body);
   21137          791 :                 break;
   21138          874 :               case OMP_MASTER:
   21139          874 :                 g = gimple_build_omp_master (body);
   21140          874 :                 break;
   21141         1954 :               case OMP_ORDERED:
   21142         1954 :                 g = gimplify_omp_ordered (*expr_p, body);
   21143         1954 :                 if (OMP_BODY (*expr_p) == NULL_TREE
   21144         1954 :                     && gimple_code (g) == GIMPLE_OMP_ORDERED)
   21145         1025 :                   gimple_omp_ordered_standalone (g);
   21146              :                 break;
   21147          500 :               case OMP_MASKED:
   21148          500 :                 gimplify_scan_omp_clauses (&OMP_MASKED_CLAUSES (*expr_p),
   21149              :                                            pre_p, ORT_WORKSHARE, OMP_MASKED);
   21150          500 :                 gimplify_adjust_omp_clauses (pre_p, body,
   21151              :                                              &OMP_MASKED_CLAUSES (*expr_p),
   21152              :                                              OMP_MASKED);
   21153         1000 :                 g = gimple_build_omp_masked (body,
   21154          500 :                                              OMP_MASKED_CLAUSES (*expr_p));
   21155          500 :                 break;
   21156          552 :               case OMP_CRITICAL:
   21157          552 :                 gimplify_scan_omp_clauses (&OMP_CRITICAL_CLAUSES (*expr_p),
   21158              :                                            pre_p, ORT_WORKSHARE, OMP_CRITICAL);
   21159          552 :                 gimplify_adjust_omp_clauses (pre_p, body,
   21160              :                                              &OMP_CRITICAL_CLAUSES (*expr_p),
   21161              :                                              OMP_CRITICAL);
   21162         1656 :                 g = gimple_build_omp_critical (body,
   21163          552 :                                                OMP_CRITICAL_NAME (*expr_p),
   21164          552 :                                                OMP_CRITICAL_CLAUSES (*expr_p));
   21165          552 :                 break;
   21166         1142 :               case OMP_SCAN:
   21167         1142 :                 gimplify_scan_omp_clauses (&OMP_SCAN_CLAUSES (*expr_p),
   21168              :                                            pre_p, ORT_WORKSHARE, OMP_SCAN);
   21169         1142 :                 gimplify_adjust_omp_clauses (pre_p, body,
   21170              :                                              &OMP_SCAN_CLAUSES (*expr_p),
   21171              :                                              OMP_SCAN);
   21172         1142 :                 g = gimple_build_omp_scan (body, OMP_SCAN_CLAUSES (*expr_p));
   21173         1142 :                 break;
   21174            0 :               default:
   21175            0 :                 gcc_unreachable ();
   21176              :               }
   21177         7092 :             gimplify_seq_add_stmt (pre_p, g);
   21178         7092 :             ret = GS_ALL_DONE;
   21179         7092 :             break;
   21180              :           }
   21181              : 
   21182          611 :         case OMP_TASKGROUP:
   21183          611 :           {
   21184          611 :             gimple_seq body = NULL;
   21185              : 
   21186          611 :             tree *pclauses = &OMP_TASKGROUP_CLAUSES (*expr_p);
   21187          611 :             bool saved_in_omp_construct = in_omp_construct;
   21188          611 :             gimplify_scan_omp_clauses (pclauses, pre_p, ORT_TASKGROUP,
   21189              :                                        OMP_TASKGROUP);
   21190          611 :             gimplify_adjust_omp_clauses (pre_p, NULL, pclauses, OMP_TASKGROUP);
   21191              : 
   21192          611 :             in_omp_construct = true;
   21193          611 :             gimplify_and_add (OMP_BODY (*expr_p), &body);
   21194          611 :             in_omp_construct = saved_in_omp_construct;
   21195          611 :             gimple_seq cleanup = NULL;
   21196          611 :             tree fn = builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_END);
   21197          611 :             gimple *g = gimple_build_call (fn, 0);
   21198          611 :             gimple_seq_add_stmt (&cleanup, g);
   21199          611 :             g = gimple_build_try (body, cleanup, GIMPLE_TRY_FINALLY);
   21200          611 :             body = NULL;
   21201          611 :             gimple_seq_add_stmt (&body, g);
   21202          611 :             g = gimple_build_omp_taskgroup (body, *pclauses);
   21203          611 :             gimplify_seq_add_stmt (pre_p, g);
   21204          611 :             ret = GS_ALL_DONE;
   21205          611 :             break;
   21206              :           }
   21207              : 
   21208          612 :         case OMP_INTEROP:
   21209          612 :           ret = gimplify_omp_interop (expr_p, pre_p);
   21210          612 :           break;
   21211        10330 :         case OMP_ATOMIC:
   21212        10330 :         case OMP_ATOMIC_READ:
   21213        10330 :         case OMP_ATOMIC_CAPTURE_OLD:
   21214        10330 :         case OMP_ATOMIC_CAPTURE_NEW:
   21215        10330 :           ret = gimplify_omp_atomic (expr_p, pre_p);
   21216        10330 :           break;
   21217              : 
   21218          852 :         case OMP_DISPATCH:
   21219          852 :           ret = gimplify_omp_dispatch (expr_p, pre_p);
   21220          852 :           break;
   21221              : 
   21222          180 :         case OMP_METADIRECTIVE:
   21223          180 :           ret = gimplify_omp_metadirective (expr_p, pre_p, post_p,
   21224              :                                             gimple_test_f, fallback);
   21225          180 :           break;
   21226              : 
   21227           36 :         case OMP_NEXT_VARIANT:
   21228           36 :         case OMP_TARGET_DEVICE_MATCHES:
   21229              :           /* These are placeholders for constants.  There's nothing to do with
   21230              :              them here but we must mark the containing function as needing
   21231              :              to run the ompdevlow pass to resolve them.  Note that
   21232              :              OMP_TARGET_DEVICE_MATCHES, in particular, may be inserted by
   21233              :              the front ends.  */
   21234           36 :           cgraph_node::get (cfun->decl)->has_omp_variant_constructs = 1;
   21235           36 :           ret = GS_ALL_DONE;
   21236           36 :           break;
   21237              : 
   21238            0 :         case OMP_DECLARE_MAPPER:
   21239            0 :           ret = gimplify_omp_declare_mapper (expr_p);
   21240            0 :           break;
   21241              : 
   21242          478 :         case TRANSACTION_EXPR:
   21243          478 :           ret = gimplify_transaction (expr_p, pre_p);
   21244          478 :           break;
   21245              : 
   21246       293775 :         case TRUTH_AND_EXPR:
   21247       293775 :         case TRUTH_OR_EXPR:
   21248       293775 :         case TRUTH_XOR_EXPR:
   21249       293775 :           {
   21250       293775 :             tree orig_type = TREE_TYPE (*expr_p);
   21251       293775 :             tree new_type, xop0, xop1;
   21252       293775 :             *expr_p = gimple_boolify (*expr_p);
   21253       293775 :             new_type = TREE_TYPE (*expr_p);
   21254       293775 :             if (!useless_type_conversion_p (orig_type, new_type))
   21255              :               {
   21256         1155 :                 *expr_p = fold_convert_loc (input_location, orig_type, *expr_p);
   21257         1155 :                 ret = GS_OK;
   21258         1155 :                 break;
   21259              :               }
   21260              : 
   21261              :           /* Boolified binary truth expressions are semantically equivalent
   21262              :              to bitwise binary expressions.  Canonicalize them to the
   21263              :              bitwise variant.  */
   21264       292620 :             switch (TREE_CODE (*expr_p))
   21265              :               {
   21266       176804 :               case TRUTH_AND_EXPR:
   21267       176804 :                 TREE_SET_CODE (*expr_p, BIT_AND_EXPR);
   21268       176804 :                 break;
   21269       115183 :               case TRUTH_OR_EXPR:
   21270       115183 :                 TREE_SET_CODE (*expr_p, BIT_IOR_EXPR);
   21271       115183 :                 break;
   21272          633 :               case TRUTH_XOR_EXPR:
   21273          633 :                 TREE_SET_CODE (*expr_p, BIT_XOR_EXPR);
   21274          633 :                 break;
   21275              :               default:
   21276              :                 break;
   21277              :               }
   21278              :             /* Now make sure that operands have compatible type to
   21279              :                expression's new_type.  */
   21280       292620 :             xop0 = TREE_OPERAND (*expr_p, 0);
   21281       292620 :             xop1 = TREE_OPERAND (*expr_p, 1);
   21282       292620 :             if (!useless_type_conversion_p (new_type, TREE_TYPE (xop0)))
   21283          326 :               TREE_OPERAND (*expr_p, 0) = fold_convert_loc (input_location,
   21284              :                                                             new_type,
   21285              :                                                             xop0);
   21286       292620 :             if (!useless_type_conversion_p (new_type, TREE_TYPE (xop1)))
   21287          366 :               TREE_OPERAND (*expr_p, 1) = fold_convert_loc (input_location,
   21288              :                                                             new_type,
   21289              :                                                             xop1);
   21290              :             /* Continue classified as tcc_binary.  */
   21291       292620 :             goto expr_2;
   21292              :           }
   21293              : 
   21294        11389 :         case VEC_COND_EXPR:
   21295        11389 :           goto expr_3;
   21296              : 
   21297       134517 :         case VEC_PERM_EXPR:
   21298              :           /* Classified as tcc_expression.  */
   21299       134517 :           goto expr_3;
   21300              : 
   21301           48 :         case BIT_INSERT_EXPR:
   21302              :           /* Argument 3 is a constant.  */
   21303           48 :           goto expr_2;
   21304              : 
   21305      3871517 :         case POINTER_PLUS_EXPR:
   21306      3871517 :           {
   21307      3871517 :             enum gimplify_status r0, r1;
   21308      3871517 :             r0 = gimplify_expr (&TREE_OPERAND (*expr_p, 0), pre_p,
   21309              :                                 post_p, is_gimple_val, fb_rvalue);
   21310      3871517 :             r1 = gimplify_expr (&TREE_OPERAND (*expr_p, 1), pre_p,
   21311              :                                 post_p, is_gimple_val, fb_rvalue);
   21312      3871517 :             recalculate_side_effects (*expr_p);
   21313      3871517 :             ret = MIN (r0, r1);
   21314              :             break;
   21315              :           }
   21316              : 
   21317     45411784 :         default:
   21318     45411784 :           switch (TREE_CODE_CLASS (TREE_CODE (*expr_p)))
   21319              :             {
   21320      7117018 :             case tcc_comparison:
   21321              :               /* Handle comparison of objects of non scalar mode aggregates
   21322              :                  with a call to memcmp.  It would be nice to only have to do
   21323              :                  this for variable-sized objects, but then we'd have to allow
   21324              :                  the same nest of reference nodes we allow for MODIFY_EXPR and
   21325              :                  that's too complex.
   21326              : 
   21327              :                  Compare scalar mode aggregates as scalar mode values.  Using
   21328              :                  memcmp for them would be very inefficient at best, and is
   21329              :                  plain wrong if bitfields are involved.  */
   21330      7117018 :               if (error_operand_p (TREE_OPERAND (*expr_p, 1)))
   21331              :                 ret = GS_ERROR;
   21332              :               else
   21333              :                 {
   21334      7117018 :                   tree type = TREE_TYPE (TREE_OPERAND (*expr_p, 1));
   21335              : 
   21336              :                   /* Vector comparisons need no boolification.  */
   21337      7117018 :                   if (TREE_CODE (type) == VECTOR_TYPE)
   21338        13441 :                     goto expr_2;
   21339      7103577 :                   else if (!AGGREGATE_TYPE_P (type))
   21340              :                     {
   21341      7103561 :                       tree org_type = TREE_TYPE (*expr_p);
   21342      7103561 :                       *expr_p = gimple_boolify (*expr_p);
   21343      7103561 :                       if (!useless_type_conversion_p (org_type,
   21344      7103561 :                                                       TREE_TYPE (*expr_p)))
   21345              :                         {
   21346        58378 :                           *expr_p = fold_convert_loc (input_location,
   21347              :                                                       org_type, *expr_p);
   21348        58378 :                           ret = GS_OK;
   21349              :                         }
   21350              :                       else
   21351      7045183 :                         goto expr_2;
   21352              :                     }
   21353           16 :                   else if (SCALAR_INT_MODE_P (TYPE_MODE (type)))
   21354           16 :                     ret = gimplify_scalar_mode_aggregate_compare (expr_p);
   21355              :                   else
   21356            0 :                     ret = gimplify_variable_sized_compare (expr_p);
   21357              :                 }
   21358              :               break;
   21359              : 
   21360              :             /* If *EXPR_P does not need to be special-cased, handle it
   21361              :                according to its class.  */
   21362      1619076 :             case tcc_unary:
   21363      1619076 :               ret = gimplify_expr (&TREE_OPERAND (*expr_p, 0), pre_p,
   21364              :                                    post_p, is_gimple_val, fb_rvalue);
   21365      1619076 :               break;
   21366              : 
   21367     26895712 :             case tcc_binary:
   21368     26895712 :             expr_2:
   21369     26895712 :               {
   21370     26895712 :                 enum gimplify_status r0, r1;
   21371              : 
   21372     26895712 :                 r0 = gimplify_expr (&TREE_OPERAND (*expr_p, 0), pre_p,
   21373              :                                     post_p, is_gimple_val, fb_rvalue);
   21374     26895712 :                 r1 = gimplify_expr (&TREE_OPERAND (*expr_p, 1), pre_p,
   21375              :                                     post_p, is_gimple_val, fb_rvalue);
   21376              : 
   21377     26895712 :                 ret = MIN (r0, r1);
   21378              :                 break;
   21379              :               }
   21380              : 
   21381       145906 :             expr_3:
   21382       145906 :               {
   21383       145906 :                 enum gimplify_status r0, r1, r2;
   21384              : 
   21385       145906 :                 r0 = gimplify_expr (&TREE_OPERAND (*expr_p, 0), pre_p,
   21386              :                                     post_p, is_gimple_val, fb_rvalue);
   21387       145906 :                 r1 = gimplify_expr (&TREE_OPERAND (*expr_p, 1), pre_p,
   21388              :                                     post_p, is_gimple_val, fb_rvalue);
   21389       145906 :                 r2 = gimplify_expr (&TREE_OPERAND (*expr_p, 2), pre_p,
   21390              :                                     post_p, is_gimple_val, fb_rvalue);
   21391              : 
   21392       145906 :                 ret = MIN (MIN (r0, r1), r2);
   21393              :                 break;
   21394              :               }
   21395              : 
   21396     17131270 :             case tcc_declaration:
   21397     17131270 :             case tcc_constant:
   21398     17131270 :               ret = GS_ALL_DONE;
   21399     17131270 :               goto dont_recalculate;
   21400              : 
   21401            0 :             default:
   21402            0 :               gcc_unreachable ();
   21403              :             }
   21404              : 
   21405     28719088 :           recalculate_side_effects (*expr_p);
   21406              : 
   21407    491516964 :         dont_recalculate:
   21408              :           break;
   21409              :         }
   21410              : 
   21411    491516964 :       gcc_assert (*expr_p || ret != GS_OK);
   21412              :     }
   21413    496486380 :   while (ret == GS_OK);
   21414              : 
   21415              :   /* If we encountered an error_mark somewhere nested inside, either
   21416              :      stub out the statement or propagate the error back out.  */
   21417    478212754 :   if (ret == GS_ERROR)
   21418              :     {
   21419         5353 :       if (is_statement)
   21420         4124 :         *expr_p = NULL;
   21421         5353 :       goto out;
   21422              :     }
   21423              : 
   21424              :   /* This was only valid as a return value from the langhook, which
   21425              :      we handled.  Make sure it doesn't escape from any other context.  */
   21426    478207401 :   gcc_assert (ret != GS_UNHANDLED);
   21427              : 
   21428    478207401 :   if (fallback == fb_none && *expr_p && !is_gimple_stmt (*expr_p))
   21429              :     {
   21430              :       /* We aren't looking for a value, and we don't have a valid
   21431              :          statement.  If it doesn't have side-effects, throw it away.
   21432              :          We can also get here with code such as "*&&L;", where L is
   21433              :          a LABEL_DECL that is marked as FORCED_LABEL.  */
   21434      1239531 :       if (TREE_CODE (*expr_p) == LABEL_DECL
   21435      1239531 :           || !TREE_SIDE_EFFECTS (*expr_p))
   21436      1238499 :         *expr_p = NULL;
   21437         1032 :       else if (!TREE_THIS_VOLATILE (*expr_p))
   21438              :         {
   21439              :           /* This is probably a _REF that contains something nested that
   21440              :              has side effects.  Recurse through the operands to find it.  */
   21441            0 :           enum tree_code code = TREE_CODE (*expr_p);
   21442              : 
   21443            0 :           switch (code)
   21444              :             {
   21445            0 :             case COMPONENT_REF:
   21446            0 :             case REALPART_EXPR:
   21447            0 :             case IMAGPART_EXPR:
   21448            0 :             case VIEW_CONVERT_EXPR:
   21449            0 :               gimplify_expr (&TREE_OPERAND (*expr_p, 0), pre_p, post_p,
   21450              :                              gimple_test_f, fallback);
   21451            0 :               break;
   21452              : 
   21453            0 :             case ARRAY_REF:
   21454            0 :             case ARRAY_RANGE_REF:
   21455            0 :               gimplify_expr (&TREE_OPERAND (*expr_p, 0), pre_p, post_p,
   21456              :                              gimple_test_f, fallback);
   21457            0 :               gimplify_expr (&TREE_OPERAND (*expr_p, 1), pre_p, post_p,
   21458              :                              gimple_test_f, fallback);
   21459            0 :               break;
   21460              : 
   21461            0 :             default:
   21462              :                /* Anything else with side-effects must be converted to
   21463              :                   a valid statement before we get here.  */
   21464            0 :               gcc_unreachable ();
   21465              :             }
   21466              : 
   21467            0 :           *expr_p = NULL;
   21468              :         }
   21469         1032 :       else if (COMPLETE_TYPE_P (TREE_TYPE (*expr_p))
   21470          982 :                && TYPE_MODE (TREE_TYPE (*expr_p)) != BLKmode
   21471         2001 :                && !is_empty_type (TREE_TYPE (*expr_p)))
   21472              :         {
   21473              :           /* Historically, the compiler has treated a bare reference
   21474              :              to a non-BLKmode volatile lvalue as forcing a load.  */
   21475          909 :           tree type = TYPE_MAIN_VARIANT (TREE_TYPE (*expr_p));
   21476              : 
   21477              :           /* Normally, we do not want to create a temporary for a
   21478              :              TREE_ADDRESSABLE type because such a type should not be
   21479              :              copied by bitwise-assignment.  However, we make an
   21480              :              exception here, as all we are doing here is ensuring that
   21481              :              we read the bytes that make up the type.  We use
   21482              :              create_tmp_var_raw because create_tmp_var will abort when
   21483              :              given a TREE_ADDRESSABLE type.  */
   21484          909 :           tree tmp = create_tmp_var_raw (type, "vol");
   21485          909 :           gimple_add_tmp_var (tmp);
   21486          909 :           gimplify_assign (tmp, *expr_p, pre_p);
   21487          909 :           *expr_p = NULL;
   21488              :         }
   21489              :       else
   21490              :         /* We can't do anything useful with a volatile reference to
   21491              :            an incomplete type, so just throw it away.  Likewise for
   21492              :            a BLKmode type, since any implicit inner load should
   21493              :            already have been turned into an explicit one by the
   21494              :            gimplification process.  */
   21495          123 :         *expr_p = NULL;
   21496              :     }
   21497              : 
   21498              :   /* If we are gimplifying at the statement level, we're done.  Tack
   21499              :      everything together and return.  */
   21500    478207401 :   if (fallback == fb_none || is_statement)
   21501              :     {
   21502              :       /* Since *EXPR_P has been converted into a GIMPLE tuple, clear
   21503              :          it out for GC to reclaim it.  */
   21504    103592607 :       *expr_p = NULL_TREE;
   21505              : 
   21506    103592607 :       if (!gimple_seq_empty_p (internal_pre)
   21507    103592607 :           || !gimple_seq_empty_p (internal_post))
   21508              :         {
   21509           19 :           gimplify_seq_add_seq (&internal_pre, internal_post);
   21510           19 :           gimplify_seq_add_seq (pre_p, internal_pre);
   21511              :         }
   21512              : 
   21513              :       /* The result of gimplifying *EXPR_P is going to be the last few
   21514              :          statements in *PRE_P and *POST_P.  Add location information
   21515              :          to all the statements that were added by the gimplification
   21516              :          helpers.  */
   21517    103592607 :       if (!gimple_seq_empty_p (*pre_p))
   21518    101052291 :         annotate_all_with_location_after (*pre_p, pre_last_gsi, input_location);
   21519              : 
   21520    103592607 :       if (!gimple_seq_empty_p (*post_p))
   21521           19 :         annotate_all_with_location_after (*post_p, post_last_gsi,
   21522              :                                           input_location);
   21523              : 
   21524    103592607 :       goto out;
   21525              :     }
   21526              : 
   21527              : #ifdef ENABLE_GIMPLE_CHECKING
   21528    374614794 :   if (*expr_p)
   21529              :     {
   21530    374614794 :       enum tree_code code = TREE_CODE (*expr_p);
   21531              :       /* These expressions should already be in gimple IR form.  */
   21532    374614794 :       gcc_assert (code != MODIFY_EXPR
   21533              :                   && code != ASM_EXPR
   21534              :                   && code != BIND_EXPR
   21535              :                   && code != CATCH_EXPR
   21536              :                   && (code != COND_EXPR || gimplify_ctxp->allow_rhs_cond_expr)
   21537              :                   && code != EH_FILTER_EXPR
   21538              :                   && code != GOTO_EXPR
   21539              :                   && code != LABEL_EXPR
   21540              :                   && code != LOOP_EXPR
   21541              :                   && code != SWITCH_EXPR
   21542              :                   && code != TRY_FINALLY_EXPR
   21543              :                   && code != EH_ELSE_EXPR
   21544              :                   && code != OACC_PARALLEL
   21545              :                   && code != OACC_KERNELS
   21546              :                   && code != OACC_SERIAL
   21547              :                   && code != OACC_DATA
   21548              :                   && code != OACC_HOST_DATA
   21549              :                   && code != OACC_DECLARE
   21550              :                   && code != OACC_UPDATE
   21551              :                   && code != OACC_ENTER_DATA
   21552              :                   && code != OACC_EXIT_DATA
   21553              :                   && code != OACC_CACHE
   21554              :                   && code != OMP_CRITICAL
   21555              :                   && code != OMP_FOR
   21556              :                   && code != OACC_LOOP
   21557              :                   && code != OMP_MASTER
   21558              :                   && code != OMP_MASKED
   21559              :                   && code != OMP_TASKGROUP
   21560              :                   && code != OMP_ORDERED
   21561              :                   && code != OMP_PARALLEL
   21562              :                   && code != OMP_SCAN
   21563              :                   && code != OMP_SECTIONS
   21564              :                   && code != OMP_SECTION
   21565              :                   && code != OMP_STRUCTURED_BLOCK
   21566              :                   && code != OMP_SINGLE
   21567              :                   && code != OMP_SCOPE
   21568              :                   && code != OMP_DISPATCH);
   21569              :     }
   21570              : #endif
   21571              : 
   21572              :   /* Otherwise we're gimplifying a subexpression, so the resulting
   21573              :      value is interesting.  If it's a valid operand that matches
   21574              :      GIMPLE_TEST_F, we're done. Unless we are handling some
   21575              :      post-effects internally; if that's the case, we need to copy into
   21576              :      a temporary before adding the post-effects to POST_P.  */
   21577    374614794 :   if (gimple_seq_empty_p (internal_post) && (*gimple_test_f) (*expr_p))
   21578    348021998 :     goto out;
   21579              : 
   21580              :   /* Otherwise, we need to create a new temporary for the gimplified
   21581              :      expression.  */
   21582              : 
   21583              :   /* We can't return an lvalue if we have an internal postqueue.  The
   21584              :      object the lvalue refers to would (probably) be modified by the
   21585              :      postqueue; we need to copy the value out first, which means an
   21586              :      rvalue.  */
   21587     26592796 :   if ((fallback & fb_lvalue)
   21588       600037 :       && gimple_seq_empty_p (internal_post)
   21589     27192833 :       && is_gimple_addressable (*expr_p))
   21590              :     {
   21591              :       /* An lvalue will do.  Take the address of the expression, store it
   21592              :          in a temporary, and replace the expression with an INDIRECT_REF of
   21593              :          that temporary.  */
   21594           15 :       tree ref_alias_type = reference_alias_ptr_type (*expr_p);
   21595           15 :       unsigned int ref_align = get_object_alignment (*expr_p);
   21596           15 :       tree ref_type = TREE_TYPE (*expr_p);
   21597           15 :       tmp = build_fold_addr_expr_loc (input_location, *expr_p);
   21598           15 :       gimplify_expr (&tmp, pre_p, post_p, is_gimple_reg, fb_rvalue);
   21599           15 :       if (TYPE_ALIGN (ref_type) != ref_align)
   21600            1 :         ref_type = build_aligned_type (ref_type, ref_align);
   21601           15 :       *expr_p = build2 (MEM_REF, ref_type,
   21602              :                         tmp, build_zero_cst (ref_alias_type));
   21603              :     }
   21604     26592781 :   else if ((fallback & fb_rvalue) && is_gimple_reg_rhs_or_call (*expr_p))
   21605              :     {
   21606              :       /* An rvalue will do.  Assign the gimplified expression into a
   21607              :          new temporary TMP and replace the original expression with
   21608              :          TMP.  First, make sure that the expression has a type so that
   21609              :          it can be assigned into a temporary.  */
   21610     26592773 :       gcc_assert (!VOID_TYPE_P (TREE_TYPE (*expr_p)));
   21611     26592773 :       *expr_p = get_formal_tmp_var (*expr_p, pre_p);
   21612              :     }
   21613              :   else
   21614              :     {
   21615              : #ifdef ENABLE_GIMPLE_CHECKING
   21616            8 :       if (!(fallback & fb_mayfail))
   21617              :         {
   21618            0 :           fprintf (stderr, "gimplification failed:\n");
   21619            0 :           print_generic_expr (stderr, *expr_p);
   21620            0 :           debug_tree (*expr_p);
   21621            0 :           internal_error ("gimplification failed");
   21622              :         }
   21623              : #endif
   21624            8 :       gcc_assert (fallback & fb_mayfail);
   21625              : 
   21626              :       /* If this is an asm statement, and the user asked for the
   21627              :          impossible, don't die.  Fail and let gimplify_asm_expr
   21628              :          issue an error.  */
   21629            8 :       ret = GS_ERROR;
   21630            8 :       goto out;
   21631              :     }
   21632              : 
   21633              :   /* Make sure the temporary matches our predicate.  */
   21634     26592788 :   gcc_assert ((*gimple_test_f) (*expr_p));
   21635              : 
   21636     26592788 :   if (!gimple_seq_empty_p (internal_post))
   21637              :     {
   21638            0 :       annotate_all_with_location (internal_post, input_location);
   21639            0 :       gimplify_seq_add_seq (pre_p, internal_post);
   21640              :     }
   21641              : 
   21642     26592788 :  out:
   21643    478212754 :   input_location = saved_location;
   21644    478212754 :   return ret;
   21645              : }
   21646              : 
   21647              : /* Like gimplify_expr but make sure the gimplified result is not itself
   21648              :    a SSA name (but a decl if it were).  Temporaries required by
   21649              :    evaluating *EXPR_P may be still SSA names.  */
   21650              : 
   21651              : static enum gimplify_status
   21652     35157230 : gimplify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
   21653              :                bool (*gimple_test_f) (tree), fallback_t fallback,
   21654              :                bool allow_ssa)
   21655              : {
   21656     35157230 :   enum gimplify_status ret = gimplify_expr (expr_p, pre_p, post_p,
   21657              :                                             gimple_test_f, fallback);
   21658     35157230 :   if (! allow_ssa
   21659       267499 :       && TREE_CODE (*expr_p) == SSA_NAME)
   21660        68724 :     *expr_p = get_initialized_tmp_var (*expr_p, pre_p, NULL, false);
   21661     35157230 :   return ret;
   21662              : }
   21663              : 
   21664              : /* Look through TYPE for variable-sized objects and gimplify each such
   21665              :    size that we find.  Add to LIST_P any statements generated.  */
   21666              : 
   21667              : void
   21668      8418435 : gimplify_type_sizes (tree type, gimple_seq *list_p)
   21669              : {
   21670      8418435 :   if (type == NULL || type == error_mark_node)
   21671              :     return;
   21672              : 
   21673      8418189 :   const bool ignored_p
   21674      8418189 :     = TYPE_NAME (type)
   21675      4193623 :       && TREE_CODE (TYPE_NAME (type)) == TYPE_DECL
   21676     12379055 :       && DECL_IGNORED_P (TYPE_NAME (type));
   21677      8418189 :   tree t;
   21678              : 
   21679              :   /* We first do the main variant, then copy into any other variants.  */
   21680      8418189 :   type = TYPE_MAIN_VARIANT (type);
   21681              : 
   21682              :   /* Avoid infinite recursion.  */
   21683      8418189 :   if (TYPE_SIZES_GIMPLIFIED (type))
   21684              :     return;
   21685              : 
   21686      2507555 :   TYPE_SIZES_GIMPLIFIED (type) = 1;
   21687              : 
   21688      2507555 :   switch (TREE_CODE (type))
   21689              :     {
   21690       501654 :     case INTEGER_TYPE:
   21691       501654 :     case ENUMERAL_TYPE:
   21692       501654 :     case BOOLEAN_TYPE:
   21693       501654 :     case REAL_TYPE:
   21694       501654 :     case FIXED_POINT_TYPE:
   21695       501654 :       gimplify_one_sizepos (&TYPE_MIN_VALUE (type), list_p);
   21696       501654 :       gimplify_one_sizepos (&TYPE_MAX_VALUE (type), list_p);
   21697              : 
   21698     10133664 :       for (t = TYPE_NEXT_VARIANT (type); t; t = TYPE_NEXT_VARIANT (t))
   21699              :         {
   21700      9632010 :           TYPE_MIN_VALUE (t) = TYPE_MIN_VALUE (type);
   21701      9632010 :           TYPE_MAX_VALUE (t) = TYPE_MAX_VALUE (type);
   21702              :         }
   21703              :       break;
   21704              : 
   21705       184356 :     case ARRAY_TYPE:
   21706              :       /* These types may not have declarations, so handle them here.  */
   21707       184356 :       gimplify_type_sizes (TREE_TYPE (type), list_p);
   21708       184356 :       gimplify_type_sizes (TYPE_DOMAIN (type), list_p);
   21709              :       /* Ensure VLA bounds aren't removed, for -O0 they should be variables
   21710              :          with assigned stack slots, for -O1+ -g they should be tracked
   21711              :          by VTA.  */
   21712       184356 :       if (!ignored_p
   21713       184356 :           && TYPE_DOMAIN (type)
   21714       368487 :           && INTEGRAL_TYPE_P (TYPE_DOMAIN (type)))
   21715              :         {
   21716       184131 :           t = TYPE_MIN_VALUE (TYPE_DOMAIN (type));
   21717       184131 :           if (t && VAR_P (t) && DECL_ARTIFICIAL (t))
   21718            0 :             DECL_IGNORED_P (t) = 0;
   21719       184131 :           t = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
   21720       184131 :           if (t && VAR_P (t) && DECL_ARTIFICIAL (t))
   21721        19162 :             DECL_IGNORED_P (t) = 0;
   21722              :         }
   21723              :       break;
   21724              : 
   21725       508289 :     case RECORD_TYPE:
   21726       508289 :     case UNION_TYPE:
   21727       508289 :     case QUAL_UNION_TYPE:
   21728     11185182 :       for (tree field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
   21729     10676893 :         if (TREE_CODE (field) == FIELD_DECL)
   21730              :           {
   21731      1102546 :             gimplify_one_sizepos (&DECL_FIELD_OFFSET (field), list_p);
   21732              :             /* Likewise, ensure variable offsets aren't removed.  */
   21733      1102546 :             if (!ignored_p
   21734      1102546 :                 && (t = DECL_FIELD_OFFSET (field))
   21735      1102534 :                 && VAR_P (t)
   21736      1102708 :                 && DECL_ARTIFICIAL (t))
   21737          162 :               DECL_IGNORED_P (t) = 0;
   21738      1102546 :             gimplify_one_sizepos (&DECL_SIZE (field), list_p);
   21739      1102546 :             gimplify_one_sizepos (&DECL_SIZE_UNIT (field), list_p);
   21740      1102546 :             gimplify_type_sizes (TREE_TYPE (field), list_p);
   21741              :           }
   21742              :       break;
   21743              : 
   21744              :     case POINTER_TYPE:
   21745              :     case REFERENCE_TYPE:
   21746              :         /* We used to recurse on the pointed-to type here, which turned out to
   21747              :            be incorrect because its definition might refer to variables not
   21748              :            yet initialized at this point if a forward declaration is involved.
   21749              : 
   21750              :            It was actually useful for anonymous pointed-to types to ensure
   21751              :            that the sizes evaluation dominates every possible later use of the
   21752              :            values.  Restricting to such types here would be safe since there
   21753              :            is no possible forward declaration around, but would introduce an
   21754              :            undesirable middle-end semantic to anonymity.  We then defer to
   21755              :            front-ends the responsibility of ensuring that the sizes are
   21756              :            evaluated both early and late enough, e.g. by attaching artificial
   21757              :            type declarations to the tree.  */
   21758              :       break;
   21759              : 
   21760              :     default:
   21761              :       break;
   21762              :     }
   21763              : 
   21764      2507555 :   gimplify_one_sizepos (&TYPE_SIZE (type), list_p);
   21765      2507555 :   gimplify_one_sizepos (&TYPE_SIZE_UNIT (type), list_p);
   21766              : 
   21767     16731224 :   for (t = TYPE_NEXT_VARIANT (type); t; t = TYPE_NEXT_VARIANT (t))
   21768              :     {
   21769     14223669 :       TYPE_SIZE (t) = TYPE_SIZE (type);
   21770     14223669 :       TYPE_SIZE_UNIT (t) = TYPE_SIZE_UNIT (type);
   21771     14223669 :       TYPE_SIZES_GIMPLIFIED (t) = 1;
   21772              :     }
   21773              : }
   21774              : 
   21775              : /* A subroutine of gimplify_type_sizes to make sure that *EXPR_P,
   21776              :    a size or position, has had all of its SAVE_EXPRs evaluated.
   21777              :    We add any required statements to *STMT_P.  */
   21778              : 
   21779              : void
   21780      9344340 : gimplify_one_sizepos (tree *expr_p, gimple_seq *stmt_p)
   21781              : {
   21782      9344340 :   tree expr = *expr_p;
   21783              : 
   21784              :   /* We don't do anything if the value isn't there, is constant, or contains
   21785              :      A PLACEHOLDER_EXPR.  We also don't want to do anything if it's already
   21786              :      a VAR_DECL.  If it's a VAR_DECL from another function, the gimplifier
   21787              :      will want to replace it with a new variable, but that will cause problems
   21788              :      if this type is from outside the function.  It's OK to have that here.  */
   21789      9344340 :   if (expr == NULL_TREE
   21790      9425980 :       || is_gimple_constant (expr)
   21791        81640 :       || VAR_P (expr)
   21792      9421823 :       || CONTAINS_PLACEHOLDER_P (expr))
   21793              :     return;
   21794              : 
   21795        77483 :   *expr_p = unshare_expr (expr);
   21796              : 
   21797              :   /* SSA names in decl/type fields are a bad idea - they'll get reclaimed
   21798              :      if the def vanishes.  */
   21799        77483 :   gimplify_expr (expr_p, stmt_p, NULL, is_gimple_val, fb_rvalue, false);
   21800              : 
   21801              :   /* If expr wasn't already is_gimple_sizepos or is_gimple_constant from the
   21802              :      FE, ensure that it is a VAR_DECL, otherwise we might handle some decls
   21803              :      as gimplify_vla_decl even when they would have all sizes INTEGER_CSTs.  */
   21804        77483 :   if (is_gimple_constant (*expr_p))
   21805          548 :     *expr_p = get_initialized_tmp_var (*expr_p, stmt_p, NULL, false);
   21806              : }
   21807              : 
   21808              : /* Gimplify the body of statements of FNDECL and return a GIMPLE_BIND node
   21809              :    containing the sequence of corresponding GIMPLE statements.  If DO_PARMS
   21810              :    is true, also gimplify the parameters.  */
   21811              : 
   21812              : gbind *
   21813      3010798 : gimplify_body (tree fndecl, bool do_parms)
   21814              : {
   21815      3010798 :   location_t saved_location = input_location;
   21816      3010798 :   gimple_seq parm_stmts, parm_cleanup = NULL, seq;
   21817      3010798 :   gimple *outer_stmt;
   21818      3010798 :   gbind *outer_bind;
   21819              : 
   21820      3010798 :   timevar_push (TV_TREE_GIMPLIFY);
   21821              : 
   21822      3010798 :   init_tree_ssa (cfun);
   21823              : 
   21824              :   /* Initialize for optimize_insn_for_s{ize,peed}_p possibly called during
   21825              :      gimplification.  */
   21826      3010798 :   default_rtl_profile ();
   21827              : 
   21828      3010798 :   gcc_assert (gimplify_ctxp == NULL);
   21829      3010798 :   push_gimplify_context (true);
   21830              : 
   21831      3010798 :   if (flag_openacc || flag_openmp)
   21832              :     {
   21833        57477 :       gcc_assert (gimplify_omp_ctxp == NULL);
   21834        57477 :       if (lookup_attribute ("omp declare target", DECL_ATTRIBUTES (fndecl)))
   21835         9780 :         gimplify_omp_ctxp = new_omp_context (ORT_IMPLICIT_TARGET);
   21836              :     }
   21837              : 
   21838              :   /* Unshare most shared trees in the body and in that of any nested functions.
   21839              :      It would seem we don't have to do this for nested functions because
   21840              :      they are supposed to be output and then the outer function gimplified
   21841              :      first, but the g++ front end doesn't always do it that way.  */
   21842      3010798 :   unshare_body (fndecl);
   21843      3010798 :   unvisit_body (fndecl);
   21844              : 
   21845              :   /* Make sure input_location isn't set to something weird.  */
   21846      3010798 :   input_location = DECL_SOURCE_LOCATION (fndecl);
   21847              : 
   21848              :   /* Resolve callee-copies.  This has to be done before processing
   21849              :      the body so that DECL_VALUE_EXPR gets processed correctly.  */
   21850      3010798 :   parm_stmts = do_parms ? gimplify_parameters (&parm_cleanup) : NULL;
   21851              : 
   21852              :   /* Gimplify the function's body.  */
   21853      3010798 :   seq = NULL;
   21854      3010798 :   gimplify_stmt (&DECL_SAVED_TREE (fndecl), &seq);
   21855      3010798 :   outer_stmt = gimple_seq_first_nondebug_stmt (seq);
   21856      3010798 :   if (!outer_stmt)
   21857              :     {
   21858        27107 :       outer_stmt = gimple_build_nop ();
   21859        27107 :       gimplify_seq_add_stmt (&seq, outer_stmt);
   21860              :     }
   21861              : 
   21862              :   /* The body must contain exactly one statement, a GIMPLE_BIND.  If this is
   21863              :      not the case, wrap everything in a GIMPLE_BIND to make it so.  */
   21864      3010798 :   if (gimple_code (outer_stmt) == GIMPLE_BIND
   21865      3010798 :       && (gimple_seq_first_nondebug_stmt (seq)
   21866      1745881 :           == gimple_seq_last_nondebug_stmt (seq)))
   21867              :     {
   21868      1680156 :       outer_bind = as_a <gbind *> (outer_stmt);
   21869      1680156 :       if (gimple_seq_first_stmt (seq) != outer_stmt
   21870      1680156 :           || gimple_seq_last_stmt (seq) != outer_stmt)
   21871              :         {
   21872              :           /* If there are debug stmts before or after outer_stmt, move them
   21873              :              inside of outer_bind body.  */
   21874            1 :           gimple_stmt_iterator gsi = gsi_for_stmt (outer_stmt, &seq);
   21875            1 :           gimple_seq second_seq = NULL;
   21876            1 :           if (gimple_seq_first_stmt (seq) != outer_stmt
   21877            2 :               && gimple_seq_last_stmt (seq) != outer_stmt)
   21878              :             {
   21879            0 :               second_seq = gsi_split_seq_after (gsi);
   21880            0 :               gsi_remove (&gsi, false);
   21881              :             }
   21882            1 :           else if (gimple_seq_first_stmt (seq) != outer_stmt)
   21883            1 :             gsi_remove (&gsi, false);
   21884              :           else
   21885              :             {
   21886            0 :               gsi_remove (&gsi, false);
   21887            0 :               second_seq = seq;
   21888            0 :               seq = NULL;
   21889              :             }
   21890            1 :           gimple_seq_add_seq_without_update (&seq,
   21891              :                                              gimple_bind_body (outer_bind));
   21892            1 :           gimple_seq_add_seq_without_update (&seq, second_seq);
   21893            1 :           gimple_bind_set_body (outer_bind, seq);
   21894              :         }
   21895              :     }
   21896              :   else
   21897      1330642 :     outer_bind = gimple_build_bind (NULL_TREE, seq, NULL);
   21898              : 
   21899      3010798 :   DECL_SAVED_TREE (fndecl) = NULL_TREE;
   21900              : 
   21901              :   /* If we had callee-copies statements, insert them at the beginning
   21902              :      of the function and clear DECL_HAS_VALUE_EXPR_P on the parameters.  */
   21903      3010798 :   if (!gimple_seq_empty_p (parm_stmts))
   21904              :     {
   21905           41 :       tree parm;
   21906              : 
   21907           41 :       gimplify_seq_add_seq (&parm_stmts, gimple_bind_body (outer_bind));
   21908           41 :       if (parm_cleanup)
   21909              :         {
   21910            0 :           gtry *g = gimple_build_try (parm_stmts, parm_cleanup,
   21911              :                                       GIMPLE_TRY_FINALLY);
   21912            0 :           parm_stmts = NULL;
   21913            0 :           gimple_seq_add_stmt (&parm_stmts, g);
   21914              :         }
   21915           41 :       gimple_bind_set_body (outer_bind, parm_stmts);
   21916              : 
   21917           41 :       for (parm = DECL_ARGUMENTS (current_function_decl);
   21918          101 :            parm; parm = DECL_CHAIN (parm))
   21919           60 :         if (DECL_HAS_VALUE_EXPR_P (parm))
   21920              :           {
   21921            0 :             DECL_HAS_VALUE_EXPR_P (parm) = 0;
   21922            0 :             DECL_IGNORED_P (parm) = 0;
   21923              :           }
   21924              :     }
   21925              : 
   21926      3010798 :   if ((flag_openacc || flag_openmp || flag_openmp_simd)
   21927        59772 :       && gimplify_omp_ctxp)
   21928              :     {
   21929         9833 :       delete_omp_context (gimplify_omp_ctxp);
   21930         9833 :       gimplify_omp_ctxp = NULL;
   21931              :     }
   21932              : 
   21933      3010798 :   pop_gimplify_context (outer_bind);
   21934      3010798 :   gcc_assert (gimplify_ctxp == NULL);
   21935              : 
   21936      3010798 :   if (flag_checking && !seen_error ())
   21937      2958490 :     verify_gimple_in_seq (gimple_bind_body (outer_bind));
   21938              : 
   21939      3010798 :   timevar_pop (TV_TREE_GIMPLIFY);
   21940      3010798 :   input_location = saved_location;
   21941              : 
   21942      3010798 :   return outer_bind;
   21943              : }
   21944              : 
   21945              : typedef char *char_p; /* For DEF_VEC_P.  */
   21946              : 
   21947              : /* Return whether we should exclude FNDECL from instrumentation.  */
   21948              : 
   21949              : static bool
   21950           43 : flag_instrument_functions_exclude_p (tree fndecl)
   21951              : {
   21952           43 :   vec<char_p> *v;
   21953              : 
   21954           43 :   v = (vec<char_p> *) flag_instrument_functions_exclude_functions;
   21955           44 :   if (v && v->length () > 0)
   21956              :     {
   21957            1 :       const char *name;
   21958            1 :       int i;
   21959            1 :       char *s;
   21960              : 
   21961            1 :       name = lang_hooks.decl_printable_name (fndecl, 1);
   21962            2 :       FOR_EACH_VEC_ELT (*v, i, s)
   21963            1 :         if (strstr (name, s) != NULL)
   21964           43 :           return true;
   21965              :     }
   21966              : 
   21967           42 :   v = (vec<char_p> *) flag_instrument_functions_exclude_files;
   21968           43 :   if (v && v->length () > 0)
   21969              :     {
   21970            1 :       const char *name;
   21971            1 :       int i;
   21972            1 :       char *s;
   21973              : 
   21974            1 :       name = DECL_SOURCE_FILE (fndecl);
   21975            1 :       FOR_EACH_VEC_ELT (*v, i, s)
   21976            1 :         if (strstr (name, s) != NULL)
   21977           43 :           return true;
   21978              :     }
   21979              : 
   21980              :   return false;
   21981              : }
   21982              : 
   21983              : /* Build a call to the instrumentation function FNCODE and add it to SEQ.
   21984              :    If COND_VAR is not NULL, it is a boolean variable guarding the call to
   21985              :    the instrumentation function.  IF STMT is not NULL, it is a statement
   21986              :    to be executed just before the call to the instrumentation function.  */
   21987              : 
   21988              : static void
   21989           82 : build_instrumentation_call (gimple_seq *seq, enum built_in_function fncode,
   21990              :                             tree cond_var, gimple *stmt)
   21991              : {
   21992              :   /* The instrumentation hooks aren't going to call the instrumented
   21993              :      function and the address they receive is expected to be matchable
   21994              :      against symbol addresses.  Make sure we don't create a trampoline,
   21995              :      in case the current function is nested.  */
   21996           82 :   tree this_fn_addr = build_fold_addr_expr (current_function_decl);
   21997           82 :   TREE_NO_TRAMPOLINE (this_fn_addr) = 1;
   21998              : 
   21999           82 :   tree label_true, label_false;
   22000           82 :   if (cond_var)
   22001              :     {
   22002           20 :       label_true = create_artificial_label (UNKNOWN_LOCATION);
   22003           20 :       label_false = create_artificial_label (UNKNOWN_LOCATION);
   22004           20 :       gcond *cond = gimple_build_cond (EQ_EXPR, cond_var, boolean_false_node,
   22005              :                                       label_true, label_false);
   22006           20 :       gimplify_seq_add_stmt (seq, cond);
   22007           20 :       gimplify_seq_add_stmt (seq, gimple_build_label (label_true));
   22008           20 :       gimplify_seq_add_stmt (seq, gimple_build_predict (PRED_COLD_LABEL,
   22009              :                                                         NOT_TAKEN));
   22010              :     }
   22011              : 
   22012           82 :   if (stmt)
   22013           10 :     gimplify_seq_add_stmt (seq, stmt);
   22014              : 
   22015           82 :   tree x = builtin_decl_implicit (BUILT_IN_RETURN_ADDRESS);
   22016           82 :   gcall *call = gimple_build_call (x, 1, integer_zero_node);
   22017           82 :   tree tmp_var = create_tmp_var (ptr_type_node, "return_addr");
   22018           82 :   gimple_call_set_lhs (call, tmp_var);
   22019           82 :   gimplify_seq_add_stmt (seq, call);
   22020           82 :   x = builtin_decl_implicit (fncode);
   22021           82 :   call = gimple_build_call (x, 2, this_fn_addr, tmp_var);
   22022           82 :   gimplify_seq_add_stmt (seq, call);
   22023              : 
   22024           82 :   if (cond_var)
   22025           20 :     gimplify_seq_add_stmt (seq, gimple_build_label (label_false));
   22026           82 : }
   22027              : 
   22028              : /* Entry point to the gimplification pass.  FNDECL is the FUNCTION_DECL
   22029              :    node for the function we want to gimplify.
   22030              : 
   22031              :    Return the sequence of GIMPLE statements corresponding to the body
   22032              :    of FNDECL.  */
   22033              : 
   22034              : void
   22035      3010285 : gimplify_function_tree (tree fndecl)
   22036              : {
   22037      3010285 :   gimple_seq seq;
   22038      3010285 :   gbind *bind;
   22039              : 
   22040      3010285 :   gcc_assert (!gimple_body (fndecl));
   22041              : 
   22042      3010285 :   if (DECL_STRUCT_FUNCTION (fndecl))
   22043      3005885 :     push_cfun (DECL_STRUCT_FUNCTION (fndecl));
   22044              :   else
   22045         4400 :     push_struct_function (fndecl);
   22046              : 
   22047      3010285 :   reset_cond_uid ();
   22048      3010285 :   if (cond_uids)
   22049              :     {
   22050           68 :       delete cond_uids;
   22051           68 :       cond_uids = NULL;
   22052              :     }
   22053              : 
   22054              :   /* Tentatively set PROP_gimple_lva here, and reset it in gimplify_va_arg_expr
   22055              :      if necessary.  */
   22056      3010285 :   cfun->curr_properties |= PROP_gimple_lva;
   22057              : 
   22058      3010285 :   if (asan_sanitize_use_after_scope ())
   22059         8680 :     asan_poisoned_variables = new hash_set<tree> ();
   22060      3010285 :   if (flag_openmp)
   22061        49739 :     omp_resolved_variant_calls = new hash_set<tree> ();
   22062              : 
   22063      3010285 :   bind = gimplify_body (fndecl, true);
   22064              : 
   22065      3010285 :   if (omp_resolved_variant_calls)
   22066              :     {
   22067        49739 :       delete omp_resolved_variant_calls;
   22068        49739 :       omp_resolved_variant_calls = NULL;
   22069              :     }
   22070      3010285 :   if (asan_poisoned_variables)
   22071              :     {
   22072         8680 :       delete asan_poisoned_variables;
   22073         8680 :       asan_poisoned_variables = NULL;
   22074              :     }
   22075              : 
   22076              :   /* The tree body of the function is no longer needed, replace it
   22077              :      with the new GIMPLE body.  */
   22078      3010285 :   seq = NULL;
   22079      3010285 :   gimple_seq_add_stmt (&seq, bind);
   22080      3010285 :   gimple_set_body (fndecl, seq);
   22081              : 
   22082              :   /* If we're instrumenting function entry/exit, then prepend the call to
   22083              :      the entry hook and wrap the whole function in a TRY_FINALLY_EXPR to
   22084              :      catch the exit hook.  */
   22085              :   /* ??? Add some way to ignore exceptions for this TFE.  */
   22086      3010285 :   if (flag_instrument_function_entry_exit
   22087           97 :       && !DECL_NO_INSTRUMENT_FUNCTION_ENTRY_EXIT (fndecl)
   22088              :       /* Do not instrument extern inline functions.  */
   22089           44 :       && !(DECL_DECLARED_INLINE_P (fndecl)
   22090            6 :            && DECL_EXTERNAL (fndecl)
   22091            1 :            && DECL_DISREGARD_INLINE_LIMITS (fndecl))
   22092      3010328 :       && !flag_instrument_functions_exclude_p (fndecl))
   22093              :     {
   22094           41 :       gimple_seq body = NULL, cleanup = NULL;
   22095           41 :       gassign *assign;
   22096           41 :       tree cond_var;
   22097              : 
   22098              :       /* If -finstrument-functions-once is specified, generate:
   22099              : 
   22100              :            static volatile bool C.0 = false;
   22101              :            bool tmp_called;
   22102              : 
   22103              :            tmp_called = C.0;
   22104              :            if (!tmp_called)
   22105              :              {
   22106              :                C.0 = true;
   22107              :                [call profiling enter function]
   22108              :              }
   22109              : 
   22110              :          without specific protection for data races.  */
   22111           41 :       if (flag_instrument_function_entry_exit > 1)
   22112              :         {
   22113           10 :           tree first_var
   22114           10 :             = build_decl (DECL_SOURCE_LOCATION (current_function_decl),
   22115              :                           VAR_DECL,
   22116              :                           create_tmp_var_name ("C"),
   22117              :                           boolean_type_node);
   22118           10 :           DECL_ARTIFICIAL (first_var) = 1;
   22119           10 :           DECL_IGNORED_P (first_var) = 1;
   22120           10 :           TREE_STATIC (first_var) = 1;
   22121           10 :           TREE_THIS_VOLATILE (first_var) = 1;
   22122           10 :           TREE_USED (first_var) = 1;
   22123           10 :           DECL_INITIAL (first_var) = boolean_false_node;
   22124           10 :           varpool_node::add (first_var);
   22125              : 
   22126           10 :           cond_var = create_tmp_var (boolean_type_node, "tmp_called");
   22127           10 :           assign = gimple_build_assign (cond_var, first_var);
   22128           10 :           gimplify_seq_add_stmt (&body, assign);
   22129              : 
   22130           10 :           assign = gimple_build_assign (first_var, boolean_true_node);
   22131              :         }
   22132              : 
   22133              :       else
   22134              :         {
   22135              :           cond_var = NULL_TREE;
   22136              :           assign = NULL;
   22137              :         }
   22138              : 
   22139           41 :       build_instrumentation_call (&body, BUILT_IN_PROFILE_FUNC_ENTER,
   22140              :                                   cond_var, assign);
   22141              : 
   22142              :       /* If -finstrument-functions-once is specified, generate:
   22143              : 
   22144              :            if (!tmp_called)
   22145              :              [call profiling exit function]
   22146              : 
   22147              :          without specific protection for data races.  */
   22148           41 :       build_instrumentation_call (&cleanup, BUILT_IN_PROFILE_FUNC_EXIT,
   22149              :                                   cond_var, NULL);
   22150              : 
   22151           41 :       gimple *tf = gimple_build_try (seq, cleanup, GIMPLE_TRY_FINALLY);
   22152           41 :       gimplify_seq_add_stmt (&body, tf);
   22153           41 :       gbind *new_bind = gimple_build_bind (NULL, body, NULL);
   22154              : 
   22155              :       /* Replace the current function body with the body
   22156              :          wrapped in the try/finally TF.  */
   22157           41 :       seq = NULL;
   22158           41 :       gimple_seq_add_stmt (&seq, new_bind);
   22159           41 :       gimple_set_body (fndecl, seq);
   22160           41 :       bind = new_bind;
   22161              :     }
   22162              : 
   22163      3010285 :   if (sanitize_flags_p (SANITIZE_THREAD)
   22164      3010285 :       && param_tsan_instrument_func_entry_exit)
   22165              :     {
   22166          963 :       gcall *call = gimple_build_call_internal (IFN_TSAN_FUNC_EXIT, 0);
   22167          963 :       gimple *tf = gimple_build_try (seq, call, GIMPLE_TRY_FINALLY);
   22168          963 :       gbind *new_bind = gimple_build_bind (NULL, tf, NULL);
   22169              :       /* Replace the current function body with the body
   22170              :          wrapped in the try/finally TF.  */
   22171          963 :       seq = NULL;
   22172          963 :       gimple_seq_add_stmt (&seq, new_bind);
   22173          963 :       gimple_set_body (fndecl, seq);
   22174              :     }
   22175              : 
   22176      3010285 :   DECL_SAVED_TREE (fndecl) = NULL_TREE;
   22177      3010285 :   cfun->curr_properties |= PROP_gimple_any;
   22178              : 
   22179      3010285 :   pop_cfun ();
   22180              : 
   22181      3010285 :   dump_function (TDI_gimple, fndecl);
   22182      3010285 : }
   22183              : 
   22184              : /* Return a dummy expression of type TYPE in order to keep going after an
   22185              :    error.  */
   22186              : 
   22187              : static tree
   22188           30 : dummy_object (tree type)
   22189              : {
   22190           30 :   tree t = build_int_cst (build_pointer_type (type), 0);
   22191           30 :   return build2 (MEM_REF, type, t, t);
   22192              : }
   22193              : 
   22194              : /* Gimplify __builtin_va_arg, aka VA_ARG_EXPR, which is not really a
   22195              :    builtin function, but a very special sort of operator.  */
   22196              : 
   22197              : enum gimplify_status
   22198        51217 : gimplify_va_arg_expr (tree *expr_p, gimple_seq *pre_p,
   22199              :                       gimple_seq *post_p ATTRIBUTE_UNUSED)
   22200              : {
   22201        51217 :   tree promoted_type, have_va_type;
   22202        51217 :   tree valist = TREE_OPERAND (*expr_p, 0);
   22203        51217 :   tree type = TREE_TYPE (*expr_p);
   22204        51217 :   tree t, tag, aptag;
   22205        51217 :   location_t loc = EXPR_LOCATION (*expr_p);
   22206              : 
   22207              :   /* Verify that valist is of the proper type.  */
   22208        51217 :   have_va_type = TREE_TYPE (valist);
   22209        51217 :   if (have_va_type == error_mark_node)
   22210              :     return GS_ERROR;
   22211        51198 :   have_va_type = targetm.canonical_va_list_type (have_va_type);
   22212        51198 :   if (have_va_type == NULL_TREE
   22213        51198 :       && POINTER_TYPE_P (TREE_TYPE (valist)))
   22214              :     /* Handle 'Case 1: Not an array type' from c-common.cc/build_va_arg.  */
   22215          270 :     have_va_type
   22216          270 :       = targetm.canonical_va_list_type (TREE_TYPE (TREE_TYPE (valist)));
   22217        51198 :   gcc_assert (have_va_type != NULL_TREE);
   22218              : 
   22219              :   /* Generate a diagnostic for requesting data of a type that cannot
   22220              :      be passed through `...' due to type promotion at the call site.  */
   22221        51198 :   if ((promoted_type = lang_hooks.types.type_promotes_to (type))
   22222              :            != type)
   22223              :     {
   22224           30 :       static bool gave_help;
   22225           30 :       bool warned;
   22226              :       /* Use the expansion point to handle cases such as passing bool (defined
   22227              :          in a system header) through `...'.  */
   22228           30 :       location_t xloc
   22229           30 :         = expansion_point_location_if_in_system_header (loc);
   22230              : 
   22231              :       /* Unfortunately, this is merely undefined, rather than a constraint
   22232              :          violation, so we cannot make this an error.  If this call is never
   22233              :          executed, the program is still strictly conforming.  */
   22234           30 :       auto_diagnostic_group d;
   22235           30 :       warned = warning_at (xloc, 0,
   22236              :                            "%qT is promoted to %qT when passed through %<...%>",
   22237              :                            type, promoted_type);
   22238           30 :       if (!gave_help && warned)
   22239              :         {
   22240           15 :           gave_help = true;
   22241           15 :           inform (xloc, "(so you should pass %qT not %qT to %<va_arg%>)",
   22242              :                   promoted_type, type);
   22243              :         }
   22244              : 
   22245              :       /* We can, however, treat "undefined" any way we please.
   22246              :          Call abort to encourage the user to fix the program.  */
   22247           23 :       if (warned)
   22248           23 :         inform (xloc, "if this code is reached, the program will abort");
   22249              :       /* Before the abort, allow the evaluation of the va_list
   22250              :          expression to exit or longjmp.  */
   22251           30 :       gimplify_and_add (valist, pre_p);
   22252           60 :       t = build_call_expr_loc (loc,
   22253              :                                builtin_decl_implicit (BUILT_IN_TRAP), 0);
   22254           30 :       gimplify_and_add (t, pre_p);
   22255              : 
   22256              :       /* This is dead code, but go ahead and finish so that the
   22257              :          mode of the result comes out right.  */
   22258           30 :       *expr_p = dummy_object (type);
   22259           30 :       return GS_ALL_DONE;
   22260           30 :     }
   22261              : 
   22262        51168 :   tag = build_int_cst (build_pointer_type (type), 0);
   22263        51168 :   aptag = build_int_cst (TREE_TYPE (valist), 0);
   22264              : 
   22265        51168 :   *expr_p = build_call_expr_internal_loc (loc, IFN_VA_ARG, type, 3,
   22266              :                                           valist, tag, aptag);
   22267              : 
   22268              :   /* Clear the tentatively set PROP_gimple_lva, to indicate that IFN_VA_ARG
   22269              :      needs to be expanded.  */
   22270        51168 :   cfun->curr_properties &= ~PROP_gimple_lva;
   22271              : 
   22272        51168 :   return GS_OK;
   22273              : }
   22274              : 
   22275              : /* Build a new GIMPLE_ASSIGN tuple and append it to the end of *SEQ_P.
   22276              : 
   22277              :    DST/SRC are the destination and source respectively.  You can pass
   22278              :    ungimplified trees in DST or SRC, in which case they will be
   22279              :    converted to a gimple operand if necessary.
   22280              : 
   22281              :    This function returns the newly created GIMPLE_ASSIGN tuple.  */
   22282              : 
   22283              : gimple *
   22284       918747 : gimplify_assign (tree dst, tree src, gimple_seq *seq_p)
   22285              : {
   22286       918747 :   tree t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
   22287       918747 :   gimplify_and_add (t, seq_p);
   22288       918747 :   ggc_free (t);
   22289       918747 :   return gimple_seq_last_stmt (*seq_p);
   22290              : }
   22291              : 
   22292              : inline hashval_t
   22293      1618290 : gimplify_hasher::hash (const elt_t *p)
   22294              : {
   22295      1618290 :   tree t = p->val;
   22296      1618290 :   return iterative_hash_expr (t, 0);
   22297              : }
   22298              : 
   22299              : inline bool
   22300       599376 : gimplify_hasher::equal (const elt_t *p1, const elt_t *p2)
   22301              : {
   22302       599376 :   tree t1 = p1->val;
   22303       599376 :   tree t2 = p2->val;
   22304       599376 :   enum tree_code code = TREE_CODE (t1);
   22305              : 
   22306       599376 :   if (TREE_CODE (t2) != code
   22307       599376 :       || TREE_TYPE (t1) != TREE_TYPE (t2))
   22308              :     return false;
   22309              : 
   22310       367319 :   if (!operand_equal_p (t1, t2, 0))
   22311              :     return false;
   22312              : 
   22313              :   return true;
   22314              : }
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.