LCOV - code coverage report
Current view: top level - gcc - omp-oacc-neuter-broadcast.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 0.7 % 940 7
Test Date: 2026-11-14 16:39:50 Functions: 5.7 % 35 2
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* OpenACC worker partitioning via middle end neutering/broadcasting scheme
       2              : 
       3              :    Copyright (C) 2015-2026 Free Software Foundation, Inc.
       4              : 
       5              :    This file is part of GCC.
       6              : 
       7              :    GCC is free software; you can redistribute it and/or modify it
       8              :    under the terms of the GNU General Public License as published
       9              :    by the Free Software Foundation; either version 3, or (at your
      10              :    option) any later version.
      11              : 
      12              :    GCC is distributed in the hope that it will be useful, but WITHOUT
      13              :    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
      14              :    or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
      15              :    License for more details.
      16              : 
      17              :    You should have received a copy of the GNU General Public License
      18              :    along with GCC; see the file COPYING3.  If not see
      19              :    <http://www.gnu.org/licenses/>.  */
      20              : 
      21              : #include "config.h"
      22              : #include "system.h"
      23              : #include "coretypes.h"
      24              : #include "backend.h"
      25              : #include "rtl.h"
      26              : #include "tree.h"
      27              : #include "gimple.h"
      28              : #include "tree-pass.h"
      29              : #include "ssa.h"
      30              : #include "cgraph.h"
      31              : #include "pretty-print.h"
      32              : #include "fold-const.h"
      33              : #include "gimplify.h"
      34              : #include "gimple-iterator.h"
      35              : #include "gimple-walk.h"
      36              : #include "tree-inline.h"
      37              : #include "langhooks.h"
      38              : #include "omp-general.h"
      39              : #include "omp-low.h"
      40              : #include "gimple-pretty-print.h"
      41              : #include "cfghooks.h"
      42              : #include "insn-config.h"
      43              : #include "recog.h"
      44              : #include "internal-fn.h"
      45              : #include "bitmap.h"
      46              : #include "tree-nested.h"
      47              : #include "stor-layout.h"
      48              : #include "tree-ssa-threadupdate.h"
      49              : #include "tree-into-ssa.h"
      50              : #include "splay-tree.h"
      51              : #include "target.h"
      52              : #include "cfgloop.h"
      53              : #include "tree-cfg.h"
      54              : #include "omp-offload.h"
      55              : #include "attribs.h"
      56              : #include "targhooks.h"
      57              : #include "diagnostic-core.h"
      58              : 
      59              : /* Loop structure of the function.  The entire function is described as
      60              :    a NULL loop.  */
      61              : /* Adapted from 'gcc/config/nvptx/nvptx.cc:struct parallel'.  */
      62              : 
      63              : struct parallel_g
      64              : {
      65              :   /* Parent parallel.  */
      66              :   parallel_g *parent;
      67              : 
      68              :   /* Next sibling parallel.  */
      69              :   parallel_g *next;
      70              : 
      71              :   /* First child parallel.  */
      72              :   parallel_g *inner;
      73              : 
      74              :   /* Partitioning mask of the parallel.  */
      75              :   unsigned mask;
      76              : 
      77              :   /* Partitioning used within inner parallels. */
      78              :   unsigned inner_mask;
      79              : 
      80              :   /* Location of parallel forked and join.  The forked is the first
      81              :      block in the parallel and the join is the first block after of
      82              :      the partition.  */
      83              :   basic_block forked_block;
      84              :   basic_block join_block;
      85              : 
      86              :   gimple *forked_stmt;
      87              :   gimple *join_stmt;
      88              : 
      89              :   gimple *fork_stmt;
      90              :   gimple *joining_stmt;
      91              : 
      92              :   /* Basic blocks in this parallel, but not in child parallels.  The
      93              :      FORKED and JOINING blocks are in the partition.  The FORK and JOIN
      94              :      blocks are not.  */
      95              :   auto_vec<basic_block> blocks;
      96              : 
      97              :   tree record_type;
      98              :   tree sender_decl;
      99              :   tree receiver_decl;
     100              : 
     101              : public:
     102              :   parallel_g (parallel_g *parent, unsigned mode);
     103              :   ~parallel_g ();
     104              : };
     105              : 
     106              : /* Constructor links the new parallel into it's parent's chain of
     107              :    children.  */
     108              : 
     109            0 : parallel_g::parallel_g (parallel_g *parent_, unsigned mask_)
     110            0 :   :parent (parent_), next (0), inner (0), mask (mask_), inner_mask (0)
     111              : {
     112            0 :   forked_block = join_block = 0;
     113            0 :   forked_stmt = join_stmt = NULL;
     114            0 :   fork_stmt = joining_stmt = NULL;
     115              : 
     116            0 :   record_type = NULL_TREE;
     117            0 :   sender_decl = NULL_TREE;
     118            0 :   receiver_decl = NULL_TREE;
     119              : 
     120            0 :   if (parent)
     121              :     {
     122            0 :       next = parent->inner;
     123            0 :       parent->inner = this;
     124              :     }
     125            0 : }
     126              : 
     127            0 : parallel_g::~parallel_g ()
     128              : {
     129            0 :   delete inner;
     130            0 :   delete next;
     131            0 : }
     132              : 
     133              : static bool
     134            0 : local_var_based_p (tree decl)
     135              : {
     136            0 :   switch (TREE_CODE (decl))
     137              :     {
     138            0 :     case VAR_DECL:
     139            0 :       return !is_global_var (decl);
     140              : 
     141            0 :     case COMPONENT_REF:
     142            0 :     case BIT_FIELD_REF:
     143            0 :     case ARRAY_REF:
     144            0 :       return local_var_based_p (TREE_OPERAND (decl, 0));
     145              : 
     146              :     default:
     147              :       return false;
     148              :     }
     149              : }
     150              : 
     151              : /* Map of basic blocks to gimple stmts.  */
     152              : typedef hash_map<basic_block, gimple *> bb_stmt_map_t;
     153              : 
     154              : /* Calls to OpenACC routines are made by all workers/wavefronts/warps, since
     155              :    the routine likely contains partitioned loops (else will do its own
     156              :    neutering and variable propagation). Return TRUE if a function call CALL
     157              :    should be made in (worker) single mode instead, rather than redundant
     158              :    mode.  */
     159              : 
     160              : static bool
     161            0 : omp_sese_active_worker_call (gcall *call)
     162              : {
     163              : #define GOMP_DIM_SEQ GOMP_DIM_MAX
     164            0 :   tree fndecl = gimple_call_fndecl (call);
     165              : 
     166            0 :   if (!fndecl)
     167              :     return true;
     168              : 
     169            0 :   tree attrs = oacc_get_fn_attrib (fndecl);
     170              : 
     171            0 :   if (!attrs)
     172              :     return true;
     173              : 
     174            0 :   int level = oacc_fn_attrib_level (attrs);
     175              : 
     176              :   /* Neither regular functions nor "seq" routines should be run by all threads
     177              :      in worker-single mode.  */
     178            0 :   return level == -1 || level == GOMP_DIM_SEQ;
     179              : #undef GOMP_DIM_SEQ
     180              : }
     181              : 
     182              : /* Split basic blocks such that each forked and join unspecs are at
     183              :    the start of their basic blocks.  Thus afterwards each block will
     184              :    have a single partitioning mode.  We also do the same for return
     185              :    insns, as they are executed by every thread.  Return the
     186              :    partitioning mode of the function as a whole.  Populate MAP with
     187              :    head and tail blocks.  We also clear the BB visited flag, which is
     188              :    used when finding partitions.  */
     189              : /* Adapted from 'gcc/config/nvptx/nvptx.cc:nvptx_split_blocks'.  */
     190              : 
     191              : static void
     192            0 : omp_sese_split_blocks (bb_stmt_map_t *map)
     193              : {
     194            0 :   auto_vec<gimple *> worklist;
     195            0 :   basic_block block;
     196              : 
     197              :   /* Locate all the reorg instructions of interest.  */
     198            0 :   FOR_ALL_BB_FN (block, cfun)
     199              :     {
     200              :       /* Clear visited flag, for use by parallel locator  */
     201            0 :       block->flags &= ~BB_VISITED;
     202              : 
     203            0 :       for (gimple_stmt_iterator gsi = gsi_start_bb (block);
     204            0 :            !gsi_end_p (gsi);
     205            0 :            gsi_next (&gsi))
     206              :         {
     207            0 :           gimple *stmt = gsi_stmt (gsi);
     208              : 
     209            0 :           if (gimple_call_internal_p (stmt, IFN_UNIQUE))
     210              :             {
     211            0 :               enum ifn_unique_kind k = ((enum ifn_unique_kind)
     212            0 :                 TREE_INT_CST_LOW (gimple_call_arg (stmt, 0)));
     213              : 
     214            0 :               if (k == IFN_UNIQUE_OACC_JOIN)
     215            0 :                 worklist.safe_push (stmt);
     216            0 :               else if (k == IFN_UNIQUE_OACC_FORK)
     217              :                 {
     218            0 :                   gcc_assert (gsi_one_before_end_p (gsi));
     219            0 :                   basic_block forked_block = single_succ (block);
     220            0 :                   gimple_stmt_iterator gsi2 = gsi_start_bb (forked_block);
     221              : 
     222              :                   /* We push a NOP as a placeholder for the "forked" stmt.
     223              :                      This is then recognized in omp_sese_find_par.  */
     224            0 :                   gimple *nop = gimple_build_nop ();
     225            0 :                   gsi_insert_before (&gsi2, nop, GSI_SAME_STMT);
     226              : 
     227            0 :                   worklist.safe_push (nop);
     228              :                 }
     229              :             }
     230            0 :           else if (gimple_code (stmt) == GIMPLE_RETURN
     231            0 :                    || gimple_code (stmt) == GIMPLE_COND
     232            0 :                    || gimple_code (stmt) == GIMPLE_SWITCH
     233            0 :                    || (gimple_code (stmt) == GIMPLE_CALL
     234            0 :                        && !gimple_call_internal_p (stmt)
     235            0 :                        && !omp_sese_active_worker_call (as_a <gcall *> (stmt))))
     236            0 :             worklist.safe_push (stmt);
     237            0 :           else if (is_gimple_assign (stmt))
     238              :             {
     239            0 :               tree lhs = gimple_assign_lhs (stmt);
     240              : 
     241              :               /* Force assignments to components/fields/elements of local
     242              :                  aggregates into fully-partitioned (redundant) mode.  This
     243              :                  avoids having to broadcast the whole aggregate.  The RHS of
     244              :                  the assignment will be propagated using the normal
     245              :                  mechanism.  */
     246              : 
     247            0 :               switch (TREE_CODE (lhs))
     248              :                 {
     249            0 :                 case COMPONENT_REF:
     250            0 :                 case BIT_FIELD_REF:
     251            0 :                 case ARRAY_REF:
     252            0 :                   {
     253            0 :                     tree aggr = TREE_OPERAND (lhs, 0);
     254              : 
     255            0 :                     if (local_var_based_p (aggr))
     256            0 :                       worklist.safe_push (stmt);
     257              :                   }
     258              :                   break;
     259              : 
     260              :                 default:
     261              :                   ;
     262              :                 }
     263              :             }
     264              :         }
     265              :     }
     266              : 
     267              :   /* Split blocks on the worklist.  */
     268              :   unsigned ix;
     269              :   gimple *stmt;
     270              : 
     271            0 :   for (ix = 0; worklist.iterate (ix, &stmt); ix++)
     272              :     {
     273            0 :       basic_block block = gimple_bb (stmt);
     274              : 
     275            0 :       if (gimple_code (stmt) == GIMPLE_COND)
     276              :         {
     277            0 :           gcond *orig_cond = as_a <gcond *> (stmt);
     278            0 :           tree_code code = gimple_expr_code (orig_cond);
     279            0 :           tree pred = make_ssa_name (boolean_type_node);
     280            0 :           gimple *asgn = gimple_build_assign (pred, code,
     281              :                            gimple_cond_lhs (orig_cond),
     282              :                            gimple_cond_rhs (orig_cond));
     283            0 :           gcond *new_cond
     284            0 :             = gimple_build_cond (NE_EXPR, pred, boolean_false_node,
     285              :                                  gimple_cond_true_label (orig_cond),
     286              :                                  gimple_cond_false_label (orig_cond));
     287              : 
     288            0 :           gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
     289            0 :           gsi_insert_before (&gsi, asgn, GSI_SAME_STMT);
     290            0 :           gsi_replace (&gsi, new_cond, true);
     291              : 
     292            0 :           edge e = split_block (block, asgn);
     293            0 :           block = e->dest;
     294            0 :           map->get_or_insert (block) = new_cond;
     295              :         }
     296            0 :       else if ((gimple_code (stmt) == GIMPLE_CALL
     297            0 :                 && !gimple_call_internal_p (stmt))
     298            0 :                || is_gimple_assign (stmt))
     299              :         {
     300            0 :           gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
     301            0 :           gsi_prev (&gsi);
     302              : 
     303            0 :           edge call = split_block (block, gsi_stmt (gsi));
     304              : 
     305            0 :           gimple *call_stmt = gsi_stmt (gsi_start_bb (call->dest));
     306              : 
     307            0 :           edge call_to_ret = split_block (call->dest, call_stmt);
     308              : 
     309            0 :           map->get_or_insert (call_to_ret->src) = call_stmt;
     310              :         }
     311              :       else
     312              :         {
     313            0 :           gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
     314            0 :           gsi_prev (&gsi);
     315              : 
     316            0 :           if (gsi_end_p (gsi))
     317            0 :             map->get_or_insert (block) = stmt;
     318              :           else
     319              :             {
     320              :               /* Split block before insn. The insn is in the new block.  */
     321            0 :               edge e = split_block (block, gsi_stmt (gsi));
     322              : 
     323            0 :               block = e->dest;
     324            0 :               map->get_or_insert (block) = stmt;
     325              :             }
     326              :         }
     327              :     }
     328            0 : }
     329              : 
     330              : static const char *
     331            0 : mask_name (unsigned mask)
     332              : {
     333            0 :   switch (mask)
     334              :     {
     335              :     case 0: return "gang redundant";
     336            0 :     case 1: return "gang partitioned";
     337            0 :     case 2: return "worker partitioned";
     338            0 :     case 3: return "gang+worker partitioned";
     339            0 :     case 4: return "vector partitioned";
     340            0 :     case 5: return "gang+vector partitioned";
     341            0 :     case 6: return "worker+vector partitioned";
     342            0 :     case 7: return "fully partitioned";
     343            0 :     default: return "<illegal>";
     344              :     }
     345              : }
     346              : 
     347              : /* Dump this parallel and all its inner parallels.  */
     348              : /* Adapted from 'gcc/config/nvptx/nvptx.cc:nvptx_dump_pars'.  */
     349              : 
     350              : static void
     351            0 : omp_sese_dump_pars (parallel_g *par, unsigned depth)
     352              : {
     353            0 :   fprintf (dump_file, "%u: mask %d (%s) head=%d, tail=%d\n",
     354              :            depth, par->mask, mask_name (par->mask),
     355            0 :            par->forked_block ? par->forked_block->index : -1,
     356            0 :            par->join_block ? par->join_block->index : -1);
     357              : 
     358            0 :   fprintf (dump_file, "    blocks:");
     359              : 
     360            0 :   basic_block block;
     361            0 :   for (unsigned ix = 0; par->blocks.iterate (ix, &block); ix++)
     362            0 :     fprintf (dump_file, " %d", block->index);
     363            0 :   fprintf (dump_file, "\n");
     364            0 :   if (par->inner)
     365            0 :     omp_sese_dump_pars (par->inner, depth + 1);
     366              : 
     367            0 :   if (par->next)
     368              :     omp_sese_dump_pars (par->next, depth);
     369            0 : }
     370              : 
     371              : /* If BLOCK contains a fork/join marker, process it to create or
     372              :    terminate a loop structure.  Add this block to the current loop,
     373              :    and then walk successor blocks.   */
     374              : /* Adapted from 'gcc/config/nvptx/nvptx.cc:nvptx_find_par'.  */
     375              : 
     376              : static parallel_g *
     377            0 : omp_sese_find_par (bb_stmt_map_t *map, parallel_g *par, basic_block block)
     378              : {
     379            0 :   if (block->flags & BB_VISITED)
     380              :     return par;
     381            0 :   block->flags |= BB_VISITED;
     382              : 
     383            0 :   if (gimple **stmtp = map->get (block))
     384              :     {
     385            0 :       gimple *stmt = *stmtp;
     386              : 
     387            0 :       if (gimple_code (stmt) == GIMPLE_COND
     388            0 :           || gimple_code (stmt) == GIMPLE_SWITCH
     389            0 :           || gimple_code (stmt) == GIMPLE_RETURN
     390            0 :           || (gimple_code (stmt) == GIMPLE_CALL
     391            0 :               && !gimple_call_internal_p (stmt))
     392            0 :           || is_gimple_assign (stmt))
     393              :         {
     394              :           /* A single block that is forced to be at the maximum partition
     395              :              level.  Make a singleton par for it.  */
     396            0 :           par = new parallel_g (par, GOMP_DIM_MASK (GOMP_DIM_GANG)
     397              :                                    | GOMP_DIM_MASK (GOMP_DIM_WORKER)
     398            0 :                                    | GOMP_DIM_MASK (GOMP_DIM_VECTOR));
     399            0 :           par->forked_block = block;
     400            0 :           par->forked_stmt = stmt;
     401            0 :           par->blocks.safe_push (block);
     402            0 :           par = par->parent;
     403            0 :           goto walk_successors;
     404              :         }
     405            0 :       else if (gimple_nop_p (stmt))
     406              :         {
     407            0 :           basic_block pred = single_pred (block);
     408            0 :           gcc_assert (pred);
     409            0 :           gimple_stmt_iterator gsi = gsi_last_bb (pred);
     410            0 :           gimple *final_stmt = gsi_stmt (gsi);
     411              : 
     412            0 :           if (gimple_call_internal_p (final_stmt, IFN_UNIQUE))
     413              :             {
     414            0 :               gcall *call = as_a <gcall *> (final_stmt);
     415            0 :               enum ifn_unique_kind k = ((enum ifn_unique_kind)
     416            0 :                 TREE_INT_CST_LOW (gimple_call_arg (call, 0)));
     417              : 
     418            0 :               if (k == IFN_UNIQUE_OACC_FORK)
     419              :                 {
     420            0 :                   HOST_WIDE_INT dim
     421            0 :                     = TREE_INT_CST_LOW (gimple_call_arg (call, 2));
     422            0 :                   unsigned mask = (dim >= 0) ? GOMP_DIM_MASK (dim) : 0;
     423              : 
     424            0 :                   par = new parallel_g (par, mask);
     425            0 :                   par->forked_block = block;
     426            0 :                   par->forked_stmt = final_stmt;
     427            0 :                   par->fork_stmt = stmt;
     428              :                 }
     429              :               else
     430            0 :                 gcc_unreachable ();
     431              :             }
     432              :           else
     433            0 :             gcc_unreachable ();
     434              :         }
     435            0 :       else if (gimple_call_internal_p (stmt, IFN_UNIQUE))
     436              :         {
     437            0 :           gcall *call = as_a <gcall *> (stmt);
     438            0 :           enum ifn_unique_kind k = ((enum ifn_unique_kind)
     439            0 :             TREE_INT_CST_LOW (gimple_call_arg (call, 0)));
     440            0 :           if (k == IFN_UNIQUE_OACC_JOIN)
     441              :             {
     442            0 :               HOST_WIDE_INT dim = TREE_INT_CST_LOW (gimple_call_arg (stmt, 2));
     443            0 :               unsigned mask = (dim >= 0) ? GOMP_DIM_MASK (dim) : 0;
     444              : 
     445            0 :               gcc_assert (par->mask == mask);
     446            0 :               par->join_block = block;
     447            0 :               par->join_stmt = stmt;
     448            0 :               par = par->parent;
     449              :             }
     450              :           else
     451            0 :             gcc_unreachable ();
     452              :         }
     453              :       else
     454            0 :         gcc_unreachable ();
     455              :     }
     456              : 
     457            0 :   if (par)
     458              :     /* Add this block onto the current loop's list of blocks.  */
     459            0 :     par->blocks.safe_push (block);
     460              :   else
     461              :     /* This must be the entry block.  Create a NULL parallel.  */
     462            0 :     par = new parallel_g (0, 0);
     463              : 
     464            0 : walk_successors:
     465              :   /* Walk successor blocks.  */
     466            0 :   edge e;
     467            0 :   edge_iterator ei;
     468              : 
     469            0 :   FOR_EACH_EDGE (e, ei, block->succs)
     470            0 :     omp_sese_find_par (map, par, e->dest);
     471              : 
     472              :   return par;
     473              : }
     474              : 
     475              : /* DFS walk the CFG looking for fork & join markers.  Construct
     476              :    loop structures as we go.  MAP is a mapping of basic blocks
     477              :    to head & tail markers, discovered when splitting blocks.  This
     478              :    speeds up the discovery.  We rely on the BB visited flag having
     479              :    been cleared when splitting blocks.  */
     480              : /* Adapted from 'gcc/config/nvptx/nvptx.cc:nvptx_discover_pars'.  */
     481              : 
     482              : static parallel_g *
     483            0 : omp_sese_discover_pars (bb_stmt_map_t *map)
     484              : {
     485            0 :   basic_block block;
     486              : 
     487              :   /* Mark exit blocks as visited.  */
     488            0 :   block = EXIT_BLOCK_PTR_FOR_FN (cfun);
     489            0 :   block->flags |= BB_VISITED;
     490              : 
     491              :   /* And entry block as not.  */
     492            0 :   block = ENTRY_BLOCK_PTR_FOR_FN (cfun);
     493            0 :   block->flags &= ~BB_VISITED;
     494              : 
     495            0 :   parallel_g *par = omp_sese_find_par (map, 0, block);
     496              : 
     497            0 :   if (dump_file)
     498              :     {
     499            0 :       fprintf (dump_file, "\nLoops\n");
     500            0 :       omp_sese_dump_pars (par, 0);
     501            0 :       fprintf (dump_file, "\n");
     502              :     }
     503              : 
     504            0 :   return par;
     505              : }
     506              : 
     507              : static void
     508            0 : populate_single_mode_bitmaps (parallel_g *par, bitmap worker_single,
     509              :                               bitmap vector_single, unsigned outer_mask,
     510              :                               int depth)
     511              : {
     512            0 :   unsigned mask = outer_mask | par->mask;
     513              : 
     514            0 :   basic_block block;
     515              : 
     516            0 :   for (unsigned i = 0; par->blocks.iterate (i, &block); i++)
     517              :     {
     518            0 :       if ((mask & GOMP_DIM_MASK (GOMP_DIM_WORKER)) == 0)
     519            0 :         bitmap_set_bit (worker_single, block->index);
     520              : 
     521            0 :       if ((mask & GOMP_DIM_MASK (GOMP_DIM_VECTOR)) == 0)
     522            0 :         bitmap_set_bit (vector_single, block->index);
     523              :     }
     524              : 
     525            0 :   if (par->inner)
     526            0 :     populate_single_mode_bitmaps (par->inner, worker_single, vector_single,
     527              :                                   mask, depth + 1);
     528            0 :   if (par->next)
     529              :     populate_single_mode_bitmaps (par->next, worker_single, vector_single,
     530              :                                   outer_mask, depth);
     531            0 : }
     532              : 
     533              : /* A map from SSA names or var decls to record fields.  */
     534              : 
     535              : typedef hash_map<tree, tree> field_map_t;
     536              : 
     537              : /* For each propagation record type, this is a map from SSA names or var decls
     538              :    to propagate, to the field in the record type that should be used for
     539              :    transmission and reception.  */
     540              : 
     541              : typedef hash_map<tree, field_map_t> record_field_map_t;
     542              : 
     543              : static void
     544            0 : install_var_field (tree var, tree record_type, field_map_t *fields)
     545              : {
     546            0 :   tree name;
     547            0 :   char tmp[20];
     548              : 
     549            0 :   if (TREE_CODE (var) == SSA_NAME)
     550              :     {
     551            0 :       name = SSA_NAME_IDENTIFIER (var);
     552            0 :       if (!name)
     553              :         {
     554            0 :           sprintf (tmp, "_%u", (unsigned) SSA_NAME_VERSION (var));
     555            0 :           name = get_identifier (tmp);
     556              :         }
     557              :     }
     558            0 :   else if (VAR_P (var))
     559              :     {
     560            0 :       name = DECL_NAME (var);
     561            0 :       if (!name)
     562              :         {
     563            0 :           sprintf (tmp, "D_%u", (unsigned) DECL_UID (var));
     564            0 :           name = get_identifier (tmp);
     565              :         }
     566              :     }
     567              :   else
     568            0 :     gcc_unreachable ();
     569              : 
     570            0 :   gcc_assert (!fields->get (var));
     571              : 
     572            0 :   tree type = TREE_TYPE (var);
     573              : 
     574            0 :   if (POINTER_TYPE_P (type)
     575            0 :       && TYPE_RESTRICT (type))
     576            0 :     type = build_qualified_type (type,
     577            0 :                                  TYPE_QUALS (type)
     578              :                                  .without (TYPE_QUAL_RESTRICT));
     579              : 
     580            0 :   tree field = build_decl (BUILTINS_LOCATION, FIELD_DECL, name, type);
     581              : 
     582            0 :   if (VAR_P (var) && type == TREE_TYPE (var))
     583              :     {
     584            0 :       SET_DECL_ALIGN (field, DECL_ALIGN (var));
     585            0 :       DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
     586            0 :       TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
     587              :     }
     588              :   else
     589            0 :     SET_DECL_ALIGN (field, TYPE_ALIGN (type));
     590              : 
     591            0 :   fields->put (var, field);
     592              : 
     593            0 :   insert_field_into_struct (record_type, field);
     594            0 : }
     595              : 
     596              : /* Sets of SSA_NAMES or VAR_DECLs to propagate.  */
     597              : typedef hash_set<tree> propagation_set;
     598              : 
     599              : static void
     600            0 : find_ssa_names_to_propagate (parallel_g *par, unsigned outer_mask,
     601              :                              bitmap worker_single, bitmap vector_single,
     602              :                              vec<propagation_set *> *prop_set)
     603              : {
     604            0 :   unsigned mask = outer_mask | par->mask;
     605              : 
     606            0 :   if (par->inner)
     607            0 :     find_ssa_names_to_propagate (par->inner, mask, worker_single,
     608              :                                  vector_single, prop_set);
     609            0 :   if (par->next)
     610            0 :     find_ssa_names_to_propagate (par->next, outer_mask, worker_single,
     611              :                                  vector_single, prop_set);
     612              : 
     613            0 :   if (mask & GOMP_DIM_MASK (GOMP_DIM_WORKER))
     614              :     {
     615              :       basic_block block;
     616              :       int ix;
     617              : 
     618            0 :       for (ix = 0; par->blocks.iterate (ix, &block); ix++)
     619              :         {
     620            0 :           for (gphi_iterator psi = gsi_start_phis (block);
     621            0 :                !gsi_end_p (psi); gsi_next (&psi))
     622              :             {
     623            0 :               gphi *phi = psi.phi ();
     624            0 :               use_operand_p use;
     625            0 :               ssa_op_iter iter;
     626              : 
     627            0 :               FOR_EACH_PHI_ARG (use, phi, iter, SSA_OP_USE)
     628              :                 {
     629            0 :                   tree var = USE_FROM_PTR (use);
     630              : 
     631            0 :                   if (TREE_CODE (var) != SSA_NAME)
     632            0 :                     continue;
     633              : 
     634            0 :                   gimple *def_stmt = SSA_NAME_DEF_STMT (var);
     635              : 
     636            0 :                   if (gimple_nop_p (def_stmt))
     637            0 :                     continue;
     638              : 
     639            0 :                   basic_block def_bb = gimple_bb (def_stmt);
     640              : 
     641            0 :                   if (bitmap_bit_p (worker_single, def_bb->index))
     642              :                     {
     643            0 :                       if (!(*prop_set)[def_bb->index])
     644            0 :                         (*prop_set)[def_bb->index] = new propagation_set;
     645              : 
     646            0 :                       propagation_set *ws_prop = (*prop_set)[def_bb->index];
     647              : 
     648            0 :                       ws_prop->add (var);
     649              :                     }
     650              :                 }
     651              :             }
     652              : 
     653            0 :           for (gimple_stmt_iterator gsi = gsi_start_bb (block);
     654            0 :                !gsi_end_p (gsi); gsi_next (&gsi))
     655              :             {
     656            0 :               use_operand_p use;
     657            0 :               ssa_op_iter iter;
     658            0 :               gimple *stmt = gsi_stmt (gsi);
     659              : 
     660            0 :               FOR_EACH_SSA_USE_OPERAND (use, stmt, iter, SSA_OP_USE)
     661              :                 {
     662            0 :                   tree var = USE_FROM_PTR (use);
     663              : 
     664            0 :                   gimple *def_stmt = SSA_NAME_DEF_STMT (var);
     665              : 
     666            0 :                   if (gimple_nop_p (def_stmt))
     667            0 :                     continue;
     668              : 
     669            0 :                   basic_block def_bb = gimple_bb (def_stmt);
     670              : 
     671            0 :                   if (bitmap_bit_p (worker_single, def_bb->index))
     672              :                     {
     673            0 :                       if (!(*prop_set)[def_bb->index])
     674            0 :                         (*prop_set)[def_bb->index] = new propagation_set;
     675              : 
     676            0 :                       propagation_set *ws_prop = (*prop_set)[def_bb->index];
     677              : 
     678            0 :                       ws_prop->add (var);
     679              :                     }
     680              :                 }
     681              :             }
     682              :         }
     683              :     }
     684            0 : }
     685              : 
     686              : /* Callback for walk_gimple_stmt to find RHS VAR_DECLs (uses) in a
     687              :    statement.  */
     688              : 
     689              : static tree
     690            0 : find_partitioned_var_uses_1 (tree *node, int *, void *data)
     691              : {
     692            0 :   walk_stmt_info *wi = (walk_stmt_info *) data;
     693            0 :   hash_set<tree> *partitioned_var_uses = (hash_set<tree> *) wi->info;
     694              : 
     695            0 :   if (!wi->is_lhs && VAR_P (*node))
     696            0 :     partitioned_var_uses->add (*node);
     697              : 
     698            0 :   return NULL_TREE;
     699              : }
     700              : 
     701              : static void
     702            0 : find_partitioned_var_uses (parallel_g *par, unsigned outer_mask,
     703              :                            hash_set<tree> *partitioned_var_uses)
     704              : {
     705            0 :   unsigned mask = outer_mask | par->mask;
     706              : 
     707            0 :   if (par->inner)
     708            0 :     find_partitioned_var_uses (par->inner, mask, partitioned_var_uses);
     709            0 :   if (par->next)
     710            0 :     find_partitioned_var_uses (par->next, outer_mask, partitioned_var_uses);
     711              : 
     712            0 :   if (mask & GOMP_DIM_MASK (GOMP_DIM_WORKER))
     713              :     {
     714              :       basic_block block;
     715              :       int ix;
     716              : 
     717            0 :       for (ix = 0; par->blocks.iterate (ix, &block); ix++)
     718            0 :         for (gimple_stmt_iterator gsi = gsi_start_bb (block);
     719            0 :              !gsi_end_p (gsi); gsi_next (&gsi))
     720              :           {
     721            0 :             walk_stmt_info wi;
     722            0 :             memset (&wi, 0, sizeof (wi));
     723            0 :             wi.info = (void *) partitioned_var_uses;
     724            0 :             walk_gimple_stmt (&gsi, NULL, find_partitioned_var_uses_1, &wi);
     725              :           }
     726              :     }
     727            0 : }
     728              : 
     729              : /* Gang-private variables (typically placed in a GPU's shared memory) do not
     730              :    need to be processed by the worker-propagation mechanism.  Populate the
     731              :    GANG_PRIVATE_VARS set with any such variables found in the current
     732              :    function.  */
     733              : 
     734              : static void
     735            0 : find_gang_private_vars (hash_set<tree> *gang_private_vars)
     736              : {
     737            0 :   basic_block block;
     738              : 
     739            0 :   FOR_EACH_BB_FN (block, cfun)
     740              :     {
     741            0 :       for (gimple_stmt_iterator gsi = gsi_start_bb (block);
     742            0 :            !gsi_end_p (gsi);
     743            0 :            gsi_next (&gsi))
     744              :         {
     745            0 :           gimple *stmt = gsi_stmt (gsi);
     746              : 
     747            0 :           if (gimple_call_internal_p (stmt, IFN_UNIQUE))
     748              :             {
     749            0 :               enum ifn_unique_kind k = ((enum ifn_unique_kind)
     750            0 :                 TREE_INT_CST_LOW (gimple_call_arg (stmt, 0)));
     751            0 :               if (k == IFN_UNIQUE_OACC_PRIVATE)
     752              :                 {
     753            0 :                   HOST_WIDE_INT level
     754            0 :                     = TREE_INT_CST_LOW (gimple_call_arg (stmt, 2));
     755            0 :                   if (level != GOMP_DIM_GANG)
     756            0 :                     continue;
     757            0 :                   for (unsigned i = 3; i < gimple_call_num_args (stmt); i++)
     758              :                     {
     759            0 :                       tree arg = gimple_call_arg (stmt, i);
     760            0 :                       gcc_assert (TREE_CODE (arg) == ADDR_EXPR);
     761            0 :                       tree decl = TREE_OPERAND (arg, 0);
     762            0 :                       gang_private_vars->add (decl);
     763              :                     }
     764              :                 }
     765              :             }
     766              :         }
     767              :     }
     768            0 : }
     769              : 
     770              : static void
     771            0 : find_local_vars_to_propagate (parallel_g *par, unsigned outer_mask,
     772              :                               hash_set<tree> *partitioned_var_uses,
     773              :                               hash_set<tree> *gang_private_vars,
     774              :                               bitmap writes_gang_private,
     775              :                               vec<propagation_set *> *prop_set)
     776              : {
     777            0 :   unsigned mask = outer_mask | par->mask;
     778              : 
     779            0 :   if (par->inner)
     780            0 :     find_local_vars_to_propagate (par->inner, mask, partitioned_var_uses,
     781              :                                   gang_private_vars, writes_gang_private,
     782              :                                   prop_set);
     783            0 :   if (par->next)
     784            0 :     find_local_vars_to_propagate (par->next, outer_mask, partitioned_var_uses,
     785              :                                   gang_private_vars, writes_gang_private,
     786              :                                   prop_set);
     787              : 
     788            0 :   if (!(mask & GOMP_DIM_MASK (GOMP_DIM_WORKER)))
     789              :     {
     790              :       basic_block block;
     791              :       int ix;
     792              : 
     793            0 :       for (ix = 0; par->blocks.iterate (ix, &block); ix++)
     794              :         {
     795            0 :           for (gimple_stmt_iterator gsi = gsi_start_bb (block);
     796            0 :                !gsi_end_p (gsi); gsi_next (&gsi))
     797              :             {
     798            0 :               gimple *stmt = gsi_stmt (gsi);
     799            0 :               tree var;
     800            0 :               unsigned i;
     801              : 
     802            0 :               FOR_EACH_LOCAL_DECL (cfun, i, var)
     803              :                 {
     804            0 :                   if (!VAR_P (var)
     805            0 :                       || is_global_var (var)
     806            0 :                       || AGGREGATE_TYPE_P (TREE_TYPE (var))
     807            0 :                       || !partitioned_var_uses->contains (var))
     808            0 :                     continue;
     809              : 
     810            0 :                   if (stmt_may_clobber_ref_p (stmt, var))
     811              :                     {
     812            0 :                       if (dump_file)
     813              :                         {
     814            0 :                           fprintf (dump_file, "bb %u: local variable may be "
     815              :                                    "clobbered in %s mode: ", block->index,
     816              :                                    mask_name (mask));
     817            0 :                           print_generic_expr (dump_file, var, TDF_SLIM);
     818            0 :                           fprintf (dump_file, "\n");
     819              :                         }
     820              : 
     821            0 :                       if (gang_private_vars->contains (var))
     822              :                         {
     823              :                           /* If we write a gang-private variable, we want a
     824              :                              barrier at the end of the block.  */
     825            0 :                           bitmap_set_bit (writes_gang_private, block->index);
     826            0 :                           continue;
     827              :                         }
     828              : 
     829            0 :                       if (!(*prop_set)[block->index])
     830            0 :                         (*prop_set)[block->index] = new propagation_set;
     831              : 
     832            0 :                       propagation_set *ws_prop
     833            0 :                         = (*prop_set)[block->index];
     834              : 
     835            0 :                       ws_prop->add (var);
     836              :                     }
     837              :                 }
     838              :             }
     839              :         }
     840              :     }
     841            0 : }
     842              : 
     843              : /* Transform basic blocks FROM, TO (which may be the same block) into:
     844              :    if (GOACC_single_start ())
     845              :      BLOCK;
     846              :    GOACC_barrier ();
     847              :                               \  |  /
     848              :                               +----+
     849              :                               |    |        (new) predicate block
     850              :                               +----+--
     851              :    \  |  /   \  |  /            |t    \
     852              :    +----+    +----+           +----+  |
     853              :    |    |    |    |     ===>  |    |  | f   (old) from block
     854              :    +----+    +----+           +----+  |
     855              :      |       t/  \f             |    /
     856              :                               +----+/
     857              :   (split  (split before       |    |        skip block
     858              :   at end)   condition)        +----+
     859              :                               t/  \f
     860              : */
     861              : 
     862              : static void
     863            0 : worker_single_simple (basic_block from, basic_block to,
     864              :                       hash_set<tree> *def_escapes_block)
     865              : {
     866            0 :   gimple *call, *cond;
     867            0 :   tree lhs, decl;
     868            0 :   basic_block skip_block;
     869              : 
     870            0 :   gimple_stmt_iterator gsi = gsi_last_bb (to);
     871            0 :   if (EDGE_COUNT (to->succs) > 1)
     872              :     {
     873            0 :       gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_COND);
     874            0 :       gsi_prev (&gsi);
     875              :     }
     876            0 :   edge e = split_block (to, gsi_stmt (gsi));
     877            0 :   skip_block = e->dest;
     878              : 
     879            0 :   gimple_stmt_iterator start = gsi_after_labels (from);
     880              : 
     881            0 :   decl = builtin_decl_explicit (BUILT_IN_GOACC_SINGLE_START);
     882            0 :   lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)));
     883            0 :   call = gimple_build_call (decl, 0);
     884            0 :   gimple_call_set_lhs (call, lhs);
     885            0 :   gsi_insert_before (&start, call, GSI_NEW_STMT);
     886            0 :   update_stmt (call);
     887              : 
     888            0 :   cond = gimple_build_cond (EQ_EXPR, lhs,
     889              :                             fold_convert_loc (UNKNOWN_LOCATION,
     890            0 :                                               TREE_TYPE (lhs),
     891              :                                               boolean_true_node),
     892              :                             NULL_TREE, NULL_TREE);
     893            0 :   gsi_insert_after (&start, cond, GSI_NEW_STMT);
     894            0 :   update_stmt (cond);
     895              : 
     896            0 :   edge et = split_block (from, cond);
     897            0 :   et->flags &= ~EDGE_FALLTHRU;
     898            0 :   et->flags |= EDGE_TRUE_VALUE;
     899              :   /* Make the active worker the more probable path so we prefer fallthrough
     900              :      (letting the idle workers jump around more).  */
     901            0 :   et->probability = profile_probability::likely ();
     902              : 
     903            0 :   edge ef = make_edge (from, skip_block, EDGE_FALSE_VALUE);
     904            0 :   ef->probability = et->probability.invert ();
     905              : 
     906            0 :   basic_block neutered = split_edge (ef);
     907            0 :   gimple_stmt_iterator neut_gsi = gsi_last_bb (neutered);
     908              : 
     909            0 :   for (gsi = gsi_start_bb (et->dest); !gsi_end_p (gsi); gsi_next (&gsi))
     910              :     {
     911            0 :       gimple *stmt = gsi_stmt (gsi);
     912            0 :       ssa_op_iter iter;
     913            0 :       tree var;
     914              : 
     915            0 :       FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_DEF)
     916              :         {
     917            0 :           if (def_escapes_block->contains (var))
     918              :             {
     919            0 :               gphi *join_phi = create_phi_node (NULL_TREE, skip_block);
     920            0 :               create_new_def_for (var, join_phi,
     921              :                                   gimple_phi_result_ptr (join_phi));
     922            0 :               add_phi_arg (join_phi, var, e, UNKNOWN_LOCATION);
     923              : 
     924            0 :               tree neutered_def = copy_ssa_name (var, NULL);
     925              :               /* We really want "don't care" or some value representing
     926              :                  undefined here, but optimizers will probably get rid of the
     927              :                  zero-assignments anyway.  */
     928            0 :               gassign *zero = gimple_build_assign (neutered_def,
     929            0 :                                 build_zero_cst (TREE_TYPE (neutered_def)));
     930              : 
     931            0 :               gsi_insert_after (&neut_gsi, zero, GSI_CONTINUE_LINKING);
     932            0 :               update_stmt (zero);
     933              : 
     934            0 :               add_phi_arg (join_phi, neutered_def, single_succ_edge (neutered),
     935              :                            UNKNOWN_LOCATION);
     936            0 :               update_stmt (join_phi);
     937              :             }
     938              :         }
     939              :     }
     940            0 : }
     941              : 
     942              : static tree
     943            0 : build_receiver_ref (tree var, tree receiver_decl, field_map_t *fields)
     944              : {
     945            0 :   tree x = build_simple_mem_ref (receiver_decl);
     946            0 :   tree field = *fields->get (var);
     947            0 :   TREE_THIS_NOTRAP (x) = 1;
     948            0 :   x = omp_build_component_ref (x, field);
     949            0 :   return x;
     950              : }
     951              : 
     952              : static tree
     953            0 : build_sender_ref (tree var, tree sender_decl, field_map_t *fields)
     954              : {
     955            0 :   if (POINTER_TYPE_P (TREE_TYPE (sender_decl)))
     956            0 :     sender_decl = build_simple_mem_ref (sender_decl);
     957            0 :   tree field = *fields->get (var);
     958            0 :   return omp_build_component_ref (sender_decl, field);
     959              : }
     960              : 
     961              : static int
     962            0 : sort_by_ssa_version_or_uid (const void *p1, const void *p2)
     963              : {
     964            0 :   const tree t1 = *(const tree *)p1;
     965            0 :   const tree t2 = *(const tree *)p2;
     966              : 
     967            0 :   if (TREE_CODE (t1) == SSA_NAME && TREE_CODE (t2) == SSA_NAME)
     968            0 :     return SSA_NAME_VERSION (t1) - SSA_NAME_VERSION (t2);
     969            0 :   else if (TREE_CODE (t1) == SSA_NAME && TREE_CODE (t2) != SSA_NAME)
     970              :     return -1;
     971            0 :   else if (TREE_CODE (t1) != SSA_NAME && TREE_CODE (t2) == SSA_NAME)
     972              :     return 1;
     973              :   else
     974            0 :     return DECL_UID (t1) - DECL_UID (t2);
     975              : }
     976              : 
     977              : static int
     978            0 : sort_by_size_then_ssa_version_or_uid (const void *p1, const void *p2)
     979              : {
     980            0 :   const tree t1 = *(const tree *)p1;
     981            0 :   const tree t2 = *(const tree *)p2;
     982            0 :   unsigned HOST_WIDE_INT s1 = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (t1)));
     983            0 :   unsigned HOST_WIDE_INT s2 = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (t2)));
     984            0 :   if (s1 != s2)
     985            0 :     return s2 - s1;
     986              :   else
     987            0 :     return sort_by_ssa_version_or_uid (p1, p2);
     988              : }
     989              : 
     990              : static void
     991            0 : worker_single_copy (basic_block from, basic_block to,
     992              :                     hash_set<tree> *def_escapes_block,
     993              :                     hash_set<tree> *worker_partitioned_uses,
     994              :                     tree record_type, record_field_map_t *record_field_map,
     995              :                     unsigned HOST_WIDE_INT placement,
     996              :                     bool isolate_broadcasts, bool has_gang_private_write)
     997              : {
     998              :   /* If we only have virtual defs, we'll have no record type, but we still want
     999              :      to emit single_copy_start and (particularly) single_copy_end to act as
    1000              :      a vdef source on the neutered edge representing memory writes on the
    1001              :      non-neutered edge.  */
    1002            0 :   if (!record_type)
    1003            0 :     record_type = char_type_node;
    1004              : 
    1005            0 :   tree sender_decl
    1006            0 :     = targetm.goacc.create_worker_broadcast_record (record_type, true,
    1007              :                                                     ".oacc_worker_o",
    1008              :                                                     placement);
    1009            0 :   tree receiver_decl
    1010            0 :     = targetm.goacc.create_worker_broadcast_record (record_type, false,
    1011              :                                                     ".oacc_worker_i",
    1012              :                                                     placement);
    1013              : 
    1014            0 :   gimple_stmt_iterator gsi = gsi_last_bb (to);
    1015            0 :   if (EDGE_COUNT (to->succs) > 1)
    1016            0 :     gsi_prev (&gsi);
    1017            0 :   edge e = split_block (to, gsi_stmt (gsi));
    1018            0 :   basic_block barrier_block = e->dest;
    1019              : 
    1020            0 :   gimple_stmt_iterator start = gsi_after_labels (from);
    1021              : 
    1022            0 :   tree decl = builtin_decl_explicit (BUILT_IN_GOACC_SINGLE_COPY_START);
    1023              : 
    1024            0 :   tree lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)));
    1025              : 
    1026            0 :   gimple *call
    1027            0 :     = gimple_build_call (decl, 1,
    1028            0 :                          POINTER_TYPE_P (TREE_TYPE (sender_decl))
    1029            0 :                          ? sender_decl : build_fold_addr_expr (sender_decl));
    1030            0 :   gimple_call_set_lhs (call, lhs);
    1031            0 :   gsi_insert_before (&start, call, GSI_NEW_STMT);
    1032            0 :   update_stmt (call);
    1033              : 
    1034              :   /* The shared-memory range for this block overflowed.  Add a barrier before
    1035              :      the GOACC_single_copy_start call.  */
    1036            0 :   if (isolate_broadcasts)
    1037              :     {
    1038            0 :       decl = builtin_decl_explicit (BUILT_IN_GOACC_BARRIER);
    1039            0 :       gimple *acc_bar = gimple_build_call (decl, 0);
    1040            0 :       gsi_insert_before (&start, acc_bar, GSI_SAME_STMT);
    1041              :     }
    1042              : 
    1043            0 :   tree conv_tmp = make_ssa_name (TREE_TYPE (receiver_decl));
    1044              : 
    1045            0 :   gimple *conv = gimple_build_assign (conv_tmp,
    1046            0 :                                       fold_convert (TREE_TYPE (receiver_decl),
    1047              :                                                     lhs));
    1048            0 :   update_stmt (conv);
    1049            0 :   gsi_insert_after (&start, conv, GSI_NEW_STMT);
    1050            0 :   gimple *asgn = gimple_build_assign (receiver_decl, conv_tmp);
    1051            0 :   gsi_insert_after (&start, asgn, GSI_NEW_STMT);
    1052            0 :   update_stmt (asgn);
    1053              : 
    1054            0 :   tree zero_ptr = build_int_cst (TREE_TYPE (receiver_decl), 0);
    1055              : 
    1056            0 :   tree recv_tmp = make_ssa_name (TREE_TYPE (receiver_decl));
    1057            0 :   asgn = gimple_build_assign (recv_tmp, receiver_decl);
    1058            0 :   gsi_insert_after (&start, asgn, GSI_NEW_STMT);
    1059            0 :   update_stmt (asgn);
    1060              : 
    1061            0 :   gimple *cond = gimple_build_cond (EQ_EXPR, recv_tmp, zero_ptr, NULL_TREE,
    1062              :                                     NULL_TREE);
    1063            0 :   update_stmt (cond);
    1064              : 
    1065            0 :   gsi_insert_after (&start, cond, GSI_NEW_STMT);
    1066              : 
    1067            0 :   edge et = split_block (from, cond);
    1068            0 :   et->flags &= ~EDGE_FALLTHRU;
    1069            0 :   et->flags |= EDGE_TRUE_VALUE;
    1070              :   /* Make the active worker the more probable path so we prefer fallthrough
    1071              :      (letting the idle workers jump around more).  */
    1072            0 :   et->probability = profile_probability::likely ();
    1073              : 
    1074            0 :   basic_block body = et->dest;
    1075              : 
    1076            0 :   edge ef = make_edge (from, barrier_block, EDGE_FALSE_VALUE);
    1077            0 :   ef->probability = et->probability.invert ();
    1078              : 
    1079            0 :   gimple_stmt_iterator bar_gsi = gsi_start_bb (barrier_block);
    1080            0 :   cond = gimple_build_cond (NE_EXPR, recv_tmp, zero_ptr, NULL_TREE, NULL_TREE);
    1081              : 
    1082            0 :   if (record_type != char_type_node || has_gang_private_write)
    1083              :     {
    1084            0 :       decl = builtin_decl_explicit (BUILT_IN_GOACC_BARRIER);
    1085            0 :       gimple *acc_bar = gimple_build_call (decl, 0);
    1086              : 
    1087            0 :       gsi_insert_before (&bar_gsi, acc_bar, GSI_NEW_STMT);
    1088            0 :       gsi_insert_after (&bar_gsi, cond, GSI_NEW_STMT);
    1089            0 :     }
    1090              :   else
    1091            0 :     gsi_insert_before (&bar_gsi, cond, GSI_NEW_STMT);
    1092              : 
    1093            0 :   edge et2 = split_block (barrier_block, cond);
    1094            0 :   et2->flags &= ~EDGE_FALLTHRU;
    1095            0 :   et2->flags |= EDGE_TRUE_VALUE;
    1096            0 :   et2->probability = profile_probability::unlikely ();
    1097              : 
    1098            0 :   basic_block exit_block = et2->dest;
    1099              : 
    1100            0 :   basic_block copyout_block = split_edge (et2);
    1101            0 :   edge ef2 = make_edge (barrier_block, exit_block, EDGE_FALSE_VALUE);
    1102            0 :   ef2->probability = et2->probability.invert ();
    1103              : 
    1104            0 :   gimple_stmt_iterator copyout_gsi = gsi_start_bb (copyout_block);
    1105              : 
    1106            0 :   edge copyout_to_exit = single_succ_edge (copyout_block);
    1107              : 
    1108            0 :   gimple_seq sender_seq = NULL;
    1109              : 
    1110              :   /* Make sure we iterate over definitions in a stable order.  */
    1111            0 :   auto_vec<tree> escape_vec (def_escapes_block->elements ());
    1112            0 :   for (hash_set<tree>::iterator it = def_escapes_block->begin ();
    1113            0 :        it != def_escapes_block->end (); ++it)
    1114            0 :     escape_vec.quick_push (*it);
    1115            0 :   escape_vec.qsort (sort_by_ssa_version_or_uid);
    1116              : 
    1117            0 :   for (unsigned i = 0; i < escape_vec.length (); i++)
    1118              :     {
    1119            0 :       tree var = escape_vec[i];
    1120              : 
    1121            0 :       if (TREE_CODE (var) == SSA_NAME && SSA_NAME_IS_VIRTUAL_OPERAND (var))
    1122            0 :         continue;
    1123              : 
    1124            0 :       tree barrier_def = 0;
    1125              : 
    1126            0 :       if (TREE_CODE (var) == SSA_NAME)
    1127              :         {
    1128            0 :           gimple *def_stmt = SSA_NAME_DEF_STMT (var);
    1129              : 
    1130            0 :           if (gimple_nop_p (def_stmt))
    1131            0 :             continue;
    1132              : 
    1133              :           /* The barrier phi takes one result from the actual work of the
    1134              :              block we're neutering, and the other result is constant zero of
    1135              :              the same type.  */
    1136              : 
    1137            0 :           gphi *barrier_phi = create_phi_node (NULL_TREE, barrier_block);
    1138            0 :           barrier_def = create_new_def_for (var, barrier_phi,
    1139              :                           gimple_phi_result_ptr (barrier_phi));
    1140              : 
    1141            0 :           add_phi_arg (barrier_phi, var, e, UNKNOWN_LOCATION);
    1142            0 :           add_phi_arg (barrier_phi, build_zero_cst (TREE_TYPE (var)), ef,
    1143              :                        UNKNOWN_LOCATION);
    1144              : 
    1145            0 :           update_stmt (barrier_phi);
    1146              :         }
    1147              :       else
    1148            0 :         gcc_assert (VAR_P (var));
    1149              : 
    1150              :       /* If we had no record type, we will have no fields map.  */
    1151            0 :       field_map_t *fields = record_field_map->get (record_type);
    1152              : 
    1153            0 :       if (worker_partitioned_uses->contains (var)
    1154            0 :           && fields
    1155            0 :           && fields->get (var))
    1156              :         {
    1157            0 :           tree neutered_def = make_ssa_name (TREE_TYPE (var));
    1158              : 
    1159              :           /* Receive definition from shared memory block.  */
    1160              : 
    1161            0 :           tree receiver_ref = build_receiver_ref (var, receiver_decl, fields);
    1162            0 :           gassign *recv = gimple_build_assign (neutered_def,
    1163              :                                                receiver_ref);
    1164            0 :           gsi_insert_after (&copyout_gsi, recv, GSI_CONTINUE_LINKING);
    1165            0 :           update_stmt (recv);
    1166              : 
    1167            0 :           if (VAR_P (var))
    1168              :             {
    1169              :               /* If it's a VAR_DECL, we only copied to an SSA temporary.  Copy
    1170              :                  to the final location now.  */
    1171            0 :               gassign *asgn = gimple_build_assign (var, neutered_def);
    1172            0 :               gsi_insert_after (&copyout_gsi, asgn, GSI_CONTINUE_LINKING);
    1173            0 :               update_stmt (asgn);
    1174              :             }
    1175              :           else
    1176              :             {
    1177              :               /* If it's an SSA name, create a new phi at the join node to
    1178              :                  represent either the output from the active worker (the
    1179              :                  barrier) or the inactive workers (the copyout block).  */
    1180            0 :               gphi *join_phi = create_phi_node (NULL_TREE, exit_block);
    1181            0 :               create_new_def_for (barrier_def, join_phi,
    1182              :                                   gimple_phi_result_ptr (join_phi));
    1183            0 :               add_phi_arg (join_phi, barrier_def, ef2, UNKNOWN_LOCATION);
    1184            0 :               add_phi_arg (join_phi, neutered_def, copyout_to_exit,
    1185              :                            UNKNOWN_LOCATION);
    1186            0 :               update_stmt (join_phi);
    1187              :             }
    1188              : 
    1189              :           /* Send definition to shared memory block.  */
    1190              : 
    1191            0 :           tree sender_ref = build_sender_ref (var, sender_decl, fields);
    1192              : 
    1193            0 :           if (TREE_CODE (var) == SSA_NAME)
    1194              :             {
    1195            0 :               gassign *send = gimple_build_assign (sender_ref, var);
    1196            0 :               gimple_seq_add_stmt (&sender_seq, send);
    1197            0 :               update_stmt (send);
    1198              :             }
    1199            0 :           else if (VAR_P (var))
    1200              :             {
    1201            0 :               tree tmp = make_ssa_name (TREE_TYPE (var));
    1202            0 :               gassign *send = gimple_build_assign (tmp, var);
    1203            0 :               gimple_seq_add_stmt (&sender_seq, send);
    1204            0 :               update_stmt (send);
    1205            0 :               send = gimple_build_assign (sender_ref, tmp);
    1206            0 :               gimple_seq_add_stmt (&sender_seq, send);
    1207            0 :               update_stmt (send);
    1208              :             }
    1209              :           else
    1210            0 :             gcc_unreachable ();
    1211              :         }
    1212              :     }
    1213              : 
    1214              :   /* The shared-memory range for this block overflowed.  Add a barrier at the
    1215              :      end.  */
    1216            0 :   if (isolate_broadcasts)
    1217              :     {
    1218            0 :       gsi = gsi_start_bb (exit_block);
    1219            0 :       decl = builtin_decl_explicit (BUILT_IN_GOACC_BARRIER);
    1220            0 :       gimple *acc_bar = gimple_build_call (decl, 0);
    1221            0 :       gsi_insert_before (&gsi, acc_bar, GSI_SAME_STMT);
    1222              :     }
    1223              : 
    1224              :   /* It's possible for the ET->DEST block (the work done by the active thread)
    1225              :      to finish with a control-flow insn, e.g. a UNIQUE function call.  Split
    1226              :      the block and add SENDER_SEQ in the latter part to avoid having control
    1227              :      flow in the middle of a BB.  */
    1228              : 
    1229            0 :   decl = builtin_decl_explicit (BUILT_IN_GOACC_SINGLE_COPY_END);
    1230            0 :   call = gimple_build_call (decl, 1,
    1231            0 :                             POINTER_TYPE_P (TREE_TYPE (sender_decl))
    1232              :                             ? sender_decl
    1233            0 :                             : build_fold_addr_expr (sender_decl));
    1234            0 :   gimple_seq_add_stmt (&sender_seq, call);
    1235              : 
    1236            0 :   gsi = gsi_last_bb (body);
    1237            0 :   gimple *last = gsi_stmt (gsi);
    1238            0 :   basic_block sender_block = split_block (body, last)->dest;
    1239            0 :   gsi = gsi_last_bb (sender_block);
    1240            0 :   gsi_insert_seq_after (&gsi, sender_seq, GSI_CONTINUE_LINKING);
    1241            0 : }
    1242              : 
    1243              : typedef hash_map<basic_block, std::pair<unsigned HOST_WIDE_INT, bool> >
    1244              :   blk_offset_map_t;
    1245              : 
    1246              : static void
    1247            0 : neuter_worker_single (parallel_g *par, unsigned outer_mask,
    1248              :                       bitmap worker_single, bitmap vector_single,
    1249              :                       vec<propagation_set *> *prop_set,
    1250              :                       hash_set<tree> *partitioned_var_uses,
    1251              :                       record_field_map_t *record_field_map,
    1252              :                       blk_offset_map_t *blk_offset_map,
    1253              :                       bitmap writes_gang_private)
    1254              : {
    1255            0 :   unsigned mask = outer_mask | par->mask;
    1256              : 
    1257            0 :   if ((mask & GOMP_DIM_MASK (GOMP_DIM_WORKER)) == 0)
    1258              :     {
    1259              :       basic_block block;
    1260              : 
    1261            0 :       for (unsigned i = 0; par->blocks.iterate (i, &block); i++)
    1262              :         {
    1263            0 :           bool has_defs = false;
    1264            0 :           hash_set<tree> def_escapes_block;
    1265            0 :           hash_set<tree> worker_partitioned_uses;
    1266            0 :           unsigned j;
    1267            0 :           tree var;
    1268              : 
    1269            0 :           FOR_EACH_SSA_NAME (j, var, cfun)
    1270              :             {
    1271            0 :               if (SSA_NAME_IS_VIRTUAL_OPERAND (var))
    1272              :                 {
    1273            0 :                   has_defs = true;
    1274            0 :                   continue;
    1275              :                 }
    1276              : 
    1277            0 :               gimple *def_stmt = SSA_NAME_DEF_STMT (var);
    1278              : 
    1279            0 :               if (gimple_nop_p (def_stmt))
    1280            0 :                 continue;
    1281              : 
    1282            0 :               if (gimple_bb (def_stmt)->index != block->index)
    1283            0 :                 continue;
    1284              : 
    1285            0 :               gimple *use_stmt;
    1286            0 :               imm_use_iterator use_iter;
    1287            0 :               bool uses_outside_block = false;
    1288            0 :               bool worker_partitioned_use = false;
    1289              : 
    1290            0 :               FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, var)
    1291              :                 {
    1292            0 :                   int blocknum = gimple_bb (use_stmt)->index;
    1293              : 
    1294              :                   /* Don't propagate SSA names that are only used in the
    1295              :                      current block, unless the usage is in a phi node: that
    1296              :                      means the name left the block, then came back in at the
    1297              :                      top.  */
    1298            0 :                   if (blocknum != block->index
    1299            0 :                       || gimple_code (use_stmt) == GIMPLE_PHI)
    1300              :                     uses_outside_block = true;
    1301            0 :                   if (!bitmap_bit_p (worker_single, blocknum))
    1302            0 :                     worker_partitioned_use = true;
    1303            0 :                 }
    1304              : 
    1305            0 :               if (uses_outside_block)
    1306            0 :                 def_escapes_block.add (var);
    1307              : 
    1308            0 :               if (worker_partitioned_use)
    1309              :                 {
    1310            0 :                   worker_partitioned_uses.add (var);
    1311            0 :                   has_defs = true;
    1312              :                 }
    1313              :             }
    1314              : 
    1315            0 :           propagation_set *ws_prop = (*prop_set)[block->index];
    1316              : 
    1317            0 :           if (ws_prop)
    1318              :             {
    1319            0 :               for (propagation_set::iterator it = ws_prop->begin ();
    1320            0 :                    it != ws_prop->end ();
    1321            0 :                    ++it)
    1322              :                 {
    1323            0 :                   tree var = *it;
    1324            0 :                   if (TREE_CODE (var) == VAR_DECL)
    1325              :                     {
    1326            0 :                       def_escapes_block.add (var);
    1327            0 :                       if (partitioned_var_uses->contains (var))
    1328              :                         {
    1329            0 :                           worker_partitioned_uses.add (var);
    1330            0 :                           has_defs = true;
    1331              :                         }
    1332              :                     }
    1333              :                 }
    1334              : 
    1335            0 :               delete ws_prop;
    1336            0 :               (*prop_set)[block->index] = 0;
    1337              :             }
    1338              : 
    1339            0 :           bool only_marker_fns = true;
    1340            0 :           bool join_block = false;
    1341              : 
    1342            0 :           for (gimple_stmt_iterator gsi = gsi_start_bb (block);
    1343            0 :                !gsi_end_p (gsi);
    1344            0 :                gsi_next (&gsi))
    1345              :             {
    1346            0 :               gimple *stmt = gsi_stmt (gsi);
    1347            0 :               if (gimple_code (stmt) == GIMPLE_CALL
    1348            0 :                   && gimple_call_internal_p (stmt, IFN_UNIQUE))
    1349              :                 {
    1350            0 :                   enum ifn_unique_kind k = ((enum ifn_unique_kind)
    1351            0 :                     TREE_INT_CST_LOW (gimple_call_arg (stmt, 0)));
    1352            0 :                   if (k != IFN_UNIQUE_OACC_PRIVATE
    1353            0 :                       && k != IFN_UNIQUE_OACC_JOIN
    1354            0 :                       && k != IFN_UNIQUE_OACC_FORK
    1355            0 :                       && k != IFN_UNIQUE_OACC_HEAD_MARK
    1356            0 :                       && k != IFN_UNIQUE_OACC_TAIL_MARK)
    1357              :                     only_marker_fns = false;
    1358            0 :                   else if (k == IFN_UNIQUE_OACC_JOIN)
    1359              :                     /* The JOIN marker is special in that it *cannot* be
    1360              :                        predicated for worker zero, because it may be lowered
    1361              :                        to a barrier instruction and all workers must typically
    1362              :                        execute that barrier.  We shouldn't be doing any
    1363              :                        broadcasts from the join block anyway.  */
    1364            0 :                     join_block = true;
    1365              :                 }
    1366            0 :               else if (gimple_code (stmt) == GIMPLE_CALL
    1367            0 :                        && gimple_call_internal_p (stmt, IFN_GOACC_LOOP))
    1368              :                 /* Empty.  */;
    1369            0 :               else if (gimple_nop_p (stmt))
    1370              :                 /* Empty.  */;
    1371              :               else
    1372            0 :                 only_marker_fns = false;
    1373              :             }
    1374              : 
    1375              :           /* We can skip predicating this block for worker zero if the only
    1376              :              thing it contains is marker functions that will be removed in the
    1377              :              oaccdevlow pass anyway.
    1378              :              Don't do this if the block has (any) phi nodes, because those
    1379              :              might define SSA names that need broadcasting.
    1380              :              TODO: We might be able to skip transforming blocks that only
    1381              :              contain some other trivial statements too.  */
    1382            0 :           if (only_marker_fns && !phi_nodes (block))
    1383            0 :             continue;
    1384              : 
    1385            0 :           gcc_assert (!join_block);
    1386              : 
    1387            0 :           if (has_defs)
    1388              :             {
    1389            0 :               tree record_type = (tree) block->aux;
    1390            0 :               std::pair<unsigned HOST_WIDE_INT, bool> *off_rngalloc
    1391            0 :                 = blk_offset_map->get (block);
    1392            0 :               gcc_assert (!record_type || off_rngalloc);
    1393            0 :               unsigned HOST_WIDE_INT offset
    1394            0 :                 = off_rngalloc ? off_rngalloc->first : 0;
    1395            0 :               bool range_allocated
    1396            0 :                 = off_rngalloc ? off_rngalloc->second : true;
    1397            0 :               bool has_gang_private_write
    1398            0 :                 = bitmap_bit_p (writes_gang_private, block->index);
    1399            0 :               worker_single_copy (block, block, &def_escapes_block,
    1400              :                                   &worker_partitioned_uses, record_type,
    1401              :                                   record_field_map,
    1402              :                                   offset, !range_allocated,
    1403              :                                   has_gang_private_write);
    1404              :             }
    1405              :           else
    1406            0 :             worker_single_simple (block, block, &def_escapes_block);
    1407            0 :         }
    1408              :     }
    1409              : 
    1410            0 :   if ((outer_mask & GOMP_DIM_MASK (GOMP_DIM_WORKER)) == 0)
    1411              :     {
    1412              :       basic_block block;
    1413              : 
    1414            0 :       for (unsigned i = 0; par->blocks.iterate (i, &block); i++)
    1415            0 :         for (gimple_stmt_iterator gsi = gsi_start_bb (block);
    1416            0 :              !gsi_end_p (gsi);
    1417            0 :              gsi_next (&gsi))
    1418              :           {
    1419            0 :             gimple *stmt = gsi_stmt (gsi);
    1420              : 
    1421            0 :             if (gimple_code (stmt) == GIMPLE_CALL
    1422            0 :                 && !gimple_call_internal_p (stmt)
    1423            0 :                 && !omp_sese_active_worker_call (as_a <gcall *> (stmt)))
    1424              :               {
    1425              :                 /* If we have an OpenACC routine call in worker-single mode,
    1426              :                    place barriers before and afterwards to prevent
    1427              :                    clobbering re-used shared memory regions (as are used
    1428              :                    for AMDGCN at present, for example).  */
    1429            0 :                 tree decl = builtin_decl_explicit (BUILT_IN_GOACC_BARRIER);
    1430            0 :                 gsi_insert_before (&gsi, gimple_build_call (decl, 0),
    1431              :                                    GSI_SAME_STMT);
    1432            0 :                 gsi_insert_after (&gsi, gimple_build_call (decl, 0),
    1433              :                                   GSI_NEW_STMT);
    1434              :               }
    1435              :           }
    1436              :     }
    1437              : 
    1438            0 :   if (par->inner)
    1439            0 :     neuter_worker_single (par->inner, mask, worker_single, vector_single,
    1440              :                           prop_set, partitioned_var_uses, record_field_map,
    1441              :                           blk_offset_map, writes_gang_private);
    1442            0 :   if (par->next)
    1443              :     neuter_worker_single (par->next, outer_mask, worker_single, vector_single,
    1444              :                           prop_set, partitioned_var_uses, record_field_map,
    1445              :                           blk_offset_map, writes_gang_private);
    1446            0 : }
    1447              : 
    1448              : static void
    1449            0 : dfs_broadcast_reachable_1 (basic_block bb, sbitmap reachable)
    1450              : {
    1451            0 :   if (bb->flags & BB_VISITED)
    1452              :     return;
    1453              : 
    1454            0 :   bb->flags |= BB_VISITED;
    1455              : 
    1456            0 :   if (bb->succs)
    1457              :     {
    1458            0 :       edge e;
    1459            0 :       edge_iterator ei;
    1460            0 :       FOR_EACH_EDGE (e, ei, bb->succs)
    1461              :         {
    1462            0 :           basic_block dest = e->dest;
    1463            0 :           if (dest->aux)
    1464            0 :             bitmap_set_bit (reachable, dest->index);
    1465              :           else
    1466            0 :             dfs_broadcast_reachable_1 (dest, reachable);
    1467              :         }
    1468              :     }
    1469              : }
    1470              : 
    1471              : typedef std::pair<int, tree> idx_decl_pair_t;
    1472              : 
    1473              : typedef auto_vec<splay_tree> used_range_vec_t;
    1474              : 
    1475              : static int
    1476            0 : sort_size_descending (const void *a, const void *b)
    1477              : {
    1478            0 :   const idx_decl_pair_t *pa = (const idx_decl_pair_t *) a;
    1479            0 :   const idx_decl_pair_t *pb = (const idx_decl_pair_t *) b;
    1480            0 :   unsigned HOST_WIDE_INT asize = tree_to_uhwi (TYPE_SIZE_UNIT (pa->second));
    1481            0 :   unsigned HOST_WIDE_INT bsize = tree_to_uhwi (TYPE_SIZE_UNIT (pb->second));
    1482            0 :   return bsize - asize;
    1483              : }
    1484              : 
    1485              : class addr_range
    1486              : {
    1487              : public:
    1488            0 :   addr_range (unsigned HOST_WIDE_INT addr_lo, unsigned HOST_WIDE_INT addr_hi)
    1489            0 :     : lo (addr_lo), hi (addr_hi)
    1490              :     { }
    1491            0 :   addr_range (const addr_range &ar) : lo (ar.lo), hi (ar.hi)
    1492              :     { }
    1493            0 :   addr_range () : lo (0), hi (0)
    1494              :     { }
    1495              : 
    1496            0 :   bool invalid () { return lo == 0 && hi == 0; }
    1497              : 
    1498              :   unsigned HOST_WIDE_INT lo;
    1499              :   unsigned HOST_WIDE_INT hi;
    1500              : };
    1501              : 
    1502              : static int
    1503            0 : splay_tree_compare_addr_range (splay_tree_key a, splay_tree_key b)
    1504              : {
    1505            0 :   addr_range *ar = (addr_range *) a;
    1506            0 :   addr_range *br = (addr_range *) b;
    1507            0 :   if (ar->lo == br->lo && ar->hi == br->hi)
    1508              :     return 0;
    1509            0 :   if (ar->hi <= br->lo)
    1510              :     return -1;
    1511            0 :   else if (ar->lo >= br->hi)
    1512            0 :     return 1;
    1513              :   return 0;
    1514              : }
    1515              : 
    1516              : static void
    1517            0 : splay_tree_free_key (splay_tree_key k)
    1518              : {
    1519            0 :   addr_range *ar = (addr_range *) k;
    1520            0 :   delete ar;
    1521            0 : }
    1522              : 
    1523              : static addr_range
    1524            0 : first_fit_range (splay_tree s, unsigned HOST_WIDE_INT size,
    1525              :                  unsigned HOST_WIDE_INT align, addr_range *bounds)
    1526              : {
    1527            0 :   splay_tree_node min = splay_tree_min (s);
    1528            0 :   if (min)
    1529              :     {
    1530              :       splay_tree_node next;
    1531            0 :       while ((next = splay_tree_successor (s, min->key)))
    1532              :         {
    1533            0 :           unsigned HOST_WIDE_INT lo = ((addr_range *) min->key)->hi;
    1534            0 :           unsigned HOST_WIDE_INT hi = ((addr_range *) next->key)->lo;
    1535            0 :           unsigned HOST_WIDE_INT base = (lo + align - 1) & ~(align - 1);
    1536            0 :           if (base + size <= hi)
    1537            0 :             return addr_range (base, base + size);
    1538              :           min = next;
    1539              :         }
    1540              : 
    1541            0 :       unsigned HOST_WIDE_INT base = ((addr_range *)min->key)->hi;
    1542            0 :       base = (base + align - 1) & ~(align - 1);
    1543            0 :       if (base + size <= bounds->hi)
    1544            0 :         return addr_range (base, base + size);
    1545              :       else
    1546            0 :         return addr_range ();
    1547              :     }
    1548              :   else
    1549              :     {
    1550            0 :       unsigned HOST_WIDE_INT lo = bounds->lo;
    1551            0 :       lo = (lo + align - 1) & ~(align - 1);
    1552            0 :       if (lo + size <= bounds->hi)
    1553            0 :         return addr_range (lo, lo + size);
    1554              :       else
    1555            0 :         return addr_range ();
    1556              :     }
    1557              : }
    1558              : 
    1559              : static int
    1560            0 : merge_ranges_1 (splay_tree_node n, void *ptr)
    1561              : {
    1562            0 :   splay_tree accum = (splay_tree) ptr;
    1563            0 :   addr_range ar = *(addr_range *) n->key;
    1564              : 
    1565            0 :   splay_tree_node old = splay_tree_lookup (accum, n->key);
    1566              : 
    1567              :   /* We might have an overlap.  Create a new range covering the
    1568              :      overlapping parts.  */
    1569            0 :   if (old)
    1570              :     {
    1571            0 :       addr_range *old_ar = (addr_range *) old->key;
    1572            0 :       ar.lo = MIN (old_ar->lo, ar.lo);
    1573            0 :       ar.hi = MAX (old_ar->hi, ar.hi);
    1574            0 :       splay_tree_remove (accum, old->key);
    1575              :     }
    1576              : 
    1577            0 :   addr_range *new_ar = new addr_range (ar);
    1578              : 
    1579            0 :   splay_tree_insert (accum, (splay_tree_key) new_ar, n->value);
    1580              : 
    1581            0 :   return 0;
    1582              : }
    1583              : 
    1584              : static void
    1585            0 : merge_ranges (splay_tree accum, splay_tree sp)
    1586              : {
    1587            0 :   splay_tree_foreach (sp, merge_ranges_1, (void *) accum);
    1588            0 : }
    1589              : 
    1590              : static void
    1591            0 : oacc_do_neutering (unsigned HOST_WIDE_INT bounds_lo,
    1592              :                    unsigned HOST_WIDE_INT bounds_hi)
    1593              : {
    1594            0 :   bb_stmt_map_t bb_stmt_map;
    1595            0 :   auto_bitmap worker_single, vector_single;
    1596              : 
    1597            0 :   omp_sese_split_blocks (&bb_stmt_map);
    1598              : 
    1599            0 :   if (dump_file)
    1600              :     {
    1601            0 :       fprintf (dump_file, "\n\nAfter splitting:\n\n");
    1602            0 :       dump_function_to_file (current_function_decl, dump_file, dump_flags);
    1603              :     }
    1604              : 
    1605            0 :   unsigned mask = 0;
    1606              : 
    1607              :   /* If this is a routine, calculate MASK as if the outer levels are already
    1608              :      partitioned.  */
    1609            0 :   {
    1610            0 :     tree attr = oacc_get_fn_attrib (current_function_decl);
    1611            0 :     tree dims = TREE_VALUE (attr);
    1612            0 :     unsigned ix;
    1613            0 :     for (ix = 0; ix != GOMP_DIM_MAX; ix++, dims = TREE_CHAIN (dims))
    1614              :       {
    1615            0 :         tree allowed = TREE_PURPOSE (dims);
    1616            0 :         if (allowed && integer_zerop (allowed))
    1617            0 :           mask |= GOMP_DIM_MASK (ix);
    1618              :       }
    1619              :   }
    1620              : 
    1621            0 :   parallel_g *par = omp_sese_discover_pars (&bb_stmt_map);
    1622            0 :   populate_single_mode_bitmaps (par, worker_single, vector_single, mask, 0);
    1623              : 
    1624            0 :   basic_block bb;
    1625            0 :   FOR_ALL_BB_FN (bb, cfun)
    1626            0 :     bb->aux = NULL;
    1627              : 
    1628            0 :   vec<propagation_set *> prop_set (vNULL);
    1629            0 :   prop_set.safe_grow_cleared (last_basic_block_for_fn (cfun), true);
    1630              : 
    1631            0 :   find_ssa_names_to_propagate (par, mask, worker_single, vector_single,
    1632              :                                &prop_set);
    1633              : 
    1634            0 :   hash_set<tree> partitioned_var_uses;
    1635            0 :   hash_set<tree> gang_private_vars;
    1636            0 :   auto_bitmap writes_gang_private;
    1637              : 
    1638            0 :   find_gang_private_vars (&gang_private_vars);
    1639            0 :   find_partitioned_var_uses (par, mask, &partitioned_var_uses);
    1640            0 :   find_local_vars_to_propagate (par, mask, &partitioned_var_uses,
    1641              :                                 &gang_private_vars, writes_gang_private,
    1642              :                                 &prop_set);
    1643              : 
    1644            0 :   record_field_map_t record_field_map;
    1645              : 
    1646            0 :   FOR_ALL_BB_FN (bb, cfun)
    1647              :     {
    1648            0 :       propagation_set *ws_prop = prop_set[bb->index];
    1649            0 :       if (ws_prop)
    1650              :         {
    1651            0 :           tree record_type = lang_hooks.types.make_type (RECORD_TYPE);
    1652            0 :           tree name = create_tmp_var_name (".oacc_ws_data_s");
    1653            0 :           name = build_decl (UNKNOWN_LOCATION, TYPE_DECL, name, record_type);
    1654            0 :           DECL_ARTIFICIAL (name) = 1;
    1655            0 :           DECL_NAMELESS (name) = 1;
    1656            0 :           TYPE_NAME (record_type) = name;
    1657            0 :           TYPE_ARTIFICIAL (record_type) = 1;
    1658              : 
    1659            0 :           auto_vec<tree> field_vec (ws_prop->elements ());
    1660            0 :           for (hash_set<tree>::iterator it = ws_prop->begin ();
    1661            0 :                it != ws_prop->end (); ++it)
    1662            0 :             field_vec.quick_push (*it);
    1663              : 
    1664            0 :           field_vec.qsort (sort_by_size_then_ssa_version_or_uid);
    1665              : 
    1666            0 :           bool existed;
    1667            0 :           field_map_t *fields
    1668            0 :             = &record_field_map.get_or_insert (record_type, &existed);
    1669            0 :           gcc_checking_assert (!existed);
    1670              : 
    1671              :           /* Insert var fields in reverse order, so the last inserted element
    1672              :              is the first in the structure.  */
    1673            0 :           for (int i = field_vec.length () - 1; i >= 0; i--)
    1674            0 :             install_var_field (field_vec[i], record_type, fields);
    1675              : 
    1676            0 :           layout_type (record_type);
    1677              : 
    1678            0 :           bb->aux = (tree) record_type;
    1679            0 :         }
    1680              :     }
    1681              : 
    1682            0 :   sbitmap *reachable
    1683            0 :     = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
    1684            0 :                             last_basic_block_for_fn (cfun));
    1685              : 
    1686            0 :   bitmap_vector_clear (reachable, last_basic_block_for_fn (cfun));
    1687              : 
    1688            0 :   auto_vec<std::pair<int, tree> > priority;
    1689              : 
    1690            0 :   FOR_ALL_BB_FN (bb, cfun)
    1691              :     {
    1692            0 :       if (bb->aux)
    1693              :         {
    1694            0 :           tree record_type = (tree) bb->aux;
    1695              : 
    1696            0 :           basic_block bb2;
    1697            0 :           FOR_ALL_BB_FN (bb2, cfun)
    1698            0 :             bb2->flags &= ~BB_VISITED;
    1699              : 
    1700            0 :           priority.safe_push (std::make_pair (bb->index, record_type));
    1701            0 :           dfs_broadcast_reachable_1 (bb, reachable[bb->index]);
    1702              :         }
    1703              :     }
    1704              : 
    1705            0 :   sbitmap *inverted
    1706            0 :     = sbitmap_vector_alloc (last_basic_block_for_fn (cfun),
    1707            0 :                             last_basic_block_for_fn (cfun));
    1708              : 
    1709            0 :   bitmap_vector_clear (inverted, last_basic_block_for_fn (cfun));
    1710              : 
    1711            0 :   for (int i = 0; i < last_basic_block_for_fn (cfun); i++)
    1712              :     {
    1713            0 :       sbitmap_iterator bi;
    1714            0 :       unsigned int j;
    1715            0 :       EXECUTE_IF_SET_IN_BITMAP (reachable[i], 0, j, bi)
    1716            0 :         bitmap_set_bit (inverted[j], i);
    1717              :     }
    1718              : 
    1719            0 :   for (int i = 0; i < last_basic_block_for_fn (cfun); i++)
    1720            0 :     bitmap_ior (reachable[i], reachable[i], inverted[i]);
    1721              : 
    1722            0 :   sbitmap_vector_free (inverted);
    1723              : 
    1724            0 :   used_range_vec_t used_ranges;
    1725              : 
    1726            0 :   used_ranges.safe_grow_cleared (last_basic_block_for_fn (cfun));
    1727              : 
    1728            0 :   blk_offset_map_t blk_offset_map;
    1729              : 
    1730            0 :   addr_range worker_shm_bounds (bounds_lo, bounds_hi);
    1731              : 
    1732            0 :   priority.qsort (sort_size_descending);
    1733            0 :   for (unsigned int i = 0; i < priority.length (); i++)
    1734              :     {
    1735            0 :       idx_decl_pair_t p = priority[i];
    1736            0 :       int blkno = p.first;
    1737            0 :       tree record_type = p.second;
    1738            0 :       HOST_WIDE_INT size = tree_to_uhwi (TYPE_SIZE_UNIT (record_type));
    1739            0 :       HOST_WIDE_INT align = TYPE_ALIGN_UNIT (record_type);
    1740              : 
    1741            0 :       splay_tree conflicts = splay_tree_new (splay_tree_compare_addr_range,
    1742              :                                              splay_tree_free_key, NULL);
    1743              : 
    1744            0 :       if (!used_ranges[blkno])
    1745            0 :         used_ranges[blkno] = splay_tree_new (splay_tree_compare_addr_range,
    1746              :                                              splay_tree_free_key, NULL);
    1747              :       else
    1748            0 :         merge_ranges (conflicts, used_ranges[blkno]);
    1749              : 
    1750            0 :       sbitmap_iterator bi;
    1751            0 :       unsigned int j;
    1752            0 :       EXECUTE_IF_SET_IN_BITMAP (reachable[blkno], 0, j, bi)
    1753            0 :         if (used_ranges[j])
    1754            0 :           merge_ranges (conflicts, used_ranges[j]);
    1755              : 
    1756            0 :       addr_range ar
    1757            0 :         = first_fit_range (conflicts, size, align, &worker_shm_bounds);
    1758              : 
    1759            0 :       splay_tree_delete (conflicts);
    1760              : 
    1761            0 :       if (ar.invalid ())
    1762              :         {
    1763            0 :           unsigned HOST_WIDE_INT base
    1764            0 :             = (bounds_lo + align - 1) & ~(align - 1);
    1765            0 :           if (base + size > bounds_hi)
    1766            0 :             error_at (UNKNOWN_LOCATION, "shared-memory region overflow");
    1767            0 :           std::pair<unsigned HOST_WIDE_INT, bool> base_inrng
    1768            0 :             = std::make_pair (base, false);
    1769            0 :           blk_offset_map.put (BASIC_BLOCK_FOR_FN (cfun, blkno), base_inrng);
    1770              :         }
    1771              :       else
    1772              :         {
    1773            0 :           splay_tree_node old = splay_tree_lookup (used_ranges[blkno],
    1774              :                                                    (splay_tree_key) &ar);
    1775            0 :           if (old)
    1776              :             {
    1777            0 :               fprintf (stderr, "trying to map [%d..%d] but [%d..%d] is "
    1778            0 :                        "already mapped in block %d\n", (int) ar.lo,
    1779            0 :                        (int) ar.hi, (int) ((addr_range *) old->key)->lo,
    1780            0 :                        (int) ((addr_range *) old->key)->hi, blkno);
    1781            0 :               abort ();
    1782              :             }
    1783              : 
    1784            0 :           addr_range *arp = new addr_range (ar);
    1785            0 :           splay_tree_insert (used_ranges[blkno], (splay_tree_key) arp,
    1786              :                              (splay_tree_value) blkno);
    1787            0 :           std::pair<unsigned HOST_WIDE_INT, bool> base_inrng
    1788            0 :             = std::make_pair (ar.lo, true);
    1789            0 :           blk_offset_map.put (BASIC_BLOCK_FOR_FN (cfun, blkno), base_inrng);
    1790              :         }
    1791              :     }
    1792              : 
    1793            0 :   sbitmap_vector_free (reachable);
    1794              : 
    1795            0 :   neuter_worker_single (par, mask, worker_single, vector_single, &prop_set,
    1796              :                         &partitioned_var_uses, &record_field_map,
    1797              :                         &blk_offset_map, writes_gang_private);
    1798              : 
    1799            0 :   record_field_map.empty ();
    1800              : 
    1801              :   /* These are supposed to have been 'delete'd by 'neuter_worker_single'.  */
    1802            0 :   for (auto it : prop_set)
    1803            0 :     gcc_checking_assert (!it);
    1804            0 :   prop_set.release ();
    1805              : 
    1806            0 :   delete par;
    1807              : 
    1808              :   /* This doesn't seem to make a difference.  */
    1809            0 :   loops_state_clear (LOOP_CLOSED_SSA);
    1810              : 
    1811              :   /* Neutering worker-single neutered blocks will invalidate dominance info.
    1812              :      It may be possible to incrementally update just the affected blocks, but
    1813              :      obliterate everything for now.  */
    1814            0 :   free_dominance_info (CDI_DOMINATORS);
    1815            0 :   free_dominance_info (CDI_POST_DOMINATORS);
    1816              : 
    1817            0 :   if (dump_file)
    1818              :     {
    1819            0 :       fprintf (dump_file, "\n\nAfter neutering:\n\n");
    1820            0 :       dump_function_to_file (current_function_decl, dump_file, dump_flags);
    1821              :     }
    1822            0 : }
    1823              : 
    1824              : static int
    1825            0 : execute_omp_oacc_neuter_broadcast ()
    1826              : {
    1827            0 :   unsigned HOST_WIDE_INT reduction_size[GOMP_DIM_MAX];
    1828            0 :   unsigned HOST_WIDE_INT private_size[GOMP_DIM_MAX];
    1829              : 
    1830            0 :   for (unsigned i = 0; i < GOMP_DIM_MAX; i++)
    1831              :     {
    1832            0 :       reduction_size[i] = 0;
    1833            0 :       private_size[i] = 0;
    1834              :     }
    1835              : 
    1836              :   /* Calculate shared memory size required for reduction variables and
    1837              :      gang-private memory for this offloaded function.  */
    1838            0 :   basic_block bb;
    1839            0 :   FOR_ALL_BB_FN (bb, cfun)
    1840              :     {
    1841            0 :       for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
    1842            0 :            !gsi_end_p (gsi);
    1843            0 :            gsi_next (&gsi))
    1844              :         {
    1845            0 :           gimple *stmt = gsi_stmt (gsi);
    1846            0 :           if (!is_gimple_call (stmt))
    1847            0 :             continue;
    1848            0 :           gcall *call = as_a <gcall *> (stmt);
    1849            0 :           if (!gimple_call_internal_p (call))
    1850            0 :             continue;
    1851            0 :           enum internal_fn ifn_code = gimple_call_internal_fn (call);
    1852            0 :           switch (ifn_code)
    1853              :             {
    1854              :             default: break;
    1855            0 :             case IFN_GOACC_REDUCTION:
    1856            0 :               if (integer_minus_onep (gimple_call_arg (call, 3)))
    1857            0 :                 continue;
    1858              :               else
    1859              :                 {
    1860            0 :                   unsigned code = TREE_INT_CST_LOW (gimple_call_arg (call, 0));
    1861              :                   /* Only count reduction variables once: the choice to pick
    1862              :                      the setup call is fairly arbitrary.  */
    1863            0 :                   if (code == IFN_GOACC_REDUCTION_SETUP)
    1864              :                     {
    1865            0 :                       int level = TREE_INT_CST_LOW (gimple_call_arg (call, 3));
    1866            0 :                       tree var = gimple_call_arg (call, 2);
    1867            0 :                       tree offset = gimple_call_arg (call, 5);
    1868            0 :                       tree var_type = TREE_TYPE (var);
    1869            0 :                       unsigned HOST_WIDE_INT limit
    1870            0 :                         = (tree_to_uhwi (offset)
    1871            0 :                            + tree_to_uhwi (TYPE_SIZE_UNIT (var_type)));
    1872            0 :                       reduction_size[level]
    1873            0 :                         = MAX (reduction_size[level], limit);
    1874              :                     }
    1875              :                 }
    1876              :               break;
    1877            0 :             case IFN_UNIQUE:
    1878            0 :               {
    1879            0 :                 enum ifn_unique_kind kind
    1880              :                   = ((enum ifn_unique_kind)
    1881            0 :                      TREE_INT_CST_LOW (gimple_call_arg (call, 0)));
    1882              : 
    1883            0 :                 if (kind == IFN_UNIQUE_OACC_PRIVATE)
    1884              :                   {
    1885            0 :                     HOST_WIDE_INT level
    1886            0 :                       = TREE_INT_CST_LOW (gimple_call_arg (call, 2));
    1887            0 :                     if (level == -1)
    1888              :                       break;
    1889            0 :                     for (unsigned i = 3;
    1890            0 :                          i < gimple_call_num_args (call);
    1891              :                          i++)
    1892              :                       {
    1893            0 :                         tree arg = gimple_call_arg (call, i);
    1894            0 :                         gcc_assert (TREE_CODE (arg) == ADDR_EXPR);
    1895            0 :                         tree decl = TREE_OPERAND (arg, 0);
    1896            0 :                         unsigned HOST_WIDE_INT align = DECL_ALIGN_UNIT (decl);
    1897            0 :                         private_size[level] = ((private_size[level] + align - 1)
    1898            0 :                                                & ~(align - 1));
    1899            0 :                         unsigned HOST_WIDE_INT decl_size
    1900            0 :                           = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (decl)));
    1901            0 :                         private_size[level] += decl_size;
    1902              :                       }
    1903              :                   }
    1904              :               }
    1905              :               break;
    1906              :             }
    1907              :         }
    1908              :     }
    1909              : 
    1910              :   int dims[GOMP_DIM_MAX];
    1911            0 :   for (unsigned i = 0; i < GOMP_DIM_MAX; i++)
    1912            0 :     dims[i] = oacc_get_fn_dim_size (current_function_decl, i);
    1913              : 
    1914              :   /* Find bounds of shared-memory buffer space we can use.  */
    1915            0 :   unsigned HOST_WIDE_INT bounds_lo = 0, bounds_hi = 0;
    1916            0 :   if (targetm.goacc.shared_mem_layout)
    1917            0 :     targetm.goacc.shared_mem_layout (&bounds_lo, &bounds_hi, dims,
    1918              :                                      private_size, reduction_size);
    1919              : 
    1920              :   /* Perform worker partitioning unless we know 'num_workers(1)'.  */
    1921            0 :   if (dims[GOMP_DIM_WORKER] != 1)
    1922            0 :     oacc_do_neutering (bounds_lo, bounds_hi);
    1923              : 
    1924            0 :   return 0;
    1925              : }
    1926              : 
    1927              : namespace {
    1928              : 
    1929              : const pass_data pass_data_omp_oacc_neuter_broadcast =
    1930              : {
    1931              :   GIMPLE_PASS, /* type */
    1932              :   "omp_oacc_neuter_broadcast", /* name */
    1933              :   OPTGROUP_OMP, /* optinfo_flags */
    1934              :   TV_NONE, /* tv_id */
    1935              :   PROP_cfg, /* properties_required */
    1936              :   0, /* properties_provided */
    1937              :   0, /* properties_destroyed */
    1938              :   0, /* todo_flags_start */
    1939              :   TODO_update_ssa | TODO_cleanup_cfg, /* todo_flags_finish */
    1940              : };
    1941              : 
    1942              : class pass_omp_oacc_neuter_broadcast : public gimple_opt_pass
    1943              : {
    1944              : public:
    1945       297180 :   pass_omp_oacc_neuter_broadcast (gcc::context *ctxt)
    1946       594360 :     : gimple_opt_pass (pass_data_omp_oacc_neuter_broadcast, ctxt)
    1947              :   {}
    1948              : 
    1949              :   /* opt_pass methods: */
    1950      1518061 :   bool gate (function *fun) final override
    1951              :   {
    1952      1518061 :     if (!flag_openacc)
    1953              :       return false;
    1954              : 
    1955        15275 :     if (!targetm.goacc.create_worker_broadcast_record)
    1956              :       return false;
    1957              : 
    1958              :     /* Only relevant for OpenACC offloaded functions.  */
    1959            0 :     tree attr = oacc_get_fn_attrib (fun->decl);
    1960            0 :     if (!attr)
    1961            0 :       return false;
    1962              : 
    1963              :     return true;
    1964              :   }
    1965              : 
    1966            0 :   unsigned int execute (function *) final override
    1967              :     {
    1968            0 :       return execute_omp_oacc_neuter_broadcast ();
    1969              :     }
    1970              : 
    1971              : }; // class pass_omp_oacc_neuter_broadcast
    1972              : 
    1973              : } // anon namespace
    1974              : 
    1975              : gimple_opt_pass *
    1976       297180 : make_pass_omp_oacc_neuter_broadcast (gcc::context *ctxt)
    1977              : {
    1978       297180 :   return new pass_omp_oacc_neuter_broadcast (ctxt);
    1979              : }
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.