Line data Source code
1 : /* Expansion pass for OMP directives. Outlines regions of certain OMP
2 : directives to separate functions, converts others into explicit calls to the
3 : runtime library (libgomp) and so forth
4 :
5 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
6 :
7 : This file is part of GCC.
8 :
9 : GCC is free software; you can redistribute it and/or modify it under
10 : the terms of the GNU General Public License as published by the Free
11 : Software Foundation; either version 3, or (at your option) any later
12 : version.
13 :
14 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 : for more details.
18 :
19 : You should have received a copy of the GNU General Public License
20 : along with GCC; see the file COPYING3. If not see
21 : <http://www.gnu.org/licenses/>. */
22 :
23 : #include "config.h"
24 : #include "system.h"
25 : #include "coretypes.h"
26 : #include "memmodel.h"
27 : #include "backend.h"
28 : #include "target.h"
29 : #include "rtl.h"
30 : #include "tree.h"
31 : #include "gimple.h"
32 : #include "cfghooks.h"
33 : #include "tree-pass.h"
34 : #include "ssa.h"
35 : #include "optabs.h"
36 : #include "cgraph.h"
37 : #include "pretty-print.h"
38 : #include "diagnostic-core.h"
39 : #include "fold-const.h"
40 : #include "stor-layout.h"
41 : #include "cfganal.h"
42 : #include "internal-fn.h"
43 : #include "gimplify.h"
44 : #include "gimple-iterator.h"
45 : #include "gimplify-me.h"
46 : #include "gimple-walk.h"
47 : #include "tree-cfg.h"
48 : #include "tree-into-ssa.h"
49 : #include "tree-ssa.h"
50 : #include "splay-tree.h"
51 : #include "cfgloop.h"
52 : #include "omp-general.h"
53 : #include "omp-offload.h"
54 : #include "tree-cfgcleanup.h"
55 : #include "alloc-pool.h"
56 : #include "symbol-summary.h"
57 : #include "gomp-constants.h"
58 : #include "gimple-pretty-print.h"
59 : #include "stringpool.h"
60 : #include "attribs.h"
61 : #include "tree-eh.h"
62 : #include "opts.h"
63 :
64 : /* OMP region information. Every parallel and workshare
65 : directive is enclosed between two markers, the OMP_* directive
66 : and a corresponding GIMPLE_OMP_RETURN statement. */
67 :
68 : struct omp_region
69 : {
70 : /* The enclosing region. */
71 : struct omp_region *outer;
72 :
73 : /* First child region. */
74 : struct omp_region *inner;
75 :
76 : /* Next peer region. */
77 : struct omp_region *next;
78 :
79 : /* Block containing the omp directive as its last stmt. */
80 : basic_block entry;
81 :
82 : /* Block containing the GIMPLE_OMP_RETURN as its last stmt. */
83 : basic_block exit;
84 :
85 : /* Block containing the GIMPLE_OMP_CONTINUE as its last stmt. */
86 : basic_block cont;
87 :
88 : /* If this is a combined parallel+workshare region, this is a list
89 : of additional arguments needed by the combined parallel+workshare
90 : library call. */
91 : vec<tree, va_gc> *ws_args;
92 :
93 : /* The code for the omp directive of this region. */
94 : enum gimple_code type;
95 :
96 : /* Schedule kind, only used for GIMPLE_OMP_FOR type regions. */
97 : enum omp_clause_schedule_kind sched_kind;
98 :
99 : /* Schedule modifiers. */
100 : unsigned char sched_modifiers;
101 :
102 : /* True if this is a combined parallel+workshare region. */
103 : bool is_combined_parallel;
104 :
105 : /* Copy of fd.lastprivate_conditional != 0. */
106 : bool has_lastprivate_conditional;
107 :
108 : /* The ordered stmt if type is GIMPLE_OMP_ORDERED and it has
109 : a depend clause. */
110 : gomp_ordered *ord_stmt;
111 : };
112 :
113 : static struct omp_region *root_omp_region;
114 : static bool omp_any_child_fn_dumped;
115 :
116 : static void expand_omp_build_assign (gimple_stmt_iterator *, tree, tree,
117 : bool = false);
118 : static gphi *find_phi_with_arg_on_edge (tree, edge);
119 : static void expand_omp (struct omp_region *region);
120 :
121 : /* Return true if REGION is a combined parallel+workshare region. */
122 :
123 : static inline bool
124 43262 : is_combined_parallel (struct omp_region *region)
125 : {
126 43262 : return region->is_combined_parallel;
127 : }
128 :
129 : /* Return true is REGION is or is contained within an offload region. */
130 :
131 : static bool
132 11168 : is_in_offload_region (struct omp_region *region)
133 : {
134 30023 : gimple *entry_stmt = last_nondebug_stmt (region->entry);
135 30023 : if (is_gimple_omp (entry_stmt)
136 29067 : && is_gimple_omp_offloaded (entry_stmt))
137 : return true;
138 25713 : else if (region->outer)
139 : return is_in_offload_region (region->outer);
140 : else
141 6858 : return (lookup_attribute ("omp declare target",
142 6858 : DECL_ATTRIBUTES (current_function_decl))
143 6858 : != NULL);
144 : }
145 :
146 : /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
147 : is the immediate dominator of PAR_ENTRY_BB, return true if there
148 : are no data dependencies that would prevent expanding the parallel
149 : directive at PAR_ENTRY_BB as a combined parallel+workshare region.
150 :
151 : When expanding a combined parallel+workshare region, the call to
152 : the child function may need additional arguments in the case of
153 : GIMPLE_OMP_FOR regions. In some cases, these arguments are
154 : computed out of variables passed in from the parent to the child
155 : via 'struct .omp_data_s'. For instance:
156 :
157 : #pragma omp parallel for schedule (guided, i * 4)
158 : for (j ...)
159 :
160 : Is lowered into:
161 :
162 : # BLOCK 2 (PAR_ENTRY_BB)
163 : .omp_data_o.i = i;
164 : #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
165 :
166 : # BLOCK 3 (WS_ENTRY_BB)
167 : .omp_data_i = &.omp_data_o;
168 : D.1667 = .omp_data_i->i;
169 : D.1598 = D.1667 * 4;
170 : #pragma omp for schedule (guided, D.1598)
171 :
172 : When we outline the parallel region, the call to the child function
173 : 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
174 : that value is computed *after* the call site. So, in principle we
175 : cannot do the transformation.
176 :
177 : To see whether the code in WS_ENTRY_BB blocks the combined
178 : parallel+workshare call, we collect all the variables used in the
179 : GIMPLE_OMP_FOR header check whether they appear on the LHS of any
180 : statement in WS_ENTRY_BB. If so, then we cannot emit the combined
181 : call.
182 :
183 : FIXME. If we had the SSA form built at this point, we could merely
184 : hoist the code in block 3 into block 2 and be done with it. But at
185 : this point we don't have dataflow information and though we could
186 : hack something up here, it is really not worth the aggravation. */
187 :
188 : static bool
189 9101 : workshare_safe_to_combine_p (basic_block ws_entry_bb)
190 : {
191 9101 : struct omp_for_data fd;
192 9101 : gimple *ws_stmt = last_nondebug_stmt (ws_entry_bb);
193 :
194 9101 : if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
195 : return true;
196 :
197 8978 : gcc_assert (gimple_code (ws_stmt) == GIMPLE_OMP_FOR);
198 8978 : if (gimple_omp_for_kind (ws_stmt) != GF_OMP_FOR_KIND_FOR)
199 : return false;
200 :
201 8961 : omp_extract_for_data (as_a <gomp_for *> (ws_stmt), &fd, NULL);
202 :
203 8961 : if (fd.collapse > 1 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
204 : return false;
205 7182 : if (fd.iter_type != long_integer_type_node)
206 : return false;
207 :
208 : /* FIXME. We give up too easily here. If any of these arguments
209 : are not constants, they will likely involve variables that have
210 : been mapped into fields of .omp_data_s for sharing with the child
211 : function. With appropriate data flow, it would be possible to
212 : see through this. */
213 1508 : if (!is_gimple_min_invariant (fd.loop.n1)
214 1278 : || !is_gimple_min_invariant (fd.loop.n2)
215 1149 : || !is_gimple_min_invariant (fd.loop.step)
216 2642 : || (fd.chunk_size && !is_gimple_min_invariant (fd.chunk_size)))
217 376 : return false;
218 :
219 : return true;
220 : }
221 :
222 : /* Adjust CHUNK_SIZE from SCHEDULE clause, depending on simd modifier
223 : presence (SIMD_SCHEDULE). */
224 :
225 : static tree
226 8341 : omp_adjust_chunk_size (tree chunk_size, bool simd_schedule, bool offload)
227 : {
228 8341 : if (!simd_schedule || integer_zerop (chunk_size))
229 8307 : return chunk_size;
230 :
231 34 : tree vf;
232 34 : tree type = TREE_TYPE (chunk_size);
233 :
234 34 : if (offload)
235 : {
236 2 : cfun->curr_properties &= ~PROP_gimple_lomp_dev;
237 2 : vf = build_call_expr_internal_loc (UNKNOWN_LOCATION, IFN_GOMP_MAX_VF,
238 : unsigned_type_node, 0);
239 2 : vf = fold_convert (type, vf);
240 : }
241 : else
242 : {
243 32 : poly_uint64 vf_num = omp_max_vf (false);
244 32 : if (known_eq (vf_num, 1U))
245 8308 : return chunk_size;
246 31 : vf = build_int_cst (type, vf_num);
247 : }
248 :
249 33 : tree vf_minus_one = fold_build2 (MINUS_EXPR, type, vf,
250 : build_int_cst (type, 1));
251 33 : tree negative_vf = fold_build1 (NEGATE_EXPR, type, vf);
252 33 : chunk_size = fold_build2 (PLUS_EXPR, type, chunk_size, vf_minus_one);
253 33 : return fold_build2 (BIT_AND_EXPR, type, chunk_size, negative_vf);
254 : }
255 :
256 : /* Collect additional arguments needed to emit a combined
257 : parallel+workshare call. WS_STMT is the workshare directive being
258 : expanded. */
259 :
260 : static vec<tree, va_gc> *
261 1200 : get_ws_args_for (gimple *par_stmt, gimple *ws_stmt, bool offload)
262 : {
263 1200 : tree t;
264 1200 : location_t loc = gimple_location (ws_stmt);
265 1200 : vec<tree, va_gc> *ws_args;
266 :
267 1200 : if (gomp_for *for_stmt = dyn_cast <gomp_for *> (ws_stmt))
268 : {
269 1084 : struct omp_for_data fd;
270 1084 : tree n1, n2;
271 :
272 1084 : omp_extract_for_data (for_stmt, &fd, NULL);
273 1084 : n1 = fd.loop.n1;
274 1084 : n2 = fd.loop.n2;
275 :
276 1084 : if (gimple_omp_for_combined_into_p (for_stmt))
277 : {
278 769 : tree innerc
279 769 : = omp_find_clause (gimple_omp_parallel_clauses (par_stmt),
280 : OMP_CLAUSE__LOOPTEMP_);
281 769 : gcc_assert (innerc);
282 769 : n1 = OMP_CLAUSE_DECL (innerc);
283 769 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
284 : OMP_CLAUSE__LOOPTEMP_);
285 769 : gcc_assert (innerc);
286 769 : n2 = OMP_CLAUSE_DECL (innerc);
287 : }
288 :
289 1084 : vec_alloc (ws_args, 3 + (fd.chunk_size != 0));
290 :
291 1084 : t = fold_convert_loc (loc, long_integer_type_node, n1);
292 1084 : ws_args->quick_push (t);
293 :
294 1084 : t = fold_convert_loc (loc, long_integer_type_node, n2);
295 1084 : ws_args->quick_push (t);
296 :
297 1084 : t = fold_convert_loc (loc, long_integer_type_node, fd.loop.step);
298 1084 : ws_args->quick_push (t);
299 :
300 1084 : if (fd.chunk_size)
301 : {
302 538 : t = fold_convert_loc (loc, long_integer_type_node, fd.chunk_size);
303 538 : t = omp_adjust_chunk_size (t, fd.simd_schedule, offload);
304 538 : ws_args->quick_push (t);
305 : }
306 :
307 1084 : return ws_args;
308 : }
309 116 : else if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
310 : {
311 : /* Number of sections is equal to the number of edges from the
312 : GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
313 : the exit of the sections region. */
314 116 : basic_block bb = single_succ (gimple_bb (ws_stmt));
315 232 : t = build_int_cst (unsigned_type_node, EDGE_COUNT (bb->succs) - 1);
316 116 : vec_alloc (ws_args, 1);
317 116 : ws_args->quick_push (t);
318 116 : return ws_args;
319 : }
320 :
321 0 : gcc_unreachable ();
322 : }
323 :
324 : /* Discover whether REGION is a combined parallel+workshare region. */
325 :
326 : static void
327 16280 : determine_parallel_type (struct omp_region *region)
328 : {
329 16280 : basic_block par_entry_bb, par_exit_bb;
330 16280 : basic_block ws_entry_bb, ws_exit_bb;
331 :
332 16280 : if (region == NULL || region->inner == NULL
333 14794 : || region->exit == NULL || region->inner->exit == NULL
334 14783 : || region->inner->cont == NULL)
335 : return;
336 :
337 : /* We only support parallel+for and parallel+sections. */
338 10322 : if (region->type != GIMPLE_OMP_PARALLEL
339 10322 : || (region->inner->type != GIMPLE_OMP_FOR
340 285 : && region->inner->type != GIMPLE_OMP_SECTIONS))
341 : return;
342 :
343 : /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
344 : WS_EXIT_BB -> PAR_EXIT_BB. */
345 10214 : par_entry_bb = region->entry;
346 10214 : par_exit_bb = region->exit;
347 10214 : ws_entry_bb = region->inner->entry;
348 10214 : ws_exit_bb = region->inner->exit;
349 :
350 : /* Give up for task reductions on the parallel, while it is implementable,
351 : adding another big set of APIs or slowing down the normal paths is
352 : not acceptable. */
353 10214 : tree pclauses
354 10214 : = gimple_omp_parallel_clauses (last_nondebug_stmt (par_entry_bb));
355 10214 : if (omp_find_clause (pclauses, OMP_CLAUSE__REDUCTEMP_))
356 : return;
357 :
358 10191 : if (single_succ (par_entry_bb) == ws_entry_bb
359 9138 : && single_succ (ws_exit_bb) == par_exit_bb
360 9101 : && workshare_safe_to_combine_p (ws_entry_bb)
361 11446 : && (gimple_omp_parallel_combined_p (last_nondebug_stmt (par_entry_bb))
362 7 : || (last_and_only_stmt (ws_entry_bb)
363 2 : && last_and_only_stmt (par_exit_bb))))
364 : {
365 1248 : gimple *par_stmt = last_nondebug_stmt (par_entry_bb);
366 1248 : gimple *ws_stmt = last_nondebug_stmt (ws_entry_bb);
367 :
368 1248 : if (region->inner->type == GIMPLE_OMP_FOR)
369 : {
370 : /* If this is a combined parallel loop, we need to determine
371 : whether or not to use the combined library calls. There
372 : are two cases where we do not apply the transformation:
373 : static loops and any kind of ordered loop. In the first
374 : case, we already open code the loop so there is no need
375 : to do anything else. In the latter case, the combined
376 : parallel loop call would still need extra synchronization
377 : to implement ordered semantics, so there would not be any
378 : gain in using the combined call. */
379 1128 : tree clauses = gimple_omp_for_clauses (ws_stmt);
380 1128 : tree c = omp_find_clause (clauses, OMP_CLAUSE_SCHEDULE);
381 1128 : if (c == NULL
382 1110 : || ((OMP_CLAUSE_SCHEDULE_KIND (c) & OMP_CLAUSE_SCHEDULE_MASK)
383 : == OMP_CLAUSE_SCHEDULE_STATIC)
384 1096 : || omp_find_clause (clauses, OMP_CLAUSE_ORDERED)
385 1084 : || omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_)
386 2212 : || ((c = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_))
387 14 : && POINTER_TYPE_P (TREE_TYPE (OMP_CLAUSE_DECL (c)))))
388 44 : return;
389 : }
390 120 : else if (region->inner->type == GIMPLE_OMP_SECTIONS
391 240 : && (omp_find_clause (gimple_omp_sections_clauses (ws_stmt),
392 : OMP_CLAUSE__REDUCTEMP_)
393 120 : || omp_find_clause (gimple_omp_sections_clauses (ws_stmt),
394 : OMP_CLAUSE__CONDTEMP_)))
395 4 : return;
396 :
397 1200 : region->is_combined_parallel = true;
398 1200 : region->inner->is_combined_parallel = true;
399 1200 : region->ws_args = get_ws_args_for (par_stmt, ws_stmt,
400 1200 : is_in_offload_region (region));
401 : }
402 : }
403 :
404 : /* Debugging dumps for parallel regions. */
405 : void dump_omp_region (FILE *, struct omp_region *, int);
406 : void debug_omp_region (struct omp_region *);
407 : void debug_all_omp_regions (void);
408 :
409 : /* Dump the parallel region tree rooted at REGION. */
410 :
411 : void
412 327 : dump_omp_region (FILE *file, struct omp_region *region, int indent)
413 : {
414 380 : fprintf (file, "%*sbb %d: %s\n", indent, "", region->entry->index,
415 380 : gimple_code_name[region->type]);
416 :
417 380 : if (region->inner)
418 71 : dump_omp_region (file, region->inner, indent + 4);
419 :
420 380 : if (region->cont)
421 : {
422 167 : fprintf (file, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent, "",
423 : region->cont->index);
424 : }
425 :
426 380 : if (region->exit)
427 357 : fprintf (file, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent, "",
428 : region->exit->index);
429 : else
430 23 : fprintf (file, "%*s[no exit marker]\n", indent, "");
431 :
432 380 : if (region->next)
433 : dump_omp_region (file, region->next, indent);
434 327 : }
435 :
436 : DEBUG_FUNCTION void
437 0 : debug_omp_region (struct omp_region *region)
438 : {
439 0 : dump_omp_region (stderr, region, 0);
440 0 : }
441 :
442 : DEBUG_FUNCTION void
443 0 : debug_all_omp_regions (void)
444 : {
445 0 : dump_omp_region (stderr, root_omp_region, 0);
446 0 : }
447 :
448 : /* Create a new parallel region starting at STMT inside region PARENT. */
449 :
450 : static struct omp_region *
451 246624 : new_omp_region (basic_block bb, enum gimple_code type,
452 : struct omp_region *parent)
453 : {
454 246624 : struct omp_region *region = XCNEW (struct omp_region);
455 :
456 246624 : region->outer = parent;
457 246624 : region->entry = bb;
458 246624 : region->type = type;
459 :
460 246624 : if (parent)
461 : {
462 : /* This is a nested region. Add it to the list of inner
463 : regions in PARENT. */
464 127269 : region->next = parent->inner;
465 127269 : parent->inner = region;
466 : }
467 : else
468 : {
469 : /* This is a toplevel region. Add it to the list of toplevel
470 : regions in ROOT_OMP_REGION. */
471 119355 : region->next = root_omp_region;
472 119355 : root_omp_region = region;
473 : }
474 :
475 246624 : return region;
476 : }
477 :
478 : /* Release the memory associated with the region tree rooted at REGION. */
479 :
480 : static void
481 246624 : free_omp_region_1 (struct omp_region *region)
482 : {
483 246624 : struct omp_region *i, *n;
484 :
485 373893 : for (i = region->inner; i ; i = n)
486 : {
487 127269 : n = i->next;
488 127269 : free_omp_region_1 (i);
489 : }
490 :
491 246624 : free (region);
492 246624 : }
493 :
494 : /* Release the memory for the entire omp region tree. */
495 :
496 : void
497 3032746 : omp_free_regions (void)
498 : {
499 3032746 : struct omp_region *r, *n;
500 3152101 : for (r = root_omp_region; r ; r = n)
501 : {
502 119355 : n = r->next;
503 119355 : free_omp_region_1 (r);
504 : }
505 3032746 : root_omp_region = NULL;
506 3032746 : }
507 :
508 : /* A convenience function to build an empty GIMPLE_COND with just the
509 : condition. */
510 :
511 : static gcond *
512 112624 : gimple_build_cond_empty (tree cond)
513 : {
514 112624 : enum tree_code pred_code;
515 112624 : tree lhs, rhs;
516 :
517 112624 : gimple_cond_get_ops_from_tree (cond, &pred_code, &lhs, &rhs);
518 112624 : return gimple_build_cond (pred_code, lhs, rhs, NULL_TREE, NULL_TREE);
519 : }
520 :
521 : /* Change DECL_CONTEXT of CHILD_FNDECL to that of the parent function.
522 : Add CHILD_FNDECL to decl chain of the supercontext of the block
523 : ENTRY_BLOCK - this is the block which originally contained the
524 : code from which CHILD_FNDECL was created.
525 :
526 : Together, these actions ensure that the debug info for the outlined
527 : function will be emitted with the correct lexical scope. */
528 :
529 : static void
530 43690 : adjust_context_and_scope (struct omp_region *region, tree entry_block,
531 : tree child_fndecl)
532 : {
533 43690 : tree parent_fndecl = NULL_TREE;
534 43690 : gimple *entry_stmt;
535 : /* OMP expansion expands inner regions before outer ones, so if
536 : we e.g. have explicit task region nested in parallel region, when
537 : expanding the task region current_function_decl will be the original
538 : source function, but we actually want to use as context the child
539 : function of the parallel. */
540 43690 : for (region = region->outer;
541 64755 : region && parent_fndecl == NULL_TREE; region = region->outer)
542 21065 : switch (region->type)
543 : {
544 6799 : case GIMPLE_OMP_PARALLEL:
545 6799 : case GIMPLE_OMP_TASK:
546 6799 : case GIMPLE_OMP_TEAMS:
547 6799 : entry_stmt = last_nondebug_stmt (region->entry);
548 6799 : parent_fndecl = gimple_omp_taskreg_child_fn (entry_stmt);
549 6799 : break;
550 4556 : case GIMPLE_OMP_TARGET:
551 4556 : entry_stmt = last_nondebug_stmt (region->entry);
552 4556 : parent_fndecl
553 4556 : = gimple_omp_target_child_fn (as_a <gomp_target *> (entry_stmt));
554 4556 : break;
555 : default:
556 : break;
557 : }
558 :
559 43690 : if (parent_fndecl == NULL_TREE)
560 35444 : parent_fndecl = current_function_decl;
561 43690 : DECL_CONTEXT (child_fndecl) = parent_fndecl;
562 :
563 43690 : if (entry_block != NULL_TREE && TREE_CODE (entry_block) == BLOCK)
564 : {
565 43656 : tree b = BLOCK_SUPERCONTEXT (entry_block);
566 43656 : if (TREE_CODE (b) == BLOCK)
567 : {
568 43521 : DECL_CHAIN (child_fndecl) = BLOCK_VARS (b);
569 43521 : BLOCK_VARS (b) = child_fndecl;
570 : }
571 : }
572 43690 : }
573 :
574 : /* Build the function calls to GOMP_parallel etc to actually
575 : generate the parallel operation. REGION is the parallel region
576 : being expanded. BB is the block where to insert the code. WS_ARGS
577 : will be set if this is a call to a combined parallel+workshare
578 : construct, it contains the list of additional arguments needed by
579 : the workshare construct. */
580 :
581 : static void
582 16280 : expand_parallel_call (struct omp_region *region, basic_block bb,
583 : gomp_parallel *entry_stmt,
584 : vec<tree, va_gc> *ws_args)
585 : {
586 16280 : tree t, t1, t2, val, cond, c, clauses, flags;
587 16280 : gimple_stmt_iterator gsi;
588 16280 : gimple *stmt;
589 16280 : enum built_in_function start_ix;
590 16280 : int start_ix2;
591 16280 : location_t clause_loc;
592 16280 : vec<tree, va_gc> *args;
593 :
594 16280 : clauses = gimple_omp_parallel_clauses (entry_stmt);
595 :
596 : /* Determine what flavor of GOMP_parallel we will be
597 : emitting. */
598 16280 : start_ix = BUILT_IN_GOMP_PARALLEL;
599 16280 : tree rtmp = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
600 16280 : if (rtmp)
601 : start_ix = BUILT_IN_GOMP_PARALLEL_REDUCTIONS;
602 16220 : else if (is_combined_parallel (region))
603 : {
604 1200 : switch (region->inner->type)
605 : {
606 1084 : case GIMPLE_OMP_FOR:
607 1084 : gcc_assert (region->inner->sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
608 1084 : switch (region->inner->sched_kind)
609 : {
610 546 : case OMP_CLAUSE_SCHEDULE_RUNTIME:
611 : /* For lastprivate(conditional:), our implementation
612 : requires monotonic behavior. */
613 546 : if (region->inner->has_lastprivate_conditional != 0)
614 : start_ix2 = 3;
615 544 : else if ((region->inner->sched_modifiers
616 : & OMP_CLAUSE_SCHEDULE_NONMONOTONIC) != 0)
617 : start_ix2 = 6;
618 530 : else if ((region->inner->sched_modifiers
619 : & OMP_CLAUSE_SCHEDULE_MONOTONIC) == 0)
620 : start_ix2 = 7;
621 : else
622 16 : start_ix2 = 3;
623 : break;
624 538 : case OMP_CLAUSE_SCHEDULE_DYNAMIC:
625 538 : case OMP_CLAUSE_SCHEDULE_GUIDED:
626 538 : if ((region->inner->sched_modifiers
627 : & OMP_CLAUSE_SCHEDULE_MONOTONIC) == 0
628 534 : && !region->inner->has_lastprivate_conditional)
629 : {
630 522 : start_ix2 = 3 + region->inner->sched_kind;
631 522 : break;
632 : }
633 : /* FALLTHRU */
634 16 : default:
635 16 : start_ix2 = region->inner->sched_kind;
636 16 : break;
637 : }
638 1084 : start_ix2 += (int) BUILT_IN_GOMP_PARALLEL_LOOP_STATIC;
639 1084 : start_ix = (enum built_in_function) start_ix2;
640 1084 : break;
641 : case GIMPLE_OMP_SECTIONS:
642 : start_ix = BUILT_IN_GOMP_PARALLEL_SECTIONS;
643 : break;
644 0 : default:
645 0 : gcc_unreachable ();
646 : }
647 : }
648 :
649 : /* By default, the value of NUM_THREADS is zero (selected at run time)
650 : and there is no conditional. */
651 16280 : cond = NULL_TREE;
652 16280 : val = build_int_cst (unsigned_type_node, 0);
653 16280 : flags = build_int_cst (unsigned_type_node, 0);
654 :
655 16280 : c = omp_find_clause (clauses, OMP_CLAUSE_IF);
656 16280 : if (c)
657 963 : cond = OMP_CLAUSE_IF_EXPR (c);
658 :
659 16280 : c = omp_find_clause (clauses, OMP_CLAUSE_NUM_THREADS);
660 16280 : if (c)
661 : {
662 2272 : val = OMP_CLAUSE_NUM_THREADS_EXPR (c);
663 2272 : clause_loc = OMP_CLAUSE_LOCATION (c);
664 : }
665 : else
666 14008 : clause_loc = gimple_location (entry_stmt);
667 :
668 16280 : c = omp_find_clause (clauses, OMP_CLAUSE_PROC_BIND);
669 16280 : if (c)
670 771 : flags = build_int_cst (unsigned_type_node, OMP_CLAUSE_PROC_BIND_KIND (c));
671 :
672 : /* Ensure 'val' is of the correct type. */
673 16280 : val = fold_convert_loc (clause_loc, unsigned_type_node, val);
674 :
675 : /* If we found the clause 'if (cond)', build either
676 : (cond != 0) or (cond ? val : 1u). */
677 16280 : if (cond)
678 : {
679 963 : cond = gimple_boolify (cond);
680 :
681 963 : if (integer_zerop (val))
682 176 : val = fold_build2_loc (clause_loc,
683 : EQ_EXPR, unsigned_type_node, cond,
684 176 : build_int_cst (TREE_TYPE (cond), 0));
685 : else
686 : {
687 787 : basic_block cond_bb, then_bb, else_bb;
688 787 : edge e, e_then, e_else;
689 787 : tree tmp_then, tmp_else, tmp_join, tmp_var;
690 :
691 787 : tmp_var = create_tmp_var (TREE_TYPE (val));
692 787 : if (gimple_in_ssa_p (cfun))
693 : {
694 0 : tmp_then = make_ssa_name (tmp_var);
695 0 : tmp_else = make_ssa_name (tmp_var);
696 0 : tmp_join = make_ssa_name (tmp_var);
697 : }
698 : else
699 : {
700 : tmp_then = tmp_var;
701 : tmp_else = tmp_var;
702 : tmp_join = tmp_var;
703 : }
704 :
705 787 : e = split_block_after_labels (bb);
706 787 : cond_bb = e->src;
707 787 : bb = e->dest;
708 787 : remove_edge (e);
709 :
710 787 : then_bb = create_empty_bb (cond_bb);
711 787 : else_bb = create_empty_bb (then_bb);
712 787 : set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
713 787 : set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
714 :
715 787 : stmt = gimple_build_cond_empty (cond);
716 787 : gsi = gsi_start_bb (cond_bb);
717 787 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
718 :
719 787 : gsi = gsi_start_bb (then_bb);
720 787 : expand_omp_build_assign (&gsi, tmp_then, val, true);
721 :
722 787 : gsi = gsi_start_bb (else_bb);
723 787 : expand_omp_build_assign (&gsi, tmp_else,
724 : build_int_cst (unsigned_type_node, 1),
725 : true);
726 :
727 787 : make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
728 787 : make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
729 787 : add_bb_to_loop (then_bb, cond_bb->loop_father);
730 787 : add_bb_to_loop (else_bb, cond_bb->loop_father);
731 787 : e_then = make_edge (then_bb, bb, EDGE_FALLTHRU);
732 787 : e_else = make_edge (else_bb, bb, EDGE_FALLTHRU);
733 :
734 787 : if (gimple_in_ssa_p (cfun))
735 : {
736 0 : gphi *phi = create_phi_node (tmp_join, bb);
737 0 : add_phi_arg (phi, tmp_then, e_then, UNKNOWN_LOCATION);
738 0 : add_phi_arg (phi, tmp_else, e_else, UNKNOWN_LOCATION);
739 : }
740 :
741 787 : val = tmp_join;
742 : }
743 :
744 963 : gsi = gsi_start_bb (bb);
745 963 : val = force_gimple_operand_gsi (&gsi, val, true, NULL_TREE,
746 : false, GSI_CONTINUE_LINKING);
747 : }
748 :
749 16280 : gsi = gsi_last_nondebug_bb (bb);
750 16280 : t = gimple_omp_parallel_data_arg (entry_stmt);
751 16280 : if (t == NULL)
752 2406 : t1 = null_pointer_node;
753 : else
754 13874 : t1 = build_fold_addr_expr (t);
755 16280 : tree child_fndecl = gimple_omp_parallel_child_fn (entry_stmt);
756 16280 : t2 = build_fold_addr_expr (child_fndecl);
757 :
758 17480 : vec_alloc (args, 4 + vec_safe_length (ws_args));
759 16280 : args->quick_push (t2);
760 16280 : args->quick_push (t1);
761 16280 : args->quick_push (val);
762 16280 : if (ws_args)
763 1200 : args->splice (*ws_args);
764 16280 : args->quick_push (flags);
765 :
766 16280 : t = build_call_expr_loc_vec (UNKNOWN_LOCATION,
767 : builtin_decl_explicit (start_ix), args);
768 :
769 16280 : if (rtmp)
770 : {
771 60 : tree type = TREE_TYPE (OMP_CLAUSE_DECL (rtmp));
772 60 : t = build2 (MODIFY_EXPR, type, OMP_CLAUSE_DECL (rtmp),
773 : fold_convert (type,
774 : fold_convert (pointer_sized_int_node, t)));
775 : }
776 16280 : force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
777 : false, GSI_CONTINUE_LINKING);
778 16280 : }
779 :
780 : /* Build the function call to GOMP_task to actually
781 : generate the task operation. BB is the block where to insert the code. */
782 :
783 : static void
784 3748 : expand_task_call (struct omp_region *region, basic_block bb,
785 : gomp_task *entry_stmt)
786 : {
787 3748 : tree t1, t2, t3;
788 3748 : gimple_stmt_iterator gsi;
789 3748 : location_t loc = gimple_location (entry_stmt);
790 :
791 3748 : tree clauses = gimple_omp_task_clauses (entry_stmt);
792 :
793 3748 : tree ifc = omp_find_clause (clauses, OMP_CLAUSE_IF);
794 3748 : tree untied = omp_find_clause (clauses, OMP_CLAUSE_UNTIED);
795 3748 : tree mergeable = omp_find_clause (clauses, OMP_CLAUSE_MERGEABLE);
796 3748 : tree depend = omp_find_clause (clauses, OMP_CLAUSE_DEPEND);
797 3748 : tree finalc = omp_find_clause (clauses, OMP_CLAUSE_FINAL);
798 3748 : tree priority = omp_find_clause (clauses, OMP_CLAUSE_PRIORITY);
799 3748 : tree detach = omp_find_clause (clauses, OMP_CLAUSE_DETACH);
800 :
801 7496 : unsigned int iflags
802 3748 : = (untied ? GOMP_TASK_FLAG_UNTIED : 0)
803 3748 : | (mergeable ? GOMP_TASK_FLAG_MERGEABLE : 0)
804 3748 : | (depend ? GOMP_TASK_FLAG_DEPEND : 0);
805 :
806 3748 : bool taskloop_p = gimple_omp_task_taskloop_p (entry_stmt);
807 3748 : tree startvar = NULL_TREE, endvar = NULL_TREE, step = NULL_TREE;
808 3748 : tree num_tasks = NULL_TREE;
809 3748 : bool ull = false;
810 3748 : if (taskloop_p)
811 : {
812 1330 : gimple *g = last_nondebug_stmt (region->outer->entry);
813 1330 : gcc_assert (gimple_code (g) == GIMPLE_OMP_FOR
814 : && gimple_omp_for_kind (g) == GF_OMP_FOR_KIND_TASKLOOP);
815 1330 : struct omp_for_data fd;
816 1330 : omp_extract_for_data (as_a <gomp_for *> (g), &fd, NULL);
817 1330 : startvar = omp_find_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
818 1330 : endvar = omp_find_clause (OMP_CLAUSE_CHAIN (startvar),
819 : OMP_CLAUSE__LOOPTEMP_);
820 1330 : startvar = OMP_CLAUSE_DECL (startvar);
821 1330 : endvar = OMP_CLAUSE_DECL (endvar);
822 1330 : step = fold_convert_loc (loc, fd.iter_type, fd.loop.step);
823 1330 : if (fd.loop.cond_code == LT_EXPR)
824 1278 : iflags |= GOMP_TASK_FLAG_UP;
825 1330 : tree tclauses = gimple_omp_for_clauses (g);
826 1330 : num_tasks = omp_find_clause (tclauses, OMP_CLAUSE_NUM_TASKS);
827 1330 : if (num_tasks)
828 : {
829 176 : if (OMP_CLAUSE_NUM_TASKS_STRICT (num_tasks))
830 3 : iflags |= GOMP_TASK_FLAG_STRICT;
831 176 : num_tasks = OMP_CLAUSE_NUM_TASKS_EXPR (num_tasks);
832 : }
833 : else
834 : {
835 1154 : num_tasks = omp_find_clause (tclauses, OMP_CLAUSE_GRAINSIZE);
836 1154 : if (num_tasks)
837 : {
838 220 : iflags |= GOMP_TASK_FLAG_GRAINSIZE;
839 220 : if (OMP_CLAUSE_GRAINSIZE_STRICT (num_tasks))
840 3 : iflags |= GOMP_TASK_FLAG_STRICT;
841 220 : num_tasks = OMP_CLAUSE_GRAINSIZE_EXPR (num_tasks);
842 : }
843 : else
844 934 : num_tasks = integer_zero_node;
845 : }
846 1330 : num_tasks = fold_convert_loc (loc, long_integer_type_node, num_tasks);
847 1330 : if (ifc == NULL_TREE)
848 959 : iflags |= GOMP_TASK_FLAG_IF;
849 1330 : if (omp_find_clause (tclauses, OMP_CLAUSE_NOGROUP))
850 57 : iflags |= GOMP_TASK_FLAG_NOGROUP;
851 1330 : ull = fd.iter_type == long_long_unsigned_type_node;
852 1330 : if (omp_find_clause (clauses, OMP_CLAUSE_REDUCTION))
853 497 : iflags |= GOMP_TASK_FLAG_REDUCTION;
854 : }
855 : else
856 : {
857 2418 : if (priority)
858 18 : iflags |= GOMP_TASK_FLAG_PRIORITY;
859 2418 : if (detach)
860 166 : iflags |= GOMP_TASK_FLAG_DETACH;
861 : }
862 :
863 3748 : tree flags = build_int_cst (unsigned_type_node, iflags);
864 :
865 3748 : tree cond = boolean_true_node;
866 3748 : if (ifc)
867 : {
868 478 : if (taskloop_p)
869 : {
870 371 : tree t = gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc));
871 371 : t = fold_build3_loc (loc, COND_EXPR, unsigned_type_node, t,
872 : build_int_cst (unsigned_type_node,
873 : GOMP_TASK_FLAG_IF),
874 : build_int_cst (unsigned_type_node, 0));
875 371 : flags = fold_build2_loc (loc, PLUS_EXPR, unsigned_type_node,
876 : flags, t);
877 : }
878 : else
879 107 : cond = gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc));
880 : }
881 :
882 3748 : if (finalc)
883 : {
884 410 : tree t = gimple_boolify (OMP_CLAUSE_FINAL_EXPR (finalc));
885 410 : t = fold_build3_loc (loc, COND_EXPR, unsigned_type_node, t,
886 : build_int_cst (unsigned_type_node,
887 : GOMP_TASK_FLAG_FINAL),
888 : build_int_cst (unsigned_type_node, 0));
889 410 : flags = fold_build2_loc (loc, PLUS_EXPR, unsigned_type_node, flags, t);
890 : }
891 3748 : if (depend)
892 1105 : depend = OMP_CLAUSE_DECL (depend);
893 : else
894 2643 : depend = build_int_cst (ptr_type_node, 0);
895 3748 : if (priority)
896 372 : priority = fold_convert (integer_type_node,
897 : OMP_CLAUSE_PRIORITY_EXPR (priority));
898 : else
899 3376 : priority = integer_zero_node;
900 :
901 3748 : gsi = gsi_last_nondebug_bb (bb);
902 :
903 3582 : detach = (detach
904 3914 : ? build_fold_addr_expr (OMP_CLAUSE_DECL (detach))
905 : : null_pointer_node);
906 :
907 3748 : tree t = gimple_omp_task_data_arg (entry_stmt);
908 3748 : if (t == NULL)
909 685 : t2 = null_pointer_node;
910 : else
911 3063 : t2 = build_fold_addr_expr_loc (loc, t);
912 3748 : t1 = build_fold_addr_expr_loc (loc, gimple_omp_task_child_fn (entry_stmt));
913 3748 : t = gimple_omp_task_copy_fn (entry_stmt);
914 3748 : if (t == NULL)
915 3268 : t3 = null_pointer_node;
916 : else
917 480 : t3 = build_fold_addr_expr_loc (loc, t);
918 :
919 3748 : if (taskloop_p)
920 2660 : t = build_call_expr (ull
921 42 : ? builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP_ULL)
922 1288 : : builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP),
923 : 11, t1, t2, t3,
924 : gimple_omp_task_arg_size (entry_stmt),
925 : gimple_omp_task_arg_align (entry_stmt), flags,
926 : num_tasks, priority, startvar, endvar, step);
927 : else
928 2418 : t = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK),
929 : 10, t1, t2, t3,
930 : gimple_omp_task_arg_size (entry_stmt),
931 : gimple_omp_task_arg_align (entry_stmt), cond, flags,
932 : depend, priority, detach);
933 :
934 3748 : force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
935 : false, GSI_CONTINUE_LINKING);
936 3748 : }
937 :
938 : /* Build the function call to GOMP_taskwait_depend to actually
939 : generate the taskwait operation. BB is the block where to insert the
940 : code. */
941 :
942 : static void
943 84 : expand_taskwait_call (basic_block bb, gomp_task *entry_stmt)
944 : {
945 84 : tree clauses = gimple_omp_task_clauses (entry_stmt);
946 84 : tree depend = omp_find_clause (clauses, OMP_CLAUSE_DEPEND);
947 84 : if (depend == NULL_TREE)
948 0 : return;
949 :
950 84 : depend = OMP_CLAUSE_DECL (depend);
951 :
952 84 : bool nowait = omp_find_clause (clauses, OMP_CLAUSE_NOWAIT) != NULL_TREE;
953 84 : gimple_stmt_iterator gsi = gsi_last_nondebug_bb (bb);
954 51 : enum built_in_function f = (nowait
955 84 : ? BUILT_IN_GOMP_TASKWAIT_DEPEND_NOWAIT
956 : : BUILT_IN_GOMP_TASKWAIT_DEPEND);
957 84 : tree t = build_call_expr (builtin_decl_explicit (f), 1, depend);
958 :
959 84 : force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
960 : false, GSI_CONTINUE_LINKING);
961 : }
962 :
963 : /* Build the function call to GOMP_teams_reg to actually
964 : generate the host teams operation. REGION is the teams region
965 : being expanded. BB is the block where to insert the code. */
966 :
967 : static void
968 2518 : expand_teams_call (basic_block bb, gomp_teams *entry_stmt)
969 : {
970 2518 : tree clauses = gimple_omp_teams_clauses (entry_stmt);
971 2518 : tree num_teams = omp_find_clause (clauses, OMP_CLAUSE_NUM_TEAMS);
972 2518 : if (num_teams == NULL_TREE)
973 2252 : num_teams = build_int_cst (unsigned_type_node, 0);
974 : else
975 : {
976 266 : num_teams = OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (num_teams);
977 266 : num_teams = fold_convert (unsigned_type_node, num_teams);
978 : }
979 2518 : tree thread_limit = omp_find_clause (clauses, OMP_CLAUSE_THREAD_LIMIT);
980 2518 : if (thread_limit == NULL_TREE)
981 2377 : thread_limit = build_int_cst (unsigned_type_node, 0);
982 : else
983 : {
984 141 : thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit);
985 141 : thread_limit = fold_convert (unsigned_type_node, thread_limit);
986 : }
987 :
988 2518 : gimple_stmt_iterator gsi = gsi_last_nondebug_bb (bb);
989 2518 : tree t = gimple_omp_teams_data_arg (entry_stmt), t1;
990 2518 : if (t == NULL)
991 1374 : t1 = null_pointer_node;
992 : else
993 1144 : t1 = build_fold_addr_expr (t);
994 2518 : tree child_fndecl = gimple_omp_teams_child_fn (entry_stmt);
995 2518 : tree t2 = build_fold_addr_expr (child_fndecl);
996 :
997 2518 : vec<tree, va_gc> *args;
998 2518 : vec_alloc (args, 5);
999 2518 : args->quick_push (t2);
1000 2518 : args->quick_push (t1);
1001 2518 : args->quick_push (num_teams);
1002 2518 : args->quick_push (thread_limit);
1003 : /* For future extensibility. */
1004 2518 : args->quick_push (build_zero_cst (unsigned_type_node));
1005 :
1006 2518 : t = build_call_expr_loc_vec (UNKNOWN_LOCATION,
1007 : builtin_decl_explicit (BUILT_IN_GOMP_TEAMS_REG),
1008 : args);
1009 :
1010 2518 : force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
1011 : false, GSI_CONTINUE_LINKING);
1012 2518 : }
1013 :
1014 : /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
1015 :
1016 : static tree
1017 43690 : vec2chain (vec<tree, va_gc> *v)
1018 : {
1019 43690 : tree chain = NULL_TREE, t;
1020 43690 : unsigned ix;
1021 :
1022 445907 : FOR_EACH_VEC_SAFE_ELT_REVERSE (v, ix, t)
1023 : {
1024 364015 : DECL_CHAIN (t) = chain;
1025 364015 : chain = t;
1026 : }
1027 :
1028 43690 : return chain;
1029 : }
1030 :
1031 : /* Remove barriers in REGION->EXIT's block. Note that this is only
1032 : valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
1033 : is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
1034 : left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
1035 : removed. */
1036 :
1037 : static void
1038 11844 : remove_exit_barrier (struct omp_region *region)
1039 : {
1040 11844 : gimple_stmt_iterator gsi;
1041 11844 : basic_block exit_bb;
1042 11844 : edge_iterator ei;
1043 11844 : edge e;
1044 11844 : gimple *stmt;
1045 11844 : int any_addressable_vars = -1;
1046 :
1047 11844 : exit_bb = region->exit;
1048 :
1049 : /* If the parallel region doesn't return, we don't have REGION->EXIT
1050 : block at all. */
1051 11844 : if (! exit_bb)
1052 1208 : return;
1053 :
1054 : /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
1055 : workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
1056 : statements that can appear in between are extremely limited -- no
1057 : memory operations at all. Here, we allow nothing at all, so the
1058 : only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
1059 11817 : gsi = gsi_last_nondebug_bb (exit_bb);
1060 11817 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
1061 11817 : gsi_prev_nondebug (&gsi);
1062 11817 : if (!gsi_end_p (gsi) && gimple_code (gsi_stmt (gsi)) != GIMPLE_LABEL)
1063 : return;
1064 :
1065 21588 : FOR_EACH_EDGE (e, ei, exit_bb->preds)
1066 : {
1067 10952 : gsi = gsi_last_nondebug_bb (e->src);
1068 10952 : if (gsi_end_p (gsi))
1069 43 : continue;
1070 10909 : stmt = gsi_stmt (gsi);
1071 10909 : if (gimple_code (stmt) == GIMPLE_OMP_RETURN
1072 10909 : && !gimple_omp_return_nowait_p (stmt))
1073 : {
1074 : /* OpenMP 3.0 tasks unfortunately prevent this optimization
1075 : in many cases. If there could be tasks queued, the barrier
1076 : might be needed to let the tasks run before some local
1077 : variable of the parallel that the task uses as shared
1078 : runs out of scope. The task can be spawned either
1079 : from within current function (this would be easy to check)
1080 : or from some function it calls and gets passed an address
1081 : of such a variable. */
1082 1057 : if (any_addressable_vars < 0)
1083 : {
1084 1057 : gomp_parallel *parallel_stmt
1085 1057 : = as_a <gomp_parallel *> (last_nondebug_stmt (region->entry));
1086 1057 : tree child_fun = gimple_omp_parallel_child_fn (parallel_stmt);
1087 1057 : tree local_decls, block, decl;
1088 1057 : unsigned ix;
1089 :
1090 1057 : any_addressable_vars = 0;
1091 8567 : FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun), ix, decl)
1092 7067 : if (TREE_ADDRESSABLE (decl))
1093 : {
1094 : any_addressable_vars = 1;
1095 : break;
1096 : }
1097 1058 : for (block = gimple_block (stmt);
1098 1058 : !any_addressable_vars
1099 1058 : && block
1100 1058 : && TREE_CODE (block) == BLOCK;
1101 1 : block = BLOCK_SUPERCONTEXT (block))
1102 : {
1103 2 : for (local_decls = BLOCK_VARS (block);
1104 2 : local_decls;
1105 0 : local_decls = DECL_CHAIN (local_decls))
1106 0 : if (TREE_ADDRESSABLE (local_decls))
1107 : {
1108 : any_addressable_vars = 1;
1109 : break;
1110 : }
1111 2 : if (block == gimple_block (parallel_stmt))
1112 : break;
1113 : }
1114 : }
1115 1057 : if (!any_addressable_vars)
1116 529 : gimple_omp_return_set_nowait (stmt);
1117 : }
1118 : }
1119 : }
1120 :
1121 : static void
1122 68668 : remove_exit_barriers (struct omp_region *region)
1123 : {
1124 68668 : if (region->type == GIMPLE_OMP_PARALLEL)
1125 11844 : remove_exit_barrier (region);
1126 :
1127 68668 : if (region->inner)
1128 : {
1129 37867 : region = region->inner;
1130 37867 : remove_exit_barriers (region);
1131 81929 : while (region->next)
1132 : {
1133 6195 : region = region->next;
1134 6195 : remove_exit_barriers (region);
1135 : }
1136 : }
1137 68668 : }
1138 :
1139 : /* Optimize omp_get_thread_num () and omp_get_num_threads ()
1140 : calls. These can't be declared as const functions, but
1141 : within one parallel body they are constant, so they can be
1142 : transformed there into __builtin_omp_get_{thread_num,num_threads} ()
1143 : which are declared const. Similarly for task body, except
1144 : that in untied task omp_get_thread_num () can change at any task
1145 : scheduling point. */
1146 :
1147 : static void
1148 16828 : optimize_omp_library_calls (gimple *entry_stmt)
1149 : {
1150 16828 : basic_block bb;
1151 16828 : gimple_stmt_iterator gsi;
1152 16828 : tree thr_num_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
1153 16828 : tree thr_num_id = DECL_ASSEMBLER_NAME (thr_num_tree);
1154 16828 : tree num_thr_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
1155 16828 : tree num_thr_id = DECL_ASSEMBLER_NAME (num_thr_tree);
1156 16828 : bool untied_task = (gimple_code (entry_stmt) == GIMPLE_OMP_TASK
1157 19249 : && omp_find_clause (gimple_omp_task_clauses (entry_stmt),
1158 16828 : OMP_CLAUSE_UNTIED) != NULL);
1159 :
1160 292944 : FOR_EACH_BB_FN (bb, cfun)
1161 1549418 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1162 : {
1163 997186 : gimple *call = gsi_stmt (gsi);
1164 997186 : tree decl;
1165 :
1166 997186 : if (is_gimple_call (call)
1167 74115 : && (decl = gimple_call_fndecl (call))
1168 66795 : && DECL_EXTERNAL (decl)
1169 55085 : && TREE_PUBLIC (decl)
1170 1052271 : && DECL_INITIAL (decl) == NULL)
1171 : {
1172 55084 : tree built_in;
1173 :
1174 55084 : if (DECL_NAME (decl) == thr_num_id)
1175 : {
1176 : /* In #pragma omp task untied omp_get_thread_num () can change
1177 : during the execution of the task region. */
1178 1278 : if (untied_task)
1179 0 : continue;
1180 1278 : built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
1181 : }
1182 53806 : else if (DECL_NAME (decl) == num_thr_id)
1183 412 : built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
1184 : else
1185 53394 : continue;
1186 :
1187 1690 : if (DECL_ASSEMBLER_NAME (decl) != DECL_ASSEMBLER_NAME (built_in)
1188 1690 : || gimple_call_num_args (call) != 0)
1189 1160 : continue;
1190 :
1191 530 : if (flag_exceptions && !TREE_NOTHROW (decl))
1192 0 : continue;
1193 :
1194 530 : if (TREE_CODE (TREE_TYPE (decl)) != FUNCTION_TYPE
1195 1060 : || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl)),
1196 530 : TREE_TYPE (TREE_TYPE (built_in))))
1197 0 : continue;
1198 :
1199 530 : gimple_call_set_fndecl (call, built_in);
1200 : }
1201 : }
1202 16828 : }
1203 :
1204 : /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
1205 : regimplified. */
1206 :
1207 : static tree
1208 263167 : expand_omp_regimplify_p (tree *tp, int *walk_subtrees, void *)
1209 : {
1210 263167 : tree t = *tp;
1211 :
1212 : /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
1213 263167 : if (VAR_P (t) && DECL_HAS_VALUE_EXPR_P (t))
1214 : return t;
1215 :
1216 263095 : if (TREE_CODE (t) == ADDR_EXPR)
1217 1763 : recompute_tree_invariant_for_addr_expr (t);
1218 :
1219 263095 : *walk_subtrees = !TYPE_P (t) && !DECL_P (t);
1220 263095 : return NULL_TREE;
1221 : }
1222 :
1223 : /* Prepend or append TO = FROM assignment before or after *GSI_P. */
1224 :
1225 : static void
1226 64771 : expand_omp_build_assign (gimple_stmt_iterator *gsi_p, tree to, tree from,
1227 : bool after)
1228 : {
1229 64771 : bool simple_p = DECL_P (to) && TREE_ADDRESSABLE (to);
1230 64771 : from = force_gimple_operand_gsi (gsi_p, from, simple_p, NULL_TREE,
1231 64771 : !after, after ? GSI_CONTINUE_LINKING
1232 : : GSI_SAME_STMT);
1233 64771 : gimple *stmt = gimple_build_assign (to, from);
1234 64771 : if (after)
1235 3469 : gsi_insert_after (gsi_p, stmt, GSI_CONTINUE_LINKING);
1236 : else
1237 61302 : gsi_insert_before (gsi_p, stmt, GSI_SAME_STMT);
1238 64771 : if (walk_tree (&from, expand_omp_regimplify_p, NULL, NULL)
1239 64771 : || walk_tree (&to, expand_omp_regimplify_p, NULL, NULL))
1240 : {
1241 56 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
1242 56 : gimple_regimplify_operands (stmt, &gsi);
1243 : }
1244 64771 : }
1245 :
1246 : /* Prepend or append LHS CODE RHS condition before or after *GSI_P. */
1247 :
1248 : static gcond *
1249 8434 : expand_omp_build_cond (gimple_stmt_iterator *gsi_p, enum tree_code code,
1250 : tree lhs, tree rhs, bool after = false)
1251 : {
1252 8434 : gcond *cond_stmt = gimple_build_cond (code, lhs, rhs, NULL_TREE, NULL_TREE);
1253 8434 : if (after)
1254 236 : gsi_insert_after (gsi_p, cond_stmt, GSI_CONTINUE_LINKING);
1255 : else
1256 8198 : gsi_insert_before (gsi_p, cond_stmt, GSI_SAME_STMT);
1257 8434 : if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), expand_omp_regimplify_p,
1258 : NULL, NULL)
1259 8434 : || walk_tree (gimple_cond_rhs_ptr (cond_stmt), expand_omp_regimplify_p,
1260 : NULL, NULL))
1261 : {
1262 6 : gimple_stmt_iterator gsi = gsi_for_stmt (cond_stmt);
1263 6 : gimple_regimplify_operands (cond_stmt, &gsi);
1264 : }
1265 8434 : return cond_stmt;
1266 : }
1267 :
1268 : /* Expand the OpenMP parallel or task directive starting at REGION. */
1269 :
1270 : static void
1271 22630 : expand_omp_taskreg (struct omp_region *region)
1272 : {
1273 22630 : basic_block entry_bb, exit_bb, new_bb;
1274 22630 : struct function *child_cfun;
1275 22630 : tree child_fn, block, t;
1276 22630 : gimple_stmt_iterator gsi;
1277 22630 : gimple *entry_stmt, *stmt;
1278 22630 : edge e;
1279 22630 : vec<tree, va_gc> *ws_args;
1280 :
1281 22630 : entry_stmt = last_nondebug_stmt (region->entry);
1282 22630 : if (gimple_code (entry_stmt) == GIMPLE_OMP_TASK
1283 22630 : && gimple_omp_task_taskwait_p (entry_stmt))
1284 : {
1285 84 : new_bb = region->entry;
1286 84 : gsi = gsi_last_nondebug_bb (region->entry);
1287 84 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TASK);
1288 84 : gsi_remove (&gsi, true);
1289 84 : expand_taskwait_call (new_bb, as_a <gomp_task *> (entry_stmt));
1290 84 : return;
1291 : }
1292 :
1293 22546 : child_fn = gimple_omp_taskreg_child_fn (entry_stmt);
1294 22546 : child_cfun = DECL_STRUCT_FUNCTION (child_fn);
1295 :
1296 22546 : entry_bb = region->entry;
1297 22546 : if (gimple_code (entry_stmt) == GIMPLE_OMP_TASK)
1298 3748 : exit_bb = region->cont;
1299 : else
1300 18798 : exit_bb = region->exit;
1301 :
1302 22546 : if (is_combined_parallel (region))
1303 1200 : ws_args = region->ws_args;
1304 : else
1305 : ws_args = NULL;
1306 :
1307 22546 : if (child_cfun->cfg)
1308 : {
1309 : /* Due to inlining, it may happen that we have already outlined
1310 : the region, in which case all we need to do is make the
1311 : sub-graph unreachable and emit the parallel call. */
1312 0 : edge entry_succ_e, exit_succ_e;
1313 :
1314 0 : entry_succ_e = single_succ_edge (entry_bb);
1315 :
1316 0 : gsi = gsi_last_nondebug_bb (entry_bb);
1317 0 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_PARALLEL
1318 : || gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TASK
1319 : || gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TEAMS);
1320 0 : gsi_remove (&gsi, true);
1321 :
1322 0 : new_bb = entry_bb;
1323 0 : if (exit_bb)
1324 : {
1325 0 : exit_succ_e = single_succ_edge (exit_bb);
1326 0 : make_edge (new_bb, exit_succ_e->dest, EDGE_FALLTHRU);
1327 : }
1328 0 : remove_edge_and_dominated_blocks (entry_succ_e);
1329 : }
1330 : else
1331 : {
1332 22546 : unsigned srcidx, dstidx, num;
1333 :
1334 : /* If the parallel region needs data sent from the parent
1335 : function, then the very first statement (except possible
1336 : tree profile counter updates) of the parallel body
1337 : is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
1338 : &.OMP_DATA_O is passed as an argument to the child function,
1339 : we need to replace it with the argument as seen by the child
1340 : function.
1341 :
1342 : In most cases, this will end up being the identity assignment
1343 : .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
1344 : a function call that has been inlined, the original PARM_DECL
1345 : .OMP_DATA_I may have been converted into a different local
1346 : variable. In which case, we need to keep the assignment. */
1347 22546 : if (gimple_omp_taskreg_data_arg (entry_stmt))
1348 : {
1349 18081 : basic_block entry_succ_bb
1350 33099 : = single_succ_p (entry_bb) ? single_succ (entry_bb)
1351 3063 : : FALLTHRU_EDGE (entry_bb)->dest;
1352 18081 : tree arg;
1353 18081 : gimple *parcopy_stmt = NULL;
1354 :
1355 36162 : for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
1356 : {
1357 18081 : gimple *stmt;
1358 :
1359 18081 : gcc_assert (!gsi_end_p (gsi));
1360 18081 : stmt = gsi_stmt (gsi);
1361 18081 : if (gimple_code (stmt) != GIMPLE_ASSIGN)
1362 0 : continue;
1363 :
1364 18081 : if (gimple_num_ops (stmt) == 2)
1365 : {
1366 18081 : tree arg = gimple_assign_rhs1 (stmt);
1367 :
1368 : /* We're ignore the subcode because we're
1369 : effectively doing a STRIP_NOPS. */
1370 :
1371 18081 : if (TREE_CODE (arg) == ADDR_EXPR
1372 18081 : && (TREE_OPERAND (arg, 0)
1373 18081 : == gimple_omp_taskreg_data_arg (entry_stmt)))
1374 : {
1375 18081 : parcopy_stmt = stmt;
1376 18081 : break;
1377 : }
1378 : }
1379 : }
1380 :
1381 18081 : gcc_assert (parcopy_stmt != NULL);
1382 18081 : arg = DECL_ARGUMENTS (child_fn);
1383 :
1384 18081 : if (!gimple_in_ssa_p (cfun))
1385 : {
1386 17897 : if (gimple_assign_lhs (parcopy_stmt) == arg)
1387 17897 : gsi_remove (&gsi, true);
1388 : else
1389 : {
1390 : /* ?? Is setting the subcode really necessary ?? */
1391 0 : gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (arg));
1392 0 : gimple_assign_set_rhs1 (parcopy_stmt, arg);
1393 : }
1394 : }
1395 : else
1396 : {
1397 184 : tree lhs = gimple_assign_lhs (parcopy_stmt);
1398 184 : gcc_assert (SSA_NAME_VAR (lhs) == arg);
1399 : /* We'd like to set the rhs to the default def in the child_fn,
1400 : but it's too early to create ssa names in the child_fn.
1401 : Instead, we set the rhs to the parm. In
1402 : move_sese_region_to_fn, we introduce a default def for the
1403 : parm, map the parm to it's default def, and once we encounter
1404 : this stmt, replace the parm with the default def. */
1405 184 : gimple_assign_set_rhs1 (parcopy_stmt, arg);
1406 184 : update_stmt (parcopy_stmt);
1407 : }
1408 : }
1409 :
1410 : /* Declare local variables needed in CHILD_CFUN. */
1411 22546 : block = DECL_INITIAL (child_fn);
1412 22546 : BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
1413 : /* The gimplifier could record temporaries in parallel/task block
1414 : rather than in containing function's local_decls chain,
1415 : which would mean cgraph missed finalizing them. Do it now. */
1416 251860 : for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
1417 229314 : if (VAR_P (t) && TREE_STATIC (t) && !DECL_EXTERNAL (t))
1418 1 : varpool_node::finalize_decl (t);
1419 22546 : DECL_SAVED_TREE (child_fn) = NULL;
1420 : /* We'll create a CFG for child_fn, so no gimple body is needed. */
1421 22546 : gimple_set_body (child_fn, NULL);
1422 22546 : TREE_USED (block) = 1;
1423 :
1424 : /* Reset DECL_CONTEXT on function arguments. */
1425 45092 : for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
1426 22546 : DECL_CONTEXT (t) = child_fn;
1427 :
1428 : /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
1429 : so that it can be moved to the child function. */
1430 22546 : gsi = gsi_last_nondebug_bb (entry_bb);
1431 22546 : stmt = gsi_stmt (gsi);
1432 22546 : gcc_assert (stmt && (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
1433 : || gimple_code (stmt) == GIMPLE_OMP_TASK
1434 : || gimple_code (stmt) == GIMPLE_OMP_TEAMS));
1435 22546 : e = split_block (entry_bb, stmt);
1436 22546 : gsi_remove (&gsi, true);
1437 22546 : entry_bb = e->dest;
1438 22546 : edge e2 = NULL;
1439 22546 : if (gimple_code (entry_stmt) != GIMPLE_OMP_TASK)
1440 18798 : single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
1441 : else
1442 : {
1443 3748 : e2 = make_edge (e->src, BRANCH_EDGE (entry_bb)->dest, EDGE_ABNORMAL);
1444 3748 : gcc_assert (e2->dest == region->exit);
1445 3748 : remove_edge (BRANCH_EDGE (entry_bb));
1446 3748 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e->src);
1447 3748 : gsi = gsi_last_nondebug_bb (region->exit);
1448 3748 : gcc_assert (!gsi_end_p (gsi)
1449 : && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
1450 3748 : gsi_remove (&gsi, true);
1451 : }
1452 :
1453 : /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
1454 22546 : if (exit_bb)
1455 : {
1456 22502 : gsi = gsi_last_nondebug_bb (exit_bb);
1457 41273 : gcc_assert (!gsi_end_p (gsi)
1458 : && (gimple_code (gsi_stmt (gsi))
1459 : == (e2 ? GIMPLE_OMP_CONTINUE : GIMPLE_OMP_RETURN)));
1460 22502 : stmt = gimple_build_return (NULL);
1461 22502 : gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
1462 22502 : gsi_remove (&gsi, true);
1463 : }
1464 :
1465 : /* Move the parallel region into CHILD_CFUN. */
1466 :
1467 22546 : if (gimple_in_ssa_p (cfun))
1468 : {
1469 196 : init_tree_ssa (child_cfun);
1470 196 : init_ssa_operands (child_cfun);
1471 196 : child_cfun->gimple_df->in_ssa_p = true;
1472 196 : block = NULL_TREE;
1473 : }
1474 : else
1475 22350 : block = gimple_block (entry_stmt);
1476 :
1477 22546 : new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
1478 22546 : if (exit_bb)
1479 22502 : single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
1480 22546 : if (e2)
1481 : {
1482 3748 : basic_block dest_bb = e2->dest;
1483 3748 : if (!exit_bb)
1484 17 : make_edge (new_bb, dest_bb, EDGE_FALLTHRU);
1485 3748 : remove_edge (e2);
1486 3748 : set_immediate_dominator (CDI_DOMINATORS, dest_bb, new_bb);
1487 : }
1488 : /* When the OMP expansion process cannot guarantee an up-to-date
1489 : loop tree arrange for the child function to fixup loops. */
1490 22546 : if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
1491 22350 : child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
1492 :
1493 : /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
1494 22546 : num = vec_safe_length (child_cfun->local_decls);
1495 1067516 : for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
1496 : {
1497 1044970 : t = (*child_cfun->local_decls)[srcidx];
1498 1044970 : if (DECL_CONTEXT (t) == cfun->decl)
1499 229314 : continue;
1500 815656 : if (srcidx != dstidx)
1501 811805 : (*child_cfun->local_decls)[dstidx] = t;
1502 815656 : dstidx++;
1503 : }
1504 22546 : if (dstidx != num)
1505 20386 : vec_safe_truncate (child_cfun->local_decls, dstidx);
1506 :
1507 : /* Inform the callgraph about the new function. */
1508 22546 : child_cfun->curr_properties = cfun->curr_properties;
1509 22546 : child_cfun->has_simduid_loops |= cfun->has_simduid_loops;
1510 22546 : child_cfun->has_force_vectorize_loops |= cfun->has_force_vectorize_loops;
1511 22546 : cgraph_node *node = cgraph_node::get_create (child_fn);
1512 22546 : node->parallelized_function = 1;
1513 45092 : node->has_omp_variant_constructs
1514 22546 : |= cgraph_node::get (cfun->decl)->has_omp_variant_constructs;
1515 22546 : cgraph_node::add_new_function (child_fn, true);
1516 :
1517 22546 : bool need_asm = DECL_ASSEMBLER_NAME_SET_P (current_function_decl)
1518 22546 : && !DECL_ASSEMBLER_NAME_SET_P (child_fn);
1519 :
1520 : /* Fix the callgraph edges for child_cfun. Those for cfun will be
1521 : fixed in a following pass. */
1522 22546 : push_cfun (child_cfun);
1523 22546 : if (need_asm)
1524 22350 : assign_assembler_name_if_needed (child_fn);
1525 :
1526 22546 : if (optimize)
1527 16828 : optimize_omp_library_calls (entry_stmt);
1528 22546 : update_max_bb_count ();
1529 22546 : cgraph_edge::rebuild_edges ();
1530 :
1531 : /* Some EH regions might become dead, see PR34608. If
1532 : pass_cleanup_cfg isn't the first pass to happen with the
1533 : new child, these dead EH edges might cause problems.
1534 : Clean them up now. */
1535 22546 : if (flag_exceptions)
1536 : {
1537 10153 : basic_block bb;
1538 10153 : bool changed = false;
1539 :
1540 134954 : FOR_EACH_BB_FN (bb, cfun)
1541 124801 : changed |= gimple_purge_dead_eh_edges (bb);
1542 10153 : if (changed)
1543 0 : cleanup_tree_cfg ();
1544 : }
1545 22546 : if (gimple_in_ssa_p (cfun))
1546 196 : update_ssa (TODO_update_ssa);
1547 22546 : if (flag_checking && !loops_state_satisfies_p (LOOPS_NEED_FIXUP))
1548 196 : verify_loop_structure ();
1549 22546 : pop_cfun ();
1550 :
1551 22546 : if (dump_file && !gimple_in_ssa_p (cfun))
1552 : {
1553 60 : omp_any_child_fn_dumped = true;
1554 60 : dump_function_header (dump_file, child_fn, dump_flags);
1555 60 : dump_function_to_file (child_fn, dump_file, dump_flags);
1556 : }
1557 : }
1558 :
1559 22546 : adjust_context_and_scope (region, gimple_block (entry_stmt), child_fn);
1560 :
1561 22546 : if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL)
1562 16280 : expand_parallel_call (region, new_bb,
1563 : as_a <gomp_parallel *> (entry_stmt), ws_args);
1564 6266 : else if (gimple_code (entry_stmt) == GIMPLE_OMP_TEAMS)
1565 2518 : expand_teams_call (new_bb, as_a <gomp_teams *> (entry_stmt));
1566 : else
1567 3748 : expand_task_call (region, new_bb, as_a <gomp_task *> (entry_stmt));
1568 : }
1569 :
1570 : /* Information about members of an OpenACC collapsed loop nest. */
1571 :
1572 : struct oacc_collapse
1573 : {
1574 : tree base; /* Base value. */
1575 : tree iters; /* Number of steps. */
1576 : tree step; /* Step size. */
1577 : tree tile; /* Tile increment (if tiled). */
1578 : tree outer; /* Tile iterator var. */
1579 : };
1580 :
1581 : /* Helper for expand_oacc_for. Determine collapsed loop information.
1582 : Fill in COUNTS array. Emit any initialization code before GSI.
1583 : Return the calculated outer loop bound of BOUND_TYPE. */
1584 :
1585 : static tree
1586 594 : expand_oacc_collapse_init (const struct omp_for_data *fd,
1587 : gimple_stmt_iterator *gsi,
1588 : oacc_collapse *counts, tree diff_type,
1589 : tree bound_type, location_t loc)
1590 : {
1591 594 : tree tiling = fd->tiling;
1592 594 : tree total = build_int_cst (bound_type, 1);
1593 594 : int ix;
1594 :
1595 594 : gcc_assert (integer_onep (fd->loop.step));
1596 594 : gcc_assert (integer_zerop (fd->loop.n1));
1597 :
1598 : /* When tiling, the first operand of the tile clause applies to the
1599 : innermost loop, and we work outwards from there. Seems
1600 : backwards, but whatever. */
1601 1890 : for (ix = fd->collapse; ix--;)
1602 : {
1603 1296 : const omp_for_data_loop *loop = &fd->loops[ix];
1604 :
1605 1296 : tree iter_type = TREE_TYPE (loop->v);
1606 1296 : tree plus_type = iter_type;
1607 :
1608 1296 : gcc_assert (loop->cond_code == LT_EXPR || loop->cond_code == GT_EXPR);
1609 :
1610 1296 : if (POINTER_TYPE_P (iter_type))
1611 0 : plus_type = sizetype;
1612 :
1613 1296 : if (tiling)
1614 : {
1615 284 : tree num = build_int_cst (integer_type_node, fd->collapse);
1616 284 : tree loop_no = build_int_cst (integer_type_node, ix);
1617 284 : tree tile = TREE_VALUE (tiling);
1618 284 : gcall *call
1619 284 : = gimple_build_call_internal (IFN_GOACC_TILE, 5, num, loop_no, tile,
1620 : /* gwv-outer=*/integer_zero_node,
1621 : /* gwv-inner=*/integer_zero_node);
1622 :
1623 284 : counts[ix].outer = create_tmp_var (iter_type, ".outer");
1624 284 : counts[ix].tile = create_tmp_var (diff_type, ".tile");
1625 284 : gimple_call_set_lhs (call, counts[ix].tile);
1626 284 : gimple_set_location (call, loc);
1627 284 : gsi_insert_before (gsi, call, GSI_SAME_STMT);
1628 :
1629 284 : tiling = TREE_CHAIN (tiling);
1630 : }
1631 : else
1632 : {
1633 1012 : counts[ix].tile = NULL;
1634 1012 : counts[ix].outer = loop->v;
1635 : }
1636 :
1637 1296 : tree b = loop->n1;
1638 1296 : tree e = loop->n2;
1639 1296 : tree s = loop->step;
1640 1296 : bool up = loop->cond_code == LT_EXPR;
1641 1344 : tree dir = build_int_cst (diff_type, up ? +1 : -1);
1642 1296 : bool negating;
1643 1296 : tree expr;
1644 :
1645 1296 : b = force_gimple_operand_gsi (gsi, b, true, NULL_TREE,
1646 : true, GSI_SAME_STMT);
1647 1296 : e = force_gimple_operand_gsi (gsi, e, true, NULL_TREE,
1648 : true, GSI_SAME_STMT);
1649 :
1650 : /* Convert the step, avoiding possible unsigned->signed overflow. */
1651 1296 : negating = !up && TYPE_UNSIGNED (TREE_TYPE (s));
1652 0 : if (negating)
1653 0 : s = fold_build1 (NEGATE_EXPR, TREE_TYPE (s), s);
1654 1296 : s = fold_convert (diff_type, s);
1655 1296 : if (negating)
1656 0 : s = fold_build1 (NEGATE_EXPR, diff_type, s);
1657 1296 : s = force_gimple_operand_gsi (gsi, s, true, NULL_TREE,
1658 : true, GSI_SAME_STMT);
1659 :
1660 : /* Determine the range, avoiding possible unsigned->signed overflow. */
1661 1296 : negating = !up && TYPE_UNSIGNED (iter_type);
1662 2592 : expr = fold_build2 (MINUS_EXPR, plus_type,
1663 : fold_convert (plus_type, negating ? b : e),
1664 : fold_convert (plus_type, negating ? e : b));
1665 1296 : expr = fold_convert (diff_type, expr);
1666 1296 : if (negating)
1667 0 : expr = fold_build1 (NEGATE_EXPR, diff_type, expr);
1668 1296 : tree range = force_gimple_operand_gsi
1669 1296 : (gsi, expr, true, NULL_TREE, true, GSI_SAME_STMT);
1670 :
1671 : /* Determine number of iterations. */
1672 1296 : expr = fold_build2 (MINUS_EXPR, diff_type, range, dir);
1673 1296 : expr = fold_build2 (PLUS_EXPR, diff_type, expr, s);
1674 1296 : expr = fold_build2 (TRUNC_DIV_EXPR, diff_type, expr, s);
1675 :
1676 1296 : tree iters = force_gimple_operand_gsi (gsi, expr, true, NULL_TREE,
1677 : true, GSI_SAME_STMT);
1678 :
1679 1296 : counts[ix].base = b;
1680 1296 : counts[ix].iters = iters;
1681 1296 : counts[ix].step = s;
1682 :
1683 1296 : total = fold_build2 (MULT_EXPR, bound_type, total,
1684 : fold_convert (bound_type, iters));
1685 : }
1686 :
1687 594 : return total;
1688 : }
1689 :
1690 : /* Emit initializers for collapsed loop members. INNER is true if
1691 : this is for the element loop of a TILE. IVAR is the outer
1692 : loop iteration variable, from which collapsed loop iteration values
1693 : are calculated. COUNTS array has been initialized by
1694 : expand_oacc_collapse_inits. */
1695 :
1696 : static void
1697 771 : expand_oacc_collapse_vars (const struct omp_for_data *fd, bool inner,
1698 : gimple_stmt_iterator *gsi,
1699 : const oacc_collapse *counts, tree ivar,
1700 : tree diff_type)
1701 : {
1702 771 : tree ivar_type = TREE_TYPE (ivar);
1703 :
1704 : /* The most rapidly changing iteration variable is the innermost
1705 : one. */
1706 2351 : for (int ix = fd->collapse; ix--;)
1707 : {
1708 1580 : const omp_for_data_loop *loop = &fd->loops[ix];
1709 1580 : const oacc_collapse *collapse = &counts[ix];
1710 1580 : tree v = inner ? loop->v : collapse->outer;
1711 1580 : tree iter_type = TREE_TYPE (v);
1712 1580 : tree plus_type = iter_type;
1713 1580 : enum tree_code plus_code = PLUS_EXPR;
1714 1580 : tree expr;
1715 :
1716 1580 : if (POINTER_TYPE_P (iter_type))
1717 : {
1718 0 : plus_code = POINTER_PLUS_EXPR;
1719 0 : plus_type = sizetype;
1720 : }
1721 :
1722 1580 : expr = ivar;
1723 1580 : if (ix)
1724 : {
1725 809 : tree mod = fold_convert (ivar_type, collapse->iters);
1726 809 : ivar = fold_build2 (TRUNC_DIV_EXPR, ivar_type, expr, mod);
1727 809 : expr = fold_build2 (TRUNC_MOD_EXPR, ivar_type, expr, mod);
1728 809 : ivar = force_gimple_operand_gsi (gsi, ivar, true, NULL_TREE,
1729 : true, GSI_SAME_STMT);
1730 : }
1731 :
1732 1580 : expr = fold_build2 (MULT_EXPR, diff_type, fold_convert (diff_type, expr),
1733 : fold_convert (diff_type, collapse->step));
1734 1580 : expr = fold_build2 (plus_code, iter_type,
1735 : inner ? collapse->outer : collapse->base,
1736 : fold_convert (plus_type, expr));
1737 1580 : expr = force_gimple_operand_gsi (gsi, expr, false, NULL_TREE,
1738 : true, GSI_SAME_STMT);
1739 1580 : gassign *ass = gimple_build_assign (v, expr);
1740 1580 : gsi_insert_before (gsi, ass, GSI_SAME_STMT);
1741 : }
1742 771 : }
1743 :
1744 : /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
1745 : of the combined collapse > 1 loop constructs, generate code like:
1746 : if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
1747 : if (cond3 is <)
1748 : adj = STEP3 - 1;
1749 : else
1750 : adj = STEP3 + 1;
1751 : count3 = (adj + N32 - N31) / STEP3;
1752 : if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
1753 : if (cond2 is <)
1754 : adj = STEP2 - 1;
1755 : else
1756 : adj = STEP2 + 1;
1757 : count2 = (adj + N22 - N21) / STEP2;
1758 : if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
1759 : if (cond1 is <)
1760 : adj = STEP1 - 1;
1761 : else
1762 : adj = STEP1 + 1;
1763 : count1 = (adj + N12 - N11) / STEP1;
1764 : count = count1 * count2 * count3;
1765 : Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
1766 : count = 0;
1767 : and set ZERO_ITER_BB to that bb. If this isn't the outermost
1768 : of the combined loop constructs, just initialize COUNTS array
1769 : from the _looptemp_ clauses. For loop nests with non-rectangular
1770 : loops, do this only for the rectangular loops. Then pick
1771 : the loops which reference outer vars in their bound expressions
1772 : and the loops which they refer to and for this sub-nest compute
1773 : number of iterations. For triangular loops use Faulhaber's formula,
1774 : otherwise as a fallback, compute by iterating the loops.
1775 : If e.g. the sub-nest is
1776 : for (I = N11; I COND1 N12; I += STEP1)
1777 : for (J = M21 * I + N21; J COND2 M22 * I + N22; J += STEP2)
1778 : for (K = M31 * J + N31; K COND3 M32 * J + N32; K += STEP3)
1779 : do:
1780 : COUNT = 0;
1781 : for (tmpi = N11; tmpi COND1 N12; tmpi += STEP1)
1782 : for (tmpj = M21 * tmpi + N21;
1783 : tmpj COND2 M22 * tmpi + N22; tmpj += STEP2)
1784 : {
1785 : int tmpk1 = M31 * tmpj + N31;
1786 : int tmpk2 = M32 * tmpj + N32;
1787 : if (tmpk1 COND3 tmpk2)
1788 : {
1789 : if (COND3 is <)
1790 : adj = STEP3 - 1;
1791 : else
1792 : adj = STEP3 + 1;
1793 : COUNT += (adj + tmpk2 - tmpk1) / STEP3;
1794 : }
1795 : }
1796 : and finally multiply the counts of the rectangular loops not
1797 : in the sub-nest with COUNT. Also, as counts[fd->last_nonrect]
1798 : store number of iterations of the loops from fd->first_nonrect
1799 : to fd->last_nonrect inclusive, i.e. the above COUNT multiplied
1800 : by the counts of rectangular loops not referenced in any non-rectangular
1801 : loops sandwiched in between those. */
1802 :
1803 : /* NOTE: It *could* be better to moosh all of the BBs together,
1804 : creating one larger BB with all the computation and the unexpected
1805 : jump at the end. I.e.
1806 :
1807 : bool zero3, zero2, zero1, zero;
1808 :
1809 : zero3 = N32 c3 N31;
1810 : count3 = (N32 - N31) /[cl] STEP3;
1811 : zero2 = N22 c2 N21;
1812 : count2 = (N22 - N21) /[cl] STEP2;
1813 : zero1 = N12 c1 N11;
1814 : count1 = (N12 - N11) /[cl] STEP1;
1815 : zero = zero3 || zero2 || zero1;
1816 : count = count1 * count2 * count3;
1817 : if (__builtin_expect(zero, false)) goto zero_iter_bb;
1818 :
1819 : After all, we expect the zero=false, and thus we expect to have to
1820 : evaluate all of the comparison expressions, so short-circuiting
1821 : oughtn't be a win. Since the condition isn't protecting a
1822 : denominator, we're not concerned about divide-by-zero, so we can
1823 : fully evaluate count even if a numerator turned out to be wrong.
1824 :
1825 : It seems like putting this all together would create much better
1826 : scheduling opportunities, and less pressure on the chip's branch
1827 : predictor. */
1828 :
1829 : static void
1830 10443 : expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
1831 : basic_block &entry_bb, tree *counts,
1832 : basic_block &zero_iter1_bb, int &first_zero_iter1,
1833 : basic_block &zero_iter2_bb, int &first_zero_iter2,
1834 : basic_block &l2_dom_bb)
1835 : {
1836 10443 : tree t, type = TREE_TYPE (fd->loop.v);
1837 10443 : edge e, ne;
1838 10443 : int i;
1839 :
1840 : /* Collapsed loops need work for expansion into SSA form. */
1841 10443 : gcc_assert (!gimple_in_ssa_p (cfun));
1842 :
1843 10443 : if (gimple_omp_for_combined_into_p (fd->for_stmt)
1844 10443 : && TREE_CODE (fd->loop.n2) != INTEGER_CST)
1845 : {
1846 2692 : gcc_assert (fd->ordered == 0);
1847 : /* First two _looptemp_ clauses are for istart/iend, counts[0]
1848 : isn't supposed to be handled, as the inner loop doesn't
1849 : use it. */
1850 2692 : tree innerc = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
1851 : OMP_CLAUSE__LOOPTEMP_);
1852 2692 : gcc_assert (innerc);
1853 10553 : for (i = 0; i < fd->collapse; i++)
1854 : {
1855 7861 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
1856 : OMP_CLAUSE__LOOPTEMP_);
1857 7861 : gcc_assert (innerc);
1858 7861 : if (i)
1859 5169 : counts[i] = OMP_CLAUSE_DECL (innerc);
1860 : else
1861 2692 : counts[0] = NULL_TREE;
1862 : }
1863 2692 : if (fd->non_rect
1864 112 : && fd->last_nonrect == fd->first_nonrect + 1
1865 2756 : && !TYPE_UNSIGNED (TREE_TYPE (fd->loops[fd->last_nonrect].v)))
1866 : {
1867 : tree c[4];
1868 300 : for (i = 0; i < 4; i++)
1869 : {
1870 240 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
1871 : OMP_CLAUSE__LOOPTEMP_);
1872 240 : gcc_assert (innerc);
1873 240 : c[i] = OMP_CLAUSE_DECL (innerc);
1874 : }
1875 60 : counts[0] = c[0];
1876 60 : fd->first_inner_iterations = c[1];
1877 60 : fd->factor = c[2];
1878 60 : fd->adjn1 = c[3];
1879 : }
1880 2692 : return;
1881 : }
1882 :
1883 8644 : for (i = fd->collapse; i < fd->ordered; i++)
1884 : {
1885 893 : tree itype = TREE_TYPE (fd->loops[i].v);
1886 893 : counts[i] = NULL_TREE;
1887 893 : t = fold_binary (fd->loops[i].cond_code, boolean_type_node,
1888 : fold_convert (itype, fd->loops[i].n1),
1889 : fold_convert (itype, fd->loops[i].n2));
1890 893 : if (t && integer_zerop (t))
1891 : {
1892 0 : for (i = fd->collapse; i < fd->ordered; i++)
1893 0 : counts[i] = build_int_cst (type, 0);
1894 : break;
1895 : }
1896 : }
1897 7751 : bool rect_count_seen = false;
1898 7751 : bool init_n2 = SSA_VAR_P (fd->loop.n2) && zero_iter1_bb;
1899 30099 : for (i = 0; i < (fd->ordered ? fd->ordered : fd->collapse); i++)
1900 : {
1901 22348 : tree itype = TREE_TYPE (fd->loops[i].v);
1902 :
1903 22348 : if (i >= fd->collapse && counts[i])
1904 0 : continue;
1905 22348 : if (fd->non_rect)
1906 : {
1907 : /* Skip loops that use outer iterators in their expressions
1908 : during this phase. */
1909 1050 : if (fd->loops[i].m1 || fd->loops[i].m2)
1910 : {
1911 432 : counts[i] = build_zero_cst (type);
1912 432 : continue;
1913 : }
1914 : }
1915 21916 : if ((SSA_VAR_P (fd->loop.n2) || i >= fd->collapse)
1916 9514 : && ((t = fold_binary (fd->loops[i].cond_code, boolean_type_node,
1917 : fold_convert (itype, fd->loops[i].n1),
1918 : fold_convert (itype, fd->loops[i].n2)))
1919 5909 : == NULL_TREE || !integer_onep (t)))
1920 : {
1921 5789 : gcond *cond_stmt;
1922 5789 : tree n1, n2;
1923 5789 : if (init_n2 && i < fd->collapse && !rect_count_seen)
1924 : {
1925 : /* When called with non-NULL zero_iter1_bb, we won't clear
1926 : fd->loop.n2 in the if (zero_iter_bb == NULL) code below
1927 : and if it is prior to storing fd->loop.n2 where
1928 : rect_count_seen is set, it could be used uninitialized.
1929 : As zero_iter1_bb in that case can be reached also if there
1930 : are non-zero iterations, the clearing can't be emitted
1931 : to the zero_iter1_bb, but needs to be done before the
1932 : condition. */
1933 1505 : gassign *assign_stmt
1934 1505 : = gimple_build_assign (fd->loop.n2, build_zero_cst (type));
1935 1505 : gsi_insert_before (gsi, assign_stmt, GSI_SAME_STMT);
1936 1505 : init_n2 = false;
1937 : }
1938 5789 : n1 = fold_convert (itype, unshare_expr (fd->loops[i].n1));
1939 5789 : n1 = force_gimple_operand_gsi (gsi, n1, true, NULL_TREE,
1940 : true, GSI_SAME_STMT);
1941 5789 : n2 = fold_convert (itype, unshare_expr (fd->loops[i].n2));
1942 5789 : n2 = force_gimple_operand_gsi (gsi, n2, true, NULL_TREE,
1943 : true, GSI_SAME_STMT);
1944 5789 : cond_stmt = expand_omp_build_cond (gsi, fd->loops[i].cond_code,
1945 : n1, n2);
1946 5789 : e = split_block (entry_bb, cond_stmt);
1947 216 : basic_block &zero_iter_bb
1948 5789 : = i < fd->collapse ? zero_iter1_bb : zero_iter2_bb;
1949 216 : int &first_zero_iter
1950 5789 : = i < fd->collapse ? first_zero_iter1 : first_zero_iter2;
1951 5789 : if (zero_iter_bb == NULL)
1952 : {
1953 626 : gassign *assign_stmt;
1954 626 : first_zero_iter = i;
1955 626 : zero_iter_bb = create_empty_bb (entry_bb);
1956 626 : add_bb_to_loop (zero_iter_bb, entry_bb->loop_father);
1957 626 : *gsi = gsi_after_labels (zero_iter_bb);
1958 626 : if (i < fd->collapse)
1959 508 : assign_stmt = gimple_build_assign (fd->loop.n2,
1960 : build_zero_cst (type));
1961 : else
1962 : {
1963 118 : counts[i] = create_tmp_reg (type, ".count");
1964 118 : assign_stmt
1965 118 : = gimple_build_assign (counts[i], build_zero_cst (type));
1966 : }
1967 626 : gsi_insert_before (gsi, assign_stmt, GSI_SAME_STMT);
1968 626 : set_immediate_dominator (CDI_DOMINATORS, zero_iter_bb,
1969 : entry_bb);
1970 : }
1971 5789 : ne = make_edge (entry_bb, zero_iter_bb, EDGE_FALSE_VALUE);
1972 5789 : ne->probability = profile_probability::very_unlikely ();
1973 5789 : e->flags = EDGE_TRUE_VALUE;
1974 5789 : e->probability = ne->probability.invert ();
1975 5789 : if (l2_dom_bb == NULL)
1976 2591 : l2_dom_bb = entry_bb;
1977 5789 : entry_bb = e->dest;
1978 5789 : *gsi = gsi_last_nondebug_bb (entry_bb);
1979 : }
1980 :
1981 21916 : if (POINTER_TYPE_P (itype))
1982 1661 : itype = signed_type_for (itype);
1983 21916 : t = build_int_cst (itype, (fd->loops[i].cond_code == LT_EXPR
1984 27020 : ? -1 : 1));
1985 21916 : t = fold_build2 (PLUS_EXPR, itype,
1986 : fold_convert (itype, fd->loops[i].step), t);
1987 21916 : t = fold_build2 (PLUS_EXPR, itype, t,
1988 : fold_convert (itype, fd->loops[i].n2));
1989 21916 : t = fold_build2 (MINUS_EXPR, itype, t,
1990 : fold_convert (itype, fd->loops[i].n1));
1991 : /* ?? We could probably use CEIL_DIV_EXPR instead of
1992 : TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
1993 : generate the same code in the end because generically we
1994 : don't know that the values involved must be negative for
1995 : GT?? */
1996 21916 : if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR)
1997 2248 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
1998 : fold_build1 (NEGATE_EXPR, itype, t),
1999 : fold_build1 (NEGATE_EXPR, itype,
2000 : fold_convert (itype,
2001 : fd->loops[i].step)));
2002 : else
2003 19668 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t,
2004 : fold_convert (itype, fd->loops[i].step));
2005 21916 : t = fold_convert (type, t);
2006 21916 : if (TREE_CODE (t) == INTEGER_CST)
2007 16057 : counts[i] = t;
2008 : else
2009 : {
2010 5859 : if (i < fd->collapse || i != first_zero_iter2)
2011 5741 : counts[i] = create_tmp_reg (type, ".count");
2012 5859 : expand_omp_build_assign (gsi, counts[i], t);
2013 : }
2014 21916 : if (SSA_VAR_P (fd->loop.n2) && i < fd->collapse)
2015 : {
2016 8621 : if (fd->non_rect && i >= fd->first_nonrect && i <= fd->last_nonrect)
2017 376 : continue;
2018 8245 : if (!rect_count_seen)
2019 : {
2020 2995 : t = counts[i];
2021 2995 : rect_count_seen = true;
2022 : }
2023 : else
2024 5250 : t = fold_build2 (MULT_EXPR, type, fd->loop.n2, counts[i]);
2025 8245 : expand_omp_build_assign (gsi, fd->loop.n2, t);
2026 : }
2027 : }
2028 7751 : if (fd->non_rect && SSA_VAR_P (fd->loop.n2))
2029 : {
2030 284 : gcc_assert (fd->last_nonrect != -1);
2031 :
2032 284 : counts[fd->last_nonrect] = create_tmp_reg (type, ".count");
2033 284 : expand_omp_build_assign (gsi, counts[fd->last_nonrect],
2034 : build_zero_cst (type));
2035 366 : for (i = fd->first_nonrect + 1; i < fd->last_nonrect; i++)
2036 113 : if (fd->loops[i].m1
2037 83 : || fd->loops[i].m2
2038 82 : || fd->loops[i].non_rect_referenced)
2039 : break;
2040 284 : if (i == fd->last_nonrect
2041 253 : && fd->loops[i].outer == fd->last_nonrect - fd->first_nonrect
2042 253 : && !POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v))
2043 511 : && !TYPE_UNSIGNED (TREE_TYPE (fd->loops[i].v)))
2044 : {
2045 225 : int o = fd->first_nonrect;
2046 225 : tree itype = TREE_TYPE (fd->loops[o].v);
2047 225 : tree n1o = create_tmp_reg (itype, ".n1o");
2048 225 : t = fold_convert (itype, unshare_expr (fd->loops[o].n1));
2049 225 : expand_omp_build_assign (gsi, n1o, t);
2050 225 : tree n2o = create_tmp_reg (itype, ".n2o");
2051 225 : t = fold_convert (itype, unshare_expr (fd->loops[o].n2));
2052 225 : expand_omp_build_assign (gsi, n2o, t);
2053 225 : if (fd->loops[i].m1 && fd->loops[i].m2)
2054 43 : t = fold_build2 (MINUS_EXPR, itype, unshare_expr (fd->loops[i].m2),
2055 : unshare_expr (fd->loops[i].m1));
2056 182 : else if (fd->loops[i].m1)
2057 162 : t = fold_build1 (NEGATE_EXPR, itype,
2058 : unshare_expr (fd->loops[i].m1));
2059 : else
2060 20 : t = unshare_expr (fd->loops[i].m2);
2061 225 : tree m2minusm1
2062 225 : = force_gimple_operand_gsi (gsi, t, true, NULL_TREE,
2063 : true, GSI_SAME_STMT);
2064 :
2065 225 : gimple_stmt_iterator gsi2 = *gsi;
2066 225 : gsi_prev (&gsi2);
2067 225 : e = split_block (entry_bb, gsi_stmt (gsi2));
2068 225 : e = split_block (e->dest, (gimple *) NULL);
2069 225 : basic_block bb1 = e->src;
2070 225 : entry_bb = e->dest;
2071 225 : *gsi = gsi_after_labels (entry_bb);
2072 :
2073 225 : gsi2 = gsi_after_labels (bb1);
2074 225 : tree ostep = fold_convert (itype, fd->loops[o].step);
2075 225 : t = build_int_cst (itype, (fd->loops[o].cond_code
2076 241 : == LT_EXPR ? -1 : 1));
2077 225 : t = fold_build2 (PLUS_EXPR, itype, ostep, t);
2078 225 : t = fold_build2 (PLUS_EXPR, itype, t, n2o);
2079 225 : t = fold_build2 (MINUS_EXPR, itype, t, n1o);
2080 225 : if (TYPE_UNSIGNED (itype)
2081 225 : && fd->loops[o].cond_code == GT_EXPR)
2082 0 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
2083 : fold_build1 (NEGATE_EXPR, itype, t),
2084 : fold_build1 (NEGATE_EXPR, itype, ostep));
2085 : else
2086 225 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, ostep);
2087 225 : tree outer_niters
2088 225 : = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2089 : true, GSI_SAME_STMT);
2090 225 : t = fold_build2 (MINUS_EXPR, itype, outer_niters,
2091 : build_one_cst (itype));
2092 225 : t = fold_build2 (MULT_EXPR, itype, t, ostep);
2093 225 : t = fold_build2 (PLUS_EXPR, itype, n1o, t);
2094 225 : tree last = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2095 : true, GSI_SAME_STMT);
2096 225 : tree n1, n2, n1e, n2e;
2097 225 : t = fold_convert (itype, unshare_expr (fd->loops[i].n1));
2098 225 : if (fd->loops[i].m1)
2099 : {
2100 205 : n1 = fold_convert (itype, unshare_expr (fd->loops[i].m1));
2101 205 : n1 = fold_build2 (MULT_EXPR, itype, n1o, n1);
2102 205 : n1 = fold_build2 (PLUS_EXPR, itype, n1, t);
2103 : }
2104 : else
2105 : n1 = t;
2106 225 : n1 = force_gimple_operand_gsi (&gsi2, n1, true, NULL_TREE,
2107 : true, GSI_SAME_STMT);
2108 225 : t = fold_convert (itype, unshare_expr (fd->loops[i].n2));
2109 225 : if (fd->loops[i].m2)
2110 : {
2111 63 : n2 = fold_convert (itype, unshare_expr (fd->loops[i].m2));
2112 63 : n2 = fold_build2 (MULT_EXPR, itype, n1o, n2);
2113 63 : n2 = fold_build2 (PLUS_EXPR, itype, n2, t);
2114 : }
2115 : else
2116 : n2 = t;
2117 225 : n2 = force_gimple_operand_gsi (&gsi2, n2, true, NULL_TREE,
2118 : true, GSI_SAME_STMT);
2119 225 : t = fold_convert (itype, unshare_expr (fd->loops[i].n1));
2120 225 : if (fd->loops[i].m1)
2121 : {
2122 205 : n1e = fold_convert (itype, unshare_expr (fd->loops[i].m1));
2123 205 : n1e = fold_build2 (MULT_EXPR, itype, last, n1e);
2124 205 : n1e = fold_build2 (PLUS_EXPR, itype, n1e, t);
2125 : }
2126 : else
2127 : n1e = t;
2128 225 : n1e = force_gimple_operand_gsi (&gsi2, n1e, true, NULL_TREE,
2129 : true, GSI_SAME_STMT);
2130 225 : t = fold_convert (itype, unshare_expr (fd->loops[i].n2));
2131 225 : if (fd->loops[i].m2)
2132 : {
2133 63 : n2e = fold_convert (itype, unshare_expr (fd->loops[i].m2));
2134 63 : n2e = fold_build2 (MULT_EXPR, itype, last, n2e);
2135 63 : n2e = fold_build2 (PLUS_EXPR, itype, n2e, t);
2136 : }
2137 : else
2138 : n2e = t;
2139 225 : n2e = force_gimple_operand_gsi (&gsi2, n2e, true, NULL_TREE,
2140 : true, GSI_SAME_STMT);
2141 225 : gcond *cond_stmt
2142 225 : = expand_omp_build_cond (&gsi2, fd->loops[i].cond_code,
2143 : n1, n2);
2144 225 : e = split_block (bb1, cond_stmt);
2145 225 : e->flags = EDGE_TRUE_VALUE;
2146 225 : e->probability = profile_probability::likely ().guessed ();
2147 225 : basic_block bb2 = e->dest;
2148 225 : gsi2 = gsi_after_labels (bb2);
2149 :
2150 225 : cond_stmt = expand_omp_build_cond (&gsi2, fd->loops[i].cond_code,
2151 : n1e, n2e);
2152 225 : e = split_block (bb2, cond_stmt);
2153 225 : e->flags = EDGE_TRUE_VALUE;
2154 225 : e->probability = profile_probability::likely ().guessed ();
2155 225 : gsi2 = gsi_after_labels (e->dest);
2156 :
2157 225 : tree step = fold_convert (itype, fd->loops[i].step);
2158 225 : t = build_int_cst (itype, (fd->loops[i].cond_code
2159 241 : == LT_EXPR ? -1 : 1));
2160 225 : t = fold_build2 (PLUS_EXPR, itype, step, t);
2161 225 : t = fold_build2 (PLUS_EXPR, itype, t, n2);
2162 225 : t = fold_build2 (MINUS_EXPR, itype, t, n1);
2163 225 : if (TYPE_UNSIGNED (itype)
2164 225 : && fd->loops[i].cond_code == GT_EXPR)
2165 0 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
2166 : fold_build1 (NEGATE_EXPR, itype, t),
2167 : fold_build1 (NEGATE_EXPR, itype, step));
2168 : else
2169 225 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
2170 225 : tree first_inner_iterations
2171 225 : = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2172 : true, GSI_SAME_STMT);
2173 225 : t = fold_build2 (MULT_EXPR, itype, m2minusm1, ostep);
2174 225 : if (TYPE_UNSIGNED (itype)
2175 225 : && fd->loops[i].cond_code == GT_EXPR)
2176 0 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
2177 : fold_build1 (NEGATE_EXPR, itype, t),
2178 : fold_build1 (NEGATE_EXPR, itype, step));
2179 : else
2180 225 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
2181 225 : tree factor
2182 225 : = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2183 : true, GSI_SAME_STMT);
2184 225 : t = fold_build2 (MINUS_EXPR, itype, outer_niters,
2185 : build_one_cst (itype));
2186 225 : t = fold_build2 (MULT_EXPR, itype, t, outer_niters);
2187 225 : t = fold_build2 (RSHIFT_EXPR, itype, t, integer_one_node);
2188 225 : t = fold_build2 (MULT_EXPR, itype, factor, t);
2189 225 : t = fold_build2 (PLUS_EXPR, itype,
2190 : fold_build2 (MULT_EXPR, itype, outer_niters,
2191 : first_inner_iterations), t);
2192 225 : expand_omp_build_assign (&gsi2, counts[fd->last_nonrect],
2193 : fold_convert (type, t));
2194 :
2195 225 : basic_block bb3 = create_empty_bb (bb1);
2196 225 : add_bb_to_loop (bb3, bb1->loop_father);
2197 :
2198 225 : e = make_edge (bb1, bb3, EDGE_FALSE_VALUE);
2199 225 : e->probability = profile_probability::unlikely ().guessed ();
2200 :
2201 225 : gsi2 = gsi_after_labels (bb3);
2202 225 : cond_stmt = expand_omp_build_cond (&gsi2, fd->loops[i].cond_code,
2203 : n1e, n2e);
2204 225 : e = split_block (bb3, cond_stmt);
2205 225 : e->flags = EDGE_TRUE_VALUE;
2206 225 : e->probability = profile_probability::likely ().guessed ();
2207 225 : basic_block bb4 = e->dest;
2208 :
2209 225 : ne = make_edge (bb3, entry_bb, EDGE_FALSE_VALUE);
2210 225 : ne->probability = e->probability.invert ();
2211 :
2212 225 : basic_block bb5 = create_empty_bb (bb2);
2213 225 : add_bb_to_loop (bb5, bb2->loop_father);
2214 :
2215 225 : ne = make_edge (bb2, bb5, EDGE_FALSE_VALUE);
2216 225 : ne->probability = profile_probability::unlikely ().guessed ();
2217 :
2218 675 : for (int j = 0; j < 2; j++)
2219 : {
2220 450 : gsi2 = gsi_after_labels (j ? bb5 : bb4);
2221 450 : t = fold_build2 (MINUS_EXPR, itype,
2222 : unshare_expr (fd->loops[i].n1),
2223 : unshare_expr (fd->loops[i].n2));
2224 450 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, m2minusm1);
2225 450 : tree tem
2226 450 : = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2227 : true, GSI_SAME_STMT);
2228 450 : t = fold_build2 (MINUS_EXPR, itype, tem, n1o);
2229 450 : t = fold_build2 (TRUNC_MOD_EXPR, itype, t, ostep);
2230 450 : t = fold_build2 (MINUS_EXPR, itype, tem, t);
2231 450 : tem = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2232 : true, GSI_SAME_STMT);
2233 450 : t = fold_convert (itype, unshare_expr (fd->loops[i].n1));
2234 450 : if (fd->loops[i].m1)
2235 : {
2236 410 : n1 = fold_convert (itype, unshare_expr (fd->loops[i].m1));
2237 410 : n1 = fold_build2 (MULT_EXPR, itype, tem, n1);
2238 410 : n1 = fold_build2 (PLUS_EXPR, itype, n1, t);
2239 : }
2240 : else
2241 : n1 = t;
2242 450 : n1 = force_gimple_operand_gsi (&gsi2, n1, true, NULL_TREE,
2243 : true, GSI_SAME_STMT);
2244 450 : t = fold_convert (itype, unshare_expr (fd->loops[i].n2));
2245 450 : if (fd->loops[i].m2)
2246 : {
2247 126 : n2 = fold_convert (itype, unshare_expr (fd->loops[i].m2));
2248 126 : n2 = fold_build2 (MULT_EXPR, itype, tem, n2);
2249 126 : n2 = fold_build2 (PLUS_EXPR, itype, n2, t);
2250 : }
2251 : else
2252 : n2 = t;
2253 450 : n2 = force_gimple_operand_gsi (&gsi2, n2, true, NULL_TREE,
2254 : true, GSI_SAME_STMT);
2255 675 : expand_omp_build_assign (&gsi2, j ? n2o : n1o, tem);
2256 :
2257 450 : cond_stmt = expand_omp_build_cond (&gsi2, fd->loops[i].cond_code,
2258 : n1, n2);
2259 450 : e = split_block (gsi_bb (gsi2), cond_stmt);
2260 450 : e->flags = j ? EDGE_TRUE_VALUE : EDGE_FALSE_VALUE;
2261 450 : e->probability = profile_probability::unlikely ().guessed ();
2262 675 : ne = make_edge (e->src, bb1,
2263 : j ? EDGE_FALSE_VALUE : EDGE_TRUE_VALUE);
2264 450 : ne->probability = e->probability.invert ();
2265 450 : gsi2 = gsi_after_labels (e->dest);
2266 :
2267 450 : t = fold_build2 (PLUS_EXPR, itype, tem, ostep);
2268 450 : expand_omp_build_assign (&gsi2, j ? n2o : n1o, t);
2269 :
2270 450 : make_edge (e->dest, bb1, EDGE_FALLTHRU);
2271 : }
2272 :
2273 225 : set_immediate_dominator (CDI_DOMINATORS, bb3, bb1);
2274 225 : set_immediate_dominator (CDI_DOMINATORS, bb5, bb2);
2275 225 : set_immediate_dominator (CDI_DOMINATORS, entry_bb, bb1);
2276 :
2277 225 : if (fd->first_nonrect + 1 == fd->last_nonrect)
2278 : {
2279 153 : fd->first_inner_iterations = first_inner_iterations;
2280 153 : fd->factor = factor;
2281 153 : fd->adjn1 = n1o;
2282 : }
2283 : }
2284 : else
2285 : {
2286 : /* Fallback implementation. Evaluate the loops with m1/m2
2287 : non-NULL as well as their outer loops at runtime using temporaries
2288 : instead of the original iteration variables, and in the
2289 : body just bump the counter. */
2290 59 : gimple_stmt_iterator gsi2 = *gsi;
2291 59 : gsi_prev (&gsi2);
2292 59 : e = split_block (entry_bb, gsi_stmt (gsi2));
2293 59 : e = split_block (e->dest, (gimple *) NULL);
2294 59 : basic_block cur_bb = e->src;
2295 59 : basic_block next_bb = e->dest;
2296 59 : entry_bb = e->dest;
2297 59 : *gsi = gsi_after_labels (entry_bb);
2298 :
2299 59 : tree *vs = XALLOCAVEC (tree, fd->last_nonrect);
2300 59 : memset (vs, 0, fd->last_nonrect * sizeof (tree));
2301 :
2302 189 : for (i = 0; i <= fd->last_nonrect; i++)
2303 : {
2304 189 : if (fd->loops[i].m1 == NULL_TREE
2305 108 : && fd->loops[i].m2 == NULL_TREE
2306 99 : && !fd->loops[i].non_rect_referenced)
2307 40 : continue;
2308 :
2309 149 : tree itype = TREE_TYPE (fd->loops[i].v);
2310 :
2311 149 : gsi2 = gsi_after_labels (cur_bb);
2312 149 : tree n1, n2;
2313 149 : t = fold_convert (itype, unshare_expr (fd->loops[i].n1));
2314 149 : if (fd->loops[i].m1 == NULL_TREE)
2315 : n1 = t;
2316 81 : else if (POINTER_TYPE_P (itype))
2317 : {
2318 30 : gcc_assert (integer_onep (fd->loops[i].m1));
2319 30 : t = unshare_expr (fd->loops[i].n1);
2320 30 : n1 = fold_build_pointer_plus (vs[i - fd->loops[i].outer], t);
2321 : }
2322 : else
2323 : {
2324 51 : n1 = fold_convert (itype, unshare_expr (fd->loops[i].m1));
2325 51 : n1 = fold_build2 (MULT_EXPR, itype,
2326 : vs[i - fd->loops[i].outer], n1);
2327 51 : n1 = fold_build2 (PLUS_EXPR, itype, n1, t);
2328 : }
2329 149 : n1 = force_gimple_operand_gsi (&gsi2, n1, true, NULL_TREE,
2330 : true, GSI_SAME_STMT);
2331 149 : if (i < fd->last_nonrect)
2332 : {
2333 90 : vs[i] = create_tmp_reg (itype, ".it");
2334 90 : expand_omp_build_assign (&gsi2, vs[i], n1);
2335 : }
2336 149 : t = fold_convert (itype, unshare_expr (fd->loops[i].n2));
2337 149 : if (fd->loops[i].m2 == NULL_TREE)
2338 : n2 = t;
2339 85 : else if (POINTER_TYPE_P (itype))
2340 : {
2341 34 : gcc_assert (integer_onep (fd->loops[i].m2));
2342 34 : t = unshare_expr (fd->loops[i].n2);
2343 34 : n2 = fold_build_pointer_plus (vs[i - fd->loops[i].outer], t);
2344 : }
2345 : else
2346 : {
2347 51 : n2 = fold_convert (itype, unshare_expr (fd->loops[i].m2));
2348 51 : n2 = fold_build2 (MULT_EXPR, itype,
2349 : vs[i - fd->loops[i].outer], n2);
2350 51 : n2 = fold_build2 (PLUS_EXPR, itype, n2, t);
2351 : }
2352 149 : n2 = force_gimple_operand_gsi (&gsi2, n2, true, NULL_TREE,
2353 : true, GSI_SAME_STMT);
2354 149 : if (POINTER_TYPE_P (itype))
2355 70 : itype = signed_type_for (itype);
2356 149 : if (i == fd->last_nonrect)
2357 : {
2358 59 : gcond *cond_stmt
2359 59 : = expand_omp_build_cond (&gsi2, fd->loops[i].cond_code,
2360 : n1, n2);
2361 59 : e = split_block (cur_bb, cond_stmt);
2362 59 : e->flags = EDGE_TRUE_VALUE;
2363 59 : ne = make_edge (cur_bb, next_bb, EDGE_FALSE_VALUE);
2364 59 : e->probability = profile_probability::likely ().guessed ();
2365 59 : ne->probability = e->probability.invert ();
2366 59 : gsi2 = gsi_after_labels (e->dest);
2367 :
2368 59 : t = build_int_cst (itype, (fd->loops[i].cond_code == LT_EXPR
2369 79 : ? -1 : 1));
2370 59 : t = fold_build2 (PLUS_EXPR, itype,
2371 : fold_convert (itype, fd->loops[i].step), t);
2372 59 : t = fold_build2 (PLUS_EXPR, itype, t,
2373 : fold_convert (itype, n2));
2374 59 : t = fold_build2 (MINUS_EXPR, itype, t,
2375 : fold_convert (itype, n1));
2376 59 : tree step = fold_convert (itype, fd->loops[i].step);
2377 59 : if (TYPE_UNSIGNED (itype)
2378 59 : && fd->loops[i].cond_code == GT_EXPR)
2379 0 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
2380 : fold_build1 (NEGATE_EXPR, itype, t),
2381 : fold_build1 (NEGATE_EXPR, itype, step));
2382 : else
2383 59 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
2384 59 : t = fold_convert (type, t);
2385 59 : t = fold_build2 (PLUS_EXPR, type,
2386 : counts[fd->last_nonrect], t);
2387 59 : t = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2388 : true, GSI_SAME_STMT);
2389 59 : expand_omp_build_assign (&gsi2, counts[fd->last_nonrect], t);
2390 59 : e = make_edge (e->dest, next_bb, EDGE_FALLTHRU);
2391 59 : set_immediate_dominator (CDI_DOMINATORS, next_bb, cur_bb);
2392 59 : break;
2393 : }
2394 90 : e = split_block (cur_bb, last_nondebug_stmt (cur_bb));
2395 :
2396 90 : basic_block new_cur_bb = create_empty_bb (cur_bb);
2397 90 : add_bb_to_loop (new_cur_bb, cur_bb->loop_father);
2398 :
2399 90 : gsi2 = gsi_after_labels (e->dest);
2400 90 : tree step = fold_convert (itype,
2401 : unshare_expr (fd->loops[i].step));
2402 90 : if (POINTER_TYPE_P (TREE_TYPE (vs[i])))
2403 38 : t = fold_build_pointer_plus (vs[i], step);
2404 : else
2405 52 : t = fold_build2 (PLUS_EXPR, itype, vs[i], step);
2406 90 : t = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2407 : true, GSI_SAME_STMT);
2408 90 : expand_omp_build_assign (&gsi2, vs[i], t);
2409 :
2410 90 : ne = split_block (e->dest, last_nondebug_stmt (e->dest));
2411 90 : gsi2 = gsi_after_labels (ne->dest);
2412 :
2413 90 : expand_omp_build_cond (&gsi2, fd->loops[i].cond_code, vs[i], n2);
2414 90 : edge e3, e4;
2415 90 : if (next_bb == entry_bb)
2416 : {
2417 59 : e3 = find_edge (ne->dest, next_bb);
2418 59 : e3->flags = EDGE_FALSE_VALUE;
2419 : }
2420 : else
2421 31 : e3 = make_edge (ne->dest, next_bb, EDGE_FALSE_VALUE);
2422 90 : e4 = make_edge (ne->dest, new_cur_bb, EDGE_TRUE_VALUE);
2423 90 : e4->probability = profile_probability::likely ().guessed ();
2424 90 : e3->probability = e4->probability.invert ();
2425 90 : basic_block esrc = e->src;
2426 90 : make_edge (e->src, ne->dest, EDGE_FALLTHRU);
2427 90 : cur_bb = new_cur_bb;
2428 90 : basic_block latch_bb = next_bb;
2429 90 : next_bb = e->dest;
2430 90 : remove_edge (e);
2431 90 : set_immediate_dominator (CDI_DOMINATORS, ne->dest, esrc);
2432 90 : set_immediate_dominator (CDI_DOMINATORS, latch_bb, ne->dest);
2433 90 : set_immediate_dominator (CDI_DOMINATORS, cur_bb, ne->dest);
2434 : }
2435 : }
2436 284 : t = NULL_TREE;
2437 691 : for (i = fd->first_nonrect; i < fd->last_nonrect; i++)
2438 407 : if (!fd->loops[i].non_rect_referenced
2439 92 : && fd->loops[i].m1 == NULL_TREE
2440 92 : && fd->loops[i].m2 == NULL_TREE)
2441 : {
2442 92 : if (t == NULL_TREE)
2443 82 : t = counts[i];
2444 : else
2445 10 : t = fold_build2 (MULT_EXPR, type, t, counts[i]);
2446 : }
2447 284 : if (t)
2448 : {
2449 82 : t = fold_build2 (MULT_EXPR, type, counts[fd->last_nonrect], t);
2450 82 : expand_omp_build_assign (gsi, counts[fd->last_nonrect], t);
2451 : }
2452 284 : if (!rect_count_seen)
2453 186 : t = counts[fd->last_nonrect];
2454 : else
2455 98 : t = fold_build2 (MULT_EXPR, type, fd->loop.n2,
2456 : counts[fd->last_nonrect]);
2457 284 : expand_omp_build_assign (gsi, fd->loop.n2, t);
2458 284 : }
2459 7467 : else if (fd->non_rect)
2460 : {
2461 117 : tree t = fd->loop.n2;
2462 117 : gcc_assert (TREE_CODE (t) == INTEGER_CST);
2463 : int non_rect_referenced = 0, non_rect = 0;
2464 358 : for (i = 0; i < fd->collapse; i++)
2465 : {
2466 240 : if ((i < fd->first_nonrect || i > fd->last_nonrect)
2467 246 : && !integer_zerop (counts[i]))
2468 6 : t = fold_build2 (TRUNC_DIV_EXPR, type, t, counts[i]);
2469 241 : if (fd->loops[i].non_rect_referenced)
2470 117 : non_rect_referenced++;
2471 241 : if (fd->loops[i].m1 || fd->loops[i].m2)
2472 117 : non_rect++;
2473 : }
2474 117 : gcc_assert (non_rect == 1 && non_rect_referenced == 1);
2475 117 : counts[fd->last_nonrect] = t;
2476 : }
2477 : }
2478 :
2479 : /* Helper function for expand_omp_{for_*,simd}. Generate code like:
2480 : T = V;
2481 : V3 = N31 + (T % count3) * STEP3;
2482 : T = T / count3;
2483 : V2 = N21 + (T % count2) * STEP2;
2484 : T = T / count2;
2485 : V1 = N11 + T * STEP1;
2486 : if this loop doesn't have an inner loop construct combined with it.
2487 : If it does have an inner loop construct combined with it and the
2488 : iteration count isn't known constant, store values from counts array
2489 : into its _looptemp_ temporaries instead.
2490 : For non-rectangular loops (between fd->first_nonrect and fd->last_nonrect
2491 : inclusive), use the count of all those loops together, and either
2492 : find quadratic etc. equation roots, or as a fallback, do:
2493 : COUNT = 0;
2494 : for (tmpi = N11; tmpi COND1 N12; tmpi += STEP1)
2495 : for (tmpj = M21 * tmpi + N21;
2496 : tmpj COND2 M22 * tmpi + N22; tmpj += STEP2)
2497 : {
2498 : int tmpk1 = M31 * tmpj + N31;
2499 : int tmpk2 = M32 * tmpj + N32;
2500 : if (tmpk1 COND3 tmpk2)
2501 : {
2502 : if (COND3 is <)
2503 : adj = STEP3 - 1;
2504 : else
2505 : adj = STEP3 + 1;
2506 : int temp = (adj + tmpk2 - tmpk1) / STEP3;
2507 : if (COUNT + temp > T)
2508 : {
2509 : V1 = tmpi;
2510 : V2 = tmpj;
2511 : V3 = tmpk1 + (T - COUNT) * STEP3;
2512 : goto done;
2513 : }
2514 : else
2515 : COUNT += temp;
2516 : }
2517 : }
2518 : done:;
2519 : but for optional innermost or outermost rectangular loops that aren't
2520 : referenced by other loop expressions keep doing the division/modulo. */
2521 :
2522 : static void
2523 10255 : expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
2524 : tree *counts, tree *nonrect_bounds,
2525 : gimple *inner_stmt, tree startvar)
2526 : {
2527 10255 : int i;
2528 10255 : if (gimple_omp_for_combined_p (fd->for_stmt))
2529 : {
2530 : /* If fd->loop.n2 is constant, then no propagation of the counts
2531 : is needed, they are constant. */
2532 4867 : if (TREE_CODE (fd->loop.n2) == INTEGER_CST)
2533 : return;
2534 :
2535 2692 : tree clauses = gimple_code (inner_stmt) != GIMPLE_OMP_FOR
2536 4158 : ? gimple_omp_taskreg_clauses (inner_stmt)
2537 1226 : : gimple_omp_for_clauses (inner_stmt);
2538 : /* First two _looptemp_ clauses are for istart/iend, counts[0]
2539 : isn't supposed to be handled, as the inner loop doesn't
2540 : use it. */
2541 2692 : tree innerc = omp_find_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
2542 2692 : gcc_assert (innerc);
2543 2692 : int count = 0;
2544 2692 : if (fd->non_rect
2545 112 : && fd->last_nonrect == fd->first_nonrect + 1
2546 2756 : && !TYPE_UNSIGNED (TREE_TYPE (fd->loops[fd->last_nonrect].v)))
2547 : count = 4;
2548 10793 : for (i = 0; i < fd->collapse + count; i++)
2549 : {
2550 8101 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
2551 : OMP_CLAUSE__LOOPTEMP_);
2552 8101 : gcc_assert (innerc);
2553 8101 : if (i)
2554 : {
2555 5409 : tree tem = OMP_CLAUSE_DECL (innerc);
2556 5409 : tree t;
2557 5409 : if (i < fd->collapse)
2558 5169 : t = counts[i];
2559 : else
2560 240 : switch (i - fd->collapse)
2561 : {
2562 60 : case 0: t = counts[0]; break;
2563 60 : case 1: t = fd->first_inner_iterations; break;
2564 60 : case 2: t = fd->factor; break;
2565 60 : case 3: t = fd->adjn1; break;
2566 0 : default: gcc_unreachable ();
2567 : }
2568 5409 : t = fold_convert (TREE_TYPE (tem), t);
2569 5409 : t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE,
2570 : false, GSI_CONTINUE_LINKING);
2571 5409 : gassign *stmt = gimple_build_assign (tem, t);
2572 5409 : gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
2573 : }
2574 : }
2575 : return;
2576 : }
2577 :
2578 5388 : tree type = TREE_TYPE (fd->loop.v);
2579 5388 : tree tem = create_tmp_reg (type, ".tem");
2580 5388 : gassign *stmt = gimple_build_assign (tem, startvar);
2581 5388 : gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
2582 :
2583 20052 : for (i = fd->collapse - 1; i >= 0; i--)
2584 : {
2585 14664 : tree vtype = TREE_TYPE (fd->loops[i].v), itype, t;
2586 14664 : itype = vtype;
2587 14664 : if (POINTER_TYPE_P (vtype))
2588 1649 : itype = signed_type_for (vtype);
2589 14664 : if (i != 0 && (i != fd->last_nonrect || fd->first_nonrect))
2590 9276 : t = fold_build2 (TRUNC_MOD_EXPR, type, tem, counts[i]);
2591 : else
2592 : t = tem;
2593 14664 : if (i == fd->last_nonrect)
2594 : {
2595 376 : t = force_gimple_operand_gsi (gsi, t, true, NULL_TREE,
2596 : false, GSI_CONTINUE_LINKING);
2597 376 : tree stopval = t;
2598 376 : tree idx = create_tmp_reg (type, ".count");
2599 376 : expand_omp_build_assign (gsi, idx,
2600 : build_zero_cst (type), true);
2601 376 : basic_block bb_triang = NULL, bb_triang_dom = NULL;
2602 376 : if (fd->first_nonrect + 1 == fd->last_nonrect
2603 272 : && (TREE_CODE (fd->loop.n2) == INTEGER_CST
2604 181 : || fd->first_inner_iterations)
2605 488 : && (optab_handler (sqrt_optab, TYPE_MODE (double_type_node))
2606 : != CODE_FOR_nothing)
2607 620 : && !integer_zerop (fd->loop.n2))
2608 : {
2609 236 : tree outer_n1 = fd->adjn1 ? fd->adjn1 : fd->loops[i - 1].n1;
2610 236 : tree itype = TREE_TYPE (fd->loops[i].v);
2611 236 : tree first_inner_iterations = fd->first_inner_iterations;
2612 236 : tree factor = fd->factor;
2613 236 : gcond *cond_stmt
2614 236 : = expand_omp_build_cond (gsi, NE_EXPR, factor,
2615 236 : build_zero_cst (TREE_TYPE (factor)),
2616 : true);
2617 236 : edge e = split_block (gsi_bb (*gsi), cond_stmt);
2618 236 : basic_block bb0 = e->src;
2619 236 : e->flags = EDGE_TRUE_VALUE;
2620 236 : e->probability = profile_probability::likely ();
2621 236 : bb_triang_dom = bb0;
2622 236 : *gsi = gsi_after_labels (e->dest);
2623 236 : tree slltype = long_long_integer_type_node;
2624 236 : tree ulltype = long_long_unsigned_type_node;
2625 236 : tree stopvalull = fold_convert (ulltype, stopval);
2626 236 : stopvalull
2627 236 : = force_gimple_operand_gsi (gsi, stopvalull, true, NULL_TREE,
2628 : false, GSI_CONTINUE_LINKING);
2629 236 : first_inner_iterations
2630 236 : = fold_convert (slltype, first_inner_iterations);
2631 236 : first_inner_iterations
2632 236 : = force_gimple_operand_gsi (gsi, first_inner_iterations, true,
2633 : NULL_TREE, false,
2634 : GSI_CONTINUE_LINKING);
2635 236 : factor = fold_convert (slltype, factor);
2636 236 : factor
2637 236 : = force_gimple_operand_gsi (gsi, factor, true, NULL_TREE,
2638 : false, GSI_CONTINUE_LINKING);
2639 236 : tree first_inner_iterationsd
2640 236 : = fold_build1 (FLOAT_EXPR, double_type_node,
2641 : first_inner_iterations);
2642 236 : first_inner_iterationsd
2643 236 : = force_gimple_operand_gsi (gsi, first_inner_iterationsd, true,
2644 : NULL_TREE, false,
2645 : GSI_CONTINUE_LINKING);
2646 236 : tree factord = fold_build1 (FLOAT_EXPR, double_type_node,
2647 : factor);
2648 236 : factord = force_gimple_operand_gsi (gsi, factord, true,
2649 : NULL_TREE, false,
2650 : GSI_CONTINUE_LINKING);
2651 236 : tree stopvald = fold_build1 (FLOAT_EXPR, double_type_node,
2652 : stopvalull);
2653 236 : stopvald = force_gimple_operand_gsi (gsi, stopvald, true,
2654 : NULL_TREE, false,
2655 : GSI_CONTINUE_LINKING);
2656 : /* Temporarily disable flag_rounding_math, values will be
2657 : decimal numbers divided by 2 and worst case imprecisions
2658 : due to too large values ought to be caught later by the
2659 : checks for fallback. */
2660 236 : int save_flag_rounding_math = flag_rounding_math;
2661 236 : flag_rounding_math = 0;
2662 236 : t = fold_build2 (RDIV_EXPR, double_type_node, factord,
2663 : build_real (double_type_node, dconst2));
2664 236 : tree t3 = fold_build2 (MINUS_EXPR, double_type_node,
2665 : first_inner_iterationsd, t);
2666 236 : t3 = force_gimple_operand_gsi (gsi, t3, true, NULL_TREE, false,
2667 : GSI_CONTINUE_LINKING);
2668 236 : t = fold_build2 (MULT_EXPR, double_type_node, factord,
2669 : build_real (double_type_node, dconst2));
2670 236 : t = fold_build2 (MULT_EXPR, double_type_node, t, stopvald);
2671 236 : t = fold_build2 (PLUS_EXPR, double_type_node, t,
2672 : fold_build2 (MULT_EXPR, double_type_node,
2673 : t3, t3));
2674 236 : flag_rounding_math = save_flag_rounding_math;
2675 236 : t = force_gimple_operand_gsi (gsi, t, true, NULL_TREE, false,
2676 : GSI_CONTINUE_LINKING);
2677 236 : if (flag_exceptions
2678 34 : && cfun->can_throw_non_call_exceptions
2679 244 : && operation_could_trap_p (LT_EXPR, true, false, NULL_TREE))
2680 : {
2681 8 : tree tem = fold_build2 (LT_EXPR, boolean_type_node, t,
2682 : build_zero_cst (double_type_node));
2683 8 : tem = force_gimple_operand_gsi (gsi, tem, true, NULL_TREE,
2684 : false, GSI_CONTINUE_LINKING);
2685 8 : cond_stmt = gimple_build_cond (NE_EXPR, tem,
2686 : boolean_false_node,
2687 : NULL_TREE, NULL_TREE);
2688 : }
2689 : else
2690 228 : cond_stmt
2691 228 : = gimple_build_cond (LT_EXPR, t,
2692 : build_zero_cst (double_type_node),
2693 : NULL_TREE, NULL_TREE);
2694 236 : gsi_insert_after (gsi, cond_stmt, GSI_CONTINUE_LINKING);
2695 236 : e = split_block (gsi_bb (*gsi), cond_stmt);
2696 236 : basic_block bb1 = e->src;
2697 236 : e->flags = EDGE_FALSE_VALUE;
2698 236 : e->probability = profile_probability::very_likely ();
2699 236 : *gsi = gsi_after_labels (e->dest);
2700 236 : gcall *call = gimple_build_call_internal (IFN_SQRT, 1, t);
2701 236 : tree sqrtr = create_tmp_var (double_type_node);
2702 236 : gimple_call_set_lhs (call, sqrtr);
2703 236 : gsi_insert_after (gsi, call, GSI_CONTINUE_LINKING);
2704 236 : t = fold_build2 (MINUS_EXPR, double_type_node, sqrtr, t3);
2705 236 : t = fold_build2 (RDIV_EXPR, double_type_node, t, factord);
2706 236 : t = fold_build1 (FIX_TRUNC_EXPR, ulltype, t);
2707 236 : tree c = create_tmp_var (ulltype);
2708 236 : tree d = create_tmp_var (ulltype);
2709 236 : expand_omp_build_assign (gsi, c, t, true);
2710 236 : t = fold_build2 (MINUS_EXPR, ulltype, c,
2711 : build_one_cst (ulltype));
2712 236 : t = fold_build2 (MULT_EXPR, ulltype, c, t);
2713 236 : t = fold_build2 (RSHIFT_EXPR, ulltype, t, integer_one_node);
2714 236 : t = fold_build2 (MULT_EXPR, ulltype,
2715 : fold_convert (ulltype, fd->factor), t);
2716 236 : tree t2
2717 236 : = fold_build2 (MULT_EXPR, ulltype, c,
2718 : fold_convert (ulltype,
2719 : fd->first_inner_iterations));
2720 236 : t = fold_build2 (PLUS_EXPR, ulltype, t, t2);
2721 236 : expand_omp_build_assign (gsi, d, t, true);
2722 236 : t = fold_build2 (MULT_EXPR, ulltype,
2723 : fold_convert (ulltype, fd->factor), c);
2724 236 : t = fold_build2 (PLUS_EXPR, ulltype,
2725 : t, fold_convert (ulltype,
2726 : fd->first_inner_iterations));
2727 236 : t2 = force_gimple_operand_gsi (gsi, t, true, NULL_TREE, false,
2728 : GSI_CONTINUE_LINKING);
2729 236 : cond_stmt = gimple_build_cond (GE_EXPR, stopvalull, d,
2730 : NULL_TREE, NULL_TREE);
2731 236 : gsi_insert_after (gsi, cond_stmt, GSI_CONTINUE_LINKING);
2732 236 : e = split_block (gsi_bb (*gsi), cond_stmt);
2733 236 : basic_block bb2 = e->src;
2734 236 : e->flags = EDGE_TRUE_VALUE;
2735 236 : e->probability = profile_probability::very_likely ();
2736 236 : *gsi = gsi_after_labels (e->dest);
2737 236 : t = fold_build2 (PLUS_EXPR, ulltype, d, t2);
2738 236 : t = force_gimple_operand_gsi (gsi, t, true, NULL_TREE, false,
2739 : GSI_CONTINUE_LINKING);
2740 236 : cond_stmt = gimple_build_cond (GE_EXPR, stopvalull, t,
2741 : NULL_TREE, NULL_TREE);
2742 236 : gsi_insert_after (gsi, cond_stmt, GSI_CONTINUE_LINKING);
2743 236 : e = split_block (gsi_bb (*gsi), cond_stmt);
2744 236 : basic_block bb3 = e->src;
2745 236 : e->flags = EDGE_FALSE_VALUE;
2746 236 : e->probability = profile_probability::very_likely ();
2747 236 : *gsi = gsi_after_labels (e->dest);
2748 236 : t = fold_convert (itype, c);
2749 236 : t = fold_build2 (MULT_EXPR, itype, t, fd->loops[i - 1].step);
2750 236 : t = fold_build2 (PLUS_EXPR, itype, outer_n1, t);
2751 236 : t = force_gimple_operand_gsi (gsi, t, true, NULL_TREE, false,
2752 : GSI_CONTINUE_LINKING);
2753 236 : expand_omp_build_assign (gsi, fd->loops[i - 1].v, t, true);
2754 236 : t2 = fold_build2 (MINUS_EXPR, ulltype, stopvalull, d);
2755 236 : t2 = fold_convert (itype, t2);
2756 236 : t2 = fold_build2 (MULT_EXPR, itype, t2, fd->loops[i].step);
2757 236 : t2 = fold_build2 (PLUS_EXPR, itype, t2, fd->loops[i].n1);
2758 236 : if (fd->loops[i].m1)
2759 : {
2760 195 : t = fold_build2 (MULT_EXPR, itype, t, fd->loops[i].m1);
2761 195 : t2 = fold_build2 (PLUS_EXPR, itype, t2, t);
2762 : }
2763 236 : expand_omp_build_assign (gsi, fd->loops[i].v, t2, true);
2764 236 : e = split_block (gsi_bb (*gsi), gsi_stmt (*gsi));
2765 236 : bb_triang = e->src;
2766 236 : *gsi = gsi_after_labels (e->dest);
2767 236 : remove_edge (e);
2768 236 : e = make_edge (bb1, gsi_bb (*gsi), EDGE_TRUE_VALUE);
2769 236 : e->probability = profile_probability::very_unlikely ();
2770 236 : e = make_edge (bb2, gsi_bb (*gsi), EDGE_FALSE_VALUE);
2771 236 : e->probability = profile_probability::very_unlikely ();
2772 236 : e = make_edge (bb3, gsi_bb (*gsi), EDGE_TRUE_VALUE);
2773 236 : e->probability = profile_probability::very_unlikely ();
2774 :
2775 236 : basic_block bb4 = create_empty_bb (bb0);
2776 236 : add_bb_to_loop (bb4, bb0->loop_father);
2777 236 : e = make_edge (bb0, bb4, EDGE_FALSE_VALUE);
2778 236 : e->probability = profile_probability::unlikely ();
2779 236 : make_edge (bb4, gsi_bb (*gsi), EDGE_FALLTHRU);
2780 236 : set_immediate_dominator (CDI_DOMINATORS, bb4, bb0);
2781 236 : set_immediate_dominator (CDI_DOMINATORS, gsi_bb (*gsi), bb0);
2782 236 : gimple_stmt_iterator gsi2 = gsi_after_labels (bb4);
2783 236 : t2 = fold_build2 (TRUNC_DIV_EXPR, type,
2784 : counts[i], counts[i - 1]);
2785 236 : t2 = force_gimple_operand_gsi (&gsi2, t2, true, NULL_TREE, false,
2786 : GSI_CONTINUE_LINKING);
2787 236 : t = fold_build2 (TRUNC_MOD_EXPR, type, stopval, t2);
2788 236 : t2 = fold_build2 (TRUNC_DIV_EXPR, type, stopval, t2);
2789 236 : t = fold_convert (itype, t);
2790 236 : t2 = fold_convert (itype, t2);
2791 236 : t = fold_build2 (MULT_EXPR, itype, t,
2792 : fold_convert (itype, fd->loops[i].step));
2793 236 : t = fold_build2 (PLUS_EXPR, itype, fd->loops[i].n1, t);
2794 236 : t2 = fold_build2 (MULT_EXPR, itype, t2,
2795 : fold_convert (itype, fd->loops[i - 1].step));
2796 236 : t2 = fold_build2 (PLUS_EXPR, itype, fd->loops[i - 1].n1, t2);
2797 236 : t2 = force_gimple_operand_gsi (&gsi2, t2, false, NULL_TREE,
2798 : false, GSI_CONTINUE_LINKING);
2799 236 : stmt = gimple_build_assign (fd->loops[i - 1].v, t2);
2800 236 : gsi_insert_after (&gsi2, stmt, GSI_CONTINUE_LINKING);
2801 236 : if (fd->loops[i].m1)
2802 : {
2803 195 : t2 = fold_build2 (MULT_EXPR, itype, fd->loops[i].m1,
2804 : fd->loops[i - 1].v);
2805 195 : t = fold_build2 (PLUS_EXPR, itype, t, t2);
2806 : }
2807 236 : t = force_gimple_operand_gsi (&gsi2, t, false, NULL_TREE,
2808 : false, GSI_CONTINUE_LINKING);
2809 236 : stmt = gimple_build_assign (fd->loops[i].v, t);
2810 236 : gsi_insert_after (&gsi2, stmt, GSI_CONTINUE_LINKING);
2811 : }
2812 : /* Fallback implementation. Evaluate the loops in between
2813 : (inclusive) fd->first_nonrect and fd->last_nonrect at
2814 : runtime using temporaries instead of the original iteration
2815 : variables, in the body just bump the counter and compare
2816 : with the desired value. */
2817 376 : gimple_stmt_iterator gsi2 = *gsi;
2818 376 : basic_block entry_bb = gsi_bb (gsi2);
2819 376 : edge e = split_block (entry_bb, gsi_stmt (gsi2));
2820 376 : e = split_block (e->dest, (gimple *) NULL);
2821 376 : basic_block dom_bb = NULL;
2822 376 : basic_block cur_bb = e->src;
2823 376 : basic_block next_bb = e->dest;
2824 376 : entry_bb = e->dest;
2825 376 : *gsi = gsi_after_labels (entry_bb);
2826 :
2827 376 : tree *vs = XALLOCAVEC (tree, fd->last_nonrect);
2828 376 : tree n1 = NULL_TREE, n2 = NULL_TREE;
2829 376 : memset (vs, 0, fd->last_nonrect * sizeof (tree));
2830 :
2831 876 : for (int j = fd->first_nonrect; j <= fd->last_nonrect; j++)
2832 : {
2833 876 : tree itype = TREE_TYPE (fd->loops[j].v);
2834 1752 : bool rect_p = (fd->loops[j].m1 == NULL_TREE
2835 527 : && fd->loops[j].m2 == NULL_TREE
2836 1345 : && !fd->loops[j].non_rect_referenced);
2837 876 : gsi2 = gsi_after_labels (cur_bb);
2838 876 : t = fold_convert (itype, unshare_expr (fd->loops[j].n1));
2839 876 : if (fd->loops[j].m1 == NULL_TREE)
2840 527 : n1 = rect_p ? build_zero_cst (type) : t;
2841 349 : else if (POINTER_TYPE_P (itype))
2842 : {
2843 30 : gcc_assert (integer_onep (fd->loops[j].m1));
2844 30 : t = unshare_expr (fd->loops[j].n1);
2845 30 : n1 = fold_build_pointer_plus (vs[j - fd->loops[j].outer], t);
2846 : }
2847 : else
2848 : {
2849 319 : n1 = fold_convert (itype, unshare_expr (fd->loops[j].m1));
2850 319 : n1 = fold_build2 (MULT_EXPR, itype,
2851 : vs[j - fd->loops[j].outer], n1);
2852 319 : n1 = fold_build2 (PLUS_EXPR, itype, n1, t);
2853 : }
2854 876 : n1 = force_gimple_operand_gsi (&gsi2, n1, true, NULL_TREE,
2855 : true, GSI_SAME_STMT);
2856 876 : if (j < fd->last_nonrect)
2857 : {
2858 907 : vs[j] = create_tmp_reg (rect_p ? type : itype, ".it");
2859 500 : expand_omp_build_assign (&gsi2, vs[j], n1);
2860 : }
2861 876 : t = fold_convert (itype, unshare_expr (fd->loops[j].n2));
2862 876 : if (fd->loops[j].m2 == NULL_TREE)
2863 646 : n2 = rect_p ? counts[j] : t;
2864 230 : else if (POINTER_TYPE_P (itype))
2865 : {
2866 34 : gcc_assert (integer_onep (fd->loops[j].m2));
2867 34 : t = unshare_expr (fd->loops[j].n2);
2868 34 : n2 = fold_build_pointer_plus (vs[j - fd->loops[j].outer], t);
2869 : }
2870 : else
2871 : {
2872 196 : n2 = fold_convert (itype, unshare_expr (fd->loops[j].m2));
2873 196 : n2 = fold_build2 (MULT_EXPR, itype,
2874 : vs[j - fd->loops[j].outer], n2);
2875 196 : n2 = fold_build2 (PLUS_EXPR, itype, n2, t);
2876 : }
2877 876 : n2 = force_gimple_operand_gsi (&gsi2, n2, true, NULL_TREE,
2878 : true, GSI_SAME_STMT);
2879 876 : if (POINTER_TYPE_P (itype))
2880 74 : itype = signed_type_for (itype);
2881 876 : if (j == fd->last_nonrect)
2882 : {
2883 376 : gcond *cond_stmt
2884 376 : = expand_omp_build_cond (&gsi2, fd->loops[i].cond_code,
2885 : n1, n2);
2886 376 : e = split_block (cur_bb, cond_stmt);
2887 376 : e->flags = EDGE_TRUE_VALUE;
2888 376 : edge ne = make_edge (cur_bb, next_bb, EDGE_FALSE_VALUE);
2889 376 : e->probability = profile_probability::likely ().guessed ();
2890 376 : ne->probability = e->probability.invert ();
2891 376 : gsi2 = gsi_after_labels (e->dest);
2892 :
2893 376 : t = build_int_cst (itype, (fd->loops[j].cond_code == LT_EXPR
2894 428 : ? -1 : 1));
2895 376 : t = fold_build2 (PLUS_EXPR, itype,
2896 : fold_convert (itype, fd->loops[j].step), t);
2897 376 : t = fold_build2 (PLUS_EXPR, itype, t,
2898 : fold_convert (itype, n2));
2899 376 : t = fold_build2 (MINUS_EXPR, itype, t,
2900 : fold_convert (itype, n1));
2901 376 : tree step = fold_convert (itype, fd->loops[j].step);
2902 376 : if (TYPE_UNSIGNED (itype)
2903 376 : && fd->loops[j].cond_code == GT_EXPR)
2904 0 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
2905 : fold_build1 (NEGATE_EXPR, itype, t),
2906 : fold_build1 (NEGATE_EXPR, itype, step));
2907 : else
2908 376 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
2909 376 : t = fold_convert (type, t);
2910 376 : t = fold_build2 (PLUS_EXPR, type, idx, t);
2911 376 : t = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2912 : true, GSI_SAME_STMT);
2913 376 : e = make_edge (e->dest, next_bb, EDGE_FALLTHRU);
2914 376 : set_immediate_dominator (CDI_DOMINATORS, next_bb, cur_bb);
2915 376 : cond_stmt
2916 376 : = gimple_build_cond (LE_EXPR, t, stopval, NULL_TREE,
2917 : NULL_TREE);
2918 376 : gsi_insert_before (&gsi2, cond_stmt, GSI_SAME_STMT);
2919 376 : e = split_block (gsi_bb (gsi2), cond_stmt);
2920 376 : e->flags = EDGE_TRUE_VALUE;
2921 376 : e->probability = profile_probability::likely ().guessed ();
2922 376 : ne = make_edge (e->src, entry_bb, EDGE_FALSE_VALUE);
2923 376 : ne->probability = e->probability.invert ();
2924 376 : gsi2 = gsi_after_labels (e->dest);
2925 376 : expand_omp_build_assign (&gsi2, idx, t);
2926 376 : set_immediate_dominator (CDI_DOMINATORS, entry_bb, dom_bb);
2927 376 : break;
2928 : }
2929 500 : e = split_block (cur_bb, last_nondebug_stmt (cur_bb));
2930 :
2931 500 : basic_block new_cur_bb = create_empty_bb (cur_bb);
2932 500 : add_bb_to_loop (new_cur_bb, cur_bb->loop_father);
2933 :
2934 500 : gsi2 = gsi_after_labels (e->dest);
2935 500 : if (rect_p)
2936 93 : t = fold_build2 (PLUS_EXPR, type, vs[j],
2937 : build_one_cst (type));
2938 : else
2939 : {
2940 407 : tree step
2941 407 : = fold_convert (itype, unshare_expr (fd->loops[j].step));
2942 407 : if (POINTER_TYPE_P (vtype))
2943 38 : t = fold_build_pointer_plus (vs[j], step);
2944 : else
2945 369 : t = fold_build2 (PLUS_EXPR, itype, vs[j], step);
2946 : }
2947 500 : t = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2948 : true, GSI_SAME_STMT);
2949 500 : expand_omp_build_assign (&gsi2, vs[j], t);
2950 :
2951 500 : edge ne = split_block (e->dest, last_nondebug_stmt (e->dest));
2952 500 : gsi2 = gsi_after_labels (ne->dest);
2953 :
2954 500 : gcond *cond_stmt;
2955 500 : if (next_bb == entry_bb)
2956 : /* No need to actually check the outermost condition. */
2957 376 : cond_stmt
2958 376 : = gimple_build_cond (EQ_EXPR, boolean_true_node,
2959 : boolean_true_node,
2960 : NULL_TREE, NULL_TREE);
2961 : else
2962 124 : cond_stmt
2963 155 : = gimple_build_cond (rect_p ? LT_EXPR
2964 31 : : fd->loops[j].cond_code,
2965 : vs[j], n2, NULL_TREE, NULL_TREE);
2966 500 : gsi_insert_before (&gsi2, cond_stmt, GSI_SAME_STMT);
2967 500 : edge e3, e4;
2968 500 : if (next_bb == entry_bb)
2969 : {
2970 376 : e3 = find_edge (ne->dest, next_bb);
2971 376 : e3->flags = EDGE_FALSE_VALUE;
2972 376 : dom_bb = ne->dest;
2973 : }
2974 : else
2975 124 : e3 = make_edge (ne->dest, next_bb, EDGE_FALSE_VALUE);
2976 500 : e4 = make_edge (ne->dest, new_cur_bb, EDGE_TRUE_VALUE);
2977 500 : e4->probability = profile_probability::likely ().guessed ();
2978 500 : e3->probability = e4->probability.invert ();
2979 500 : basic_block esrc = e->src;
2980 500 : make_edge (e->src, ne->dest, EDGE_FALLTHRU);
2981 500 : cur_bb = new_cur_bb;
2982 500 : basic_block latch_bb = next_bb;
2983 500 : next_bb = e->dest;
2984 500 : remove_edge (e);
2985 500 : set_immediate_dominator (CDI_DOMINATORS, ne->dest, esrc);
2986 500 : set_immediate_dominator (CDI_DOMINATORS, latch_bb, ne->dest);
2987 500 : set_immediate_dominator (CDI_DOMINATORS, cur_bb, ne->dest);
2988 : }
2989 1252 : for (int j = fd->last_nonrect; j >= fd->first_nonrect; j--)
2990 : {
2991 876 : tree vtype = TREE_TYPE (fd->loops[j].v);
2992 876 : tree itype = vtype;
2993 876 : if (POINTER_TYPE_P (itype))
2994 74 : itype = signed_type_for (itype);
2995 1752 : bool rect_p = (fd->loops[j].m1 == NULL_TREE
2996 527 : && fd->loops[j].m2 == NULL_TREE
2997 1345 : && !fd->loops[j].non_rect_referenced);
2998 876 : if (j == fd->last_nonrect)
2999 : {
3000 376 : t = fold_build2 (MINUS_EXPR, type, stopval, idx);
3001 376 : t = fold_convert (itype, t);
3002 376 : tree t2
3003 376 : = fold_convert (itype, unshare_expr (fd->loops[j].step));
3004 376 : t = fold_build2 (MULT_EXPR, itype, t, t2);
3005 376 : if (POINTER_TYPE_P (vtype))
3006 32 : t = fold_build_pointer_plus (n1, t);
3007 : else
3008 344 : t = fold_build2 (PLUS_EXPR, itype, n1, t);
3009 : }
3010 500 : else if (rect_p)
3011 : {
3012 93 : t = fold_convert (itype, vs[j]);
3013 93 : t = fold_build2 (MULT_EXPR, itype, t,
3014 : fold_convert (itype, fd->loops[j].step));
3015 93 : if (POINTER_TYPE_P (vtype))
3016 4 : t = fold_build_pointer_plus (fd->loops[j].n1, t);
3017 : else
3018 89 : t = fold_build2 (PLUS_EXPR, itype, fd->loops[j].n1, t);
3019 : }
3020 : else
3021 407 : t = vs[j];
3022 876 : t = force_gimple_operand_gsi (gsi, t, false,
3023 : NULL_TREE, true,
3024 : GSI_SAME_STMT);
3025 876 : stmt = gimple_build_assign (fd->loops[j].v, t);
3026 876 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
3027 : }
3028 376 : if (gsi_end_p (*gsi))
3029 752 : *gsi = gsi_last_bb (gsi_bb (*gsi));
3030 : else
3031 0 : gsi_prev (gsi);
3032 376 : if (bb_triang)
3033 : {
3034 236 : e = split_block (gsi_bb (*gsi), gsi_stmt (*gsi));
3035 236 : make_edge (bb_triang, e->dest, EDGE_FALLTHRU);
3036 236 : *gsi = gsi_after_labels (e->dest);
3037 236 : if (!gsi_end_p (*gsi))
3038 0 : gsi_insert_before (gsi, gimple_build_nop (), GSI_NEW_STMT);
3039 236 : set_immediate_dominator (CDI_DOMINATORS, e->dest, bb_triang_dom);
3040 : }
3041 : }
3042 : else
3043 : {
3044 14288 : t = fold_convert (itype, t);
3045 14288 : t = fold_build2 (MULT_EXPR, itype, t,
3046 : fold_convert (itype, fd->loops[i].step));
3047 14288 : if (POINTER_TYPE_P (vtype))
3048 1617 : t = fold_build_pointer_plus (fd->loops[i].n1, t);
3049 : else
3050 12671 : t = fold_build2 (PLUS_EXPR, itype, fd->loops[i].n1, t);
3051 14288 : t = force_gimple_operand_gsi (gsi, t,
3052 14288 : DECL_P (fd->loops[i].v)
3053 14288 : && TREE_ADDRESSABLE (fd->loops[i].v),
3054 : NULL_TREE, false,
3055 : GSI_CONTINUE_LINKING);
3056 14288 : stmt = gimple_build_assign (fd->loops[i].v, t);
3057 14288 : gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
3058 : }
3059 14664 : if (i != 0 && (i != fd->last_nonrect || fd->first_nonrect))
3060 : {
3061 9276 : t = fold_build2 (TRUNC_DIV_EXPR, type, tem, counts[i]);
3062 9276 : t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE,
3063 : false, GSI_CONTINUE_LINKING);
3064 9276 : stmt = gimple_build_assign (tem, t);
3065 9276 : gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
3066 : }
3067 14664 : if (i == fd->last_nonrect)
3068 376 : i = fd->first_nonrect;
3069 : }
3070 5388 : if (fd->non_rect)
3071 1351 : for (i = 0; i <= fd->last_nonrect; i++)
3072 975 : if (fd->loops[i].m2)
3073 : {
3074 230 : tree itype = TREE_TYPE (fd->loops[i].v);
3075 :
3076 230 : tree t;
3077 230 : if (POINTER_TYPE_P (itype))
3078 : {
3079 34 : gcc_assert (integer_onep (fd->loops[i].m2));
3080 34 : t = fold_build_pointer_plus (fd->loops[i - fd->loops[i].outer].v,
3081 : unshare_expr (fd->loops[i].n2));
3082 : }
3083 : else
3084 : {
3085 196 : t = fold_convert (itype, unshare_expr (fd->loops[i].m2));
3086 196 : t = fold_build2 (MULT_EXPR, itype,
3087 : fd->loops[i - fd->loops[i].outer].v, t);
3088 196 : t = fold_build2 (PLUS_EXPR, itype, t,
3089 : fold_convert (itype,
3090 : unshare_expr (fd->loops[i].n2)));
3091 : }
3092 230 : nonrect_bounds[i] = create_tmp_reg (itype, ".bound");
3093 230 : t = force_gimple_operand_gsi (gsi, t, false,
3094 : NULL_TREE, false,
3095 : GSI_CONTINUE_LINKING);
3096 230 : stmt = gimple_build_assign (nonrect_bounds[i], t);
3097 230 : gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
3098 : }
3099 : }
3100 :
3101 : /* Helper function for expand_omp_for_*. Generate code like:
3102 : L10:
3103 : V3 += STEP3;
3104 : if (V3 cond3 N32) goto BODY_BB; else goto L11;
3105 : L11:
3106 : V3 = N31;
3107 : V2 += STEP2;
3108 : if (V2 cond2 N22) goto BODY_BB; else goto L12;
3109 : L12:
3110 : V2 = N21;
3111 : V1 += STEP1;
3112 : goto BODY_BB;
3113 : For non-rectangular loops, use temporaries stored in nonrect_bounds
3114 : for the upper bounds if M?2 multiplier is present. Given e.g.
3115 : for (V1 = N11; V1 cond1 N12; V1 += STEP1)
3116 : for (V2 = N21; V2 cond2 N22; V2 += STEP2)
3117 : for (V3 = N31; V3 cond3 N32; V3 += STEP3)
3118 : for (V4 = N41 + M41 * V2; V4 cond4 N42 + M42 * V2; V4 += STEP4)
3119 : do:
3120 : L10:
3121 : V4 += STEP4;
3122 : if (V4 cond4 NONRECT_BOUND4) goto BODY_BB; else goto L11;
3123 : L11:
3124 : V4 = N41 + M41 * V2; // This can be left out if the loop
3125 : // refers to the immediate parent loop
3126 : V3 += STEP3;
3127 : if (V3 cond3 N32) goto BODY_BB; else goto L12;
3128 : L12:
3129 : V3 = N31;
3130 : V2 += STEP2;
3131 : if (V2 cond2 N22) goto L120; else goto L13;
3132 : L120:
3133 : V4 = N41 + M41 * V2;
3134 : NONRECT_BOUND4 = N42 + M42 * V2;
3135 : if (V4 cond4 NONRECT_BOUND4) goto BODY_BB; else goto L12;
3136 : L13:
3137 : V2 = N21;
3138 : V1 += STEP1;
3139 : goto L120; */
3140 :
3141 : static basic_block
3142 2575 : extract_omp_for_update_vars (struct omp_for_data *fd, tree *nonrect_bounds,
3143 : basic_block cont_bb, basic_block body_bb)
3144 : {
3145 2575 : basic_block last_bb, bb, collapse_bb = NULL;
3146 2575 : int i;
3147 2575 : gimple_stmt_iterator gsi;
3148 2575 : edge e;
3149 2575 : tree t;
3150 2575 : gimple *stmt;
3151 :
3152 2575 : last_bb = cont_bb;
3153 9682 : for (i = fd->collapse - 1; i >= 0; i--)
3154 : {
3155 7107 : tree vtype = TREE_TYPE (fd->loops[i].v);
3156 :
3157 7107 : bb = create_empty_bb (last_bb);
3158 7107 : add_bb_to_loop (bb, last_bb->loop_father);
3159 7107 : gsi = gsi_start_bb (bb);
3160 :
3161 7107 : if (i < fd->collapse - 1)
3162 : {
3163 4532 : e = make_edge (last_bb, bb, EDGE_FALSE_VALUE);
3164 4532 : e->probability = profile_probability::guessed_always () / 8;
3165 :
3166 4532 : struct omp_for_data_loop *l = &fd->loops[i + 1];
3167 4532 : if (l->m1 == NULL_TREE || l->outer != 1)
3168 : {
3169 4324 : t = l->n1;
3170 4324 : if (l->m1)
3171 : {
3172 56 : if (POINTER_TYPE_P (TREE_TYPE (l->v)))
3173 3 : t = fold_build_pointer_plus (fd->loops[i + 1 - l->outer].v,
3174 : t);
3175 : else
3176 : {
3177 53 : tree t2
3178 53 : = fold_build2 (MULT_EXPR, TREE_TYPE (t),
3179 : fd->loops[i + 1 - l->outer].v, l->m1);
3180 53 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (t), t2, t);
3181 : }
3182 : }
3183 4324 : t = force_gimple_operand_gsi (&gsi, t,
3184 4324 : DECL_P (l->v)
3185 4324 : && TREE_ADDRESSABLE (l->v),
3186 : NULL_TREE, false,
3187 : GSI_CONTINUE_LINKING);
3188 4324 : stmt = gimple_build_assign (l->v, t);
3189 4324 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
3190 : }
3191 : }
3192 : else
3193 : collapse_bb = bb;
3194 :
3195 7107 : set_immediate_dominator (CDI_DOMINATORS, bb, last_bb);
3196 :
3197 7107 : if (POINTER_TYPE_P (vtype))
3198 902 : t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step);
3199 : else
3200 6205 : t = fold_build2 (PLUS_EXPR, vtype, fd->loops[i].v, fd->loops[i].step);
3201 7107 : t = force_gimple_operand_gsi (&gsi, t,
3202 7107 : DECL_P (fd->loops[i].v)
3203 7107 : && TREE_ADDRESSABLE (fd->loops[i].v),
3204 : NULL_TREE, false, GSI_CONTINUE_LINKING);
3205 7107 : stmt = gimple_build_assign (fd->loops[i].v, t);
3206 7107 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
3207 :
3208 7107 : if (fd->loops[i].non_rect_referenced)
3209 : {
3210 308 : basic_block update_bb = NULL, prev_bb = NULL;
3211 714 : for (int j = i + 1; j <= fd->last_nonrect; j++)
3212 406 : if (j - fd->loops[j].outer == i)
3213 : {
3214 308 : tree n1, n2;
3215 308 : struct omp_for_data_loop *l = &fd->loops[j];
3216 308 : basic_block this_bb = create_empty_bb (last_bb);
3217 308 : add_bb_to_loop (this_bb, last_bb->loop_father);
3218 308 : gimple_stmt_iterator gsi2 = gsi_start_bb (this_bb);
3219 308 : if (prev_bb)
3220 : {
3221 0 : e = make_edge (prev_bb, this_bb, EDGE_TRUE_VALUE);
3222 0 : e->probability
3223 0 : = profile_probability::guessed_always ().apply_scale (7,
3224 : 8);
3225 0 : set_immediate_dominator (CDI_DOMINATORS, this_bb, prev_bb);
3226 : }
3227 308 : if (l->m1)
3228 : {
3229 264 : if (POINTER_TYPE_P (TREE_TYPE (l->v)))
3230 26 : t = fold_build_pointer_plus (fd->loops[i].v, l->n1);
3231 : else
3232 : {
3233 238 : t = fold_build2 (MULT_EXPR, TREE_TYPE (l->m1), l->m1,
3234 : fd->loops[i].v);
3235 238 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (l->v),
3236 : t, l->n1);
3237 : }
3238 264 : n1 = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
3239 : false,
3240 : GSI_CONTINUE_LINKING);
3241 264 : stmt = gimple_build_assign (l->v, n1);
3242 264 : gsi_insert_after (&gsi2, stmt, GSI_CONTINUE_LINKING);
3243 264 : n1 = l->v;
3244 : }
3245 : else
3246 44 : n1 = force_gimple_operand_gsi (&gsi2, l->n1, true,
3247 : NULL_TREE, false,
3248 : GSI_CONTINUE_LINKING);
3249 308 : if (l->m2)
3250 : {
3251 204 : if (POINTER_TYPE_P (TREE_TYPE (l->v)))
3252 30 : t = fold_build_pointer_plus (fd->loops[i].v, l->n2);
3253 : else
3254 : {
3255 174 : t = fold_build2 (MULT_EXPR, TREE_TYPE (l->m2), l->m2,
3256 : fd->loops[i].v);
3257 174 : t = fold_build2 (PLUS_EXPR,
3258 : TREE_TYPE (nonrect_bounds[j]),
3259 : t, unshare_expr (l->n2));
3260 : }
3261 204 : n2 = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
3262 : false,
3263 : GSI_CONTINUE_LINKING);
3264 204 : stmt = gimple_build_assign (nonrect_bounds[j], n2);
3265 204 : gsi_insert_after (&gsi2, stmt, GSI_CONTINUE_LINKING);
3266 204 : n2 = nonrect_bounds[j];
3267 : }
3268 : else
3269 104 : n2 = force_gimple_operand_gsi (&gsi2, unshare_expr (l->n2),
3270 : true, NULL_TREE, false,
3271 : GSI_CONTINUE_LINKING);
3272 308 : gcond *cond_stmt
3273 308 : = gimple_build_cond (l->cond_code, n1, n2,
3274 : NULL_TREE, NULL_TREE);
3275 308 : gsi_insert_after (&gsi2, cond_stmt, GSI_CONTINUE_LINKING);
3276 308 : if (update_bb == NULL)
3277 308 : update_bb = this_bb;
3278 308 : e = make_edge (this_bb, bb, EDGE_FALSE_VALUE);
3279 308 : e->probability = profile_probability::guessed_always () / 8;
3280 308 : if (prev_bb == NULL)
3281 308 : set_immediate_dominator (CDI_DOMINATORS, this_bb, bb);
3282 308 : prev_bb = this_bb;
3283 : }
3284 308 : e = make_edge (prev_bb, body_bb, EDGE_TRUE_VALUE);
3285 308 : e->probability
3286 308 : = profile_probability::guessed_always ().apply_scale (7, 8);
3287 308 : body_bb = update_bb;
3288 : }
3289 :
3290 7107 : if (i > 0)
3291 : {
3292 4532 : if (fd->loops[i].m2)
3293 204 : t = nonrect_bounds[i];
3294 : else
3295 4328 : t = unshare_expr (fd->loops[i].n2);
3296 4532 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
3297 : false, GSI_CONTINUE_LINKING);
3298 4532 : tree v = fd->loops[i].v;
3299 4532 : if (DECL_P (v) && TREE_ADDRESSABLE (v))
3300 0 : v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE,
3301 : false, GSI_CONTINUE_LINKING);
3302 4532 : t = build2 (fd->loops[i].cond_code, boolean_type_node, v, t);
3303 4532 : stmt = gimple_build_cond_empty (t);
3304 4532 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
3305 4532 : if (walk_tree (gimple_cond_lhs_ptr (as_a <gcond *> (stmt)),
3306 : expand_omp_regimplify_p, NULL, NULL)
3307 4532 : || walk_tree (gimple_cond_rhs_ptr (as_a <gcond *> (stmt)),
3308 : expand_omp_regimplify_p, NULL, NULL))
3309 4 : gimple_regimplify_operands (stmt, &gsi);
3310 4532 : e = make_edge (bb, body_bb, EDGE_TRUE_VALUE);
3311 4532 : e->probability = profile_probability::guessed_always ().apply_scale (7, 8);
3312 : }
3313 : else
3314 2575 : make_edge (bb, body_bb, EDGE_FALLTHRU);
3315 7107 : set_immediate_dominator (CDI_DOMINATORS, bb, last_bb);
3316 7107 : last_bb = bb;
3317 : }
3318 :
3319 2575 : return collapse_bb;
3320 : }
3321 :
3322 : /* Expand #pragma omp ordered depend(source). */
3323 :
3324 : static void
3325 335 : expand_omp_ordered_source (gimple_stmt_iterator *gsi, struct omp_for_data *fd,
3326 : tree *counts, location_t loc)
3327 : {
3328 56 : enum built_in_function source_ix
3329 335 : = fd->iter_type == long_integer_type_node
3330 335 : ? BUILT_IN_GOMP_DOACROSS_POST : BUILT_IN_GOMP_DOACROSS_ULL_POST;
3331 335 : gimple *g
3332 335 : = gimple_build_call (builtin_decl_explicit (source_ix), 1,
3333 335 : build_fold_addr_expr (counts[fd->ordered]));
3334 335 : gimple_set_location (g, loc);
3335 335 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
3336 335 : }
3337 :
3338 : /* Expand a single depend from #pragma omp ordered depend(sink:...). */
3339 :
3340 : static void
3341 466 : expand_omp_ordered_sink (gimple_stmt_iterator *gsi, struct omp_for_data *fd,
3342 : tree *counts, tree c, location_t loc,
3343 : basic_block cont_bb)
3344 : {
3345 466 : auto_vec<tree, 10> args;
3346 73 : enum built_in_function sink_ix
3347 466 : = fd->iter_type == long_integer_type_node
3348 466 : ? BUILT_IN_GOMP_DOACROSS_WAIT : BUILT_IN_GOMP_DOACROSS_ULL_WAIT;
3349 466 : tree t, off, coff = NULL_TREE, deps = OMP_CLAUSE_DECL (c), cond = NULL_TREE;
3350 466 : int i;
3351 466 : gimple_stmt_iterator gsi2 = *gsi;
3352 466 : bool warned_step = false;
3353 :
3354 466 : if (deps == NULL)
3355 : {
3356 : /* Handle doacross(sink: omp_cur_iteration - 1). */
3357 52 : gsi_prev (&gsi2);
3358 52 : edge e1 = split_block (gsi_bb (gsi2), gsi_stmt (gsi2));
3359 52 : edge e2 = split_block_after_labels (e1->dest);
3360 52 : gsi2 = gsi_after_labels (e1->dest);
3361 52 : *gsi = gsi_last_bb (e1->src);
3362 52 : gimple_stmt_iterator gsi3 = *gsi;
3363 :
3364 52 : if (counts[fd->collapse - 1])
3365 : {
3366 8 : gcc_assert (fd->collapse == 1);
3367 8 : t = counts[fd->collapse - 1];
3368 : }
3369 44 : else if (fd->collapse > 1)
3370 28 : t = fd->loop.v;
3371 : else
3372 : {
3373 16 : t = fold_build2 (MINUS_EXPR, TREE_TYPE (fd->loops[0].v),
3374 : fd->loops[0].v, fd->loops[0].n1);
3375 16 : t = fold_convert (fd->iter_type, t);
3376 : }
3377 :
3378 52 : t = force_gimple_operand_gsi (gsi, t, true, NULL_TREE,
3379 : false, GSI_CONTINUE_LINKING);
3380 52 : gsi_insert_after (gsi, gimple_build_cond (NE_EXPR, t,
3381 52 : build_zero_cst (TREE_TYPE (t)),
3382 : NULL_TREE, NULL_TREE),
3383 : GSI_NEW_STMT);
3384 :
3385 52 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (t), t,
3386 : build_minus_one_cst (TREE_TYPE (t)));
3387 52 : t = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
3388 : true, GSI_SAME_STMT);
3389 52 : args.safe_push (t);
3390 281 : for (i = fd->collapse; i < fd->ordered; i++)
3391 : {
3392 229 : t = counts[fd->ordered + 2 + (i - fd->collapse)];
3393 229 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (t), t,
3394 : build_minus_one_cst (TREE_TYPE (t)));
3395 229 : t = fold_convert (fd->iter_type, t);
3396 229 : t = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
3397 : true, GSI_SAME_STMT);
3398 229 : args.safe_push (t);
3399 : }
3400 :
3401 52 : gimple *g = gimple_build_call_vec (builtin_decl_explicit (sink_ix),
3402 : args);
3403 52 : gimple_set_location (g, loc);
3404 52 : gsi_insert_before (&gsi2, g, GSI_SAME_STMT);
3405 :
3406 52 : edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE);
3407 52 : e3->probability = profile_probability::guessed_always () / 8;
3408 52 : e1->probability = e3->probability.invert ();
3409 52 : e1->flags = EDGE_TRUE_VALUE;
3410 52 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
3411 :
3412 52 : if (fd->ordered > fd->collapse && cont_bb)
3413 : {
3414 33 : if (counts[fd->ordered + 1] == NULL_TREE)
3415 33 : counts[fd->ordered + 1]
3416 33 : = create_tmp_var (boolean_type_node, ".first");
3417 :
3418 33 : edge e4;
3419 33 : if (gsi_end_p (gsi3))
3420 25 : e4 = split_block_after_labels (e1->src);
3421 : else
3422 : {
3423 8 : gsi_prev (&gsi3);
3424 8 : e4 = split_block (gsi_bb (gsi3), gsi_stmt (gsi3));
3425 : }
3426 33 : gsi3 = gsi_last_bb (e4->src);
3427 :
3428 33 : gsi_insert_after (&gsi3,
3429 33 : gimple_build_cond (NE_EXPR,
3430 33 : counts[fd->ordered + 1],
3431 : boolean_false_node,
3432 : NULL_TREE, NULL_TREE),
3433 : GSI_NEW_STMT);
3434 :
3435 33 : edge e5 = make_edge (e4->src, e2->dest, EDGE_FALSE_VALUE);
3436 33 : e4->probability = profile_probability::guessed_always () / 8;
3437 33 : e5->probability = e4->probability.invert ();
3438 33 : e4->flags = EDGE_TRUE_VALUE;
3439 33 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e4->src);
3440 : }
3441 :
3442 52 : *gsi = gsi_after_labels (e2->dest);
3443 52 : return;
3444 : }
3445 504 : for (i = 0; i < fd->ordered; i++)
3446 : {
3447 502 : tree step = NULL_TREE;
3448 502 : off = TREE_PURPOSE (deps);
3449 502 : if (TREE_CODE (off) == TRUNC_DIV_EXPR)
3450 : {
3451 83 : step = TREE_OPERAND (off, 1);
3452 83 : off = TREE_OPERAND (off, 0);
3453 : }
3454 502 : if (!integer_zerop (off))
3455 : {
3456 412 : gcc_assert (fd->loops[i].cond_code == LT_EXPR
3457 : || fd->loops[i].cond_code == GT_EXPR);
3458 412 : bool forward = fd->loops[i].cond_code == LT_EXPR;
3459 412 : if (step)
3460 : {
3461 : /* Non-simple Fortran DO loops. If step is variable,
3462 : we don't know at compile even the direction, so can't
3463 : warn. */
3464 83 : if (TREE_CODE (step) != INTEGER_CST)
3465 : break;
3466 0 : forward = tree_int_cst_sgn (step) != -1;
3467 : }
3468 329 : if (forward ^ OMP_CLAUSE_DOACROSS_SINK_NEGATIVE (deps))
3469 17 : warning_at (loc, OPT_Wopenmp,
3470 : "%qs clause with %<sink%> modifier "
3471 : "waiting for lexically later iteration",
3472 17 : OMP_CLAUSE_DOACROSS_DEPEND (c)
3473 : ? "depend" : "doacross");
3474 : break;
3475 : }
3476 90 : deps = TREE_CHAIN (deps);
3477 : }
3478 : /* If all offsets corresponding to the collapsed loops are zero,
3479 : this depend clause can be ignored. FIXME: but there is still a
3480 : flush needed. We need to emit one __sync_synchronize () for it
3481 : though (perhaps conditionally)? Solve this together with the
3482 : conservative dependence folding optimization.
3483 : if (i >= fd->collapse)
3484 : return; */
3485 :
3486 414 : deps = OMP_CLAUSE_DECL (c);
3487 414 : gsi_prev (&gsi2);
3488 414 : edge e1 = split_block (gsi_bb (gsi2), gsi_stmt (gsi2));
3489 414 : edge e2 = split_block_after_labels (e1->dest);
3490 :
3491 414 : gsi2 = gsi_after_labels (e1->dest);
3492 414 : *gsi = gsi_last_bb (e1->src);
3493 2806 : for (i = 0; i < fd->ordered; i++)
3494 : {
3495 2392 : tree itype = TREE_TYPE (fd->loops[i].v);
3496 2392 : tree step = NULL_TREE;
3497 2392 : tree orig_off = NULL_TREE;
3498 2392 : if (POINTER_TYPE_P (itype))
3499 9 : itype = sizetype;
3500 2392 : if (i)
3501 1978 : deps = TREE_CHAIN (deps);
3502 2392 : off = TREE_PURPOSE (deps);
3503 2392 : if (TREE_CODE (off) == TRUNC_DIV_EXPR)
3504 : {
3505 204 : step = TREE_OPERAND (off, 1);
3506 204 : off = TREE_OPERAND (off, 0);
3507 204 : gcc_assert (fd->loops[i].cond_code == LT_EXPR
3508 : && integer_onep (fd->loops[i].step)
3509 : && !POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v)));
3510 : }
3511 2392 : tree s = fold_convert_loc (loc, itype, step ? step : fd->loops[i].step);
3512 2392 : if (step)
3513 : {
3514 204 : off = fold_convert_loc (loc, itype, off);
3515 204 : orig_off = off;
3516 204 : off = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, off, s);
3517 : }
3518 :
3519 2392 : if (integer_zerop (off))
3520 1516 : t = boolean_true_node;
3521 : else
3522 : {
3523 876 : tree a;
3524 876 : tree co = fold_convert_loc (loc, itype, off);
3525 876 : if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v)))
3526 : {
3527 9 : if (OMP_CLAUSE_DOACROSS_SINK_NEGATIVE (deps))
3528 0 : co = fold_build1_loc (loc, NEGATE_EXPR, itype, co);
3529 9 : a = fold_build2_loc (loc, POINTER_PLUS_EXPR,
3530 9 : TREE_TYPE (fd->loops[i].v), fd->loops[i].v,
3531 : co);
3532 : }
3533 867 : else if (OMP_CLAUSE_DOACROSS_SINK_NEGATIVE (deps))
3534 640 : a = fold_build2_loc (loc, MINUS_EXPR, TREE_TYPE (fd->loops[i].v),
3535 : fd->loops[i].v, co);
3536 : else
3537 227 : a = fold_build2_loc (loc, PLUS_EXPR, TREE_TYPE (fd->loops[i].v),
3538 : fd->loops[i].v, co);
3539 876 : if (step)
3540 : {
3541 204 : tree t1, t2;
3542 204 : if (OMP_CLAUSE_DOACROSS_SINK_NEGATIVE (deps))
3543 149 : t1 = fold_build2_loc (loc, GE_EXPR, boolean_type_node, a,
3544 149 : fd->loops[i].n1);
3545 : else
3546 55 : t1 = fold_build2_loc (loc, LT_EXPR, boolean_type_node, a,
3547 55 : fd->loops[i].n2);
3548 204 : if (OMP_CLAUSE_DOACROSS_SINK_NEGATIVE (deps))
3549 149 : t2 = fold_build2_loc (loc, LT_EXPR, boolean_type_node, a,
3550 149 : fd->loops[i].n2);
3551 : else
3552 55 : t2 = fold_build2_loc (loc, GE_EXPR, boolean_type_node, a,
3553 55 : fd->loops[i].n1);
3554 204 : t = fold_build2_loc (loc, LT_EXPR, boolean_type_node,
3555 204 : step, build_int_cst (TREE_TYPE (step), 0));
3556 204 : if (TREE_CODE (step) != INTEGER_CST)
3557 : {
3558 204 : t1 = unshare_expr (t1);
3559 204 : t1 = force_gimple_operand_gsi (gsi, t1, true, NULL_TREE,
3560 : false, GSI_CONTINUE_LINKING);
3561 204 : t2 = unshare_expr (t2);
3562 204 : t2 = force_gimple_operand_gsi (gsi, t2, true, NULL_TREE,
3563 : false, GSI_CONTINUE_LINKING);
3564 : }
3565 204 : t = fold_build3_loc (loc, COND_EXPR, boolean_type_node,
3566 : t, t2, t1);
3567 : }
3568 672 : else if (fd->loops[i].cond_code == LT_EXPR)
3569 : {
3570 520 : if (OMP_CLAUSE_DOACROSS_SINK_NEGATIVE (deps))
3571 433 : t = fold_build2_loc (loc, GE_EXPR, boolean_type_node, a,
3572 : fd->loops[i].n1);
3573 : else
3574 87 : t = fold_build2_loc (loc, LT_EXPR, boolean_type_node, a,
3575 : fd->loops[i].n2);
3576 : }
3577 152 : else if (OMP_CLAUSE_DOACROSS_SINK_NEGATIVE (deps))
3578 58 : t = fold_build2_loc (loc, GT_EXPR, boolean_type_node, a,
3579 : fd->loops[i].n2);
3580 : else
3581 94 : t = fold_build2_loc (loc, LE_EXPR, boolean_type_node, a,
3582 : fd->loops[i].n1);
3583 : }
3584 2392 : if (cond)
3585 1978 : cond = fold_build2_loc (loc, BIT_AND_EXPR, boolean_type_node, cond, t);
3586 : else
3587 414 : cond = t;
3588 :
3589 2392 : off = fold_convert_loc (loc, itype, off);
3590 :
3591 2392 : if (step
3592 4411 : || (fd->loops[i].cond_code == LT_EXPR
3593 2188 : ? !integer_onep (fd->loops[i].step)
3594 169 : : !integer_minus_onep (fd->loops[i].step)))
3595 : {
3596 382 : if (step == NULL_TREE
3597 178 : && TYPE_UNSIGNED (itype)
3598 393 : && fd->loops[i].cond_code == GT_EXPR)
3599 9 : t = fold_build2_loc (loc, TRUNC_MOD_EXPR, itype, off,
3600 : fold_build1_loc (loc, NEGATE_EXPR, itype,
3601 : s));
3602 : else
3603 542 : t = fold_build2_loc (loc, TRUNC_MOD_EXPR, itype,
3604 : orig_off ? orig_off : off, s);
3605 382 : t = fold_build2_loc (loc, EQ_EXPR, boolean_type_node, t,
3606 : build_int_cst (itype, 0));
3607 382 : if (integer_zerop (t) && !warned_step)
3608 : {
3609 5 : warning_at (loc, OPT_Wopenmp,
3610 : "%qs clause with %<sink%> modifier refers to "
3611 : "iteration never in the iteration space",
3612 5 : OMP_CLAUSE_DOACROSS_DEPEND (c)
3613 : ? "depend" : "doacross");
3614 5 : warned_step = true;
3615 : }
3616 382 : cond = fold_build2_loc (loc, BIT_AND_EXPR, boolean_type_node,
3617 : cond, t);
3618 : }
3619 :
3620 2392 : if (i <= fd->collapse - 1 && fd->collapse > 1)
3621 322 : t = fd->loop.v;
3622 2070 : else if (counts[i])
3623 166 : t = counts[i];
3624 : else
3625 : {
3626 1904 : t = fold_build2_loc (loc, MINUS_EXPR, TREE_TYPE (fd->loops[i].v),
3627 1904 : fd->loops[i].v, fd->loops[i].n1);
3628 1904 : t = fold_convert_loc (loc, fd->iter_type, t);
3629 : }
3630 2392 : if (step)
3631 : /* We have divided off by step already earlier. */;
3632 2188 : else if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR)
3633 9 : off = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, off,
3634 : fold_build1_loc (loc, NEGATE_EXPR, itype,
3635 : s));
3636 : else
3637 2179 : off = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, off, s);
3638 2392 : if (OMP_CLAUSE_DOACROSS_SINK_NEGATIVE (deps))
3639 640 : off = fold_build1_loc (loc, NEGATE_EXPR, itype, off);
3640 2392 : off = fold_convert_loc (loc, fd->iter_type, off);
3641 2392 : if (i <= fd->collapse - 1 && fd->collapse > 1)
3642 : {
3643 322 : if (i)
3644 161 : off = fold_build2_loc (loc, PLUS_EXPR, fd->iter_type, coff,
3645 : off);
3646 322 : if (i < fd->collapse - 1)
3647 : {
3648 322 : coff = fold_build2_loc (loc, MULT_EXPR, fd->iter_type, off,
3649 161 : counts[i]);
3650 161 : continue;
3651 : }
3652 : }
3653 2231 : off = unshare_expr (off);
3654 2231 : t = fold_build2_loc (loc, PLUS_EXPR, fd->iter_type, t, off);
3655 2231 : t = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
3656 : true, GSI_SAME_STMT);
3657 2231 : args.safe_push (t);
3658 : }
3659 414 : gimple *g = gimple_build_call_vec (builtin_decl_explicit (sink_ix), args);
3660 414 : gimple_set_location (g, loc);
3661 414 : gsi_insert_before (&gsi2, g, GSI_SAME_STMT);
3662 :
3663 414 : cond = unshare_expr (cond);
3664 414 : cond = force_gimple_operand_gsi (gsi, cond, true, NULL_TREE, false,
3665 : GSI_CONTINUE_LINKING);
3666 414 : gsi_insert_after (gsi, gimple_build_cond_empty (cond), GSI_NEW_STMT);
3667 414 : edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE);
3668 414 : e3->probability = profile_probability::guessed_always () / 8;
3669 414 : e1->probability = e3->probability.invert ();
3670 414 : e1->flags = EDGE_TRUE_VALUE;
3671 414 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
3672 :
3673 414 : *gsi = gsi_after_labels (e2->dest);
3674 466 : }
3675 :
3676 : /* Expand all #pragma omp ordered depend(source) and
3677 : #pragma omp ordered depend(sink:...) constructs in the current
3678 : #pragma omp for ordered(n) region. */
3679 :
3680 : static void
3681 335 : expand_omp_ordered_source_sink (struct omp_region *region,
3682 : struct omp_for_data *fd, tree *counts,
3683 : basic_block cont_bb)
3684 : {
3685 335 : struct omp_region *inner;
3686 335 : int i;
3687 1563 : for (i = fd->collapse - 1; i < fd->ordered; i++)
3688 1228 : if (i == fd->collapse - 1 && fd->collapse > 1)
3689 147 : counts[i] = NULL_TREE;
3690 1081 : else if (i >= fd->collapse && !cont_bb)
3691 0 : counts[i] = build_zero_cst (fd->iter_type);
3692 2154 : else if (!POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v))
3693 2154 : && integer_onep (fd->loops[i].step))
3694 987 : counts[i] = NULL_TREE;
3695 : else
3696 94 : counts[i] = create_tmp_var (fd->iter_type, ".orditer");
3697 335 : tree atype
3698 335 : = build_array_type_nelts (fd->iter_type, fd->ordered - fd->collapse + 1);
3699 335 : counts[fd->ordered] = create_tmp_var (atype, ".orditera");
3700 335 : TREE_ADDRESSABLE (counts[fd->ordered]) = 1;
3701 335 : counts[fd->ordered + 1] = NULL_TREE;
3702 :
3703 1615 : for (inner = region->inner; inner; inner = inner->next)
3704 1280 : if (inner->type == GIMPLE_OMP_ORDERED)
3705 : {
3706 709 : gomp_ordered *ord_stmt = inner->ord_stmt;
3707 709 : gimple_stmt_iterator gsi = gsi_for_stmt (ord_stmt);
3708 709 : location_t loc = gimple_location (ord_stmt);
3709 709 : tree c;
3710 1175 : for (c = gimple_omp_ordered_clauses (ord_stmt);
3711 1175 : c; c = OMP_CLAUSE_CHAIN (c))
3712 801 : if (OMP_CLAUSE_DOACROSS_KIND (c) == OMP_CLAUSE_DOACROSS_SOURCE)
3713 : break;
3714 709 : if (c)
3715 335 : expand_omp_ordered_source (&gsi, fd, counts, loc);
3716 1510 : for (c = gimple_omp_ordered_clauses (ord_stmt);
3717 1510 : c; c = OMP_CLAUSE_CHAIN (c))
3718 801 : if (OMP_CLAUSE_DOACROSS_KIND (c) == OMP_CLAUSE_DOACROSS_SINK)
3719 466 : expand_omp_ordered_sink (&gsi, fd, counts, c, loc, cont_bb);
3720 709 : gsi_remove (&gsi, true);
3721 : }
3722 335 : }
3723 :
3724 : /* Wrap the body into fd->ordered - fd->collapse loops that aren't
3725 : collapsed. */
3726 :
3727 : static basic_block
3728 335 : expand_omp_for_ordered_loops (struct omp_for_data *fd, tree *counts,
3729 : basic_block cont_bb, basic_block body_bb,
3730 : basic_block l0_bb, bool ordered_lastprivate)
3731 : {
3732 335 : if (fd->ordered == fd->collapse)
3733 : return cont_bb;
3734 :
3735 171 : if (!cont_bb)
3736 : {
3737 0 : gimple_stmt_iterator gsi = gsi_after_labels (body_bb);
3738 0 : for (int i = fd->collapse; i < fd->ordered; i++)
3739 : {
3740 0 : tree type = TREE_TYPE (fd->loops[i].v);
3741 0 : tree n1 = fold_convert (type, fd->loops[i].n1);
3742 0 : expand_omp_build_assign (&gsi, fd->loops[i].v, n1);
3743 0 : tree aref = build4 (ARRAY_REF, fd->iter_type, counts[fd->ordered],
3744 0 : size_int (i - fd->collapse + 1),
3745 : NULL_TREE, NULL_TREE);
3746 0 : expand_omp_build_assign (&gsi, aref, build_zero_cst (fd->iter_type));
3747 : }
3748 0 : return NULL;
3749 : }
3750 :
3751 1064 : for (int i = fd->ordered - 1; i >= fd->collapse; i--)
3752 : {
3753 893 : tree t, type = TREE_TYPE (fd->loops[i].v);
3754 893 : gimple_stmt_iterator gsi = gsi_after_labels (body_bb);
3755 893 : if (counts[fd->ordered + 1] && i == fd->collapse)
3756 33 : expand_omp_build_assign (&gsi, counts[fd->ordered + 1],
3757 : boolean_true_node);
3758 893 : expand_omp_build_assign (&gsi, fd->loops[i].v,
3759 893 : fold_convert (type, fd->loops[i].n1));
3760 893 : if (counts[i])
3761 57 : expand_omp_build_assign (&gsi, counts[i],
3762 : build_zero_cst (fd->iter_type));
3763 893 : tree aref = build4 (ARRAY_REF, fd->iter_type, counts[fd->ordered],
3764 893 : size_int (i - fd->collapse + 1),
3765 : NULL_TREE, NULL_TREE);
3766 893 : expand_omp_build_assign (&gsi, aref, build_zero_cst (fd->iter_type));
3767 893 : if (!gsi_end_p (gsi))
3768 893 : gsi_prev (&gsi);
3769 : else
3770 0 : gsi = gsi_last_bb (body_bb);
3771 893 : edge e1 = split_block (body_bb, gsi_stmt (gsi));
3772 893 : basic_block new_body = e1->dest;
3773 893 : if (body_bb == cont_bb)
3774 0 : cont_bb = new_body;
3775 893 : edge e2 = NULL;
3776 893 : basic_block new_header;
3777 893 : if (EDGE_COUNT (cont_bb->preds) > 0)
3778 : {
3779 864 : gsi = gsi_last_bb (cont_bb);
3780 864 : if (POINTER_TYPE_P (type))
3781 0 : t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step);
3782 : else
3783 864 : t = fold_build2 (PLUS_EXPR, type, fd->loops[i].v,
3784 : fold_convert (type, fd->loops[i].step));
3785 864 : expand_omp_build_assign (&gsi, fd->loops[i].v, t);
3786 864 : if (counts[i])
3787 : {
3788 57 : t = fold_build2 (PLUS_EXPR, fd->iter_type, counts[i],
3789 : build_int_cst (fd->iter_type, 1));
3790 57 : expand_omp_build_assign (&gsi, counts[i], t);
3791 57 : t = counts[i];
3792 : }
3793 : else
3794 : {
3795 807 : t = fold_build2 (MINUS_EXPR, TREE_TYPE (fd->loops[i].v),
3796 : fd->loops[i].v, fd->loops[i].n1);
3797 807 : t = fold_convert (fd->iter_type, t);
3798 807 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
3799 : true, GSI_SAME_STMT);
3800 : }
3801 864 : aref = build4 (ARRAY_REF, fd->iter_type, counts[fd->ordered],
3802 864 : size_int (i - fd->collapse + 1),
3803 : NULL_TREE, NULL_TREE);
3804 864 : expand_omp_build_assign (&gsi, aref, t);
3805 864 : if (counts[fd->ordered + 1] && i == fd->ordered - 1)
3806 30 : expand_omp_build_assign (&gsi, counts[fd->ordered + 1],
3807 : boolean_false_node);
3808 864 : gsi_prev (&gsi);
3809 864 : e2 = split_block (cont_bb, gsi_stmt (gsi));
3810 864 : new_header = e2->dest;
3811 : }
3812 : else
3813 : new_header = cont_bb;
3814 893 : gsi = gsi_after_labels (new_header);
3815 893 : tree v = force_gimple_operand_gsi (&gsi, fd->loops[i].v, true, NULL_TREE,
3816 : true, GSI_SAME_STMT);
3817 893 : tree n2
3818 893 : = force_gimple_operand_gsi (&gsi, fold_convert (type, fd->loops[i].n2),
3819 : true, NULL_TREE, true, GSI_SAME_STMT);
3820 893 : t = build2 (fd->loops[i].cond_code, boolean_type_node, v, n2);
3821 893 : gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_NEW_STMT);
3822 893 : edge e3 = split_block (new_header, gsi_stmt (gsi));
3823 893 : cont_bb = e3->dest;
3824 893 : remove_edge (e1);
3825 893 : make_edge (body_bb, new_header, EDGE_FALLTHRU);
3826 893 : e3->flags = EDGE_FALSE_VALUE;
3827 893 : e3->probability = profile_probability::guessed_always () / 8;
3828 893 : e1 = make_edge (new_header, new_body, EDGE_TRUE_VALUE);
3829 893 : e1->probability = e3->probability.invert ();
3830 :
3831 893 : set_immediate_dominator (CDI_DOMINATORS, new_header, body_bb);
3832 893 : set_immediate_dominator (CDI_DOMINATORS, new_body, new_header);
3833 :
3834 893 : if (e2)
3835 : {
3836 864 : class loop *loop = alloc_loop ();
3837 864 : loop->header = new_header;
3838 864 : loop->latch = e2->src;
3839 864 : add_loop (loop, l0_bb->loop_father);
3840 : }
3841 : }
3842 :
3843 : /* If there are any lastprivate clauses and it is possible some loops
3844 : might have zero iterations, ensure all the decls are initialized,
3845 : otherwise we could crash evaluating C++ class iterators with lastprivate
3846 : clauses. */
3847 : bool need_inits = false;
3848 171 : for (int i = fd->collapse; ordered_lastprivate && i < fd->ordered; i++)
3849 0 : if (need_inits)
3850 : {
3851 0 : tree type = TREE_TYPE (fd->loops[i].v);
3852 0 : gimple_stmt_iterator gsi = gsi_after_labels (body_bb);
3853 0 : expand_omp_build_assign (&gsi, fd->loops[i].v,
3854 : fold_convert (type, fd->loops[i].n1));
3855 : }
3856 : else
3857 : {
3858 0 : tree type = TREE_TYPE (fd->loops[i].v);
3859 0 : tree this_cond = fold_build2 (fd->loops[i].cond_code,
3860 : boolean_type_node,
3861 : fold_convert (type, fd->loops[i].n1),
3862 : fold_convert (type, fd->loops[i].n2));
3863 0 : if (!integer_onep (this_cond))
3864 0 : need_inits = true;
3865 : }
3866 :
3867 : return cont_bb;
3868 : }
3869 :
3870 : /* A subroutine of expand_omp_for. Generate code for a parallel
3871 : loop with any schedule. Given parameters:
3872 :
3873 : for (V = N1; V cond N2; V += STEP) BODY;
3874 :
3875 : where COND is "<" or ">", we generate pseudocode
3876 :
3877 : more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
3878 : if (more) goto L0; else goto L3;
3879 : L0:
3880 : V = istart0;
3881 : iend = iend0;
3882 : L1:
3883 : BODY;
3884 : V += STEP;
3885 : if (V cond iend) goto L1; else goto L2;
3886 : L2:
3887 : if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
3888 : L3:
3889 :
3890 : If this is a combined omp parallel loop, instead of the call to
3891 : GOMP_loop_foo_start, we call GOMP_loop_foo_next.
3892 : If this is gimple_omp_for_combined_p loop, then instead of assigning
3893 : V and iend in L0 we assign the first two _looptemp_ clause decls of the
3894 : inner GIMPLE_OMP_FOR and V += STEP; and
3895 : if (V cond iend) goto L1; else goto L2; are removed.
3896 :
3897 : For collapsed loops, given parameters:
3898 : collapse(3)
3899 : for (V1 = N11; V1 cond1 N12; V1 += STEP1)
3900 : for (V2 = N21; V2 cond2 N22; V2 += STEP2)
3901 : for (V3 = N31; V3 cond3 N32; V3 += STEP3)
3902 : BODY;
3903 :
3904 : we generate pseudocode
3905 :
3906 : if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
3907 : if (cond3 is <)
3908 : adj = STEP3 - 1;
3909 : else
3910 : adj = STEP3 + 1;
3911 : count3 = (adj + N32 - N31) / STEP3;
3912 : if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
3913 : if (cond2 is <)
3914 : adj = STEP2 - 1;
3915 : else
3916 : adj = STEP2 + 1;
3917 : count2 = (adj + N22 - N21) / STEP2;
3918 : if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
3919 : if (cond1 is <)
3920 : adj = STEP1 - 1;
3921 : else
3922 : adj = STEP1 + 1;
3923 : count1 = (adj + N12 - N11) / STEP1;
3924 : count = count1 * count2 * count3;
3925 : goto Z1;
3926 : Z0:
3927 : count = 0;
3928 : Z1:
3929 : more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
3930 : if (more) goto L0; else goto L3;
3931 : L0:
3932 : V = istart0;
3933 : T = V;
3934 : V3 = N31 + (T % count3) * STEP3;
3935 : T = T / count3;
3936 : V2 = N21 + (T % count2) * STEP2;
3937 : T = T / count2;
3938 : V1 = N11 + T * STEP1;
3939 : iend = iend0;
3940 : L1:
3941 : BODY;
3942 : V += 1;
3943 : if (V < iend) goto L10; else goto L2;
3944 : L10:
3945 : V3 += STEP3;
3946 : if (V3 cond3 N32) goto L1; else goto L11;
3947 : L11:
3948 : V3 = N31;
3949 : V2 += STEP2;
3950 : if (V2 cond2 N22) goto L1; else goto L12;
3951 : L12:
3952 : V2 = N21;
3953 : V1 += STEP1;
3954 : goto L1;
3955 : L2:
3956 : if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
3957 : L3:
3958 :
3959 : */
3960 :
3961 : static void
3962 4128 : expand_omp_for_generic (struct omp_region *region,
3963 : struct omp_for_data *fd,
3964 : enum built_in_function start_fn,
3965 : enum built_in_function next_fn,
3966 : tree sched_arg,
3967 : gimple *inner_stmt)
3968 : {
3969 4128 : tree type, istart0, iend0, iend;
3970 4128 : tree t, vmain, vback, bias = NULL_TREE;
3971 4128 : basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, collapse_bb;
3972 4128 : basic_block l2_bb = NULL, l3_bb = NULL;
3973 4128 : gimple_stmt_iterator gsi;
3974 4128 : gassign *assign_stmt;
3975 4128 : bool in_combined_parallel = is_combined_parallel (region);
3976 4128 : bool broken_loop = region->cont == NULL;
3977 4128 : edge e, ne;
3978 4128 : tree *counts = NULL;
3979 4128 : int i;
3980 4128 : bool ordered_lastprivate = false;
3981 4128 : bool offload = is_in_offload_region (region);
3982 :
3983 4128 : gcc_assert (!broken_loop || !in_combined_parallel);
3984 4128 : gcc_assert (fd->iter_type == long_integer_type_node
3985 : || !in_combined_parallel);
3986 :
3987 4128 : entry_bb = region->entry;
3988 4128 : cont_bb = region->cont;
3989 4128 : collapse_bb = NULL;
3990 4128 : gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
3991 4128 : gcc_assert (broken_loop
3992 : || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
3993 4128 : l0_bb = split_edge (FALLTHRU_EDGE (entry_bb));
3994 4128 : l1_bb = single_succ (l0_bb);
3995 4128 : if (!broken_loop)
3996 : {
3997 3721 : l2_bb = create_empty_bb (cont_bb);
3998 3721 : gcc_assert (BRANCH_EDGE (cont_bb)->dest == l1_bb
3999 : || (single_succ_edge (BRANCH_EDGE (cont_bb)->dest)->dest
4000 : == l1_bb));
4001 3721 : gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
4002 : }
4003 : else
4004 : l2_bb = NULL;
4005 4128 : l3_bb = BRANCH_EDGE (entry_bb)->dest;
4006 4128 : exit_bb = region->exit;
4007 :
4008 4128 : gsi = gsi_last_nondebug_bb (entry_bb);
4009 :
4010 4128 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
4011 4128 : if (fd->ordered
4012 4128 : && omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
4013 : OMP_CLAUSE_LASTPRIVATE))
4014 4128 : ordered_lastprivate = false;
4015 4128 : tree reductions = NULL_TREE;
4016 4128 : tree mem = NULL_TREE, cond_var = NULL_TREE, condtemp = NULL_TREE;
4017 4128 : tree memv = NULL_TREE;
4018 4128 : if (fd->lastprivate_conditional)
4019 : {
4020 46 : tree c = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
4021 : OMP_CLAUSE__CONDTEMP_);
4022 46 : if (fd->have_pointer_condtemp)
4023 28 : condtemp = OMP_CLAUSE_DECL (c);
4024 46 : c = omp_find_clause (OMP_CLAUSE_CHAIN (c), OMP_CLAUSE__CONDTEMP_);
4025 46 : cond_var = OMP_CLAUSE_DECL (c);
4026 : }
4027 4128 : if (sched_arg)
4028 : {
4029 169 : if (fd->have_reductemp)
4030 : {
4031 149 : tree c = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
4032 : OMP_CLAUSE__REDUCTEMP_);
4033 149 : reductions = OMP_CLAUSE_DECL (c);
4034 149 : gcc_assert (TREE_CODE (reductions) == SSA_NAME);
4035 149 : gimple *g = SSA_NAME_DEF_STMT (reductions);
4036 149 : reductions = gimple_assign_rhs1 (g);
4037 149 : OMP_CLAUSE_DECL (c) = reductions;
4038 149 : entry_bb = gimple_bb (g);
4039 149 : edge e = split_block (entry_bb, g);
4040 149 : if (region->entry == entry_bb)
4041 8 : region->entry = e->dest;
4042 298 : gsi = gsi_last_bb (entry_bb);
4043 : }
4044 : else
4045 20 : reductions = null_pointer_node;
4046 169 : if (fd->have_pointer_condtemp)
4047 : {
4048 28 : tree type = TREE_TYPE (condtemp);
4049 28 : memv = create_tmp_var (type);
4050 28 : TREE_ADDRESSABLE (memv) = 1;
4051 28 : unsigned HOST_WIDE_INT sz
4052 28 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (type)));
4053 28 : sz *= fd->lastprivate_conditional;
4054 28 : expand_omp_build_assign (&gsi, memv, build_int_cst (type, sz),
4055 : false);
4056 28 : mem = build_fold_addr_expr (memv);
4057 : }
4058 : else
4059 141 : mem = null_pointer_node;
4060 : }
4061 4128 : if (fd->collapse > 1 || fd->ordered)
4062 : {
4063 1663 : int first_zero_iter1 = -1, first_zero_iter2 = -1;
4064 1663 : basic_block zero_iter1_bb = NULL, zero_iter2_bb = NULL, l2_dom_bb = NULL;
4065 :
4066 1663 : counts = XALLOCAVEC (tree, fd->ordered
4067 : ? fd->ordered + 2
4068 : + (fd->ordered - fd->collapse)
4069 : : fd->collapse);
4070 1663 : expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
4071 : zero_iter1_bb, first_zero_iter1,
4072 : zero_iter2_bb, first_zero_iter2, l2_dom_bb);
4073 :
4074 1663 : if (zero_iter1_bb)
4075 : {
4076 : /* Some counts[i] vars might be uninitialized if
4077 : some loop has zero iterations. But the body shouldn't
4078 : be executed in that case, so just avoid uninit warnings. */
4079 1913 : for (i = first_zero_iter1;
4080 1913 : i < (fd->ordered ? fd->ordered : fd->collapse); i++)
4081 1502 : if (SSA_VAR_P (counts[i]))
4082 950 : suppress_warning (counts[i], OPT_Wuninitialized);
4083 411 : gsi_prev (&gsi);
4084 411 : e = split_block (entry_bb, gsi_stmt (gsi));
4085 411 : entry_bb = e->dest;
4086 411 : make_edge (zero_iter1_bb, entry_bb, EDGE_FALLTHRU);
4087 411 : gsi = gsi_last_nondebug_bb (entry_bb);
4088 411 : set_immediate_dominator (CDI_DOMINATORS, entry_bb,
4089 : get_immediate_dominator (CDI_DOMINATORS,
4090 : zero_iter1_bb));
4091 : }
4092 1663 : if (zero_iter2_bb)
4093 : {
4094 : /* Some counts[i] vars might be uninitialized if
4095 : some loop has zero iterations. But the body shouldn't
4096 : be executed in that case, so just avoid uninit warnings. */
4097 394 : for (i = first_zero_iter2; i < fd->ordered; i++)
4098 276 : if (SSA_VAR_P (counts[i]))
4099 216 : suppress_warning (counts[i], OPT_Wuninitialized);
4100 118 : if (zero_iter1_bb)
4101 103 : make_edge (zero_iter2_bb, entry_bb, EDGE_FALLTHRU);
4102 : else
4103 : {
4104 15 : gsi_prev (&gsi);
4105 15 : e = split_block (entry_bb, gsi_stmt (gsi));
4106 15 : entry_bb = e->dest;
4107 15 : make_edge (zero_iter2_bb, entry_bb, EDGE_FALLTHRU);
4108 15 : gsi = gsi_last_nondebug_bb (entry_bb);
4109 15 : set_immediate_dominator (CDI_DOMINATORS, entry_bb,
4110 : get_immediate_dominator
4111 : (CDI_DOMINATORS, zero_iter2_bb));
4112 : }
4113 : }
4114 1663 : if (fd->collapse == 1)
4115 : {
4116 188 : counts[0] = fd->loop.n2;
4117 188 : fd->loop = fd->loops[0];
4118 : }
4119 : }
4120 :
4121 4128 : type = TREE_TYPE (fd->loop.v);
4122 4128 : istart0 = create_tmp_var (fd->iter_type, ".istart0");
4123 4128 : iend0 = create_tmp_var (fd->iter_type, ".iend0");
4124 4128 : TREE_ADDRESSABLE (istart0) = 1;
4125 4128 : TREE_ADDRESSABLE (iend0) = 1;
4126 :
4127 : /* See if we need to bias by LLONG_MIN. */
4128 4128 : if (fd->iter_type == long_long_unsigned_type_node
4129 778 : && (TREE_CODE (type) == INTEGER_TYPE || TREE_CODE (type) == BITINT_TYPE)
4130 530 : && !TYPE_UNSIGNED (type)
4131 4128 : && fd->ordered == 0)
4132 : {
4133 0 : tree n1, n2;
4134 :
4135 0 : if (fd->loop.cond_code == LT_EXPR)
4136 : {
4137 0 : n1 = fd->loop.n1;
4138 0 : n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step);
4139 : }
4140 : else
4141 : {
4142 0 : n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step);
4143 0 : n2 = fd->loop.n1;
4144 : }
4145 0 : if (TREE_CODE (n1) != INTEGER_CST
4146 0 : || TREE_CODE (n2) != INTEGER_CST
4147 0 : || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0)))
4148 0 : bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type));
4149 : }
4150 :
4151 4128 : gimple_stmt_iterator gsif = gsi;
4152 4128 : gsi_prev (&gsif);
4153 :
4154 4128 : tree arr = NULL_TREE;
4155 4128 : if (in_combined_parallel)
4156 : {
4157 1084 : gcc_assert (fd->ordered == 0);
4158 : /* In a combined parallel loop, emit a call to
4159 : GOMP_loop_foo_next. */
4160 1084 : t = build_call_expr (builtin_decl_explicit (next_fn), 2,
4161 : build_fold_addr_expr (istart0),
4162 : build_fold_addr_expr (iend0));
4163 : }
4164 : else
4165 : {
4166 3044 : tree t0, t1, t2, t3, t4;
4167 : /* If this is not a combined parallel loop, emit a call to
4168 : GOMP_loop_foo_start in ENTRY_BB. */
4169 3044 : t4 = build_fold_addr_expr (iend0);
4170 3044 : t3 = build_fold_addr_expr (istart0);
4171 3044 : if (fd->ordered)
4172 : {
4173 670 : t0 = build_int_cst (unsigned_type_node,
4174 335 : fd->ordered - fd->collapse + 1);
4175 335 : arr = create_tmp_var (build_array_type_nelts (fd->iter_type,
4176 335 : fd->ordered
4177 335 : - fd->collapse + 1),
4178 : ".omp_counts");
4179 335 : DECL_NAMELESS (arr) = 1;
4180 335 : TREE_ADDRESSABLE (arr) = 1;
4181 335 : TREE_STATIC (arr) = 1;
4182 335 : vec<constructor_elt, va_gc> *v;
4183 335 : vec_alloc (v, fd->ordered - fd->collapse + 1);
4184 335 : int idx;
4185 :
4186 1563 : for (idx = 0; idx < fd->ordered - fd->collapse + 1; idx++)
4187 : {
4188 1228 : tree c;
4189 1228 : if (idx == 0 && fd->collapse > 1)
4190 147 : c = fd->loop.n2;
4191 : else
4192 1081 : c = counts[idx + fd->collapse - 1];
4193 1228 : tree purpose = size_int (idx);
4194 1228 : CONSTRUCTOR_APPEND_ELT (v, purpose, c);
4195 1228 : if (TREE_CODE (c) != INTEGER_CST)
4196 433 : TREE_STATIC (arr) = 0;
4197 : }
4198 :
4199 335 : DECL_INITIAL (arr) = build_constructor (TREE_TYPE (arr), v);
4200 335 : if (!TREE_STATIC (arr))
4201 232 : force_gimple_operand_gsi (&gsi, build1 (DECL_EXPR,
4202 : void_type_node, arr),
4203 : true, NULL_TREE, true, GSI_SAME_STMT);
4204 335 : t1 = build_fold_addr_expr (arr);
4205 335 : t2 = NULL_TREE;
4206 : }
4207 : else
4208 : {
4209 2709 : t2 = fold_convert (fd->iter_type, fd->loop.step);
4210 2709 : t1 = fd->loop.n2;
4211 2709 : t0 = fd->loop.n1;
4212 2709 : if (gimple_omp_for_combined_into_p (fd->for_stmt))
4213 : {
4214 1152 : tree innerc
4215 1152 : = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
4216 : OMP_CLAUSE__LOOPTEMP_);
4217 1152 : gcc_assert (innerc);
4218 1152 : t0 = OMP_CLAUSE_DECL (innerc);
4219 1152 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
4220 : OMP_CLAUSE__LOOPTEMP_);
4221 1152 : gcc_assert (innerc);
4222 1152 : t1 = OMP_CLAUSE_DECL (innerc);
4223 : }
4224 5306 : if (POINTER_TYPE_P (TREE_TYPE (t0))
4225 2709 : && TYPE_PRECISION (TREE_TYPE (t0))
4226 112 : != TYPE_PRECISION (fd->iter_type))
4227 : {
4228 : /* Avoid casting pointers to integer of a different size. */
4229 0 : tree itype = signed_type_for (type);
4230 0 : t1 = fold_convert (fd->iter_type, fold_convert (itype, t1));
4231 0 : t0 = fold_convert (fd->iter_type, fold_convert (itype, t0));
4232 : }
4233 : else
4234 : {
4235 2709 : t1 = fold_convert (fd->iter_type, t1);
4236 2709 : t0 = fold_convert (fd->iter_type, t0);
4237 : }
4238 2709 : if (bias)
4239 : {
4240 0 : t1 = fold_build2 (PLUS_EXPR, fd->iter_type, t1, bias);
4241 0 : t0 = fold_build2 (PLUS_EXPR, fd->iter_type, t0, bias);
4242 : }
4243 : }
4244 3044 : if (fd->iter_type == long_integer_type_node || fd->ordered)
4245 : {
4246 2322 : if (fd->chunk_size)
4247 : {
4248 1573 : t = fold_convert (fd->iter_type, fd->chunk_size);
4249 1573 : t = omp_adjust_chunk_size (t, fd->simd_schedule, offload);
4250 1573 : if (sched_arg)
4251 : {
4252 135 : if (fd->ordered)
4253 36 : t = build_call_expr (builtin_decl_explicit (start_fn),
4254 : 8, t0, t1, sched_arg, t, t3, t4,
4255 : reductions, mem);
4256 : else
4257 99 : t = build_call_expr (builtin_decl_explicit (start_fn),
4258 : 9, t0, t1, t2, sched_arg, t, t3, t4,
4259 : reductions, mem);
4260 : }
4261 1438 : else if (fd->ordered)
4262 299 : t = build_call_expr (builtin_decl_explicit (start_fn),
4263 : 5, t0, t1, t, t3, t4);
4264 : else
4265 1139 : t = build_call_expr (builtin_decl_explicit (start_fn),
4266 : 6, t0, t1, t2, t, t3, t4);
4267 : }
4268 749 : else if (fd->ordered)
4269 0 : t = build_call_expr (builtin_decl_explicit (start_fn),
4270 : 4, t0, t1, t3, t4);
4271 : else
4272 749 : t = build_call_expr (builtin_decl_explicit (start_fn),
4273 : 5, t0, t1, t2, t3, t4);
4274 : }
4275 : else
4276 : {
4277 722 : tree t5;
4278 722 : tree c_bool_type;
4279 722 : tree bfn_decl;
4280 :
4281 : /* The GOMP_loop_ull_*start functions have additional boolean
4282 : argument, true for < loops and false for > loops.
4283 : In Fortran, the C bool type can be different from
4284 : boolean_type_node. */
4285 722 : bfn_decl = builtin_decl_explicit (start_fn);
4286 722 : c_bool_type = TREE_TYPE (TREE_TYPE (bfn_decl));
4287 722 : t5 = build_int_cst (c_bool_type,
4288 942 : fd->loop.cond_code == LT_EXPR ? 1 : 0);
4289 722 : if (fd->chunk_size)
4290 : {
4291 390 : tree bfn_decl = builtin_decl_explicit (start_fn);
4292 390 : t = fold_convert (fd->iter_type, fd->chunk_size);
4293 390 : t = omp_adjust_chunk_size (t, fd->simd_schedule, offload);
4294 390 : if (sched_arg)
4295 34 : t = build_call_expr (bfn_decl, 10, t5, t0, t1, t2, sched_arg,
4296 : t, t3, t4, reductions, mem);
4297 : else
4298 356 : t = build_call_expr (bfn_decl, 7, t5, t0, t1, t2, t, t3, t4);
4299 : }
4300 : else
4301 332 : t = build_call_expr (builtin_decl_explicit (start_fn),
4302 : 6, t5, t0, t1, t2, t3, t4);
4303 : }
4304 : }
4305 4128 : if (TREE_TYPE (t) != boolean_type_node)
4306 0 : t = fold_build2 (NE_EXPR, boolean_type_node,
4307 : t, build_int_cst (TREE_TYPE (t), 0));
4308 4128 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4309 : true, GSI_SAME_STMT);
4310 4128 : if (arr && !TREE_STATIC (arr))
4311 : {
4312 232 : tree clobber = build_clobber (TREE_TYPE (arr));
4313 232 : gsi_insert_before (&gsi, gimple_build_assign (arr, clobber),
4314 : GSI_SAME_STMT);
4315 : }
4316 4128 : if (fd->have_pointer_condtemp)
4317 28 : expand_omp_build_assign (&gsi, condtemp, memv, false);
4318 4128 : if (fd->have_reductemp)
4319 : {
4320 149 : gimple *g = gsi_stmt (gsi);
4321 149 : gsi_remove (&gsi, true);
4322 149 : release_ssa_name (gimple_assign_lhs (g));
4323 :
4324 149 : entry_bb = region->entry;
4325 149 : gsi = gsi_last_nondebug_bb (entry_bb);
4326 :
4327 149 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
4328 : }
4329 4128 : gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
4330 :
4331 : /* Remove the GIMPLE_OMP_FOR statement. */
4332 4128 : gsi_remove (&gsi, true);
4333 :
4334 4128 : if (gsi_end_p (gsif))
4335 1203 : gsif = gsi_after_labels (gsi_bb (gsif));
4336 4128 : gsi_next (&gsif);
4337 :
4338 : /* Iteration setup for sequential loop goes in L0_BB. */
4339 4128 : tree startvar = fd->loop.v;
4340 4128 : tree endvar = NULL_TREE;
4341 :
4342 4128 : if (gimple_omp_for_combined_p (fd->for_stmt))
4343 : {
4344 1389 : gcc_assert (gimple_code (inner_stmt) == GIMPLE_OMP_FOR
4345 : && gimple_omp_for_kind (inner_stmt)
4346 : == GF_OMP_FOR_KIND_SIMD);
4347 1389 : tree innerc = omp_find_clause (gimple_omp_for_clauses (inner_stmt),
4348 : OMP_CLAUSE__LOOPTEMP_);
4349 1389 : gcc_assert (innerc);
4350 1389 : startvar = OMP_CLAUSE_DECL (innerc);
4351 1389 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
4352 : OMP_CLAUSE__LOOPTEMP_);
4353 1389 : gcc_assert (innerc);
4354 1389 : endvar = OMP_CLAUSE_DECL (innerc);
4355 : }
4356 :
4357 4128 : gsi = gsi_start_bb (l0_bb);
4358 4128 : t = istart0;
4359 4128 : if (fd->ordered && fd->collapse == 1)
4360 188 : t = fold_build2 (MULT_EXPR, fd->iter_type, t,
4361 : fold_convert (fd->iter_type, fd->loop.step));
4362 3940 : else if (bias)
4363 0 : t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
4364 4128 : if (fd->ordered && fd->collapse == 1)
4365 : {
4366 188 : if (POINTER_TYPE_P (TREE_TYPE (startvar)))
4367 8 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (startvar),
4368 : fd->loop.n1, fold_convert (sizetype, t));
4369 : else
4370 : {
4371 180 : t = fold_convert (TREE_TYPE (startvar), t);
4372 180 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (startvar),
4373 : fd->loop.n1, t);
4374 : }
4375 : }
4376 : else
4377 : {
4378 3940 : if (POINTER_TYPE_P (TREE_TYPE (startvar)))
4379 240 : t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t);
4380 3940 : t = fold_convert (TREE_TYPE (startvar), t);
4381 : }
4382 4128 : t = force_gimple_operand_gsi (&gsi, t,
4383 4128 : DECL_P (startvar)
4384 4128 : && TREE_ADDRESSABLE (startvar),
4385 : NULL_TREE, false, GSI_CONTINUE_LINKING);
4386 4128 : assign_stmt = gimple_build_assign (startvar, t);
4387 4128 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
4388 4128 : if (cond_var)
4389 : {
4390 46 : tree itype = TREE_TYPE (cond_var);
4391 : /* For lastprivate(conditional:) itervar, we need some iteration
4392 : counter that starts at unsigned non-zero and increases.
4393 : Prefer as few IVs as possible, so if we can use startvar
4394 : itself, use that, or startvar + constant (those would be
4395 : incremented with step), and as last resort use the s0 + 1
4396 : incremented by 1. */
4397 46 : if ((fd->ordered && fd->collapse == 1)
4398 46 : || bias
4399 46 : || POINTER_TYPE_P (type)
4400 46 : || TREE_CODE (fd->loop.n1) != INTEGER_CST
4401 40 : || fd->loop.cond_code != LT_EXPR)
4402 6 : t = fold_build2 (PLUS_EXPR, itype, fold_convert (itype, istart0),
4403 : build_int_cst (itype, 1));
4404 40 : else if (tree_int_cst_sgn (fd->loop.n1) == 1)
4405 9 : t = fold_convert (itype, t);
4406 : else
4407 : {
4408 31 : tree c = fold_convert (itype, fd->loop.n1);
4409 31 : c = fold_build2 (MINUS_EXPR, itype, build_int_cst (itype, 1), c);
4410 31 : t = fold_build2 (PLUS_EXPR, itype, fold_convert (itype, t), c);
4411 : }
4412 46 : t = force_gimple_operand_gsi (&gsi, t, false,
4413 : NULL_TREE, false, GSI_CONTINUE_LINKING);
4414 46 : assign_stmt = gimple_build_assign (cond_var, t);
4415 46 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
4416 : }
4417 :
4418 4128 : t = iend0;
4419 4128 : if (fd->ordered && fd->collapse == 1)
4420 188 : t = fold_build2 (MULT_EXPR, fd->iter_type, t,
4421 : fold_convert (fd->iter_type, fd->loop.step));
4422 3940 : else if (bias)
4423 0 : t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
4424 4128 : if (fd->ordered && fd->collapse == 1)
4425 : {
4426 188 : if (POINTER_TYPE_P (TREE_TYPE (startvar)))
4427 8 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (startvar),
4428 : fd->loop.n1, fold_convert (sizetype, t));
4429 : else
4430 : {
4431 180 : t = fold_convert (TREE_TYPE (startvar), t);
4432 180 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (startvar),
4433 : fd->loop.n1, t);
4434 : }
4435 : }
4436 : else
4437 : {
4438 3940 : if (POINTER_TYPE_P (TREE_TYPE (startvar)))
4439 240 : t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t);
4440 3940 : t = fold_convert (TREE_TYPE (startvar), t);
4441 : }
4442 4128 : iend = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4443 : false, GSI_CONTINUE_LINKING);
4444 4128 : if (endvar)
4445 : {
4446 1389 : assign_stmt = gimple_build_assign (endvar, iend);
4447 1389 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
4448 1389 : if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (iend)))
4449 857 : assign_stmt = gimple_build_assign (fd->loop.v, iend);
4450 : else
4451 532 : assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, iend);
4452 1389 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
4453 : }
4454 : /* Handle linear clause adjustments. */
4455 4128 : tree itercnt = NULL_TREE;
4456 4128 : if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_FOR)
4457 26943 : for (tree c = gimple_omp_for_clauses (fd->for_stmt);
4458 26943 : c; c = OMP_CLAUSE_CHAIN (c))
4459 22815 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
4460 22815 : && !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
4461 : {
4462 66 : tree d = OMP_CLAUSE_DECL (c);
4463 66 : tree t = d, a, dest;
4464 66 : if (omp_privatize_by_reference (t))
4465 2 : t = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), t);
4466 66 : tree type = TREE_TYPE (t);
4467 66 : if (POINTER_TYPE_P (type))
4468 0 : type = sizetype;
4469 66 : dest = unshare_expr (t);
4470 66 : tree v = create_tmp_var (TREE_TYPE (t), NULL);
4471 66 : expand_omp_build_assign (&gsif, v, t);
4472 66 : if (itercnt == NULL_TREE)
4473 : {
4474 66 : itercnt = startvar;
4475 66 : tree n1 = fd->loop.n1;
4476 66 : if (POINTER_TYPE_P (TREE_TYPE (itercnt)))
4477 : {
4478 0 : itercnt
4479 0 : = fold_convert (signed_type_for (TREE_TYPE (itercnt)),
4480 : itercnt);
4481 0 : n1 = fold_convert (TREE_TYPE (itercnt), n1);
4482 : }
4483 66 : itercnt = fold_build2 (MINUS_EXPR, TREE_TYPE (itercnt),
4484 : itercnt, n1);
4485 66 : itercnt = fold_build2 (EXACT_DIV_EXPR, TREE_TYPE (itercnt),
4486 : itercnt, fd->loop.step);
4487 66 : itercnt = force_gimple_operand_gsi (&gsi, itercnt, true,
4488 : NULL_TREE, false,
4489 : GSI_CONTINUE_LINKING);
4490 : }
4491 66 : a = fold_build2 (MULT_EXPR, type,
4492 : fold_convert (type, itercnt),
4493 : fold_convert (type, OMP_CLAUSE_LINEAR_STEP (c)));
4494 66 : t = fold_build2 (type == TREE_TYPE (t) ? PLUS_EXPR
4495 : : POINTER_PLUS_EXPR, TREE_TYPE (t), v, a);
4496 66 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4497 : false, GSI_CONTINUE_LINKING);
4498 66 : expand_omp_build_assign (&gsi, dest, t, true);
4499 : }
4500 4128 : if (fd->collapse > 1)
4501 1475 : expand_omp_for_init_vars (fd, &gsi, counts, NULL, inner_stmt, startvar);
4502 :
4503 4128 : if (fd->ordered)
4504 : {
4505 : /* Until now, counts array contained number of iterations or
4506 : variable containing it for ith loop. From now on, we usually need
4507 : those counts only for collapsed loops, and only for the 2nd
4508 : till the last collapsed one. Move those one element earlier,
4509 : we'll use counts[fd->collapse - 1] for the first source/sink
4510 : iteration counter and so on and counts[fd->ordered]
4511 : as the array holding the current counter values for
4512 : depend(source). For doacross(sink:omp_cur_iteration - 1) we need
4513 : the counts from fd->collapse to fd->ordered - 1; make a copy of
4514 : those to counts[fd->ordered + 2] and onwards.
4515 : counts[fd->ordered + 1] can be a flag whether it is the first
4516 : iteration with a new collapsed counter (used only if
4517 : fd->ordered > fd->collapse). */
4518 335 : if (fd->ordered > fd->collapse)
4519 171 : memcpy (counts + fd->ordered + 2, counts + fd->collapse,
4520 171 : (fd->ordered - fd->collapse) * sizeof (counts[0]));
4521 335 : if (fd->collapse > 1)
4522 147 : memmove (counts, counts + 1, (fd->collapse - 1) * sizeof (counts[0]));
4523 335 : if (broken_loop)
4524 : {
4525 : int i;
4526 41 : for (i = fd->collapse; i < fd->ordered; i++)
4527 : {
4528 33 : tree type = TREE_TYPE (fd->loops[i].v);
4529 33 : tree this_cond
4530 33 : = fold_build2 (fd->loops[i].cond_code, boolean_type_node,
4531 : fold_convert (type, fd->loops[i].n1),
4532 : fold_convert (type, fd->loops[i].n2));
4533 33 : if (!integer_onep (this_cond))
4534 : break;
4535 : }
4536 37 : if (i < fd->ordered)
4537 : {
4538 29 : if (entry_bb->loop_father != l0_bb->loop_father)
4539 : {
4540 4 : remove_bb_from_loops (l0_bb);
4541 4 : add_bb_to_loop (l0_bb, entry_bb->loop_father);
4542 4 : gcc_assert (single_succ (l0_bb) == l1_bb);
4543 : }
4544 29 : cont_bb
4545 29 : = create_empty_bb (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb);
4546 29 : add_bb_to_loop (cont_bb, l0_bb->loop_father);
4547 29 : gimple_stmt_iterator gsi = gsi_after_labels (cont_bb);
4548 29 : gimple *g = gimple_build_omp_continue (fd->loop.v, fd->loop.v);
4549 29 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
4550 29 : make_edge (cont_bb, l3_bb, EDGE_FALLTHRU);
4551 29 : make_edge (cont_bb, l1_bb, 0);
4552 29 : l2_bb = create_empty_bb (cont_bb);
4553 29 : broken_loop = false;
4554 : }
4555 : }
4556 335 : expand_omp_ordered_source_sink (region, fd, counts, cont_bb);
4557 335 : cont_bb = expand_omp_for_ordered_loops (fd, counts, cont_bb, l1_bb,
4558 : l0_bb, ordered_lastprivate);
4559 335 : if (counts[fd->collapse - 1])
4560 : {
4561 37 : gcc_assert (fd->collapse == 1);
4562 37 : gsi = gsi_last_bb (l0_bb);
4563 37 : expand_omp_build_assign (&gsi, counts[fd->collapse - 1],
4564 : istart0, true);
4565 37 : if (cont_bb)
4566 : {
4567 33 : gsi = gsi_last_bb (cont_bb);
4568 33 : t = fold_build2 (PLUS_EXPR, fd->iter_type,
4569 : counts[fd->collapse - 1],
4570 : build_int_cst (fd->iter_type, 1));
4571 33 : expand_omp_build_assign (&gsi, counts[fd->collapse - 1], t);
4572 66 : tree aref = build4 (ARRAY_REF, fd->iter_type,
4573 33 : counts[fd->ordered], size_zero_node,
4574 : NULL_TREE, NULL_TREE);
4575 33 : expand_omp_build_assign (&gsi, aref, counts[fd->collapse - 1]);
4576 : }
4577 37 : t = counts[fd->collapse - 1];
4578 : }
4579 298 : else if (fd->collapse > 1)
4580 147 : t = fd->loop.v;
4581 : else
4582 : {
4583 151 : t = fold_build2 (MINUS_EXPR, TREE_TYPE (fd->loops[0].v),
4584 : fd->loops[0].v, fd->loops[0].n1);
4585 151 : t = fold_convert (fd->iter_type, t);
4586 : }
4587 335 : gsi = gsi_last_bb (l0_bb);
4588 335 : tree aref = build4 (ARRAY_REF, fd->iter_type, counts[fd->ordered],
4589 : size_zero_node, NULL_TREE, NULL_TREE);
4590 335 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4591 : false, GSI_CONTINUE_LINKING);
4592 335 : expand_omp_build_assign (&gsi, aref, t, true);
4593 : }
4594 :
4595 4128 : if (!broken_loop)
4596 : {
4597 : /* Code to control the increment and predicate for the sequential
4598 : loop goes in the CONT_BB. */
4599 3750 : gsi = gsi_last_nondebug_bb (cont_bb);
4600 3750 : gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
4601 3750 : gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE);
4602 3750 : vmain = gimple_omp_continue_control_use (cont_stmt);
4603 3750 : vback = gimple_omp_continue_control_def (cont_stmt);
4604 :
4605 3750 : if (cond_var)
4606 : {
4607 46 : tree itype = TREE_TYPE (cond_var);
4608 46 : tree t2;
4609 46 : if ((fd->ordered && fd->collapse == 1)
4610 46 : || bias
4611 46 : || POINTER_TYPE_P (type)
4612 46 : || TREE_CODE (fd->loop.n1) != INTEGER_CST
4613 40 : || fd->loop.cond_code != LT_EXPR)
4614 6 : t2 = build_int_cst (itype, 1);
4615 : else
4616 40 : t2 = fold_convert (itype, fd->loop.step);
4617 46 : t2 = fold_build2 (PLUS_EXPR, itype, cond_var, t2);
4618 46 : t2 = force_gimple_operand_gsi (&gsi, t2, false,
4619 : NULL_TREE, true, GSI_SAME_STMT);
4620 46 : assign_stmt = gimple_build_assign (cond_var, t2);
4621 46 : gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
4622 : }
4623 :
4624 3750 : if (!gimple_omp_for_combined_p (fd->for_stmt))
4625 : {
4626 2361 : if (POINTER_TYPE_P (type))
4627 160 : t = fold_build_pointer_plus (vmain, fd->loop.step);
4628 : else
4629 2201 : t = fold_build2 (PLUS_EXPR, type, vmain, fd->loop.step);
4630 2361 : t = force_gimple_operand_gsi (&gsi, t,
4631 2361 : DECL_P (vback)
4632 2361 : && TREE_ADDRESSABLE (vback),
4633 : NULL_TREE, true, GSI_SAME_STMT);
4634 2361 : assign_stmt = gimple_build_assign (vback, t);
4635 2361 : gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
4636 :
4637 2361 : if (fd->ordered && counts[fd->collapse - 1] == NULL_TREE)
4638 : {
4639 294 : tree tem;
4640 294 : if (fd->collapse > 1)
4641 143 : tem = fd->loop.v;
4642 : else
4643 : {
4644 151 : tem = fold_build2 (MINUS_EXPR, TREE_TYPE (fd->loops[0].v),
4645 : fd->loops[0].v, fd->loops[0].n1);
4646 151 : tem = fold_convert (fd->iter_type, tem);
4647 : }
4648 588 : tree aref = build4 (ARRAY_REF, fd->iter_type,
4649 294 : counts[fd->ordered], size_zero_node,
4650 : NULL_TREE, NULL_TREE);
4651 294 : tem = force_gimple_operand_gsi (&gsi, tem, true, NULL_TREE,
4652 : true, GSI_SAME_STMT);
4653 294 : expand_omp_build_assign (&gsi, aref, tem);
4654 : }
4655 :
4656 2361 : t = build2 (fd->loop.cond_code, boolean_type_node,
4657 2361 : DECL_P (vback) && TREE_ADDRESSABLE (vback) ? t : vback,
4658 : iend);
4659 2361 : gcond *cond_stmt = gimple_build_cond_empty (t);
4660 2361 : gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
4661 : }
4662 :
4663 : /* Remove GIMPLE_OMP_CONTINUE. */
4664 3750 : gsi_remove (&gsi, true);
4665 :
4666 3750 : if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
4667 675 : collapse_bb = extract_omp_for_update_vars (fd, NULL, cont_bb, l1_bb);
4668 :
4669 : /* Emit code to get the next parallel iteration in L2_BB. */
4670 3750 : gsi = gsi_start_bb (l2_bb);
4671 :
4672 3750 : t = build_call_expr (builtin_decl_explicit (next_fn), 2,
4673 : build_fold_addr_expr (istart0),
4674 : build_fold_addr_expr (iend0));
4675 3750 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4676 : false, GSI_CONTINUE_LINKING);
4677 3750 : if (TREE_TYPE (t) != boolean_type_node)
4678 0 : t = fold_build2 (NE_EXPR, boolean_type_node,
4679 : t, build_int_cst (TREE_TYPE (t), 0));
4680 3750 : gcond *cond_stmt = gimple_build_cond_empty (t);
4681 3750 : gsi_insert_after (&gsi, cond_stmt, GSI_CONTINUE_LINKING);
4682 : }
4683 :
4684 : /* Add the loop cleanup function. */
4685 4128 : gsi = gsi_last_nondebug_bb (exit_bb);
4686 4128 : if (gimple_omp_return_nowait_p (gsi_stmt (gsi)))
4687 3047 : t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT);
4688 1081 : else if (gimple_omp_return_lhs (gsi_stmt (gsi)))
4689 2 : t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL);
4690 : else
4691 1079 : t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END);
4692 4128 : gcall *call_stmt = gimple_build_call (t, 0);
4693 4128 : if (fd->ordered)
4694 : {
4695 335 : tree arr = counts[fd->ordered];
4696 335 : tree clobber = build_clobber (TREE_TYPE (arr));
4697 335 : gsi_insert_after (&gsi, gimple_build_assign (arr, clobber),
4698 : GSI_SAME_STMT);
4699 : }
4700 4128 : if (gimple_omp_return_lhs (gsi_stmt (gsi)))
4701 : {
4702 2 : gimple_call_set_lhs (call_stmt, gimple_omp_return_lhs (gsi_stmt (gsi)));
4703 2 : if (fd->have_reductemp)
4704 : {
4705 0 : gimple *g = gimple_build_assign (reductions, NOP_EXPR,
4706 : gimple_call_lhs (call_stmt));
4707 0 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
4708 : }
4709 : }
4710 4128 : gsi_insert_after (&gsi, call_stmt, GSI_SAME_STMT);
4711 4128 : gsi_remove (&gsi, true);
4712 :
4713 : /* Connect the new blocks. */
4714 4128 : find_edge (entry_bb, l0_bb)->flags = EDGE_TRUE_VALUE;
4715 4128 : find_edge (entry_bb, l3_bb)->flags = EDGE_FALSE_VALUE;
4716 :
4717 4128 : if (!broken_loop)
4718 : {
4719 3750 : gimple_seq phis;
4720 :
4721 3750 : e = find_edge (cont_bb, l3_bb);
4722 3750 : ne = make_edge (l2_bb, l3_bb, EDGE_FALSE_VALUE);
4723 :
4724 3750 : phis = phi_nodes (l3_bb);
4725 3772 : for (gsi = gsi_start (phis); !gsi_end_p (gsi); gsi_next (&gsi))
4726 : {
4727 11 : gimple *phi = gsi_stmt (gsi);
4728 11 : SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, ne),
4729 : PHI_ARG_DEF_FROM_EDGE (phi, e));
4730 : }
4731 3750 : remove_edge (e);
4732 :
4733 3750 : make_edge (cont_bb, l2_bb, EDGE_FALSE_VALUE);
4734 3750 : e = find_edge (cont_bb, l1_bb);
4735 3750 : if (e == NULL)
4736 : {
4737 13 : e = BRANCH_EDGE (cont_bb);
4738 13 : gcc_assert (single_succ (e->dest) == l1_bb);
4739 : }
4740 3750 : if (gimple_omp_for_combined_p (fd->for_stmt))
4741 : {
4742 1389 : remove_edge (e);
4743 1389 : e = NULL;
4744 : }
4745 2361 : else if (fd->collapse > 1)
4746 : {
4747 675 : remove_edge (e);
4748 675 : e = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
4749 : }
4750 : else
4751 1686 : e->flags = EDGE_TRUE_VALUE;
4752 3750 : if (e)
4753 : {
4754 2361 : e->probability = profile_probability::guessed_always ().apply_scale (7, 8);
4755 2361 : find_edge (cont_bb, l2_bb)->probability = e->probability.invert ();
4756 : }
4757 : else
4758 : {
4759 1389 : e = find_edge (cont_bb, l2_bb);
4760 1389 : e->flags = EDGE_FALLTHRU;
4761 : }
4762 3750 : make_edge (l2_bb, l0_bb, EDGE_TRUE_VALUE);
4763 :
4764 3750 : if (gimple_in_ssa_p (cfun))
4765 : {
4766 : /* Add phis to the outer loop that connect to the phis in the inner,
4767 : original loop, and move the loop entry value of the inner phi to
4768 : the loop entry value of the outer phi. */
4769 13 : gphi_iterator psi;
4770 24 : for (psi = gsi_start_phis (l3_bb); !gsi_end_p (psi); gsi_next (&psi))
4771 : {
4772 11 : location_t locus;
4773 11 : gphi *nphi;
4774 11 : gphi *exit_phi = psi.phi ();
4775 :
4776 22 : if (virtual_operand_p (gimple_phi_result (exit_phi)))
4777 6 : continue;
4778 :
4779 5 : edge l2_to_l3 = find_edge (l2_bb, l3_bb);
4780 5 : tree exit_res = PHI_ARG_DEF_FROM_EDGE (exit_phi, l2_to_l3);
4781 :
4782 5 : basic_block latch = BRANCH_EDGE (cont_bb)->dest;
4783 5 : edge latch_to_l1 = find_edge (latch, l1_bb);
4784 5 : gphi *inner_phi
4785 5 : = find_phi_with_arg_on_edge (exit_res, latch_to_l1);
4786 :
4787 5 : tree t = gimple_phi_result (exit_phi);
4788 5 : tree new_res = copy_ssa_name (t, NULL);
4789 5 : nphi = create_phi_node (new_res, l0_bb);
4790 :
4791 5 : edge l0_to_l1 = find_edge (l0_bb, l1_bb);
4792 5 : t = PHI_ARG_DEF_FROM_EDGE (inner_phi, l0_to_l1);
4793 5 : locus = gimple_phi_arg_location_from_edge (inner_phi, l0_to_l1);
4794 5 : edge entry_to_l0 = find_edge (entry_bb, l0_bb);
4795 5 : add_phi_arg (nphi, t, entry_to_l0, locus);
4796 :
4797 5 : edge l2_to_l0 = find_edge (l2_bb, l0_bb);
4798 5 : add_phi_arg (nphi, exit_res, l2_to_l0, UNKNOWN_LOCATION);
4799 :
4800 5 : add_phi_arg (inner_phi, new_res, l0_to_l1, UNKNOWN_LOCATION);
4801 : }
4802 : }
4803 :
4804 3750 : set_immediate_dominator (CDI_DOMINATORS, l2_bb,
4805 : recompute_dominator (CDI_DOMINATORS, l2_bb));
4806 3750 : set_immediate_dominator (CDI_DOMINATORS, l3_bb,
4807 : recompute_dominator (CDI_DOMINATORS, l3_bb));
4808 3750 : set_immediate_dominator (CDI_DOMINATORS, l0_bb,
4809 : recompute_dominator (CDI_DOMINATORS, l0_bb));
4810 3750 : set_immediate_dominator (CDI_DOMINATORS, l1_bb,
4811 : recompute_dominator (CDI_DOMINATORS, l1_bb));
4812 :
4813 : /* We enter expand_omp_for_generic with a loop. This original loop may
4814 : have its own loop struct, or it may be part of an outer loop struct
4815 : (which may be the fake loop). */
4816 3750 : class loop *outer_loop = entry_bb->loop_father;
4817 3750 : bool orig_loop_has_loop_struct = l1_bb->loop_father != outer_loop;
4818 :
4819 3750 : add_bb_to_loop (l2_bb, outer_loop);
4820 :
4821 : /* We've added a new loop around the original loop. Allocate the
4822 : corresponding loop struct. */
4823 3750 : class loop *new_loop = alloc_loop ();
4824 3750 : new_loop->header = l0_bb;
4825 3750 : new_loop->latch = l2_bb;
4826 3750 : add_loop (new_loop, outer_loop);
4827 :
4828 : /* Allocate a loop structure for the original loop unless we already
4829 : had one. */
4830 3750 : if (!orig_loop_has_loop_struct
4831 3750 : && !gimple_omp_for_combined_p (fd->for_stmt))
4832 : {
4833 2344 : class loop *orig_loop = alloc_loop ();
4834 2344 : orig_loop->header = l1_bb;
4835 : /* The loop may have multiple latches. */
4836 2344 : add_loop (orig_loop, new_loop);
4837 : }
4838 : }
4839 4128 : }
4840 :
4841 : /* Helper function for expand_omp_for_static_nochunk. If PTR is NULL,
4842 : compute needed allocation size. If !ALLOC of team allocations,
4843 : if ALLOC of thread allocation. SZ is the initial needed size for
4844 : other purposes, ALLOC_ALIGN guaranteed alignment of allocation in bytes,
4845 : CNT number of elements of each array, for !ALLOC this is
4846 : omp_get_num_threads (), for ALLOC number of iterations handled by the
4847 : current thread. If PTR is non-NULL, it is the start of the allocation
4848 : and this routine shall assign to OMP_CLAUSE_DECL (c) of those _scantemp_
4849 : clauses pointers to the corresponding arrays. */
4850 :
4851 : static tree
4852 708 : expand_omp_scantemp_alloc (tree clauses, tree ptr, unsigned HOST_WIDE_INT sz,
4853 : unsigned HOST_WIDE_INT alloc_align, tree cnt,
4854 : gimple_stmt_iterator *gsi, bool alloc)
4855 : {
4856 708 : tree eltsz = NULL_TREE;
4857 708 : unsigned HOST_WIDE_INT preval = 0;
4858 708 : if (ptr && sz)
4859 5 : ptr = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (ptr),
4860 : ptr, size_int (sz));
4861 5832 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
4862 5124 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__SCANTEMP_
4863 3088 : && !OMP_CLAUSE__SCANTEMP__CONTROL (c)
4864 6796 : && (!OMP_CLAUSE__SCANTEMP__ALLOC (c)) != alloc)
4865 : {
4866 836 : tree pointee_type = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
4867 836 : unsigned HOST_WIDE_INT al = TYPE_ALIGN_UNIT (pointee_type);
4868 836 : if (tree_fits_uhwi_p (TYPE_SIZE_UNIT (pointee_type)))
4869 : {
4870 836 : unsigned HOST_WIDE_INT szl
4871 836 : = tree_to_uhwi (TYPE_SIZE_UNIT (pointee_type));
4872 836 : szl = least_bit_hwi (szl);
4873 836 : if (szl)
4874 836 : al = MIN (al, szl);
4875 : }
4876 836 : if (ptr == NULL_TREE)
4877 : {
4878 418 : if (eltsz == NULL_TREE)
4879 354 : eltsz = TYPE_SIZE_UNIT (pointee_type);
4880 : else
4881 64 : eltsz = size_binop (PLUS_EXPR, eltsz,
4882 : TYPE_SIZE_UNIT (pointee_type));
4883 : }
4884 836 : if (preval == 0 && al <= alloc_align)
4885 : {
4886 708 : unsigned HOST_WIDE_INT diff = ROUND_UP (sz, al) - sz;
4887 708 : sz += diff;
4888 708 : if (diff && ptr)
4889 0 : ptr = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (ptr),
4890 : ptr, size_int (diff));
4891 : }
4892 128 : else if (al > preval)
4893 : {
4894 32 : if (ptr)
4895 : {
4896 16 : ptr = fold_convert (pointer_sized_int_node, ptr);
4897 16 : ptr = fold_build2 (PLUS_EXPR, pointer_sized_int_node, ptr,
4898 : build_int_cst (pointer_sized_int_node,
4899 : al - 1));
4900 16 : ptr = fold_build2 (BIT_AND_EXPR, pointer_sized_int_node, ptr,
4901 : build_int_cst (pointer_sized_int_node,
4902 : -(HOST_WIDE_INT) al));
4903 16 : ptr = fold_convert (ptr_type_node, ptr);
4904 : }
4905 : else
4906 16 : sz += al - 1;
4907 : }
4908 836 : if (tree_fits_uhwi_p (TYPE_SIZE_UNIT (pointee_type)))
4909 : preval = al;
4910 : else
4911 0 : preval = 1;
4912 836 : if (ptr)
4913 : {
4914 418 : expand_omp_build_assign (gsi, OMP_CLAUSE_DECL (c), ptr, false);
4915 418 : ptr = OMP_CLAUSE_DECL (c);
4916 418 : ptr = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (ptr), ptr,
4917 : size_binop (MULT_EXPR, cnt,
4918 : TYPE_SIZE_UNIT (pointee_type)));
4919 : }
4920 : }
4921 :
4922 708 : if (ptr == NULL_TREE)
4923 : {
4924 354 : eltsz = size_binop (MULT_EXPR, eltsz, cnt);
4925 354 : if (sz)
4926 13 : eltsz = size_binop (PLUS_EXPR, eltsz, size_int (sz));
4927 354 : return eltsz;
4928 : }
4929 : else
4930 : return ptr;
4931 : }
4932 :
4933 : /* Return the last _looptemp_ clause if one has been created for
4934 : lastprivate on distribute parallel for{, simd} or taskloop.
4935 : FD is the loop data and INNERC should be the second _looptemp_
4936 : clause (the one holding the end of the range).
4937 : This is followed by collapse - 1 _looptemp_ clauses for the
4938 : counts[1] and up, and for triangular loops followed by 4
4939 : further _looptemp_ clauses (one for counts[0], one first_inner_iterations,
4940 : one factor and one adjn1). After this there is optionally one
4941 : _looptemp_ clause that this function returns. */
4942 :
4943 : static tree
4944 1634 : find_lastprivate_looptemp (struct omp_for_data *fd, tree innerc)
4945 : {
4946 1634 : gcc_assert (innerc);
4947 1634 : int count = fd->collapse - 1;
4948 1634 : if (fd->non_rect
4949 24 : && fd->last_nonrect == fd->first_nonrect + 1
4950 1646 : && !TYPE_UNSIGNED (TREE_TYPE (fd->loops[fd->last_nonrect].v)))
4951 12 : count += 4;
4952 4811 : for (int i = 0; i < count; i++)
4953 : {
4954 3177 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
4955 : OMP_CLAUSE__LOOPTEMP_);
4956 3177 : gcc_assert (innerc);
4957 : }
4958 1634 : return omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
4959 1634 : OMP_CLAUSE__LOOPTEMP_);
4960 : }
4961 :
4962 : /* A subroutine of expand_omp_for. Generate code for a parallel
4963 : loop with static schedule and no specified chunk size. Given
4964 : parameters:
4965 :
4966 : for (V = N1; V cond N2; V += STEP) BODY;
4967 :
4968 : where COND is "<" or ">", we generate pseudocode
4969 :
4970 : if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
4971 : if (cond is <)
4972 : adj = STEP - 1;
4973 : else
4974 : adj = STEP + 1;
4975 : if ((__typeof (V)) -1 > 0 && cond is >)
4976 : n = -(adj + N2 - N1) / -STEP;
4977 : else
4978 : n = (adj + N2 - N1) / STEP;
4979 : q = n / nthreads;
4980 : tt = n % nthreads;
4981 : if (threadid < tt) goto L3; else goto L4;
4982 : L3:
4983 : tt = 0;
4984 : q = q + 1;
4985 : L4:
4986 : s0 = q * threadid + tt;
4987 : e0 = s0 + q;
4988 : V = s0 * STEP + N1;
4989 : if (s0 >= e0) goto L2; else goto L0;
4990 : L0:
4991 : e = e0 * STEP + N1;
4992 : L1:
4993 : BODY;
4994 : V += STEP;
4995 : if (V cond e) goto L1;
4996 : L2:
4997 : */
4998 :
4999 : static void
5000 13893 : expand_omp_for_static_nochunk (struct omp_region *region,
5001 : struct omp_for_data *fd,
5002 : gimple *inner_stmt)
5003 : {
5004 13893 : tree n, q, s0, e0, e, t, tt, nthreads = NULL_TREE, threadid;
5005 13893 : tree type, itype, vmain, vback;
5006 13893 : basic_block entry_bb, second_bb, third_bb, exit_bb, seq_start_bb;
5007 13893 : basic_block body_bb, cont_bb, collapse_bb = NULL;
5008 13893 : basic_block fin_bb, fourth_bb = NULL, fifth_bb = NULL, sixth_bb = NULL;
5009 13893 : basic_block exit1_bb = NULL, exit2_bb = NULL, exit3_bb = NULL;
5010 13893 : gimple_stmt_iterator gsi, gsip;
5011 13893 : edge ep;
5012 13893 : bool broken_loop = region->cont == NULL;
5013 13893 : tree *counts = NULL;
5014 13893 : tree n1, n2, step;
5015 13893 : tree reductions = NULL_TREE;
5016 13893 : tree cond_var = NULL_TREE, condtemp = NULL_TREE;
5017 :
5018 13893 : itype = type = TREE_TYPE (fd->loop.v);
5019 13893 : if (POINTER_TYPE_P (type))
5020 606 : itype = signed_type_for (type);
5021 :
5022 13893 : entry_bb = region->entry;
5023 13893 : cont_bb = region->cont;
5024 13893 : gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
5025 13893 : fin_bb = BRANCH_EDGE (entry_bb)->dest;
5026 13893 : gcc_assert (broken_loop
5027 : || (fin_bb == FALLTHRU_EDGE (cont_bb)->dest));
5028 13893 : seq_start_bb = split_edge (FALLTHRU_EDGE (entry_bb));
5029 13893 : body_bb = single_succ (seq_start_bb);
5030 13893 : if (!broken_loop)
5031 : {
5032 13409 : gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb
5033 : || single_succ (BRANCH_EDGE (cont_bb)->dest) == body_bb);
5034 13409 : gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
5035 : }
5036 13893 : exit_bb = region->exit;
5037 :
5038 : /* Iteration space partitioning goes in ENTRY_BB. */
5039 13893 : gsi = gsi_last_nondebug_bb (entry_bb);
5040 13893 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5041 13893 : gsip = gsi;
5042 13893 : gsi_prev (&gsip);
5043 :
5044 13893 : if (fd->collapse > 1)
5045 : {
5046 3915 : int first_zero_iter = -1, dummy = -1;
5047 3915 : basic_block l2_dom_bb = NULL, dummy_bb = NULL;
5048 :
5049 3915 : counts = XALLOCAVEC (tree, fd->collapse);
5050 3915 : expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
5051 : fin_bb, first_zero_iter,
5052 : dummy_bb, dummy, l2_dom_bb);
5053 3915 : t = NULL_TREE;
5054 : }
5055 9978 : else if (gimple_omp_for_combined_into_p (fd->for_stmt))
5056 1998 : t = integer_one_node;
5057 : else
5058 7980 : t = fold_binary (fd->loop.cond_code, boolean_type_node,
5059 : fold_convert (type, fd->loop.n1),
5060 : fold_convert (type, fd->loop.n2));
5061 13893 : if (fd->collapse == 1
5062 9978 : && TYPE_UNSIGNED (type)
5063 15906 : && (t == NULL_TREE || !integer_onep (t)))
5064 : {
5065 523 : n1 = fold_convert (type, unshare_expr (fd->loop.n1));
5066 523 : n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE,
5067 : true, GSI_SAME_STMT);
5068 523 : n2 = fold_convert (type, unshare_expr (fd->loop.n2));
5069 523 : n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
5070 : true, GSI_SAME_STMT);
5071 523 : gcond *cond_stmt = expand_omp_build_cond (&gsi, fd->loop.cond_code,
5072 : n1, n2);
5073 523 : ep = split_block (entry_bb, cond_stmt);
5074 523 : ep->flags = EDGE_TRUE_VALUE;
5075 523 : entry_bb = ep->dest;
5076 523 : ep->probability = profile_probability::very_likely ();
5077 523 : ep = make_edge (ep->src, fin_bb, EDGE_FALSE_VALUE);
5078 523 : ep->probability = profile_probability::very_unlikely ();
5079 523 : if (gimple_in_ssa_p (cfun))
5080 : {
5081 22 : int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx;
5082 22 : for (gphi_iterator gpi = gsi_start_phis (fin_bb);
5083 44 : !gsi_end_p (gpi); gsi_next (&gpi))
5084 : {
5085 22 : gphi *phi = gpi.phi ();
5086 22 : add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
5087 : ep, UNKNOWN_LOCATION);
5088 : }
5089 : }
5090 1046 : gsi = gsi_last_bb (entry_bb);
5091 : }
5092 :
5093 13893 : if (fd->lastprivate_conditional)
5094 : {
5095 102 : tree clauses = gimple_omp_for_clauses (fd->for_stmt);
5096 102 : tree c = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_);
5097 102 : if (fd->have_pointer_condtemp)
5098 38 : condtemp = OMP_CLAUSE_DECL (c);
5099 102 : c = omp_find_clause (OMP_CLAUSE_CHAIN (c), OMP_CLAUSE__CONDTEMP_);
5100 102 : cond_var = OMP_CLAUSE_DECL (c);
5101 : }
5102 13893 : if (fd->have_reductemp
5103 : /* For scan, we don't want to reinitialize condtemp before the
5104 : second loop. */
5105 13840 : || (fd->have_pointer_condtemp && !fd->have_scantemp)
5106 13822 : || fd->have_nonctrl_scantemp)
5107 : {
5108 248 : tree t1 = build_int_cst (long_integer_type_node, 0);
5109 248 : tree t2 = build_int_cst (long_integer_type_node, 1);
5110 248 : tree t3 = build_int_cstu (long_integer_type_node,
5111 : (HOST_WIDE_INT_1U << 31) + 1);
5112 248 : tree clauses = gimple_omp_for_clauses (fd->for_stmt);
5113 248 : gimple_stmt_iterator gsi2 = gsi_none ();
5114 248 : gimple *g = NULL;
5115 248 : tree mem = null_pointer_node, memv = NULL_TREE;
5116 248 : unsigned HOST_WIDE_INT condtemp_sz = 0;
5117 248 : unsigned HOST_WIDE_INT alloc_align = 0;
5118 248 : if (fd->have_reductemp)
5119 : {
5120 53 : gcc_assert (!fd->have_nonctrl_scantemp);
5121 53 : tree c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
5122 53 : reductions = OMP_CLAUSE_DECL (c);
5123 53 : gcc_assert (TREE_CODE (reductions) == SSA_NAME);
5124 53 : g = SSA_NAME_DEF_STMT (reductions);
5125 53 : reductions = gimple_assign_rhs1 (g);
5126 53 : OMP_CLAUSE_DECL (c) = reductions;
5127 53 : gsi2 = gsi_for_stmt (g);
5128 : }
5129 : else
5130 : {
5131 195 : if (gsi_end_p (gsip))
5132 0 : gsi2 = gsi_after_labels (region->entry);
5133 : else
5134 195 : gsi2 = gsip;
5135 : reductions = null_pointer_node;
5136 : }
5137 248 : if (fd->have_pointer_condtemp || fd->have_nonctrl_scantemp)
5138 : {
5139 205 : tree type;
5140 205 : if (fd->have_pointer_condtemp)
5141 33 : type = TREE_TYPE (condtemp);
5142 : else
5143 172 : type = ptr_type_node;
5144 205 : memv = create_tmp_var (type);
5145 205 : TREE_ADDRESSABLE (memv) = 1;
5146 205 : unsigned HOST_WIDE_INT sz = 0;
5147 205 : tree size = NULL_TREE;
5148 205 : if (fd->have_pointer_condtemp)
5149 : {
5150 33 : sz = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (type)));
5151 33 : sz *= fd->lastprivate_conditional;
5152 33 : condtemp_sz = sz;
5153 : }
5154 205 : if (fd->have_nonctrl_scantemp)
5155 : {
5156 177 : nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
5157 177 : gimple *g = gimple_build_call (nthreads, 0);
5158 177 : nthreads = create_tmp_var (integer_type_node);
5159 177 : gimple_call_set_lhs (g, nthreads);
5160 177 : gsi_insert_before (&gsi2, g, GSI_SAME_STMT);
5161 177 : nthreads = fold_convert (sizetype, nthreads);
5162 177 : alloc_align = TYPE_ALIGN_UNIT (long_long_integer_type_node);
5163 177 : size = expand_omp_scantemp_alloc (clauses, NULL_TREE, sz,
5164 : alloc_align, nthreads, NULL,
5165 : false);
5166 177 : size = fold_convert (type, size);
5167 : }
5168 : else
5169 28 : size = build_int_cst (type, sz);
5170 205 : expand_omp_build_assign (&gsi2, memv, size, false);
5171 205 : mem = build_fold_addr_expr (memv);
5172 : }
5173 248 : tree t
5174 248 : = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_LOOP_START),
5175 : 9, t1, t2, t2, t3, t1, null_pointer_node,
5176 : null_pointer_node, reductions, mem);
5177 248 : force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
5178 : true, GSI_SAME_STMT);
5179 248 : if (fd->have_pointer_condtemp)
5180 33 : expand_omp_build_assign (&gsi2, condtemp, memv, false);
5181 248 : if (fd->have_nonctrl_scantemp)
5182 : {
5183 177 : tree ptr = fd->have_pointer_condtemp ? condtemp : memv;
5184 177 : expand_omp_scantemp_alloc (clauses, ptr, condtemp_sz,
5185 : alloc_align, nthreads, &gsi2, false);
5186 : }
5187 248 : if (fd->have_reductemp)
5188 : {
5189 53 : gsi_remove (&gsi2, true);
5190 53 : release_ssa_name (gimple_assign_lhs (g));
5191 : }
5192 : }
5193 :
5194 13893 : n1 = fd->loop.n1;
5195 13893 : n2 = fd->loop.n2;
5196 13893 : step = fd->loop.step;
5197 13893 : if (gimple_omp_for_combined_into_p (fd->for_stmt))
5198 : {
5199 3042 : tree innerc = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
5200 : OMP_CLAUSE__LOOPTEMP_);
5201 3042 : gcc_assert (innerc);
5202 3042 : n1 = OMP_CLAUSE_DECL (innerc);
5203 3042 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
5204 : OMP_CLAUSE__LOOPTEMP_);
5205 3042 : gcc_assert (innerc);
5206 3042 : n2 = OMP_CLAUSE_DECL (innerc);
5207 : }
5208 13893 : n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
5209 : true, NULL_TREE, true, GSI_SAME_STMT);
5210 13893 : n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
5211 : true, NULL_TREE, true, GSI_SAME_STMT);
5212 13893 : step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
5213 : true, NULL_TREE, true, GSI_SAME_STMT);
5214 :
5215 14912 : t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
5216 13893 : t = fold_build2 (PLUS_EXPR, itype, step, t);
5217 13893 : t = fold_build2 (PLUS_EXPR, itype, t, n2);
5218 13893 : t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1));
5219 13893 : if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
5220 414 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
5221 : fold_build1 (NEGATE_EXPR, itype, t),
5222 : fold_build1 (NEGATE_EXPR, itype, step));
5223 : else
5224 13479 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
5225 13893 : t = fold_convert (itype, t);
5226 13893 : n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
5227 :
5228 13893 : {
5229 : /* Fetch the thread/team id and the number of threads/teams in a single
5230 : call to GOMP_loop_static_worksharing or
5231 : GOMP_distribute_static_worksharing. The helper returns both values packed
5232 : into one complex int, with the id as the imaginary part and the count as
5233 : the real part. Returning (rather than writing through pointers) keeps
5234 : both values as plain SSA names, which lets later passes - notably IPA-CP
5235 : propagating constants into the outlined kernel - reason about them.
5236 : Also pass the total number of iterations for OMPT. */
5237 13893 : tree decl;
5238 13893 : switch (gimple_omp_for_kind (fd->for_stmt))
5239 : {
5240 9359 : case GF_OMP_FOR_KIND_FOR:
5241 18718 : decl = builtin_decl_explicit (
5242 9359 : flag_openmp_ompt ? BUILT_IN_GOMP_LOOP_STATIC_WORKSHARING_START
5243 : : BUILT_IN_GOMP_LOOP_STATIC_WORKSHARING);
5244 9359 : break;
5245 4534 : case GF_OMP_FOR_KIND_DISTRIBUTE:
5246 9068 : decl = builtin_decl_explicit (
5247 4534 : flag_openmp_ompt ? BUILT_IN_GOMP_DISTRIBUTE_STATIC_WORKSHARING_START
5248 : : BUILT_IN_GOMP_DISTRIBUTE_STATIC_WORKSHARING);
5249 4534 : break;
5250 0 : default:
5251 0 : gcc_unreachable ();
5252 : }
5253 13893 : tree n_ull = fold_convert (long_long_unsigned_type_node, n);
5254 13893 : tree packed = build_call_expr (decl, 1, n_ull);
5255 13893 : packed = force_gimple_operand_gsi (&gsi, packed, true, NULL_TREE,
5256 : true, GSI_SAME_STMT);
5257 13893 : threadid = fold_build1 (IMAGPART_EXPR, integer_type_node, packed);
5258 13893 : threadid = fold_convert (itype, threadid);
5259 13893 : threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE,
5260 : true, GSI_SAME_STMT);
5261 13893 : nthreads = fold_build1 (REALPART_EXPR, integer_type_node, packed);
5262 13893 : nthreads = fold_convert (itype, nthreads);
5263 13893 : nthreads = force_gimple_operand_gsi (&gsi, nthreads, true, NULL_TREE,
5264 : true, GSI_SAME_STMT);
5265 : }
5266 :
5267 13893 : q = create_tmp_reg (itype, "q");
5268 13893 : t = fold_build2 (TRUNC_DIV_EXPR, itype, n, nthreads);
5269 13893 : t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
5270 13893 : gsi_insert_before (&gsi, gimple_build_assign (q, t), GSI_SAME_STMT);
5271 :
5272 13893 : tt = create_tmp_reg (itype, "tt");
5273 13893 : t = fold_build2 (TRUNC_MOD_EXPR, itype, n, nthreads);
5274 13893 : t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
5275 13893 : gsi_insert_before (&gsi, gimple_build_assign (tt, t), GSI_SAME_STMT);
5276 :
5277 13893 : t = build2 (LT_EXPR, boolean_type_node, threadid, tt);
5278 13893 : gcond *cond_stmt = gimple_build_cond_empty (t);
5279 13893 : gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
5280 :
5281 13893 : second_bb = split_block (entry_bb, cond_stmt)->dest;
5282 13893 : gsi = gsi_last_nondebug_bb (second_bb);
5283 13893 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5284 :
5285 13893 : gsi_insert_before (&gsi, gimple_build_assign (tt, build_int_cst (itype, 0)),
5286 : GSI_SAME_STMT);
5287 13893 : gassign *assign_stmt
5288 13893 : = gimple_build_assign (q, PLUS_EXPR, q, build_int_cst (itype, 1));
5289 13893 : gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
5290 :
5291 13893 : third_bb = split_block (second_bb, assign_stmt)->dest;
5292 13893 : gsi = gsi_last_nondebug_bb (third_bb);
5293 13893 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5294 :
5295 13893 : if (fd->have_nonctrl_scantemp)
5296 : {
5297 177 : tree clauses = gimple_omp_for_clauses (fd->for_stmt);
5298 177 : tree controlp = NULL_TREE, controlb = NULL_TREE;
5299 772 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
5300 772 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__SCANTEMP_
5301 772 : && OMP_CLAUSE__SCANTEMP__CONTROL (c))
5302 : {
5303 354 : if (TREE_TYPE (OMP_CLAUSE_DECL (c)) == boolean_type_node)
5304 177 : controlb = OMP_CLAUSE_DECL (c);
5305 : else
5306 177 : controlp = OMP_CLAUSE_DECL (c);
5307 354 : if (controlb && controlp)
5308 : break;
5309 : }
5310 177 : gcc_assert (controlp && controlb);
5311 177 : tree cnt = create_tmp_var (sizetype);
5312 177 : gimple *g = gimple_build_assign (cnt, NOP_EXPR, q);
5313 177 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5314 177 : unsigned HOST_WIDE_INT alloc_align = TYPE_ALIGN_UNIT (ptr_type_node);
5315 177 : tree sz = expand_omp_scantemp_alloc (clauses, NULL_TREE, 0,
5316 : alloc_align, cnt, NULL, true);
5317 177 : tree size = create_tmp_var (sizetype);
5318 177 : expand_omp_build_assign (&gsi, size, sz, false);
5319 177 : tree cmp = fold_build2 (GT_EXPR, boolean_type_node,
5320 : size, size_int (16384));
5321 177 : expand_omp_build_assign (&gsi, controlb, cmp);
5322 177 : g = gimple_build_cond (NE_EXPR, controlb, boolean_false_node,
5323 : NULL_TREE, NULL_TREE);
5324 177 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5325 177 : fourth_bb = split_block (third_bb, g)->dest;
5326 177 : gsi = gsi_last_nondebug_bb (fourth_bb);
5327 : /* FIXME: Once we have allocators, this should use allocator. */
5328 177 : g = gimple_build_call (builtin_decl_explicit (BUILT_IN_MALLOC), 1, size);
5329 177 : gimple_call_set_lhs (g, controlp);
5330 177 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5331 177 : expand_omp_scantemp_alloc (clauses, controlp, 0, alloc_align, cnt,
5332 : &gsi, true);
5333 177 : gsi_prev (&gsi);
5334 177 : g = gsi_stmt (gsi);
5335 177 : fifth_bb = split_block (fourth_bb, g)->dest;
5336 177 : gsi = gsi_last_nondebug_bb (fifth_bb);
5337 :
5338 354 : g = gimple_build_call (builtin_decl_implicit (BUILT_IN_STACK_SAVE), 0);
5339 177 : gimple_call_set_lhs (g, controlp);
5340 177 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5341 177 : tree alloca_decl = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
5342 1458 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
5343 1281 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__SCANTEMP_
5344 1281 : && OMP_CLAUSE__SCANTEMP__ALLOC (c))
5345 : {
5346 209 : tree tmp = create_tmp_var (sizetype);
5347 209 : tree pointee_type = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
5348 209 : g = gimple_build_assign (tmp, MULT_EXPR, cnt,
5349 209 : TYPE_SIZE_UNIT (pointee_type));
5350 209 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5351 209 : g = gimple_build_call (alloca_decl, 2, tmp,
5352 209 : size_int (TYPE_ALIGN (pointee_type)));
5353 209 : gimple_call_set_lhs (g, OMP_CLAUSE_DECL (c));
5354 209 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5355 : }
5356 :
5357 177 : sixth_bb = split_block (fifth_bb, g)->dest;
5358 177 : gsi = gsi_last_nondebug_bb (sixth_bb);
5359 : }
5360 :
5361 13893 : t = build2 (MULT_EXPR, itype, q, threadid);
5362 13893 : t = build2 (PLUS_EXPR, itype, t, tt);
5363 13893 : s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
5364 :
5365 13893 : t = fold_build2 (PLUS_EXPR, itype, s0, q);
5366 13893 : e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
5367 :
5368 13893 : t = build2 (GE_EXPR, boolean_type_node, s0, e0);
5369 13893 : gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
5370 :
5371 : /* Remove the GIMPLE_OMP_FOR statement. */
5372 13893 : gsi_remove (&gsi, true);
5373 :
5374 : /* Setup code for sequential iteration goes in SEQ_START_BB. */
5375 13893 : gsi = gsi_start_bb (seq_start_bb);
5376 :
5377 13893 : tree startvar = fd->loop.v;
5378 13893 : tree endvar = NULL_TREE;
5379 :
5380 13893 : if (gimple_omp_for_combined_p (fd->for_stmt))
5381 : {
5382 7060 : tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
5383 10583 : ? gimple_omp_parallel_clauses (inner_stmt)
5384 3537 : : gimple_omp_for_clauses (inner_stmt);
5385 7060 : tree innerc = omp_find_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
5386 7060 : gcc_assert (innerc);
5387 7060 : startvar = OMP_CLAUSE_DECL (innerc);
5388 7060 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
5389 : OMP_CLAUSE__LOOPTEMP_);
5390 7060 : gcc_assert (innerc);
5391 7060 : endvar = OMP_CLAUSE_DECL (innerc);
5392 2311 : if (fd->collapse > 1 && TREE_CODE (fd->loop.n2) != INTEGER_CST
5393 8382 : && gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
5394 : {
5395 829 : innerc = find_lastprivate_looptemp (fd, innerc);
5396 829 : if (innerc)
5397 : {
5398 : /* If needed (distribute parallel for with lastprivate),
5399 : propagate down the total number of iterations. */
5400 375 : tree t = fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc)),
5401 : fd->loop.n2);
5402 375 : t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, false,
5403 : GSI_CONTINUE_LINKING);
5404 375 : assign_stmt = gimple_build_assign (OMP_CLAUSE_DECL (innerc), t);
5405 375 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
5406 : }
5407 : }
5408 : }
5409 13893 : t = fold_convert (itype, s0);
5410 13893 : t = fold_build2 (MULT_EXPR, itype, t, step);
5411 13893 : if (POINTER_TYPE_P (type))
5412 : {
5413 606 : t = fold_build_pointer_plus (n1, t);
5414 670 : if (!POINTER_TYPE_P (TREE_TYPE (startvar))
5415 670 : && TYPE_PRECISION (TREE_TYPE (startvar)) > TYPE_PRECISION (type))
5416 0 : t = fold_convert (signed_type_for (type), t);
5417 : }
5418 : else
5419 13287 : t = fold_build2 (PLUS_EXPR, type, t, n1);
5420 13893 : t = fold_convert (TREE_TYPE (startvar), t);
5421 13893 : t = force_gimple_operand_gsi (&gsi, t,
5422 13893 : DECL_P (startvar)
5423 13893 : && TREE_ADDRESSABLE (startvar),
5424 : NULL_TREE, false, GSI_CONTINUE_LINKING);
5425 13893 : assign_stmt = gimple_build_assign (startvar, t);
5426 13893 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
5427 13893 : if (cond_var)
5428 : {
5429 102 : tree itype = TREE_TYPE (cond_var);
5430 : /* For lastprivate(conditional:) itervar, we need some iteration
5431 : counter that starts at unsigned non-zero and increases.
5432 : Prefer as few IVs as possible, so if we can use startvar
5433 : itself, use that, or startvar + constant (those would be
5434 : incremented with step), and as last resort use the s0 + 1
5435 : incremented by 1. */
5436 102 : if (POINTER_TYPE_P (type)
5437 102 : || TREE_CODE (n1) != INTEGER_CST
5438 96 : || fd->loop.cond_code != LT_EXPR)
5439 6 : t = fold_build2 (PLUS_EXPR, itype, fold_convert (itype, s0),
5440 : build_int_cst (itype, 1));
5441 96 : else if (tree_int_cst_sgn (n1) == 1)
5442 12 : t = fold_convert (itype, t);
5443 : else
5444 : {
5445 84 : tree c = fold_convert (itype, n1);
5446 84 : c = fold_build2 (MINUS_EXPR, itype, build_int_cst (itype, 1), c);
5447 84 : t = fold_build2 (PLUS_EXPR, itype, fold_convert (itype, t), c);
5448 : }
5449 102 : t = force_gimple_operand_gsi (&gsi, t, false,
5450 : NULL_TREE, false, GSI_CONTINUE_LINKING);
5451 102 : assign_stmt = gimple_build_assign (cond_var, t);
5452 102 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
5453 : }
5454 :
5455 13893 : t = fold_convert (itype, e0);
5456 13893 : t = fold_build2 (MULT_EXPR, itype, t, step);
5457 13893 : if (POINTER_TYPE_P (type))
5458 : {
5459 606 : t = fold_build_pointer_plus (n1, t);
5460 670 : if (!POINTER_TYPE_P (TREE_TYPE (startvar))
5461 670 : && TYPE_PRECISION (TREE_TYPE (startvar)) > TYPE_PRECISION (type))
5462 0 : t = fold_convert (signed_type_for (type), t);
5463 : }
5464 : else
5465 13287 : t = fold_build2 (PLUS_EXPR, type, t, n1);
5466 13893 : t = fold_convert (TREE_TYPE (startvar), t);
5467 13893 : e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5468 : false, GSI_CONTINUE_LINKING);
5469 13893 : if (endvar)
5470 : {
5471 7060 : assign_stmt = gimple_build_assign (endvar, e);
5472 7060 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
5473 7060 : if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e)))
5474 6291 : assign_stmt = gimple_build_assign (fd->loop.v, e);
5475 : else
5476 769 : assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e);
5477 7060 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
5478 : }
5479 : /* Handle linear clause adjustments. */
5480 13893 : tree itercnt = NULL_TREE;
5481 13893 : tree *nonrect_bounds = NULL;
5482 13893 : if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_FOR)
5483 44825 : for (tree c = gimple_omp_for_clauses (fd->for_stmt);
5484 44825 : c; c = OMP_CLAUSE_CHAIN (c))
5485 35466 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
5486 35466 : && !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
5487 : {
5488 75 : tree d = OMP_CLAUSE_DECL (c);
5489 75 : tree t = d, a, dest;
5490 75 : if (omp_privatize_by_reference (t))
5491 13 : t = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), t);
5492 75 : if (itercnt == NULL_TREE)
5493 : {
5494 75 : if (gimple_omp_for_combined_into_p (fd->for_stmt))
5495 : {
5496 0 : itercnt = fold_build2 (MINUS_EXPR, itype,
5497 : fold_convert (itype, n1),
5498 : fold_convert (itype, fd->loop.n1));
5499 0 : itercnt = fold_build2 (EXACT_DIV_EXPR, itype, itercnt, step);
5500 0 : itercnt = fold_build2 (PLUS_EXPR, itype, itercnt, s0);
5501 0 : itercnt = force_gimple_operand_gsi (&gsi, itercnt, true,
5502 : NULL_TREE, false,
5503 : GSI_CONTINUE_LINKING);
5504 : }
5505 : else
5506 : itercnt = s0;
5507 : }
5508 75 : tree type = TREE_TYPE (t);
5509 75 : if (POINTER_TYPE_P (type))
5510 0 : type = sizetype;
5511 75 : a = fold_build2 (MULT_EXPR, type,
5512 : fold_convert (type, itercnt),
5513 : fold_convert (type, OMP_CLAUSE_LINEAR_STEP (c)));
5514 75 : dest = unshare_expr (t);
5515 75 : t = fold_build2 (type == TREE_TYPE (t) ? PLUS_EXPR
5516 : : POINTER_PLUS_EXPR, TREE_TYPE (t), t, a);
5517 75 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5518 : false, GSI_CONTINUE_LINKING);
5519 75 : expand_omp_build_assign (&gsi, dest, t, true);
5520 : }
5521 13893 : if (fd->collapse > 1)
5522 : {
5523 3915 : if (fd->non_rect)
5524 : {
5525 376 : nonrect_bounds = XALLOCAVEC (tree, fd->last_nonrect + 1);
5526 376 : memset (nonrect_bounds, 0, sizeof (tree) * (fd->last_nonrect + 1));
5527 : }
5528 3915 : expand_omp_for_init_vars (fd, &gsi, counts, nonrect_bounds, inner_stmt,
5529 : startvar);
5530 : }
5531 :
5532 13893 : if (!broken_loop)
5533 : {
5534 : /* The code controlling the sequential loop replaces the
5535 : GIMPLE_OMP_CONTINUE. */
5536 13409 : gsi = gsi_last_nondebug_bb (cont_bb);
5537 13409 : gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
5538 13409 : gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE);
5539 13409 : vmain = gimple_omp_continue_control_use (cont_stmt);
5540 13409 : vback = gimple_omp_continue_control_def (cont_stmt);
5541 :
5542 13409 : if (cond_var)
5543 : {
5544 102 : tree itype = TREE_TYPE (cond_var);
5545 102 : tree t2;
5546 102 : if (POINTER_TYPE_P (type)
5547 102 : || TREE_CODE (n1) != INTEGER_CST
5548 96 : || fd->loop.cond_code != LT_EXPR)
5549 6 : t2 = build_int_cst (itype, 1);
5550 : else
5551 96 : t2 = fold_convert (itype, step);
5552 102 : t2 = fold_build2 (PLUS_EXPR, itype, cond_var, t2);
5553 102 : t2 = force_gimple_operand_gsi (&gsi, t2, false,
5554 : NULL_TREE, true, GSI_SAME_STMT);
5555 102 : assign_stmt = gimple_build_assign (cond_var, t2);
5556 102 : gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
5557 : }
5558 :
5559 13409 : if (!gimple_omp_for_combined_p (fd->for_stmt))
5560 : {
5561 6349 : if (POINTER_TYPE_P (type))
5562 316 : t = fold_build_pointer_plus (vmain, step);
5563 : else
5564 6033 : t = fold_build2 (PLUS_EXPR, type, vmain, step);
5565 6349 : t = force_gimple_operand_gsi (&gsi, t,
5566 6349 : DECL_P (vback)
5567 6349 : && TREE_ADDRESSABLE (vback),
5568 : NULL_TREE, true, GSI_SAME_STMT);
5569 6349 : assign_stmt = gimple_build_assign (vback, t);
5570 6349 : gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
5571 :
5572 6349 : t = build2 (fd->loop.cond_code, boolean_type_node,
5573 6349 : DECL_P (vback) && TREE_ADDRESSABLE (vback)
5574 : ? t : vback, e);
5575 6349 : gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
5576 : }
5577 :
5578 : /* Remove the GIMPLE_OMP_CONTINUE statement. */
5579 13409 : gsi_remove (&gsi, true);
5580 :
5581 13409 : if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
5582 1376 : collapse_bb = extract_omp_for_update_vars (fd, nonrect_bounds,
5583 : cont_bb, body_bb);
5584 : }
5585 :
5586 13893 : if (flag_openmp_ompt_detailed)
5587 : {
5588 : /* Insert call to GOMP_*_static_worksharing_dispatch at the end of
5589 : seq_start_bb. */
5590 8 : gsi = gsi_last_nondebug_bb (seq_start_bb);
5591 8 : tree decl;
5592 8 : switch (gimple_omp_for_kind (fd->for_stmt))
5593 : {
5594 4 : case GF_OMP_FOR_KIND_FOR:
5595 4 : decl = builtin_decl_explicit (
5596 : BUILT_IN_GOMP_LOOP_STATIC_WORKSHARING_DISPATCH);
5597 4 : break;
5598 4 : case GF_OMP_FOR_KIND_DISTRIBUTE:
5599 4 : decl = builtin_decl_explicit (
5600 : BUILT_IN_GOMP_DISTRIBUTE_STATIC_WORKSHARING_DISPATCH);
5601 4 : break;
5602 0 : default:
5603 0 : gcc_unreachable ();
5604 : }
5605 8 : gcall *g = gimple_build_call (decl, 0);
5606 8 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5607 : }
5608 :
5609 13893 : gsi = gsi_last_nondebug_bb (exit_bb);
5610 13893 : if (flag_openmp_ompt)
5611 : {
5612 : /* Insert call to GOMP_*_static_worksharing_end at the end of exit_bb.
5613 : */
5614 16 : tree decl;
5615 16 : switch (gimple_omp_for_kind (fd->for_stmt))
5616 : {
5617 8 : case GF_OMP_FOR_KIND_FOR:
5618 8 : decl
5619 8 : = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_STATIC_WORKSHARING_END);
5620 8 : break;
5621 8 : case GF_OMP_FOR_KIND_DISTRIBUTE:
5622 8 : decl = builtin_decl_explicit (
5623 : BUILT_IN_GOMP_DISTRIBUTE_STATIC_WORKSHARING_END);
5624 8 : break;
5625 0 : default:
5626 0 : gcc_unreachable ();
5627 : }
5628 16 : gcall *g = gimple_build_call (decl, 0);
5629 16 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
5630 : }
5631 :
5632 : /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
5633 13893 : if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
5634 : {
5635 2323 : t = gimple_omp_return_lhs (gsi_stmt (gsi));
5636 2323 : if (fd->have_reductemp
5637 2270 : || ((fd->have_pointer_condtemp || fd->have_scantemp)
5638 63 : && !fd->have_nonctrl_scantemp))
5639 : {
5640 116 : tree fn;
5641 116 : if (t)
5642 4 : fn = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL);
5643 : else
5644 112 : fn = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END);
5645 116 : gcall *g = gimple_build_call (fn, 0);
5646 116 : if (t)
5647 : {
5648 4 : gimple_call_set_lhs (g, t);
5649 4 : if (fd->have_reductemp)
5650 4 : gsi_insert_after (&gsi, gimple_build_assign (reductions,
5651 : NOP_EXPR, t),
5652 : GSI_SAME_STMT);
5653 : }
5654 116 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
5655 116 : }
5656 : else
5657 2207 : gsi_insert_after (&gsi,
5658 : omp_build_barrier (t,
5659 : GOMP_BARRIER_IMPLICIT_WORKSHARE),
5660 : GSI_SAME_STMT);
5661 : }
5662 11570 : else if ((fd->have_pointer_condtemp || fd->have_scantemp)
5663 309 : && !fd->have_nonctrl_scantemp)
5664 : {
5665 132 : tree fn = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT);
5666 132 : gcall *g = gimple_build_call (fn, 0);
5667 132 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
5668 : }
5669 13893 : if (fd->have_scantemp && !fd->have_nonctrl_scantemp)
5670 : {
5671 177 : tree clauses = gimple_omp_for_clauses (fd->for_stmt);
5672 177 : tree controlp = NULL_TREE, controlb = NULL_TREE;
5673 354 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
5674 354 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__SCANTEMP_
5675 354 : && OMP_CLAUSE__SCANTEMP__CONTROL (c))
5676 : {
5677 354 : if (TREE_TYPE (OMP_CLAUSE_DECL (c)) == boolean_type_node)
5678 177 : controlb = OMP_CLAUSE_DECL (c);
5679 : else
5680 177 : controlp = OMP_CLAUSE_DECL (c);
5681 354 : if (controlb && controlp)
5682 : break;
5683 : }
5684 177 : gcc_assert (controlp && controlb);
5685 177 : gimple *g = gimple_build_cond (NE_EXPR, controlb, boolean_false_node,
5686 : NULL_TREE, NULL_TREE);
5687 177 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5688 177 : exit1_bb = split_block (exit_bb, g)->dest;
5689 177 : gsi = gsi_after_labels (exit1_bb);
5690 177 : g = gimple_build_call (builtin_decl_explicit (BUILT_IN_FREE), 1,
5691 : controlp);
5692 177 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5693 177 : exit2_bb = split_block (exit1_bb, g)->dest;
5694 177 : gsi = gsi_after_labels (exit2_bb);
5695 354 : g = gimple_build_call (builtin_decl_implicit (BUILT_IN_STACK_RESTORE), 1,
5696 : controlp);
5697 177 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5698 177 : exit3_bb = split_block (exit2_bb, g)->dest;
5699 177 : gsi = gsi_after_labels (exit3_bb);
5700 : }
5701 :
5702 13893 : gsi_remove (&gsi, true);
5703 :
5704 : /* Connect all the blocks. */
5705 13893 : ep = make_edge (entry_bb, third_bb, EDGE_FALSE_VALUE);
5706 13893 : ep->probability = profile_probability::guessed_always ().apply_scale (3, 4);
5707 13893 : ep = find_edge (entry_bb, second_bb);
5708 13893 : ep->flags = EDGE_TRUE_VALUE;
5709 13893 : ep->probability = profile_probability::guessed_always () / 4;
5710 13893 : if (fourth_bb)
5711 : {
5712 177 : ep = make_edge (third_bb, fifth_bb, EDGE_FALSE_VALUE);
5713 177 : ep->probability = profile_probability::guessed_always () / 2;
5714 177 : ep = find_edge (third_bb, fourth_bb);
5715 177 : ep->flags = EDGE_TRUE_VALUE;
5716 177 : ep->probability = profile_probability::guessed_always () / 2;
5717 177 : ep = find_edge (fourth_bb, fifth_bb);
5718 177 : redirect_edge_and_branch (ep, sixth_bb);
5719 : }
5720 : else
5721 : sixth_bb = third_bb;
5722 13893 : find_edge (sixth_bb, seq_start_bb)->flags = EDGE_FALSE_VALUE;
5723 13893 : find_edge (sixth_bb, fin_bb)->flags = EDGE_TRUE_VALUE;
5724 13893 : if (exit1_bb)
5725 : {
5726 177 : ep = make_edge (exit_bb, exit2_bb, EDGE_FALSE_VALUE);
5727 177 : ep->probability = profile_probability::guessed_always () / 2;
5728 177 : ep = find_edge (exit_bb, exit1_bb);
5729 177 : ep->flags = EDGE_TRUE_VALUE;
5730 177 : ep->probability = profile_probability::guessed_always () / 2;
5731 177 : ep = find_edge (exit1_bb, exit2_bb);
5732 177 : redirect_edge_and_branch (ep, exit3_bb);
5733 : }
5734 :
5735 13893 : if (!broken_loop)
5736 : {
5737 13409 : ep = find_edge (cont_bb, body_bb);
5738 13409 : if (ep == NULL)
5739 : {
5740 171 : ep = BRANCH_EDGE (cont_bb);
5741 171 : gcc_assert (single_succ (ep->dest) == body_bb);
5742 : }
5743 13409 : if (gimple_omp_for_combined_p (fd->for_stmt))
5744 : {
5745 7060 : remove_edge (ep);
5746 7060 : ep = NULL;
5747 : }
5748 6349 : else if (fd->collapse > 1)
5749 : {
5750 1376 : remove_edge (ep);
5751 1376 : ep = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
5752 : }
5753 : else
5754 4973 : ep->flags = EDGE_TRUE_VALUE;
5755 25442 : find_edge (cont_bb, fin_bb)->flags
5756 14785 : = ep ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
5757 : }
5758 :
5759 13893 : set_immediate_dominator (CDI_DOMINATORS, second_bb, entry_bb);
5760 13893 : set_immediate_dominator (CDI_DOMINATORS, third_bb, entry_bb);
5761 13893 : if (fourth_bb)
5762 : {
5763 177 : set_immediate_dominator (CDI_DOMINATORS, fifth_bb, third_bb);
5764 177 : set_immediate_dominator (CDI_DOMINATORS, sixth_bb, third_bb);
5765 : }
5766 13893 : set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, sixth_bb);
5767 :
5768 13893 : set_immediate_dominator (CDI_DOMINATORS, body_bb,
5769 : recompute_dominator (CDI_DOMINATORS, body_bb));
5770 13893 : set_immediate_dominator (CDI_DOMINATORS, fin_bb,
5771 : recompute_dominator (CDI_DOMINATORS, fin_bb));
5772 13893 : if (exit1_bb)
5773 : {
5774 177 : set_immediate_dominator (CDI_DOMINATORS, exit2_bb, exit_bb);
5775 177 : set_immediate_dominator (CDI_DOMINATORS, exit3_bb, exit_bb);
5776 : }
5777 :
5778 13893 : class loop *loop = body_bb->loop_father;
5779 13893 : if (loop != entry_bb->loop_father)
5780 : {
5781 333 : gcc_assert (broken_loop || loop->header == body_bb);
5782 171 : gcc_assert (broken_loop
5783 : || loop->latch == region->cont
5784 : || single_pred (loop->latch) == region->cont);
5785 333 : return;
5786 : }
5787 :
5788 13560 : if (!broken_loop && !gimple_omp_for_combined_p (fd->for_stmt))
5789 : {
5790 6178 : loop = alloc_loop ();
5791 6178 : loop->header = body_bb;
5792 6178 : if (collapse_bb == NULL)
5793 4802 : loop->latch = cont_bb;
5794 6178 : add_loop (loop, body_bb->loop_father);
5795 : }
5796 : }
5797 :
5798 : /* Return phi in E->DEST with ARG on edge E. */
5799 :
5800 : static gphi *
5801 14 : find_phi_with_arg_on_edge (tree arg, edge e)
5802 : {
5803 14 : basic_block bb = e->dest;
5804 :
5805 14 : for (gphi_iterator gpi = gsi_start_phis (bb);
5806 14 : !gsi_end_p (gpi);
5807 0 : gsi_next (&gpi))
5808 : {
5809 14 : gphi *phi = gpi.phi ();
5810 14 : if (PHI_ARG_DEF_FROM_EDGE (phi, e) == arg)
5811 14 : return phi;
5812 : }
5813 :
5814 0 : return NULL;
5815 : }
5816 :
5817 : /* A subroutine of expand_omp_for. Generate code for a parallel
5818 : loop with static schedule and a specified chunk size. Given
5819 : parameters:
5820 :
5821 : for (V = N1; V cond N2; V += STEP) BODY;
5822 :
5823 : where COND is "<" or ">", we generate pseudocode
5824 :
5825 : if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5826 : if (cond is <)
5827 : adj = STEP - 1;
5828 : else
5829 : adj = STEP + 1;
5830 : if ((__typeof (V)) -1 > 0 && cond is >)
5831 : n = -(adj + N2 - N1) / -STEP;
5832 : else
5833 : n = (adj + N2 - N1) / STEP;
5834 : trip = 0;
5835 : V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
5836 : here so that V is defined
5837 : if the loop is not entered
5838 : L0:
5839 : s0 = (trip * nthreads + threadid) * CHUNK;
5840 : e0 = min (s0 + CHUNK, n);
5841 : if (s0 < n) goto L1; else goto L4;
5842 : L1:
5843 : V = s0 * STEP + N1;
5844 : e = e0 * STEP + N1;
5845 : L2:
5846 : BODY;
5847 : V += STEP;
5848 : if (V cond e) goto L2; else goto L3;
5849 : L3:
5850 : trip += 1;
5851 : goto L0;
5852 : L4:
5853 : */
5854 :
5855 : static void
5856 5840 : expand_omp_for_static_chunk (struct omp_region *region,
5857 : struct omp_for_data *fd, gimple *inner_stmt)
5858 : {
5859 5840 : tree n, s0, e0, e, t;
5860 5840 : tree trip_var, trip_init, trip_main, trip_back, nthreads, threadid;
5861 5840 : tree type, itype, vmain, vback, vextra;
5862 5840 : basic_block entry_bb, exit_bb, body_bb, seq_start_bb, iter_part_bb;
5863 5840 : basic_block trip_update_bb = NULL, cont_bb, collapse_bb = NULL, fin_bb;
5864 5840 : gimple_stmt_iterator gsi, gsip;
5865 5840 : edge se;
5866 5840 : bool broken_loop = region->cont == NULL;
5867 5840 : tree *counts = NULL;
5868 5840 : tree n1, n2, step;
5869 5840 : tree reductions = NULL_TREE;
5870 5840 : tree cond_var = NULL_TREE, condtemp = NULL_TREE;
5871 :
5872 5840 : itype = type = TREE_TYPE (fd->loop.v);
5873 5840 : if (POINTER_TYPE_P (type))
5874 316 : itype = signed_type_for (type);
5875 :
5876 5840 : entry_bb = region->entry;
5877 5840 : se = split_block (entry_bb, last_nondebug_stmt (entry_bb));
5878 5840 : entry_bb = se->src;
5879 5840 : iter_part_bb = se->dest;
5880 5840 : cont_bb = region->cont;
5881 5840 : gcc_assert (EDGE_COUNT (iter_part_bb->succs) == 2);
5882 5840 : fin_bb = BRANCH_EDGE (iter_part_bb)->dest;
5883 5840 : gcc_assert (broken_loop
5884 : || fin_bb == FALLTHRU_EDGE (cont_bb)->dest);
5885 5840 : seq_start_bb = split_edge (FALLTHRU_EDGE (iter_part_bb));
5886 5840 : body_bb = single_succ (seq_start_bb);
5887 5840 : if (!broken_loop)
5888 : {
5889 5583 : gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb
5890 : || single_succ (BRANCH_EDGE (cont_bb)->dest) == body_bb);
5891 5583 : gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
5892 5583 : trip_update_bb = split_edge (FALLTHRU_EDGE (cont_bb));
5893 : }
5894 5840 : exit_bb = region->exit;
5895 :
5896 : /* Trip and adjustment setup goes in ENTRY_BB. */
5897 5840 : gsi = gsi_last_nondebug_bb (entry_bb);
5898 5840 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5899 5840 : gsip = gsi;
5900 5840 : gsi_prev (&gsip);
5901 :
5902 5840 : if (fd->collapse > 1)
5903 : {
5904 2226 : int first_zero_iter = -1, dummy = -1;
5905 2226 : basic_block l2_dom_bb = NULL, dummy_bb = NULL;
5906 :
5907 2226 : counts = XALLOCAVEC (tree, fd->collapse);
5908 2226 : expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
5909 : fin_bb, first_zero_iter,
5910 : dummy_bb, dummy, l2_dom_bb);
5911 2226 : t = NULL_TREE;
5912 : }
5913 3614 : else if (gimple_omp_for_combined_into_p (fd->for_stmt))
5914 813 : t = integer_one_node;
5915 : else
5916 2801 : t = fold_binary (fd->loop.cond_code, boolean_type_node,
5917 : fold_convert (type, fd->loop.n1),
5918 : fold_convert (type, fd->loop.n2));
5919 5840 : if (fd->collapse == 1
5920 3614 : && TYPE_UNSIGNED (type)
5921 7027 : && (t == NULL_TREE || !integer_onep (t)))
5922 : {
5923 236 : n1 = fold_convert (type, unshare_expr (fd->loop.n1));
5924 236 : n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE,
5925 : true, GSI_SAME_STMT);
5926 236 : n2 = fold_convert (type, unshare_expr (fd->loop.n2));
5927 236 : n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
5928 : true, GSI_SAME_STMT);
5929 236 : gcond *cond_stmt = expand_omp_build_cond (&gsi, fd->loop.cond_code,
5930 : n1, n2);
5931 236 : se = split_block (entry_bb, cond_stmt);
5932 236 : se->flags = EDGE_TRUE_VALUE;
5933 236 : entry_bb = se->dest;
5934 236 : se->probability = profile_probability::very_likely ();
5935 236 : se = make_edge (se->src, fin_bb, EDGE_FALSE_VALUE);
5936 236 : se->probability = profile_probability::very_unlikely ();
5937 236 : if (gimple_in_ssa_p (cfun))
5938 : {
5939 1 : int dest_idx = find_edge (iter_part_bb, fin_bb)->dest_idx;
5940 1 : for (gphi_iterator gpi = gsi_start_phis (fin_bb);
5941 2 : !gsi_end_p (gpi); gsi_next (&gpi))
5942 : {
5943 1 : gphi *phi = gpi.phi ();
5944 1 : add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
5945 : se, UNKNOWN_LOCATION);
5946 : }
5947 : }
5948 472 : gsi = gsi_last_bb (entry_bb);
5949 : }
5950 :
5951 5840 : if (fd->lastprivate_conditional)
5952 : {
5953 49 : tree clauses = gimple_omp_for_clauses (fd->for_stmt);
5954 49 : tree c = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_);
5955 49 : if (fd->have_pointer_condtemp)
5956 29 : condtemp = OMP_CLAUSE_DECL (c);
5957 49 : c = omp_find_clause (OMP_CLAUSE_CHAIN (c), OMP_CLAUSE__CONDTEMP_);
5958 49 : cond_var = OMP_CLAUSE_DECL (c);
5959 : }
5960 5840 : if (fd->have_reductemp || fd->have_pointer_condtemp)
5961 : {
5962 44 : tree t1 = build_int_cst (long_integer_type_node, 0);
5963 44 : tree t2 = build_int_cst (long_integer_type_node, 1);
5964 44 : tree t3 = build_int_cstu (long_integer_type_node,
5965 : (HOST_WIDE_INT_1U << 31) + 1);
5966 44 : tree clauses = gimple_omp_for_clauses (fd->for_stmt);
5967 44 : gimple_stmt_iterator gsi2 = gsi_none ();
5968 44 : gimple *g = NULL;
5969 44 : tree mem = null_pointer_node, memv = NULL_TREE;
5970 44 : if (fd->have_reductemp)
5971 : {
5972 25 : tree c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
5973 25 : reductions = OMP_CLAUSE_DECL (c);
5974 25 : gcc_assert (TREE_CODE (reductions) == SSA_NAME);
5975 25 : g = SSA_NAME_DEF_STMT (reductions);
5976 25 : reductions = gimple_assign_rhs1 (g);
5977 25 : OMP_CLAUSE_DECL (c) = reductions;
5978 25 : gsi2 = gsi_for_stmt (g);
5979 : }
5980 : else
5981 : {
5982 19 : if (gsi_end_p (gsip))
5983 0 : gsi2 = gsi_after_labels (region->entry);
5984 : else
5985 19 : gsi2 = gsip;
5986 : reductions = null_pointer_node;
5987 : }
5988 44 : if (fd->have_pointer_condtemp)
5989 : {
5990 29 : tree type = TREE_TYPE (condtemp);
5991 29 : memv = create_tmp_var (type);
5992 29 : TREE_ADDRESSABLE (memv) = 1;
5993 29 : unsigned HOST_WIDE_INT sz
5994 29 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (type)));
5995 29 : sz *= fd->lastprivate_conditional;
5996 29 : expand_omp_build_assign (&gsi2, memv, build_int_cst (type, sz),
5997 : false);
5998 29 : mem = build_fold_addr_expr (memv);
5999 : }
6000 44 : tree t
6001 44 : = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_LOOP_START),
6002 : 9, t1, t2, t2, t3, t1, null_pointer_node,
6003 : null_pointer_node, reductions, mem);
6004 44 : force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
6005 : true, GSI_SAME_STMT);
6006 44 : if (fd->have_pointer_condtemp)
6007 29 : expand_omp_build_assign (&gsi2, condtemp, memv, false);
6008 44 : if (fd->have_reductemp)
6009 : {
6010 25 : gsi_remove (&gsi2, true);
6011 25 : release_ssa_name (gimple_assign_lhs (g));
6012 : }
6013 : }
6014 :
6015 5840 : n1 = fd->loop.n1;
6016 5840 : n2 = fd->loop.n2;
6017 5840 : step = fd->loop.step;
6018 5840 : if (gimple_omp_for_combined_into_p (fd->for_stmt))
6019 : {
6020 1301 : tree innerc = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
6021 : OMP_CLAUSE__LOOPTEMP_);
6022 1301 : gcc_assert (innerc);
6023 1301 : n1 = OMP_CLAUSE_DECL (innerc);
6024 1301 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
6025 : OMP_CLAUSE__LOOPTEMP_);
6026 1301 : gcc_assert (innerc);
6027 1301 : n2 = OMP_CLAUSE_DECL (innerc);
6028 : }
6029 5840 : n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
6030 : true, NULL_TREE, true, GSI_SAME_STMT);
6031 5840 : n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
6032 : true, NULL_TREE, true, GSI_SAME_STMT);
6033 5840 : step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
6034 : true, NULL_TREE, true, GSI_SAME_STMT);
6035 5840 : tree chunk_size = fold_convert (itype, fd->chunk_size);
6036 5840 : chunk_size = omp_adjust_chunk_size (chunk_size, fd->simd_schedule,
6037 5840 : is_in_offload_region (region));
6038 5840 : chunk_size
6039 5840 : = force_gimple_operand_gsi (&gsi, chunk_size, true, NULL_TREE, true,
6040 : GSI_SAME_STMT);
6041 :
6042 6446 : t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
6043 5840 : t = fold_build2 (PLUS_EXPR, itype, step, t);
6044 5840 : t = fold_build2 (PLUS_EXPR, itype, t, n2);
6045 5840 : t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1));
6046 5840 : if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
6047 284 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
6048 : fold_build1 (NEGATE_EXPR, itype, t),
6049 : fold_build1 (NEGATE_EXPR, itype, step));
6050 : else
6051 5556 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
6052 5840 : t = fold_convert (itype, t);
6053 5840 : n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6054 : true, GSI_SAME_STMT);
6055 :
6056 5840 : {
6057 : /* Fetch the thread/team id and the number of threads/teams in a single
6058 : call to GOMP_loop_static_worksharing or
6059 : GOMP_distribute_static_worksharing. The helper returns both values packed
6060 : into one complex int, with the id as the imaginary part and the count as
6061 : the real part. Returning (rather than writing through pointers) keeps
6062 : both values as plain SSA names, which lets later passes - notably IPA-CP
6063 : propagating constants into the outlined kernel - reason about them.
6064 : Also pass the total number of iterations for OMPT. */
6065 5840 : tree decl;
6066 5840 : switch (gimple_omp_for_kind (fd->for_stmt))
6067 : {
6068 2375 : case GF_OMP_FOR_KIND_FOR:
6069 4750 : decl = builtin_decl_explicit (
6070 2375 : flag_openmp_ompt ? BUILT_IN_GOMP_LOOP_STATIC_WORKSHARING_START
6071 : : BUILT_IN_GOMP_LOOP_STATIC_WORKSHARING);
6072 2375 : break;
6073 3465 : case GF_OMP_FOR_KIND_DISTRIBUTE:
6074 6930 : decl = builtin_decl_explicit (
6075 3465 : flag_openmp_ompt ? BUILT_IN_GOMP_DISTRIBUTE_STATIC_WORKSHARING_START
6076 : : BUILT_IN_GOMP_DISTRIBUTE_STATIC_WORKSHARING);
6077 3465 : break;
6078 0 : default:
6079 0 : gcc_unreachable ();
6080 : }
6081 5840 : tree n_ull = fold_convert (long_long_unsigned_type_node, n);
6082 5840 : tree packed = build_call_expr (decl, 1, n_ull);
6083 5840 : packed = force_gimple_operand_gsi (&gsi, packed, true, NULL_TREE, true,
6084 : GSI_SAME_STMT);
6085 5840 : threadid = fold_build1 (IMAGPART_EXPR, integer_type_node, packed);
6086 5840 : threadid = fold_convert (itype, threadid);
6087 5840 : threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE, true,
6088 : GSI_SAME_STMT);
6089 5840 : nthreads = fold_build1 (REALPART_EXPR, integer_type_node, packed);
6090 5840 : nthreads = fold_convert (itype, nthreads);
6091 5840 : nthreads = force_gimple_operand_gsi (&gsi, nthreads, true, NULL_TREE, true,
6092 : GSI_SAME_STMT);
6093 : }
6094 :
6095 5840 : trip_var = create_tmp_reg (itype, ".trip");
6096 5840 : if (gimple_in_ssa_p (cfun))
6097 : {
6098 12 : trip_init = make_ssa_name (trip_var);
6099 12 : trip_main = make_ssa_name (trip_var);
6100 12 : trip_back = make_ssa_name (trip_var);
6101 : }
6102 : else
6103 : {
6104 : trip_init = trip_var;
6105 : trip_main = trip_var;
6106 : trip_back = trip_var;
6107 : }
6108 :
6109 5840 : gassign *assign_stmt
6110 5840 : = gimple_build_assign (trip_init, build_int_cst (itype, 0));
6111 5840 : gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
6112 :
6113 5840 : t = fold_build2 (MULT_EXPR, itype, threadid, chunk_size);
6114 5840 : t = fold_build2 (MULT_EXPR, itype, t, step);
6115 5840 : if (POINTER_TYPE_P (type))
6116 316 : t = fold_build_pointer_plus (n1, t);
6117 : else
6118 5524 : t = fold_build2 (PLUS_EXPR, type, t, n1);
6119 5840 : vextra = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6120 : true, GSI_SAME_STMT);
6121 :
6122 : /* Remove the GIMPLE_OMP_FOR. */
6123 5840 : gsi_remove (&gsi, true);
6124 :
6125 5840 : gimple_stmt_iterator gsif = gsi;
6126 :
6127 : /* Iteration space partitioning goes in ITER_PART_BB. */
6128 5840 : gsi = gsi_last_bb (iter_part_bb);
6129 :
6130 5840 : t = fold_build2 (MULT_EXPR, itype, trip_main, nthreads);
6131 5840 : t = fold_build2 (PLUS_EXPR, itype, t, threadid);
6132 5840 : t = fold_build2 (MULT_EXPR, itype, t, chunk_size);
6133 5840 : s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6134 : false, GSI_CONTINUE_LINKING);
6135 :
6136 5840 : t = fold_build2 (PLUS_EXPR, itype, s0, chunk_size);
6137 5840 : t = fold_build2 (MIN_EXPR, itype, t, n);
6138 5840 : e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6139 : false, GSI_CONTINUE_LINKING);
6140 :
6141 5840 : t = build2 (LT_EXPR, boolean_type_node, s0, n);
6142 5840 : gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_CONTINUE_LINKING);
6143 :
6144 : /* Setup code for sequential iteration goes in SEQ_START_BB. */
6145 5840 : gsi = gsi_start_bb (seq_start_bb);
6146 :
6147 5840 : tree startvar = fd->loop.v;
6148 5840 : tree endvar = NULL_TREE;
6149 :
6150 5840 : if (gimple_omp_for_combined_p (fd->for_stmt))
6151 : {
6152 4202 : tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
6153 6943 : ? gimple_omp_parallel_clauses (inner_stmt)
6154 1461 : : gimple_omp_for_clauses (inner_stmt);
6155 4202 : tree innerc = omp_find_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
6156 4202 : gcc_assert (innerc);
6157 4202 : startvar = OMP_CLAUSE_DECL (innerc);
6158 4202 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
6159 : OMP_CLAUSE__LOOPTEMP_);
6160 4202 : gcc_assert (innerc);
6161 4202 : endvar = OMP_CLAUSE_DECL (innerc);
6162 1663 : if (fd->collapse > 1 && TREE_CODE (fd->loop.n2) != INTEGER_CST
6163 5087 : && gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
6164 : {
6165 704 : innerc = find_lastprivate_looptemp (fd, innerc);
6166 704 : if (innerc)
6167 : {
6168 : /* If needed (distribute parallel for with lastprivate),
6169 : propagate down the total number of iterations. */
6170 320 : tree t = fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc)),
6171 : fd->loop.n2);
6172 320 : t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, false,
6173 : GSI_CONTINUE_LINKING);
6174 320 : assign_stmt = gimple_build_assign (OMP_CLAUSE_DECL (innerc), t);
6175 320 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6176 : }
6177 : }
6178 : }
6179 :
6180 5840 : t = fold_convert (itype, s0);
6181 5840 : t = fold_build2 (MULT_EXPR, itype, t, step);
6182 5840 : if (POINTER_TYPE_P (type))
6183 : {
6184 316 : t = fold_build_pointer_plus (n1, t);
6185 380 : if (!POINTER_TYPE_P (TREE_TYPE (startvar))
6186 380 : && TYPE_PRECISION (TREE_TYPE (startvar)) > TYPE_PRECISION (type))
6187 0 : t = fold_convert (signed_type_for (type), t);
6188 : }
6189 : else
6190 5524 : t = fold_build2 (PLUS_EXPR, type, t, n1);
6191 5840 : t = fold_convert (TREE_TYPE (startvar), t);
6192 5840 : t = force_gimple_operand_gsi (&gsi, t,
6193 5840 : DECL_P (startvar)
6194 5840 : && TREE_ADDRESSABLE (startvar),
6195 : NULL_TREE, false, GSI_CONTINUE_LINKING);
6196 5840 : assign_stmt = gimple_build_assign (startvar, t);
6197 5840 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6198 5840 : if (cond_var)
6199 : {
6200 49 : tree itype = TREE_TYPE (cond_var);
6201 : /* For lastprivate(conditional:) itervar, we need some iteration
6202 : counter that starts at unsigned non-zero and increases.
6203 : Prefer as few IVs as possible, so if we can use startvar
6204 : itself, use that, or startvar + constant (those would be
6205 : incremented with step), and as last resort use the s0 + 1
6206 : incremented by 1. */
6207 49 : if (POINTER_TYPE_P (type)
6208 49 : || TREE_CODE (n1) != INTEGER_CST
6209 43 : || fd->loop.cond_code != LT_EXPR)
6210 6 : t = fold_build2 (PLUS_EXPR, itype, fold_convert (itype, s0),
6211 : build_int_cst (itype, 1));
6212 43 : else if (tree_int_cst_sgn (n1) == 1)
6213 21 : t = fold_convert (itype, t);
6214 : else
6215 : {
6216 22 : tree c = fold_convert (itype, n1);
6217 22 : c = fold_build2 (MINUS_EXPR, itype, build_int_cst (itype, 1), c);
6218 22 : t = fold_build2 (PLUS_EXPR, itype, fold_convert (itype, t), c);
6219 : }
6220 49 : t = force_gimple_operand_gsi (&gsi, t, false,
6221 : NULL_TREE, false, GSI_CONTINUE_LINKING);
6222 49 : assign_stmt = gimple_build_assign (cond_var, t);
6223 49 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6224 : }
6225 :
6226 5840 : t = fold_convert (itype, e0);
6227 5840 : t = fold_build2 (MULT_EXPR, itype, t, step);
6228 5840 : if (POINTER_TYPE_P (type))
6229 : {
6230 316 : t = fold_build_pointer_plus (n1, t);
6231 380 : if (!POINTER_TYPE_P (TREE_TYPE (startvar))
6232 380 : && TYPE_PRECISION (TREE_TYPE (startvar)) > TYPE_PRECISION (type))
6233 0 : t = fold_convert (signed_type_for (type), t);
6234 : }
6235 : else
6236 5524 : t = fold_build2 (PLUS_EXPR, type, t, n1);
6237 5840 : t = fold_convert (TREE_TYPE (startvar), t);
6238 5840 : e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6239 : false, GSI_CONTINUE_LINKING);
6240 5840 : if (endvar)
6241 : {
6242 4202 : assign_stmt = gimple_build_assign (endvar, e);
6243 4202 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6244 4202 : if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e)))
6245 3434 : assign_stmt = gimple_build_assign (fd->loop.v, e);
6246 : else
6247 768 : assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e);
6248 4202 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6249 : }
6250 : /* Handle linear clause adjustments. */
6251 5840 : tree itercnt = NULL_TREE, itercntbias = NULL_TREE;
6252 5840 : if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_FOR)
6253 15776 : for (tree c = gimple_omp_for_clauses (fd->for_stmt);
6254 15776 : c; c = OMP_CLAUSE_CHAIN (c))
6255 13401 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
6256 13401 : && !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
6257 : {
6258 62 : tree d = OMP_CLAUSE_DECL (c);
6259 62 : tree t = d, a, dest;
6260 62 : if (omp_privatize_by_reference (t))
6261 3 : t = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), t);
6262 62 : tree type = TREE_TYPE (t);
6263 62 : if (POINTER_TYPE_P (type))
6264 0 : type = sizetype;
6265 62 : dest = unshare_expr (t);
6266 62 : tree v = create_tmp_var (TREE_TYPE (t), NULL);
6267 62 : expand_omp_build_assign (&gsif, v, t);
6268 62 : if (itercnt == NULL_TREE)
6269 : {
6270 62 : if (gimple_omp_for_combined_into_p (fd->for_stmt))
6271 : {
6272 0 : itercntbias
6273 0 : = fold_build2 (MINUS_EXPR, itype, fold_convert (itype, n1),
6274 : fold_convert (itype, fd->loop.n1));
6275 0 : itercntbias = fold_build2 (EXACT_DIV_EXPR, itype,
6276 : itercntbias, step);
6277 0 : itercntbias
6278 0 : = force_gimple_operand_gsi (&gsif, itercntbias, true,
6279 : NULL_TREE, true,
6280 : GSI_SAME_STMT);
6281 0 : itercnt = fold_build2 (PLUS_EXPR, itype, itercntbias, s0);
6282 0 : itercnt = force_gimple_operand_gsi (&gsi, itercnt, true,
6283 : NULL_TREE, false,
6284 : GSI_CONTINUE_LINKING);
6285 : }
6286 : else
6287 : itercnt = s0;
6288 : }
6289 62 : a = fold_build2 (MULT_EXPR, type,
6290 : fold_convert (type, itercnt),
6291 : fold_convert (type, OMP_CLAUSE_LINEAR_STEP (c)));
6292 62 : t = fold_build2 (type == TREE_TYPE (t) ? PLUS_EXPR
6293 : : POINTER_PLUS_EXPR, TREE_TYPE (t), v, a);
6294 62 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6295 : false, GSI_CONTINUE_LINKING);
6296 62 : expand_omp_build_assign (&gsi, dest, t, true);
6297 : }
6298 5840 : if (fd->collapse > 1)
6299 2226 : expand_omp_for_init_vars (fd, &gsi, counts, NULL, inner_stmt, startvar);
6300 :
6301 5840 : if (!broken_loop)
6302 : {
6303 : /* The code controlling the sequential loop goes in CONT_BB,
6304 : replacing the GIMPLE_OMP_CONTINUE. */
6305 5583 : gsi = gsi_last_nondebug_bb (cont_bb);
6306 5583 : gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
6307 5583 : vmain = gimple_omp_continue_control_use (cont_stmt);
6308 5583 : vback = gimple_omp_continue_control_def (cont_stmt);
6309 :
6310 5583 : if (cond_var)
6311 : {
6312 49 : tree itype = TREE_TYPE (cond_var);
6313 49 : tree t2;
6314 49 : if (POINTER_TYPE_P (type)
6315 49 : || TREE_CODE (n1) != INTEGER_CST
6316 43 : || fd->loop.cond_code != LT_EXPR)
6317 6 : t2 = build_int_cst (itype, 1);
6318 : else
6319 43 : t2 = fold_convert (itype, step);
6320 49 : t2 = fold_build2 (PLUS_EXPR, itype, cond_var, t2);
6321 49 : t2 = force_gimple_operand_gsi (&gsi, t2, false,
6322 : NULL_TREE, true, GSI_SAME_STMT);
6323 49 : assign_stmt = gimple_build_assign (cond_var, t2);
6324 49 : gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
6325 : }
6326 :
6327 5583 : if (!gimple_omp_for_combined_p (fd->for_stmt))
6328 : {
6329 1381 : if (POINTER_TYPE_P (type))
6330 96 : t = fold_build_pointer_plus (vmain, step);
6331 : else
6332 1285 : t = fold_build2 (PLUS_EXPR, type, vmain, step);
6333 1381 : t = force_gimple_operand_gsi (&gsi, t,
6334 1381 : DECL_P (vback)
6335 1381 : && TREE_ADDRESSABLE (vback), NULL_TREE,
6336 : true, GSI_SAME_STMT);
6337 1381 : assign_stmt = gimple_build_assign (vback, t);
6338 1381 : gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
6339 :
6340 1381 : if (tree_int_cst_equal (fd->chunk_size, integer_one_node))
6341 93 : t = build2 (EQ_EXPR, boolean_type_node,
6342 : build_int_cst (itype, 0),
6343 : build_int_cst (itype, 1));
6344 : else
6345 1288 : t = build2 (fd->loop.cond_code, boolean_type_node,
6346 1288 : DECL_P (vback) && TREE_ADDRESSABLE (vback)
6347 : ? t : vback, e);
6348 1381 : gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
6349 : }
6350 :
6351 : /* Remove GIMPLE_OMP_CONTINUE. */
6352 5583 : gsi_remove (&gsi, true);
6353 :
6354 5583 : if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
6355 443 : collapse_bb = extract_omp_for_update_vars (fd, NULL, cont_bb, body_bb);
6356 :
6357 : /* Trip update code goes into TRIP_UPDATE_BB. */
6358 5583 : gsi = gsi_start_bb (trip_update_bb);
6359 :
6360 5583 : t = build_int_cst (itype, 1);
6361 5583 : t = build2 (PLUS_EXPR, itype, trip_main, t);
6362 5583 : assign_stmt = gimple_build_assign (trip_back, t);
6363 5583 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6364 : }
6365 :
6366 5840 : gsi = gsi_last_nondebug_bb (exit_bb);
6367 5840 : if (flag_openmp_ompt)
6368 : {
6369 : /* Insert call to GOMP_*_static_worksharing_end at the end of exit_bb.
6370 : */
6371 16 : tree decl;
6372 16 : switch (gimple_omp_for_kind (fd->for_stmt))
6373 : {
6374 8 : case GF_OMP_FOR_KIND_FOR:
6375 8 : decl
6376 8 : = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_STATIC_WORKSHARING_END);
6377 8 : break;
6378 8 : case GF_OMP_FOR_KIND_DISTRIBUTE:
6379 8 : decl = builtin_decl_explicit (
6380 : BUILT_IN_GOMP_DISTRIBUTE_STATIC_WORKSHARING_END);
6381 8 : break;
6382 0 : default:
6383 0 : gcc_unreachable ();
6384 : }
6385 16 : gcall *g = gimple_build_call (decl, 0);
6386 16 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
6387 : }
6388 :
6389 : /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6390 5840 : if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
6391 : {
6392 310 : t = gimple_omp_return_lhs (gsi_stmt (gsi));
6393 310 : if (fd->have_reductemp || fd->have_pointer_condtemp)
6394 : {
6395 35 : tree fn;
6396 35 : if (t)
6397 0 : fn = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL);
6398 : else
6399 35 : fn = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END);
6400 35 : gcall *g = gimple_build_call (fn, 0);
6401 35 : if (t)
6402 : {
6403 0 : gimple_call_set_lhs (g, t);
6404 0 : if (fd->have_reductemp)
6405 0 : gsi_insert_after (&gsi, gimple_build_assign (reductions,
6406 : NOP_EXPR, t),
6407 : GSI_SAME_STMT);
6408 : }
6409 35 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
6410 35 : }
6411 : else
6412 275 : gsi_insert_after (&gsi,
6413 : omp_build_barrier (t,
6414 : GOMP_BARRIER_IMPLICIT_WORKSHARE),
6415 : GSI_SAME_STMT);
6416 : }
6417 5530 : else if (fd->have_pointer_condtemp)
6418 : {
6419 9 : tree fn = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT);
6420 9 : gcall *g = gimple_build_call (fn, 0);
6421 9 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
6422 : }
6423 5840 : gsi_remove (&gsi, true);
6424 :
6425 5840 : if (flag_openmp_ompt_detailed)
6426 : {
6427 : /* Insert call to GOMP_*_static_worksharing_dispatch at the end of
6428 : seq_start_bb. */
6429 8 : gsi = gsi_last_nondebug_bb (seq_start_bb);
6430 8 : tree decl;
6431 8 : switch (gimple_omp_for_kind (fd->for_stmt))
6432 : {
6433 4 : case GF_OMP_FOR_KIND_FOR:
6434 4 : decl = builtin_decl_explicit (
6435 : BUILT_IN_GOMP_LOOP_STATIC_WORKSHARING_DISPATCH);
6436 4 : break;
6437 4 : case GF_OMP_FOR_KIND_DISTRIBUTE:
6438 4 : decl = builtin_decl_explicit (
6439 : BUILT_IN_GOMP_DISTRIBUTE_STATIC_WORKSHARING_DISPATCH);
6440 4 : break;
6441 0 : default:
6442 0 : gcc_unreachable ();
6443 : }
6444 8 : gcall *g = gimple_build_call (decl, 0);
6445 8 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
6446 : }
6447 :
6448 : /* Connect the new blocks. */
6449 5840 : find_edge (iter_part_bb, seq_start_bb)->flags = EDGE_TRUE_VALUE;
6450 5840 : find_edge (iter_part_bb, fin_bb)->flags = EDGE_FALSE_VALUE;
6451 :
6452 5840 : if (!broken_loop)
6453 : {
6454 5583 : se = find_edge (cont_bb, body_bb);
6455 5583 : if (se == NULL)
6456 : {
6457 12 : se = BRANCH_EDGE (cont_bb);
6458 12 : gcc_assert (single_succ (se->dest) == body_bb);
6459 : }
6460 5583 : if (gimple_omp_for_combined_p (fd->for_stmt))
6461 : {
6462 4202 : remove_edge (se);
6463 4202 : se = NULL;
6464 : }
6465 1381 : else if (fd->collapse > 1)
6466 : {
6467 443 : remove_edge (se);
6468 443 : se = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
6469 : }
6470 : else
6471 938 : se->flags = EDGE_TRUE_VALUE;
6472 10723 : find_edge (cont_bb, trip_update_bb)->flags
6473 443 : = se ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
6474 :
6475 5583 : redirect_edge_and_branch (single_succ_edge (trip_update_bb),
6476 : iter_part_bb);
6477 : }
6478 :
6479 5840 : if (gimple_in_ssa_p (cfun))
6480 : {
6481 12 : gphi_iterator psi;
6482 12 : gphi *phi;
6483 12 : edge re, ene;
6484 12 : edge_var_map *vm;
6485 12 : size_t i;
6486 :
6487 12 : gcc_assert (fd->collapse == 1 && !broken_loop);
6488 :
6489 : /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6490 : remove arguments of the phi nodes in fin_bb. We need to create
6491 : appropriate phi nodes in iter_part_bb instead. */
6492 12 : se = find_edge (iter_part_bb, fin_bb);
6493 12 : re = single_succ_edge (trip_update_bb);
6494 12 : vec<edge_var_map> *head = redirect_edge_var_map_vector (re);
6495 12 : ene = single_succ_edge (entry_bb);
6496 :
6497 12 : psi = gsi_start_phis (fin_bb);
6498 21 : for (i = 0; !gsi_end_p (psi) && head->iterate (i, &vm);
6499 9 : gsi_next (&psi), ++i)
6500 : {
6501 9 : gphi *nphi;
6502 9 : location_t locus;
6503 :
6504 9 : phi = psi.phi ();
6505 9 : if (operand_equal_p (gimple_phi_arg_def (phi, 0),
6506 9 : redirect_edge_var_map_def (vm), 0))
6507 0 : continue;
6508 :
6509 9 : t = gimple_phi_result (phi);
6510 9 : gcc_assert (t == redirect_edge_var_map_result (vm));
6511 :
6512 9 : if (!single_pred_p (fin_bb))
6513 1 : t = copy_ssa_name (t, phi);
6514 :
6515 9 : nphi = create_phi_node (t, iter_part_bb);
6516 :
6517 9 : t = PHI_ARG_DEF_FROM_EDGE (phi, se);
6518 9 : locus = gimple_phi_arg_location_from_edge (phi, se);
6519 :
6520 : /* A special case -- fd->loop.v is not yet computed in
6521 : iter_part_bb, we need to use vextra instead. */
6522 9 : if (t == fd->loop.v)
6523 0 : t = vextra;
6524 9 : add_phi_arg (nphi, t, ene, locus);
6525 9 : locus = redirect_edge_var_map_location (vm);
6526 9 : tree back_arg = redirect_edge_var_map_def (vm);
6527 9 : add_phi_arg (nphi, back_arg, re, locus);
6528 9 : edge ce = find_edge (cont_bb, body_bb);
6529 9 : if (ce == NULL)
6530 : {
6531 9 : ce = BRANCH_EDGE (cont_bb);
6532 9 : gcc_assert (single_succ (ce->dest) == body_bb);
6533 : ce = single_succ_edge (ce->dest);
6534 : }
6535 9 : gphi *inner_loop_phi = find_phi_with_arg_on_edge (back_arg, ce);
6536 9 : gcc_assert (inner_loop_phi != NULL);
6537 9 : add_phi_arg (inner_loop_phi, gimple_phi_result (nphi),
6538 : find_edge (seq_start_bb, body_bb), locus);
6539 :
6540 9 : if (!single_pred_p (fin_bb))
6541 1 : add_phi_arg (phi, gimple_phi_result (nphi), se, locus);
6542 : }
6543 21 : gcc_assert (gsi_end_p (psi) && (head == NULL || i == head->length ()));
6544 12 : redirect_edge_var_map_clear (re);
6545 12 : if (single_pred_p (fin_bb))
6546 27 : while (1)
6547 : {
6548 19 : psi = gsi_start_phis (fin_bb);
6549 19 : if (gsi_end_p (psi))
6550 : break;
6551 8 : remove_phi_node (&psi, false);
6552 : }
6553 :
6554 : /* Make phi node for trip. */
6555 12 : phi = create_phi_node (trip_main, iter_part_bb);
6556 12 : add_phi_arg (phi, trip_back, single_succ_edge (trip_update_bb),
6557 : UNKNOWN_LOCATION);
6558 12 : add_phi_arg (phi, trip_init, single_succ_edge (entry_bb),
6559 : UNKNOWN_LOCATION);
6560 : }
6561 :
6562 5840 : if (!broken_loop)
6563 5583 : set_immediate_dominator (CDI_DOMINATORS, trip_update_bb, cont_bb);
6564 5840 : set_immediate_dominator (CDI_DOMINATORS, iter_part_bb,
6565 : recompute_dominator (CDI_DOMINATORS, iter_part_bb));
6566 5840 : set_immediate_dominator (CDI_DOMINATORS, fin_bb,
6567 : recompute_dominator (CDI_DOMINATORS, fin_bb));
6568 5840 : set_immediate_dominator (CDI_DOMINATORS, seq_start_bb,
6569 : recompute_dominator (CDI_DOMINATORS, seq_start_bb));
6570 5840 : set_immediate_dominator (CDI_DOMINATORS, body_bb,
6571 : recompute_dominator (CDI_DOMINATORS, body_bb));
6572 :
6573 5840 : if (!broken_loop)
6574 : {
6575 5583 : class loop *loop = body_bb->loop_father;
6576 5583 : class loop *trip_loop = alloc_loop ();
6577 5583 : trip_loop->header = iter_part_bb;
6578 5583 : trip_loop->latch = trip_update_bb;
6579 5583 : add_loop (trip_loop, iter_part_bb->loop_father);
6580 :
6581 5583 : if (loop != entry_bb->loop_father)
6582 : {
6583 12 : gcc_assert (loop->header == body_bb);
6584 12 : gcc_assert (loop->latch == region->cont
6585 : || single_pred (loop->latch) == region->cont);
6586 12 : trip_loop->inner = loop;
6587 12 : return;
6588 : }
6589 :
6590 5571 : if (!gimple_omp_for_combined_p (fd->for_stmt))
6591 : {
6592 1369 : loop = alloc_loop ();
6593 1369 : loop->header = body_bb;
6594 1369 : if (collapse_bb == NULL)
6595 926 : loop->latch = cont_bb;
6596 1369 : add_loop (loop, trip_loop);
6597 : }
6598 : }
6599 : }
6600 :
6601 : /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
6602 : loop. Given parameters:
6603 :
6604 : for (V = N1; V cond N2; V += STEP) BODY;
6605 :
6606 : where COND is "<" or ">", we generate pseudocode
6607 :
6608 : V = N1;
6609 : goto L1;
6610 : L0:
6611 : BODY;
6612 : V += STEP;
6613 : L1:
6614 : if (V cond N2) goto L0; else goto L2;
6615 : L2:
6616 :
6617 : For collapsed loops, emit the outer loops as scalar
6618 : and only try to vectorize the innermost loop. */
6619 :
6620 : static void
6621 9355 : expand_omp_simd (struct omp_region *region, struct omp_for_data *fd)
6622 : {
6623 9355 : tree type, t;
6624 9355 : basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, l2_bb, l2_dom_bb;
6625 9355 : gimple_stmt_iterator gsi;
6626 9355 : gimple *stmt;
6627 9355 : gcond *cond_stmt;
6628 9355 : bool broken_loop = region->cont == NULL;
6629 9355 : edge e, ne;
6630 9355 : tree *counts = NULL;
6631 9355 : int i;
6632 9355 : int safelen_int = INT_MAX;
6633 9355 : bool dont_vectorize = false;
6634 9355 : tree safelen = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
6635 : OMP_CLAUSE_SAFELEN);
6636 9355 : tree simduid = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
6637 : OMP_CLAUSE__SIMDUID_);
6638 9355 : tree ifc = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
6639 : OMP_CLAUSE_IF);
6640 9355 : tree simdlen = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
6641 : OMP_CLAUSE_SIMDLEN);
6642 9355 : tree condtemp = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
6643 : OMP_CLAUSE__CONDTEMP_);
6644 9355 : tree n1, n2;
6645 9432 : tree cond_var = condtemp ? OMP_CLAUSE_DECL (condtemp) : NULL_TREE;
6646 :
6647 9355 : if (safelen)
6648 : {
6649 5351 : poly_uint64 val;
6650 5351 : safelen = OMP_CLAUSE_SAFELEN_EXPR (safelen);
6651 5351 : if (!poly_int_tree_p (safelen, &val))
6652 : safelen_int = 0;
6653 : else
6654 5336 : safelen_int = MIN (constant_lower_bound (val), INT_MAX);
6655 5336 : if (safelen_int == 1)
6656 1789 : safelen_int = 0;
6657 : }
6658 759 : if ((ifc && integer_zerop (OMP_CLAUSE_IF_EXPR (ifc)))
6659 9990 : || (simdlen && integer_onep (OMP_CLAUSE_SIMDLEN_EXPR (simdlen))))
6660 : {
6661 : safelen_int = 0;
6662 : dont_vectorize = true;
6663 : }
6664 9355 : type = TREE_TYPE (fd->loop.v);
6665 9355 : entry_bb = region->entry;
6666 9355 : cont_bb = region->cont;
6667 9355 : gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
6668 9355 : gcc_assert (broken_loop
6669 : || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
6670 9355 : l0_bb = FALLTHRU_EDGE (entry_bb)->dest;
6671 9355 : if (!broken_loop)
6672 : {
6673 8283 : gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb);
6674 8283 : gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
6675 8283 : l1_bb = split_block (cont_bb, last_nondebug_stmt (cont_bb))->dest;
6676 8283 : l2_bb = BRANCH_EDGE (entry_bb)->dest;
6677 : }
6678 : else
6679 : {
6680 1072 : BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL;
6681 1072 : l1_bb = split_edge (BRANCH_EDGE (entry_bb));
6682 1072 : l2_bb = single_succ (l1_bb);
6683 : }
6684 9355 : exit_bb = region->exit;
6685 9355 : l2_dom_bb = NULL;
6686 :
6687 9355 : gsi = gsi_last_nondebug_bb (entry_bb);
6688 :
6689 9355 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
6690 : /* Not needed in SSA form right now. */
6691 9355 : gcc_assert (!gimple_in_ssa_p (cfun));
6692 9355 : if (fd->collapse > 1
6693 9355 : && (gimple_omp_for_combined_into_p (fd->for_stmt)
6694 496 : || broken_loop))
6695 : {
6696 2277 : int first_zero_iter = -1, dummy = -1;
6697 2277 : basic_block zero_iter_bb = l2_bb, dummy_bb = NULL;
6698 :
6699 2277 : counts = XALLOCAVEC (tree, fd->collapse);
6700 2277 : expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
6701 : zero_iter_bb, first_zero_iter,
6702 : dummy_bb, dummy, l2_dom_bb);
6703 : }
6704 9355 : if (l2_dom_bb == NULL)
6705 9347 : l2_dom_bb = l1_bb;
6706 :
6707 9355 : n1 = fd->loop.n1;
6708 9355 : n2 = fd->loop.n2;
6709 9355 : if (gimple_omp_for_combined_into_p (fd->for_stmt))
6710 : {
6711 7004 : tree innerc = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
6712 : OMP_CLAUSE__LOOPTEMP_);
6713 7004 : gcc_assert (innerc);
6714 7004 : n1 = OMP_CLAUSE_DECL (innerc);
6715 7004 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
6716 : OMP_CLAUSE__LOOPTEMP_);
6717 7004 : gcc_assert (innerc);
6718 7004 : n2 = OMP_CLAUSE_DECL (innerc);
6719 : }
6720 9355 : tree step = fd->loop.step;
6721 9355 : tree orig_step = step; /* May be different from step if is_simt. */
6722 :
6723 9355 : bool is_simt = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
6724 9355 : OMP_CLAUSE__SIMT_);
6725 9355 : if (is_simt)
6726 : {
6727 0 : cfun->curr_properties &= ~PROP_gimple_lomp_dev;
6728 0 : is_simt = safelen_int > 1;
6729 : }
6730 9355 : tree simt_lane = NULL_TREE, simt_maxlane = NULL_TREE;
6731 9355 : if (is_simt)
6732 : {
6733 0 : simt_lane = create_tmp_var (unsigned_type_node);
6734 0 : gimple *g = gimple_build_call_internal (IFN_GOMP_SIMT_LANE, 0);
6735 0 : gimple_call_set_lhs (g, simt_lane);
6736 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
6737 0 : tree offset = fold_build2 (MULT_EXPR, TREE_TYPE (step), step,
6738 : fold_convert (TREE_TYPE (step), simt_lane));
6739 0 : n1 = fold_convert (type, n1);
6740 0 : if (POINTER_TYPE_P (type))
6741 0 : n1 = fold_build_pointer_plus (n1, offset);
6742 : else
6743 0 : n1 = fold_build2 (PLUS_EXPR, type, n1, fold_convert (type, offset));
6744 :
6745 : /* Collapsed loops not handled for SIMT yet: limit to one lane only. */
6746 0 : if (fd->collapse > 1)
6747 0 : simt_maxlane = build_one_cst (unsigned_type_node);
6748 0 : else if (safelen_int < omp_max_simt_vf ())
6749 0 : simt_maxlane = build_int_cst (unsigned_type_node, safelen_int);
6750 0 : tree vf
6751 0 : = build_call_expr_internal_loc (UNKNOWN_LOCATION, IFN_GOMP_SIMT_VF,
6752 : unsigned_type_node, 0);
6753 0 : if (simt_maxlane)
6754 0 : vf = fold_build2 (MIN_EXPR, unsigned_type_node, vf, simt_maxlane);
6755 0 : vf = fold_convert (TREE_TYPE (step), vf);
6756 0 : step = fold_build2 (MULT_EXPR, TREE_TYPE (step), step, vf);
6757 : }
6758 :
6759 9355 : tree n2var = NULL_TREE;
6760 9355 : tree n2v = NULL_TREE;
6761 9355 : tree *nonrect_bounds = NULL;
6762 9355 : tree min_arg1 = NULL_TREE, min_arg2 = NULL_TREE;
6763 9355 : if (fd->collapse > 1)
6764 : {
6765 2753 : if (broken_loop || gimple_omp_for_combined_into_p (fd->for_stmt))
6766 : {
6767 2277 : if (fd->non_rect)
6768 : {
6769 99 : nonrect_bounds = XALLOCAVEC (tree, fd->last_nonrect + 1);
6770 99 : memset (nonrect_bounds, 0,
6771 : sizeof (tree) * (fd->last_nonrect + 1));
6772 : }
6773 2277 : expand_omp_build_assign (&gsi, fd->loop.v, fold_convert (type, n1));
6774 2277 : gcc_assert (entry_bb == gsi_bb (gsi));
6775 2277 : gcc_assert (fd->for_stmt == gsi_stmt (gsi));
6776 2277 : gsi_prev (&gsi);
6777 2277 : entry_bb = split_block (entry_bb, gsi_stmt (gsi))->dest;
6778 2277 : expand_omp_for_init_vars (fd, &gsi, counts, nonrect_bounds,
6779 : NULL, n1);
6780 2277 : gsi = gsi_for_stmt (fd->for_stmt);
6781 : }
6782 2753 : if (broken_loop)
6783 : ;
6784 2217 : else if (gimple_omp_for_combined_into_p (fd->for_stmt))
6785 : {
6786 : /* Compute in n2var the limit for the first innermost loop,
6787 : i.e. fd->loop.v + MIN (n2 - fd->loop.v, cnt)
6788 : where cnt is how many iterations would the loop have if
6789 : all further iterations were assigned to the current task. */
6790 1741 : n2var = create_tmp_var (type);
6791 1741 : i = fd->collapse - 1;
6792 1741 : tree itype = TREE_TYPE (fd->loops[i].v);
6793 1741 : if (POINTER_TYPE_P (itype))
6794 266 : itype = signed_type_for (itype);
6795 1741 : t = build_int_cst (itype, (fd->loops[i].cond_code == LT_EXPR
6796 2511 : ? -1 : 1));
6797 1741 : t = fold_build2 (PLUS_EXPR, itype,
6798 : fold_convert (itype, fd->loops[i].step), t);
6799 1741 : t = fold_build2 (PLUS_EXPR, itype, t,
6800 : fold_convert (itype, fd->loops[i].n2));
6801 1741 : if (fd->loops[i].m2)
6802 : {
6803 26 : tree t2 = fold_convert (itype,
6804 : fd->loops[i - fd->loops[i].outer].v);
6805 26 : tree t3 = fold_convert (itype, fd->loops[i].m2);
6806 26 : t2 = fold_build2 (MULT_EXPR, TREE_TYPE (t), t2, t3);
6807 26 : t = fold_build2 (PLUS_EXPR, itype, t, t2);
6808 : }
6809 1741 : t = fold_build2 (MINUS_EXPR, itype, t,
6810 : fold_convert (itype, fd->loops[i].v));
6811 1741 : if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR)
6812 256 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
6813 : fold_build1 (NEGATE_EXPR, itype, t),
6814 : fold_build1 (NEGATE_EXPR, itype,
6815 : fold_convert (itype,
6816 : fd->loops[i].step)));
6817 : else
6818 1485 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t,
6819 : fold_convert (itype, fd->loops[i].step));
6820 1741 : t = fold_convert (type, t);
6821 1741 : tree t2 = fold_build2 (MINUS_EXPR, type, n2, n1);
6822 1741 : min_arg1 = create_tmp_var (type);
6823 1741 : expand_omp_build_assign (&gsi, min_arg1, t2);
6824 1741 : min_arg2 = create_tmp_var (type);
6825 1741 : expand_omp_build_assign (&gsi, min_arg2, t);
6826 : }
6827 : else
6828 : {
6829 476 : if (TREE_CODE (n2) == INTEGER_CST)
6830 : {
6831 : /* Indicate for lastprivate handling that at least one iteration
6832 : has been performed, without wasting runtime. */
6833 138 : if (integer_nonzerop (n2))
6834 130 : expand_omp_build_assign (&gsi, fd->loop.v,
6835 : fold_convert (type, n2));
6836 : else
6837 : /* Indicate that no iteration has been performed. */
6838 8 : expand_omp_build_assign (&gsi, fd->loop.v,
6839 : build_one_cst (type));
6840 : }
6841 : else
6842 : {
6843 338 : expand_omp_build_assign (&gsi, fd->loop.v,
6844 : build_zero_cst (type));
6845 338 : expand_omp_build_assign (&gsi, n2, build_one_cst (type));
6846 : }
6847 476 : for (i = 0; i < fd->collapse; i++)
6848 : {
6849 476 : t = fold_convert (TREE_TYPE (fd->loops[i].v), fd->loops[i].n1);
6850 476 : if (fd->loops[i].m1)
6851 : {
6852 0 : tree t2
6853 0 : = fold_convert (TREE_TYPE (t),
6854 : fd->loops[i - fd->loops[i].outer].v);
6855 0 : tree t3 = fold_convert (TREE_TYPE (t), fd->loops[i].m1);
6856 0 : t2 = fold_build2 (MULT_EXPR, TREE_TYPE (t), t2, t3);
6857 0 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (t), t, t2);
6858 : }
6859 476 : expand_omp_build_assign (&gsi, fd->loops[i].v, t);
6860 : /* For normal non-combined collapsed loops just initialize
6861 : the outermost iterator in the entry_bb. */
6862 476 : if (!broken_loop)
6863 : break;
6864 : }
6865 : }
6866 : }
6867 : else
6868 6602 : expand_omp_build_assign (&gsi, fd->loop.v, fold_convert (type, n1));
6869 9355 : tree altv = NULL_TREE, altn2 = NULL_TREE;
6870 9355 : if (fd->collapse == 1
6871 6602 : && !broken_loop
6872 6066 : && TREE_CODE (orig_step) != INTEGER_CST)
6873 : {
6874 : /* The vectorizer currently punts on loops with non-constant steps
6875 : for the main IV (can't compute number of iterations and gives up
6876 : because of that). As for OpenMP loops it is always possible to
6877 : compute the number of iterations upfront, use an alternate IV
6878 : as the loop iterator:
6879 : altn2 = n1 < n2 ? (n2 - n1 + step - 1) / step : 0;
6880 : for (i = n1, altv = 0; altv < altn2; altv++, i += step) */
6881 176 : altv = create_tmp_var (unsigned_type_for (TREE_TYPE (fd->loops[0].v)));
6882 176 : expand_omp_build_assign (&gsi, altv, build_zero_cst (TREE_TYPE (altv)));
6883 176 : tree itype = TREE_TYPE (fd->loop.v);
6884 176 : if (POINTER_TYPE_P (itype))
6885 0 : itype = signed_type_for (itype);
6886 334 : t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
6887 176 : t = fold_build2 (PLUS_EXPR, itype,
6888 : fold_convert (itype, step), t);
6889 176 : t = fold_build2 (PLUS_EXPR, itype, t, fold_convert (itype, n2));
6890 176 : t = fold_build2 (MINUS_EXPR, itype, t,
6891 : fold_convert (itype, fd->loop.v));
6892 176 : if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
6893 0 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
6894 : fold_build1 (NEGATE_EXPR, itype, t),
6895 : fold_build1 (NEGATE_EXPR, itype,
6896 : fold_convert (itype, step)));
6897 : else
6898 176 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t,
6899 : fold_convert (itype, step));
6900 176 : t = fold_convert (TREE_TYPE (altv), t);
6901 176 : altn2 = create_tmp_var (TREE_TYPE (altv));
6902 176 : expand_omp_build_assign (&gsi, altn2, t);
6903 176 : tree t2 = fold_convert (TREE_TYPE (fd->loop.v), n2);
6904 176 : t2 = fold_build2 (fd->loop.cond_code, boolean_type_node, fd->loop.v, t2);
6905 176 : t2 = force_gimple_operand_gsi (&gsi, t2, true, NULL_TREE,
6906 : true, GSI_SAME_STMT);
6907 176 : gassign *g = gimple_build_assign (altn2, COND_EXPR, t2, altn2,
6908 176 : build_zero_cst (TREE_TYPE (altv)));
6909 176 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
6910 176 : }
6911 9179 : else if (fd->collapse > 1
6912 2753 : && !broken_loop
6913 2217 : && !gimple_omp_for_combined_into_p (fd->for_stmt)
6914 9655 : && TREE_CODE (fd->loops[fd->collapse - 1].step) != INTEGER_CST)
6915 : {
6916 22 : altv = create_tmp_var (unsigned_type_for (TREE_TYPE (fd->loops[0].v)));
6917 22 : altn2 = create_tmp_var (TREE_TYPE (altv));
6918 : }
6919 9355 : if (cond_var)
6920 : {
6921 77 : if (POINTER_TYPE_P (type)
6922 77 : || TREE_CODE (n1) != INTEGER_CST
6923 15 : || fd->loop.cond_code != LT_EXPR
6924 92 : || tree_int_cst_sgn (n1) != 1)
6925 72 : expand_omp_build_assign (&gsi, cond_var,
6926 72 : build_one_cst (TREE_TYPE (cond_var)));
6927 : else
6928 5 : expand_omp_build_assign (&gsi, cond_var,
6929 5 : fold_convert (TREE_TYPE (cond_var), n1));
6930 : }
6931 :
6932 : /* Remove the GIMPLE_OMP_FOR statement. */
6933 9355 : gsi_remove (&gsi, true);
6934 :
6935 9355 : if (!broken_loop)
6936 : {
6937 : /* Code to control the increment goes in the CONT_BB. */
6938 8283 : gsi = gsi_last_nondebug_bb (cont_bb);
6939 8283 : stmt = gsi_stmt (gsi);
6940 8283 : gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
6941 :
6942 8283 : if (fd->collapse == 1
6943 8283 : || gimple_omp_for_combined_into_p (fd->for_stmt))
6944 : {
6945 7807 : if (POINTER_TYPE_P (type))
6946 286 : t = fold_build_pointer_plus (fd->loop.v, step);
6947 : else
6948 7521 : t = fold_build2 (PLUS_EXPR, type, fd->loop.v, step);
6949 7807 : expand_omp_build_assign (&gsi, fd->loop.v, t);
6950 : }
6951 476 : else if (TREE_CODE (n2) != INTEGER_CST)
6952 338 : expand_omp_build_assign (&gsi, fd->loop.v, build_one_cst (type));
6953 8283 : if (altv)
6954 : {
6955 198 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (altv), altv,
6956 : build_one_cst (TREE_TYPE (altv)));
6957 198 : expand_omp_build_assign (&gsi, altv, t);
6958 : }
6959 :
6960 8283 : if (fd->collapse > 1)
6961 : {
6962 2217 : i = fd->collapse - 1;
6963 2217 : if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v)))
6964 280 : t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step);
6965 : else
6966 : {
6967 1937 : t = fold_convert (TREE_TYPE (fd->loops[i].v),
6968 : fd->loops[i].step);
6969 1937 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loops[i].v),
6970 : fd->loops[i].v, t);
6971 : }
6972 2217 : expand_omp_build_assign (&gsi, fd->loops[i].v, t);
6973 : }
6974 8283 : if (cond_var)
6975 : {
6976 77 : if (POINTER_TYPE_P (type)
6977 77 : || TREE_CODE (n1) != INTEGER_CST
6978 15 : || fd->loop.cond_code != LT_EXPR
6979 92 : || tree_int_cst_sgn (n1) != 1)
6980 72 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (cond_var), cond_var,
6981 : build_one_cst (TREE_TYPE (cond_var)));
6982 : else
6983 5 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (cond_var), cond_var,
6984 : fold_convert (TREE_TYPE (cond_var), step));
6985 77 : expand_omp_build_assign (&gsi, cond_var, t);
6986 : }
6987 :
6988 : /* Remove GIMPLE_OMP_CONTINUE. */
6989 8283 : gsi_remove (&gsi, true);
6990 : }
6991 :
6992 : /* Emit the condition in L1_BB. */
6993 9355 : gsi = gsi_start_bb (l1_bb);
6994 :
6995 9355 : if (altv)
6996 198 : t = build2 (LT_EXPR, boolean_type_node, altv, altn2);
6997 9157 : else if (fd->collapse > 1
6998 2731 : && !gimple_omp_for_combined_into_p (fd->for_stmt)
6999 9631 : && !broken_loop)
7000 : {
7001 454 : i = fd->collapse - 1;
7002 454 : tree itype = TREE_TYPE (fd->loops[i].v);
7003 454 : if (fd->loops[i].m2)
7004 203 : t = n2v = create_tmp_var (itype);
7005 : else
7006 251 : t = fold_convert (itype, unshare_expr (fd->loops[i].n2));
7007 454 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7008 : false, GSI_CONTINUE_LINKING);
7009 454 : tree v = fd->loops[i].v;
7010 454 : if (DECL_P (v) && TREE_ADDRESSABLE (v))
7011 0 : v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE,
7012 : false, GSI_CONTINUE_LINKING);
7013 454 : t = build2 (fd->loops[i].cond_code, boolean_type_node, v, t);
7014 : }
7015 : else
7016 : {
7017 8703 : if (fd->collapse > 1 && !broken_loop)
7018 : t = n2var;
7019 : else
7020 6962 : t = fold_convert (type, unshare_expr (n2));
7021 8703 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7022 : false, GSI_CONTINUE_LINKING);
7023 8703 : tree v = fd->loop.v;
7024 8703 : if (DECL_P (v) && TREE_ADDRESSABLE (v))
7025 0 : v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE,
7026 : false, GSI_CONTINUE_LINKING);
7027 8703 : t = build2 (fd->loop.cond_code, boolean_type_node, v, t);
7028 : }
7029 9355 : cond_stmt = gimple_build_cond_empty (t);
7030 9355 : gsi_insert_after (&gsi, cond_stmt, GSI_CONTINUE_LINKING);
7031 9355 : if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), expand_omp_regimplify_p,
7032 : NULL, NULL)
7033 9355 : || walk_tree (gimple_cond_rhs_ptr (cond_stmt), expand_omp_regimplify_p,
7034 : NULL, NULL))
7035 : {
7036 0 : gsi = gsi_for_stmt (cond_stmt);
7037 0 : gimple_regimplify_operands (cond_stmt, &gsi);
7038 : }
7039 :
7040 : /* Add 'V -= STEP * (SIMT_VF - 1)' after the loop. */
7041 9355 : if (is_simt)
7042 : {
7043 0 : gsi = gsi_start_bb (l2_bb);
7044 0 : step = fold_build2 (MINUS_EXPR, TREE_TYPE (step), orig_step, step);
7045 0 : if (POINTER_TYPE_P (type))
7046 0 : t = fold_build_pointer_plus (fd->loop.v, step);
7047 : else
7048 0 : t = fold_build2 (PLUS_EXPR, type, fd->loop.v, step);
7049 0 : expand_omp_build_assign (&gsi, fd->loop.v, t);
7050 : }
7051 :
7052 : /* Remove GIMPLE_OMP_RETURN. */
7053 9355 : gsi = gsi_last_nondebug_bb (exit_bb);
7054 9355 : gsi_remove (&gsi, true);
7055 :
7056 : /* Connect the new blocks. */
7057 9355 : remove_edge (FALLTHRU_EDGE (entry_bb));
7058 :
7059 9355 : if (!broken_loop)
7060 : {
7061 8283 : remove_edge (BRANCH_EDGE (entry_bb));
7062 8283 : make_edge (entry_bb, l1_bb, EDGE_FALLTHRU);
7063 :
7064 8283 : e = BRANCH_EDGE (l1_bb);
7065 8283 : ne = FALLTHRU_EDGE (l1_bb);
7066 8283 : e->flags = EDGE_TRUE_VALUE;
7067 : }
7068 : else
7069 : {
7070 1072 : single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
7071 :
7072 1072 : ne = single_succ_edge (l1_bb);
7073 1072 : e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE);
7074 :
7075 : }
7076 9355 : ne->flags = EDGE_FALSE_VALUE;
7077 9355 : e->probability = profile_probability::guessed_always ().apply_scale (7, 8);
7078 9355 : ne->probability = e->probability.invert ();
7079 :
7080 9355 : set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb);
7081 9355 : set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb);
7082 :
7083 9355 : if (simt_maxlane)
7084 : {
7085 0 : cond_stmt = gimple_build_cond (LT_EXPR, simt_lane, simt_maxlane,
7086 : NULL_TREE, NULL_TREE);
7087 0 : gsi = gsi_last_bb (entry_bb);
7088 0 : gsi_insert_after (&gsi, cond_stmt, GSI_NEW_STMT);
7089 0 : make_edge (entry_bb, l2_bb, EDGE_FALSE_VALUE);
7090 0 : FALLTHRU_EDGE (entry_bb)->flags = EDGE_TRUE_VALUE;
7091 0 : FALLTHRU_EDGE (entry_bb)->probability
7092 0 : = profile_probability::guessed_always ().apply_scale (7, 8);
7093 0 : BRANCH_EDGE (entry_bb)->probability
7094 0 : = FALLTHRU_EDGE (entry_bb)->probability.invert ();
7095 0 : l2_dom_bb = entry_bb;
7096 : }
7097 9355 : set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb);
7098 :
7099 9355 : if (!broken_loop && fd->collapse > 1)
7100 : {
7101 2217 : basic_block last_bb = l1_bb;
7102 2217 : basic_block init_bb = NULL;
7103 6108 : for (i = fd->collapse - 2; i >= 0; i--)
7104 : {
7105 3891 : tree nextn2v = NULL_TREE;
7106 3891 : if (EDGE_SUCC (last_bb, 0)->flags & EDGE_FALSE_VALUE)
7107 : e = EDGE_SUCC (last_bb, 0);
7108 : else
7109 2217 : e = EDGE_SUCC (last_bb, 1);
7110 3891 : basic_block bb = split_edge (e);
7111 3891 : if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v)))
7112 553 : t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step);
7113 : else
7114 : {
7115 3338 : t = fold_convert (TREE_TYPE (fd->loops[i].v),
7116 : fd->loops[i].step);
7117 3338 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loops[i].v),
7118 : fd->loops[i].v, t);
7119 : }
7120 3891 : gsi = gsi_after_labels (bb);
7121 3891 : expand_omp_build_assign (&gsi, fd->loops[i].v, t);
7122 :
7123 3891 : bb = split_block (bb, last_nondebug_stmt (bb))->dest;
7124 3891 : gsi = gsi_start_bb (bb);
7125 3891 : tree itype = TREE_TYPE (fd->loops[i].v);
7126 3891 : if (fd->loops[i].m2)
7127 0 : t = nextn2v = create_tmp_var (itype);
7128 : else
7129 3891 : t = fold_convert (itype, unshare_expr (fd->loops[i].n2));
7130 3891 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7131 : false, GSI_CONTINUE_LINKING);
7132 3891 : tree v = fd->loops[i].v;
7133 3891 : if (DECL_P (v) && TREE_ADDRESSABLE (v))
7134 0 : v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE,
7135 : false, GSI_CONTINUE_LINKING);
7136 3891 : t = build2 (fd->loops[i].cond_code, boolean_type_node, v, t);
7137 3891 : cond_stmt = gimple_build_cond_empty (t);
7138 3891 : gsi_insert_after (&gsi, cond_stmt, GSI_CONTINUE_LINKING);
7139 3891 : if (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
7140 : expand_omp_regimplify_p, NULL, NULL)
7141 3891 : || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
7142 : expand_omp_regimplify_p, NULL, NULL))
7143 : {
7144 6 : gsi = gsi_for_stmt (cond_stmt);
7145 6 : gimple_regimplify_operands (cond_stmt, &gsi);
7146 : }
7147 3891 : ne = single_succ_edge (bb);
7148 3891 : ne->flags = EDGE_FALSE_VALUE;
7149 :
7150 3891 : init_bb = create_empty_bb (bb);
7151 3891 : set_immediate_dominator (CDI_DOMINATORS, init_bb, bb);
7152 3891 : add_bb_to_loop (init_bb, bb->loop_father);
7153 3891 : e = make_edge (bb, init_bb, EDGE_TRUE_VALUE);
7154 3891 : e->probability
7155 3891 : = profile_probability::guessed_always ().apply_scale (7, 8);
7156 3891 : ne->probability = e->probability.invert ();
7157 :
7158 3891 : gsi = gsi_after_labels (init_bb);
7159 3891 : if (fd->loops[i + 1].m1)
7160 : {
7161 242 : tree t2 = fold_convert (TREE_TYPE (fd->loops[i + 1].v),
7162 : fd->loops[i + 1
7163 : - fd->loops[i + 1].outer].v);
7164 242 : if (POINTER_TYPE_P (TREE_TYPE (t2)))
7165 6 : t = fold_build_pointer_plus (t2, fd->loops[i + 1].n1);
7166 : else
7167 : {
7168 236 : t = fold_convert (TREE_TYPE (fd->loops[i + 1].v),
7169 : fd->loops[i + 1].n1);
7170 236 : tree t3 = fold_convert (TREE_TYPE (t), fd->loops[i + 1].m1);
7171 236 : t2 = fold_build2 (MULT_EXPR, TREE_TYPE (t), t2, t3);
7172 236 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (t), t, t2);
7173 : }
7174 : }
7175 : else
7176 3649 : t = fold_convert (TREE_TYPE (fd->loops[i + 1].v),
7177 : fd->loops[i + 1].n1);
7178 3891 : expand_omp_build_assign (&gsi, fd->loops[i + 1].v, t);
7179 3891 : if (fd->loops[i + 1].m2)
7180 : {
7181 241 : if (i + 2 == fd->collapse && (n2var || altv))
7182 : {
7183 38 : gcc_assert (n2v == NULL_TREE);
7184 38 : n2v = create_tmp_var (TREE_TYPE (fd->loops[i + 1].v));
7185 : }
7186 241 : tree t2 = fold_convert (TREE_TYPE (fd->loops[i + 1].v),
7187 : fd->loops[i + 1
7188 : - fd->loops[i + 1].outer].v);
7189 241 : if (POINTER_TYPE_P (TREE_TYPE (t2)))
7190 6 : t = fold_build_pointer_plus (t2, fd->loops[i + 1].n2);
7191 : else
7192 : {
7193 235 : t = fold_convert (TREE_TYPE (fd->loops[i + 1].v),
7194 : fd->loops[i + 1].n2);
7195 235 : tree t3 = fold_convert (TREE_TYPE (t), fd->loops[i + 1].m2);
7196 235 : t2 = fold_build2 (MULT_EXPR, TREE_TYPE (t), t2, t3);
7197 235 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (t), t, t2);
7198 : }
7199 241 : expand_omp_build_assign (&gsi, n2v, t);
7200 : }
7201 3891 : if (i + 2 == fd->collapse && n2var)
7202 : {
7203 : /* For composite simd, n2 is the first iteration the current
7204 : task shouldn't already handle, so we effectively want to use
7205 : for (V3 = N31; V < N2 && V3 < N32; V++, V3 += STEP3)
7206 : as the vectorized loop. Except the vectorizer will not
7207 : vectorize that, so instead compute N2VAR as
7208 : N2VAR = V + MIN (N2 - V, COUNTS3) and use
7209 : for (V3 = N31; V < N2VAR; V++, V3 += STEP3)
7210 : as the loop to vectorize. */
7211 1741 : tree t2 = fold_build2 (MINUS_EXPR, type, n2, fd->loop.v);
7212 1741 : if (fd->loops[i + 1].m1 || fd->loops[i + 1].m2)
7213 : {
7214 99 : tree itype = TREE_TYPE (fd->loops[i].v);
7215 99 : if (POINTER_TYPE_P (itype))
7216 4 : itype = signed_type_for (itype);
7217 99 : t = build_int_cst (itype, (fd->loops[i + 1].cond_code
7218 99 : == LT_EXPR ? -1 : 1));
7219 99 : t = fold_build2 (PLUS_EXPR, itype,
7220 : fold_convert (itype,
7221 : fd->loops[i + 1].step), t);
7222 99 : if (fd->loops[i + 1].m2 == NULL_TREE)
7223 73 : t = fold_build2 (PLUS_EXPR, itype, t,
7224 : fold_convert (itype,
7225 : fd->loops[i + 1].n2));
7226 26 : else if (POINTER_TYPE_P (TREE_TYPE (n2v)))
7227 : {
7228 4 : t = fold_build_pointer_plus (n2v, t);
7229 4 : t = fold_convert (itype, t);
7230 : }
7231 : else
7232 22 : t = fold_build2 (PLUS_EXPR, itype, t, n2v);
7233 99 : t = fold_build2 (MINUS_EXPR, itype, t,
7234 : fold_convert (itype, fd->loops[i + 1].v));
7235 99 : tree step = fold_convert (itype, fd->loops[i + 1].step);
7236 99 : if (TYPE_UNSIGNED (itype)
7237 99 : && fd->loops[i + 1].cond_code == GT_EXPR)
7238 0 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
7239 : fold_build1 (NEGATE_EXPR, itype, t),
7240 : fold_build1 (NEGATE_EXPR, itype, step));
7241 : else
7242 99 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
7243 99 : t = fold_convert (type, t);
7244 99 : }
7245 : else
7246 1642 : t = counts[i + 1];
7247 1741 : expand_omp_build_assign (&gsi, min_arg1, t2);
7248 1741 : expand_omp_build_assign (&gsi, min_arg2, t);
7249 1741 : e = split_block (init_bb, last_nondebug_stmt (init_bb));
7250 1741 : gsi = gsi_after_labels (e->dest);
7251 1741 : init_bb = e->dest;
7252 1741 : remove_edge (FALLTHRU_EDGE (entry_bb));
7253 1741 : make_edge (entry_bb, init_bb, EDGE_FALLTHRU);
7254 1741 : set_immediate_dominator (CDI_DOMINATORS, init_bb, entry_bb);
7255 1741 : set_immediate_dominator (CDI_DOMINATORS, l1_bb, init_bb);
7256 1741 : t = fold_build2 (MIN_EXPR, type, min_arg1, min_arg2);
7257 1741 : t = fold_build2 (PLUS_EXPR, type, fd->loop.v, t);
7258 1741 : expand_omp_build_assign (&gsi, n2var, t);
7259 : }
7260 3891 : if (i + 2 == fd->collapse && altv)
7261 : {
7262 : /* The vectorizer currently punts on loops with non-constant
7263 : steps for the main IV (can't compute number of iterations
7264 : and gives up because of that). As for OpenMP loops it is
7265 : always possible to compute the number of iterations upfront,
7266 : use an alternate IV as the loop iterator. */
7267 22 : expand_omp_build_assign (&gsi, altv,
7268 22 : build_zero_cst (TREE_TYPE (altv)));
7269 22 : tree itype = TREE_TYPE (fd->loops[i + 1].v);
7270 22 : if (POINTER_TYPE_P (itype))
7271 0 : itype = signed_type_for (itype);
7272 22 : t = build_int_cst (itype, (fd->loops[i + 1].cond_code == LT_EXPR
7273 26 : ? -1 : 1));
7274 22 : t = fold_build2 (PLUS_EXPR, itype,
7275 : fold_convert (itype, fd->loops[i + 1].step), t);
7276 22 : t = fold_build2 (PLUS_EXPR, itype, t,
7277 : fold_convert (itype,
7278 : fd->loops[i + 1].m2
7279 : ? n2v : fd->loops[i + 1].n2));
7280 22 : t = fold_build2 (MINUS_EXPR, itype, t,
7281 : fold_convert (itype, fd->loops[i + 1].v));
7282 22 : tree step = fold_convert (itype, fd->loops[i + 1].step);
7283 22 : if (TYPE_UNSIGNED (itype)
7284 22 : && fd->loops[i + 1].cond_code == GT_EXPR)
7285 0 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
7286 : fold_build1 (NEGATE_EXPR, itype, t),
7287 : fold_build1 (NEGATE_EXPR, itype, step));
7288 : else
7289 22 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
7290 22 : t = fold_convert (TREE_TYPE (altv), t);
7291 22 : expand_omp_build_assign (&gsi, altn2, t);
7292 22 : tree t2 = fold_convert (TREE_TYPE (fd->loops[i + 1].v),
7293 : fd->loops[i + 1].m2
7294 : ? n2v : fd->loops[i + 1].n2);
7295 22 : t2 = fold_build2 (fd->loops[i + 1].cond_code, boolean_type_node,
7296 : fd->loops[i + 1].v, t2);
7297 22 : t2 = force_gimple_operand_gsi (&gsi, t2, true, NULL_TREE,
7298 : true, GSI_SAME_STMT);
7299 22 : gassign *g
7300 22 : = gimple_build_assign (altn2, COND_EXPR, t2, altn2,
7301 22 : build_zero_cst (TREE_TYPE (altv)));
7302 22 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7303 : }
7304 3891 : n2v = nextn2v;
7305 :
7306 3891 : make_edge (init_bb, last_bb, EDGE_FALLTHRU);
7307 3891 : if (!gimple_omp_for_combined_into_p (fd->for_stmt))
7308 : {
7309 787 : e = find_edge (entry_bb, last_bb);
7310 787 : redirect_edge_succ (e, bb);
7311 787 : set_immediate_dominator (CDI_DOMINATORS, bb, entry_bb);
7312 787 : set_immediate_dominator (CDI_DOMINATORS, last_bb, init_bb);
7313 : }
7314 :
7315 3891 : last_bb = bb;
7316 : }
7317 : }
7318 9355 : if (!broken_loop)
7319 : {
7320 8283 : class loop *loop = alloc_loop ();
7321 8283 : loop->header = l1_bb;
7322 8283 : loop->latch = cont_bb;
7323 8283 : add_loop (loop, l1_bb->loop_father);
7324 8283 : loop->safelen = safelen_int;
7325 8283 : if (simduid)
7326 : {
7327 3473 : loop->simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
7328 3473 : cfun->has_simduid_loops = true;
7329 : }
7330 : /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
7331 : the loop. */
7332 8283 : if ((flag_tree_loop_vectorize
7333 3297 : || !OPTION_SET_P (flag_tree_loop_vectorize))
7334 8283 : && flag_tree_loop_optimize
7335 8282 : && loop->safelen > 1)
7336 : {
7337 6495 : loop->force_vectorize = true;
7338 6495 : if (simdlen && tree_fits_uhwi_p (OMP_CLAUSE_SIMDLEN_EXPR (simdlen)))
7339 : {
7340 60 : unsigned HOST_WIDE_INT v
7341 60 : = tree_to_uhwi (OMP_CLAUSE_SIMDLEN_EXPR (simdlen));
7342 60 : if (v < INT_MAX && v <= (unsigned HOST_WIDE_INT) loop->safelen)
7343 60 : loop->simdlen = v;
7344 : }
7345 6495 : cfun->has_force_vectorize_loops = true;
7346 6495 : }
7347 1788 : else if (dont_vectorize)
7348 234 : loop->dont_vectorize = true;
7349 : }
7350 1072 : else if (simduid)
7351 530 : cfun->has_simduid_loops = true;
7352 9355 : }
7353 :
7354 : /* Taskloop construct is represented after gimplification with
7355 : two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
7356 : in between them. This routine expands the outer GIMPLE_OMP_FOR,
7357 : which should just compute all the needed loop temporaries
7358 : for GIMPLE_OMP_TASK. */
7359 :
7360 : static void
7361 1330 : expand_omp_taskloop_for_outer (struct omp_region *region,
7362 : struct omp_for_data *fd,
7363 : gimple *inner_stmt)
7364 : {
7365 1330 : tree type, bias = NULL_TREE;
7366 1330 : basic_block entry_bb, cont_bb, exit_bb;
7367 1330 : gimple_stmt_iterator gsi;
7368 1330 : gassign *assign_stmt;
7369 1330 : tree *counts = NULL;
7370 1330 : int i;
7371 :
7372 1330 : gcc_assert (inner_stmt);
7373 1330 : gcc_assert (region->cont);
7374 1330 : gcc_assert (gimple_code (inner_stmt) == GIMPLE_OMP_TASK
7375 : && gimple_omp_task_taskloop_p (inner_stmt));
7376 1330 : type = TREE_TYPE (fd->loop.v);
7377 :
7378 : /* See if we need to bias by LLONG_MIN. */
7379 1330 : if (fd->iter_type == long_long_unsigned_type_node
7380 42 : && (TREE_CODE (type) == INTEGER_TYPE || TREE_CODE (type) == BITINT_TYPE)
7381 1358 : && !TYPE_UNSIGNED (type))
7382 : {
7383 0 : tree n1, n2;
7384 :
7385 0 : if (fd->loop.cond_code == LT_EXPR)
7386 : {
7387 0 : n1 = fd->loop.n1;
7388 0 : n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step);
7389 : }
7390 : else
7391 : {
7392 0 : n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step);
7393 0 : n2 = fd->loop.n1;
7394 : }
7395 0 : if (TREE_CODE (n1) != INTEGER_CST
7396 0 : || TREE_CODE (n2) != INTEGER_CST
7397 0 : || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0)))
7398 0 : bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type));
7399 : }
7400 :
7401 1330 : entry_bb = region->entry;
7402 1330 : cont_bb = region->cont;
7403 1330 : gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
7404 1330 : gcc_assert (BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
7405 1330 : exit_bb = region->exit;
7406 :
7407 1330 : gsi = gsi_last_nondebug_bb (entry_bb);
7408 1330 : gimple *for_stmt = gsi_stmt (gsi);
7409 1330 : gcc_assert (gimple_code (for_stmt) == GIMPLE_OMP_FOR);
7410 1330 : if (fd->collapse > 1)
7411 : {
7412 181 : int first_zero_iter = -1, dummy = -1;
7413 181 : basic_block zero_iter_bb = NULL, dummy_bb = NULL, l2_dom_bb = NULL;
7414 :
7415 181 : counts = XALLOCAVEC (tree, fd->collapse);
7416 181 : expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
7417 : zero_iter_bb, first_zero_iter,
7418 : dummy_bb, dummy, l2_dom_bb);
7419 :
7420 181 : if (zero_iter_bb)
7421 : {
7422 : /* Some counts[i] vars might be uninitialized if
7423 : some loop has zero iterations. But the body shouldn't
7424 : be executed in that case, so just avoid uninit warnings. */
7425 336 : for (i = first_zero_iter; i < fd->collapse; i++)
7426 239 : if (SSA_VAR_P (counts[i]))
7427 226 : suppress_warning (counts[i], OPT_Wuninitialized);
7428 97 : gsi_prev (&gsi);
7429 97 : edge e = split_block (entry_bb, gsi_stmt (gsi));
7430 97 : entry_bb = e->dest;
7431 97 : make_edge (zero_iter_bb, entry_bb, EDGE_FALLTHRU);
7432 97 : gsi = gsi_last_bb (entry_bb);
7433 97 : set_immediate_dominator (CDI_DOMINATORS, entry_bb,
7434 : get_immediate_dominator (CDI_DOMINATORS,
7435 : zero_iter_bb));
7436 : }
7437 : }
7438 :
7439 1330 : tree t0, t1;
7440 1330 : t1 = fd->loop.n2;
7441 1330 : t0 = fd->loop.n1;
7442 2646 : if (POINTER_TYPE_P (TREE_TYPE (t0))
7443 1330 : && TYPE_PRECISION (TREE_TYPE (t0))
7444 14 : != TYPE_PRECISION (fd->iter_type))
7445 : {
7446 : /* Avoid casting pointers to integer of a different size. */
7447 0 : tree itype = signed_type_for (type);
7448 0 : t1 = fold_convert (fd->iter_type, fold_convert (itype, t1));
7449 0 : t0 = fold_convert (fd->iter_type, fold_convert (itype, t0));
7450 : }
7451 : else
7452 : {
7453 1330 : t1 = fold_convert (fd->iter_type, t1);
7454 1330 : t0 = fold_convert (fd->iter_type, t0);
7455 : }
7456 1330 : if (bias)
7457 : {
7458 0 : t1 = fold_build2 (PLUS_EXPR, fd->iter_type, t1, bias);
7459 0 : t0 = fold_build2 (PLUS_EXPR, fd->iter_type, t0, bias);
7460 : }
7461 :
7462 1330 : tree innerc = omp_find_clause (gimple_omp_task_clauses (inner_stmt),
7463 : OMP_CLAUSE__LOOPTEMP_);
7464 1330 : gcc_assert (innerc);
7465 1330 : tree startvar = OMP_CLAUSE_DECL (innerc);
7466 1330 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc), OMP_CLAUSE__LOOPTEMP_);
7467 1330 : gcc_assert (innerc);
7468 1330 : tree endvar = OMP_CLAUSE_DECL (innerc);
7469 1330 : if (fd->collapse > 1 && TREE_CODE (fd->loop.n2) != INTEGER_CST)
7470 : {
7471 101 : innerc = find_lastprivate_looptemp (fd, innerc);
7472 101 : if (innerc)
7473 : {
7474 : /* If needed (inner taskloop has lastprivate clause), propagate
7475 : down the total number of iterations. */
7476 31 : tree t = force_gimple_operand_gsi (&gsi, fd->loop.n2, false,
7477 : NULL_TREE, false,
7478 : GSI_CONTINUE_LINKING);
7479 31 : assign_stmt = gimple_build_assign (OMP_CLAUSE_DECL (innerc), t);
7480 31 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7481 : }
7482 : }
7483 :
7484 1330 : t0 = force_gimple_operand_gsi (&gsi, t0, false, NULL_TREE, false,
7485 : GSI_CONTINUE_LINKING);
7486 1330 : assign_stmt = gimple_build_assign (startvar, t0);
7487 1330 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7488 :
7489 1330 : t1 = force_gimple_operand_gsi (&gsi, t1, false, NULL_TREE, false,
7490 : GSI_CONTINUE_LINKING);
7491 1330 : assign_stmt = gimple_build_assign (endvar, t1);
7492 1330 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7493 1330 : if (fd->collapse > 1)
7494 181 : expand_omp_for_init_vars (fd, &gsi, counts, NULL, inner_stmt, startvar);
7495 :
7496 : /* Remove the GIMPLE_OMP_FOR statement. */
7497 1330 : gsi = gsi_for_stmt (for_stmt);
7498 1330 : gsi_remove (&gsi, true);
7499 :
7500 1330 : gsi = gsi_last_nondebug_bb (cont_bb);
7501 1330 : gsi_remove (&gsi, true);
7502 :
7503 1330 : gsi = gsi_last_nondebug_bb (exit_bb);
7504 1330 : gsi_remove (&gsi, true);
7505 :
7506 1330 : FALLTHRU_EDGE (entry_bb)->probability = profile_probability::always ();
7507 1330 : remove_edge (BRANCH_EDGE (entry_bb));
7508 1330 : FALLTHRU_EDGE (cont_bb)->probability = profile_probability::always ();
7509 1330 : remove_edge (BRANCH_EDGE (cont_bb));
7510 1330 : set_immediate_dominator (CDI_DOMINATORS, exit_bb, cont_bb);
7511 1330 : set_immediate_dominator (CDI_DOMINATORS, region->entry,
7512 : recompute_dominator (CDI_DOMINATORS, region->entry));
7513 1330 : }
7514 :
7515 : /* Taskloop construct is represented after gimplification with
7516 : two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
7517 : in between them. This routine expands the inner GIMPLE_OMP_FOR.
7518 : GOMP_taskloop{,_ull} function arranges for each task to be given just
7519 : a single range of iterations. */
7520 :
7521 : static void
7522 1330 : expand_omp_taskloop_for_inner (struct omp_region *region,
7523 : struct omp_for_data *fd,
7524 : gimple *inner_stmt)
7525 : {
7526 1330 : tree e, t, type, itype, vmain, vback, bias = NULL_TREE;
7527 1330 : basic_block entry_bb, exit_bb, body_bb, cont_bb, collapse_bb = NULL;
7528 1330 : basic_block fin_bb;
7529 1330 : gimple_stmt_iterator gsi;
7530 1330 : edge ep;
7531 1330 : bool broken_loop = region->cont == NULL;
7532 1330 : tree *counts = NULL;
7533 1330 : tree n1, n2, step;
7534 :
7535 1330 : itype = type = TREE_TYPE (fd->loop.v);
7536 1330 : if (POINTER_TYPE_P (type))
7537 14 : itype = signed_type_for (type);
7538 :
7539 : /* See if we need to bias by LLONG_MIN. */
7540 1330 : if (fd->iter_type == long_long_unsigned_type_node
7541 42 : && (TREE_CODE (type) == INTEGER_TYPE || TREE_CODE (type) == BITINT_TYPE)
7542 1358 : && !TYPE_UNSIGNED (type))
7543 : {
7544 0 : tree n1, n2;
7545 :
7546 0 : if (fd->loop.cond_code == LT_EXPR)
7547 : {
7548 0 : n1 = fd->loop.n1;
7549 0 : n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step);
7550 : }
7551 : else
7552 : {
7553 0 : n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step);
7554 0 : n2 = fd->loop.n1;
7555 : }
7556 0 : if (TREE_CODE (n1) != INTEGER_CST
7557 0 : || TREE_CODE (n2) != INTEGER_CST
7558 0 : || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0)))
7559 0 : bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type));
7560 : }
7561 :
7562 1330 : entry_bb = region->entry;
7563 1330 : cont_bb = region->cont;
7564 1330 : gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
7565 1330 : fin_bb = BRANCH_EDGE (entry_bb)->dest;
7566 1330 : gcc_assert (broken_loop
7567 : || (fin_bb == FALLTHRU_EDGE (cont_bb)->dest));
7568 1330 : body_bb = FALLTHRU_EDGE (entry_bb)->dest;
7569 1330 : if (!broken_loop)
7570 : {
7571 1314 : gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
7572 1314 : gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
7573 : }
7574 1330 : exit_bb = region->exit;
7575 :
7576 : /* Iteration space partitioning goes in ENTRY_BB. */
7577 1330 : gsi = gsi_last_nondebug_bb (entry_bb);
7578 1330 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
7579 :
7580 1330 : if (fd->collapse > 1)
7581 : {
7582 181 : int first_zero_iter = -1, dummy = -1;
7583 181 : basic_block l2_dom_bb = NULL, dummy_bb = NULL;
7584 :
7585 181 : counts = XALLOCAVEC (tree, fd->collapse);
7586 181 : expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
7587 : fin_bb, first_zero_iter,
7588 : dummy_bb, dummy, l2_dom_bb);
7589 181 : t = NULL_TREE;
7590 : }
7591 : else
7592 1330 : t = integer_one_node;
7593 :
7594 1330 : step = fd->loop.step;
7595 1330 : tree innerc = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
7596 : OMP_CLAUSE__LOOPTEMP_);
7597 1330 : gcc_assert (innerc);
7598 1330 : n1 = OMP_CLAUSE_DECL (innerc);
7599 1330 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc), OMP_CLAUSE__LOOPTEMP_);
7600 1330 : gcc_assert (innerc);
7601 1330 : n2 = OMP_CLAUSE_DECL (innerc);
7602 1330 : if (bias)
7603 : {
7604 0 : n1 = fold_build2 (PLUS_EXPR, fd->iter_type, n1, bias);
7605 0 : n2 = fold_build2 (PLUS_EXPR, fd->iter_type, n2, bias);
7606 : }
7607 1330 : n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
7608 : true, NULL_TREE, true, GSI_SAME_STMT);
7609 1330 : n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
7610 : true, NULL_TREE, true, GSI_SAME_STMT);
7611 1330 : step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
7612 : true, NULL_TREE, true, GSI_SAME_STMT);
7613 :
7614 1330 : tree startvar = fd->loop.v;
7615 1330 : tree endvar = NULL_TREE;
7616 :
7617 1330 : if (gimple_omp_for_combined_p (fd->for_stmt))
7618 : {
7619 617 : tree clauses = gimple_omp_for_clauses (inner_stmt);
7620 617 : tree innerc = omp_find_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
7621 617 : gcc_assert (innerc);
7622 617 : startvar = OMP_CLAUSE_DECL (innerc);
7623 617 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
7624 : OMP_CLAUSE__LOOPTEMP_);
7625 617 : gcc_assert (innerc);
7626 617 : endvar = OMP_CLAUSE_DECL (innerc);
7627 : }
7628 1330 : t = fold_convert (TREE_TYPE (startvar), n1);
7629 1330 : t = force_gimple_operand_gsi (&gsi, t,
7630 1330 : DECL_P (startvar)
7631 1330 : && TREE_ADDRESSABLE (startvar),
7632 : NULL_TREE, false, GSI_CONTINUE_LINKING);
7633 1330 : gimple *assign_stmt = gimple_build_assign (startvar, t);
7634 1330 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7635 :
7636 1330 : t = fold_convert (TREE_TYPE (startvar), n2);
7637 1330 : e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7638 : false, GSI_CONTINUE_LINKING);
7639 1330 : if (endvar)
7640 : {
7641 617 : assign_stmt = gimple_build_assign (endvar, e);
7642 617 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7643 617 : if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e)))
7644 529 : assign_stmt = gimple_build_assign (fd->loop.v, e);
7645 : else
7646 88 : assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e);
7647 617 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7648 : }
7649 :
7650 1330 : tree *nonrect_bounds = NULL;
7651 1330 : if (fd->collapse > 1)
7652 : {
7653 181 : if (fd->non_rect)
7654 : {
7655 19 : nonrect_bounds = XALLOCAVEC (tree, fd->last_nonrect + 1);
7656 19 : memset (nonrect_bounds, 0, sizeof (tree) * (fd->last_nonrect + 1));
7657 : }
7658 181 : gcc_assert (gsi_bb (gsi) == entry_bb);
7659 181 : expand_omp_for_init_vars (fd, &gsi, counts, nonrect_bounds, inner_stmt,
7660 : startvar);
7661 181 : entry_bb = gsi_bb (gsi);
7662 : }
7663 :
7664 1330 : if (!broken_loop)
7665 : {
7666 : /* The code controlling the sequential loop replaces the
7667 : GIMPLE_OMP_CONTINUE. */
7668 1314 : gsi = gsi_last_nondebug_bb (cont_bb);
7669 1314 : gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
7670 1314 : gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE);
7671 1314 : vmain = gimple_omp_continue_control_use (cont_stmt);
7672 1314 : vback = gimple_omp_continue_control_def (cont_stmt);
7673 :
7674 1314 : if (!gimple_omp_for_combined_p (fd->for_stmt))
7675 : {
7676 697 : if (POINTER_TYPE_P (type))
7677 8 : t = fold_build_pointer_plus (vmain, step);
7678 : else
7679 689 : t = fold_build2 (PLUS_EXPR, type, vmain, step);
7680 697 : t = force_gimple_operand_gsi (&gsi, t,
7681 697 : DECL_P (vback)
7682 697 : && TREE_ADDRESSABLE (vback),
7683 : NULL_TREE, true, GSI_SAME_STMT);
7684 697 : assign_stmt = gimple_build_assign (vback, t);
7685 697 : gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
7686 :
7687 697 : t = build2 (fd->loop.cond_code, boolean_type_node,
7688 697 : DECL_P (vback) && TREE_ADDRESSABLE (vback)
7689 : ? t : vback, e);
7690 697 : gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
7691 : }
7692 :
7693 : /* Remove the GIMPLE_OMP_CONTINUE statement. */
7694 1314 : gsi_remove (&gsi, true);
7695 :
7696 1314 : if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
7697 81 : collapse_bb = extract_omp_for_update_vars (fd, nonrect_bounds,
7698 : cont_bb, body_bb);
7699 : }
7700 :
7701 : /* Remove the GIMPLE_OMP_FOR statement. */
7702 1330 : gsi = gsi_for_stmt (fd->for_stmt);
7703 1330 : gsi_remove (&gsi, true);
7704 :
7705 : /* Remove the GIMPLE_OMP_RETURN statement. */
7706 1330 : gsi = gsi_last_nondebug_bb (exit_bb);
7707 1330 : gsi_remove (&gsi, true);
7708 :
7709 1330 : FALLTHRU_EDGE (entry_bb)->probability = profile_probability::always ();
7710 1330 : if (!broken_loop)
7711 1314 : remove_edge (BRANCH_EDGE (entry_bb));
7712 : else
7713 : {
7714 16 : remove_edge_and_dominated_blocks (BRANCH_EDGE (entry_bb));
7715 16 : region->outer->cont = NULL;
7716 : }
7717 :
7718 : /* Connect all the blocks. */
7719 1330 : if (!broken_loop)
7720 : {
7721 1314 : ep = find_edge (cont_bb, body_bb);
7722 1314 : if (gimple_omp_for_combined_p (fd->for_stmt))
7723 : {
7724 617 : remove_edge (ep);
7725 617 : ep = NULL;
7726 : }
7727 697 : else if (fd->collapse > 1)
7728 : {
7729 81 : remove_edge (ep);
7730 81 : ep = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
7731 : }
7732 : else
7733 616 : ep->flags = EDGE_TRUE_VALUE;
7734 2547 : find_edge (cont_bb, fin_bb)->flags
7735 1395 : = ep ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
7736 : }
7737 :
7738 1330 : set_immediate_dominator (CDI_DOMINATORS, body_bb,
7739 : recompute_dominator (CDI_DOMINATORS, body_bb));
7740 1330 : if (!broken_loop)
7741 1314 : set_immediate_dominator (CDI_DOMINATORS, fin_bb,
7742 : recompute_dominator (CDI_DOMINATORS, fin_bb));
7743 :
7744 1314 : if (!broken_loop && !gimple_omp_for_combined_p (fd->for_stmt))
7745 : {
7746 697 : class loop *loop = alloc_loop ();
7747 697 : loop->header = body_bb;
7748 697 : if (collapse_bb == NULL)
7749 616 : loop->latch = cont_bb;
7750 697 : add_loop (loop, body_bb->loop_father);
7751 : }
7752 1330 : }
7753 :
7754 : /* A subroutine of expand_omp_for. Generate code for an OpenACC
7755 : partitioned loop. The lowering here is abstracted, in that the
7756 : loop parameters are passed through internal functions, which are
7757 : further lowered by oacc_device_lower, once we get to the target
7758 : compiler. The loop is of the form:
7759 :
7760 : for (V = B; V LTGT E; V += S) {BODY}
7761 :
7762 : where LTGT is < or >. We may have a specified chunking size, CHUNKING
7763 : (constant 0 for no chunking) and we will have a GWV partitioning
7764 : mask, specifying dimensions over which the loop is to be
7765 : partitioned (see note below). We generate code that looks like
7766 : (this ignores tiling):
7767 :
7768 : <entry_bb> [incoming FALL->body, BRANCH->exit]
7769 : typedef signedintify (typeof (V)) T; // underlying signed integral type
7770 : T range = E - B;
7771 : T chunk_no = 0;
7772 : T DIR = LTGT == '<' ? +1 : -1;
7773 : T chunk_max = GOACC_LOOP_CHUNK (dir, range, S, CHUNK_SIZE, GWV);
7774 : T step = GOACC_LOOP_STEP (dir, range, S, CHUNK_SIZE, GWV);
7775 :
7776 : <head_bb> [created by splitting end of entry_bb]
7777 : T offset = GOACC_LOOP_OFFSET (dir, range, S, CHUNK_SIZE, GWV, chunk_no);
7778 : T bound = GOACC_LOOP_BOUND (dir, range, S, CHUNK_SIZE, GWV, offset);
7779 : if (!(offset LTGT bound)) goto bottom_bb;
7780 :
7781 : <body_bb> [incoming]
7782 : V = B + offset;
7783 : {BODY}
7784 :
7785 : <cont_bb> [incoming, may == body_bb FALL->exit_bb, BRANCH->body_bb]
7786 : offset += step;
7787 : if (offset LTGT bound) goto body_bb; [*]
7788 :
7789 : <bottom_bb> [created by splitting start of exit_bb] insert BRANCH->head_bb
7790 : chunk_no++;
7791 : if (chunk < chunk_max) goto head_bb;
7792 :
7793 : <exit_bb> [incoming]
7794 : V = B + ((range -/+ 1) / S +/- 1) * S [*]
7795 :
7796 : [*] Needed if V live at end of loop. */
7797 :
7798 : static void
7799 11711 : expand_oacc_for (struct omp_region *region, struct omp_for_data *fd)
7800 : {
7801 11711 : bool is_oacc_kernels_parallelized
7802 11711 : = (lookup_attribute ("oacc kernels parallelized",
7803 11711 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
7804 11711 : {
7805 11711 : bool is_oacc_kernels
7806 11711 : = (lookup_attribute ("oacc kernels",
7807 11711 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
7808 11711 : if (is_oacc_kernels_parallelized)
7809 386 : gcc_checking_assert (is_oacc_kernels);
7810 : }
7811 23422 : gcc_assert (gimple_in_ssa_p (cfun) == is_oacc_kernels_parallelized);
7812 : /* In the following, some of the 'gimple_in_ssa_p (cfun)' conditionals are
7813 : for SSA specifics, and some are for 'parloops' OpenACC
7814 : 'kernels'-parallelized specifics. */
7815 :
7816 11711 : tree v = fd->loop.v;
7817 11711 : enum tree_code cond_code = fd->loop.cond_code;
7818 11711 : enum tree_code plus_code = PLUS_EXPR;
7819 :
7820 11711 : tree chunk_size = integer_minus_one_node;
7821 11711 : tree gwv = integer_zero_node;
7822 11711 : tree iter_type = TREE_TYPE (v);
7823 11711 : tree diff_type = iter_type;
7824 11711 : tree plus_type = iter_type;
7825 11711 : struct oacc_collapse *counts = NULL;
7826 :
7827 11711 : gcc_checking_assert (gimple_omp_for_kind (fd->for_stmt)
7828 : == GF_OMP_FOR_KIND_OACC_LOOP);
7829 11711 : gcc_assert (!gimple_omp_for_combined_into_p (fd->for_stmt));
7830 11711 : gcc_assert (cond_code == LT_EXPR || cond_code == GT_EXPR);
7831 :
7832 11711 : if (POINTER_TYPE_P (iter_type))
7833 : {
7834 52 : plus_code = POINTER_PLUS_EXPR;
7835 52 : plus_type = sizetype;
7836 : }
7837 24124 : for (int ix = fd->collapse; ix--;)
7838 : {
7839 12413 : tree diff_type2 = TREE_TYPE (fd->loops[ix].step);
7840 12413 : if (TYPE_PRECISION (diff_type) < TYPE_PRECISION (diff_type2))
7841 0 : diff_type = diff_type2;
7842 : }
7843 11711 : if (POINTER_TYPE_P (diff_type) || TYPE_UNSIGNED (diff_type))
7844 1002 : diff_type = signed_type_for (diff_type);
7845 11711 : if (TYPE_PRECISION (diff_type) < TYPE_PRECISION (integer_type_node))
7846 23 : diff_type = integer_type_node;
7847 :
7848 11711 : basic_block entry_bb = region->entry; /* BB ending in OMP_FOR */
7849 11711 : basic_block exit_bb = region->exit; /* BB ending in OMP_RETURN */
7850 11711 : basic_block cont_bb = region->cont; /* BB ending in OMP_CONTINUE */
7851 11711 : basic_block bottom_bb = NULL;
7852 :
7853 : /* entry_bb has two successors; the branch edge is to the exit
7854 : block (or to finalization blocks preceding it), fallthrough edge
7855 : to body. */
7856 11711 : gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
7857 :
7858 : /* If cont_bb non-NULL, it has 2 successors. The branch successor is
7859 : body_bb, or to a block whose only successor is the body_bb. Its
7860 : fallthrough successor is the final block (same as the branch
7861 : successor of the entry_bb), possibly via finalization blocks. */
7862 11711 : if (cont_bb)
7863 : {
7864 11672 : basic_block body_bb = FALLTHRU_EDGE (entry_bb)->dest;
7865 11672 : basic_block bed = BRANCH_EDGE (cont_bb)->dest;
7866 :
7867 11672 : gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
7868 11672 : gcc_assert (bed == body_bb || single_succ_edge (bed)->dest == body_bb);
7869 : }
7870 : else
7871 39 : gcc_assert (!gimple_in_ssa_p (cfun));
7872 :
7873 11711 : tree chunk_no;
7874 11711 : tree chunk_max = NULL_TREE;
7875 11711 : tree bound, offset;
7876 11711 : tree step = create_tmp_var (diff_type, ".step");
7877 11711 : bool up = cond_code == LT_EXPR;
7878 11871 : tree dir = build_int_cst (diff_type, up ? +1 : -1);
7879 11711 : bool chunking = !gimple_in_ssa_p (cfun);
7880 11711 : bool negating;
7881 :
7882 : /* Tiling vars. */
7883 11711 : tree tile_size = NULL_TREE;
7884 11711 : tree element_s = NULL_TREE;
7885 11711 : tree e_bound = NULL_TREE, e_offset = NULL_TREE, e_step = NULL_TREE;
7886 11711 : basic_block elem_body_bb = NULL;
7887 11711 : basic_block elem_cont_bb = NULL;
7888 :
7889 : /* SSA instances. */
7890 11711 : tree offset_incr = NULL_TREE;
7891 11711 : tree offset_init = NULL_TREE;
7892 :
7893 11711 : gimple_stmt_iterator gsi;
7894 11711 : gassign *ass;
7895 11711 : gcall *call;
7896 11711 : gimple *stmt;
7897 11711 : tree expr;
7898 11711 : location_t loc;
7899 11711 : edge split, be, fte;
7900 :
7901 : /* Split the end of entry_bb to create head_bb. */
7902 11711 : split = split_block (entry_bb, last_nondebug_stmt (entry_bb));
7903 11711 : basic_block head_bb = split->dest;
7904 11711 : entry_bb = split->src;
7905 :
7906 : /* Chunk setup goes at end of entry_bb, replacing the omp_for. */
7907 11711 : gsi = gsi_last_nondebug_bb (entry_bb);
7908 11711 : gomp_for *for_stmt = as_a <gomp_for *> (gsi_stmt (gsi));
7909 11711 : loc = gimple_location (for_stmt);
7910 :
7911 11711 : if (gimple_in_ssa_p (cfun))
7912 : {
7913 386 : offset_init = gimple_omp_for_index (for_stmt, 0);
7914 386 : gcc_assert (integer_zerop (fd->loop.n1));
7915 : /* The SSA parallelizer does gang parallelism. */
7916 386 : gwv = build_int_cst (integer_type_node, GOMP_DIM_MASK (GOMP_DIM_GANG));
7917 : }
7918 :
7919 11711 : if (fd->collapse > 1 || fd->tiling)
7920 : {
7921 1188 : gcc_assert (!gimple_in_ssa_p (cfun) && up);
7922 594 : counts = XALLOCAVEC (struct oacc_collapse, fd->collapse);
7923 594 : tree total = expand_oacc_collapse_init (fd, &gsi, counts, diff_type,
7924 594 : TREE_TYPE (fd->loop.n2), loc);
7925 :
7926 594 : if (SSA_VAR_P (fd->loop.n2))
7927 : {
7928 107 : total = force_gimple_operand_gsi (&gsi, total, false, NULL_TREE,
7929 : true, GSI_SAME_STMT);
7930 107 : ass = gimple_build_assign (fd->loop.n2, total);
7931 107 : gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
7932 : }
7933 : }
7934 :
7935 11711 : tree b = fd->loop.n1;
7936 11711 : tree e = fd->loop.n2;
7937 11711 : tree s = fd->loop.step;
7938 :
7939 11711 : b = force_gimple_operand_gsi (&gsi, b, true, NULL_TREE, true, GSI_SAME_STMT);
7940 11711 : e = force_gimple_operand_gsi (&gsi, e, true, NULL_TREE, true, GSI_SAME_STMT);
7941 :
7942 : /* Convert the step, avoiding possible unsigned->signed overflow. */
7943 11711 : negating = !up && TYPE_UNSIGNED (TREE_TYPE (s));
7944 32 : if (negating)
7945 32 : s = fold_build1 (NEGATE_EXPR, TREE_TYPE (s), s);
7946 11711 : s = fold_convert (diff_type, s);
7947 11711 : if (negating)
7948 32 : s = fold_build1 (NEGATE_EXPR, diff_type, s);
7949 11711 : s = force_gimple_operand_gsi (&gsi, s, true, NULL_TREE, true, GSI_SAME_STMT);
7950 :
7951 11711 : if (!chunking)
7952 386 : chunk_size = integer_zero_node;
7953 11711 : expr = fold_convert (diff_type, chunk_size);
7954 11711 : chunk_size = force_gimple_operand_gsi (&gsi, expr, true,
7955 : NULL_TREE, true, GSI_SAME_STMT);
7956 :
7957 11711 : if (fd->tiling)
7958 : {
7959 : /* Determine the tile size and element step,
7960 : modify the outer loop step size. */
7961 177 : tile_size = create_tmp_var (diff_type, ".tile_size");
7962 177 : expr = build_int_cst (diff_type, 1);
7963 461 : for (int ix = 0; ix < fd->collapse; ix++)
7964 284 : expr = fold_build2 (MULT_EXPR, diff_type, counts[ix].tile, expr);
7965 177 : expr = force_gimple_operand_gsi (&gsi, expr, true,
7966 : NULL_TREE, true, GSI_SAME_STMT);
7967 177 : ass = gimple_build_assign (tile_size, expr);
7968 177 : gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
7969 :
7970 177 : element_s = create_tmp_var (diff_type, ".element_s");
7971 177 : ass = gimple_build_assign (element_s, s);
7972 177 : gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
7973 :
7974 177 : expr = fold_build2 (MULT_EXPR, diff_type, s, tile_size);
7975 177 : s = force_gimple_operand_gsi (&gsi, expr, true,
7976 : NULL_TREE, true, GSI_SAME_STMT);
7977 : }
7978 :
7979 : /* Determine the range, avoiding possible unsigned->signed overflow. */
7980 11711 : negating = !up && TYPE_UNSIGNED (iter_type);
7981 23374 : expr = fold_build2 (MINUS_EXPR, plus_type,
7982 : fold_convert (plus_type, negating ? b : e),
7983 : fold_convert (plus_type, negating ? e : b));
7984 11711 : expr = fold_convert (diff_type, expr);
7985 11711 : if (negating)
7986 48 : expr = fold_build1 (NEGATE_EXPR, diff_type, expr);
7987 11711 : tree range = force_gimple_operand_gsi (&gsi, expr, true,
7988 : NULL_TREE, true, GSI_SAME_STMT);
7989 :
7990 11711 : chunk_no = build_int_cst (diff_type, 0);
7991 11711 : if (chunking)
7992 : {
7993 11325 : gcc_assert (!gimple_in_ssa_p (cfun));
7994 :
7995 11325 : expr = chunk_no;
7996 11325 : chunk_max = create_tmp_var (diff_type, ".chunk_max");
7997 11325 : chunk_no = create_tmp_var (diff_type, ".chunk_no");
7998 :
7999 11325 : ass = gimple_build_assign (chunk_no, expr);
8000 11325 : gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
8001 :
8002 11325 : call = gimple_build_call_internal (IFN_GOACC_LOOP, 6,
8003 : build_int_cst (integer_type_node,
8004 : IFN_GOACC_LOOP_CHUNKS),
8005 : dir, range, s, chunk_size, gwv);
8006 11325 : gimple_call_set_lhs (call, chunk_max);
8007 11325 : gimple_set_location (call, loc);
8008 11325 : gsi_insert_before (&gsi, call, GSI_SAME_STMT);
8009 : }
8010 : else
8011 : chunk_size = chunk_no;
8012 :
8013 11711 : call = gimple_build_call_internal (IFN_GOACC_LOOP, 6,
8014 : build_int_cst (integer_type_node,
8015 : IFN_GOACC_LOOP_STEP),
8016 : dir, range, s, chunk_size, gwv);
8017 11711 : gimple_call_set_lhs (call, step);
8018 11711 : gimple_set_location (call, loc);
8019 11711 : gsi_insert_before (&gsi, call, GSI_SAME_STMT);
8020 :
8021 : /* Remove the GIMPLE_OMP_FOR. */
8022 11711 : gsi_remove (&gsi, true);
8023 :
8024 : /* Fixup edges from head_bb. */
8025 11711 : be = BRANCH_EDGE (head_bb);
8026 11711 : fte = FALLTHRU_EDGE (head_bb);
8027 11711 : be->flags |= EDGE_FALSE_VALUE;
8028 11711 : fte->flags ^= EDGE_FALLTHRU | EDGE_TRUE_VALUE;
8029 :
8030 11711 : basic_block body_bb = fte->dest;
8031 :
8032 11711 : if (gimple_in_ssa_p (cfun))
8033 : {
8034 386 : gsi = gsi_last_nondebug_bb (cont_bb);
8035 386 : gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
8036 :
8037 386 : offset = gimple_omp_continue_control_use (cont_stmt);
8038 386 : offset_incr = gimple_omp_continue_control_def (cont_stmt);
8039 : }
8040 : else
8041 : {
8042 11325 : offset = create_tmp_var (diff_type, ".offset");
8043 11325 : offset_init = offset_incr = offset;
8044 : }
8045 11711 : bound = create_tmp_var (TREE_TYPE (offset), ".bound");
8046 :
8047 : /* Loop offset & bound go into head_bb. */
8048 11711 : gsi = gsi_start_bb (head_bb);
8049 :
8050 11711 : call = gimple_build_call_internal (IFN_GOACC_LOOP, 7,
8051 : build_int_cst (integer_type_node,
8052 : IFN_GOACC_LOOP_OFFSET),
8053 : dir, range, s,
8054 : chunk_size, gwv, chunk_no);
8055 11711 : gimple_call_set_lhs (call, offset_init);
8056 11711 : gimple_set_location (call, loc);
8057 11711 : gsi_insert_after (&gsi, call, GSI_CONTINUE_LINKING);
8058 :
8059 11711 : call = gimple_build_call_internal (IFN_GOACC_LOOP, 7,
8060 : build_int_cst (integer_type_node,
8061 : IFN_GOACC_LOOP_BOUND),
8062 : dir, range, s,
8063 : chunk_size, gwv, offset_init);
8064 11711 : gimple_call_set_lhs (call, bound);
8065 11711 : gimple_set_location (call, loc);
8066 11711 : gsi_insert_after (&gsi, call, GSI_CONTINUE_LINKING);
8067 :
8068 11711 : expr = build2 (cond_code, boolean_type_node, offset_init, bound);
8069 11711 : gsi_insert_after (&gsi, gimple_build_cond_empty (expr),
8070 : GSI_CONTINUE_LINKING);
8071 :
8072 : /* V assignment goes into body_bb. */
8073 11711 : if (!gimple_in_ssa_p (cfun))
8074 : {
8075 11325 : gsi = gsi_start_bb (body_bb);
8076 :
8077 11325 : expr = build2 (plus_code, iter_type, b,
8078 : fold_convert (plus_type, offset));
8079 11325 : expr = force_gimple_operand_gsi (&gsi, expr, false, NULL_TREE,
8080 : true, GSI_SAME_STMT);
8081 11325 : ass = gimple_build_assign (v, expr);
8082 11325 : gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
8083 :
8084 11325 : if (fd->collapse > 1 || fd->tiling)
8085 594 : expand_oacc_collapse_vars (fd, false, &gsi, counts, v, diff_type);
8086 :
8087 11325 : if (fd->tiling)
8088 : {
8089 : /* Determine the range of the element loop -- usually simply
8090 : the tile_size, but could be smaller if the final
8091 : iteration of the outer loop is a partial tile. */
8092 177 : tree e_range = create_tmp_var (diff_type, ".e_range");
8093 :
8094 177 : expr = build2 (MIN_EXPR, diff_type,
8095 : build2 (MINUS_EXPR, diff_type, bound, offset),
8096 : build2 (MULT_EXPR, diff_type, tile_size,
8097 : element_s));
8098 177 : expr = force_gimple_operand_gsi (&gsi, expr, false, NULL_TREE,
8099 : true, GSI_SAME_STMT);
8100 177 : ass = gimple_build_assign (e_range, expr);
8101 177 : gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
8102 :
8103 : /* Determine bound, offset & step of inner loop. */
8104 177 : e_bound = create_tmp_var (diff_type, ".e_bound");
8105 177 : e_offset = create_tmp_var (diff_type, ".e_offset");
8106 177 : e_step = create_tmp_var (diff_type, ".e_step");
8107 :
8108 : /* Mark these as element loops. */
8109 177 : tree t, e_gwv = integer_minus_one_node;
8110 177 : tree chunk = build_int_cst (diff_type, 0); /* Never chunked. */
8111 :
8112 177 : t = build_int_cst (integer_type_node, IFN_GOACC_LOOP_OFFSET);
8113 177 : call = gimple_build_call_internal (IFN_GOACC_LOOP, 7, t, dir, e_range,
8114 : element_s, chunk, e_gwv, chunk);
8115 177 : gimple_call_set_lhs (call, e_offset);
8116 177 : gimple_set_location (call, loc);
8117 177 : gsi_insert_before (&gsi, call, GSI_SAME_STMT);
8118 :
8119 177 : t = build_int_cst (integer_type_node, IFN_GOACC_LOOP_BOUND);
8120 177 : call = gimple_build_call_internal (IFN_GOACC_LOOP, 7, t, dir, e_range,
8121 : element_s, chunk, e_gwv, e_offset);
8122 177 : gimple_call_set_lhs (call, e_bound);
8123 177 : gimple_set_location (call, loc);
8124 177 : gsi_insert_before (&gsi, call, GSI_SAME_STMT);
8125 :
8126 177 : t = build_int_cst (integer_type_node, IFN_GOACC_LOOP_STEP);
8127 177 : call = gimple_build_call_internal (IFN_GOACC_LOOP, 6, t, dir, e_range,
8128 : element_s, chunk, e_gwv);
8129 177 : gimple_call_set_lhs (call, e_step);
8130 177 : gimple_set_location (call, loc);
8131 177 : gsi_insert_before (&gsi, call, GSI_SAME_STMT);
8132 :
8133 : /* Add test and split block. */
8134 177 : expr = build2 (cond_code, boolean_type_node, e_offset, e_bound);
8135 177 : stmt = gimple_build_cond_empty (expr);
8136 177 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
8137 177 : split = split_block (body_bb, stmt);
8138 177 : elem_body_bb = split->dest;
8139 177 : if (cont_bb == body_bb)
8140 146 : cont_bb = elem_body_bb;
8141 177 : body_bb = split->src;
8142 :
8143 177 : split->flags ^= EDGE_FALLTHRU | EDGE_TRUE_VALUE;
8144 :
8145 : /* Add a dummy exit for the tiled block when cont_bb is missing. */
8146 177 : if (cont_bb == NULL)
8147 : {
8148 5 : edge e = make_edge (body_bb, exit_bb, EDGE_FALSE_VALUE);
8149 5 : e->probability = profile_probability::even ();
8150 5 : split->probability = profile_probability::even ();
8151 : }
8152 :
8153 : /* Initialize the user's loop vars. */
8154 177 : gsi = gsi_start_bb (elem_body_bb);
8155 177 : expand_oacc_collapse_vars (fd, true, &gsi, counts, e_offset,
8156 : diff_type);
8157 : }
8158 : }
8159 :
8160 : /* Loop increment goes into cont_bb. If this is not a loop, we
8161 : will have spawned threads as if it was, and each one will
8162 : execute one iteration. The specification is not explicit about
8163 : whether such constructs are ill-formed or not, and they can
8164 : occur, especially when noreturn routines are involved. */
8165 11711 : if (cont_bb)
8166 : {
8167 11672 : gsi = gsi_last_nondebug_bb (cont_bb);
8168 11672 : gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
8169 11672 : loc = gimple_location (cont_stmt);
8170 :
8171 11672 : if (fd->tiling)
8172 : {
8173 : /* Insert element loop increment and test. */
8174 172 : expr = build2 (PLUS_EXPR, diff_type, e_offset, e_step);
8175 172 : expr = force_gimple_operand_gsi (&gsi, expr, false, NULL_TREE,
8176 : true, GSI_SAME_STMT);
8177 172 : ass = gimple_build_assign (e_offset, expr);
8178 172 : gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
8179 172 : expr = build2 (cond_code, boolean_type_node, e_offset, e_bound);
8180 :
8181 172 : stmt = gimple_build_cond_empty (expr);
8182 172 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
8183 172 : split = split_block (cont_bb, stmt);
8184 172 : elem_cont_bb = split->src;
8185 172 : cont_bb = split->dest;
8186 :
8187 172 : split->flags ^= EDGE_FALLTHRU | EDGE_FALSE_VALUE;
8188 172 : split->probability = profile_probability::unlikely ().guessed ();
8189 172 : edge latch_edge
8190 172 : = make_edge (elem_cont_bb, elem_body_bb, EDGE_TRUE_VALUE);
8191 172 : latch_edge->probability = profile_probability::likely ().guessed ();
8192 :
8193 172 : edge skip_edge = make_edge (body_bb, cont_bb, EDGE_FALSE_VALUE);
8194 172 : skip_edge->probability = profile_probability::unlikely ().guessed ();
8195 172 : edge loop_entry_edge = EDGE_SUCC (body_bb, 1 - skip_edge->dest_idx);
8196 172 : loop_entry_edge->probability
8197 172 : = profile_probability::likely ().guessed ();
8198 :
8199 172 : gsi = gsi_for_stmt (cont_stmt);
8200 : }
8201 :
8202 : /* Increment offset. */
8203 11672 : if (gimple_in_ssa_p (cfun))
8204 386 : expr = build2 (plus_code, iter_type, offset,
8205 : fold_convert (plus_type, step));
8206 : else
8207 11286 : expr = build2 (PLUS_EXPR, diff_type, offset, step);
8208 11672 : expr = force_gimple_operand_gsi (&gsi, expr, false, NULL_TREE,
8209 : true, GSI_SAME_STMT);
8210 11672 : ass = gimple_build_assign (offset_incr, expr);
8211 11672 : gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
8212 11672 : expr = build2 (cond_code, boolean_type_node, offset_incr, bound);
8213 11672 : gsi_insert_before (&gsi, gimple_build_cond_empty (expr), GSI_SAME_STMT);
8214 :
8215 : /* Remove the GIMPLE_OMP_CONTINUE. */
8216 11672 : gsi_remove (&gsi, true);
8217 :
8218 : /* Fixup edges from cont_bb. */
8219 11672 : be = BRANCH_EDGE (cont_bb);
8220 11672 : fte = FALLTHRU_EDGE (cont_bb);
8221 11672 : be->flags |= EDGE_TRUE_VALUE;
8222 11672 : fte->flags ^= EDGE_FALLTHRU | EDGE_FALSE_VALUE;
8223 :
8224 11672 : if (chunking)
8225 : {
8226 : /* Split the beginning of exit_bb to make bottom_bb. We
8227 : need to insert a nop at the start, because splitting is
8228 : after a stmt, not before. */
8229 11286 : gsi = gsi_start_bb (exit_bb);
8230 11286 : stmt = gimple_build_nop ();
8231 11286 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
8232 11286 : split = split_block (exit_bb, stmt);
8233 11286 : bottom_bb = split->src;
8234 11286 : exit_bb = split->dest;
8235 11286 : gsi = gsi_last_bb (bottom_bb);
8236 :
8237 : /* Chunk increment and test goes into bottom_bb. */
8238 11286 : expr = build2 (PLUS_EXPR, diff_type, chunk_no,
8239 : build_int_cst (diff_type, 1));
8240 11286 : ass = gimple_build_assign (chunk_no, expr);
8241 11286 : gsi_insert_after (&gsi, ass, GSI_CONTINUE_LINKING);
8242 :
8243 : /* Chunk test at end of bottom_bb. */
8244 11286 : expr = build2 (LT_EXPR, boolean_type_node, chunk_no, chunk_max);
8245 11286 : gsi_insert_after (&gsi, gimple_build_cond_empty (expr),
8246 : GSI_CONTINUE_LINKING);
8247 :
8248 : /* Fixup edges from bottom_bb. */
8249 11286 : split->flags ^= EDGE_FALLTHRU | EDGE_FALSE_VALUE;
8250 11286 : split->probability = profile_probability::unlikely ().guessed ();
8251 11286 : edge latch_edge = make_edge (bottom_bb, head_bb, EDGE_TRUE_VALUE);
8252 11286 : latch_edge->probability = profile_probability::likely ().guessed ();
8253 : }
8254 : }
8255 :
8256 11711 : gsi = gsi_last_nondebug_bb (exit_bb);
8257 11711 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
8258 11711 : loc = gimple_location (gsi_stmt (gsi));
8259 :
8260 11711 : if (!gimple_in_ssa_p (cfun))
8261 : {
8262 : /* Insert the final value of V, in case it is live. This is the
8263 : value for the only thread that survives past the join. */
8264 11325 : expr = fold_build2 (MINUS_EXPR, diff_type, range, dir);
8265 11325 : expr = fold_build2 (PLUS_EXPR, diff_type, expr, s);
8266 11325 : expr = fold_build2 (TRUNC_DIV_EXPR, diff_type, expr, s);
8267 11325 : expr = fold_build2 (MULT_EXPR, diff_type, expr, s);
8268 11325 : expr = build2 (plus_code, iter_type, b, fold_convert (plus_type, expr));
8269 11325 : expr = force_gimple_operand_gsi (&gsi, expr, false, NULL_TREE,
8270 : true, GSI_SAME_STMT);
8271 11325 : ass = gimple_build_assign (v, expr);
8272 11325 : gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
8273 : }
8274 :
8275 : /* Remove the OMP_RETURN. */
8276 11711 : gsi_remove (&gsi, true);
8277 :
8278 11711 : if (cont_bb)
8279 : {
8280 : /* We now have one, two or three nested loops. Update the loop
8281 : structures. */
8282 11672 : class loop *parent = entry_bb->loop_father;
8283 11672 : class loop *body = body_bb->loop_father;
8284 :
8285 11672 : if (chunking)
8286 : {
8287 11286 : class loop *chunk_loop = alloc_loop ();
8288 11286 : chunk_loop->header = head_bb;
8289 11286 : chunk_loop->latch = bottom_bb;
8290 11286 : add_loop (chunk_loop, parent);
8291 11286 : parent = chunk_loop;
8292 : }
8293 386 : else if (parent != body)
8294 : {
8295 386 : gcc_assert (body->header == body_bb);
8296 386 : gcc_assert (body->latch == cont_bb
8297 : || single_pred (body->latch) == cont_bb);
8298 : parent = NULL;
8299 : }
8300 :
8301 11286 : if (parent)
8302 : {
8303 11286 : class loop *body_loop = alloc_loop ();
8304 11286 : body_loop->header = body_bb;
8305 11286 : body_loop->latch = cont_bb;
8306 11286 : add_loop (body_loop, parent);
8307 :
8308 11286 : if (fd->tiling)
8309 : {
8310 : /* Insert tiling's element loop. */
8311 172 : class loop *inner_loop = alloc_loop ();
8312 172 : inner_loop->header = elem_body_bb;
8313 172 : inner_loop->latch = elem_cont_bb;
8314 172 : add_loop (inner_loop, body_loop);
8315 : }
8316 : }
8317 : }
8318 11711 : }
8319 :
8320 : /* Expand the OMP loop defined by REGION. */
8321 :
8322 : static void
8323 47587 : expand_omp_for (struct omp_region *region, gimple *inner_stmt)
8324 : {
8325 47587 : struct omp_for_data fd;
8326 47587 : struct omp_for_data_loop *loops;
8327 :
8328 47587 : loops = XALLOCAVEC (struct omp_for_data_loop,
8329 : gimple_omp_for_collapse
8330 : (last_nondebug_stmt (region->entry)));
8331 47587 : omp_extract_for_data (as_a <gomp_for *> (last_nondebug_stmt (region->entry)),
8332 : &fd, loops);
8333 47587 : region->sched_kind = fd.sched_kind;
8334 47587 : region->sched_modifiers = fd.sched_modifiers;
8335 47587 : region->has_lastprivate_conditional = fd.lastprivate_conditional != 0;
8336 47587 : if (fd.non_rect && !gimple_omp_for_combined_into_p (fd.for_stmt))
8337 : {
8338 2344 : for (int i = fd.first_nonrect; i <= fd.last_nonrect; i++)
8339 1644 : if ((loops[i].m1 || loops[i].m2)
8340 731 : && (loops[i].m1 == NULL_TREE
8341 506 : || TREE_CODE (loops[i].m1) == INTEGER_CST)
8342 612 : && (loops[i].m2 == NULL_TREE
8343 343 : || TREE_CODE (loops[i].m2) == INTEGER_CST)
8344 596 : && TREE_CODE (loops[i].step) == INTEGER_CST
8345 586 : && TREE_CODE (loops[i - loops[i].outer].step) == INTEGER_CST)
8346 : {
8347 586 : tree t;
8348 586 : tree itype = TREE_TYPE (loops[i].v);
8349 586 : if (loops[i].m1 && loops[i].m2)
8350 108 : t = fold_build2 (MINUS_EXPR, itype, loops[i].m2, loops[i].m1);
8351 478 : else if (loops[i].m1)
8352 267 : t = fold_build1 (NEGATE_EXPR, itype, loops[i].m1);
8353 : else
8354 : t = loops[i].m2;
8355 586 : t = fold_build2 (MULT_EXPR, itype, t,
8356 : fold_convert (itype,
8357 : loops[i - loops[i].outer].step));
8358 586 : if (TYPE_UNSIGNED (itype) && loops[i].cond_code == GT_EXPR)
8359 3 : t = fold_build2 (TRUNC_MOD_EXPR, itype,
8360 : fold_build1 (NEGATE_EXPR, itype, t),
8361 : fold_build1 (NEGATE_EXPR, itype,
8362 : fold_convert (itype,
8363 : loops[i].step)));
8364 : else
8365 583 : t = fold_build2 (TRUNC_MOD_EXPR, itype, t,
8366 : fold_convert (itype, loops[i].step));
8367 586 : if (integer_nonzerop (t))
8368 8 : error_at (gimple_location (fd.for_stmt),
8369 : "invalid OpenMP non-rectangular loop step; "
8370 : "%<(%E - %E) * %E%> is not a multiple of loop %d "
8371 : "step %qE",
8372 8 : loops[i].m2 ? loops[i].m2 : integer_zero_node,
8373 8 : loops[i].m1 ? loops[i].m1 : integer_zero_node,
8374 8 : loops[i - loops[i].outer].step, i + 1,
8375 : loops[i].step);
8376 : }
8377 : }
8378 :
8379 47587 : gcc_assert (EDGE_COUNT (region->entry->succs) == 2);
8380 47587 : BRANCH_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
8381 47587 : FALLTHRU_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
8382 47587 : if (region->cont)
8383 : {
8384 45312 : gcc_assert (EDGE_COUNT (region->cont->succs) == 2);
8385 45312 : BRANCH_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
8386 45312 : FALLTHRU_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
8387 : }
8388 : else
8389 : /* If there isn't a continue then this is a degenerate case where
8390 : the introduction of abnormal edges during lowering will prevent
8391 : original loops from being detected. Fix that up. */
8392 2275 : loops_state_set (LOOPS_NEED_FIXUP);
8393 :
8394 47587 : if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_SIMD)
8395 9355 : expand_omp_simd (region, &fd);
8396 38232 : else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
8397 : {
8398 11711 : gcc_assert (!inner_stmt && !fd.non_rect);
8399 11711 : expand_oacc_for (region, &fd);
8400 : }
8401 26521 : else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_TASKLOOP)
8402 : {
8403 2660 : if (gimple_omp_for_combined_into_p (fd.for_stmt))
8404 1330 : expand_omp_taskloop_for_inner (region, &fd, inner_stmt);
8405 : else
8406 1330 : expand_omp_taskloop_for_outer (region, &fd, inner_stmt);
8407 : }
8408 23861 : else if (fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
8409 20185 : && !fd.have_ordered)
8410 : {
8411 19733 : if (fd.chunk_size == NULL)
8412 13893 : expand_omp_for_static_nochunk (region, &fd, inner_stmt);
8413 : else
8414 5840 : expand_omp_for_static_chunk (region, &fd, inner_stmt);
8415 : }
8416 : else
8417 : {
8418 4128 : int fn_index, start_ix, next_ix;
8419 4128 : unsigned HOST_WIDE_INT sched = 0;
8420 4128 : tree sched_arg = NULL_TREE;
8421 :
8422 4128 : gcc_assert (gimple_omp_for_kind (fd.for_stmt)
8423 : == GF_OMP_FOR_KIND_FOR && !fd.non_rect);
8424 4128 : if (fd.chunk_size == NULL
8425 1665 : && fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
8426 0 : fd.chunk_size = integer_zero_node;
8427 4128 : switch (fd.sched_kind)
8428 : {
8429 1665 : case OMP_CLAUSE_SCHEDULE_RUNTIME:
8430 1665 : if ((fd.sched_modifiers & OMP_CLAUSE_SCHEDULE_NONMONOTONIC) != 0
8431 28 : && fd.lastprivate_conditional == 0)
8432 : {
8433 28 : gcc_assert (!fd.have_ordered);
8434 : fn_index = 6;
8435 : sched = 4;
8436 : }
8437 1637 : else if ((fd.sched_modifiers & OMP_CLAUSE_SCHEDULE_MONOTONIC) == 0
8438 1605 : && !fd.have_ordered
8439 1561 : && fd.lastprivate_conditional == 0)
8440 : fn_index = 7;
8441 : else
8442 : {
8443 87 : fn_index = 3;
8444 87 : sched = (HOST_WIDE_INT_1U << 31);
8445 : }
8446 : break;
8447 2011 : case OMP_CLAUSE_SCHEDULE_DYNAMIC:
8448 2011 : case OMP_CLAUSE_SCHEDULE_GUIDED:
8449 2011 : if ((fd.sched_modifiers & OMP_CLAUSE_SCHEDULE_MONOTONIC) == 0
8450 1955 : && !fd.have_ordered
8451 1792 : && fd.lastprivate_conditional == 0)
8452 : {
8453 1765 : fn_index = 3 + fd.sched_kind;
8454 1765 : sched = (fd.sched_kind == OMP_CLAUSE_SCHEDULE_GUIDED) + 2;
8455 : break;
8456 : }
8457 246 : fn_index = fd.sched_kind;
8458 246 : sched = (fd.sched_kind == OMP_CLAUSE_SCHEDULE_GUIDED) + 2;
8459 246 : sched += (HOST_WIDE_INT_1U << 31);
8460 246 : break;
8461 452 : case OMP_CLAUSE_SCHEDULE_STATIC:
8462 452 : gcc_assert (fd.have_ordered);
8463 : fn_index = 0;
8464 : sched = (HOST_WIDE_INT_1U << 31) + 1;
8465 : break;
8466 0 : default:
8467 0 : gcc_unreachable ();
8468 : }
8469 4128 : if (!fd.ordered)
8470 3793 : fn_index += fd.have_ordered * 8;
8471 4128 : if (fd.ordered)
8472 335 : start_ix = ((int)BUILT_IN_GOMP_LOOP_DOACROSS_STATIC_START) + fn_index;
8473 : else
8474 3793 : start_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_START) + fn_index;
8475 4128 : next_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT) + fn_index;
8476 4128 : if (fd.have_reductemp || fd.have_pointer_condtemp)
8477 : {
8478 169 : if (fd.ordered)
8479 : start_ix = (int)BUILT_IN_GOMP_LOOP_DOACROSS_START;
8480 133 : else if (fd.have_ordered)
8481 : start_ix = (int)BUILT_IN_GOMP_LOOP_ORDERED_START;
8482 : else
8483 95 : start_ix = (int)BUILT_IN_GOMP_LOOP_START;
8484 169 : sched_arg = build_int_cstu (long_integer_type_node, sched);
8485 169 : if (!fd.chunk_size)
8486 38 : fd.chunk_size = integer_zero_node;
8487 : }
8488 4128 : if (fd.iter_type == long_long_unsigned_type_node)
8489 : {
8490 778 : start_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
8491 : - (int)BUILT_IN_GOMP_LOOP_STATIC_START);
8492 778 : next_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
8493 : - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT);
8494 : }
8495 4128 : expand_omp_for_generic (region, &fd, (enum built_in_function) start_ix,
8496 : (enum built_in_function) next_ix, sched_arg,
8497 : inner_stmt);
8498 : }
8499 47587 : }
8500 :
8501 : /* Expand code for an OpenMP sections directive. In pseudo code, we generate
8502 :
8503 : v = GOMP_sections_start (n);
8504 : L0:
8505 : switch (v)
8506 : {
8507 : case 0:
8508 : goto L2;
8509 : case 1:
8510 : section 1;
8511 : goto L1;
8512 : case 2:
8513 : ...
8514 : case n:
8515 : ...
8516 : default:
8517 : abort ();
8518 : }
8519 : L1:
8520 : v = GOMP_sections_next ();
8521 : goto L0;
8522 : L2:
8523 : reduction;
8524 :
8525 : If this is a combined parallel sections, replace the call to
8526 : GOMP_sections_start with call to GOMP_sections_next. */
8527 :
8528 : static void
8529 386 : expand_omp_sections (struct omp_region *region)
8530 : {
8531 386 : tree t, u, vin = NULL, vmain, vnext, l2;
8532 386 : unsigned len;
8533 386 : basic_block entry_bb, l0_bb, l1_bb, l2_bb, default_bb;
8534 386 : gimple_stmt_iterator si, switch_si;
8535 386 : gomp_sections *sections_stmt;
8536 386 : gimple *stmt;
8537 386 : gomp_continue *cont;
8538 386 : edge_iterator ei;
8539 386 : edge e;
8540 386 : struct omp_region *inner;
8541 386 : unsigned i, casei;
8542 386 : bool exit_reachable = region->cont != NULL;
8543 :
8544 386 : gcc_assert (region->exit != NULL);
8545 386 : entry_bb = region->entry;
8546 386 : l0_bb = single_succ (entry_bb);
8547 386 : l1_bb = region->cont;
8548 386 : l2_bb = region->exit;
8549 735 : if (single_pred_p (l2_bb) && single_pred (l2_bb) == l0_bb)
8550 322 : l2 = gimple_block_label (l2_bb);
8551 : else
8552 : {
8553 : /* This can happen if there are reductions. */
8554 64 : len = EDGE_COUNT (l0_bb->succs);
8555 64 : gcc_assert (len > 0);
8556 64 : e = EDGE_SUCC (l0_bb, len - 1);
8557 64 : si = gsi_last_nondebug_bb (e->dest);
8558 64 : l2 = NULL_TREE;
8559 64 : if (gsi_end_p (si)
8560 64 : || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
8561 64 : l2 = gimple_block_label (e->dest);
8562 : else
8563 0 : FOR_EACH_EDGE (e, ei, l0_bb->succs)
8564 : {
8565 0 : si = gsi_last_nondebug_bb (e->dest);
8566 0 : if (gsi_end_p (si)
8567 0 : || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
8568 : {
8569 0 : l2 = gimple_block_label (e->dest);
8570 0 : break;
8571 : }
8572 : }
8573 : }
8574 386 : if (exit_reachable)
8575 340 : default_bb = create_empty_bb (l1_bb->prev_bb);
8576 : else
8577 46 : default_bb = create_empty_bb (l0_bb);
8578 :
8579 : /* We will build a switch() with enough cases for all the
8580 : GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
8581 : and a default case to abort if something goes wrong. */
8582 386 : len = EDGE_COUNT (l0_bb->succs);
8583 :
8584 : /* Use vec::quick_push on label_vec throughout, since we know the size
8585 : in advance. */
8586 386 : auto_vec<tree> label_vec (len);
8587 :
8588 : /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
8589 : GIMPLE_OMP_SECTIONS statement. */
8590 386 : si = gsi_last_nondebug_bb (entry_bb);
8591 386 : sections_stmt = as_a <gomp_sections *> (gsi_stmt (si));
8592 386 : gcc_assert (gimple_code (sections_stmt) == GIMPLE_OMP_SECTIONS);
8593 386 : vin = gimple_omp_sections_control (sections_stmt);
8594 386 : tree clauses = gimple_omp_sections_clauses (sections_stmt);
8595 386 : tree reductmp = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
8596 386 : tree condtmp = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_);
8597 386 : tree cond_var = NULL_TREE;
8598 386 : if (reductmp || condtmp)
8599 : {
8600 18 : tree reductions = null_pointer_node, mem = null_pointer_node;
8601 18 : tree memv = NULL_TREE, condtemp = NULL_TREE;
8602 18 : gimple_stmt_iterator gsi = gsi_none ();
8603 18 : gimple *g = NULL;
8604 18 : if (reductmp)
8605 : {
8606 8 : reductions = OMP_CLAUSE_DECL (reductmp);
8607 8 : gcc_assert (TREE_CODE (reductions) == SSA_NAME);
8608 8 : g = SSA_NAME_DEF_STMT (reductions);
8609 8 : reductions = gimple_assign_rhs1 (g);
8610 8 : OMP_CLAUSE_DECL (reductmp) = reductions;
8611 8 : gsi = gsi_for_stmt (g);
8612 : }
8613 : else
8614 10 : gsi = si;
8615 18 : if (condtmp)
8616 : {
8617 12 : condtemp = OMP_CLAUSE_DECL (condtmp);
8618 12 : tree c = omp_find_clause (OMP_CLAUSE_CHAIN (condtmp),
8619 : OMP_CLAUSE__CONDTEMP_);
8620 12 : cond_var = OMP_CLAUSE_DECL (c);
8621 12 : tree type = TREE_TYPE (condtemp);
8622 12 : memv = create_tmp_var (type);
8623 12 : TREE_ADDRESSABLE (memv) = 1;
8624 12 : unsigned cnt = 0;
8625 74 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
8626 62 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
8627 62 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
8628 24 : ++cnt;
8629 12 : unsigned HOST_WIDE_INT sz
8630 12 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (type))) * cnt;
8631 12 : expand_omp_build_assign (&gsi, memv, build_int_cst (type, sz),
8632 : false);
8633 12 : mem = build_fold_addr_expr (memv);
8634 : }
8635 18 : t = build_int_cst (unsigned_type_node, len - 1);
8636 18 : u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS2_START);
8637 18 : stmt = gimple_build_call (u, 3, t, reductions, mem);
8638 18 : gimple_call_set_lhs (stmt, vin);
8639 18 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
8640 18 : if (condtmp)
8641 : {
8642 12 : expand_omp_build_assign (&gsi, condtemp, memv, false);
8643 12 : tree t = build2 (PLUS_EXPR, TREE_TYPE (cond_var),
8644 12 : vin, build_one_cst (TREE_TYPE (cond_var)));
8645 12 : expand_omp_build_assign (&gsi, cond_var, t, false);
8646 : }
8647 18 : if (reductmp)
8648 : {
8649 8 : gsi_remove (&gsi, true);
8650 8 : release_ssa_name (gimple_assign_lhs (g));
8651 : }
8652 : }
8653 368 : else if (!is_combined_parallel (region))
8654 : {
8655 : /* If we are not inside a combined parallel+sections region,
8656 : call GOMP_sections_start. */
8657 252 : t = build_int_cst (unsigned_type_node, len - 1);
8658 252 : u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START);
8659 252 : stmt = gimple_build_call (u, 1, t);
8660 : }
8661 : else
8662 : {
8663 : /* Otherwise, call GOMP_sections_next. */
8664 116 : u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
8665 116 : stmt = gimple_build_call (u, 0);
8666 : }
8667 386 : if (!reductmp && !condtmp)
8668 : {
8669 368 : gimple_call_set_lhs (stmt, vin);
8670 368 : gsi_insert_after (&si, stmt, GSI_SAME_STMT);
8671 : }
8672 386 : gsi_remove (&si, true);
8673 :
8674 : /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
8675 : L0_BB. */
8676 386 : switch_si = gsi_last_nondebug_bb (l0_bb);
8677 386 : gcc_assert (gimple_code (gsi_stmt (switch_si)) == GIMPLE_OMP_SECTIONS_SWITCH);
8678 386 : if (exit_reachable)
8679 : {
8680 340 : cont = as_a <gomp_continue *> (last_nondebug_stmt (l1_bb));
8681 340 : gcc_assert (gimple_code (cont) == GIMPLE_OMP_CONTINUE);
8682 340 : vmain = gimple_omp_continue_control_use (cont);
8683 340 : vnext = gimple_omp_continue_control_def (cont);
8684 : }
8685 : else
8686 : {
8687 : vmain = vin;
8688 : vnext = NULL_TREE;
8689 : }
8690 :
8691 386 : t = build_case_label (build_int_cst (unsigned_type_node, 0), NULL, l2);
8692 386 : label_vec.quick_push (t);
8693 386 : i = 1;
8694 :
8695 : /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
8696 386 : for (inner = region->inner, casei = 1;
8697 1276 : inner;
8698 890 : inner = inner->next, i++, casei++)
8699 : {
8700 890 : basic_block s_entry_bb, s_exit_bb;
8701 :
8702 : /* Skip optional reduction region. */
8703 890 : if (inner->type == GIMPLE_OMP_ATOMIC_LOAD)
8704 : {
8705 27 : --i;
8706 27 : --casei;
8707 27 : continue;
8708 : }
8709 :
8710 863 : s_entry_bb = inner->entry;
8711 863 : s_exit_bb = inner->exit;
8712 :
8713 863 : t = gimple_block_label (s_entry_bb);
8714 863 : u = build_int_cst (unsigned_type_node, casei);
8715 863 : u = build_case_label (u, NULL, t);
8716 863 : label_vec.quick_push (u);
8717 :
8718 863 : si = gsi_last_nondebug_bb (s_entry_bb);
8719 863 : gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SECTION);
8720 863 : gcc_assert (i < len || gimple_omp_section_last_p (gsi_stmt (si)));
8721 863 : gsi_remove (&si, true);
8722 863 : single_succ_edge (s_entry_bb)->flags = EDGE_FALLTHRU;
8723 :
8724 863 : if (s_exit_bb == NULL)
8725 122 : continue;
8726 :
8727 741 : si = gsi_last_nondebug_bb (s_exit_bb);
8728 741 : gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
8729 741 : gsi_remove (&si, true);
8730 :
8731 741 : single_succ_edge (s_exit_bb)->flags = EDGE_FALLTHRU;
8732 : }
8733 :
8734 : /* Error handling code goes in DEFAULT_BB. */
8735 386 : t = gimple_block_label (default_bb);
8736 386 : u = build_case_label (NULL, NULL, t);
8737 386 : make_edge (l0_bb, default_bb, 0);
8738 386 : add_bb_to_loop (default_bb, current_loops->tree_root);
8739 :
8740 386 : stmt = gimple_build_switch (vmain, u, label_vec);
8741 386 : gsi_insert_after (&switch_si, stmt, GSI_SAME_STMT);
8742 386 : gsi_remove (&switch_si, true);
8743 :
8744 386 : si = gsi_start_bb (default_bb);
8745 386 : stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP), 0);
8746 386 : gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING);
8747 :
8748 386 : if (exit_reachable)
8749 : {
8750 340 : tree bfn_decl;
8751 :
8752 : /* Code to get the next section goes in L1_BB. */
8753 340 : si = gsi_last_nondebug_bb (l1_bb);
8754 340 : gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CONTINUE);
8755 :
8756 340 : bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
8757 340 : stmt = gimple_build_call (bfn_decl, 0);
8758 340 : gimple_call_set_lhs (stmt, vnext);
8759 340 : gsi_insert_before (&si, stmt, GSI_SAME_STMT);
8760 340 : if (cond_var)
8761 : {
8762 12 : tree t = build2 (PLUS_EXPR, TREE_TYPE (cond_var),
8763 12 : vnext, build_one_cst (TREE_TYPE (cond_var)));
8764 12 : expand_omp_build_assign (&si, cond_var, t, false);
8765 : }
8766 340 : gsi_remove (&si, true);
8767 :
8768 340 : single_succ_edge (l1_bb)->flags = EDGE_FALLTHRU;
8769 : }
8770 :
8771 : /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
8772 386 : si = gsi_last_nondebug_bb (l2_bb);
8773 386 : if (gimple_omp_return_nowait_p (gsi_stmt (si)))
8774 255 : t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT);
8775 131 : else if (gimple_omp_return_lhs (gsi_stmt (si)))
8776 0 : t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL);
8777 : else
8778 131 : t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END);
8779 386 : stmt = gimple_build_call (t, 0);
8780 386 : if (gimple_omp_return_lhs (gsi_stmt (si)))
8781 0 : gimple_call_set_lhs (stmt, gimple_omp_return_lhs (gsi_stmt (si)));
8782 386 : gsi_insert_after (&si, stmt, GSI_SAME_STMT);
8783 386 : gsi_remove (&si, true);
8784 :
8785 386 : set_immediate_dominator (CDI_DOMINATORS, default_bb, l0_bb);
8786 386 : }
8787 :
8788 : /* Expand code for an OpenMP single or scope directive. We've already expanded
8789 : much of the code, here we simply place the GOMP_barrier call. */
8790 :
8791 : static void
8792 1272 : expand_omp_single (struct omp_region *region)
8793 : {
8794 1272 : basic_block entry_bb, exit_bb;
8795 1272 : gimple_stmt_iterator si;
8796 :
8797 1272 : entry_bb = region->entry;
8798 1272 : exit_bb = region->exit;
8799 :
8800 1272 : si = gsi_last_nondebug_bb (entry_bb);
8801 1272 : enum gimple_code code = gimple_code (gsi_stmt (si));
8802 1272 : gcc_assert (code == GIMPLE_OMP_SINGLE || code == GIMPLE_OMP_SCOPE);
8803 1272 : gsi_remove (&si, true);
8804 1272 : single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
8805 :
8806 1272 : if (exit_bb == NULL)
8807 : {
8808 8 : gcc_assert (code == GIMPLE_OMP_SCOPE);
8809 8 : return;
8810 : }
8811 :
8812 1264 : si = gsi_last_nondebug_bb (exit_bb);
8813 1264 : if (!gimple_omp_return_nowait_p (gsi_stmt (si)))
8814 : {
8815 985 : tree t = gimple_omp_return_lhs (gsi_stmt (si));
8816 985 : gsi_insert_after (&si,
8817 : omp_build_barrier (t, GOMP_BARRIER_IMPLICIT_WORKSHARE),
8818 : GSI_SAME_STMT);
8819 : }
8820 1264 : gsi_remove (&si, true);
8821 1264 : single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
8822 : }
8823 :
8824 : /* Generic expansion for OpenMP synchronization directives: master,
8825 : ordered and critical. All we need to do here is remove the entry
8826 : and exit markers for REGION. */
8827 :
8828 : static void
8829 10814 : expand_omp_synch (struct omp_region *region)
8830 : {
8831 10814 : basic_block entry_bb, exit_bb;
8832 10814 : gimple_stmt_iterator si;
8833 :
8834 10814 : entry_bb = region->entry;
8835 10814 : exit_bb = region->exit;
8836 :
8837 10814 : si = gsi_last_nondebug_bb (entry_bb);
8838 10814 : gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE
8839 : || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_MASTER
8840 : || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_MASKED
8841 : || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TASKGROUP
8842 : || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ORDERED
8843 : || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CRITICAL
8844 : || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TEAMS);
8845 10814 : if (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TEAMS
8846 10814 : && gimple_omp_teams_host (as_a <gomp_teams *> (gsi_stmt (si))))
8847 : {
8848 2518 : expand_omp_taskreg (region);
8849 2518 : return;
8850 : }
8851 8296 : gsi_remove (&si, true);
8852 8296 : single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
8853 :
8854 8296 : if (exit_bb)
8855 : {
8856 7738 : si = gsi_last_nondebug_bb (exit_bb);
8857 7738 : gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
8858 7738 : gsi_remove (&si, true);
8859 7738 : single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
8860 : }
8861 : }
8862 :
8863 : /* Translate enum omp_memory_order to enum memmodel for the embedded
8864 : fail clause in there. */
8865 :
8866 : static enum memmodel
8867 2936 : omp_memory_order_to_fail_memmodel (enum omp_memory_order mo)
8868 : {
8869 2936 : switch (mo & OMP_FAIL_MEMORY_ORDER_MASK)
8870 : {
8871 2665 : case OMP_FAIL_MEMORY_ORDER_UNSPECIFIED:
8872 2665 : switch (mo & OMP_MEMORY_ORDER_MASK)
8873 : {
8874 : case OMP_MEMORY_ORDER_RELAXED: return MEMMODEL_RELAXED;
8875 : case OMP_MEMORY_ORDER_ACQUIRE: return MEMMODEL_ACQUIRE;
8876 : case OMP_MEMORY_ORDER_RELEASE: return MEMMODEL_RELAXED;
8877 : case OMP_MEMORY_ORDER_ACQ_REL: return MEMMODEL_ACQUIRE;
8878 : case OMP_MEMORY_ORDER_SEQ_CST: return MEMMODEL_SEQ_CST;
8879 0 : default: break;
8880 : }
8881 0 : gcc_unreachable ();
8882 : case OMP_FAIL_MEMORY_ORDER_RELAXED: return MEMMODEL_RELAXED;
8883 : case OMP_FAIL_MEMORY_ORDER_ACQUIRE: return MEMMODEL_ACQUIRE;
8884 : case OMP_FAIL_MEMORY_ORDER_SEQ_CST: return MEMMODEL_SEQ_CST;
8885 0 : default: gcc_unreachable ();
8886 : }
8887 : }
8888 :
8889 : /* Translate enum omp_memory_order to enum memmodel. The two enums
8890 : are using different numbers so that OMP_MEMORY_ORDER_UNSPECIFIED
8891 : is 0 and omp_memory_order has the fail mode encoded in it too. */
8892 :
8893 : static enum memmodel
8894 9996 : omp_memory_order_to_memmodel (enum omp_memory_order mo)
8895 : {
8896 9996 : enum memmodel ret, fail_ret;
8897 9996 : switch (mo & OMP_MEMORY_ORDER_MASK)
8898 : {
8899 : case OMP_MEMORY_ORDER_RELAXED: ret = MEMMODEL_RELAXED; break;
8900 : case OMP_MEMORY_ORDER_ACQUIRE: ret = MEMMODEL_ACQUIRE; break;
8901 : case OMP_MEMORY_ORDER_RELEASE: ret = MEMMODEL_RELEASE; break;
8902 : case OMP_MEMORY_ORDER_ACQ_REL: ret = MEMMODEL_ACQ_REL; break;
8903 : case OMP_MEMORY_ORDER_SEQ_CST: ret = MEMMODEL_SEQ_CST; break;
8904 0 : default: gcc_unreachable ();
8905 : }
8906 : /* If we drop the -Winvalid-memory-model warning for C++17 P0418R2,
8907 : we can just return ret here unconditionally. Otherwise, work around
8908 : it here and make sure fail memmodel is not stronger. */
8909 9996 : if ((mo & OMP_FAIL_MEMORY_ORDER_MASK) == OMP_FAIL_MEMORY_ORDER_UNSPECIFIED)
8910 : return ret;
8911 142 : fail_ret = omp_memory_order_to_fail_memmodel (mo);
8912 142 : if (fail_ret > ret)
8913 13 : return fail_ret;
8914 : return ret;
8915 : }
8916 :
8917 : /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8918 : operation as a normal volatile load. */
8919 :
8920 : static bool
8921 1183 : expand_omp_atomic_load (basic_block load_bb, tree addr,
8922 : tree loaded_val, int index)
8923 : {
8924 1183 : enum built_in_function tmpbase;
8925 1183 : gimple_stmt_iterator gsi;
8926 1183 : basic_block store_bb;
8927 1183 : location_t loc;
8928 1183 : gimple *stmt;
8929 1183 : tree decl, type, itype;
8930 :
8931 1183 : gsi = gsi_last_nondebug_bb (load_bb);
8932 1183 : stmt = gsi_stmt (gsi);
8933 1183 : gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
8934 1183 : loc = gimple_location (stmt);
8935 :
8936 : /* ??? If the target does not implement atomic_load_optab[mode], and mode
8937 : is smaller than word size, then expand_atomic_load assumes that the load
8938 : is atomic. We could avoid the builtin entirely in this case. */
8939 :
8940 1183 : tmpbase = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1);
8941 1183 : decl = builtin_decl_explicit (tmpbase);
8942 1183 : if (decl == NULL_TREE)
8943 : return false;
8944 :
8945 1183 : type = TREE_TYPE (loaded_val);
8946 1183 : itype = TREE_TYPE (TREE_TYPE (decl));
8947 :
8948 1183 : enum omp_memory_order omo = gimple_omp_atomic_memory_order (stmt);
8949 1183 : tree mo = build_int_cst (integer_type_node,
8950 1183 : omp_memory_order_to_memmodel (omo));
8951 1183 : gcall *call = gimple_build_call (decl, 2, addr, mo);
8952 1183 : gimple_set_location (call, loc);
8953 2366 : gimple_set_vuse (call, gimple_vuse (stmt));
8954 1183 : gimple *repl;
8955 1183 : if (!useless_type_conversion_p (type, itype))
8956 : {
8957 1181 : tree lhs = make_ssa_name (itype);
8958 1181 : gimple_call_set_lhs (call, lhs);
8959 1181 : gsi_insert_before (&gsi, call, GSI_SAME_STMT);
8960 1181 : repl = gimple_build_assign (loaded_val,
8961 : build1 (VIEW_CONVERT_EXPR, type, lhs));
8962 1181 : gimple_set_location (repl, loc);
8963 : }
8964 : else
8965 : {
8966 2 : gimple_call_set_lhs (call, loaded_val);
8967 : repl = call;
8968 : }
8969 1183 : gsi_replace (&gsi, repl, true);
8970 :
8971 1183 : store_bb = single_succ (load_bb);
8972 1183 : gsi = gsi_last_nondebug_bb (store_bb);
8973 1183 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
8974 1183 : gsi_remove (&gsi, true);
8975 :
8976 1183 : return true;
8977 : }
8978 :
8979 : /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8980 : operation as a normal volatile store. */
8981 :
8982 : static bool
8983 856 : expand_omp_atomic_store (basic_block load_bb, tree addr,
8984 : tree loaded_val, tree stored_val, int index)
8985 : {
8986 856 : enum built_in_function tmpbase;
8987 856 : gimple_stmt_iterator gsi;
8988 856 : basic_block store_bb = single_succ (load_bb);
8989 856 : location_t loc;
8990 856 : gimple *stmt;
8991 856 : tree decl, type, itype;
8992 856 : machine_mode imode;
8993 856 : bool exchange;
8994 :
8995 856 : gsi = gsi_last_nondebug_bb (load_bb);
8996 856 : stmt = gsi_stmt (gsi);
8997 856 : gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
8998 :
8999 : /* If the load value is needed, then this isn't a store but an exchange. */
9000 856 : exchange = gimple_omp_atomic_need_value_p (stmt);
9001 :
9002 856 : gsi = gsi_last_nondebug_bb (store_bb);
9003 856 : stmt = gsi_stmt (gsi);
9004 856 : gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE);
9005 856 : loc = gimple_location (stmt);
9006 :
9007 : /* ??? If the target does not implement atomic_store_optab[mode], and mode
9008 : is smaller than word size, then expand_atomic_store assumes that the store
9009 : is atomic. We could avoid the builtin entirely in this case. */
9010 :
9011 856 : tmpbase = (exchange ? BUILT_IN_ATOMIC_EXCHANGE_N : BUILT_IN_ATOMIC_STORE_N);
9012 856 : tmpbase = (enum built_in_function) ((int) tmpbase + index + 1);
9013 856 : decl = builtin_decl_explicit (tmpbase);
9014 856 : if (decl == NULL_TREE)
9015 : return false;
9016 :
9017 856 : type = TREE_TYPE (stored_val);
9018 :
9019 : /* Dig out the type of the function's second argument. */
9020 856 : itype = TREE_TYPE (decl);
9021 856 : itype = TYPE_ARG_TYPES (itype);
9022 856 : itype = TREE_CHAIN (itype);
9023 856 : itype = TREE_VALUE (itype);
9024 856 : imode = TYPE_MODE (itype);
9025 :
9026 856 : if (exchange && !can_atomic_exchange_p (imode, true))
9027 : return false;
9028 :
9029 856 : if (!useless_type_conversion_p (itype, type))
9030 856 : stored_val = fold_build1_loc (loc, VIEW_CONVERT_EXPR, itype, stored_val);
9031 856 : enum omp_memory_order omo = gimple_omp_atomic_memory_order (stmt);
9032 856 : tree mo = build_int_cst (integer_type_node,
9033 856 : omp_memory_order_to_memmodel (omo));
9034 856 : stored_val = force_gimple_operand_gsi (&gsi, stored_val, true, NULL_TREE,
9035 : true, GSI_SAME_STMT);
9036 856 : gcall *call = gimple_build_call (decl, 3, addr, stored_val, mo);
9037 856 : gimple_set_location (call, loc);
9038 1712 : gimple_set_vuse (call, gimple_vuse (stmt));
9039 1712 : gimple_set_vdef (call, gimple_vdef (stmt));
9040 :
9041 856 : gimple *repl = call;
9042 856 : if (exchange)
9043 : {
9044 88 : if (!useless_type_conversion_p (type, itype))
9045 : {
9046 88 : tree lhs = make_ssa_name (itype);
9047 88 : gimple_call_set_lhs (call, lhs);
9048 88 : gsi_insert_before (&gsi, call, GSI_SAME_STMT);
9049 88 : repl = gimple_build_assign (loaded_val,
9050 : build1 (VIEW_CONVERT_EXPR, type, lhs));
9051 88 : gimple_set_location (repl, loc);
9052 : }
9053 : else
9054 0 : gimple_call_set_lhs (call, loaded_val);
9055 : }
9056 856 : gsi_replace (&gsi, repl, true);
9057 :
9058 : /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
9059 856 : gsi = gsi_last_nondebug_bb (load_bb);
9060 856 : gsi_remove (&gsi, true);
9061 :
9062 856 : return true;
9063 : }
9064 :
9065 : /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
9066 : operation as a __atomic_fetch_op builtin. INDEX is log2 of the
9067 : size of the data type, and thus usable to find the index of the builtin
9068 : decl. Returns false if the expression is not of the proper form. */
9069 :
9070 : static bool
9071 5828 : expand_omp_atomic_fetch_op (basic_block load_bb,
9072 : tree addr, tree loaded_val,
9073 : tree stored_val, int index)
9074 : {
9075 5828 : enum built_in_function oldbase, newbase, tmpbase;
9076 5828 : tree decl, itype, call;
9077 5828 : tree lhs, rhs;
9078 5828 : basic_block store_bb = single_succ (load_bb);
9079 5828 : gimple_stmt_iterator gsi;
9080 5828 : gimple *stmt;
9081 5828 : location_t loc;
9082 5828 : enum tree_code code;
9083 5828 : bool need_old, need_new;
9084 5828 : machine_mode imode;
9085 :
9086 : /* We expect to find the following sequences:
9087 :
9088 : load_bb:
9089 : GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
9090 :
9091 : store_bb:
9092 : val = tmp OP something; (or: something OP tmp)
9093 : GIMPLE_OMP_STORE (val)
9094 :
9095 : ???FIXME: Allow a more flexible sequence.
9096 : Perhaps use data flow to pick the statements.
9097 :
9098 : */
9099 :
9100 5828 : gsi = gsi_after_labels (store_bb);
9101 5828 : stmt = gsi_stmt (gsi);
9102 5828 : if (is_gimple_debug (stmt))
9103 : {
9104 0 : gsi_next_nondebug (&gsi);
9105 0 : if (gsi_end_p (gsi))
9106 : return false;
9107 5828 : stmt = gsi_stmt (gsi);
9108 : }
9109 5828 : loc = gimple_location (stmt);
9110 5828 : if (!is_gimple_assign (stmt))
9111 : return false;
9112 5828 : gsi_next_nondebug (&gsi);
9113 5828 : if (gimple_code (gsi_stmt (gsi)) != GIMPLE_OMP_ATOMIC_STORE)
9114 : return false;
9115 5163 : need_new = gimple_omp_atomic_need_value_p (gsi_stmt (gsi));
9116 5163 : need_old = gimple_omp_atomic_need_value_p (last_nondebug_stmt (load_bb));
9117 5163 : enum omp_memory_order omo
9118 5163 : = gimple_omp_atomic_memory_order (last_nondebug_stmt (load_bb));
9119 5163 : enum memmodel mo = omp_memory_order_to_memmodel (omo);
9120 5163 : gcc_checking_assert (!need_old || !need_new);
9121 :
9122 5163 : if (!operand_equal_p (gimple_assign_lhs (stmt), stored_val, 0))
9123 : return false;
9124 :
9125 : /* Check for one of the supported fetch-op operations. */
9126 5163 : code = gimple_assign_rhs_code (stmt);
9127 5163 : switch (code)
9128 : {
9129 : case PLUS_EXPR:
9130 : case POINTER_PLUS_EXPR:
9131 : oldbase = BUILT_IN_ATOMIC_FETCH_ADD_N;
9132 : newbase = BUILT_IN_ATOMIC_ADD_FETCH_N;
9133 : break;
9134 79 : case MINUS_EXPR:
9135 79 : oldbase = BUILT_IN_ATOMIC_FETCH_SUB_N;
9136 79 : newbase = BUILT_IN_ATOMIC_SUB_FETCH_N;
9137 79 : break;
9138 112 : case BIT_AND_EXPR:
9139 112 : oldbase = BUILT_IN_ATOMIC_FETCH_AND_N;
9140 112 : newbase = BUILT_IN_ATOMIC_AND_FETCH_N;
9141 112 : break;
9142 227 : case BIT_IOR_EXPR:
9143 227 : oldbase = BUILT_IN_ATOMIC_FETCH_OR_N;
9144 227 : newbase = BUILT_IN_ATOMIC_OR_FETCH_N;
9145 227 : break;
9146 111 : case BIT_XOR_EXPR:
9147 111 : oldbase = BUILT_IN_ATOMIC_FETCH_XOR_N;
9148 111 : newbase = BUILT_IN_ATOMIC_XOR_FETCH_N;
9149 111 : break;
9150 : default:
9151 : return false;
9152 : }
9153 :
9154 : /* Make sure the expression is of the proper form. */
9155 4524 : if (operand_equal_p (gimple_assign_rhs1 (stmt), loaded_val, 0))
9156 4342 : rhs = gimple_assign_rhs2 (stmt);
9157 182 : else if (commutative_tree_code (gimple_assign_rhs_code (stmt))
9158 182 : && operand_equal_p (gimple_assign_rhs2 (stmt), loaded_val, 0))
9159 139 : rhs = gimple_assign_rhs1 (stmt);
9160 : else
9161 43 : return false;
9162 :
9163 8962 : tmpbase = ((enum built_in_function)
9164 4481 : ((need_new ? newbase : oldbase) + index + 1));
9165 4481 : decl = builtin_decl_explicit (tmpbase);
9166 4481 : if (decl == NULL_TREE)
9167 : return false;
9168 4481 : itype = TREE_TYPE (TREE_TYPE (decl));
9169 4481 : imode = TYPE_MODE (itype);
9170 :
9171 : /* We could test all of the various optabs involved, but the fact of the
9172 : matter is that (with the exception of i486 vs i586 and xadd) all targets
9173 : that support any atomic operation optab also implements compare-and-swap.
9174 : Let optabs.cc take care of expanding any compare-and-swap loop. */
9175 4481 : if (!can_compare_and_swap_p (imode, true) || !can_atomic_load_p (imode))
9176 4 : return false;
9177 :
9178 4477 : gsi = gsi_last_nondebug_bb (load_bb);
9179 4477 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_LOAD);
9180 :
9181 : /* OpenMP does not imply any barrier-like semantics on its atomic ops.
9182 : It only requires that the operation happen atomically. Thus we can
9183 : use the RELAXED memory model. */
9184 4477 : call = build_call_expr_loc (loc, decl, 3, addr,
9185 : fold_convert_loc (loc, itype, rhs),
9186 4477 : build_int_cst (NULL, mo));
9187 :
9188 4477 : if (need_old || need_new)
9189 : {
9190 552 : lhs = need_old ? loaded_val : stored_val;
9191 552 : call = fold_convert_loc (loc, TREE_TYPE (lhs), call);
9192 552 : call = build2_loc (loc, MODIFY_EXPR, void_type_node, lhs, call);
9193 : }
9194 : else
9195 3925 : call = fold_convert_loc (loc, void_type_node, call);
9196 4477 : force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
9197 4477 : gsi_remove (&gsi, true);
9198 :
9199 4477 : gsi = gsi_last_nondebug_bb (store_bb);
9200 4477 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
9201 4477 : gsi_remove (&gsi, true);
9202 4477 : gsi = gsi_last_nondebug_bb (store_bb);
9203 4477 : stmt = gsi_stmt (gsi);
9204 4477 : gsi_remove (&gsi, true);
9205 :
9206 4477 : if (gimple_in_ssa_p (cfun))
9207 29 : release_defs (stmt);
9208 :
9209 : return true;
9210 : }
9211 :
9212 : /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
9213 : compare and exchange as an ATOMIC_COMPARE_EXCHANGE internal function.
9214 : Returns false if the expression is not of the proper form. */
9215 :
9216 : static bool
9217 2217 : expand_omp_atomic_cas (basic_block load_bb, tree addr,
9218 : tree loaded_val, tree stored_val, int index)
9219 : {
9220 : /* We expect to find the following sequences:
9221 :
9222 : load_bb:
9223 : GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
9224 :
9225 : store_bb:
9226 : val = tmp == e ? d : tmp;
9227 : GIMPLE_OMP_ATOMIC_STORE (val)
9228 :
9229 : or in store_bb instead:
9230 : tmp2 = tmp == e;
9231 : val = tmp2 ? d : tmp;
9232 : GIMPLE_OMP_ATOMIC_STORE (val)
9233 :
9234 : or:
9235 : tmp3 = VIEW_CONVERT_EXPR<integral_type>(tmp);
9236 : val = e == tmp3 ? d : tmp;
9237 : GIMPLE_OMP_ATOMIC_STORE (val)
9238 :
9239 : etc. */
9240 :
9241 :
9242 2217 : basic_block store_bb = single_succ (load_bb);
9243 2217 : gimple_stmt_iterator gsi = gsi_last_nondebug_bb (store_bb);
9244 2217 : gimple *store_stmt = gsi_stmt (gsi);
9245 2217 : if (!store_stmt || gimple_code (store_stmt) != GIMPLE_OMP_ATOMIC_STORE)
9246 : return false;
9247 2217 : gsi_prev_nondebug (&gsi);
9248 2217 : if (gsi_end_p (gsi))
9249 : return false;
9250 2211 : gimple *condexpr_stmt = gsi_stmt (gsi);
9251 2211 : if (!is_gimple_assign (condexpr_stmt)
9252 2211 : || gimple_assign_rhs_code (condexpr_stmt) != COND_EXPR)
9253 : return false;
9254 408 : if (!operand_equal_p (gimple_assign_lhs (condexpr_stmt), stored_val, 0))
9255 : return false;
9256 408 : gimple *cond_stmt = NULL;
9257 408 : gimple *vce_stmt = NULL;
9258 408 : gsi_prev_nondebug (&gsi);
9259 408 : if (!gsi_end_p (gsi))
9260 : {
9261 408 : cond_stmt = gsi_stmt (gsi);
9262 408 : if (!is_gimple_assign (cond_stmt))
9263 : return false;
9264 408 : if (gimple_assign_rhs_code (cond_stmt) == EQ_EXPR)
9265 : {
9266 323 : gsi_prev_nondebug (&gsi);
9267 323 : if (!gsi_end_p (gsi))
9268 : {
9269 89 : vce_stmt = gsi_stmt (gsi);
9270 89 : if (!is_gimple_assign (vce_stmt)
9271 89 : || gimple_assign_rhs_code (vce_stmt) != VIEW_CONVERT_EXPR)
9272 : return false;
9273 : }
9274 : }
9275 85 : else if (gimple_assign_rhs_code (cond_stmt) == VIEW_CONVERT_EXPR)
9276 : std::swap (vce_stmt, cond_stmt);
9277 : else
9278 : return false;
9279 69 : if (vce_stmt)
9280 : {
9281 69 : tree vce_rhs = gimple_assign_rhs1 (vce_stmt);
9282 69 : if (TREE_CODE (vce_rhs) != VIEW_CONVERT_EXPR
9283 69 : || !operand_equal_p (TREE_OPERAND (vce_rhs, 0), loaded_val))
9284 12 : return false;
9285 114 : if (!INTEGRAL_TYPE_P (TREE_TYPE (vce_rhs))
9286 57 : || !SCALAR_FLOAT_TYPE_P (TREE_TYPE (loaded_val))
9287 114 : || !tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vce_rhs)),
9288 57 : TYPE_SIZE (TREE_TYPE (loaded_val))))
9289 0 : return false;
9290 57 : gsi_prev_nondebug (&gsi);
9291 57 : if (!gsi_end_p (gsi))
9292 : return false;
9293 : }
9294 : }
9295 291 : tree cond = gimple_assign_rhs1 (condexpr_stmt);
9296 291 : tree cond_op1, cond_op2;
9297 291 : if (cond_stmt)
9298 : {
9299 : /* We should now always get a separate cond_stmt. */
9300 291 : if (!operand_equal_p (cond, gimple_assign_lhs (cond_stmt)))
9301 : return false;
9302 291 : cond_op1 = gimple_assign_rhs1 (cond_stmt);
9303 291 : cond_op2 = gimple_assign_rhs2 (cond_stmt);
9304 : }
9305 0 : else if (TREE_CODE (cond) != EQ_EXPR && TREE_CODE (cond) != NE_EXPR)
9306 : return false;
9307 : else
9308 : {
9309 0 : cond_op1 = TREE_OPERAND (cond, 0);
9310 0 : cond_op2 = TREE_OPERAND (cond, 1);
9311 : }
9312 291 : tree d;
9313 291 : if (TREE_CODE (cond) == NE_EXPR)
9314 : {
9315 0 : if (!operand_equal_p (gimple_assign_rhs2 (condexpr_stmt), loaded_val))
9316 : return false;
9317 0 : d = gimple_assign_rhs3 (condexpr_stmt);
9318 : }
9319 291 : else if (!operand_equal_p (gimple_assign_rhs3 (condexpr_stmt), loaded_val))
9320 : return false;
9321 : else
9322 291 : d = gimple_assign_rhs2 (condexpr_stmt);
9323 291 : tree e = vce_stmt ? gimple_assign_lhs (vce_stmt) : loaded_val;
9324 291 : if (operand_equal_p (e, cond_op1))
9325 : e = cond_op2;
9326 21 : else if (operand_equal_p (e, cond_op2))
9327 : e = cond_op1;
9328 : else
9329 : return false;
9330 :
9331 291 : location_t loc = gimple_location (store_stmt);
9332 291 : gimple *load_stmt = last_nondebug_stmt (load_bb);
9333 291 : bool need_new = gimple_omp_atomic_need_value_p (store_stmt);
9334 291 : bool need_old = gimple_omp_atomic_need_value_p (load_stmt);
9335 291 : bool weak = gimple_omp_atomic_weak_p (load_stmt);
9336 291 : enum omp_memory_order omo = gimple_omp_atomic_memory_order (load_stmt);
9337 291 : tree mo = build_int_cst (NULL, omp_memory_order_to_memmodel (omo));
9338 291 : tree fmo = build_int_cst (NULL, omp_memory_order_to_fail_memmodel (omo));
9339 291 : gcc_checking_assert (!need_old || !need_new);
9340 :
9341 291 : enum built_in_function fncode
9342 : = (enum built_in_function) ((int) BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
9343 291 : + index + 1);
9344 291 : tree cmpxchg = builtin_decl_explicit (fncode);
9345 291 : if (cmpxchg == NULL_TREE)
9346 : return false;
9347 291 : tree itype = TREE_TYPE (TREE_TYPE (cmpxchg));
9348 :
9349 291 : if (!can_compare_and_swap_p (TYPE_MODE (itype), true)
9350 291 : || !can_atomic_load_p (TYPE_MODE (itype)))
9351 1 : return false;
9352 :
9353 290 : tree type = TYPE_MAIN_VARIANT (TREE_TYPE (loaded_val));
9354 290 : if (SCALAR_FLOAT_TYPE_P (type) && !vce_stmt)
9355 : return false;
9356 :
9357 289 : gsi = gsi_for_stmt (store_stmt);
9358 289 : if (!useless_type_conversion_p (itype, TREE_TYPE (e)))
9359 : {
9360 232 : tree ne = create_tmp_reg (itype);
9361 232 : gimple *g = gimple_build_assign (ne, NOP_EXPR, e);
9362 232 : gimple_set_location (g, loc);
9363 232 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9364 232 : e = ne;
9365 : }
9366 289 : if (!useless_type_conversion_p (itype, TREE_TYPE (d)))
9367 : {
9368 289 : tree nd = create_tmp_reg (itype);
9369 289 : enum tree_code code;
9370 289 : if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (d)))
9371 : {
9372 57 : code = VIEW_CONVERT_EXPR;
9373 57 : d = build1 (VIEW_CONVERT_EXPR, itype, d);
9374 : }
9375 : else
9376 : code = NOP_EXPR;
9377 289 : gimple *g = gimple_build_assign (nd, code, d);
9378 289 : gimple_set_location (g, loc);
9379 289 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9380 289 : d = nd;
9381 : }
9382 :
9383 289 : tree ctype = build_complex_type (itype);
9384 289 : int flag = int_size_in_bytes (itype) + (weak ? 256 : 0);
9385 289 : gimple *g
9386 289 : = gimple_build_call_internal (IFN_ATOMIC_COMPARE_EXCHANGE, 6, addr, e, d,
9387 289 : build_int_cst (integer_type_node, flag),
9388 : mo, fmo);
9389 289 : tree cres = create_tmp_reg (ctype);
9390 289 : gimple_call_set_lhs (g, cres);
9391 289 : gimple_set_location (g, loc);
9392 289 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9393 :
9394 289 : if (cond_stmt || need_old || need_new)
9395 : {
9396 289 : tree im = create_tmp_reg (itype);
9397 289 : g = gimple_build_assign (im, IMAGPART_EXPR,
9398 : build1 (IMAGPART_EXPR, itype, cres));
9399 289 : gimple_set_location (g, loc);
9400 289 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9401 :
9402 289 : tree re = NULL_TREE;
9403 289 : if (need_old || need_new)
9404 : {
9405 199 : re = create_tmp_reg (itype);
9406 199 : g = gimple_build_assign (re, REALPART_EXPR,
9407 : build1 (REALPART_EXPR, itype, cres));
9408 199 : gimple_set_location (g, loc);
9409 199 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9410 : }
9411 :
9412 289 : if (cond_stmt)
9413 : {
9414 289 : g = gimple_build_assign (cond, NOP_EXPR, im);
9415 289 : gimple_set_location (g, loc);
9416 289 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9417 : }
9418 :
9419 289 : if (need_new)
9420 : {
9421 44 : g = gimple_build_assign (create_tmp_reg (itype), COND_EXPR,
9422 : cond_stmt
9423 0 : ? cond : build2 (NE_EXPR, boolean_type_node,
9424 : im, build_zero_cst (itype)),
9425 : d, re);
9426 44 : gimple_set_location (g, loc);
9427 44 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9428 44 : re = gimple_assign_lhs (g);
9429 : }
9430 :
9431 289 : if (need_old || need_new)
9432 : {
9433 199 : tree v = need_old ? loaded_val : stored_val;
9434 199 : enum tree_code code;
9435 199 : if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (v)))
9436 : {
9437 38 : code = VIEW_CONVERT_EXPR;
9438 38 : re = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (v), re);
9439 : }
9440 161 : else if (!useless_type_conversion_p (TREE_TYPE (v), itype))
9441 : code = NOP_EXPR;
9442 : else
9443 0 : code = TREE_CODE (re);
9444 199 : g = gimple_build_assign (v, code, re);
9445 199 : gimple_set_location (g, loc);
9446 199 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9447 : }
9448 : }
9449 :
9450 289 : gsi_remove (&gsi, true);
9451 289 : gsi = gsi_for_stmt (load_stmt);
9452 289 : gsi_remove (&gsi, true);
9453 289 : gsi = gsi_for_stmt (condexpr_stmt);
9454 289 : gsi_remove (&gsi, true);
9455 289 : if (cond_stmt)
9456 : {
9457 289 : gsi = gsi_for_stmt (cond_stmt);
9458 289 : gsi_remove (&gsi, true);
9459 : }
9460 289 : if (vce_stmt)
9461 : {
9462 57 : gsi = gsi_for_stmt (vce_stmt);
9463 57 : gsi_remove (&gsi, true);
9464 : }
9465 :
9466 : return true;
9467 : }
9468 :
9469 : /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
9470 :
9471 : oldval = *addr;
9472 : repeat:
9473 : newval = rhs; // with oldval replacing *addr in rhs
9474 : oldval = __sync_val_compare_and_swap (addr, oldval, newval);
9475 : if (oldval != newval)
9476 : goto repeat;
9477 :
9478 : INDEX is log2 of the size of the data type, and thus usable to find the
9479 : index of the builtin decl. */
9480 :
9481 : static bool
9482 2586 : expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb,
9483 : tree addr, tree loaded_val, tree stored_val,
9484 : int index)
9485 : {
9486 2586 : tree loadedi, storedi, initial, new_storedi, old_vali;
9487 2586 : tree type, itype, cmpxchg, iaddr, atype;
9488 2586 : gimple_stmt_iterator si;
9489 2586 : basic_block loop_header = single_succ (load_bb);
9490 2586 : gimple *phi, *stmt;
9491 2586 : edge e;
9492 2586 : enum built_in_function fncode;
9493 :
9494 2586 : fncode = (enum built_in_function)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
9495 2586 : + index + 1);
9496 2586 : cmpxchg = builtin_decl_explicit (fncode);
9497 2586 : if (cmpxchg == NULL_TREE)
9498 : return false;
9499 2586 : type = TYPE_MAIN_VARIANT (TREE_TYPE (loaded_val));
9500 2586 : atype = type;
9501 2586 : itype = TREE_TYPE (TREE_TYPE (cmpxchg));
9502 :
9503 2586 : if (!can_compare_and_swap_p (TYPE_MODE (itype), true)
9504 2586 : || !can_atomic_load_p (TYPE_MODE (itype)))
9505 83 : return false;
9506 :
9507 : /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
9508 2503 : si = gsi_last_nondebug_bb (load_bb);
9509 2503 : gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
9510 2503 : location_t loc = gimple_location (gsi_stmt (si));
9511 2503 : enum omp_memory_order omo = gimple_omp_atomic_memory_order (gsi_stmt (si));
9512 2503 : tree mo = build_int_cst (NULL, omp_memory_order_to_memmodel (omo));
9513 2503 : tree fmo = build_int_cst (NULL, omp_memory_order_to_fail_memmodel (omo));
9514 :
9515 : /* For floating-point values, we'll need to view-convert them to integers
9516 : so that we can perform the atomic compare and swap. Simplify the
9517 : following code by always setting up the "i"ntegral variables. */
9518 2503 : if (!INTEGRAL_TYPE_P (type) && !POINTER_TYPE_P (type))
9519 : {
9520 845 : tree iaddr_val;
9521 :
9522 845 : iaddr = create_tmp_reg (build_pointer_type_for_mode (itype, ptr_mode,
9523 : true));
9524 845 : atype = itype;
9525 845 : iaddr_val
9526 845 : = force_gimple_operand_gsi (&si,
9527 845 : fold_convert (TREE_TYPE (iaddr), addr),
9528 : false, NULL_TREE, true, GSI_SAME_STMT);
9529 845 : stmt = gimple_build_assign (iaddr, iaddr_val);
9530 845 : gsi_insert_before (&si, stmt, GSI_SAME_STMT);
9531 845 : loadedi = create_tmp_var (itype);
9532 845 : if (gimple_in_ssa_p (cfun))
9533 15 : loadedi = make_ssa_name (loadedi);
9534 : }
9535 : else
9536 : {
9537 : iaddr = addr;
9538 : loadedi = loaded_val;
9539 : }
9540 :
9541 2503 : fncode = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1);
9542 2503 : tree loaddecl = builtin_decl_explicit (fncode);
9543 2503 : if (loaddecl)
9544 2503 : initial
9545 2503 : = fold_convert (atype,
9546 : build_call_expr (loaddecl, 2, iaddr,
9547 : build_int_cst (NULL_TREE,
9548 : MEMMODEL_RELAXED)));
9549 : else
9550 : {
9551 0 : tree off
9552 0 : = build_int_cst (build_pointer_type_for_mode (atype, ptr_mode,
9553 : true), 0);
9554 0 : initial = build2 (MEM_REF, atype, iaddr, off);
9555 : }
9556 :
9557 2503 : initial
9558 2503 : = force_gimple_operand_gsi (&si, initial, true, NULL_TREE, true,
9559 : GSI_SAME_STMT);
9560 :
9561 : /* Move the value to the LOADEDI temporary. */
9562 2503 : if (gimple_in_ssa_p (cfun))
9563 : {
9564 39 : gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header)));
9565 39 : phi = create_phi_node (loadedi, loop_header);
9566 39 : SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, single_succ_edge (load_bb)),
9567 : initial);
9568 : }
9569 : else
9570 2464 : gsi_insert_before (&si,
9571 2464 : gimple_build_assign (loadedi, initial),
9572 : GSI_SAME_STMT);
9573 2503 : if (loadedi != loaded_val)
9574 : {
9575 845 : gimple_stmt_iterator gsi2;
9576 845 : tree x;
9577 :
9578 845 : x = build1 (VIEW_CONVERT_EXPR, type, loadedi);
9579 845 : gsi2 = gsi_start_bb (loop_header);
9580 845 : if (gimple_in_ssa_p (cfun))
9581 : {
9582 15 : gassign *stmt;
9583 15 : x = force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
9584 : true, GSI_SAME_STMT);
9585 15 : stmt = gimple_build_assign (loaded_val, x);
9586 15 : gsi_insert_before (&gsi2, stmt, GSI_SAME_STMT);
9587 : }
9588 : else
9589 : {
9590 830 : x = build2 (MODIFY_EXPR, TREE_TYPE (loaded_val), loaded_val, x);
9591 830 : force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
9592 : true, GSI_SAME_STMT);
9593 : }
9594 : }
9595 2503 : gsi_remove (&si, true);
9596 :
9597 2503 : si = gsi_last_nondebug_bb (store_bb);
9598 2503 : gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
9599 :
9600 2503 : if (iaddr == addr)
9601 : storedi = stored_val;
9602 : else
9603 845 : storedi
9604 845 : = force_gimple_operand_gsi (&si,
9605 : build1 (VIEW_CONVERT_EXPR, itype,
9606 : stored_val), true, NULL_TREE, true,
9607 : GSI_SAME_STMT);
9608 :
9609 : /* Build the compare&swap statement. */
9610 2503 : tree ctype = build_complex_type (itype);
9611 2503 : int flag = int_size_in_bytes (itype);
9612 2503 : new_storedi = build_call_expr_internal_loc (loc, IFN_ATOMIC_COMPARE_EXCHANGE,
9613 : ctype, 6, iaddr, loadedi,
9614 : storedi,
9615 : build_int_cst (integer_type_node,
9616 2503 : flag),
9617 : mo, fmo);
9618 2503 : new_storedi = build1 (REALPART_EXPR, itype, new_storedi);
9619 2503 : new_storedi = force_gimple_operand_gsi (&si,
9620 2503 : fold_convert (TREE_TYPE (loadedi),
9621 : new_storedi),
9622 : true, NULL_TREE,
9623 : true, GSI_SAME_STMT);
9624 :
9625 2503 : if (gimple_in_ssa_p (cfun))
9626 : old_vali = loadedi;
9627 : else
9628 : {
9629 2464 : old_vali = create_tmp_var (TREE_TYPE (loadedi));
9630 2464 : stmt = gimple_build_assign (old_vali, loadedi);
9631 2464 : gsi_insert_before (&si, stmt, GSI_SAME_STMT);
9632 :
9633 2464 : stmt = gimple_build_assign (loadedi, new_storedi);
9634 2464 : gsi_insert_before (&si, stmt, GSI_SAME_STMT);
9635 : }
9636 :
9637 : /* Note that we always perform the comparison as an integer, even for
9638 : floating point. This allows the atomic operation to properly
9639 : succeed even with NaNs and -0.0. */
9640 2503 : tree ne = build2 (NE_EXPR, boolean_type_node, new_storedi, old_vali);
9641 2503 : stmt = gimple_build_cond_empty (ne);
9642 2503 : gsi_insert_before (&si, stmt, GSI_SAME_STMT);
9643 :
9644 : /* Update cfg. */
9645 2503 : e = single_succ_edge (store_bb);
9646 2503 : e->flags &= ~EDGE_FALLTHRU;
9647 2503 : e->flags |= EDGE_FALSE_VALUE;
9648 : /* Expect no looping. */
9649 2503 : e->probability = profile_probability::guessed_always ();
9650 :
9651 2503 : e = make_edge (store_bb, loop_header, EDGE_TRUE_VALUE);
9652 2503 : e->probability = profile_probability::guessed_never ();
9653 :
9654 : /* Copy the new value to loadedi (we already did that before the condition
9655 : if we are not in SSA). */
9656 2503 : if (gimple_in_ssa_p (cfun))
9657 : {
9658 39 : phi = gimple_seq_first_stmt (phi_nodes (loop_header));
9659 39 : SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e), new_storedi);
9660 : }
9661 :
9662 : /* Remove GIMPLE_OMP_ATOMIC_STORE. */
9663 2503 : stmt = gsi_stmt (si);
9664 2503 : gsi_remove (&si, true);
9665 2503 : if (gimple_in_ssa_p (cfun))
9666 39 : release_defs (stmt);
9667 :
9668 2503 : class loop *loop = alloc_loop ();
9669 2503 : loop->header = loop_header;
9670 2503 : loop->latch = store_bb;
9671 2503 : add_loop (loop, loop_header->loop_father);
9672 :
9673 2503 : return true;
9674 : }
9675 :
9676 : /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
9677 :
9678 : GOMP_atomic_start ();
9679 : *addr = rhs;
9680 : GOMP_atomic_end ();
9681 :
9682 : The result is not globally atomic, but works so long as all parallel
9683 : references are within #pragma omp atomic directives. According to
9684 : responses received from omp@openmp.org, appears to be within spec.
9685 : Which makes sense, since that's how several other compilers handle
9686 : this situation as well.
9687 : LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
9688 : expanding. STORED_VAL is the operand of the matching
9689 : GIMPLE_OMP_ATOMIC_STORE.
9690 :
9691 : We replace
9692 : GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
9693 : loaded_val = *addr;
9694 :
9695 : and replace
9696 : GIMPLE_OMP_ATOMIC_STORE (stored_val) with
9697 : *addr = stored_val;
9698 : */
9699 :
9700 : static bool
9701 293 : expand_omp_atomic_mutex (basic_block load_bb, basic_block store_bb,
9702 : tree addr, tree loaded_val, tree stored_val)
9703 : {
9704 293 : gimple_stmt_iterator si;
9705 293 : gassign *stmt;
9706 293 : tree t;
9707 :
9708 293 : si = gsi_last_nondebug_bb (load_bb);
9709 293 : gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
9710 :
9711 293 : t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START);
9712 293 : t = build_call_expr (t, 0);
9713 293 : force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
9714 :
9715 293 : tree mem = build_simple_mem_ref (addr);
9716 293 : TREE_TYPE (mem) = TREE_TYPE (loaded_val);
9717 293 : TREE_OPERAND (mem, 1)
9718 293 : = fold_convert (build_pointer_type_for_mode (TREE_TYPE (mem), ptr_mode,
9719 : true),
9720 : TREE_OPERAND (mem, 1));
9721 293 : stmt = gimple_build_assign (loaded_val, mem);
9722 293 : gsi_insert_before (&si, stmt, GSI_SAME_STMT);
9723 293 : gsi_remove (&si, true);
9724 :
9725 293 : si = gsi_last_nondebug_bb (store_bb);
9726 293 : gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
9727 :
9728 293 : stmt = gimple_build_assign (unshare_expr (mem), stored_val);
9729 586 : gimple_set_vuse (stmt, gimple_vuse (gsi_stmt (si)));
9730 586 : gimple_set_vdef (stmt, gimple_vdef (gsi_stmt (si)));
9731 293 : gsi_insert_before (&si, stmt, GSI_SAME_STMT);
9732 :
9733 293 : t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END);
9734 293 : t = build_call_expr (t, 0);
9735 293 : force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
9736 293 : gsi_remove (&si, true);
9737 293 : return true;
9738 : }
9739 :
9740 : /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
9741 : using expand_omp_atomic_fetch_op. If it failed, we try to
9742 : call expand_omp_atomic_pipeline, and if it fails too, the
9743 : ultimate fallback is wrapping the operation in a mutex
9744 : (expand_omp_atomic_mutex). REGION is the atomic region built
9745 : by build_omp_regions_1(). */
9746 :
9747 : static void
9748 9601 : expand_omp_atomic (struct omp_region *region)
9749 : {
9750 9601 : basic_block load_bb = region->entry, store_bb = region->exit;
9751 9601 : gomp_atomic_load *load
9752 9601 : = as_a <gomp_atomic_load *> (last_nondebug_stmt (load_bb));
9753 9601 : gomp_atomic_store *store
9754 9601 : = as_a <gomp_atomic_store *> (last_nondebug_stmt (store_bb));
9755 9601 : tree loaded_val = gimple_omp_atomic_load_lhs (load);
9756 9601 : tree addr = gimple_omp_atomic_load_rhs (load);
9757 9601 : tree stored_val = gimple_omp_atomic_store_val (store);
9758 9601 : tree type = TYPE_MAIN_VARIANT (TREE_TYPE (loaded_val));
9759 9601 : HOST_WIDE_INT index;
9760 :
9761 : /* Make sure the type is one of the supported sizes. */
9762 9601 : index = tree_to_uhwi (TYPE_SIZE_UNIT (type));
9763 9601 : index = exact_log2 (index);
9764 9567 : if (index >= 0 && index <= 4)
9765 : {
9766 9567 : unsigned int align = TYPE_ALIGN_UNIT (type);
9767 :
9768 : /* __sync builtins require strict data alignment. */
9769 19134 : if (exact_log2 (align) >= index)
9770 : {
9771 : /* Atomic load. */
9772 9391 : scalar_mode smode;
9773 9391 : if (loaded_val == stored_val
9774 1187 : && (is_int_mode (TYPE_MODE (type), &smode)
9775 1337 : || is_float_mode (TYPE_MODE (type), &smode))
9776 1187 : && GET_MODE_BITSIZE (smode) <= BITS_PER_WORD
9777 10574 : && expand_omp_atomic_load (load_bb, addr, loaded_val, index))
9778 9601 : return;
9779 :
9780 : /* Atomic store. */
9781 8208 : if ((is_int_mode (TYPE_MODE (type), &smode)
9782 9253 : || is_float_mode (TYPE_MODE (type), &smode))
9783 8208 : && GET_MODE_BITSIZE (smode) <= BITS_PER_WORD
9784 8125 : && store_bb == single_succ (load_bb)
9785 7510 : && first_stmt (store_bb) == store
9786 856 : && expand_omp_atomic_store (load_bb, addr, loaded_val,
9787 : stored_val, index))
9788 : return;
9789 :
9790 : /* When possible, use specialized atomic update functions. */
9791 985 : if ((INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type))
9792 6375 : && store_bb == single_succ (load_bb)
9793 13180 : && expand_omp_atomic_fetch_op (load_bb, addr,
9794 : loaded_val, stored_val, index))
9795 : return;
9796 :
9797 : /* When possible, use ATOMIC_COMPARE_EXCHANGE ifn without a loop. */
9798 2875 : if (store_bb == single_succ (load_bb)
9799 2256 : && !gimple_in_ssa_p (cfun)
9800 5092 : && expand_omp_atomic_cas (load_bb, addr, loaded_val, stored_val,
9801 : index))
9802 : return;
9803 :
9804 : /* If we don't have specialized __sync builtins, try and implement
9805 : as a compare and swap loop. */
9806 2586 : if (expand_omp_atomic_pipeline (load_bb, store_bb, addr,
9807 : loaded_val, stored_val, index))
9808 : return;
9809 : }
9810 : }
9811 :
9812 : /* The ultimate fallback is wrapping the operation in a mutex. */
9813 293 : expand_omp_atomic_mutex (load_bb, store_bb, addr, loaded_val, stored_val);
9814 : }
9815 :
9816 : /* Mark the loops inside the kernels region starting at REGION_ENTRY and ending
9817 : at REGION_EXIT. */
9818 :
9819 : static void
9820 1651 : mark_loops_in_oacc_kernels_region (basic_block region_entry,
9821 : basic_block region_exit)
9822 : {
9823 1651 : class loop *outer = region_entry->loop_father;
9824 1651 : gcc_assert (region_exit == NULL || outer == region_exit->loop_father);
9825 :
9826 : /* Don't parallelize the kernels region if it contains more than one outer
9827 : loop. */
9828 1651 : unsigned int nr_outer_loops = 0;
9829 1651 : class loop *single_outer = NULL;
9830 33062 : for (class loop *loop = outer->inner; loop != NULL; loop = loop->next)
9831 : {
9832 31411 : gcc_assert (loop_outer (loop) == outer);
9833 :
9834 31411 : if (!dominated_by_p (CDI_DOMINATORS, loop->header, region_entry))
9835 23379 : continue;
9836 :
9837 14588 : if (region_exit != NULL
9838 8032 : && dominated_by_p (CDI_DOMINATORS, loop->header, region_exit))
9839 6556 : continue;
9840 :
9841 1476 : nr_outer_loops++;
9842 1476 : single_outer = loop;
9843 : }
9844 1651 : if (nr_outer_loops != 1)
9845 : return;
9846 :
9847 1336 : for (class loop *loop = single_outer->inner;
9848 2293 : loop != NULL;
9849 957 : loop = loop->inner)
9850 1045 : if (loop->next)
9851 : return;
9852 :
9853 : /* Mark the loops in the region. */
9854 3269 : for (class loop *loop = single_outer; loop != NULL; loop = loop->inner)
9855 2021 : loop->in_oacc_kernels_region = true;
9856 : }
9857 :
9858 : /* Build target argument identifier from the DEVICE identifier, value
9859 : identifier ID and whether the element also has a SUBSEQUENT_PARAM. */
9860 :
9861 : static tree
9862 23498 : get_target_argument_identifier_1 (int device, bool subseqent_param, int id)
9863 : {
9864 23498 : tree t = build_int_cst (integer_type_node, device);
9865 23498 : if (subseqent_param)
9866 703 : t = fold_build2 (BIT_IOR_EXPR, integer_type_node, t,
9867 : build_int_cst (integer_type_node,
9868 : GOMP_TARGET_ARG_SUBSEQUENT_PARAM));
9869 23498 : t = fold_build2 (BIT_IOR_EXPR, integer_type_node, t,
9870 : build_int_cst (integer_type_node, id));
9871 23498 : return t;
9872 : }
9873 :
9874 : /* Like above but return it in type that can be directly stored as an element
9875 : of the argument array. */
9876 :
9877 : static tree
9878 703 : get_target_argument_identifier (int device, bool subseqent_param, int id)
9879 : {
9880 703 : tree t = get_target_argument_identifier_1 (device, subseqent_param, id);
9881 703 : return fold_convert (ptr_type_node, t);
9882 : }
9883 :
9884 : /* Return a target argument consisting of DEVICE identifier, value identifier
9885 : ID, and the actual VALUE. */
9886 :
9887 : static tree
9888 22795 : get_target_argument_value (gimple_stmt_iterator *gsi, int device, int id,
9889 : tree value)
9890 : {
9891 22795 : tree t = fold_build2 (LSHIFT_EXPR, integer_type_node,
9892 : fold_convert (integer_type_node, value),
9893 : build_int_cst (unsigned_type_node,
9894 : GOMP_TARGET_ARG_VALUE_SHIFT));
9895 22795 : t = fold_build2 (BIT_IOR_EXPR, integer_type_node, t,
9896 : get_target_argument_identifier_1 (device, false, id));
9897 22795 : t = fold_convert (ptr_type_node, t);
9898 22795 : return force_gimple_operand_gsi (gsi, t, true, NULL, true, GSI_SAME_STMT);
9899 : }
9900 :
9901 : /* If VALUE is an integer constant greater than -2^15 and smaller than 2^15,
9902 : push one argument to ARGS with both the DEVICE, ID and VALUE embedded in it,
9903 : otherwise push an identifier (with DEVICE and ID) and the VALUE in two
9904 : arguments. */
9905 :
9906 : static void
9907 23498 : push_target_argument_according_to_value (gimple_stmt_iterator *gsi, int device,
9908 : int id, tree value, vec <tree> *args)
9909 : {
9910 23498 : if (tree_fits_shwi_p (value)
9911 22795 : && tree_to_shwi (value) > -(1 << 15)
9912 22795 : && tree_to_shwi (value) < (1 << 15))
9913 22795 : args->quick_push (get_target_argument_value (gsi, device, id, value));
9914 : else
9915 : {
9916 703 : args->quick_push (get_target_argument_identifier (device, true, id));
9917 703 : value = fold_convert (ptr_type_node, value);
9918 703 : value = force_gimple_operand_gsi (gsi, value, true, NULL, true,
9919 : GSI_SAME_STMT);
9920 703 : args->quick_push (value);
9921 : }
9922 23498 : }
9923 :
9924 : /* Create an array of arguments that is then passed to GOMP_target. */
9925 :
9926 : static tree
9927 11749 : get_target_arguments (gimple_stmt_iterator *gsi, gomp_target *tgt_stmt)
9928 : {
9929 11749 : auto_vec <tree, 6> args;
9930 11749 : tree clauses = gimple_omp_target_clauses (tgt_stmt);
9931 11749 : tree t, c = omp_find_clause (clauses, OMP_CLAUSE_NUM_TEAMS);
9932 11749 : if (c)
9933 11749 : t = OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (c);
9934 : else
9935 0 : t = integer_minus_one_node;
9936 11749 : push_target_argument_according_to_value (gsi, GOMP_TARGET_ARG_DEVICE_ALL,
9937 : GOMP_TARGET_ARG_NUM_TEAMS, t, &args);
9938 :
9939 11749 : c = omp_find_clause (clauses, OMP_CLAUSE_THREAD_LIMIT);
9940 11749 : if (c)
9941 11749 : t = OMP_CLAUSE_THREAD_LIMIT_EXPR (c);
9942 : else
9943 0 : t = integer_minus_one_node;
9944 11749 : push_target_argument_according_to_value (gsi, GOMP_TARGET_ARG_DEVICE_ALL,
9945 : GOMP_TARGET_ARG_THREAD_LIMIT, t,
9946 : &args);
9947 :
9948 : /* Produce more, perhaps device specific, arguments here. */
9949 :
9950 23498 : tree argarray = create_tmp_var (build_array_type_nelts (ptr_type_node,
9951 11749 : args.length () + 1),
9952 : ".omp_target_args");
9953 35950 : for (unsigned i = 0; i < args.length (); i++)
9954 : {
9955 24201 : tree ref = build4 (ARRAY_REF, ptr_type_node, argarray,
9956 24201 : build_int_cst (integer_type_node, i),
9957 : NULL_TREE, NULL_TREE);
9958 24201 : gsi_insert_before (gsi, gimple_build_assign (ref, args[i]),
9959 : GSI_SAME_STMT);
9960 : }
9961 23498 : tree ref = build4 (ARRAY_REF, ptr_type_node, argarray,
9962 11749 : build_int_cst (integer_type_node, args.length ()),
9963 : NULL_TREE, NULL_TREE);
9964 11749 : gsi_insert_before (gsi, gimple_build_assign (ref, null_pointer_node),
9965 : GSI_SAME_STMT);
9966 11749 : TREE_ADDRESSABLE (argarray) = 1;
9967 11749 : return build_fold_addr_expr (argarray);
9968 11749 : }
9969 :
9970 : /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
9971 :
9972 : static void
9973 37102 : expand_omp_target (struct omp_region *region)
9974 : {
9975 37102 : basic_block entry_bb, exit_bb, new_bb;
9976 37102 : struct function *child_cfun;
9977 37102 : tree child_fn, child_fn2, block, t, c;
9978 37102 : gimple_stmt_iterator gsi;
9979 37102 : gomp_target *entry_stmt;
9980 37102 : gimple *stmt;
9981 37102 : edge e;
9982 37102 : bool offloaded;
9983 37102 : int target_kind;
9984 :
9985 37102 : entry_stmt = as_a <gomp_target *> (last_nondebug_stmt (region->entry));
9986 37102 : target_kind = gimple_omp_target_kind (entry_stmt);
9987 37102 : new_bb = region->entry;
9988 :
9989 37102 : offloaded = is_gimple_omp_offloaded (entry_stmt);
9990 37102 : switch (target_kind)
9991 : {
9992 37102 : case GF_OMP_TARGET_KIND_REGION:
9993 37102 : case GF_OMP_TARGET_KIND_UPDATE:
9994 37102 : case GF_OMP_TARGET_KIND_ENTER_DATA:
9995 37102 : case GF_OMP_TARGET_KIND_EXIT_DATA:
9996 37102 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
9997 37102 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
9998 37102 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
9999 37102 : case GF_OMP_TARGET_KIND_OACC_UPDATE:
10000 37102 : case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
10001 37102 : case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
10002 37102 : case GF_OMP_TARGET_KIND_OACC_DECLARE:
10003 37102 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
10004 37102 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
10005 37102 : case GF_OMP_TARGET_KIND_DATA:
10006 37102 : case GF_OMP_TARGET_KIND_OACC_DATA:
10007 37102 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
10008 37102 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
10009 37102 : break;
10010 0 : default:
10011 0 : gcc_unreachable ();
10012 : }
10013 :
10014 37102 : tree clauses = gimple_omp_target_clauses (entry_stmt);
10015 :
10016 37102 : bool is_ancestor = false;
10017 37102 : child_fn = child_fn2 = NULL_TREE;
10018 37102 : child_cfun = NULL;
10019 37102 : if (offloaded)
10020 : {
10021 21144 : c = omp_find_clause (clauses, OMP_CLAUSE_DEVICE);
10022 21144 : if (ENABLE_OFFLOADING && c)
10023 : is_ancestor = OMP_CLAUSE_DEVICE_ANCESTOR (c);
10024 21144 : child_fn = gimple_omp_target_child_fn (entry_stmt);
10025 21144 : child_cfun = DECL_STRUCT_FUNCTION (child_fn);
10026 : }
10027 :
10028 : /* Supported by expand_omp_taskreg, but not here. */
10029 21144 : if (child_cfun != NULL)
10030 21144 : gcc_checking_assert (!child_cfun->cfg);
10031 37102 : gcc_checking_assert (!gimple_in_ssa_p (cfun));
10032 :
10033 37102 : entry_bb = region->entry;
10034 37102 : exit_bb = region->exit;
10035 :
10036 37102 : if (target_kind == GF_OMP_TARGET_KIND_OACC_KERNELS)
10037 1651 : mark_loops_in_oacc_kernels_region (region->entry, region->exit);
10038 :
10039 : /* Going on, all OpenACC compute constructs are mapped to
10040 : 'BUILT_IN_GOACC_PARALLEL', and get their compute regions outlined.
10041 : To distinguish between them, we attach attributes. */
10042 37102 : switch (target_kind)
10043 : {
10044 6825 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
10045 6825 : DECL_ATTRIBUTES (child_fn)
10046 6825 : = tree_cons (get_identifier ("oacc parallel"),
10047 6825 : NULL_TREE, DECL_ATTRIBUTES (child_fn));
10048 6825 : break;
10049 1651 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
10050 1651 : DECL_ATTRIBUTES (child_fn)
10051 1651 : = tree_cons (get_identifier ("oacc kernels"),
10052 1651 : NULL_TREE, DECL_ATTRIBUTES (child_fn));
10053 1651 : break;
10054 756 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
10055 756 : DECL_ATTRIBUTES (child_fn)
10056 756 : = tree_cons (get_identifier ("oacc serial"),
10057 756 : NULL_TREE, DECL_ATTRIBUTES (child_fn));
10058 756 : break;
10059 54 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
10060 54 : DECL_ATTRIBUTES (child_fn)
10061 54 : = tree_cons (get_identifier ("oacc parallel_kernels_parallelized"),
10062 54 : NULL_TREE, DECL_ATTRIBUTES (child_fn));
10063 54 : break;
10064 109 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
10065 109 : DECL_ATTRIBUTES (child_fn)
10066 109 : = tree_cons (get_identifier ("oacc parallel_kernels_gang_single"),
10067 109 : NULL_TREE, DECL_ATTRIBUTES (child_fn));
10068 109 : break;
10069 27707 : default:
10070 : /* Make sure we don't miss any. */
10071 27707 : gcc_checking_assert (!(is_gimple_omp_oacc (entry_stmt)
10072 : && is_gimple_omp_offloaded (entry_stmt)));
10073 : break;
10074 : }
10075 :
10076 37102 : if (offloaded)
10077 : {
10078 21144 : unsigned srcidx, dstidx, num;
10079 :
10080 : /* If the offloading region needs data sent from the parent
10081 : function, then the very first statement (except possible
10082 : tree profile counter updates) of the offloading body
10083 : is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
10084 : &.OMP_DATA_O is passed as an argument to the child function,
10085 : we need to replace it with the argument as seen by the child
10086 : function.
10087 :
10088 : In most cases, this will end up being the identity assignment
10089 : .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
10090 : a function call that has been inlined, the original PARM_DECL
10091 : .OMP_DATA_I may have been converted into a different local
10092 : variable. In which case, we need to keep the assignment. */
10093 21144 : tree data_arg = gimple_omp_target_data_arg (entry_stmt);
10094 21144 : if (data_arg)
10095 : {
10096 16509 : basic_block entry_succ_bb = single_succ (entry_bb);
10097 16509 : gimple_stmt_iterator gsi;
10098 16509 : tree arg;
10099 16509 : gimple *tgtcopy_stmt = NULL;
10100 16509 : tree sender = TREE_VEC_ELT (data_arg, 0);
10101 :
10102 33018 : for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
10103 : {
10104 16509 : gcc_assert (!gsi_end_p (gsi));
10105 16509 : stmt = gsi_stmt (gsi);
10106 16509 : if (gimple_code (stmt) != GIMPLE_ASSIGN)
10107 0 : continue;
10108 :
10109 16509 : if (gimple_num_ops (stmt) == 2)
10110 : {
10111 16509 : tree arg = gimple_assign_rhs1 (stmt);
10112 :
10113 : /* We're ignoring the subcode because we're
10114 : effectively doing a STRIP_NOPS. */
10115 :
10116 16509 : if ((TREE_CODE (arg) == ADDR_EXPR
10117 16402 : && TREE_OPERAND (arg, 0) == sender)
10118 16509 : || arg == sender)
10119 : {
10120 16509 : tgtcopy_stmt = stmt;
10121 16509 : break;
10122 : }
10123 : }
10124 : }
10125 :
10126 16509 : gcc_assert (tgtcopy_stmt != NULL);
10127 16509 : arg = DECL_ARGUMENTS (child_fn);
10128 :
10129 16509 : gcc_assert (gimple_assign_lhs (tgtcopy_stmt) == arg);
10130 16509 : gsi_remove (&gsi, true);
10131 : }
10132 :
10133 : /* Declare local variables needed in CHILD_CFUN. */
10134 21144 : block = DECL_INITIAL (child_fn);
10135 21144 : BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
10136 : /* The gimplifier could record temporaries in the offloading block
10137 : rather than in containing function's local_decls chain,
10138 : which would mean cgraph missed finalizing them. Do it now. */
10139 155845 : for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
10140 134701 : if (VAR_P (t) && TREE_STATIC (t) && !DECL_EXTERNAL (t))
10141 0 : varpool_node::finalize_decl (t);
10142 21144 : DECL_SAVED_TREE (child_fn) = NULL;
10143 : /* We'll create a CFG for child_fn, so no gimple body is needed. */
10144 21144 : gimple_set_body (child_fn, NULL);
10145 21144 : TREE_USED (block) = 1;
10146 :
10147 : /* Reset DECL_CONTEXT on function arguments. */
10148 42288 : for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
10149 21144 : DECL_CONTEXT (t) = child_fn;
10150 :
10151 : /* Split ENTRY_BB at GIMPLE_*,
10152 : so that it can be moved to the child function. */
10153 21144 : gsi = gsi_last_nondebug_bb (entry_bb);
10154 21144 : stmt = gsi_stmt (gsi);
10155 21144 : gcc_assert (stmt
10156 : && gimple_code (stmt) == gimple_code (entry_stmt));
10157 21144 : e = split_block (entry_bb, stmt);
10158 21144 : gsi_remove (&gsi, true);
10159 21144 : entry_bb = e->dest;
10160 21144 : single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
10161 :
10162 : /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
10163 21144 : if (exit_bb)
10164 : {
10165 21068 : gsi = gsi_last_nondebug_bb (exit_bb);
10166 21068 : gcc_assert (!gsi_end_p (gsi)
10167 : && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
10168 21068 : stmt = gimple_build_return (NULL);
10169 21068 : gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
10170 21068 : gsi_remove (&gsi, true);
10171 : }
10172 :
10173 : /* Move the offloading region into CHILD_CFUN. */
10174 :
10175 21144 : block = gimple_block (entry_stmt);
10176 :
10177 21144 : new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
10178 21144 : if (exit_bb)
10179 21068 : single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
10180 : /* When the OMP expansion process cannot guarantee an up-to-date
10181 : loop tree arrange for the child function to fixup loops. */
10182 21144 : if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
10183 21144 : child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
10184 :
10185 : /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
10186 21144 : num = vec_safe_length (child_cfun->local_decls);
10187 642904 : for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
10188 : {
10189 621760 : t = (*child_cfun->local_decls)[srcidx];
10190 621760 : if (DECL_CONTEXT (t) == cfun->decl)
10191 134701 : continue;
10192 487059 : if (srcidx != dstidx)
10193 472085 : (*child_cfun->local_decls)[dstidx] = t;
10194 487059 : dstidx++;
10195 : }
10196 21144 : if (dstidx != num)
10197 17816 : vec_safe_truncate (child_cfun->local_decls, dstidx);
10198 :
10199 : /* Inform the callgraph about the new function. */
10200 21144 : child_cfun->curr_properties = cfun->curr_properties;
10201 21144 : child_cfun->has_simduid_loops |= cfun->has_simduid_loops;
10202 21144 : child_cfun->has_force_vectorize_loops |= cfun->has_force_vectorize_loops;
10203 21144 : cgraph_node *node = cgraph_node::get_create (child_fn);
10204 21144 : node->parallelized_function = 1;
10205 42288 : node->has_omp_variant_constructs
10206 21144 : |= cgraph_node::get (cfun->decl)->has_omp_variant_constructs;
10207 21144 : cgraph_node::add_new_function (child_fn, true);
10208 :
10209 : /* Add the new function to the offload table. */
10210 21144 : if (ENABLE_OFFLOADING)
10211 : {
10212 : if (in_lto_p)
10213 : DECL_PRESERVE_P (child_fn) = 1;
10214 : if (!is_ancestor)
10215 : vec_safe_push (offload_funcs, child_fn);
10216 : }
10217 :
10218 21144 : bool need_asm = DECL_ASSEMBLER_NAME_SET_P (current_function_decl)
10219 21144 : && !DECL_ASSEMBLER_NAME_SET_P (child_fn);
10220 :
10221 : /* Fix the callgraph edges for child_cfun. Those for cfun will be
10222 : fixed in a following pass. */
10223 21144 : push_cfun (child_cfun);
10224 21144 : if (need_asm)
10225 21138 : assign_assembler_name_if_needed (child_fn);
10226 21144 : cgraph_edge::rebuild_edges ();
10227 :
10228 : /* Some EH regions might become dead, see PR34608. If
10229 : pass_cleanup_cfg isn't the first pass to happen with the
10230 : new child, these dead EH edges might cause problems.
10231 : Clean them up now. */
10232 21144 : if (flag_exceptions)
10233 : {
10234 8282 : basic_block bb;
10235 8282 : bool changed = false;
10236 :
10237 118988 : FOR_EACH_BB_FN (bb, cfun)
10238 110706 : changed |= gimple_purge_dead_eh_edges (bb);
10239 8282 : if (changed)
10240 0 : cleanup_tree_cfg ();
10241 : }
10242 21144 : if (flag_checking && !loops_state_satisfies_p (LOOPS_NEED_FIXUP))
10243 0 : verify_loop_structure ();
10244 21144 : pop_cfun ();
10245 :
10246 21144 : if (dump_file && !gimple_in_ssa_p (cfun))
10247 : {
10248 31 : omp_any_child_fn_dumped = true;
10249 31 : dump_function_header (dump_file, child_fn, dump_flags);
10250 31 : dump_function_to_file (child_fn, dump_file, dump_flags);
10251 : }
10252 :
10253 21144 : adjust_context_and_scope (region, gimple_block (entry_stmt), child_fn);
10254 :
10255 : /* Handle the case that an inner ancestor:1 target is called by an outer
10256 : target region. */
10257 21144 : if (is_ancestor)
10258 : {
10259 : cgraph_node *fn2_node;
10260 : child_fn2 = build_decl (DECL_SOURCE_LOCATION (child_fn),
10261 : FUNCTION_DECL,
10262 : clone_function_name (child_fn, "nohost"),
10263 : TREE_TYPE (child_fn));
10264 : if (in_lto_p)
10265 : DECL_PRESERVE_P (child_fn2) = 1;
10266 : TREE_STATIC (child_fn2) = 1;
10267 : DECL_ARTIFICIAL (child_fn2) = 1;
10268 : DECL_IGNORED_P (child_fn2) = 0;
10269 : TREE_PUBLIC (child_fn2) = 0;
10270 : DECL_UNINLINABLE (child_fn2) = 1;
10271 : DECL_EXTERNAL (child_fn2) = 0;
10272 : DECL_CONTEXT (child_fn2) = DECL_CONTEXT (child_fn);
10273 : DECL_INITIAL (child_fn2) = make_node (BLOCK);
10274 : BLOCK_SUPERCONTEXT (DECL_INITIAL (child_fn2)) = child_fn2;
10275 : DECL_ATTRIBUTES (child_fn)
10276 : = remove_attribute ("omp target entrypoint",
10277 : DECL_ATTRIBUTES (child_fn));
10278 : DECL_ATTRIBUTES (child_fn2)
10279 : = tree_cons (get_identifier ("omp target device_ancestor_nohost"),
10280 : NULL_TREE, copy_list (DECL_ATTRIBUTES (child_fn)));
10281 : DECL_ATTRIBUTES (child_fn)
10282 : = tree_cons (get_identifier ("omp target device_ancestor_host"),
10283 : NULL_TREE, DECL_ATTRIBUTES (child_fn));
10284 : DECL_FUNCTION_SPECIFIC_OPTIMIZATION (child_fn2)
10285 : = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (current_function_decl);
10286 : DECL_FUNCTION_SPECIFIC_TARGET (child_fn2)
10287 : = DECL_FUNCTION_SPECIFIC_TARGET (current_function_decl);
10288 : DECL_FUNCTION_VERSIONED (child_fn2)
10289 : = DECL_FUNCTION_VERSIONED (current_function_decl);
10290 :
10291 : fn2_node = cgraph_node::get_create (child_fn2);
10292 : fn2_node->offloadable = 1;
10293 : fn2_node->force_output = 1;
10294 : node->offloadable = 0;
10295 :
10296 : /* Enable pass_omp_device_lower pass. */
10297 : fn2_node = cgraph_node::get (DECL_CONTEXT (child_fn));
10298 : fn2_node->has_omp_variant_constructs = 1;
10299 :
10300 : t = build_decl (DECL_SOURCE_LOCATION (child_fn),
10301 : RESULT_DECL, NULL_TREE, void_type_node);
10302 : DECL_ARTIFICIAL (t) = 1;
10303 : DECL_IGNORED_P (t) = 1;
10304 : DECL_CONTEXT (t) = child_fn2;
10305 : DECL_RESULT (child_fn2) = t;
10306 : DECL_SAVED_TREE (child_fn2) = build1 (RETURN_EXPR,
10307 : void_type_node, NULL);
10308 : tree tmp = DECL_ARGUMENTS (child_fn);
10309 : t = build_decl (DECL_SOURCE_LOCATION (child_fn), PARM_DECL,
10310 : DECL_NAME (tmp), TREE_TYPE (tmp));
10311 : DECL_ARTIFICIAL (t) = 1;
10312 : DECL_NAMELESS (t) = 1;
10313 : DECL_ARG_TYPE (t) = ptr_type_node;
10314 : DECL_CONTEXT (t) = current_function_decl;
10315 : TREE_USED (t) = 1;
10316 : TREE_READONLY (t) = 1;
10317 : DECL_ARGUMENTS (child_fn2) = t;
10318 : gcc_assert (TREE_CHAIN (tmp) == NULL_TREE);
10319 :
10320 : gimplify_function_tree (child_fn2);
10321 : cgraph_node::add_new_function (child_fn2, true);
10322 :
10323 : vec_safe_push (offload_funcs, child_fn2);
10324 : if (dump_file && !gimple_in_ssa_p (cfun))
10325 : {
10326 : dump_function_header (dump_file, child_fn2, dump_flags);
10327 : dump_function_to_file (child_fn2, dump_file, dump_flags);
10328 : }
10329 : }
10330 : }
10331 :
10332 : /* Emit a library call to launch the offloading region, or do data
10333 : transfers. */
10334 37102 : tree t1, t2, t3, t4, depend;
10335 37102 : enum built_in_function start_ix;
10336 37102 : unsigned int flags_i = 0;
10337 :
10338 37102 : switch (gimple_omp_target_kind (entry_stmt))
10339 : {
10340 : case GF_OMP_TARGET_KIND_REGION:
10341 : start_ix = BUILT_IN_GOMP_TARGET;
10342 : break;
10343 : case GF_OMP_TARGET_KIND_DATA:
10344 : start_ix = BUILT_IN_GOMP_TARGET_DATA;
10345 : break;
10346 : case GF_OMP_TARGET_KIND_UPDATE:
10347 : start_ix = BUILT_IN_GOMP_TARGET_UPDATE;
10348 : break;
10349 : case GF_OMP_TARGET_KIND_ENTER_DATA:
10350 : start_ix = BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA;
10351 : break;
10352 : case GF_OMP_TARGET_KIND_EXIT_DATA:
10353 : start_ix = BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA;
10354 : flags_i |= GOMP_TARGET_FLAG_EXIT_DATA;
10355 : break;
10356 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
10357 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
10358 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
10359 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
10360 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
10361 : start_ix = BUILT_IN_GOACC_PARALLEL;
10362 : break;
10363 : case GF_OMP_TARGET_KIND_OACC_DATA:
10364 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
10365 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
10366 : start_ix = BUILT_IN_GOACC_DATA_START;
10367 : break;
10368 : case GF_OMP_TARGET_KIND_OACC_UPDATE:
10369 : start_ix = BUILT_IN_GOACC_UPDATE;
10370 : break;
10371 : case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
10372 : start_ix = BUILT_IN_GOACC_ENTER_DATA;
10373 : break;
10374 : case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
10375 : start_ix = BUILT_IN_GOACC_EXIT_DATA;
10376 : break;
10377 : case GF_OMP_TARGET_KIND_OACC_DECLARE:
10378 : start_ix = BUILT_IN_GOACC_DECLARE;
10379 : break;
10380 0 : default:
10381 0 : gcc_unreachable ();
10382 : }
10383 :
10384 37102 : tree device = NULL_TREE;
10385 37102 : location_t device_loc = UNKNOWN_LOCATION;
10386 37102 : tree goacc_flags = NULL_TREE;
10387 37102 : bool need_device_adjustment = false;
10388 37102 : gimple_stmt_iterator adj_gsi;
10389 37102 : if (is_gimple_omp_oacc (entry_stmt))
10390 : {
10391 : /* By default, no GOACC_FLAGs are set. */
10392 14785 : goacc_flags = integer_zero_node;
10393 : }
10394 : else
10395 : {
10396 22317 : c = omp_find_clause (clauses, OMP_CLAUSE_DEVICE);
10397 22317 : if (c)
10398 : {
10399 928 : device = OMP_CLAUSE_DEVICE_ID (c);
10400 : /* Ensure 'device' is of the correct type. */
10401 928 : device = fold_convert_loc (device_loc, integer_type_node, device);
10402 928 : if (TREE_CODE (device) == INTEGER_CST)
10403 : {
10404 54 : if (wi::to_wide (device) == GOMP_DEVICE_ICV)
10405 2 : device = build_int_cst (integer_type_node,
10406 : GOMP_DEVICE_HOST_FALLBACK);
10407 52 : else if (wi::to_wide (device) == GOMP_DEVICE_HOST_FALLBACK)
10408 0 : device = build_int_cst (integer_type_node,
10409 : GOMP_DEVICE_HOST_FALLBACK - 1);
10410 : }
10411 : else
10412 : need_device_adjustment = true;
10413 928 : device_loc = OMP_CLAUSE_LOCATION (c);
10414 928 : if (OMP_CLAUSE_DEVICE_ANCESTOR (c))
10415 41 : device = build_int_cst (integer_type_node,
10416 : GOMP_DEVICE_HOST_FALLBACK);
10417 : }
10418 : else
10419 : {
10420 : /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
10421 : library choose). */
10422 21389 : device = build_int_cst (integer_type_node, GOMP_DEVICE_ICV);
10423 21389 : device_loc = gimple_location (entry_stmt);
10424 : }
10425 :
10426 22317 : c = omp_find_clause (clauses, OMP_CLAUSE_NOWAIT);
10427 : /* FIXME: in_reduction(...) nowait is unimplemented yet, pretend
10428 : nowait doesn't appear. */
10429 22317 : if (c && omp_find_clause (clauses, OMP_CLAUSE_IN_REDUCTION))
10430 : c = NULL;
10431 22053 : if (c)
10432 137 : flags_i |= GOMP_TARGET_FLAG_NOWAIT;
10433 : }
10434 :
10435 : /* By default, there is no conditional. */
10436 37102 : tree cond = NULL_TREE;
10437 37102 : c = omp_find_clause (clauses, OMP_CLAUSE_IF);
10438 37102 : if (c)
10439 1741 : cond = OMP_CLAUSE_IF_EXPR (c);
10440 : /* If we found the clause 'if (cond)', build:
10441 : OpenACC: goacc_flags = (cond ? goacc_flags
10442 : : goacc_flags | GOACC_FLAG_HOST_FALLBACK)
10443 : OpenMP: device = (cond ? device : GOMP_DEVICE_HOST_FALLBACK) */
10444 1741 : if (cond)
10445 : {
10446 1741 : tree *tp;
10447 1741 : if (is_gimple_omp_oacc (entry_stmt))
10448 : tp = &goacc_flags;
10449 : else
10450 666 : tp = &device;
10451 :
10452 1741 : cond = gimple_boolify (cond);
10453 :
10454 1741 : basic_block cond_bb, then_bb, else_bb;
10455 1741 : edge e;
10456 1741 : tree tmp_var = create_tmp_var (TREE_TYPE (*tp));
10457 1741 : if (offloaded)
10458 806 : e = split_block_after_labels (new_bb);
10459 : else
10460 : {
10461 935 : gsi = gsi_last_nondebug_bb (new_bb);
10462 935 : gsi_prev (&gsi);
10463 935 : e = split_block (new_bb, gsi_stmt (gsi));
10464 : }
10465 1741 : cond_bb = e->src;
10466 1741 : new_bb = e->dest;
10467 1741 : remove_edge (e);
10468 :
10469 1741 : then_bb = create_empty_bb (cond_bb);
10470 1741 : else_bb = create_empty_bb (then_bb);
10471 1741 : set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
10472 1741 : set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
10473 :
10474 1741 : stmt = gimple_build_cond_empty (cond);
10475 1741 : gsi = gsi_last_bb (cond_bb);
10476 1741 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
10477 :
10478 1741 : gsi = gsi_start_bb (then_bb);
10479 1741 : stmt = gimple_build_assign (tmp_var, *tp);
10480 1741 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
10481 1741 : adj_gsi = gsi;
10482 :
10483 1741 : gsi = gsi_start_bb (else_bb);
10484 1741 : if (is_gimple_omp_oacc (entry_stmt))
10485 1075 : stmt = gimple_build_assign (tmp_var,
10486 : BIT_IOR_EXPR,
10487 : *tp,
10488 : build_int_cst (integer_type_node,
10489 : GOACC_FLAG_HOST_FALLBACK));
10490 : else
10491 666 : stmt = gimple_build_assign (tmp_var,
10492 : build_int_cst (integer_type_node,
10493 : GOMP_DEVICE_HOST_FALLBACK));
10494 1741 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
10495 :
10496 1741 : make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
10497 1741 : make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
10498 1741 : add_bb_to_loop (then_bb, cond_bb->loop_father);
10499 1741 : add_bb_to_loop (else_bb, cond_bb->loop_father);
10500 1741 : make_edge (then_bb, new_bb, EDGE_FALLTHRU);
10501 1741 : make_edge (else_bb, new_bb, EDGE_FALLTHRU);
10502 :
10503 1741 : *tp = tmp_var;
10504 :
10505 1741 : gsi = gsi_last_nondebug_bb (new_bb);
10506 : }
10507 : else
10508 : {
10509 35361 : gsi = gsi_last_nondebug_bb (new_bb);
10510 :
10511 35361 : if (device != NULL_TREE)
10512 21651 : device = force_gimple_operand_gsi (&gsi, device, true, NULL_TREE,
10513 : true, GSI_SAME_STMT);
10514 35361 : if (need_device_adjustment)
10515 : {
10516 400 : tree tmp_var = create_tmp_var (TREE_TYPE (device));
10517 400 : stmt = gimple_build_assign (tmp_var, device);
10518 400 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
10519 400 : adj_gsi = gsi_for_stmt (stmt);
10520 400 : device = tmp_var;
10521 : }
10522 : }
10523 :
10524 37102 : if ((c = omp_find_clause (clauses, OMP_CLAUSE_SELF)) != NULL_TREE)
10525 : {
10526 324 : gcc_assert ((is_gimple_omp_oacc (entry_stmt) && offloaded)
10527 : || (gimple_omp_target_kind (entry_stmt)
10528 : == GF_OMP_TARGET_KIND_OACC_DATA_KERNELS));
10529 :
10530 87 : edge e;
10531 87 : if (offloaded)
10532 237 : e = split_block_after_labels (new_bb);
10533 : else
10534 : {
10535 87 : gsi = gsi_last_nondebug_bb (new_bb);
10536 87 : gsi_prev (&gsi);
10537 87 : e = split_block (new_bb, gsi_stmt (gsi));
10538 : }
10539 324 : basic_block cond_bb = e->src;
10540 324 : new_bb = e->dest;
10541 324 : remove_edge (e);
10542 :
10543 324 : basic_block then_bb = create_empty_bb (cond_bb);
10544 324 : basic_block else_bb = create_empty_bb (then_bb);
10545 324 : set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
10546 324 : set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
10547 :
10548 324 : tree self_cond = gimple_boolify (OMP_CLAUSE_SELF_EXPR (c));
10549 324 : stmt = gimple_build_cond_empty (self_cond);
10550 324 : gsi = gsi_last_bb (cond_bb);
10551 324 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
10552 :
10553 324 : tree tmp_var = create_tmp_var (TREE_TYPE (goacc_flags));
10554 324 : stmt = gimple_build_assign (tmp_var, BIT_IOR_EXPR, goacc_flags,
10555 : build_int_cst (integer_type_node,
10556 : GOACC_FLAG_LOCAL_DEVICE));
10557 324 : gsi = gsi_start_bb (then_bb);
10558 324 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
10559 :
10560 324 : gsi = gsi_start_bb (else_bb);
10561 324 : stmt = gimple_build_assign (tmp_var, goacc_flags);
10562 324 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
10563 :
10564 324 : make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
10565 324 : make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
10566 324 : add_bb_to_loop (then_bb, cond_bb->loop_father);
10567 324 : add_bb_to_loop (else_bb, cond_bb->loop_father);
10568 324 : make_edge (then_bb, new_bb, EDGE_FALLTHRU);
10569 324 : make_edge (else_bb, new_bb, EDGE_FALLTHRU);
10570 :
10571 324 : goacc_flags = tmp_var;
10572 324 : gsi = gsi_last_nondebug_bb (new_bb);
10573 : }
10574 :
10575 37102 : if (need_device_adjustment)
10576 : {
10577 874 : tree uns = fold_convert (unsigned_type_node, device);
10578 874 : uns = force_gimple_operand_gsi (&adj_gsi, uns, true, NULL_TREE,
10579 : false, GSI_CONTINUE_LINKING);
10580 874 : edge e = split_block (gsi_bb (adj_gsi), gsi_stmt (adj_gsi));
10581 874 : basic_block cond_bb = e->src;
10582 874 : basic_block else_bb = e->dest;
10583 874 : if (gsi_bb (adj_gsi) == new_bb)
10584 : {
10585 400 : new_bb = else_bb;
10586 400 : gsi = gsi_last_nondebug_bb (new_bb);
10587 : }
10588 :
10589 874 : basic_block then_bb = create_empty_bb (cond_bb);
10590 874 : set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
10591 :
10592 874 : cond = build2 (GT_EXPR, boolean_type_node, uns,
10593 : build_int_cst (unsigned_type_node,
10594 : GOMP_DEVICE_HOST_FALLBACK - 1));
10595 874 : stmt = gimple_build_cond_empty (cond);
10596 874 : adj_gsi = gsi_last_bb (cond_bb);
10597 874 : gsi_insert_after (&adj_gsi, stmt, GSI_CONTINUE_LINKING);
10598 :
10599 874 : adj_gsi = gsi_start_bb (then_bb);
10600 874 : tree add = build2 (PLUS_EXPR, integer_type_node, device,
10601 : build_int_cst (integer_type_node, -1));
10602 874 : stmt = gimple_build_assign (device, add);
10603 874 : gsi_insert_after (&adj_gsi, stmt, GSI_CONTINUE_LINKING);
10604 :
10605 874 : make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
10606 874 : e->flags = EDGE_FALSE_VALUE;
10607 874 : add_bb_to_loop (then_bb, cond_bb->loop_father);
10608 874 : make_edge (then_bb, else_bb, EDGE_FALLTHRU);
10609 : }
10610 :
10611 37102 : t = gimple_omp_target_data_arg (entry_stmt);
10612 37102 : if (t == NULL)
10613 : {
10614 4872 : t1 = size_zero_node;
10615 4872 : t2 = build_zero_cst (ptr_type_node);
10616 4872 : t3 = t2;
10617 4872 : t4 = t2;
10618 : }
10619 32230 : else if (TREE_VEC_LENGTH (t) == 3 || is_gimple_omp_oacc (entry_stmt))
10620 : {
10621 32088 : t1 = TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t, 1))));
10622 32088 : t1 = size_binop (PLUS_EXPR, t1, size_int (1));
10623 32088 : t2 = build_fold_addr_expr (TREE_VEC_ELT (t, 0));
10624 32088 : t3 = build_fold_addr_expr (TREE_VEC_ELT (t, 1));
10625 32088 : t4 = build_fold_addr_expr (TREE_VEC_ELT (t, 2));
10626 : }
10627 : else
10628 : {
10629 142 : t1 = force_gimple_operand_gsi (&gsi, TREE_VEC_ELT (t, 3), true, NULL_TREE,
10630 : true, GSI_SAME_STMT);
10631 142 : t2 = force_gimple_operand_gsi (&gsi, TREE_VEC_ELT (t, 0), true, NULL_TREE,
10632 : true, GSI_SAME_STMT);
10633 142 : t3 = force_gimple_operand_gsi (&gsi, TREE_VEC_ELT (t, 1), true, NULL_TREE,
10634 : true, GSI_SAME_STMT);
10635 142 : t4 = force_gimple_operand_gsi (&gsi, TREE_VEC_ELT (t, 2), true, NULL_TREE,
10636 : true, GSI_SAME_STMT);
10637 : }
10638 :
10639 37102 : gimple *g;
10640 37102 : bool tagging = false;
10641 : /* The maximum number used by any start_ix, without varargs. */
10642 37102 : auto_vec<tree, 11> args;
10643 37102 : if (is_gimple_omp_oacc (entry_stmt))
10644 : {
10645 14785 : tree goacc_flags_m = fold_build1 (GOACC_FLAGS_MARSHAL_OP,
10646 : TREE_TYPE (goacc_flags), goacc_flags);
10647 14785 : goacc_flags_m = force_gimple_operand_gsi (&gsi, goacc_flags_m, true,
10648 : NULL_TREE, true,
10649 : GSI_SAME_STMT);
10650 14785 : args.quick_push (goacc_flags_m);
10651 : }
10652 : else
10653 22317 : args.quick_push (device);
10654 37102 : if (offloaded)
10655 21144 : args.quick_push (build_fold_addr_expr (child_fn2 ? child_fn2 : child_fn));
10656 37102 : args.quick_push (t1);
10657 37102 : args.quick_push (t2);
10658 37102 : args.quick_push (t3);
10659 37102 : args.quick_push (t4);
10660 37102 : switch (start_ix)
10661 : {
10662 : case BUILT_IN_GOACC_DATA_START:
10663 : case BUILT_IN_GOACC_DECLARE:
10664 : case BUILT_IN_GOMP_TARGET_DATA:
10665 : break;
10666 20683 : case BUILT_IN_GOMP_TARGET:
10667 20683 : case BUILT_IN_GOMP_TARGET_UPDATE:
10668 20683 : case BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA:
10669 20683 : args.quick_push (build_int_cst (unsigned_type_node, flags_i));
10670 20683 : c = omp_find_clause (clauses, OMP_CLAUSE_DEPEND);
10671 20683 : if (c)
10672 329 : depend = OMP_CLAUSE_DECL (c);
10673 : else
10674 20354 : depend = build_int_cst (ptr_type_node, 0);
10675 20683 : args.quick_push (depend);
10676 20683 : if (start_ix == BUILT_IN_GOMP_TARGET)
10677 11749 : args.quick_push (get_target_arguments (&gsi, entry_stmt));
10678 : break;
10679 9395 : case BUILT_IN_GOACC_PARALLEL:
10680 9395 : if (lookup_attribute ("oacc serial", DECL_ATTRIBUTES (child_fn)) != NULL)
10681 : {
10682 : tree dims = NULL_TREE;
10683 : unsigned int ix;
10684 :
10685 : /* For serial constructs we set all dimensions to 1. */
10686 3024 : for (ix = GOMP_DIM_MAX; ix--;)
10687 2268 : dims = tree_cons (NULL_TREE, integer_one_node, dims);
10688 756 : oacc_replace_fn_attrib (child_fn, dims);
10689 : }
10690 : else
10691 8639 : oacc_set_fn_attrib (child_fn, clauses, &args);
10692 : tagging = true;
10693 : /* FALLTHRU */
10694 12136 : case BUILT_IN_GOACC_ENTER_DATA:
10695 12136 : case BUILT_IN_GOACC_EXIT_DATA:
10696 12136 : case BUILT_IN_GOACC_UPDATE:
10697 12136 : {
10698 12136 : tree t_async = NULL_TREE;
10699 :
10700 : /* If present, use the value specified by the respective
10701 : clause, making sure that is of the correct type. */
10702 12136 : c = omp_find_clause (clauses, OMP_CLAUSE_ASYNC);
10703 12136 : if (c)
10704 1681 : t_async = fold_convert_loc (OMP_CLAUSE_LOCATION (c),
10705 : integer_type_node,
10706 1681 : OMP_CLAUSE_ASYNC_EXPR (c));
10707 10455 : else if (!tagging)
10708 : /* Default values for t_async. */
10709 2431 : t_async = fold_convert_loc (gimple_location (entry_stmt),
10710 : integer_type_node,
10711 : build_int_cst (integer_type_node,
10712 : GOMP_ASYNC_SYNC));
10713 12136 : if (tagging && t_async)
10714 : {
10715 1371 : unsigned HOST_WIDE_INT i_async = GOMP_LAUNCH_OP_MAX;
10716 :
10717 1371 : if (TREE_CODE (t_async) == INTEGER_CST)
10718 : {
10719 : /* See if we can pack the async arg in to the tag's
10720 : operand. */
10721 1331 : i_async = TREE_INT_CST_LOW (t_async);
10722 1331 : if (i_async < GOMP_LAUNCH_OP_MAX)
10723 : t_async = NULL_TREE;
10724 : else
10725 825 : i_async = GOMP_LAUNCH_OP_MAX;
10726 : }
10727 1371 : args.safe_push (oacc_launch_pack (GOMP_LAUNCH_ASYNC, NULL_TREE,
10728 : i_async));
10729 : }
10730 12136 : if (t_async)
10731 3566 : args.safe_push (force_gimple_operand_gsi (&gsi, t_async, true,
10732 : NULL_TREE, true,
10733 : GSI_SAME_STMT));
10734 :
10735 : /* Save the argument index, and ... */
10736 12136 : unsigned t_wait_idx = args.length ();
10737 12136 : unsigned num_waits = 0;
10738 12136 : c = omp_find_clause (clauses, OMP_CLAUSE_WAIT);
10739 12136 : if (!tagging || c)
10740 : /* ... push a placeholder. */
10741 2944 : args.safe_push (integer_zero_node);
10742 :
10743 13986 : for (; c; c = OMP_CLAUSE_CHAIN (c))
10744 1850 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_WAIT)
10745 : {
10746 422 : tree arg = fold_convert_loc (OMP_CLAUSE_LOCATION (c),
10747 : integer_type_node,
10748 422 : OMP_CLAUSE_WAIT_EXPR (c));
10749 422 : arg = force_gimple_operand_gsi (&gsi, arg, true, NULL_TREE, true,
10750 : GSI_SAME_STMT);
10751 422 : args.safe_push (arg);
10752 422 : num_waits++;
10753 : }
10754 :
10755 12136 : if (!tagging || num_waits)
10756 : {
10757 203 : tree len;
10758 :
10759 : /* Now that we know the number, update the placeholder. */
10760 203 : if (tagging)
10761 203 : len = oacc_launch_pack (GOMP_LAUNCH_WAIT, NULL_TREE, num_waits);
10762 : else
10763 2741 : len = build_int_cst (integer_type_node, num_waits);
10764 2944 : len = fold_convert_loc (gimple_location (entry_stmt),
10765 : unsigned_type_node, len);
10766 2944 : args[t_wait_idx] = len;
10767 : }
10768 : }
10769 12136 : break;
10770 0 : default:
10771 0 : gcc_unreachable ();
10772 : }
10773 23885 : if (tagging)
10774 : /* Push terminal marker - zero. */
10775 9395 : args.safe_push (oacc_launch_pack (0, NULL_TREE, 0));
10776 :
10777 37102 : if (child_fn2)
10778 : {
10779 : g = gimple_build_call_internal (IFN_GOMP_TARGET_REV, 1,
10780 : build_fold_addr_expr (child_fn));
10781 : gimple_set_location (g, gimple_location (entry_stmt));
10782 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
10783 : }
10784 :
10785 37102 : g = gimple_build_call_vec (builtin_decl_explicit (start_ix), args);
10786 37102 : gimple_set_location (g, gimple_location (entry_stmt));
10787 37102 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
10788 37102 : if (!offloaded)
10789 : {
10790 15958 : g = gsi_stmt (gsi);
10791 15958 : gcc_assert (g && gimple_code (g) == GIMPLE_OMP_TARGET);
10792 15958 : gsi_remove (&gsi, true);
10793 : }
10794 37102 : }
10795 :
10796 : /* Expand the parallel region tree rooted at REGION. Expansion
10797 : proceeds in depth-first order. Innermost regions are expanded
10798 : first. This way, parallel regions that require a new function to
10799 : be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
10800 : internal dependencies in their body. */
10801 :
10802 : static void
10803 81859 : expand_omp (struct omp_region *region)
10804 : {
10805 81859 : omp_any_child_fn_dumped = false;
10806 210305 : while (region)
10807 : {
10808 128446 : location_t saved_location;
10809 128446 : gimple *inner_stmt = NULL;
10810 :
10811 : /* First, determine whether this is a combined parallel+workshare
10812 : region. */
10813 128446 : if (region->type == GIMPLE_OMP_PARALLEL)
10814 16280 : determine_parallel_type (region);
10815 :
10816 128446 : if (region->type == GIMPLE_OMP_FOR
10817 128446 : && gimple_omp_for_combined_p (last_nondebug_stmt (region->entry)))
10818 14598 : inner_stmt = last_nondebug_stmt (region->inner->entry);
10819 :
10820 128446 : if (region->inner)
10821 57253 : expand_omp (region->inner);
10822 :
10823 128446 : saved_location = input_location;
10824 128446 : if (gimple_has_location (last_nondebug_stmt (region->entry)))
10825 128262 : input_location = gimple_location (last_nondebug_stmt (region->entry));
10826 :
10827 128446 : switch (region->type)
10828 : {
10829 20112 : case GIMPLE_OMP_PARALLEL:
10830 20112 : case GIMPLE_OMP_TASK:
10831 20112 : expand_omp_taskreg (region);
10832 20112 : break;
10833 :
10834 47587 : case GIMPLE_OMP_FOR:
10835 47587 : expand_omp_for (region, inner_stmt);
10836 47587 : break;
10837 :
10838 386 : case GIMPLE_OMP_SECTIONS:
10839 386 : expand_omp_sections (region);
10840 386 : break;
10841 :
10842 : case GIMPLE_OMP_SECTION:
10843 : /* Individual omp sections are handled together with their
10844 : parent GIMPLE_OMP_SECTIONS region. */
10845 : break;
10846 :
10847 0 : case GIMPLE_OMP_STRUCTURED_BLOCK:
10848 : /* We should have gotten rid of these in gimple lowering. */
10849 0 : gcc_unreachable ();
10850 :
10851 1272 : case GIMPLE_OMP_SINGLE:
10852 1272 : case GIMPLE_OMP_SCOPE:
10853 1272 : expand_omp_single (region);
10854 1272 : break;
10855 :
10856 1120 : case GIMPLE_OMP_ORDERED:
10857 1120 : {
10858 1120 : gomp_ordered *ord_stmt
10859 1120 : = as_a <gomp_ordered *> (last_nondebug_stmt (region->entry));
10860 1120 : if (gimple_omp_ordered_standalone_p (ord_stmt))
10861 : {
10862 : /* We'll expand these when expanding corresponding
10863 : worksharing region with ordered(n) clause. */
10864 709 : gcc_assert (region->outer
10865 : && region->outer->type == GIMPLE_OMP_FOR);
10866 709 : region->ord_stmt = ord_stmt;
10867 709 : break;
10868 : }
10869 : }
10870 : /* FALLTHRU */
10871 10814 : case GIMPLE_OMP_MASTER:
10872 10814 : case GIMPLE_OMP_MASKED:
10873 10814 : case GIMPLE_OMP_TASKGROUP:
10874 10814 : case GIMPLE_OMP_CRITICAL:
10875 10814 : case GIMPLE_OMP_TEAMS:
10876 10814 : expand_omp_synch (region);
10877 10814 : break;
10878 :
10879 9601 : case GIMPLE_OMP_ATOMIC_LOAD:
10880 9601 : expand_omp_atomic (region);
10881 9601 : break;
10882 :
10883 37102 : case GIMPLE_OMP_TARGET:
10884 37102 : expand_omp_target (region);
10885 37102 : break;
10886 :
10887 0 : default:
10888 0 : gcc_unreachable ();
10889 : }
10890 :
10891 128446 : input_location = saved_location;
10892 128446 : region = region->next;
10893 : }
10894 81859 : if (omp_any_child_fn_dumped)
10895 : {
10896 85 : if (dump_file)
10897 85 : dump_function_header (dump_file, current_function_decl, dump_flags);
10898 85 : omp_any_child_fn_dumped = false;
10899 : }
10900 81859 : }
10901 :
10902 : /* Helper for build_omp_regions. Scan the dominator tree starting at
10903 : block BB. PARENT is the region that contains BB. If SINGLE_TREE is
10904 : true, the function ends once a single tree is built (otherwise, whole
10905 : forest of OMP constructs may be built). */
10906 :
10907 : static void
10908 1399017 : build_omp_regions_1 (basic_block bb, struct omp_region *parent,
10909 : bool single_tree)
10910 : {
10911 1399017 : gimple_stmt_iterator gsi;
10912 1399017 : gimple *stmt;
10913 1399017 : basic_block son;
10914 :
10915 1399017 : gsi = gsi_last_nondebug_bb (bb);
10916 1399017 : if (!gsi_end_p (gsi) && is_gimple_omp (gsi_stmt (gsi)))
10917 : {
10918 289135 : struct omp_region *region;
10919 289135 : enum gimple_code code;
10920 :
10921 289135 : stmt = gsi_stmt (gsi);
10922 289135 : code = gimple_code (stmt);
10923 289135 : if (code == GIMPLE_OMP_RETURN)
10924 : {
10925 : /* STMT is the return point out of region PARENT. Mark it
10926 : as the exit point and make PARENT the immediately
10927 : enclosing region. */
10928 101303 : gcc_assert (parent);
10929 101303 : region = parent;
10930 101303 : region->exit = bb;
10931 101303 : parent = parent->outer;
10932 : }
10933 187832 : else if (code == GIMPLE_OMP_ATOMIC_STORE)
10934 : {
10935 : /* GIMPLE_OMP_ATOMIC_STORE is analogous to
10936 : GIMPLE_OMP_RETURN, but matches with
10937 : GIMPLE_OMP_ATOMIC_LOAD. */
10938 9601 : gcc_assert (parent);
10939 9601 : gcc_assert (parent->type == GIMPLE_OMP_ATOMIC_LOAD);
10940 9601 : region = parent;
10941 9601 : region->exit = bb;
10942 9601 : parent = parent->outer;
10943 : }
10944 178231 : else if (code == GIMPLE_OMP_CONTINUE)
10945 : {
10946 49399 : gcc_assert (parent);
10947 49399 : parent->cont = bb;
10948 : }
10949 128832 : else if (code == GIMPLE_OMP_SECTIONS_SWITCH)
10950 : {
10951 : /* GIMPLE_OMP_SECTIONS_SWITCH is part of
10952 : GIMPLE_OMP_SECTIONS, and we do nothing for it. */
10953 : }
10954 : else
10955 : {
10956 128446 : region = new_omp_region (bb, code, parent);
10957 : /* Otherwise... */
10958 128446 : if (code == GIMPLE_OMP_TARGET)
10959 : {
10960 37102 : switch (gimple_omp_target_kind (stmt))
10961 : {
10962 : case GF_OMP_TARGET_KIND_REGION:
10963 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
10964 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
10965 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
10966 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
10967 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
10968 : break;
10969 : case GF_OMP_TARGET_KIND_UPDATE:
10970 : case GF_OMP_TARGET_KIND_ENTER_DATA:
10971 : case GF_OMP_TARGET_KIND_EXIT_DATA:
10972 : case GF_OMP_TARGET_KIND_DATA:
10973 : case GF_OMP_TARGET_KIND_OACC_DATA:
10974 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
10975 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
10976 : case GF_OMP_TARGET_KIND_OACC_UPDATE:
10977 : case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
10978 : case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
10979 : case GF_OMP_TARGET_KIND_OACC_DECLARE:
10980 : /* ..., other than for those stand-alone directives...
10981 : To be precise, target data isn't stand-alone, but
10982 : gimplifier put the end API call into try finally block
10983 : for it, so omp expansion can treat it as such. */
10984 : region = NULL;
10985 : break;
10986 0 : default:
10987 0 : gcc_unreachable ();
10988 : }
10989 : }
10990 91344 : else if (code == GIMPLE_OMP_ORDERED
10991 91344 : && gimple_omp_ordered_standalone_p (stmt))
10992 : /* #pragma omp ordered depend is also just a stand-alone
10993 : directive. */
10994 : region = NULL;
10995 90635 : else if (code == GIMPLE_OMP_TASK
10996 90635 : && gimple_omp_task_taskwait_p (stmt))
10997 : /* #pragma omp taskwait depend(...) is a stand-alone directive. */
10998 : region = NULL;
10999 90551 : else if (code == GIMPLE_OMP_TASKGROUP)
11000 : /* #pragma omp taskgroup isn't a stand-alone directive, but
11001 : gimplifier put the end API call into try finally block
11002 : for it, so omp expansion can treat it as such. */
11003 : region = NULL;
11004 : /* ..., this directive becomes the parent for a new region. */
11005 : if (region)
11006 : parent = region;
11007 : }
11008 : }
11009 :
11010 1399017 : if (single_tree && !parent)
11011 0 : return;
11012 :
11013 1399017 : for (son = first_dom_son (CDI_DOMINATORS, bb);
11014 2721323 : son;
11015 1322306 : son = next_dom_son (CDI_DOMINATORS, son))
11016 1322306 : build_omp_regions_1 (son, parent, single_tree);
11017 : }
11018 :
11019 : /* Builds the tree of OMP regions rooted at ROOT, storing it to
11020 : root_omp_region. */
11021 :
11022 : static void
11023 0 : build_omp_regions_root (basic_block root)
11024 : {
11025 0 : gcc_assert (root_omp_region == NULL);
11026 0 : build_omp_regions_1 (root, NULL, true);
11027 0 : gcc_assert (root_omp_region != NULL);
11028 0 : }
11029 :
11030 : /* Expands omp construct (and its subconstructs) starting in HEAD. */
11031 :
11032 : void
11033 0 : omp_expand_local (basic_block head)
11034 : {
11035 0 : build_omp_regions_root (head);
11036 0 : if (dump_file && (dump_flags & TDF_DETAILS))
11037 : {
11038 0 : fprintf (dump_file, "\nOMP region tree\n\n");
11039 0 : dump_omp_region (dump_file, root_omp_region, 0);
11040 0 : fprintf (dump_file, "\n");
11041 : }
11042 :
11043 0 : remove_exit_barriers (root_omp_region);
11044 0 : expand_omp (root_omp_region);
11045 :
11046 0 : omp_free_regions ();
11047 0 : }
11048 :
11049 : /* Scan the CFG and build a tree of OMP regions. Return the root of
11050 : the OMP region tree. */
11051 :
11052 : static void
11053 76711 : build_omp_regions (void)
11054 : {
11055 76711 : gcc_assert (root_omp_region == NULL);
11056 76711 : calculate_dominance_info (CDI_DOMINATORS);
11057 76711 : build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, false);
11058 76711 : }
11059 :
11060 : /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
11061 :
11062 : static unsigned int
11063 76711 : execute_expand_omp (void)
11064 : {
11065 76711 : build_omp_regions ();
11066 :
11067 76711 : if (!root_omp_region)
11068 : return 0;
11069 :
11070 24606 : if (dump_file)
11071 : {
11072 256 : fprintf (dump_file, "\nOMP region tree\n\n");
11073 256 : dump_omp_region (dump_file, root_omp_region, 0);
11074 256 : fprintf (dump_file, "\n");
11075 : }
11076 :
11077 24606 : remove_exit_barriers (root_omp_region);
11078 :
11079 24606 : expand_omp (root_omp_region);
11080 :
11081 24606 : omp_free_regions ();
11082 :
11083 24606 : return (TODO_cleanup_cfg
11084 48663 : | (gimple_in_ssa_p (cfun) ? TODO_update_ssa_only_virtuals : 0));
11085 : }
11086 :
11087 : /* OMP expansion -- the default pass, run before creation of SSA form. */
11088 :
11089 : namespace {
11090 :
11091 : const pass_data pass_data_expand_omp =
11092 : {
11093 : GIMPLE_PASS, /* type */
11094 : "ompexp", /* name */
11095 : OPTGROUP_OMP, /* optinfo_flags */
11096 : TV_NONE, /* tv_id */
11097 : PROP_gimple_any, /* properties_required */
11098 : PROP_gimple_eomp, /* properties_provided */
11099 : 0, /* properties_destroyed */
11100 : 0, /* todo_flags_start */
11101 : 0, /* todo_flags_finish */
11102 : };
11103 :
11104 : class pass_expand_omp : public gimple_opt_pass
11105 : {
11106 : public:
11107 292371 : pass_expand_omp (gcc::context *ctxt)
11108 584742 : : gimple_opt_pass (pass_data_expand_omp, ctxt)
11109 : {}
11110 :
11111 : /* opt_pass methods: */
11112 3008124 : unsigned int execute (function *) final override
11113 : {
11114 3000965 : bool gate = ((flag_openacc != 0 || flag_openmp != 0
11115 2953565 : || flag_openmp_simd != 0)
11116 3057781 : && !seen_error ());
11117 :
11118 : /* This pass always runs, to provide PROP_gimple_eomp.
11119 : But often, there is nothing to do. */
11120 3008124 : if (!gate)
11121 2955371 : return 0;
11122 :
11123 52753 : return execute_expand_omp ();
11124 : }
11125 :
11126 : }; // class pass_expand_omp
11127 :
11128 : } // anon namespace
11129 :
11130 : gimple_opt_pass *
11131 292371 : make_pass_expand_omp (gcc::context *ctxt)
11132 : {
11133 292371 : return new pass_expand_omp (ctxt);
11134 : }
11135 :
11136 : namespace {
11137 :
11138 : const pass_data pass_data_expand_omp_ssa =
11139 : {
11140 : GIMPLE_PASS, /* type */
11141 : "ompexpssa", /* name */
11142 : OPTGROUP_OMP, /* optinfo_flags */
11143 : TV_NONE, /* tv_id */
11144 : PROP_cfg | PROP_ssa, /* properties_required */
11145 : PROP_gimple_eomp, /* properties_provided */
11146 : 0, /* properties_destroyed */
11147 : 0, /* todo_flags_start */
11148 : TODO_cleanup_cfg | TODO_rebuild_alias, /* todo_flags_finish */
11149 : };
11150 :
11151 : class pass_expand_omp_ssa : public gimple_opt_pass
11152 : {
11153 : public:
11154 584742 : pass_expand_omp_ssa (gcc::context *ctxt)
11155 1169484 : : gimple_opt_pass (pass_data_expand_omp_ssa, ctxt)
11156 : {}
11157 :
11158 : /* opt_pass methods: */
11159 245008 : bool gate (function *fun) final override
11160 : {
11161 245008 : return !(fun->curr_properties & PROP_gimple_eomp);
11162 : }
11163 23958 : unsigned int execute (function *) final override
11164 : {
11165 23958 : return execute_expand_omp ();
11166 : }
11167 292371 : opt_pass * clone () final override
11168 : {
11169 292371 : return new pass_expand_omp_ssa (m_ctxt);
11170 : }
11171 :
11172 : }; // class pass_expand_omp_ssa
11173 :
11174 : } // anon namespace
11175 :
11176 : gimple_opt_pass *
11177 292371 : make_pass_expand_omp_ssa (gcc::context *ctxt)
11178 : {
11179 292371 : return new pass_expand_omp_ssa (ctxt);
11180 : }
11181 :
11182 : /* Called from tree-cfg.cc::make_edges to create cfg edges for all relevant
11183 : GIMPLE_* codes. */
11184 :
11185 : bool
11186 290205 : omp_make_gimple_edges (basic_block bb, struct omp_region **region,
11187 : int *region_idx)
11188 : {
11189 290205 : gimple *last = last_nondebug_stmt (bb);
11190 290205 : enum gimple_code code = gimple_code (last);
11191 290205 : struct omp_region *cur_region = *region;
11192 290205 : bool fallthru = false;
11193 :
11194 290205 : switch (code)
11195 : {
11196 75153 : case GIMPLE_OMP_PARALLEL:
11197 75153 : case GIMPLE_OMP_FOR:
11198 75153 : case GIMPLE_OMP_SINGLE:
11199 75153 : case GIMPLE_OMP_TEAMS:
11200 75153 : case GIMPLE_OMP_MASTER:
11201 75153 : case GIMPLE_OMP_MASKED:
11202 75153 : case GIMPLE_OMP_SCOPE:
11203 75153 : case GIMPLE_OMP_CRITICAL:
11204 75153 : case GIMPLE_OMP_SECTION:
11205 75153 : cur_region = new_omp_region (bb, code, cur_region);
11206 75153 : fallthru = true;
11207 75153 : break;
11208 :
11209 536 : case GIMPLE_OMP_TASKGROUP:
11210 536 : cur_region = new_omp_region (bb, code, cur_region);
11211 536 : fallthru = true;
11212 536 : cur_region = cur_region->outer;
11213 536 : break;
11214 :
11215 3832 : case GIMPLE_OMP_TASK:
11216 3832 : cur_region = new_omp_region (bb, code, cur_region);
11217 3832 : fallthru = true;
11218 3832 : if (gimple_omp_task_taskwait_p (last))
11219 84 : cur_region = cur_region->outer;
11220 : break;
11221 :
11222 1124 : case GIMPLE_OMP_ORDERED:
11223 1124 : cur_region = new_omp_region (bb, code, cur_region);
11224 1124 : fallthru = true;
11225 1124 : if (gimple_omp_ordered_standalone_p (last))
11226 713 : cur_region = cur_region->outer;
11227 : break;
11228 :
11229 37147 : case GIMPLE_OMP_TARGET:
11230 37147 : cur_region = new_omp_region (bb, code, cur_region);
11231 37147 : fallthru = true;
11232 37147 : switch (gimple_omp_target_kind (last))
11233 : {
11234 : case GF_OMP_TARGET_KIND_REGION:
11235 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
11236 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
11237 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
11238 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
11239 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
11240 : break;
11241 15960 : case GF_OMP_TARGET_KIND_UPDATE:
11242 15960 : case GF_OMP_TARGET_KIND_ENTER_DATA:
11243 15960 : case GF_OMP_TARGET_KIND_EXIT_DATA:
11244 15960 : case GF_OMP_TARGET_KIND_DATA:
11245 15960 : case GF_OMP_TARGET_KIND_OACC_DATA:
11246 15960 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
11247 15960 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
11248 15960 : case GF_OMP_TARGET_KIND_OACC_UPDATE:
11249 15960 : case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
11250 15960 : case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
11251 15960 : case GF_OMP_TARGET_KIND_OACC_DECLARE:
11252 15960 : cur_region = cur_region->outer;
11253 15960 : break;
11254 0 : default:
11255 0 : gcc_unreachable ();
11256 : }
11257 : break;
11258 :
11259 386 : case GIMPLE_OMP_SECTIONS:
11260 386 : cur_region = new_omp_region (bb, code, cur_region);
11261 386 : fallthru = true;
11262 386 : break;
11263 :
11264 : case GIMPLE_OMP_SECTIONS_SWITCH:
11265 : fallthru = false;
11266 : break;
11267 :
11268 19586 : case GIMPLE_OMP_ATOMIC_LOAD:
11269 19586 : case GIMPLE_OMP_ATOMIC_STORE:
11270 19586 : fallthru = true;
11271 19586 : break;
11272 :
11273 100885 : case GIMPLE_OMP_RETURN:
11274 : /* In the case of a GIMPLE_OMP_SECTION, the edge will go
11275 : somewhere other than the next block. This will be
11276 : created later. */
11277 100885 : cur_region->exit = bb;
11278 100885 : if (cur_region->type == GIMPLE_OMP_TASK)
11279 : /* Add an edge corresponding to not scheduling the task
11280 : immediately. */
11281 3748 : make_edge (cur_region->entry, bb, EDGE_ABNORMAL);
11282 100885 : fallthru = cur_region->type != GIMPLE_OMP_SECTION;
11283 100885 : cur_region = cur_region->outer;
11284 100885 : break;
11285 :
11286 51170 : case GIMPLE_OMP_CONTINUE:
11287 51170 : cur_region->cont = bb;
11288 51170 : switch (cur_region->type)
11289 : {
11290 47036 : case GIMPLE_OMP_FOR:
11291 : /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
11292 : succs edges as abnormal to prevent splitting
11293 : them. */
11294 47036 : single_succ_edge (cur_region->entry)->flags |= EDGE_ABNORMAL;
11295 : /* Make the loopback edge. */
11296 47036 : make_edge (bb, single_succ (cur_region->entry),
11297 : EDGE_ABNORMAL);
11298 :
11299 : /* Create an edge from GIMPLE_OMP_FOR to exit, which
11300 : corresponds to the case that the body of the loop
11301 : is not executed at all. */
11302 47036 : make_edge (cur_region->entry, bb->next_bb, EDGE_ABNORMAL);
11303 47036 : make_edge (bb, bb->next_bb, EDGE_FALLTHRU | EDGE_ABNORMAL);
11304 47036 : fallthru = false;
11305 47036 : break;
11306 :
11307 386 : case GIMPLE_OMP_SECTIONS:
11308 : /* Wire up the edges into and out of the nested sections. */
11309 386 : {
11310 386 : basic_block switch_bb = single_succ (cur_region->entry);
11311 :
11312 386 : struct omp_region *i;
11313 1249 : for (i = cur_region->inner; i ; i = i->next)
11314 : {
11315 863 : gcc_assert (i->type == GIMPLE_OMP_SECTION);
11316 863 : make_edge (switch_bb, i->entry, 0);
11317 863 : make_edge (i->exit, bb, EDGE_FALLTHRU);
11318 : }
11319 :
11320 : /* Make the loopback edge to the block with
11321 : GIMPLE_OMP_SECTIONS_SWITCH. */
11322 386 : make_edge (bb, switch_bb, 0);
11323 :
11324 : /* Make the edge from the switch to exit. */
11325 386 : make_edge (switch_bb, bb->next_bb, 0);
11326 386 : fallthru = false;
11327 : }
11328 386 : break;
11329 :
11330 : case GIMPLE_OMP_TASK:
11331 : fallthru = true;
11332 : break;
11333 :
11334 0 : default:
11335 0 : gcc_unreachable ();
11336 : }
11337 : break;
11338 :
11339 0 : default:
11340 0 : gcc_unreachable ();
11341 : }
11342 :
11343 290205 : if (*region != cur_region)
11344 : {
11345 201770 : *region = cur_region;
11346 201770 : if (cur_region)
11347 160535 : *region_idx = cur_region->entry->index;
11348 : else
11349 41235 : *region_idx = 0;
11350 : }
11351 :
11352 290205 : return fallthru;
11353 : }
|
