Line data Source code
1 : /* Expansion pass for OMP directives. Outlines regions of certain OMP
2 : directives to separate functions, converts others into explicit calls to the
3 : runtime library (libgomp) and so forth
4 :
5 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
6 :
7 : This file is part of GCC.
8 :
9 : GCC is free software; you can redistribute it and/or modify it under
10 : the terms of the GNU General Public License as published by the Free
11 : Software Foundation; either version 3, or (at your option) any later
12 : version.
13 :
14 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 : for more details.
18 :
19 : You should have received a copy of the GNU General Public License
20 : along with GCC; see the file COPYING3. If not see
21 : <http://www.gnu.org/licenses/>. */
22 :
23 : #include "config.h"
24 : #include "system.h"
25 : #include "coretypes.h"
26 : #include "memmodel.h"
27 : #include "backend.h"
28 : #include "target.h"
29 : #include "rtl.h"
30 : #include "tree.h"
31 : #include "gimple.h"
32 : #include "cfghooks.h"
33 : #include "tree-pass.h"
34 : #include "ssa.h"
35 : #include "optabs.h"
36 : #include "cgraph.h"
37 : #include "pretty-print.h"
38 : #include "diagnostic-core.h"
39 : #include "fold-const.h"
40 : #include "stor-layout.h"
41 : #include "cfganal.h"
42 : #include "internal-fn.h"
43 : #include "gimplify.h"
44 : #include "gimple-iterator.h"
45 : #include "gimplify-me.h"
46 : #include "gimple-walk.h"
47 : #include "tree-cfg.h"
48 : #include "tree-into-ssa.h"
49 : #include "tree-ssa.h"
50 : #include "splay-tree.h"
51 : #include "cfgloop.h"
52 : #include "omp-general.h"
53 : #include "omp-offload.h"
54 : #include "tree-cfgcleanup.h"
55 : #include "alloc-pool.h"
56 : #include "symbol-summary.h"
57 : #include "gomp-constants.h"
58 : #include "gimple-pretty-print.h"
59 : #include "stringpool.h"
60 : #include "attribs.h"
61 : #include "tree-eh.h"
62 : #include "opts.h"
63 :
64 : /* OMP region information. Every parallel and workshare
65 : directive is enclosed between two markers, the OMP_* directive
66 : and a corresponding GIMPLE_OMP_RETURN statement. */
67 :
68 : struct omp_region
69 : {
70 : /* The enclosing region. */
71 : struct omp_region *outer;
72 :
73 : /* First child region. */
74 : struct omp_region *inner;
75 :
76 : /* Next peer region. */
77 : struct omp_region *next;
78 :
79 : /* Block containing the omp directive as its last stmt. */
80 : basic_block entry;
81 :
82 : /* Block containing the GIMPLE_OMP_RETURN as its last stmt. */
83 : basic_block exit;
84 :
85 : /* Block containing the GIMPLE_OMP_CONTINUE as its last stmt. */
86 : basic_block cont;
87 :
88 : /* If this is a combined parallel+workshare region, this is a list
89 : of additional arguments needed by the combined parallel+workshare
90 : library call. */
91 : vec<tree, va_gc> *ws_args;
92 :
93 : /* The code for the omp directive of this region. */
94 : enum gimple_code type;
95 :
96 : /* Schedule kind, only used for GIMPLE_OMP_FOR type regions. */
97 : enum omp_clause_schedule_kind sched_kind;
98 :
99 : /* Schedule modifiers. */
100 : unsigned char sched_modifiers;
101 :
102 : /* True if this is a combined parallel+workshare region. */
103 : bool is_combined_parallel;
104 :
105 : /* Copy of fd.lastprivate_conditional != 0. */
106 : bool has_lastprivate_conditional;
107 :
108 : /* The ordered stmt if type is GIMPLE_OMP_ORDERED and it has
109 : a depend clause. */
110 : gomp_ordered *ord_stmt;
111 : };
112 :
113 : static struct omp_region *root_omp_region;
114 : static bool omp_any_child_fn_dumped;
115 :
116 : static void expand_omp_build_assign (gimple_stmt_iterator *, tree, tree,
117 : bool = false);
118 : static gphi *find_phi_with_arg_on_edge (tree, edge);
119 : static void expand_omp (struct omp_region *region);
120 :
121 : /* Return true if REGION is a combined parallel+workshare region. */
122 :
123 : static inline bool
124 43626 : is_combined_parallel (struct omp_region *region)
125 : {
126 43626 : return region->is_combined_parallel;
127 : }
128 :
129 : /* Return true is REGION is or is contained within an offload region. */
130 :
131 : static bool
132 11189 : is_in_offload_region (struct omp_region *region)
133 : {
134 30060 : gimple *entry_stmt = last_nondebug_stmt (region->entry);
135 30060 : if (is_gimple_omp (entry_stmt)
136 29104 : && is_gimple_omp_offloaded (entry_stmt))
137 : return true;
138 25750 : else if (region->outer)
139 : return is_in_offload_region (region->outer);
140 : else
141 6879 : return (lookup_attribute ("omp declare target",
142 6879 : DECL_ATTRIBUTES (current_function_decl))
143 6879 : != NULL);
144 : }
145 :
146 : /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
147 : is the immediate dominator of PAR_ENTRY_BB, return true if there
148 : are no data dependencies that would prevent expanding the parallel
149 : directive at PAR_ENTRY_BB as a combined parallel+workshare region.
150 :
151 : When expanding a combined parallel+workshare region, the call to
152 : the child function may need additional arguments in the case of
153 : GIMPLE_OMP_FOR regions. In some cases, these arguments are
154 : computed out of variables passed in from the parent to the child
155 : via 'struct .omp_data_s'. For instance:
156 :
157 : #pragma omp parallel for schedule (guided, i * 4)
158 : for (j ...)
159 :
160 : Is lowered into:
161 :
162 : # BLOCK 2 (PAR_ENTRY_BB)
163 : .omp_data_o.i = i;
164 : #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
165 :
166 : # BLOCK 3 (WS_ENTRY_BB)
167 : .omp_data_i = &.omp_data_o;
168 : D.1667 = .omp_data_i->i;
169 : D.1598 = D.1667 * 4;
170 : #pragma omp for schedule (guided, D.1598)
171 :
172 : When we outline the parallel region, the call to the child function
173 : 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
174 : that value is computed *after* the call site. So, in principle we
175 : cannot do the transformation.
176 :
177 : To see whether the code in WS_ENTRY_BB blocks the combined
178 : parallel+workshare call, we collect all the variables used in the
179 : GIMPLE_OMP_FOR header check whether they appear on the LHS of any
180 : statement in WS_ENTRY_BB. If so, then we cannot emit the combined
181 : call.
182 :
183 : FIXME. If we had the SSA form built at this point, we could merely
184 : hoist the code in block 3 into block 2 and be done with it. But at
185 : this point we don't have dataflow information and though we could
186 : hack something up here, it is really not worth the aggravation. */
187 :
188 : static bool
189 9110 : workshare_safe_to_combine_p (basic_block ws_entry_bb)
190 : {
191 9110 : struct omp_for_data fd;
192 9110 : gimple *ws_stmt = last_nondebug_stmt (ws_entry_bb);
193 :
194 9110 : if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
195 : return true;
196 :
197 8987 : gcc_assert (gimple_code (ws_stmt) == GIMPLE_OMP_FOR);
198 8987 : if (gimple_omp_for_kind (ws_stmt) != GF_OMP_FOR_KIND_FOR)
199 : return false;
200 :
201 8970 : omp_extract_for_data (as_a <gomp_for *> (ws_stmt), &fd, NULL);
202 :
203 8970 : if (fd.collapse > 1 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
204 : return false;
205 7191 : if (fd.iter_type != long_integer_type_node)
206 : return false;
207 :
208 : /* FIXME. We give up too easily here. If any of these arguments
209 : are not constants, they will likely involve variables that have
210 : been mapped into fields of .omp_data_s for sharing with the child
211 : function. With appropriate data flow, it would be possible to
212 : see through this. */
213 1508 : if (!is_gimple_min_invariant (fd.loop.n1)
214 1278 : || !is_gimple_min_invariant (fd.loop.n2)
215 1149 : || !is_gimple_min_invariant (fd.loop.step)
216 2642 : || (fd.chunk_size && !is_gimple_min_invariant (fd.chunk_size)))
217 376 : return false;
218 :
219 : return true;
220 : }
221 :
222 : /* Adjust CHUNK_SIZE from SCHEDULE clause, depending on simd modifier
223 : presence (SIMD_SCHEDULE). */
224 :
225 : static tree
226 8362 : omp_adjust_chunk_size (tree chunk_size, bool simd_schedule, bool offload)
227 : {
228 8362 : if (!simd_schedule || integer_zerop (chunk_size))
229 : return chunk_size;
230 :
231 34 : tree vf;
232 34 : tree type = TREE_TYPE (chunk_size);
233 :
234 34 : if (offload)
235 : {
236 2 : cfun->curr_properties &= ~PROP_gimple_lomp_dev;
237 2 : vf = build_call_expr_internal_loc (UNKNOWN_LOCATION, IFN_GOMP_MAX_VF,
238 : unsigned_type_node, 0);
239 2 : vf = fold_convert (type, vf);
240 : }
241 : else
242 : {
243 32 : poly_uint64 vf_num = omp_max_vf (false);
244 32 : if (known_eq (vf_num, 1U))
245 8362 : return chunk_size;
246 31 : vf = build_int_cst (type, vf_num);
247 : }
248 :
249 33 : tree vf_minus_one = fold_build2 (MINUS_EXPR, type, vf,
250 : build_int_cst (type, 1));
251 33 : tree negative_vf = fold_build1 (NEGATE_EXPR, type, vf);
252 33 : chunk_size = fold_build2 (PLUS_EXPR, type, chunk_size, vf_minus_one);
253 33 : return fold_build2 (BIT_AND_EXPR, type, chunk_size, negative_vf);
254 : }
255 :
256 : /* Collect additional arguments needed to emit a combined
257 : parallel+workshare call. WS_STMT is the workshare directive being
258 : expanded. */
259 :
260 : static vec<tree, va_gc> *
261 1200 : get_ws_args_for (gimple *par_stmt, gimple *ws_stmt, bool offload)
262 : {
263 1200 : tree t;
264 1200 : location_t loc = gimple_location (ws_stmt);
265 1200 : vec<tree, va_gc> *ws_args;
266 :
267 1200 : if (gomp_for *for_stmt = dyn_cast <gomp_for *> (ws_stmt))
268 : {
269 1084 : struct omp_for_data fd;
270 1084 : tree n1, n2;
271 :
272 1084 : omp_extract_for_data (for_stmt, &fd, NULL);
273 1084 : n1 = fd.loop.n1;
274 1084 : n2 = fd.loop.n2;
275 :
276 1084 : if (gimple_omp_for_combined_into_p (for_stmt))
277 : {
278 769 : tree innerc
279 769 : = omp_find_clause (gimple_omp_parallel_clauses (par_stmt),
280 : OMP_CLAUSE__LOOPTEMP_);
281 769 : gcc_assert (innerc);
282 769 : n1 = OMP_CLAUSE_DECL (innerc);
283 769 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
284 : OMP_CLAUSE__LOOPTEMP_);
285 769 : gcc_assert (innerc);
286 769 : n2 = OMP_CLAUSE_DECL (innerc);
287 : }
288 :
289 1084 : vec_alloc (ws_args, 3 + (fd.chunk_size != 0));
290 :
291 1084 : t = fold_convert_loc (loc, long_integer_type_node, n1);
292 1084 : ws_args->quick_push (t);
293 :
294 1084 : t = fold_convert_loc (loc, long_integer_type_node, n2);
295 1084 : ws_args->quick_push (t);
296 :
297 1084 : t = fold_convert_loc (loc, long_integer_type_node, fd.loop.step);
298 1084 : ws_args->quick_push (t);
299 :
300 1084 : if (fd.chunk_size)
301 : {
302 538 : t = fold_convert_loc (loc, long_integer_type_node, fd.chunk_size);
303 538 : t = omp_adjust_chunk_size (t, fd.simd_schedule, offload);
304 538 : ws_args->quick_push (t);
305 : }
306 :
307 1084 : return ws_args;
308 : }
309 116 : else if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
310 : {
311 : /* Number of sections is equal to the number of edges from the
312 : GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
313 : the exit of the sections region. */
314 116 : basic_block bb = single_succ (gimple_bb (ws_stmt));
315 232 : t = build_int_cst (unsigned_type_node, EDGE_COUNT (bb->succs) - 1);
316 116 : vec_alloc (ws_args, 1);
317 116 : ws_args->quick_push (t);
318 116 : return ws_args;
319 : }
320 :
321 0 : gcc_unreachable ();
322 : }
323 :
324 : /* Discover whether REGION is a combined parallel+workshare region. */
325 :
326 : static void
327 16400 : determine_parallel_type (struct omp_region *region)
328 : {
329 16400 : basic_block par_entry_bb, par_exit_bb;
330 16400 : basic_block ws_entry_bb, ws_exit_bb;
331 :
332 16400 : if (region == NULL || region->inner == NULL
333 14872 : || region->exit == NULL || region->inner->exit == NULL
334 14861 : || region->inner->cont == NULL)
335 : return;
336 :
337 : /* We only support parallel+for and parallel+sections. */
338 10387 : if (region->type != GIMPLE_OMP_PARALLEL
339 10387 : || (region->inner->type != GIMPLE_OMP_FOR
340 297 : && region->inner->type != GIMPLE_OMP_SECTIONS))
341 : return;
342 :
343 : /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
344 : WS_EXIT_BB -> PAR_EXIT_BB. */
345 10267 : par_entry_bb = region->entry;
346 10267 : par_exit_bb = region->exit;
347 10267 : ws_entry_bb = region->inner->entry;
348 10267 : ws_exit_bb = region->inner->exit;
349 :
350 : /* Give up for task reductions on the parallel, while it is implementable,
351 : adding another big set of APIs or slowing down the normal paths is
352 : not acceptable. */
353 10267 : tree pclauses
354 10267 : = gimple_omp_parallel_clauses (last_nondebug_stmt (par_entry_bb));
355 10267 : if (omp_find_clause (pclauses, OMP_CLAUSE__REDUCTEMP_))
356 : return;
357 :
358 10244 : if (single_succ (par_entry_bb) == ws_entry_bb
359 9147 : && single_succ (ws_exit_bb) == par_exit_bb
360 9110 : && workshare_safe_to_combine_p (ws_entry_bb)
361 11499 : && (gimple_omp_parallel_combined_p (last_nondebug_stmt (par_entry_bb))
362 7 : || (last_and_only_stmt (ws_entry_bb)
363 2 : && last_and_only_stmt (par_exit_bb))))
364 : {
365 1248 : gimple *par_stmt = last_nondebug_stmt (par_entry_bb);
366 1248 : gimple *ws_stmt = last_nondebug_stmt (ws_entry_bb);
367 :
368 1248 : if (region->inner->type == GIMPLE_OMP_FOR)
369 : {
370 : /* If this is a combined parallel loop, we need to determine
371 : whether or not to use the combined library calls. There
372 : are two cases where we do not apply the transformation:
373 : static loops and any kind of ordered loop. In the first
374 : case, we already open code the loop so there is no need
375 : to do anything else. In the latter case, the combined
376 : parallel loop call would still need extra synchronization
377 : to implement ordered semantics, so there would not be any
378 : gain in using the combined call. */
379 1128 : tree clauses = gimple_omp_for_clauses (ws_stmt);
380 1128 : tree c = omp_find_clause (clauses, OMP_CLAUSE_SCHEDULE);
381 1128 : if (c == NULL
382 1110 : || ((OMP_CLAUSE_SCHEDULE_KIND (c) & OMP_CLAUSE_SCHEDULE_MASK)
383 : == OMP_CLAUSE_SCHEDULE_STATIC)
384 1096 : || omp_find_clause (clauses, OMP_CLAUSE_ORDERED)
385 1084 : || omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_)
386 2212 : || ((c = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_))
387 14 : && POINTER_TYPE_P (TREE_TYPE (OMP_CLAUSE_DECL (c)))))
388 : return;
389 : }
390 120 : else if (region->inner->type == GIMPLE_OMP_SECTIONS
391 240 : && (omp_find_clause (gimple_omp_sections_clauses (ws_stmt),
392 : OMP_CLAUSE__REDUCTEMP_)
393 120 : || omp_find_clause (gimple_omp_sections_clauses (ws_stmt),
394 : OMP_CLAUSE__CONDTEMP_)))
395 : return;
396 :
397 1200 : region->is_combined_parallel = true;
398 1200 : region->inner->is_combined_parallel = true;
399 1200 : region->ws_args = get_ws_args_for (par_stmt, ws_stmt,
400 : is_in_offload_region (region));
401 : }
402 : }
403 :
404 : /* Debugging dumps for parallel regions. */
405 : void dump_omp_region (FILE *, struct omp_region *, int);
406 : void debug_omp_region (struct omp_region *);
407 : void debug_all_omp_regions (void);
408 :
409 : /* Dump the parallel region tree rooted at REGION. */
410 :
411 : void
412 402 : dump_omp_region (FILE *file, struct omp_region *region, int indent)
413 : {
414 473 : fprintf (file, "%*sbb %d: %s\n", indent, "", region->entry->index,
415 473 : gimple_code_name[region->type]);
416 :
417 473 : if (region->inner)
418 118 : dump_omp_region (file, region->inner, indent + 4);
419 :
420 473 : if (region->cont)
421 : {
422 223 : fprintf (file, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent, "",
423 : region->cont->index);
424 : }
425 :
426 473 : if (region->exit)
427 450 : fprintf (file, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent, "",
428 : region->exit->index);
429 : else
430 23 : fprintf (file, "%*s[no exit marker]\n", indent, "");
431 :
432 473 : if (region->next)
433 : dump_omp_region (file, region->next, indent);
434 402 : }
435 :
436 : DEBUG_FUNCTION void
437 0 : debug_omp_region (struct omp_region *region)
438 : {
439 0 : dump_omp_region (stderr, region, 0);
440 0 : }
441 :
442 : DEBUG_FUNCTION void
443 0 : debug_all_omp_regions (void)
444 : {
445 0 : dump_omp_region (stderr, root_omp_region, 0);
446 0 : }
447 :
448 : /* Create a new parallel region starting at STMT inside region PARENT. */
449 :
450 : static struct omp_region *
451 247721 : new_omp_region (basic_block bb, enum gimple_code type,
452 : struct omp_region *parent)
453 : {
454 247721 : struct omp_region *region = XCNEW (struct omp_region);
455 :
456 247721 : region->outer = parent;
457 247721 : region->entry = bb;
458 247721 : region->type = type;
459 :
460 247721 : if (parent)
461 : {
462 : /* This is a nested region. Add it to the list of inner
463 : regions in PARENT. */
464 127890 : region->next = parent->inner;
465 127890 : parent->inner = region;
466 : }
467 : else
468 : {
469 : /* This is a toplevel region. Add it to the list of toplevel
470 : regions in ROOT_OMP_REGION. */
471 119831 : region->next = root_omp_region;
472 119831 : root_omp_region = region;
473 : }
474 :
475 247721 : return region;
476 : }
477 :
478 : /* Release the memory associated with the region tree rooted at REGION. */
479 :
480 : static void
481 247721 : free_omp_region_1 (struct omp_region *region)
482 : {
483 247721 : struct omp_region *i, *n;
484 :
485 375611 : for (i = region->inner; i ; i = n)
486 : {
487 127890 : n = i->next;
488 127890 : free_omp_region_1 (i);
489 : }
490 :
491 247721 : free (region);
492 247721 : }
493 :
494 : /* Release the memory for the entire omp region tree. */
495 :
496 : void
497 3051790 : omp_free_regions (void)
498 : {
499 3051790 : struct omp_region *r, *n;
500 3171621 : for (r = root_omp_region; r ; r = n)
501 : {
502 119831 : n = r->next;
503 119831 : free_omp_region_1 (r);
504 : }
505 3051790 : root_omp_region = NULL;
506 3051790 : }
507 :
508 : /* A convenience function to build an empty GIMPLE_COND with just the
509 : condition. */
510 :
511 : static gcond *
512 113167 : gimple_build_cond_empty (tree cond)
513 : {
514 113167 : enum tree_code pred_code;
515 113167 : tree lhs, rhs;
516 :
517 113167 : gimple_cond_get_ops_from_tree (cond, &pred_code, &lhs, &rhs);
518 113167 : return gimple_build_cond (pred_code, lhs, rhs, NULL_TREE, NULL_TREE);
519 : }
520 :
521 : /* Change DECL_CONTEXT of CHILD_FNDECL to that of the parent function.
522 : Add CHILD_FNDECL to decl chain of the supercontext of the block
523 : ENTRY_BLOCK - this is the block which originally contained the
524 : code from which CHILD_FNDECL was created.
525 :
526 : Together, these actions ensure that the debug info for the outlined
527 : function will be emitted with the correct lexical scope. */
528 :
529 : static void
530 43969 : adjust_context_and_scope (struct omp_region *region, tree entry_block,
531 : tree child_fndecl)
532 : {
533 43969 : tree parent_fndecl = NULL_TREE;
534 43969 : gimple *entry_stmt;
535 : /* OMP expansion expands inner regions before outer ones, so if
536 : we e.g. have explicit task region nested in parallel region, when
537 : expanding the task region current_function_decl will be the original
538 : source function, but we actually want to use as context the child
539 : function of the parallel. */
540 43969 : for (region = region->outer;
541 65143 : region && parent_fndecl == NULL_TREE; region = region->outer)
542 21174 : switch (region->type)
543 : {
544 6870 : case GIMPLE_OMP_PARALLEL:
545 6870 : case GIMPLE_OMP_TASK:
546 6870 : case GIMPLE_OMP_TEAMS:
547 6870 : entry_stmt = last_nondebug_stmt (region->entry);
548 6870 : parent_fndecl = gimple_omp_taskreg_child_fn (entry_stmt);
549 6870 : break;
550 4559 : case GIMPLE_OMP_TARGET:
551 4559 : entry_stmt = last_nondebug_stmt (region->entry);
552 4559 : parent_fndecl
553 4559 : = gimple_omp_target_child_fn (as_a <gomp_target *> (entry_stmt));
554 4559 : break;
555 : default:
556 : break;
557 : }
558 :
559 43969 : if (parent_fndecl == NULL_TREE)
560 35651 : parent_fndecl = current_function_decl;
561 43969 : DECL_CONTEXT (child_fndecl) = parent_fndecl;
562 :
563 43969 : if (entry_block != NULL_TREE && TREE_CODE (entry_block) == BLOCK)
564 : {
565 43935 : tree b = BLOCK_SUPERCONTEXT (entry_block);
566 43935 : if (TREE_CODE (b) == BLOCK)
567 : {
568 43801 : DECL_CHAIN (child_fndecl) = BLOCK_VARS (b);
569 43801 : BLOCK_VARS (b) = child_fndecl;
570 : }
571 : }
572 43969 : }
573 :
574 : /* Build the function calls to GOMP_parallel etc to actually
575 : generate the parallel operation. REGION is the parallel region
576 : being expanded. BB is the block where to insert the code. WS_ARGS
577 : will be set if this is a call to a combined parallel+workshare
578 : construct, it contains the list of additional arguments needed by
579 : the workshare construct. */
580 :
581 : static void
582 16400 : expand_parallel_call (struct omp_region *region, basic_block bb,
583 : gomp_parallel *entry_stmt,
584 : vec<tree, va_gc> *ws_args)
585 : {
586 16400 : tree t, t1, t2, val, cond, c, clauses, flags;
587 16400 : gimple_stmt_iterator gsi;
588 16400 : gimple *stmt;
589 16400 : enum built_in_function start_ix;
590 16400 : int start_ix2;
591 16400 : location_t clause_loc;
592 16400 : vec<tree, va_gc> *args;
593 :
594 16400 : clauses = gimple_omp_parallel_clauses (entry_stmt);
595 :
596 : /* Determine what flavor of GOMP_parallel we will be
597 : emitting. */
598 16400 : start_ix = BUILT_IN_GOMP_PARALLEL;
599 16400 : tree rtmp = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
600 16400 : if (rtmp)
601 : start_ix = BUILT_IN_GOMP_PARALLEL_REDUCTIONS;
602 16340 : else if (is_combined_parallel (region))
603 : {
604 1200 : switch (region->inner->type)
605 : {
606 1084 : case GIMPLE_OMP_FOR:
607 1084 : gcc_assert (region->inner->sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
608 1084 : switch (region->inner->sched_kind)
609 : {
610 546 : case OMP_CLAUSE_SCHEDULE_RUNTIME:
611 : /* For lastprivate(conditional:), our implementation
612 : requires monotonic behavior. */
613 546 : if (region->inner->has_lastprivate_conditional != 0)
614 : start_ix2 = 3;
615 544 : else if ((region->inner->sched_modifiers
616 : & OMP_CLAUSE_SCHEDULE_NONMONOTONIC) != 0)
617 : start_ix2 = 6;
618 530 : else if ((region->inner->sched_modifiers
619 : & OMP_CLAUSE_SCHEDULE_MONOTONIC) == 0)
620 : start_ix2 = 7;
621 : else
622 16 : start_ix2 = 3;
623 : break;
624 538 : case OMP_CLAUSE_SCHEDULE_DYNAMIC:
625 538 : case OMP_CLAUSE_SCHEDULE_GUIDED:
626 538 : if ((region->inner->sched_modifiers
627 : & OMP_CLAUSE_SCHEDULE_MONOTONIC) == 0
628 534 : && !region->inner->has_lastprivate_conditional)
629 : {
630 522 : start_ix2 = 3 + region->inner->sched_kind;
631 522 : break;
632 : }
633 : /* FALLTHRU */
634 16 : default:
635 16 : start_ix2 = region->inner->sched_kind;
636 16 : break;
637 : }
638 1084 : start_ix2 += (int) BUILT_IN_GOMP_PARALLEL_LOOP_STATIC;
639 1084 : start_ix = (enum built_in_function) start_ix2;
640 1084 : break;
641 : case GIMPLE_OMP_SECTIONS:
642 : start_ix = BUILT_IN_GOMP_PARALLEL_SECTIONS;
643 : break;
644 0 : default:
645 0 : gcc_unreachable ();
646 : }
647 : }
648 :
649 : /* By default, the value of NUM_THREADS is zero (selected at run time)
650 : and there is no conditional. */
651 16400 : cond = NULL_TREE;
652 16400 : val = build_int_cst (unsigned_type_node, 0);
653 16400 : flags = build_int_cst (unsigned_type_node, 0);
654 :
655 16400 : c = omp_find_clause (clauses, OMP_CLAUSE_IF);
656 16400 : if (c)
657 963 : cond = OMP_CLAUSE_IF_EXPR (c);
658 :
659 16400 : c = omp_find_clause (clauses, OMP_CLAUSE_NUM_THREADS);
660 16400 : if (c)
661 : {
662 2289 : val = OMP_CLAUSE_NUM_THREADS_EXPR (c);
663 2289 : clause_loc = OMP_CLAUSE_LOCATION (c);
664 2289 : if (TREE_CODE (val) == TREE_LIST)
665 : {
666 957 : if (OMP_CLAUSE_NUM_THREADS_DIMS (c)
667 957 : && (TREE_CHAIN (val) || OMP_CLAUSE_NUM_THREADS_STRICT (c)))
668 : /* Accept 'dim(1),relaxed'. */
669 1 : sorry_at (clause_loc,
670 : "%<num_threads%> clause with %<dims%> modifier");
671 956 : else if (TREE_CHAIN (val))
672 1 : sorry_at (clause_loc,
673 : "%<num_threads%> clause with more than one argument");
674 955 : else if (OMP_CLAUSE_NUM_THREADS_STRICT (c))
675 0 : sorry_at (clause_loc,
676 : "%<num_threads%> clause with %<strict%> modifier");
677 957 : val = TREE_VALUE (val);
678 : }
679 1332 : else if (OMP_CLAUSE_NUM_THREADS_STRICT (c))
680 0 : sorry_at (clause_loc,
681 : "%<num_threads%> clause with %<strict%> modifier");
682 : /* FIXME: Handle OMP_CLAUSE_MESSAGE (only relevant with 'strict'). */
683 : }
684 : else
685 14111 : clause_loc = gimple_location (entry_stmt);
686 :
687 16400 : c = omp_find_clause (clauses, OMP_CLAUSE_PROC_BIND);
688 16400 : if (c)
689 771 : flags = build_int_cst (unsigned_type_node, OMP_CLAUSE_PROC_BIND_KIND (c));
690 :
691 : /* Ensure 'val' is of the correct type. */
692 16400 : val = fold_convert_loc (clause_loc, unsigned_type_node, val);
693 :
694 : /* If we found the clause 'if (cond)', build either
695 : (cond != 0) or (cond ? val : 1u). */
696 16400 : if (cond)
697 : {
698 963 : cond = gimple_boolify (cond);
699 :
700 963 : if (integer_zerop (val))
701 176 : val = fold_build2_loc (clause_loc,
702 : EQ_EXPR, unsigned_type_node, cond,
703 176 : build_int_cst (TREE_TYPE (cond), 0));
704 : else
705 : {
706 787 : basic_block cond_bb, then_bb, else_bb;
707 787 : edge e, e_then, e_else;
708 787 : tree tmp_then, tmp_else, tmp_join, tmp_var;
709 :
710 787 : tmp_var = create_tmp_var (TREE_TYPE (val));
711 787 : if (gimple_in_ssa_p (cfun))
712 : {
713 0 : tmp_then = make_ssa_name (tmp_var);
714 0 : tmp_else = make_ssa_name (tmp_var);
715 0 : tmp_join = make_ssa_name (tmp_var);
716 : }
717 : else
718 : {
719 : tmp_then = tmp_var;
720 : tmp_else = tmp_var;
721 : tmp_join = tmp_var;
722 : }
723 :
724 787 : e = split_block_after_labels (bb);
725 787 : cond_bb = e->src;
726 787 : bb = e->dest;
727 787 : remove_edge (e);
728 :
729 787 : then_bb = create_empty_bb (cond_bb);
730 787 : else_bb = create_empty_bb (then_bb);
731 787 : set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
732 787 : set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
733 :
734 787 : stmt = gimple_build_cond_empty (cond);
735 787 : gsi = gsi_start_bb (cond_bb);
736 787 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
737 :
738 787 : gsi = gsi_start_bb (then_bb);
739 787 : expand_omp_build_assign (&gsi, tmp_then, val, true);
740 :
741 787 : gsi = gsi_start_bb (else_bb);
742 787 : expand_omp_build_assign (&gsi, tmp_else,
743 : build_int_cst (unsigned_type_node, 1),
744 : true);
745 :
746 787 : make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
747 787 : make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
748 787 : add_bb_to_loop (then_bb, cond_bb->loop_father);
749 787 : add_bb_to_loop (else_bb, cond_bb->loop_father);
750 787 : e_then = make_edge (then_bb, bb, EDGE_FALLTHRU);
751 787 : e_else = make_edge (else_bb, bb, EDGE_FALLTHRU);
752 :
753 787 : if (gimple_in_ssa_p (cfun))
754 : {
755 0 : gphi *phi = create_phi_node (tmp_join, bb);
756 0 : add_phi_arg (phi, tmp_then, e_then, UNKNOWN_LOCATION);
757 0 : add_phi_arg (phi, tmp_else, e_else, UNKNOWN_LOCATION);
758 : }
759 :
760 787 : val = tmp_join;
761 : }
762 :
763 963 : gsi = gsi_start_bb (bb);
764 963 : val = force_gimple_operand_gsi (&gsi, val, true, NULL_TREE,
765 : false, GSI_CONTINUE_LINKING);
766 : }
767 :
768 16400 : gsi = gsi_last_nondebug_bb (bb);
769 16400 : t = gimple_omp_parallel_data_arg (entry_stmt);
770 16400 : if (t == NULL)
771 2413 : t1 = null_pointer_node;
772 : else
773 13987 : t1 = build_fold_addr_expr (t);
774 16400 : tree child_fndecl = gimple_omp_parallel_child_fn (entry_stmt);
775 16400 : t2 = build_fold_addr_expr (child_fndecl);
776 :
777 17600 : vec_alloc (args, 4 + vec_safe_length (ws_args));
778 16400 : args->quick_push (t2);
779 16400 : args->quick_push (t1);
780 16400 : args->quick_push (val);
781 16400 : if (ws_args)
782 1200 : args->splice (*ws_args);
783 16400 : args->quick_push (flags);
784 :
785 16400 : t = build_call_expr_loc_vec (UNKNOWN_LOCATION,
786 : builtin_decl_explicit (start_ix), args);
787 :
788 16400 : if (rtmp)
789 : {
790 60 : tree type = TREE_TYPE (OMP_CLAUSE_DECL (rtmp));
791 60 : t = build2 (MODIFY_EXPR, type, OMP_CLAUSE_DECL (rtmp),
792 : fold_convert (type,
793 : fold_convert (pointer_sized_int_node, t)));
794 : }
795 16400 : force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
796 : false, GSI_CONTINUE_LINKING);
797 16400 : }
798 :
799 : /* Build the function call to GOMP_task to actually
800 : generate the task operation. BB is the block where to insert the code. */
801 :
802 : static void
803 3812 : expand_task_call (struct omp_region *region, basic_block bb,
804 : gomp_task *entry_stmt)
805 : {
806 3812 : tree t1, t2, t3;
807 3812 : gimple_stmt_iterator gsi;
808 3812 : location_t loc = gimple_location (entry_stmt);
809 :
810 3812 : tree clauses = gimple_omp_task_clauses (entry_stmt);
811 :
812 3812 : tree ifc = omp_find_clause (clauses, OMP_CLAUSE_IF);
813 3812 : tree untied = omp_find_clause (clauses, OMP_CLAUSE_UNTIED);
814 3812 : tree mergeable = omp_find_clause (clauses, OMP_CLAUSE_MERGEABLE);
815 3812 : tree depend = omp_find_clause (clauses, OMP_CLAUSE_DEPEND);
816 3812 : tree finalc = omp_find_clause (clauses, OMP_CLAUSE_FINAL);
817 3812 : tree priority = omp_find_clause (clauses, OMP_CLAUSE_PRIORITY);
818 3812 : tree detach = omp_find_clause (clauses, OMP_CLAUSE_DETACH);
819 :
820 7624 : unsigned int iflags
821 3812 : = (untied ? GOMP_TASK_FLAG_UNTIED : 0)
822 3812 : | (mergeable ? GOMP_TASK_FLAG_MERGEABLE : 0)
823 3812 : | (depend ? GOMP_TASK_FLAG_DEPEND : 0);
824 :
825 3812 : bool taskloop_p = gimple_omp_task_taskloop_p (entry_stmt);
826 3812 : tree startvar = NULL_TREE, endvar = NULL_TREE, step = NULL_TREE;
827 3812 : tree num_tasks = NULL_TREE;
828 3812 : bool ull = false;
829 3812 : if (taskloop_p)
830 : {
831 1330 : gimple *g = last_nondebug_stmt (region->outer->entry);
832 1330 : gcc_assert (gimple_code (g) == GIMPLE_OMP_FOR
833 : && gimple_omp_for_kind (g) == GF_OMP_FOR_KIND_TASKLOOP);
834 1330 : struct omp_for_data fd;
835 1330 : omp_extract_for_data (as_a <gomp_for *> (g), &fd, NULL);
836 1330 : startvar = omp_find_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
837 1330 : endvar = omp_find_clause (OMP_CLAUSE_CHAIN (startvar),
838 : OMP_CLAUSE__LOOPTEMP_);
839 1330 : startvar = OMP_CLAUSE_DECL (startvar);
840 1330 : endvar = OMP_CLAUSE_DECL (endvar);
841 1330 : step = fold_convert_loc (loc, fd.iter_type, fd.loop.step);
842 1330 : if (fd.loop.cond_code == LT_EXPR)
843 1278 : iflags |= GOMP_TASK_FLAG_UP;
844 1330 : tree tclauses = gimple_omp_for_clauses (g);
845 1330 : num_tasks = omp_find_clause (tclauses, OMP_CLAUSE_NUM_TASKS);
846 1330 : if (num_tasks)
847 : {
848 176 : if (OMP_CLAUSE_NUM_TASKS_STRICT (num_tasks))
849 3 : iflags |= GOMP_TASK_FLAG_STRICT;
850 176 : num_tasks = OMP_CLAUSE_NUM_TASKS_EXPR (num_tasks);
851 : }
852 : else
853 : {
854 1154 : num_tasks = omp_find_clause (tclauses, OMP_CLAUSE_GRAINSIZE);
855 1154 : if (num_tasks)
856 : {
857 220 : iflags |= GOMP_TASK_FLAG_GRAINSIZE;
858 220 : if (OMP_CLAUSE_GRAINSIZE_STRICT (num_tasks))
859 3 : iflags |= GOMP_TASK_FLAG_STRICT;
860 220 : num_tasks = OMP_CLAUSE_GRAINSIZE_EXPR (num_tasks);
861 : }
862 : else
863 934 : num_tasks = integer_zero_node;
864 : }
865 1330 : num_tasks = fold_convert_loc (loc, long_integer_type_node, num_tasks);
866 1330 : if (ifc == NULL_TREE)
867 959 : iflags |= GOMP_TASK_FLAG_IF;
868 1330 : if (omp_find_clause (tclauses, OMP_CLAUSE_NOGROUP))
869 57 : iflags |= GOMP_TASK_FLAG_NOGROUP;
870 1330 : ull = fd.iter_type == long_long_unsigned_type_node;
871 1330 : if (omp_find_clause (clauses, OMP_CLAUSE_REDUCTION))
872 497 : iflags |= GOMP_TASK_FLAG_REDUCTION;
873 : }
874 : else
875 : {
876 2482 : if (priority)
877 18 : iflags |= GOMP_TASK_FLAG_PRIORITY;
878 2482 : if (detach)
879 166 : iflags |= GOMP_TASK_FLAG_DETACH;
880 : }
881 :
882 3812 : tree flags = build_int_cst (unsigned_type_node, iflags);
883 :
884 3812 : tree cond = boolean_true_node;
885 3812 : if (ifc)
886 : {
887 484 : if (taskloop_p)
888 : {
889 371 : tree t = gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc));
890 371 : t = fold_build3_loc (loc, COND_EXPR, unsigned_type_node, t,
891 : build_int_cst (unsigned_type_node,
892 : GOMP_TASK_FLAG_IF),
893 : build_int_cst (unsigned_type_node, 0));
894 371 : flags = fold_build2_loc (loc, PLUS_EXPR, unsigned_type_node,
895 : flags, t);
896 : }
897 : else
898 113 : cond = gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc));
899 : }
900 :
901 3812 : if (finalc)
902 : {
903 410 : tree t = gimple_boolify (OMP_CLAUSE_FINAL_EXPR (finalc));
904 410 : t = fold_build3_loc (loc, COND_EXPR, unsigned_type_node, t,
905 : build_int_cst (unsigned_type_node,
906 : GOMP_TASK_FLAG_FINAL),
907 : build_int_cst (unsigned_type_node, 0));
908 410 : flags = fold_build2_loc (loc, PLUS_EXPR, unsigned_type_node, flags, t);
909 : }
910 3812 : if (depend)
911 1105 : depend = OMP_CLAUSE_DECL (depend);
912 : else
913 2707 : depend = build_int_cst (ptr_type_node, 0);
914 3812 : if (priority)
915 372 : priority = fold_convert (integer_type_node,
916 : OMP_CLAUSE_PRIORITY_EXPR (priority));
917 : else
918 3440 : priority = integer_zero_node;
919 :
920 3812 : gsi = gsi_last_nondebug_bb (bb);
921 :
922 3646 : detach = (detach
923 3978 : ? build_fold_addr_expr (OMP_CLAUSE_DECL (detach))
924 : : null_pointer_node);
925 :
926 3812 : tree t = gimple_omp_task_data_arg (entry_stmt);
927 3812 : if (t == NULL)
928 719 : t2 = null_pointer_node;
929 : else
930 3093 : t2 = build_fold_addr_expr_loc (loc, t);
931 3812 : t1 = build_fold_addr_expr_loc (loc, gimple_omp_task_child_fn (entry_stmt));
932 3812 : t = gimple_omp_task_copy_fn (entry_stmt);
933 3812 : if (t == NULL)
934 3308 : t3 = null_pointer_node;
935 : else
936 504 : t3 = build_fold_addr_expr_loc (loc, t);
937 :
938 3812 : if (taskloop_p)
939 2660 : t = build_call_expr (ull
940 42 : ? builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP_ULL)
941 1288 : : builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP),
942 : 11, t1, t2, t3,
943 : gimple_omp_task_arg_size (entry_stmt),
944 : gimple_omp_task_arg_align (entry_stmt), flags,
945 : num_tasks, priority, startvar, endvar, step);
946 : else
947 2482 : t = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK),
948 : 10, t1, t2, t3,
949 : gimple_omp_task_arg_size (entry_stmt),
950 : gimple_omp_task_arg_align (entry_stmt), cond, flags,
951 : depend, priority, detach);
952 :
953 3812 : force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
954 : false, GSI_CONTINUE_LINKING);
955 3812 : }
956 :
957 : /* Build the function call to GOMP_taskwait_depend to actually
958 : generate the taskwait operation. BB is the block where to insert the
959 : code. */
960 :
961 : static void
962 84 : expand_taskwait_call (basic_block bb, gomp_task *entry_stmt)
963 : {
964 84 : tree clauses = gimple_omp_task_clauses (entry_stmt);
965 84 : tree depend = omp_find_clause (clauses, OMP_CLAUSE_DEPEND);
966 84 : if (depend == NULL_TREE)
967 0 : return;
968 :
969 84 : depend = OMP_CLAUSE_DECL (depend);
970 :
971 84 : bool nowait = omp_find_clause (clauses, OMP_CLAUSE_NOWAIT) != NULL_TREE;
972 84 : gimple_stmt_iterator gsi = gsi_last_nondebug_bb (bb);
973 51 : enum built_in_function f = (nowait
974 84 : ? BUILT_IN_GOMP_TASKWAIT_DEPEND_NOWAIT
975 : : BUILT_IN_GOMP_TASKWAIT_DEPEND);
976 84 : tree t = build_call_expr (builtin_decl_explicit (f), 1, depend);
977 :
978 84 : force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
979 : false, GSI_CONTINUE_LINKING);
980 : }
981 :
982 : /* Build the function call to GOMP_teams_reg to actually
983 : generate the host teams operation. REGION is the teams region
984 : being expanded. BB is the block where to insert the code. */
985 :
986 : static void
987 2572 : expand_teams_call (basic_block bb, gomp_teams *entry_stmt)
988 : {
989 2572 : tree clauses = gimple_omp_teams_clauses (entry_stmt);
990 2572 : tree num_teams = omp_find_clause (clauses, OMP_CLAUSE_NUM_TEAMS);
991 2572 : tree thread_limit = omp_find_clause (clauses, OMP_CLAUSE_THREAD_LIMIT);
992 2572 : tree message = omp_find_clause (clauses, OMP_CLAUSE_MESSAGE);
993 2572 : if (message
994 2572 : && ((thread_limit && OMP_CLAUSE_THREAD_LIMIT_STRICT (thread_limit))
995 1 : || (num_teams && (TREE_CODE (num_teams) != INTEGER_CST
996 0 : || tree_int_cst_sgn (num_teams) < 0))))
997 0 : sorry_at (OMP_CLAUSE_LOCATION (message), "%<message%> clause");
998 2572 : if (num_teams == NULL_TREE)
999 2301 : num_teams = build_int_cst (unsigned_type_node, 0);
1000 : else
1001 : {
1002 271 : tree expr = OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (num_teams);
1003 : /* FIXME: Implement lower-bound handling as well; however, currently,
1004 : on the host the use-specified value is always used. */
1005 271 : if (TREE_CODE (expr) == TREE_LIST
1006 28 : && OMP_CLAUSE_NUM_TEAMS_DIMS (num_teams)
1007 275 : && TREE_CHAIN (expr))
1008 : {
1009 1 : sorry_at (OMP_CLAUSE_LOCATION (num_teams),
1010 : "%<num_teams%> clause with %<dims%> modifier");
1011 1 : expr = TREE_VALUE (expr);
1012 : }
1013 270 : else if (TREE_CODE (expr) == TREE_LIST)
1014 27 : expr = (TREE_CHAIN (expr)
1015 27 : ? TREE_VALUE (TREE_CHAIN (expr)) : TREE_VALUE (expr));
1016 271 : num_teams = fold_convert (unsigned_type_node, expr);
1017 : }
1018 2572 : if (thread_limit == NULL_TREE)
1019 2426 : thread_limit = build_int_cst (unsigned_type_node, 0);
1020 : else
1021 : {
1022 146 : tree expr = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit);
1023 146 : if (TREE_CODE (expr) == TREE_LIST)
1024 : {
1025 15 : if (OMP_CLAUSE_THREAD_LIMIT_DIMS (thread_limit)
1026 15 : && (TREE_CHAIN (expr)
1027 0 : || OMP_CLAUSE_THREAD_LIMIT_STRICT (thread_limit)))
1028 1 : sorry_at (OMP_CLAUSE_LOCATION (thread_limit),
1029 : "%<thread_limit%> clause with %<dims%> modifier");
1030 14 : else if (OMP_CLAUSE_THREAD_LIMIT_STRICT (thread_limit))
1031 0 : sorry_at (OMP_CLAUSE_LOCATION (thread_limit),
1032 : "%<thread_limit%> clause with %<strict%> modifier");
1033 15 : expr = TREE_VALUE (expr);
1034 : }
1035 131 : else if (OMP_CLAUSE_THREAD_LIMIT_STRICT (thread_limit))
1036 0 : sorry_at (OMP_CLAUSE_LOCATION (thread_limit),
1037 : "%<thread_limit%> clause with %<strict%> modifier");
1038 146 : thread_limit = fold_convert (unsigned_type_node, expr);
1039 : }
1040 :
1041 2572 : gimple_stmt_iterator gsi = gsi_last_nondebug_bb (bb);
1042 2572 : tree t = gimple_omp_teams_data_arg (entry_stmt), t1;
1043 2572 : if (t == NULL)
1044 1379 : t1 = null_pointer_node;
1045 : else
1046 1193 : t1 = build_fold_addr_expr (t);
1047 2572 : tree child_fndecl = gimple_omp_teams_child_fn (entry_stmt);
1048 2572 : tree t2 = build_fold_addr_expr (child_fndecl);
1049 :
1050 2572 : vec<tree, va_gc> *args;
1051 2572 : vec_alloc (args, 5);
1052 2572 : args->quick_push (t2);
1053 2572 : args->quick_push (t1);
1054 2572 : args->quick_push (num_teams);
1055 2572 : args->quick_push (thread_limit);
1056 : /* For future extensibility: flags. */
1057 2572 : args->quick_push (build_zero_cst (unsigned_type_node));
1058 :
1059 2572 : t = build_call_expr_loc_vec (UNKNOWN_LOCATION,
1060 : builtin_decl_explicit (BUILT_IN_GOMP_TEAMS_REG),
1061 : args);
1062 :
1063 2572 : force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
1064 : false, GSI_CONTINUE_LINKING);
1065 2572 : }
1066 :
1067 : /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
1068 :
1069 : static tree
1070 43969 : vec2chain (vec<tree, va_gc> *v)
1071 : {
1072 43969 : tree chain = NULL_TREE, t;
1073 43969 : unsigned ix;
1074 :
1075 450902 : FOR_EACH_VEC_SAFE_ELT_REVERSE (v, ix, t)
1076 : {
1077 368510 : DECL_CHAIN (t) = chain;
1078 368510 : chain = t;
1079 : }
1080 :
1081 43969 : return chain;
1082 : }
1083 :
1084 : /* Remove barriers in REGION->EXIT's block. Note that this is only
1085 : valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
1086 : is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
1087 : left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
1088 : removed. */
1089 :
1090 : static void
1091 11879 : remove_exit_barrier (struct omp_region *region)
1092 : {
1093 11879 : gimple_stmt_iterator gsi;
1094 11879 : basic_block exit_bb;
1095 11879 : edge_iterator ei;
1096 11879 : edge e;
1097 11879 : gimple *stmt;
1098 11879 : int any_addressable_vars = -1;
1099 :
1100 11879 : exit_bb = region->exit;
1101 :
1102 : /* If the parallel region doesn't return, we don't have REGION->EXIT
1103 : block at all. */
1104 11879 : if (! exit_bb)
1105 1220 : return;
1106 :
1107 : /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
1108 : workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
1109 : statements that can appear in between are extremely limited -- no
1110 : memory operations at all. Here, we allow nothing at all, so the
1111 : only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
1112 11852 : gsi = gsi_last_nondebug_bb (exit_bb);
1113 11852 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
1114 11852 : gsi_prev_nondebug (&gsi);
1115 11852 : if (!gsi_end_p (gsi) && gimple_code (gsi_stmt (gsi)) != GIMPLE_LABEL)
1116 : return;
1117 :
1118 21688 : FOR_EACH_EDGE (e, ei, exit_bb->preds)
1119 : {
1120 11029 : gsi = gsi_last_nondebug_bb (e->src);
1121 11029 : if (gsi_end_p (gsi))
1122 43 : continue;
1123 10986 : stmt = gsi_stmt (gsi);
1124 10986 : if (gimple_code (stmt) == GIMPLE_OMP_RETURN
1125 10986 : && !gimple_omp_return_nowait_p (stmt))
1126 : {
1127 : /* OpenMP 3.0 tasks unfortunately prevent this optimization
1128 : in many cases. If there could be tasks queued, the barrier
1129 : might be needed to let the tasks run before some local
1130 : variable of the parallel that the task uses as shared
1131 : runs out of scope. The task can be spawned either
1132 : from within current function (this would be easy to check)
1133 : or from some function it calls and gets passed an address
1134 : of such a variable. */
1135 1057 : if (any_addressable_vars < 0)
1136 : {
1137 1057 : gomp_parallel *parallel_stmt
1138 1057 : = as_a <gomp_parallel *> (last_nondebug_stmt (region->entry));
1139 1057 : tree child_fun = gimple_omp_parallel_child_fn (parallel_stmt);
1140 1057 : tree local_decls, block, decl;
1141 1057 : unsigned ix;
1142 :
1143 1057 : any_addressable_vars = 0;
1144 8563 : FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun), ix, decl)
1145 7063 : if (TREE_ADDRESSABLE (decl))
1146 : {
1147 : any_addressable_vars = 1;
1148 : break;
1149 : }
1150 1058 : for (block = gimple_block (stmt);
1151 1058 : !any_addressable_vars
1152 1058 : && block
1153 1058 : && TREE_CODE (block) == BLOCK;
1154 1 : block = BLOCK_SUPERCONTEXT (block))
1155 : {
1156 2 : for (local_decls = BLOCK_VARS (block);
1157 2 : local_decls;
1158 0 : local_decls = DECL_CHAIN (local_decls))
1159 0 : if (TREE_ADDRESSABLE (local_decls))
1160 : {
1161 : any_addressable_vars = 1;
1162 : break;
1163 : }
1164 2 : if (block == gimple_block (parallel_stmt))
1165 : break;
1166 : }
1167 : }
1168 1057 : if (!any_addressable_vars)
1169 529 : gimple_omp_return_set_nowait (stmt);
1170 : }
1171 : }
1172 : }
1173 :
1174 : static void
1175 68933 : remove_exit_barriers (struct omp_region *region)
1176 : {
1177 68933 : if (region->type == GIMPLE_OMP_PARALLEL)
1178 11879 : remove_exit_barrier (region);
1179 :
1180 68933 : if (region->inner)
1181 : {
1182 37953 : region = region->inner;
1183 37953 : remove_exit_barriers (region);
1184 82183 : while (region->next)
1185 : {
1186 6277 : region = region->next;
1187 6277 : remove_exit_barriers (region);
1188 : }
1189 : }
1190 68933 : }
1191 :
1192 : /* Optimize omp_get_thread_num () and omp_get_num_threads ()
1193 : calls. These can't be declared as const functions, but
1194 : within one parallel body they are constant, so they can be
1195 : transformed there into __builtin_omp_get_{thread_num,num_threads} ()
1196 : which are declared const. Similarly for task body, except
1197 : that in untied task omp_get_thread_num () can change at any task
1198 : scheduling point. */
1199 :
1200 : static void
1201 16911 : optimize_omp_library_calls (gimple *entry_stmt)
1202 : {
1203 16911 : basic_block bb;
1204 16911 : gimple_stmt_iterator gsi;
1205 16911 : tree thr_num_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
1206 16911 : tree thr_num_id = DECL_ASSEMBLER_NAME (thr_num_tree);
1207 16911 : tree num_thr_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
1208 16911 : tree num_thr_id = DECL_ASSEMBLER_NAME (num_thr_tree);
1209 16911 : bool untied_task = (gimple_code (entry_stmt) == GIMPLE_OMP_TASK
1210 19357 : && omp_find_clause (gimple_omp_task_clauses (entry_stmt),
1211 16911 : OMP_CLAUSE_UNTIED) != NULL);
1212 :
1213 294744 : FOR_EACH_BB_FN (bb, cfun)
1214 1558570 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1215 : {
1216 1002904 : gimple *call = gsi_stmt (gsi);
1217 1002904 : tree decl;
1218 :
1219 1002904 : if (is_gimple_call (call)
1220 75244 : && (decl = gimple_call_fndecl (call))
1221 67924 : && DECL_EXTERNAL (decl)
1222 56202 : && TREE_PUBLIC (decl)
1223 1059106 : && DECL_INITIAL (decl) == NULL)
1224 : {
1225 56201 : tree built_in;
1226 :
1227 56201 : if (DECL_NAME (decl) == thr_num_id)
1228 : {
1229 : /* In #pragma omp task untied omp_get_thread_num () can change
1230 : during the execution of the task region. */
1231 1308 : if (untied_task)
1232 0 : continue;
1233 1308 : built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
1234 : }
1235 54893 : else if (DECL_NAME (decl) == num_thr_id)
1236 412 : built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
1237 : else
1238 54481 : continue;
1239 :
1240 1720 : if (DECL_ASSEMBLER_NAME (decl) != DECL_ASSEMBLER_NAME (built_in)
1241 1720 : || gimple_call_num_args (call) != 0)
1242 1190 : continue;
1243 :
1244 530 : if (flag_exceptions && !TREE_NOTHROW (decl))
1245 0 : continue;
1246 :
1247 530 : if (TREE_CODE (TREE_TYPE (decl)) != FUNCTION_TYPE
1248 1060 : || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl)),
1249 530 : TREE_TYPE (TREE_TYPE (built_in))))
1250 0 : continue;
1251 :
1252 530 : gimple_call_set_fndecl (call, built_in);
1253 : }
1254 : }
1255 16911 : }
1256 :
1257 : /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
1258 : regimplified. */
1259 :
1260 : static tree
1261 263563 : expand_omp_regimplify_p (tree *tp, int *walk_subtrees, void *)
1262 : {
1263 263563 : tree t = *tp;
1264 :
1265 : /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
1266 263563 : if (VAR_P (t) && DECL_HAS_VALUE_EXPR_P (t))
1267 : return t;
1268 :
1269 263467 : if (TREE_CODE (t) == ADDR_EXPR)
1270 1763 : recompute_tree_invariant_for_addr_expr (t);
1271 :
1272 263467 : *walk_subtrees = !TYPE_P (t) && !DECL_P (t);
1273 263467 : return NULL_TREE;
1274 : }
1275 :
1276 : /* Prepend or append TO = FROM assignment before or after *GSI_P. */
1277 :
1278 : static void
1279 64885 : expand_omp_build_assign (gimple_stmt_iterator *gsi_p, tree to, tree from,
1280 : bool after)
1281 : {
1282 64885 : bool simple_p = DECL_P (to) && TREE_ADDRESSABLE (to);
1283 126277 : from = force_gimple_operand_gsi (gsi_p, from, simple_p, NULL_TREE,
1284 : !after, after ? GSI_CONTINUE_LINKING
1285 : : GSI_SAME_STMT);
1286 64885 : gimple *stmt = gimple_build_assign (to, from);
1287 64885 : if (after)
1288 3493 : gsi_insert_after (gsi_p, stmt, GSI_CONTINUE_LINKING);
1289 : else
1290 61392 : gsi_insert_before (gsi_p, stmt, GSI_SAME_STMT);
1291 64885 : if (walk_tree (&from, expand_omp_regimplify_p, NULL, NULL)
1292 64885 : || walk_tree (&to, expand_omp_regimplify_p, NULL, NULL))
1293 : {
1294 80 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
1295 80 : gimple_regimplify_operands (stmt, &gsi);
1296 : }
1297 64885 : }
1298 :
1299 : /* Prepend or append LHS CODE RHS condition before or after *GSI_P. */
1300 :
1301 : static gcond *
1302 8434 : expand_omp_build_cond (gimple_stmt_iterator *gsi_p, enum tree_code code,
1303 : tree lhs, tree rhs, bool after = false)
1304 : {
1305 8434 : gcond *cond_stmt = gimple_build_cond (code, lhs, rhs, NULL_TREE, NULL_TREE);
1306 8434 : if (after)
1307 236 : gsi_insert_after (gsi_p, cond_stmt, GSI_CONTINUE_LINKING);
1308 : else
1309 8198 : gsi_insert_before (gsi_p, cond_stmt, GSI_SAME_STMT);
1310 8434 : if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), expand_omp_regimplify_p,
1311 : NULL, NULL)
1312 8434 : || walk_tree (gimple_cond_rhs_ptr (cond_stmt), expand_omp_regimplify_p,
1313 : NULL, NULL))
1314 : {
1315 6 : gimple_stmt_iterator gsi = gsi_for_stmt (cond_stmt);
1316 6 : gimple_regimplify_operands (cond_stmt, &gsi);
1317 : }
1318 8434 : return cond_stmt;
1319 : }
1320 :
1321 : /* Expand the OpenMP parallel or task directive starting at REGION. */
1322 :
1323 : static void
1324 22868 : expand_omp_taskreg (struct omp_region *region)
1325 : {
1326 22868 : basic_block entry_bb, exit_bb, new_bb;
1327 22868 : struct function *child_cfun;
1328 22868 : tree child_fn, block, t;
1329 22868 : gimple_stmt_iterator gsi;
1330 22868 : gimple *entry_stmt, *stmt;
1331 22868 : edge e;
1332 22868 : vec<tree, va_gc> *ws_args;
1333 :
1334 22868 : entry_stmt = last_nondebug_stmt (region->entry);
1335 22868 : if (gimple_code (entry_stmt) == GIMPLE_OMP_TASK
1336 22868 : && gimple_omp_task_taskwait_p (entry_stmt))
1337 : {
1338 84 : new_bb = region->entry;
1339 84 : gsi = gsi_last_nondebug_bb (region->entry);
1340 84 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TASK);
1341 84 : gsi_remove (&gsi, true);
1342 84 : expand_taskwait_call (new_bb, as_a <gomp_task *> (entry_stmt));
1343 84 : return;
1344 : }
1345 :
1346 22784 : child_fn = gimple_omp_taskreg_child_fn (entry_stmt);
1347 22784 : child_cfun = DECL_STRUCT_FUNCTION (child_fn);
1348 :
1349 22784 : entry_bb = region->entry;
1350 22784 : if (gimple_code (entry_stmt) == GIMPLE_OMP_TASK)
1351 3812 : exit_bb = region->cont;
1352 : else
1353 18972 : exit_bb = region->exit;
1354 :
1355 22784 : if (is_combined_parallel (region))
1356 1200 : ws_args = region->ws_args;
1357 : else
1358 : ws_args = NULL;
1359 :
1360 22784 : if (child_cfun->cfg)
1361 : {
1362 : /* Due to inlining, it may happen that we have already outlined
1363 : the region, in which case all we need to do is make the
1364 : sub-graph unreachable and emit the parallel call. */
1365 0 : edge entry_succ_e, exit_succ_e;
1366 :
1367 0 : entry_succ_e = single_succ_edge (entry_bb);
1368 :
1369 0 : gsi = gsi_last_nondebug_bb (entry_bb);
1370 0 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_PARALLEL
1371 : || gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TASK
1372 : || gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TEAMS);
1373 0 : gsi_remove (&gsi, true);
1374 :
1375 0 : new_bb = entry_bb;
1376 0 : if (exit_bb)
1377 : {
1378 0 : exit_succ_e = single_succ_edge (exit_bb);
1379 0 : make_edge (new_bb, exit_succ_e->dest, EDGE_FALLTHRU);
1380 : }
1381 0 : remove_edge_and_dominated_blocks (entry_succ_e);
1382 : }
1383 : else
1384 : {
1385 22784 : unsigned srcidx, dstidx, num;
1386 :
1387 : /* If the parallel region needs data sent from the parent
1388 : function, then the very first statement (except possible
1389 : tree profile counter updates) of the parallel body
1390 : is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
1391 : &.OMP_DATA_O is passed as an argument to the child function,
1392 : we need to replace it with the argument as seen by the child
1393 : function.
1394 :
1395 : In most cases, this will end up being the identity assignment
1396 : .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
1397 : a function call that has been inlined, the original PARM_DECL
1398 : .OMP_DATA_I may have been converted into a different local
1399 : variable. In which case, we need to keep the assignment. */
1400 22784 : if (gimple_omp_taskreg_data_arg (entry_stmt))
1401 : {
1402 18273 : basic_block entry_succ_bb
1403 33453 : = single_succ_p (entry_bb) ? single_succ (entry_bb)
1404 3093 : : FALLTHRU_EDGE (entry_bb)->dest;
1405 18273 : tree arg;
1406 18273 : gimple *parcopy_stmt = NULL;
1407 :
1408 36546 : for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
1409 : {
1410 18273 : gimple *stmt;
1411 :
1412 18273 : gcc_assert (!gsi_end_p (gsi));
1413 18273 : stmt = gsi_stmt (gsi);
1414 18273 : if (gimple_code (stmt) != GIMPLE_ASSIGN)
1415 0 : continue;
1416 :
1417 18273 : if (gimple_num_ops (stmt) == 2)
1418 : {
1419 18273 : tree arg = gimple_assign_rhs1 (stmt);
1420 :
1421 : /* We're ignore the subcode because we're
1422 : effectively doing a STRIP_NOPS. */
1423 :
1424 18273 : if (TREE_CODE (arg) == ADDR_EXPR
1425 18273 : && (TREE_OPERAND (arg, 0)
1426 18273 : == gimple_omp_taskreg_data_arg (entry_stmt)))
1427 : {
1428 18273 : parcopy_stmt = stmt;
1429 18273 : break;
1430 : }
1431 : }
1432 0 : }
1433 :
1434 18273 : gcc_assert (parcopy_stmt != NULL);
1435 18273 : arg = DECL_ARGUMENTS (child_fn);
1436 :
1437 18273 : if (!gimple_in_ssa_p (cfun))
1438 : {
1439 18090 : if (gimple_assign_lhs (parcopy_stmt) == arg)
1440 18090 : gsi_remove (&gsi, true);
1441 : else
1442 : {
1443 : /* ?? Is setting the subcode really necessary ?? */
1444 0 : gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (arg));
1445 0 : gimple_assign_set_rhs1 (parcopy_stmt, arg);
1446 : }
1447 : }
1448 : else
1449 : {
1450 183 : tree lhs = gimple_assign_lhs (parcopy_stmt);
1451 183 : gcc_assert (SSA_NAME_VAR (lhs) == arg);
1452 : /* We'd like to set the rhs to the default def in the child_fn,
1453 : but it's too early to create ssa names in the child_fn.
1454 : Instead, we set the rhs to the parm. In
1455 : move_sese_region_to_fn, we introduce a default def for the
1456 : parm, map the parm to it's default def, and once we encounter
1457 : this stmt, replace the parm with the default def. */
1458 183 : gimple_assign_set_rhs1 (parcopy_stmt, arg);
1459 183 : update_stmt (parcopy_stmt);
1460 : }
1461 : }
1462 :
1463 : /* Declare local variables needed in CHILD_CFUN. */
1464 22784 : block = DECL_INITIAL (child_fn);
1465 22784 : BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
1466 : /* The gimplifier could record temporaries in parallel/task block
1467 : rather than in containing function's local_decls chain,
1468 : which would mean cgraph missed finalizing them. Do it now. */
1469 254071 : for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
1470 231287 : if (VAR_P (t) && TREE_STATIC (t) && !DECL_EXTERNAL (t))
1471 1 : varpool_node::finalize_decl (t);
1472 22784 : DECL_SAVED_TREE (child_fn) = NULL;
1473 : /* We'll create a CFG for child_fn, so no gimple body is needed. */
1474 22784 : gimple_set_body (child_fn, NULL);
1475 22784 : TREE_USED (block) = 1;
1476 :
1477 : /* Reset DECL_CONTEXT on function arguments. */
1478 45568 : for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
1479 22784 : DECL_CONTEXT (t) = child_fn;
1480 :
1481 : /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
1482 : so that it can be moved to the child function. */
1483 22784 : gsi = gsi_last_nondebug_bb (entry_bb);
1484 22784 : stmt = gsi_stmt (gsi);
1485 22784 : gcc_assert (stmt && (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
1486 : || gimple_code (stmt) == GIMPLE_OMP_TASK
1487 : || gimple_code (stmt) == GIMPLE_OMP_TEAMS));
1488 22784 : e = split_block (entry_bb, stmt);
1489 22784 : gsi_remove (&gsi, true);
1490 22784 : entry_bb = e->dest;
1491 22784 : edge e2 = NULL;
1492 22784 : if (gimple_code (entry_stmt) != GIMPLE_OMP_TASK)
1493 18972 : single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
1494 : else
1495 : {
1496 3812 : e2 = make_edge (e->src, BRANCH_EDGE (entry_bb)->dest, EDGE_ABNORMAL);
1497 3812 : gcc_assert (e2->dest == region->exit);
1498 3812 : remove_edge (BRANCH_EDGE (entry_bb));
1499 3812 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e->src);
1500 3812 : gsi = gsi_last_nondebug_bb (region->exit);
1501 3812 : gcc_assert (!gsi_end_p (gsi)
1502 : && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
1503 3812 : gsi_remove (&gsi, true);
1504 : }
1505 :
1506 : /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
1507 22784 : if (exit_bb)
1508 : {
1509 22740 : gsi = gsi_last_nondebug_bb (exit_bb);
1510 41685 : gcc_assert (!gsi_end_p (gsi)
1511 : && (gimple_code (gsi_stmt (gsi))
1512 : == (e2 ? GIMPLE_OMP_CONTINUE : GIMPLE_OMP_RETURN)));
1513 22740 : stmt = gimple_build_return (NULL);
1514 22740 : gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
1515 22740 : gsi_remove (&gsi, true);
1516 : }
1517 :
1518 : /* Move the parallel region into CHILD_CFUN. */
1519 :
1520 22784 : if (gimple_in_ssa_p (cfun))
1521 : {
1522 195 : init_tree_ssa (child_cfun);
1523 195 : init_ssa_operands (child_cfun);
1524 195 : child_cfun->gimple_df->in_ssa_p = true;
1525 195 : block = NULL_TREE;
1526 : }
1527 : else
1528 22589 : block = gimple_block (entry_stmt);
1529 :
1530 22784 : new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
1531 22784 : if (exit_bb)
1532 22740 : single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
1533 22784 : if (e2)
1534 : {
1535 3812 : basic_block dest_bb = e2->dest;
1536 3812 : if (!exit_bb)
1537 17 : make_edge (new_bb, dest_bb, EDGE_FALLTHRU);
1538 3812 : remove_edge (e2);
1539 3812 : set_immediate_dominator (CDI_DOMINATORS, dest_bb, new_bb);
1540 : }
1541 : /* When the OMP expansion process cannot guarantee an up-to-date
1542 : loop tree arrange for the child function to fixup loops. */
1543 22784 : if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
1544 22589 : child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
1545 :
1546 : /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
1547 22784 : num = vec_safe_length (child_cfun->local_decls);
1548 1077179 : for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
1549 : {
1550 1054395 : t = (*child_cfun->local_decls)[srcidx];
1551 1054395 : if (DECL_CONTEXT (t) == cfun->decl)
1552 231287 : continue;
1553 823108 : if (srcidx != dstidx)
1554 819261 : (*child_cfun->local_decls)[dstidx] = t;
1555 823108 : dstidx++;
1556 : }
1557 22784 : if (dstidx != num)
1558 20588 : vec_safe_truncate (child_cfun->local_decls, dstidx);
1559 :
1560 : /* Inform the callgraph about the new function. */
1561 22784 : child_cfun->curr_properties = cfun->curr_properties;
1562 22784 : child_cfun->has_simduid_loops |= cfun->has_simduid_loops;
1563 22784 : child_cfun->has_force_vectorize_loops |= cfun->has_force_vectorize_loops;
1564 22784 : cgraph_node *node = cgraph_node::get_create (child_fn);
1565 22784 : node->parallelized_function = 1;
1566 45568 : node->has_omp_variant_constructs
1567 22784 : |= cgraph_node::get (cfun->decl)->has_omp_variant_constructs;
1568 22784 : cgraph_node::add_new_function (child_fn, true);
1569 :
1570 22784 : bool need_asm = DECL_ASSEMBLER_NAME_SET_P (current_function_decl)
1571 22784 : && !DECL_ASSEMBLER_NAME_SET_P (child_fn);
1572 :
1573 : /* Fix the callgraph edges for child_cfun. Those for cfun will be
1574 : fixed in a following pass. */
1575 22784 : push_cfun (child_cfun);
1576 22784 : if (need_asm)
1577 22589 : assign_assembler_name_if_needed (child_fn);
1578 :
1579 22784 : if (optimize)
1580 16911 : optimize_omp_library_calls (entry_stmt);
1581 22784 : update_max_bb_count ();
1582 22784 : cgraph_edge::rebuild_edges ();
1583 :
1584 : /* Some EH regions might become dead, see PR34608. If
1585 : pass_cleanup_cfg isn't the first pass to happen with the
1586 : new child, these dead EH edges might cause problems.
1587 : Clean them up now. */
1588 22784 : if (flag_exceptions)
1589 : {
1590 10289 : basic_block bb;
1591 10289 : bool changed = false;
1592 :
1593 136412 : FOR_EACH_BB_FN (bb, cfun)
1594 126123 : changed |= gimple_purge_dead_eh_edges (bb);
1595 10289 : if (changed)
1596 0 : cleanup_tree_cfg ();
1597 : }
1598 22784 : if (gimple_in_ssa_p (cfun))
1599 195 : update_ssa (TODO_update_ssa);
1600 22784 : if (flag_checking && !loops_state_satisfies_p (LOOPS_NEED_FIXUP))
1601 195 : verify_loop_structure ();
1602 22784 : pop_cfun ();
1603 :
1604 22784 : if (dump_file && !gimple_in_ssa_p (cfun))
1605 : {
1606 88 : omp_any_child_fn_dumped = true;
1607 88 : dump_function_header (dump_file, child_fn, dump_flags);
1608 88 : dump_function_to_file (child_fn, dump_file, dump_flags);
1609 : }
1610 : }
1611 :
1612 22784 : adjust_context_and_scope (region, gimple_block (entry_stmt), child_fn);
1613 :
1614 22784 : if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL)
1615 16400 : expand_parallel_call (region, new_bb,
1616 : as_a <gomp_parallel *> (entry_stmt), ws_args);
1617 6384 : else if (gimple_code (entry_stmt) == GIMPLE_OMP_TEAMS)
1618 2572 : expand_teams_call (new_bb, as_a <gomp_teams *> (entry_stmt));
1619 : else
1620 3812 : expand_task_call (region, new_bb, as_a <gomp_task *> (entry_stmt));
1621 : }
1622 :
1623 : /* Information about members of an OpenACC collapsed loop nest. */
1624 :
1625 : struct oacc_collapse
1626 : {
1627 : tree base; /* Base value. */
1628 : tree iters; /* Number of steps. */
1629 : tree step; /* Step size. */
1630 : tree tile; /* Tile increment (if tiled). */
1631 : tree outer; /* Tile iterator var. */
1632 : };
1633 :
1634 : /* Helper for expand_oacc_for. Determine collapsed loop information.
1635 : Fill in COUNTS array. Emit any initialization code before GSI.
1636 : Return the calculated outer loop bound of BOUND_TYPE. */
1637 :
1638 : static tree
1639 594 : expand_oacc_collapse_init (const struct omp_for_data *fd,
1640 : gimple_stmt_iterator *gsi,
1641 : oacc_collapse *counts, tree diff_type,
1642 : tree bound_type, location_t loc)
1643 : {
1644 594 : tree tiling = fd->tiling;
1645 594 : tree total = build_int_cst (bound_type, 1);
1646 594 : int ix;
1647 :
1648 594 : gcc_assert (integer_onep (fd->loop.step));
1649 594 : gcc_assert (integer_zerop (fd->loop.n1));
1650 :
1651 : /* When tiling, the first operand of the tile clause applies to the
1652 : innermost loop, and we work outwards from there. Seems
1653 : backwards, but whatever. */
1654 1890 : for (ix = fd->collapse; ix--;)
1655 : {
1656 1296 : const omp_for_data_loop *loop = &fd->loops[ix];
1657 :
1658 1296 : tree iter_type = TREE_TYPE (loop->v);
1659 1296 : tree plus_type = iter_type;
1660 :
1661 1296 : gcc_assert (loop->cond_code == LT_EXPR || loop->cond_code == GT_EXPR);
1662 :
1663 1296 : if (POINTER_TYPE_P (iter_type))
1664 0 : plus_type = sizetype;
1665 :
1666 1296 : if (tiling)
1667 : {
1668 284 : tree num = build_int_cst (integer_type_node, fd->collapse);
1669 284 : tree loop_no = build_int_cst (integer_type_node, ix);
1670 284 : tree tile = TREE_VALUE (tiling);
1671 284 : gcall *call
1672 284 : = gimple_build_call_internal (IFN_GOACC_TILE, 5, num, loop_no, tile,
1673 : /* gwv-outer=*/integer_zero_node,
1674 : /* gwv-inner=*/integer_zero_node);
1675 :
1676 284 : counts[ix].outer = create_tmp_var (iter_type, ".outer");
1677 284 : counts[ix].tile = create_tmp_var (diff_type, ".tile");
1678 284 : gimple_call_set_lhs (call, counts[ix].tile);
1679 284 : gimple_set_location (call, loc);
1680 284 : gsi_insert_before (gsi, call, GSI_SAME_STMT);
1681 :
1682 284 : tiling = TREE_CHAIN (tiling);
1683 : }
1684 : else
1685 : {
1686 1012 : counts[ix].tile = NULL;
1687 1012 : counts[ix].outer = loop->v;
1688 : }
1689 :
1690 1296 : tree b = loop->n1;
1691 1296 : tree e = loop->n2;
1692 1296 : tree s = loop->step;
1693 1296 : bool up = loop->cond_code == LT_EXPR;
1694 1344 : tree dir = build_int_cst (diff_type, up ? +1 : -1);
1695 1296 : bool negating;
1696 1296 : tree expr;
1697 :
1698 1296 : b = force_gimple_operand_gsi (gsi, b, true, NULL_TREE,
1699 : true, GSI_SAME_STMT);
1700 1296 : e = force_gimple_operand_gsi (gsi, e, true, NULL_TREE,
1701 : true, GSI_SAME_STMT);
1702 :
1703 : /* Convert the step, avoiding possible unsigned->signed overflow. */
1704 1296 : negating = !up && TYPE_UNSIGNED (TREE_TYPE (s));
1705 0 : if (negating)
1706 0 : s = fold_build1 (NEGATE_EXPR, TREE_TYPE (s), s);
1707 1296 : s = fold_convert (diff_type, s);
1708 1296 : if (negating)
1709 0 : s = fold_build1 (NEGATE_EXPR, diff_type, s);
1710 1296 : s = force_gimple_operand_gsi (gsi, s, true, NULL_TREE,
1711 : true, GSI_SAME_STMT);
1712 :
1713 : /* Determine the range, avoiding possible unsigned->signed overflow. */
1714 1296 : negating = !up && TYPE_UNSIGNED (iter_type);
1715 2592 : expr = fold_build2 (MINUS_EXPR, plus_type,
1716 : fold_convert (plus_type, negating ? b : e),
1717 : fold_convert (plus_type, negating ? e : b));
1718 1296 : expr = fold_convert (diff_type, expr);
1719 1296 : if (negating)
1720 0 : expr = fold_build1 (NEGATE_EXPR, diff_type, expr);
1721 1296 : tree range = force_gimple_operand_gsi
1722 1296 : (gsi, expr, true, NULL_TREE, true, GSI_SAME_STMT);
1723 :
1724 : /* Determine number of iterations. */
1725 1296 : expr = fold_build2 (MINUS_EXPR, diff_type, range, dir);
1726 1296 : expr = fold_build2 (PLUS_EXPR, diff_type, expr, s);
1727 1296 : expr = fold_build2 (TRUNC_DIV_EXPR, diff_type, expr, s);
1728 :
1729 1296 : tree iters = force_gimple_operand_gsi (gsi, expr, true, NULL_TREE,
1730 : true, GSI_SAME_STMT);
1731 :
1732 1296 : counts[ix].base = b;
1733 1296 : counts[ix].iters = iters;
1734 1296 : counts[ix].step = s;
1735 :
1736 1296 : total = fold_build2 (MULT_EXPR, bound_type, total,
1737 : fold_convert (bound_type, iters));
1738 : }
1739 :
1740 594 : return total;
1741 : }
1742 :
1743 : /* Emit initializers for collapsed loop members. INNER is true if
1744 : this is for the element loop of a TILE. IVAR is the outer
1745 : loop iteration variable, from which collapsed loop iteration values
1746 : are calculated. COUNTS array has been initialized by
1747 : expand_oacc_collapse_inits. */
1748 :
1749 : static void
1750 771 : expand_oacc_collapse_vars (const struct omp_for_data *fd, bool inner,
1751 : gimple_stmt_iterator *gsi,
1752 : const oacc_collapse *counts, tree ivar,
1753 : tree diff_type)
1754 : {
1755 771 : tree ivar_type = TREE_TYPE (ivar);
1756 :
1757 : /* The most rapidly changing iteration variable is the innermost
1758 : one. */
1759 2351 : for (int ix = fd->collapse; ix--;)
1760 : {
1761 1580 : const omp_for_data_loop *loop = &fd->loops[ix];
1762 1580 : const oacc_collapse *collapse = &counts[ix];
1763 1580 : tree v = inner ? loop->v : collapse->outer;
1764 1580 : tree iter_type = TREE_TYPE (v);
1765 1580 : tree plus_type = iter_type;
1766 1580 : enum tree_code plus_code = PLUS_EXPR;
1767 1580 : tree expr;
1768 :
1769 1580 : if (POINTER_TYPE_P (iter_type))
1770 : {
1771 0 : plus_code = POINTER_PLUS_EXPR;
1772 0 : plus_type = sizetype;
1773 : }
1774 :
1775 1580 : expr = ivar;
1776 1580 : if (ix)
1777 : {
1778 809 : tree mod = fold_convert (ivar_type, collapse->iters);
1779 809 : ivar = fold_build2 (TRUNC_DIV_EXPR, ivar_type, expr, mod);
1780 809 : expr = fold_build2 (TRUNC_MOD_EXPR, ivar_type, expr, mod);
1781 809 : ivar = force_gimple_operand_gsi (gsi, ivar, true, NULL_TREE,
1782 : true, GSI_SAME_STMT);
1783 : }
1784 :
1785 1580 : expr = fold_build2 (MULT_EXPR, diff_type, fold_convert (diff_type, expr),
1786 : fold_convert (diff_type, collapse->step));
1787 1580 : expr = fold_build2 (plus_code, iter_type,
1788 : inner ? collapse->outer : collapse->base,
1789 : fold_convert (plus_type, expr));
1790 1580 : expr = force_gimple_operand_gsi (gsi, expr, false, NULL_TREE,
1791 : true, GSI_SAME_STMT);
1792 1580 : gassign *ass = gimple_build_assign (v, expr);
1793 1580 : gsi_insert_before (gsi, ass, GSI_SAME_STMT);
1794 : }
1795 771 : }
1796 :
1797 : /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
1798 : of the combined collapse > 1 loop constructs, generate code like:
1799 : if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
1800 : if (cond3 is <)
1801 : adj = STEP3 - 1;
1802 : else
1803 : adj = STEP3 + 1;
1804 : count3 = (adj + N32 - N31) / STEP3;
1805 : if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
1806 : if (cond2 is <)
1807 : adj = STEP2 - 1;
1808 : else
1809 : adj = STEP2 + 1;
1810 : count2 = (adj + N22 - N21) / STEP2;
1811 : if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
1812 : if (cond1 is <)
1813 : adj = STEP1 - 1;
1814 : else
1815 : adj = STEP1 + 1;
1816 : count1 = (adj + N12 - N11) / STEP1;
1817 : count = count1 * count2 * count3;
1818 : Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
1819 : count = 0;
1820 : and set ZERO_ITER_BB to that bb. If this isn't the outermost
1821 : of the combined loop constructs, just initialize COUNTS array
1822 : from the _looptemp_ clauses. For loop nests with non-rectangular
1823 : loops, do this only for the rectangular loops. Then pick
1824 : the loops which reference outer vars in their bound expressions
1825 : and the loops which they refer to and for this sub-nest compute
1826 : number of iterations. For triangular loops use Faulhaber's formula,
1827 : otherwise as a fallback, compute by iterating the loops.
1828 : If e.g. the sub-nest is
1829 : for (I = N11; I COND1 N12; I += STEP1)
1830 : for (J = M21 * I + N21; J COND2 M22 * I + N22; J += STEP2)
1831 : for (K = M31 * J + N31; K COND3 M32 * J + N32; K += STEP3)
1832 : do:
1833 : COUNT = 0;
1834 : for (tmpi = N11; tmpi COND1 N12; tmpi += STEP1)
1835 : for (tmpj = M21 * tmpi + N21;
1836 : tmpj COND2 M22 * tmpi + N22; tmpj += STEP2)
1837 : {
1838 : int tmpk1 = M31 * tmpj + N31;
1839 : int tmpk2 = M32 * tmpj + N32;
1840 : if (tmpk1 COND3 tmpk2)
1841 : {
1842 : if (COND3 is <)
1843 : adj = STEP3 - 1;
1844 : else
1845 : adj = STEP3 + 1;
1846 : COUNT += (adj + tmpk2 - tmpk1) / STEP3;
1847 : }
1848 : }
1849 : and finally multiply the counts of the rectangular loops not
1850 : in the sub-nest with COUNT. Also, as counts[fd->last_nonrect]
1851 : store number of iterations of the loops from fd->first_nonrect
1852 : to fd->last_nonrect inclusive, i.e. the above COUNT multiplied
1853 : by the counts of rectangular loops not referenced in any non-rectangular
1854 : loops sandwiched in between those. */
1855 :
1856 : /* NOTE: It *could* be better to moosh all of the BBs together,
1857 : creating one larger BB with all the computation and the unexpected
1858 : jump at the end. I.e.
1859 :
1860 : bool zero3, zero2, zero1, zero;
1861 :
1862 : zero3 = N32 c3 N31;
1863 : count3 = (N32 - N31) /[cl] STEP3;
1864 : zero2 = N22 c2 N21;
1865 : count2 = (N22 - N21) /[cl] STEP2;
1866 : zero1 = N12 c1 N11;
1867 : count1 = (N12 - N11) /[cl] STEP1;
1868 : zero = zero3 || zero2 || zero1;
1869 : count = count1 * count2 * count3;
1870 : if (__builtin_expect(zero, false)) goto zero_iter_bb;
1871 :
1872 : After all, we expect the zero=false, and thus we expect to have to
1873 : evaluate all of the comparison expressions, so short-circuiting
1874 : oughtn't be a win. Since the condition isn't protecting a
1875 : denominator, we're not concerned about divide-by-zero, so we can
1876 : fully evaluate count even if a numerator turned out to be wrong.
1877 :
1878 : It seems like putting this all together would create much better
1879 : scheduling opportunities, and less pressure on the chip's branch
1880 : predictor. */
1881 :
1882 : static void
1883 10461 : expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
1884 : basic_block &entry_bb, tree *counts,
1885 : basic_block &zero_iter1_bb, int &first_zero_iter1,
1886 : basic_block &zero_iter2_bb, int &first_zero_iter2,
1887 : basic_block &l2_dom_bb)
1888 : {
1889 10461 : tree t, type = TREE_TYPE (fd->loop.v);
1890 10461 : edge e, ne;
1891 10461 : int i;
1892 :
1893 : /* Collapsed loops need work for expansion into SSA form. */
1894 10461 : gcc_assert (!gimple_in_ssa_p (cfun));
1895 :
1896 10461 : if (gimple_omp_for_combined_into_p (fd->for_stmt)
1897 10461 : && TREE_CODE (fd->loop.n2) != INTEGER_CST)
1898 : {
1899 2692 : gcc_assert (fd->ordered == 0);
1900 : /* First two _looptemp_ clauses are for istart/iend, counts[0]
1901 : isn't supposed to be handled, as the inner loop doesn't
1902 : use it. */
1903 2692 : tree innerc = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
1904 : OMP_CLAUSE__LOOPTEMP_);
1905 2692 : gcc_assert (innerc);
1906 10553 : for (i = 0; i < fd->collapse; i++)
1907 : {
1908 7861 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
1909 : OMP_CLAUSE__LOOPTEMP_);
1910 7861 : gcc_assert (innerc);
1911 7861 : if (i)
1912 5169 : counts[i] = OMP_CLAUSE_DECL (innerc);
1913 : else
1914 2692 : counts[0] = NULL_TREE;
1915 : }
1916 2692 : if (fd->non_rect
1917 112 : && fd->last_nonrect == fd->first_nonrect + 1
1918 2756 : && !TYPE_UNSIGNED (TREE_TYPE (fd->loops[fd->last_nonrect].v)))
1919 : {
1920 : tree c[4];
1921 300 : for (i = 0; i < 4; i++)
1922 : {
1923 240 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
1924 : OMP_CLAUSE__LOOPTEMP_);
1925 240 : gcc_assert (innerc);
1926 240 : c[i] = OMP_CLAUSE_DECL (innerc);
1927 : }
1928 60 : counts[0] = c[0];
1929 60 : fd->first_inner_iterations = c[1];
1930 60 : fd->factor = c[2];
1931 60 : fd->adjn1 = c[3];
1932 : }
1933 : return;
1934 : }
1935 :
1936 8662 : for (i = fd->collapse; i < fd->ordered; i++)
1937 : {
1938 893 : tree itype = TREE_TYPE (fd->loops[i].v);
1939 893 : counts[i] = NULL_TREE;
1940 893 : t = fold_binary (fd->loops[i].cond_code, boolean_type_node,
1941 : fold_convert (itype, fd->loops[i].n1),
1942 : fold_convert (itype, fd->loops[i].n2));
1943 893 : if (t && integer_zerop (t))
1944 : {
1945 0 : for (i = fd->collapse; i < fd->ordered; i++)
1946 0 : counts[i] = build_int_cst (type, 0);
1947 : break;
1948 : }
1949 : }
1950 7769 : bool rect_count_seen = false;
1951 7769 : bool init_n2 = SSA_VAR_P (fd->loop.n2) && zero_iter1_bb;
1952 30153 : for (i = 0; i < (fd->ordered ? fd->ordered : fd->collapse); i++)
1953 : {
1954 22384 : tree itype = TREE_TYPE (fd->loops[i].v);
1955 :
1956 22384 : if (i >= fd->collapse && counts[i])
1957 0 : continue;
1958 22384 : if (fd->non_rect)
1959 : {
1960 : /* Skip loops that use outer iterators in their expressions
1961 : during this phase. */
1962 1050 : if (fd->loops[i].m1 || fd->loops[i].m2)
1963 : {
1964 432 : counts[i] = build_zero_cst (type);
1965 432 : continue;
1966 : }
1967 : }
1968 21952 : if ((SSA_VAR_P (fd->loop.n2) || i >= fd->collapse)
1969 9514 : && ((t = fold_binary (fd->loops[i].cond_code, boolean_type_node,
1970 : fold_convert (itype, fd->loops[i].n1),
1971 : fold_convert (itype, fd->loops[i].n2)))
1972 5909 : == NULL_TREE || !integer_onep (t)))
1973 : {
1974 5789 : gcond *cond_stmt;
1975 5789 : tree n1, n2;
1976 5789 : if (init_n2 && i < fd->collapse && !rect_count_seen)
1977 : {
1978 : /* When called with non-NULL zero_iter1_bb, we won't clear
1979 : fd->loop.n2 in the if (zero_iter_bb == NULL) code below
1980 : and if it is prior to storing fd->loop.n2 where
1981 : rect_count_seen is set, it could be used uninitialized.
1982 : As zero_iter1_bb in that case can be reached also if there
1983 : are non-zero iterations, the clearing can't be emitted
1984 : to the zero_iter1_bb, but needs to be done before the
1985 : condition. */
1986 1505 : gassign *assign_stmt
1987 1505 : = gimple_build_assign (fd->loop.n2, build_zero_cst (type));
1988 1505 : gsi_insert_before (gsi, assign_stmt, GSI_SAME_STMT);
1989 1505 : init_n2 = false;
1990 : }
1991 5789 : n1 = fold_convert (itype, unshare_expr (fd->loops[i].n1));
1992 5789 : n1 = force_gimple_operand_gsi (gsi, n1, true, NULL_TREE,
1993 : true, GSI_SAME_STMT);
1994 5789 : n2 = fold_convert (itype, unshare_expr (fd->loops[i].n2));
1995 5789 : n2 = force_gimple_operand_gsi (gsi, n2, true, NULL_TREE,
1996 : true, GSI_SAME_STMT);
1997 5789 : cond_stmt = expand_omp_build_cond (gsi, fd->loops[i].cond_code,
1998 : n1, n2);
1999 5789 : e = split_block (entry_bb, cond_stmt);
2000 216 : basic_block &zero_iter_bb
2001 5789 : = i < fd->collapse ? zero_iter1_bb : zero_iter2_bb;
2002 216 : int &first_zero_iter
2003 5789 : = i < fd->collapse ? first_zero_iter1 : first_zero_iter2;
2004 5789 : if (zero_iter_bb == NULL)
2005 : {
2006 626 : gassign *assign_stmt;
2007 626 : first_zero_iter = i;
2008 626 : zero_iter_bb = create_empty_bb (entry_bb);
2009 626 : add_bb_to_loop (zero_iter_bb, entry_bb->loop_father);
2010 626 : *gsi = gsi_after_labels (zero_iter_bb);
2011 626 : if (i < fd->collapse)
2012 508 : assign_stmt = gimple_build_assign (fd->loop.n2,
2013 : build_zero_cst (type));
2014 : else
2015 : {
2016 118 : counts[i] = create_tmp_reg (type, ".count");
2017 118 : assign_stmt
2018 118 : = gimple_build_assign (counts[i], build_zero_cst (type));
2019 : }
2020 626 : gsi_insert_before (gsi, assign_stmt, GSI_SAME_STMT);
2021 626 : set_immediate_dominator (CDI_DOMINATORS, zero_iter_bb,
2022 : entry_bb);
2023 : }
2024 5789 : ne = make_edge (entry_bb, zero_iter_bb, EDGE_FALSE_VALUE);
2025 5789 : ne->probability = profile_probability::very_unlikely ();
2026 5789 : e->flags = EDGE_TRUE_VALUE;
2027 5789 : e->probability = ne->probability.invert ();
2028 5789 : if (l2_dom_bb == NULL)
2029 2591 : l2_dom_bb = entry_bb;
2030 5789 : entry_bb = e->dest;
2031 5789 : *gsi = gsi_last_nondebug_bb (entry_bb);
2032 : }
2033 :
2034 21952 : if (POINTER_TYPE_P (itype))
2035 1661 : itype = signed_type_for (itype);
2036 21952 : t = build_int_cst (itype, (fd->loops[i].cond_code == LT_EXPR
2037 27056 : ? -1 : 1));
2038 21952 : t = fold_build2 (PLUS_EXPR, itype,
2039 : fold_convert (itype, fd->loops[i].step), t);
2040 21952 : t = fold_build2 (PLUS_EXPR, itype, t,
2041 : fold_convert (itype, fd->loops[i].n2));
2042 21952 : t = fold_build2 (MINUS_EXPR, itype, t,
2043 : fold_convert (itype, fd->loops[i].n1));
2044 : /* ?? We could probably use CEIL_DIV_EXPR instead of
2045 : TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
2046 : generate the same code in the end because generically we
2047 : don't know that the values involved must be negative for
2048 : GT?? */
2049 21952 : if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR)
2050 2248 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
2051 : fold_build1 (NEGATE_EXPR, itype, t),
2052 : fold_build1 (NEGATE_EXPR, itype,
2053 : fold_convert (itype,
2054 : fd->loops[i].step)));
2055 : else
2056 19704 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t,
2057 : fold_convert (itype, fd->loops[i].step));
2058 21952 : t = fold_convert (type, t);
2059 21952 : if (TREE_CODE (t) == INTEGER_CST)
2060 16093 : counts[i] = t;
2061 : else
2062 : {
2063 5859 : if (i < fd->collapse || i != first_zero_iter2)
2064 5741 : counts[i] = create_tmp_reg (type, ".count");
2065 5859 : expand_omp_build_assign (gsi, counts[i], t);
2066 : }
2067 21952 : if (SSA_VAR_P (fd->loop.n2) && i < fd->collapse)
2068 : {
2069 8621 : if (fd->non_rect && i >= fd->first_nonrect && i <= fd->last_nonrect)
2070 376 : continue;
2071 8245 : if (!rect_count_seen)
2072 : {
2073 2995 : t = counts[i];
2074 2995 : rect_count_seen = true;
2075 : }
2076 : else
2077 5250 : t = fold_build2 (MULT_EXPR, type, fd->loop.n2, counts[i]);
2078 8245 : expand_omp_build_assign (gsi, fd->loop.n2, t);
2079 : }
2080 : }
2081 7769 : if (fd->non_rect && SSA_VAR_P (fd->loop.n2))
2082 : {
2083 284 : gcc_assert (fd->last_nonrect != -1);
2084 :
2085 284 : counts[fd->last_nonrect] = create_tmp_reg (type, ".count");
2086 284 : expand_omp_build_assign (gsi, counts[fd->last_nonrect],
2087 : build_zero_cst (type));
2088 366 : for (i = fd->first_nonrect + 1; i < fd->last_nonrect; i++)
2089 113 : if (fd->loops[i].m1
2090 83 : || fd->loops[i].m2
2091 82 : || fd->loops[i].non_rect_referenced)
2092 : break;
2093 284 : if (i == fd->last_nonrect
2094 253 : && fd->loops[i].outer == fd->last_nonrect - fd->first_nonrect
2095 253 : && !POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v))
2096 511 : && !TYPE_UNSIGNED (TREE_TYPE (fd->loops[i].v)))
2097 : {
2098 225 : int o = fd->first_nonrect;
2099 225 : tree itype = TREE_TYPE (fd->loops[o].v);
2100 225 : tree n1o = create_tmp_reg (itype, ".n1o");
2101 225 : t = fold_convert (itype, unshare_expr (fd->loops[o].n1));
2102 225 : expand_omp_build_assign (gsi, n1o, t);
2103 225 : tree n2o = create_tmp_reg (itype, ".n2o");
2104 225 : t = fold_convert (itype, unshare_expr (fd->loops[o].n2));
2105 225 : expand_omp_build_assign (gsi, n2o, t);
2106 225 : if (fd->loops[i].m1 && fd->loops[i].m2)
2107 43 : t = fold_build2 (MINUS_EXPR, itype, unshare_expr (fd->loops[i].m2),
2108 : unshare_expr (fd->loops[i].m1));
2109 182 : else if (fd->loops[i].m1)
2110 162 : t = fold_build1 (NEGATE_EXPR, itype,
2111 : unshare_expr (fd->loops[i].m1));
2112 : else
2113 20 : t = unshare_expr (fd->loops[i].m2);
2114 225 : tree m2minusm1
2115 225 : = force_gimple_operand_gsi (gsi, t, true, NULL_TREE,
2116 : true, GSI_SAME_STMT);
2117 :
2118 225 : gimple_stmt_iterator gsi2 = *gsi;
2119 225 : gsi_prev (&gsi2);
2120 225 : e = split_block (entry_bb, gsi_stmt (gsi2));
2121 225 : e = split_block (e->dest, (gimple *) NULL);
2122 225 : basic_block bb1 = e->src;
2123 225 : entry_bb = e->dest;
2124 225 : *gsi = gsi_after_labels (entry_bb);
2125 :
2126 225 : gsi2 = gsi_after_labels (bb1);
2127 225 : tree ostep = fold_convert (itype, fd->loops[o].step);
2128 225 : t = build_int_cst (itype, (fd->loops[o].cond_code
2129 241 : == LT_EXPR ? -1 : 1));
2130 225 : t = fold_build2 (PLUS_EXPR, itype, ostep, t);
2131 225 : t = fold_build2 (PLUS_EXPR, itype, t, n2o);
2132 225 : t = fold_build2 (MINUS_EXPR, itype, t, n1o);
2133 225 : if (TYPE_UNSIGNED (itype)
2134 225 : && fd->loops[o].cond_code == GT_EXPR)
2135 0 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
2136 : fold_build1 (NEGATE_EXPR, itype, t),
2137 : fold_build1 (NEGATE_EXPR, itype, ostep));
2138 : else
2139 225 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, ostep);
2140 225 : tree outer_niters
2141 225 : = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2142 : true, GSI_SAME_STMT);
2143 225 : t = fold_build2 (MINUS_EXPR, itype, outer_niters,
2144 : build_one_cst (itype));
2145 225 : t = fold_build2 (MULT_EXPR, itype, t, ostep);
2146 225 : t = fold_build2 (PLUS_EXPR, itype, n1o, t);
2147 225 : tree last = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2148 : true, GSI_SAME_STMT);
2149 225 : tree n1, n2, n1e, n2e;
2150 225 : t = fold_convert (itype, unshare_expr (fd->loops[i].n1));
2151 225 : if (fd->loops[i].m1)
2152 : {
2153 205 : n1 = fold_convert (itype, unshare_expr (fd->loops[i].m1));
2154 205 : n1 = fold_build2 (MULT_EXPR, itype, n1o, n1);
2155 205 : n1 = fold_build2 (PLUS_EXPR, itype, n1, t);
2156 : }
2157 : else
2158 : n1 = t;
2159 225 : n1 = force_gimple_operand_gsi (&gsi2, n1, true, NULL_TREE,
2160 : true, GSI_SAME_STMT);
2161 225 : t = fold_convert (itype, unshare_expr (fd->loops[i].n2));
2162 225 : if (fd->loops[i].m2)
2163 : {
2164 63 : n2 = fold_convert (itype, unshare_expr (fd->loops[i].m2));
2165 63 : n2 = fold_build2 (MULT_EXPR, itype, n1o, n2);
2166 63 : n2 = fold_build2 (PLUS_EXPR, itype, n2, t);
2167 : }
2168 : else
2169 : n2 = t;
2170 225 : n2 = force_gimple_operand_gsi (&gsi2, n2, true, NULL_TREE,
2171 : true, GSI_SAME_STMT);
2172 225 : t = fold_convert (itype, unshare_expr (fd->loops[i].n1));
2173 225 : if (fd->loops[i].m1)
2174 : {
2175 205 : n1e = fold_convert (itype, unshare_expr (fd->loops[i].m1));
2176 205 : n1e = fold_build2 (MULT_EXPR, itype, last, n1e);
2177 205 : n1e = fold_build2 (PLUS_EXPR, itype, n1e, t);
2178 : }
2179 : else
2180 : n1e = t;
2181 225 : n1e = force_gimple_operand_gsi (&gsi2, n1e, true, NULL_TREE,
2182 : true, GSI_SAME_STMT);
2183 225 : t = fold_convert (itype, unshare_expr (fd->loops[i].n2));
2184 225 : if (fd->loops[i].m2)
2185 : {
2186 63 : n2e = fold_convert (itype, unshare_expr (fd->loops[i].m2));
2187 63 : n2e = fold_build2 (MULT_EXPR, itype, last, n2e);
2188 63 : n2e = fold_build2 (PLUS_EXPR, itype, n2e, t);
2189 : }
2190 : else
2191 : n2e = t;
2192 225 : n2e = force_gimple_operand_gsi (&gsi2, n2e, true, NULL_TREE,
2193 : true, GSI_SAME_STMT);
2194 225 : gcond *cond_stmt
2195 225 : = expand_omp_build_cond (&gsi2, fd->loops[i].cond_code,
2196 : n1, n2);
2197 225 : e = split_block (bb1, cond_stmt);
2198 225 : e->flags = EDGE_TRUE_VALUE;
2199 225 : e->probability = profile_probability::likely ().guessed ();
2200 225 : basic_block bb2 = e->dest;
2201 225 : gsi2 = gsi_after_labels (bb2);
2202 :
2203 225 : cond_stmt = expand_omp_build_cond (&gsi2, fd->loops[i].cond_code,
2204 : n1e, n2e);
2205 225 : e = split_block (bb2, cond_stmt);
2206 225 : e->flags = EDGE_TRUE_VALUE;
2207 225 : e->probability = profile_probability::likely ().guessed ();
2208 225 : gsi2 = gsi_after_labels (e->dest);
2209 :
2210 225 : tree step = fold_convert (itype, fd->loops[i].step);
2211 225 : t = build_int_cst (itype, (fd->loops[i].cond_code
2212 241 : == LT_EXPR ? -1 : 1));
2213 225 : t = fold_build2 (PLUS_EXPR, itype, step, t);
2214 225 : t = fold_build2 (PLUS_EXPR, itype, t, n2);
2215 225 : t = fold_build2 (MINUS_EXPR, itype, t, n1);
2216 225 : if (TYPE_UNSIGNED (itype)
2217 225 : && fd->loops[i].cond_code == GT_EXPR)
2218 0 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
2219 : fold_build1 (NEGATE_EXPR, itype, t),
2220 : fold_build1 (NEGATE_EXPR, itype, step));
2221 : else
2222 225 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
2223 225 : tree first_inner_iterations
2224 225 : = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2225 : true, GSI_SAME_STMT);
2226 225 : t = fold_build2 (MULT_EXPR, itype, m2minusm1, ostep);
2227 225 : if (TYPE_UNSIGNED (itype)
2228 225 : && fd->loops[i].cond_code == GT_EXPR)
2229 0 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
2230 : fold_build1 (NEGATE_EXPR, itype, t),
2231 : fold_build1 (NEGATE_EXPR, itype, step));
2232 : else
2233 225 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
2234 225 : tree factor
2235 225 : = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2236 : true, GSI_SAME_STMT);
2237 225 : t = fold_build2 (MINUS_EXPR, itype, outer_niters,
2238 : build_one_cst (itype));
2239 225 : t = fold_build2 (MULT_EXPR, itype, t, outer_niters);
2240 225 : t = fold_build2 (RSHIFT_EXPR, itype, t, integer_one_node);
2241 225 : t = fold_build2 (MULT_EXPR, itype, factor, t);
2242 225 : t = fold_build2 (PLUS_EXPR, itype,
2243 : fold_build2 (MULT_EXPR, itype, outer_niters,
2244 : first_inner_iterations), t);
2245 225 : expand_omp_build_assign (&gsi2, counts[fd->last_nonrect],
2246 : fold_convert (type, t));
2247 :
2248 225 : basic_block bb3 = create_empty_bb (bb1);
2249 225 : add_bb_to_loop (bb3, bb1->loop_father);
2250 :
2251 225 : e = make_edge (bb1, bb3, EDGE_FALSE_VALUE);
2252 225 : e->probability = profile_probability::unlikely ().guessed ();
2253 :
2254 225 : gsi2 = gsi_after_labels (bb3);
2255 225 : cond_stmt = expand_omp_build_cond (&gsi2, fd->loops[i].cond_code,
2256 : n1e, n2e);
2257 225 : e = split_block (bb3, cond_stmt);
2258 225 : e->flags = EDGE_TRUE_VALUE;
2259 225 : e->probability = profile_probability::likely ().guessed ();
2260 225 : basic_block bb4 = e->dest;
2261 :
2262 225 : ne = make_edge (bb3, entry_bb, EDGE_FALSE_VALUE);
2263 225 : ne->probability = e->probability.invert ();
2264 :
2265 225 : basic_block bb5 = create_empty_bb (bb2);
2266 225 : add_bb_to_loop (bb5, bb2->loop_father);
2267 :
2268 225 : ne = make_edge (bb2, bb5, EDGE_FALSE_VALUE);
2269 225 : ne->probability = profile_probability::unlikely ().guessed ();
2270 :
2271 675 : for (int j = 0; j < 2; j++)
2272 : {
2273 450 : gsi2 = gsi_after_labels (j ? bb5 : bb4);
2274 450 : t = fold_build2 (MINUS_EXPR, itype,
2275 : unshare_expr (fd->loops[i].n1),
2276 : unshare_expr (fd->loops[i].n2));
2277 450 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, m2minusm1);
2278 450 : tree tem
2279 450 : = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2280 : true, GSI_SAME_STMT);
2281 450 : t = fold_build2 (MINUS_EXPR, itype, tem, n1o);
2282 450 : t = fold_build2 (TRUNC_MOD_EXPR, itype, t, ostep);
2283 450 : t = fold_build2 (MINUS_EXPR, itype, tem, t);
2284 450 : tem = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2285 : true, GSI_SAME_STMT);
2286 450 : t = fold_convert (itype, unshare_expr (fd->loops[i].n1));
2287 450 : if (fd->loops[i].m1)
2288 : {
2289 410 : n1 = fold_convert (itype, unshare_expr (fd->loops[i].m1));
2290 410 : n1 = fold_build2 (MULT_EXPR, itype, tem, n1);
2291 410 : n1 = fold_build2 (PLUS_EXPR, itype, n1, t);
2292 : }
2293 : else
2294 : n1 = t;
2295 450 : n1 = force_gimple_operand_gsi (&gsi2, n1, true, NULL_TREE,
2296 : true, GSI_SAME_STMT);
2297 450 : t = fold_convert (itype, unshare_expr (fd->loops[i].n2));
2298 450 : if (fd->loops[i].m2)
2299 : {
2300 126 : n2 = fold_convert (itype, unshare_expr (fd->loops[i].m2));
2301 126 : n2 = fold_build2 (MULT_EXPR, itype, tem, n2);
2302 126 : n2 = fold_build2 (PLUS_EXPR, itype, n2, t);
2303 : }
2304 : else
2305 : n2 = t;
2306 450 : n2 = force_gimple_operand_gsi (&gsi2, n2, true, NULL_TREE,
2307 : true, GSI_SAME_STMT);
2308 675 : expand_omp_build_assign (&gsi2, j ? n2o : n1o, tem);
2309 :
2310 450 : cond_stmt = expand_omp_build_cond (&gsi2, fd->loops[i].cond_code,
2311 : n1, n2);
2312 450 : e = split_block (gsi_bb (gsi2), cond_stmt);
2313 450 : e->flags = j ? EDGE_TRUE_VALUE : EDGE_FALSE_VALUE;
2314 450 : e->probability = profile_probability::unlikely ().guessed ();
2315 675 : ne = make_edge (e->src, bb1,
2316 : j ? EDGE_FALSE_VALUE : EDGE_TRUE_VALUE);
2317 450 : ne->probability = e->probability.invert ();
2318 450 : gsi2 = gsi_after_labels (e->dest);
2319 :
2320 450 : t = fold_build2 (PLUS_EXPR, itype, tem, ostep);
2321 450 : expand_omp_build_assign (&gsi2, j ? n2o : n1o, t);
2322 :
2323 450 : make_edge (e->dest, bb1, EDGE_FALLTHRU);
2324 : }
2325 :
2326 225 : set_immediate_dominator (CDI_DOMINATORS, bb3, bb1);
2327 225 : set_immediate_dominator (CDI_DOMINATORS, bb5, bb2);
2328 225 : set_immediate_dominator (CDI_DOMINATORS, entry_bb, bb1);
2329 :
2330 225 : if (fd->first_nonrect + 1 == fd->last_nonrect)
2331 : {
2332 153 : fd->first_inner_iterations = first_inner_iterations;
2333 153 : fd->factor = factor;
2334 153 : fd->adjn1 = n1o;
2335 : }
2336 : }
2337 : else
2338 : {
2339 : /* Fallback implementation. Evaluate the loops with m1/m2
2340 : non-NULL as well as their outer loops at runtime using temporaries
2341 : instead of the original iteration variables, and in the
2342 : body just bump the counter. */
2343 59 : gimple_stmt_iterator gsi2 = *gsi;
2344 59 : gsi_prev (&gsi2);
2345 59 : e = split_block (entry_bb, gsi_stmt (gsi2));
2346 59 : e = split_block (e->dest, (gimple *) NULL);
2347 59 : basic_block cur_bb = e->src;
2348 59 : basic_block next_bb = e->dest;
2349 59 : entry_bb = e->dest;
2350 59 : *gsi = gsi_after_labels (entry_bb);
2351 :
2352 59 : tree *vs = XALLOCAVEC (tree, fd->last_nonrect);
2353 59 : memset (vs, 0, fd->last_nonrect * sizeof (tree));
2354 :
2355 189 : for (i = 0; i <= fd->last_nonrect; i++)
2356 : {
2357 189 : if (fd->loops[i].m1 == NULL_TREE
2358 108 : && fd->loops[i].m2 == NULL_TREE
2359 99 : && !fd->loops[i].non_rect_referenced)
2360 40 : continue;
2361 :
2362 149 : tree itype = TREE_TYPE (fd->loops[i].v);
2363 :
2364 149 : gsi2 = gsi_after_labels (cur_bb);
2365 149 : tree n1, n2;
2366 149 : t = fold_convert (itype, unshare_expr (fd->loops[i].n1));
2367 149 : if (fd->loops[i].m1 == NULL_TREE)
2368 : n1 = t;
2369 81 : else if (POINTER_TYPE_P (itype))
2370 : {
2371 30 : gcc_assert (integer_onep (fd->loops[i].m1));
2372 30 : t = unshare_expr (fd->loops[i].n1);
2373 30 : n1 = fold_build_pointer_plus (vs[i - fd->loops[i].outer], t);
2374 : }
2375 : else
2376 : {
2377 51 : n1 = fold_convert (itype, unshare_expr (fd->loops[i].m1));
2378 51 : n1 = fold_build2 (MULT_EXPR, itype,
2379 : vs[i - fd->loops[i].outer], n1);
2380 51 : n1 = fold_build2 (PLUS_EXPR, itype, n1, t);
2381 : }
2382 149 : n1 = force_gimple_operand_gsi (&gsi2, n1, true, NULL_TREE,
2383 : true, GSI_SAME_STMT);
2384 149 : if (i < fd->last_nonrect)
2385 : {
2386 90 : vs[i] = create_tmp_reg (itype, ".it");
2387 90 : expand_omp_build_assign (&gsi2, vs[i], n1);
2388 : }
2389 149 : t = fold_convert (itype, unshare_expr (fd->loops[i].n2));
2390 149 : if (fd->loops[i].m2 == NULL_TREE)
2391 : n2 = t;
2392 85 : else if (POINTER_TYPE_P (itype))
2393 : {
2394 34 : gcc_assert (integer_onep (fd->loops[i].m2));
2395 34 : t = unshare_expr (fd->loops[i].n2);
2396 34 : n2 = fold_build_pointer_plus (vs[i - fd->loops[i].outer], t);
2397 : }
2398 : else
2399 : {
2400 51 : n2 = fold_convert (itype, unshare_expr (fd->loops[i].m2));
2401 51 : n2 = fold_build2 (MULT_EXPR, itype,
2402 : vs[i - fd->loops[i].outer], n2);
2403 51 : n2 = fold_build2 (PLUS_EXPR, itype, n2, t);
2404 : }
2405 149 : n2 = force_gimple_operand_gsi (&gsi2, n2, true, NULL_TREE,
2406 : true, GSI_SAME_STMT);
2407 149 : if (POINTER_TYPE_P (itype))
2408 70 : itype = signed_type_for (itype);
2409 149 : if (i == fd->last_nonrect)
2410 : {
2411 59 : gcond *cond_stmt
2412 59 : = expand_omp_build_cond (&gsi2, fd->loops[i].cond_code,
2413 : n1, n2);
2414 59 : e = split_block (cur_bb, cond_stmt);
2415 59 : e->flags = EDGE_TRUE_VALUE;
2416 59 : ne = make_edge (cur_bb, next_bb, EDGE_FALSE_VALUE);
2417 59 : e->probability = profile_probability::likely ().guessed ();
2418 59 : ne->probability = e->probability.invert ();
2419 59 : gsi2 = gsi_after_labels (e->dest);
2420 :
2421 59 : t = build_int_cst (itype, (fd->loops[i].cond_code == LT_EXPR
2422 79 : ? -1 : 1));
2423 59 : t = fold_build2 (PLUS_EXPR, itype,
2424 : fold_convert (itype, fd->loops[i].step), t);
2425 59 : t = fold_build2 (PLUS_EXPR, itype, t,
2426 : fold_convert (itype, n2));
2427 59 : t = fold_build2 (MINUS_EXPR, itype, t,
2428 : fold_convert (itype, n1));
2429 59 : tree step = fold_convert (itype, fd->loops[i].step);
2430 59 : if (TYPE_UNSIGNED (itype)
2431 59 : && fd->loops[i].cond_code == GT_EXPR)
2432 0 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
2433 : fold_build1 (NEGATE_EXPR, itype, t),
2434 : fold_build1 (NEGATE_EXPR, itype, step));
2435 : else
2436 59 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
2437 59 : t = fold_convert (type, t);
2438 59 : t = fold_build2 (PLUS_EXPR, type,
2439 : counts[fd->last_nonrect], t);
2440 59 : t = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2441 : true, GSI_SAME_STMT);
2442 59 : expand_omp_build_assign (&gsi2, counts[fd->last_nonrect], t);
2443 59 : e = make_edge (e->dest, next_bb, EDGE_FALLTHRU);
2444 59 : set_immediate_dominator (CDI_DOMINATORS, next_bb, cur_bb);
2445 59 : break;
2446 : }
2447 90 : e = split_block (cur_bb, last_nondebug_stmt (cur_bb));
2448 :
2449 90 : basic_block new_cur_bb = create_empty_bb (cur_bb);
2450 90 : add_bb_to_loop (new_cur_bb, cur_bb->loop_father);
2451 :
2452 90 : gsi2 = gsi_after_labels (e->dest);
2453 90 : tree step = fold_convert (itype,
2454 : unshare_expr (fd->loops[i].step));
2455 90 : if (POINTER_TYPE_P (TREE_TYPE (vs[i])))
2456 38 : t = fold_build_pointer_plus (vs[i], step);
2457 : else
2458 52 : t = fold_build2 (PLUS_EXPR, itype, vs[i], step);
2459 90 : t = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2460 : true, GSI_SAME_STMT);
2461 90 : expand_omp_build_assign (&gsi2, vs[i], t);
2462 :
2463 90 : ne = split_block (e->dest, last_nondebug_stmt (e->dest));
2464 90 : gsi2 = gsi_after_labels (ne->dest);
2465 :
2466 90 : expand_omp_build_cond (&gsi2, fd->loops[i].cond_code, vs[i], n2);
2467 90 : edge e3, e4;
2468 90 : if (next_bb == entry_bb)
2469 : {
2470 59 : e3 = find_edge (ne->dest, next_bb);
2471 59 : e3->flags = EDGE_FALSE_VALUE;
2472 : }
2473 : else
2474 31 : e3 = make_edge (ne->dest, next_bb, EDGE_FALSE_VALUE);
2475 90 : e4 = make_edge (ne->dest, new_cur_bb, EDGE_TRUE_VALUE);
2476 90 : e4->probability = profile_probability::likely ().guessed ();
2477 90 : e3->probability = e4->probability.invert ();
2478 90 : basic_block esrc = e->src;
2479 90 : make_edge (e->src, ne->dest, EDGE_FALLTHRU);
2480 90 : cur_bb = new_cur_bb;
2481 90 : basic_block latch_bb = next_bb;
2482 90 : next_bb = e->dest;
2483 90 : remove_edge (e);
2484 90 : set_immediate_dominator (CDI_DOMINATORS, ne->dest, esrc);
2485 90 : set_immediate_dominator (CDI_DOMINATORS, latch_bb, ne->dest);
2486 90 : set_immediate_dominator (CDI_DOMINATORS, cur_bb, ne->dest);
2487 : }
2488 : }
2489 284 : t = NULL_TREE;
2490 691 : for (i = fd->first_nonrect; i < fd->last_nonrect; i++)
2491 407 : if (!fd->loops[i].non_rect_referenced
2492 92 : && fd->loops[i].m1 == NULL_TREE
2493 92 : && fd->loops[i].m2 == NULL_TREE)
2494 : {
2495 92 : if (t == NULL_TREE)
2496 82 : t = counts[i];
2497 : else
2498 10 : t = fold_build2 (MULT_EXPR, type, t, counts[i]);
2499 : }
2500 284 : if (t)
2501 : {
2502 82 : t = fold_build2 (MULT_EXPR, type, counts[fd->last_nonrect], t);
2503 82 : expand_omp_build_assign (gsi, counts[fd->last_nonrect], t);
2504 : }
2505 284 : if (!rect_count_seen)
2506 186 : t = counts[fd->last_nonrect];
2507 : else
2508 98 : t = fold_build2 (MULT_EXPR, type, fd->loop.n2,
2509 : counts[fd->last_nonrect]);
2510 284 : expand_omp_build_assign (gsi, fd->loop.n2, t);
2511 284 : }
2512 7485 : else if (fd->non_rect)
2513 : {
2514 117 : tree t = fd->loop.n2;
2515 117 : gcc_assert (TREE_CODE (t) == INTEGER_CST);
2516 : int non_rect_referenced = 0, non_rect = 0;
2517 358 : for (i = 0; i < fd->collapse; i++)
2518 : {
2519 240 : if ((i < fd->first_nonrect || i > fd->last_nonrect)
2520 246 : && !integer_zerop (counts[i]))
2521 6 : t = fold_build2 (TRUNC_DIV_EXPR, type, t, counts[i]);
2522 241 : if (fd->loops[i].non_rect_referenced)
2523 117 : non_rect_referenced++;
2524 241 : if (fd->loops[i].m1 || fd->loops[i].m2)
2525 117 : non_rect++;
2526 : }
2527 117 : gcc_assert (non_rect == 1 && non_rect_referenced == 1);
2528 117 : counts[fd->last_nonrect] = t;
2529 : }
2530 : }
2531 :
2532 : /* Helper function for expand_omp_{for_*,simd}. Generate code like:
2533 : T = V;
2534 : V3 = N31 + (T % count3) * STEP3;
2535 : T = T / count3;
2536 : V2 = N21 + (T % count2) * STEP2;
2537 : T = T / count2;
2538 : V1 = N11 + T * STEP1;
2539 : if this loop doesn't have an inner loop construct combined with it.
2540 : If it does have an inner loop construct combined with it and the
2541 : iteration count isn't known constant, store values from counts array
2542 : into its _looptemp_ temporaries instead.
2543 : For non-rectangular loops (between fd->first_nonrect and fd->last_nonrect
2544 : inclusive), use the count of all those loops together, and either
2545 : find quadratic etc. equation roots, or as a fallback, do:
2546 : COUNT = 0;
2547 : for (tmpi = N11; tmpi COND1 N12; tmpi += STEP1)
2548 : for (tmpj = M21 * tmpi + N21;
2549 : tmpj COND2 M22 * tmpi + N22; tmpj += STEP2)
2550 : {
2551 : int tmpk1 = M31 * tmpj + N31;
2552 : int tmpk2 = M32 * tmpj + N32;
2553 : if (tmpk1 COND3 tmpk2)
2554 : {
2555 : if (COND3 is <)
2556 : adj = STEP3 - 1;
2557 : else
2558 : adj = STEP3 + 1;
2559 : int temp = (adj + tmpk2 - tmpk1) / STEP3;
2560 : if (COUNT + temp > T)
2561 : {
2562 : V1 = tmpi;
2563 : V2 = tmpj;
2564 : V3 = tmpk1 + (T - COUNT) * STEP3;
2565 : goto done;
2566 : }
2567 : else
2568 : COUNT += temp;
2569 : }
2570 : }
2571 : done:;
2572 : but for optional innermost or outermost rectangular loops that aren't
2573 : referenced by other loop expressions keep doing the division/modulo. */
2574 :
2575 : static void
2576 10273 : expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
2577 : tree *counts, tree *nonrect_bounds,
2578 : gimple *inner_stmt, tree startvar)
2579 : {
2580 10273 : int i;
2581 10273 : if (gimple_omp_for_combined_p (fd->for_stmt))
2582 : {
2583 : /* If fd->loop.n2 is constant, then no propagation of the counts
2584 : is needed, they are constant. */
2585 4867 : if (TREE_CODE (fd->loop.n2) == INTEGER_CST)
2586 : return;
2587 :
2588 2692 : tree clauses = gimple_code (inner_stmt) != GIMPLE_OMP_FOR
2589 4158 : ? gimple_omp_taskreg_clauses (inner_stmt)
2590 1226 : : gimple_omp_for_clauses (inner_stmt);
2591 : /* First two _looptemp_ clauses are for istart/iend, counts[0]
2592 : isn't supposed to be handled, as the inner loop doesn't
2593 : use it. */
2594 2692 : tree innerc = omp_find_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
2595 2692 : gcc_assert (innerc);
2596 2692 : int count = 0;
2597 2692 : if (fd->non_rect
2598 112 : && fd->last_nonrect == fd->first_nonrect + 1
2599 2756 : && !TYPE_UNSIGNED (TREE_TYPE (fd->loops[fd->last_nonrect].v)))
2600 : count = 4;
2601 10793 : for (i = 0; i < fd->collapse + count; i++)
2602 : {
2603 8101 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
2604 : OMP_CLAUSE__LOOPTEMP_);
2605 8101 : gcc_assert (innerc);
2606 8101 : if (i)
2607 : {
2608 5409 : tree tem = OMP_CLAUSE_DECL (innerc);
2609 5409 : tree t;
2610 5409 : if (i < fd->collapse)
2611 5169 : t = counts[i];
2612 : else
2613 240 : switch (i - fd->collapse)
2614 : {
2615 60 : case 0: t = counts[0]; break;
2616 60 : case 1: t = fd->first_inner_iterations; break;
2617 60 : case 2: t = fd->factor; break;
2618 60 : case 3: t = fd->adjn1; break;
2619 0 : default: gcc_unreachable ();
2620 : }
2621 5409 : t = fold_convert (TREE_TYPE (tem), t);
2622 5409 : t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE,
2623 : false, GSI_CONTINUE_LINKING);
2624 5409 : gassign *stmt = gimple_build_assign (tem, t);
2625 5409 : gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
2626 : }
2627 : }
2628 : return;
2629 : }
2630 :
2631 5406 : tree type = TREE_TYPE (fd->loop.v);
2632 5406 : tree tem = create_tmp_reg (type, ".tem");
2633 5406 : gassign *stmt = gimple_build_assign (tem, startvar);
2634 5406 : gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
2635 :
2636 20106 : for (i = fd->collapse - 1; i >= 0; i--)
2637 : {
2638 14700 : tree vtype = TREE_TYPE (fd->loops[i].v), itype, t;
2639 14700 : itype = vtype;
2640 14700 : if (POINTER_TYPE_P (vtype))
2641 1649 : itype = signed_type_for (vtype);
2642 14700 : if (i != 0 && (i != fd->last_nonrect || fd->first_nonrect))
2643 9294 : t = fold_build2 (TRUNC_MOD_EXPR, type, tem, counts[i]);
2644 : else
2645 : t = tem;
2646 14700 : if (i == fd->last_nonrect)
2647 : {
2648 376 : t = force_gimple_operand_gsi (gsi, t, true, NULL_TREE,
2649 : false, GSI_CONTINUE_LINKING);
2650 376 : tree stopval = t;
2651 376 : tree idx = create_tmp_reg (type, ".count");
2652 376 : expand_omp_build_assign (gsi, idx,
2653 : build_zero_cst (type), true);
2654 376 : basic_block bb_triang = NULL, bb_triang_dom = NULL;
2655 376 : if (fd->first_nonrect + 1 == fd->last_nonrect
2656 272 : && (TREE_CODE (fd->loop.n2) == INTEGER_CST
2657 181 : || fd->first_inner_iterations)
2658 488 : && (optab_handler (sqrt_optab, TYPE_MODE (double_type_node))
2659 : != CODE_FOR_nothing)
2660 620 : && !integer_zerop (fd->loop.n2))
2661 : {
2662 236 : tree outer_n1 = fd->adjn1 ? fd->adjn1 : fd->loops[i - 1].n1;
2663 236 : tree itype = TREE_TYPE (fd->loops[i].v);
2664 236 : tree first_inner_iterations = fd->first_inner_iterations;
2665 236 : tree factor = fd->factor;
2666 236 : gcond *cond_stmt
2667 236 : = expand_omp_build_cond (gsi, NE_EXPR, factor,
2668 236 : build_zero_cst (TREE_TYPE (factor)),
2669 : true);
2670 236 : edge e = split_block (gsi_bb (*gsi), cond_stmt);
2671 236 : basic_block bb0 = e->src;
2672 236 : e->flags = EDGE_TRUE_VALUE;
2673 236 : e->probability = profile_probability::likely ();
2674 236 : bb_triang_dom = bb0;
2675 236 : *gsi = gsi_after_labels (e->dest);
2676 236 : tree slltype = long_long_integer_type_node;
2677 236 : tree ulltype = long_long_unsigned_type_node;
2678 236 : tree stopvalull = fold_convert (ulltype, stopval);
2679 236 : stopvalull
2680 236 : = force_gimple_operand_gsi (gsi, stopvalull, true, NULL_TREE,
2681 : false, GSI_CONTINUE_LINKING);
2682 236 : first_inner_iterations
2683 236 : = fold_convert (slltype, first_inner_iterations);
2684 236 : first_inner_iterations
2685 236 : = force_gimple_operand_gsi (gsi, first_inner_iterations, true,
2686 : NULL_TREE, false,
2687 : GSI_CONTINUE_LINKING);
2688 236 : factor = fold_convert (slltype, factor);
2689 236 : factor
2690 236 : = force_gimple_operand_gsi (gsi, factor, true, NULL_TREE,
2691 : false, GSI_CONTINUE_LINKING);
2692 236 : tree first_inner_iterationsd
2693 236 : = fold_build1 (FLOAT_EXPR, double_type_node,
2694 : first_inner_iterations);
2695 236 : first_inner_iterationsd
2696 236 : = force_gimple_operand_gsi (gsi, first_inner_iterationsd, true,
2697 : NULL_TREE, false,
2698 : GSI_CONTINUE_LINKING);
2699 236 : tree factord = fold_build1 (FLOAT_EXPR, double_type_node,
2700 : factor);
2701 236 : factord = force_gimple_operand_gsi (gsi, factord, true,
2702 : NULL_TREE, false,
2703 : GSI_CONTINUE_LINKING);
2704 236 : tree stopvald = fold_build1 (FLOAT_EXPR, double_type_node,
2705 : stopvalull);
2706 236 : stopvald = force_gimple_operand_gsi (gsi, stopvald, true,
2707 : NULL_TREE, false,
2708 : GSI_CONTINUE_LINKING);
2709 : /* Temporarily disable flag_rounding_math, values will be
2710 : decimal numbers divided by 2 and worst case imprecisions
2711 : due to too large values ought to be caught later by the
2712 : checks for fallback. */
2713 236 : int save_flag_rounding_math = flag_rounding_math;
2714 236 : flag_rounding_math = 0;
2715 236 : t = fold_build2 (RDIV_EXPR, double_type_node, factord,
2716 : build_real (double_type_node, dconst2));
2717 236 : tree t3 = fold_build2 (MINUS_EXPR, double_type_node,
2718 : first_inner_iterationsd, t);
2719 236 : t3 = force_gimple_operand_gsi (gsi, t3, true, NULL_TREE, false,
2720 : GSI_CONTINUE_LINKING);
2721 236 : t = fold_build2 (MULT_EXPR, double_type_node, factord,
2722 : build_real (double_type_node, dconst2));
2723 236 : t = fold_build2 (MULT_EXPR, double_type_node, t, stopvald);
2724 236 : t = fold_build2 (PLUS_EXPR, double_type_node, t,
2725 : fold_build2 (MULT_EXPR, double_type_node,
2726 : t3, t3));
2727 236 : flag_rounding_math = save_flag_rounding_math;
2728 236 : t = force_gimple_operand_gsi (gsi, t, true, NULL_TREE, false,
2729 : GSI_CONTINUE_LINKING);
2730 236 : if (flag_exceptions
2731 34 : && cfun->can_throw_non_call_exceptions
2732 244 : && operation_could_trap_p (LT_EXPR, true, false, NULL_TREE))
2733 : {
2734 8 : tree tem = fold_build2 (LT_EXPR, boolean_type_node, t,
2735 : build_zero_cst (double_type_node));
2736 8 : tem = force_gimple_operand_gsi (gsi, tem, true, NULL_TREE,
2737 : false, GSI_CONTINUE_LINKING);
2738 8 : cond_stmt = gimple_build_cond (NE_EXPR, tem,
2739 : boolean_false_node,
2740 : NULL_TREE, NULL_TREE);
2741 : }
2742 : else
2743 228 : cond_stmt
2744 228 : = gimple_build_cond (LT_EXPR, t,
2745 : build_zero_cst (double_type_node),
2746 : NULL_TREE, NULL_TREE);
2747 236 : gsi_insert_after (gsi, cond_stmt, GSI_CONTINUE_LINKING);
2748 236 : e = split_block (gsi_bb (*gsi), cond_stmt);
2749 236 : basic_block bb1 = e->src;
2750 236 : e->flags = EDGE_FALSE_VALUE;
2751 236 : e->probability = profile_probability::very_likely ();
2752 236 : *gsi = gsi_after_labels (e->dest);
2753 236 : gcall *call = gimple_build_call_internal (IFN_SQRT, 1, t);
2754 236 : tree sqrtr = create_tmp_var (double_type_node);
2755 236 : gimple_call_set_lhs (call, sqrtr);
2756 236 : gsi_insert_after (gsi, call, GSI_CONTINUE_LINKING);
2757 236 : t = fold_build2 (MINUS_EXPR, double_type_node, sqrtr, t3);
2758 236 : t = fold_build2 (RDIV_EXPR, double_type_node, t, factord);
2759 236 : t = fold_build1 (FIX_TRUNC_EXPR, ulltype, t);
2760 236 : tree c = create_tmp_var (ulltype);
2761 236 : tree d = create_tmp_var (ulltype);
2762 236 : expand_omp_build_assign (gsi, c, t, true);
2763 236 : t = fold_build2 (MINUS_EXPR, ulltype, c,
2764 : build_one_cst (ulltype));
2765 236 : t = fold_build2 (MULT_EXPR, ulltype, c, t);
2766 236 : t = fold_build2 (RSHIFT_EXPR, ulltype, t, integer_one_node);
2767 236 : t = fold_build2 (MULT_EXPR, ulltype,
2768 : fold_convert (ulltype, fd->factor), t);
2769 236 : tree t2
2770 236 : = fold_build2 (MULT_EXPR, ulltype, c,
2771 : fold_convert (ulltype,
2772 : fd->first_inner_iterations));
2773 236 : t = fold_build2 (PLUS_EXPR, ulltype, t, t2);
2774 236 : expand_omp_build_assign (gsi, d, t, true);
2775 236 : t = fold_build2 (MULT_EXPR, ulltype,
2776 : fold_convert (ulltype, fd->factor), c);
2777 236 : t = fold_build2 (PLUS_EXPR, ulltype,
2778 : t, fold_convert (ulltype,
2779 : fd->first_inner_iterations));
2780 236 : t2 = force_gimple_operand_gsi (gsi, t, true, NULL_TREE, false,
2781 : GSI_CONTINUE_LINKING);
2782 236 : cond_stmt = gimple_build_cond (GE_EXPR, stopvalull, d,
2783 : NULL_TREE, NULL_TREE);
2784 236 : gsi_insert_after (gsi, cond_stmt, GSI_CONTINUE_LINKING);
2785 236 : e = split_block (gsi_bb (*gsi), cond_stmt);
2786 236 : basic_block bb2 = e->src;
2787 236 : e->flags = EDGE_TRUE_VALUE;
2788 236 : e->probability = profile_probability::very_likely ();
2789 236 : *gsi = gsi_after_labels (e->dest);
2790 236 : t = fold_build2 (PLUS_EXPR, ulltype, d, t2);
2791 236 : t = force_gimple_operand_gsi (gsi, t, true, NULL_TREE, false,
2792 : GSI_CONTINUE_LINKING);
2793 236 : cond_stmt = gimple_build_cond (GE_EXPR, stopvalull, t,
2794 : NULL_TREE, NULL_TREE);
2795 236 : gsi_insert_after (gsi, cond_stmt, GSI_CONTINUE_LINKING);
2796 236 : e = split_block (gsi_bb (*gsi), cond_stmt);
2797 236 : basic_block bb3 = e->src;
2798 236 : e->flags = EDGE_FALSE_VALUE;
2799 236 : e->probability = profile_probability::very_likely ();
2800 236 : *gsi = gsi_after_labels (e->dest);
2801 236 : t = fold_convert (itype, c);
2802 236 : t = fold_build2 (MULT_EXPR, itype, t, fd->loops[i - 1].step);
2803 236 : t = fold_build2 (PLUS_EXPR, itype, outer_n1, t);
2804 236 : t = force_gimple_operand_gsi (gsi, t, true, NULL_TREE, false,
2805 : GSI_CONTINUE_LINKING);
2806 236 : expand_omp_build_assign (gsi, fd->loops[i - 1].v, t, true);
2807 236 : t2 = fold_build2 (MINUS_EXPR, ulltype, stopvalull, d);
2808 236 : t2 = fold_convert (itype, t2);
2809 236 : t2 = fold_build2 (MULT_EXPR, itype, t2, fd->loops[i].step);
2810 236 : t2 = fold_build2 (PLUS_EXPR, itype, t2, fd->loops[i].n1);
2811 236 : if (fd->loops[i].m1)
2812 : {
2813 195 : t = fold_build2 (MULT_EXPR, itype, t, fd->loops[i].m1);
2814 195 : t2 = fold_build2 (PLUS_EXPR, itype, t2, t);
2815 : }
2816 236 : expand_omp_build_assign (gsi, fd->loops[i].v, t2, true);
2817 236 : e = split_block (gsi_bb (*gsi), gsi_stmt (*gsi));
2818 236 : bb_triang = e->src;
2819 236 : *gsi = gsi_after_labels (e->dest);
2820 236 : remove_edge (e);
2821 236 : e = make_edge (bb1, gsi_bb (*gsi), EDGE_TRUE_VALUE);
2822 236 : e->probability = profile_probability::very_unlikely ();
2823 236 : e = make_edge (bb2, gsi_bb (*gsi), EDGE_FALSE_VALUE);
2824 236 : e->probability = profile_probability::very_unlikely ();
2825 236 : e = make_edge (bb3, gsi_bb (*gsi), EDGE_TRUE_VALUE);
2826 236 : e->probability = profile_probability::very_unlikely ();
2827 :
2828 236 : basic_block bb4 = create_empty_bb (bb0);
2829 236 : add_bb_to_loop (bb4, bb0->loop_father);
2830 236 : e = make_edge (bb0, bb4, EDGE_FALSE_VALUE);
2831 236 : e->probability = profile_probability::unlikely ();
2832 236 : make_edge (bb4, gsi_bb (*gsi), EDGE_FALLTHRU);
2833 236 : set_immediate_dominator (CDI_DOMINATORS, bb4, bb0);
2834 236 : set_immediate_dominator (CDI_DOMINATORS, gsi_bb (*gsi), bb0);
2835 236 : gimple_stmt_iterator gsi2 = gsi_after_labels (bb4);
2836 236 : t2 = fold_build2 (TRUNC_DIV_EXPR, type,
2837 : counts[i], counts[i - 1]);
2838 236 : t2 = force_gimple_operand_gsi (&gsi2, t2, true, NULL_TREE, false,
2839 : GSI_CONTINUE_LINKING);
2840 236 : t = fold_build2 (TRUNC_MOD_EXPR, type, stopval, t2);
2841 236 : t2 = fold_build2 (TRUNC_DIV_EXPR, type, stopval, t2);
2842 236 : t = fold_convert (itype, t);
2843 236 : t2 = fold_convert (itype, t2);
2844 236 : t = fold_build2 (MULT_EXPR, itype, t,
2845 : fold_convert (itype, fd->loops[i].step));
2846 236 : t = fold_build2 (PLUS_EXPR, itype, fd->loops[i].n1, t);
2847 236 : t2 = fold_build2 (MULT_EXPR, itype, t2,
2848 : fold_convert (itype, fd->loops[i - 1].step));
2849 236 : t2 = fold_build2 (PLUS_EXPR, itype, fd->loops[i - 1].n1, t2);
2850 236 : t2 = force_gimple_operand_gsi (&gsi2, t2, false, NULL_TREE,
2851 : false, GSI_CONTINUE_LINKING);
2852 236 : stmt = gimple_build_assign (fd->loops[i - 1].v, t2);
2853 236 : gsi_insert_after (&gsi2, stmt, GSI_CONTINUE_LINKING);
2854 236 : if (fd->loops[i].m1)
2855 : {
2856 195 : t2 = fold_build2 (MULT_EXPR, itype, fd->loops[i].m1,
2857 : fd->loops[i - 1].v);
2858 195 : t = fold_build2 (PLUS_EXPR, itype, t, t2);
2859 : }
2860 236 : t = force_gimple_operand_gsi (&gsi2, t, false, NULL_TREE,
2861 : false, GSI_CONTINUE_LINKING);
2862 236 : stmt = gimple_build_assign (fd->loops[i].v, t);
2863 236 : gsi_insert_after (&gsi2, stmt, GSI_CONTINUE_LINKING);
2864 : }
2865 : /* Fallback implementation. Evaluate the loops in between
2866 : (inclusive) fd->first_nonrect and fd->last_nonrect at
2867 : runtime using temporaries instead of the original iteration
2868 : variables, in the body just bump the counter and compare
2869 : with the desired value. */
2870 376 : gimple_stmt_iterator gsi2 = *gsi;
2871 376 : basic_block entry_bb = gsi_bb (gsi2);
2872 376 : edge e = split_block (entry_bb, gsi_stmt (gsi2));
2873 376 : e = split_block (e->dest, (gimple *) NULL);
2874 376 : basic_block dom_bb = NULL;
2875 376 : basic_block cur_bb = e->src;
2876 376 : basic_block next_bb = e->dest;
2877 376 : entry_bb = e->dest;
2878 376 : *gsi = gsi_after_labels (entry_bb);
2879 :
2880 376 : tree *vs = XALLOCAVEC (tree, fd->last_nonrect);
2881 376 : tree n1 = NULL_TREE, n2 = NULL_TREE;
2882 376 : memset (vs, 0, fd->last_nonrect * sizeof (tree));
2883 :
2884 876 : for (int j = fd->first_nonrect; j <= fd->last_nonrect; j++)
2885 : {
2886 876 : tree itype = TREE_TYPE (fd->loops[j].v);
2887 1752 : bool rect_p = (fd->loops[j].m1 == NULL_TREE
2888 527 : && fd->loops[j].m2 == NULL_TREE
2889 1345 : && !fd->loops[j].non_rect_referenced);
2890 876 : gsi2 = gsi_after_labels (cur_bb);
2891 876 : t = fold_convert (itype, unshare_expr (fd->loops[j].n1));
2892 876 : if (fd->loops[j].m1 == NULL_TREE)
2893 527 : n1 = rect_p ? build_zero_cst (type) : t;
2894 349 : else if (POINTER_TYPE_P (itype))
2895 : {
2896 30 : gcc_assert (integer_onep (fd->loops[j].m1));
2897 30 : t = unshare_expr (fd->loops[j].n1);
2898 30 : n1 = fold_build_pointer_plus (vs[j - fd->loops[j].outer], t);
2899 : }
2900 : else
2901 : {
2902 319 : n1 = fold_convert (itype, unshare_expr (fd->loops[j].m1));
2903 319 : n1 = fold_build2 (MULT_EXPR, itype,
2904 : vs[j - fd->loops[j].outer], n1);
2905 319 : n1 = fold_build2 (PLUS_EXPR, itype, n1, t);
2906 : }
2907 876 : n1 = force_gimple_operand_gsi (&gsi2, n1, true, NULL_TREE,
2908 : true, GSI_SAME_STMT);
2909 876 : if (j < fd->last_nonrect)
2910 : {
2911 907 : vs[j] = create_tmp_reg (rect_p ? type : itype, ".it");
2912 500 : expand_omp_build_assign (&gsi2, vs[j], n1);
2913 : }
2914 876 : t = fold_convert (itype, unshare_expr (fd->loops[j].n2));
2915 876 : if (fd->loops[j].m2 == NULL_TREE)
2916 646 : n2 = rect_p ? counts[j] : t;
2917 230 : else if (POINTER_TYPE_P (itype))
2918 : {
2919 34 : gcc_assert (integer_onep (fd->loops[j].m2));
2920 34 : t = unshare_expr (fd->loops[j].n2);
2921 34 : n2 = fold_build_pointer_plus (vs[j - fd->loops[j].outer], t);
2922 : }
2923 : else
2924 : {
2925 196 : n2 = fold_convert (itype, unshare_expr (fd->loops[j].m2));
2926 196 : n2 = fold_build2 (MULT_EXPR, itype,
2927 : vs[j - fd->loops[j].outer], n2);
2928 196 : n2 = fold_build2 (PLUS_EXPR, itype, n2, t);
2929 : }
2930 876 : n2 = force_gimple_operand_gsi (&gsi2, n2, true, NULL_TREE,
2931 : true, GSI_SAME_STMT);
2932 876 : if (POINTER_TYPE_P (itype))
2933 74 : itype = signed_type_for (itype);
2934 876 : if (j == fd->last_nonrect)
2935 : {
2936 376 : gcond *cond_stmt
2937 376 : = expand_omp_build_cond (&gsi2, fd->loops[i].cond_code,
2938 : n1, n2);
2939 376 : e = split_block (cur_bb, cond_stmt);
2940 376 : e->flags = EDGE_TRUE_VALUE;
2941 376 : edge ne = make_edge (cur_bb, next_bb, EDGE_FALSE_VALUE);
2942 376 : e->probability = profile_probability::likely ().guessed ();
2943 376 : ne->probability = e->probability.invert ();
2944 376 : gsi2 = gsi_after_labels (e->dest);
2945 :
2946 376 : t = build_int_cst (itype, (fd->loops[j].cond_code == LT_EXPR
2947 428 : ? -1 : 1));
2948 376 : t = fold_build2 (PLUS_EXPR, itype,
2949 : fold_convert (itype, fd->loops[j].step), t);
2950 376 : t = fold_build2 (PLUS_EXPR, itype, t,
2951 : fold_convert (itype, n2));
2952 376 : t = fold_build2 (MINUS_EXPR, itype, t,
2953 : fold_convert (itype, n1));
2954 376 : tree step = fold_convert (itype, fd->loops[j].step);
2955 376 : if (TYPE_UNSIGNED (itype)
2956 376 : && fd->loops[j].cond_code == GT_EXPR)
2957 0 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
2958 : fold_build1 (NEGATE_EXPR, itype, t),
2959 : fold_build1 (NEGATE_EXPR, itype, step));
2960 : else
2961 376 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
2962 376 : t = fold_convert (type, t);
2963 376 : t = fold_build2 (PLUS_EXPR, type, idx, t);
2964 376 : t = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
2965 : true, GSI_SAME_STMT);
2966 376 : e = make_edge (e->dest, next_bb, EDGE_FALLTHRU);
2967 376 : set_immediate_dominator (CDI_DOMINATORS, next_bb, cur_bb);
2968 376 : cond_stmt
2969 376 : = gimple_build_cond (LE_EXPR, t, stopval, NULL_TREE,
2970 : NULL_TREE);
2971 376 : gsi_insert_before (&gsi2, cond_stmt, GSI_SAME_STMT);
2972 376 : e = split_block (gsi_bb (gsi2), cond_stmt);
2973 376 : e->flags = EDGE_TRUE_VALUE;
2974 376 : e->probability = profile_probability::likely ().guessed ();
2975 376 : ne = make_edge (e->src, entry_bb, EDGE_FALSE_VALUE);
2976 376 : ne->probability = e->probability.invert ();
2977 376 : gsi2 = gsi_after_labels (e->dest);
2978 376 : expand_omp_build_assign (&gsi2, idx, t);
2979 376 : set_immediate_dominator (CDI_DOMINATORS, entry_bb, dom_bb);
2980 376 : break;
2981 : }
2982 500 : e = split_block (cur_bb, last_nondebug_stmt (cur_bb));
2983 :
2984 500 : basic_block new_cur_bb = create_empty_bb (cur_bb);
2985 500 : add_bb_to_loop (new_cur_bb, cur_bb->loop_father);
2986 :
2987 500 : gsi2 = gsi_after_labels (e->dest);
2988 500 : if (rect_p)
2989 93 : t = fold_build2 (PLUS_EXPR, type, vs[j],
2990 : build_one_cst (type));
2991 : else
2992 : {
2993 407 : tree step
2994 407 : = fold_convert (itype, unshare_expr (fd->loops[j].step));
2995 407 : if (POINTER_TYPE_P (vtype))
2996 38 : t = fold_build_pointer_plus (vs[j], step);
2997 : else
2998 369 : t = fold_build2 (PLUS_EXPR, itype, vs[j], step);
2999 : }
3000 500 : t = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
3001 : true, GSI_SAME_STMT);
3002 500 : expand_omp_build_assign (&gsi2, vs[j], t);
3003 :
3004 500 : edge ne = split_block (e->dest, last_nondebug_stmt (e->dest));
3005 500 : gsi2 = gsi_after_labels (ne->dest);
3006 :
3007 500 : gcond *cond_stmt;
3008 500 : if (next_bb == entry_bb)
3009 : /* No need to actually check the outermost condition. */
3010 376 : cond_stmt
3011 376 : = gimple_build_cond (EQ_EXPR, boolean_true_node,
3012 : boolean_true_node,
3013 : NULL_TREE, NULL_TREE);
3014 : else
3015 124 : cond_stmt
3016 155 : = gimple_build_cond (rect_p ? LT_EXPR
3017 31 : : fd->loops[j].cond_code,
3018 : vs[j], n2, NULL_TREE, NULL_TREE);
3019 500 : gsi_insert_before (&gsi2, cond_stmt, GSI_SAME_STMT);
3020 500 : edge e3, e4;
3021 500 : if (next_bb == entry_bb)
3022 : {
3023 376 : e3 = find_edge (ne->dest, next_bb);
3024 376 : e3->flags = EDGE_FALSE_VALUE;
3025 376 : dom_bb = ne->dest;
3026 : }
3027 : else
3028 124 : e3 = make_edge (ne->dest, next_bb, EDGE_FALSE_VALUE);
3029 500 : e4 = make_edge (ne->dest, new_cur_bb, EDGE_TRUE_VALUE);
3030 500 : e4->probability = profile_probability::likely ().guessed ();
3031 500 : e3->probability = e4->probability.invert ();
3032 500 : basic_block esrc = e->src;
3033 500 : make_edge (e->src, ne->dest, EDGE_FALLTHRU);
3034 500 : cur_bb = new_cur_bb;
3035 500 : basic_block latch_bb = next_bb;
3036 500 : next_bb = e->dest;
3037 500 : remove_edge (e);
3038 500 : set_immediate_dominator (CDI_DOMINATORS, ne->dest, esrc);
3039 500 : set_immediate_dominator (CDI_DOMINATORS, latch_bb, ne->dest);
3040 500 : set_immediate_dominator (CDI_DOMINATORS, cur_bb, ne->dest);
3041 : }
3042 1252 : for (int j = fd->last_nonrect; j >= fd->first_nonrect; j--)
3043 : {
3044 876 : tree vtype = TREE_TYPE (fd->loops[j].v);
3045 876 : tree itype = vtype;
3046 876 : if (POINTER_TYPE_P (itype))
3047 74 : itype = signed_type_for (itype);
3048 1752 : bool rect_p = (fd->loops[j].m1 == NULL_TREE
3049 527 : && fd->loops[j].m2 == NULL_TREE
3050 1345 : && !fd->loops[j].non_rect_referenced);
3051 876 : if (j == fd->last_nonrect)
3052 : {
3053 376 : t = fold_build2 (MINUS_EXPR, type, stopval, idx);
3054 376 : t = fold_convert (itype, t);
3055 376 : tree t2
3056 376 : = fold_convert (itype, unshare_expr (fd->loops[j].step));
3057 376 : t = fold_build2 (MULT_EXPR, itype, t, t2);
3058 376 : if (POINTER_TYPE_P (vtype))
3059 32 : t = fold_build_pointer_plus (n1, t);
3060 : else
3061 344 : t = fold_build2 (PLUS_EXPR, itype, n1, t);
3062 : }
3063 500 : else if (rect_p)
3064 : {
3065 93 : t = fold_convert (itype, vs[j]);
3066 93 : t = fold_build2 (MULT_EXPR, itype, t,
3067 : fold_convert (itype, fd->loops[j].step));
3068 93 : if (POINTER_TYPE_P (vtype))
3069 4 : t = fold_build_pointer_plus (fd->loops[j].n1, t);
3070 : else
3071 89 : t = fold_build2 (PLUS_EXPR, itype, fd->loops[j].n1, t);
3072 : }
3073 : else
3074 407 : t = vs[j];
3075 876 : t = force_gimple_operand_gsi (gsi, t, false,
3076 : NULL_TREE, true,
3077 : GSI_SAME_STMT);
3078 876 : stmt = gimple_build_assign (fd->loops[j].v, t);
3079 876 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
3080 : }
3081 376 : if (gsi_end_p (*gsi))
3082 752 : *gsi = gsi_last_bb (gsi_bb (*gsi));
3083 : else
3084 0 : gsi_prev (gsi);
3085 376 : if (bb_triang)
3086 : {
3087 236 : e = split_block (gsi_bb (*gsi), gsi_stmt (*gsi));
3088 236 : make_edge (bb_triang, e->dest, EDGE_FALLTHRU);
3089 236 : *gsi = gsi_after_labels (e->dest);
3090 236 : if (!gsi_end_p (*gsi))
3091 0 : gsi_insert_before (gsi, gimple_build_nop (), GSI_NEW_STMT);
3092 236 : set_immediate_dominator (CDI_DOMINATORS, e->dest, bb_triang_dom);
3093 : }
3094 : }
3095 : else
3096 : {
3097 14324 : t = fold_convert (itype, t);
3098 14324 : t = fold_build2 (MULT_EXPR, itype, t,
3099 : fold_convert (itype, fd->loops[i].step));
3100 14324 : if (POINTER_TYPE_P (vtype))
3101 1617 : t = fold_build_pointer_plus (fd->loops[i].n1, t);
3102 : else
3103 12707 : t = fold_build2 (PLUS_EXPR, itype, fd->loops[i].n1, t);
3104 14324 : t = force_gimple_operand_gsi (gsi, t,
3105 14324 : DECL_P (fd->loops[i].v)
3106 14324 : && TREE_ADDRESSABLE (fd->loops[i].v),
3107 : NULL_TREE, false,
3108 : GSI_CONTINUE_LINKING);
3109 14324 : stmt = gimple_build_assign (fd->loops[i].v, t);
3110 14324 : gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
3111 : }
3112 14700 : if (i != 0 && (i != fd->last_nonrect || fd->first_nonrect))
3113 : {
3114 9294 : t = fold_build2 (TRUNC_DIV_EXPR, type, tem, counts[i]);
3115 9294 : t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE,
3116 : false, GSI_CONTINUE_LINKING);
3117 9294 : stmt = gimple_build_assign (tem, t);
3118 9294 : gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
3119 : }
3120 14700 : if (i == fd->last_nonrect)
3121 376 : i = fd->first_nonrect;
3122 : }
3123 5406 : if (fd->non_rect)
3124 1351 : for (i = 0; i <= fd->last_nonrect; i++)
3125 975 : if (fd->loops[i].m2)
3126 : {
3127 230 : tree itype = TREE_TYPE (fd->loops[i].v);
3128 :
3129 230 : tree t;
3130 230 : if (POINTER_TYPE_P (itype))
3131 : {
3132 34 : gcc_assert (integer_onep (fd->loops[i].m2));
3133 34 : t = fold_build_pointer_plus (fd->loops[i - fd->loops[i].outer].v,
3134 : unshare_expr (fd->loops[i].n2));
3135 : }
3136 : else
3137 : {
3138 196 : t = fold_convert (itype, unshare_expr (fd->loops[i].m2));
3139 196 : t = fold_build2 (MULT_EXPR, itype,
3140 : fd->loops[i - fd->loops[i].outer].v, t);
3141 196 : t = fold_build2 (PLUS_EXPR, itype, t,
3142 : fold_convert (itype,
3143 : unshare_expr (fd->loops[i].n2)));
3144 : }
3145 230 : nonrect_bounds[i] = create_tmp_reg (itype, ".bound");
3146 230 : t = force_gimple_operand_gsi (gsi, t, false,
3147 : NULL_TREE, false,
3148 : GSI_CONTINUE_LINKING);
3149 230 : stmt = gimple_build_assign (nonrect_bounds[i], t);
3150 230 : gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
3151 : }
3152 : }
3153 :
3154 : /* Helper function for expand_omp_for_*. Generate code like:
3155 : L10:
3156 : V3 += STEP3;
3157 : if (V3 cond3 N32) goto BODY_BB; else goto L11;
3158 : L11:
3159 : V3 = N31;
3160 : V2 += STEP2;
3161 : if (V2 cond2 N22) goto BODY_BB; else goto L12;
3162 : L12:
3163 : V2 = N21;
3164 : V1 += STEP1;
3165 : goto BODY_BB;
3166 : For non-rectangular loops, use temporaries stored in nonrect_bounds
3167 : for the upper bounds if M?2 multiplier is present. Given e.g.
3168 : for (V1 = N11; V1 cond1 N12; V1 += STEP1)
3169 : for (V2 = N21; V2 cond2 N22; V2 += STEP2)
3170 : for (V3 = N31; V3 cond3 N32; V3 += STEP3)
3171 : for (V4 = N41 + M41 * V2; V4 cond4 N42 + M42 * V2; V4 += STEP4)
3172 : do:
3173 : L10:
3174 : V4 += STEP4;
3175 : if (V4 cond4 NONRECT_BOUND4) goto BODY_BB; else goto L11;
3176 : L11:
3177 : V4 = N41 + M41 * V2; // This can be left out if the loop
3178 : // refers to the immediate parent loop
3179 : V3 += STEP3;
3180 : if (V3 cond3 N32) goto BODY_BB; else goto L12;
3181 : L12:
3182 : V3 = N31;
3183 : V2 += STEP2;
3184 : if (V2 cond2 N22) goto L120; else goto L13;
3185 : L120:
3186 : V4 = N41 + M41 * V2;
3187 : NONRECT_BOUND4 = N42 + M42 * V2;
3188 : if (V4 cond4 NONRECT_BOUND4) goto BODY_BB; else goto L12;
3189 : L13:
3190 : V2 = N21;
3191 : V1 += STEP1;
3192 : goto L120; */
3193 :
3194 : static basic_block
3195 2593 : extract_omp_for_update_vars (struct omp_for_data *fd, tree *nonrect_bounds,
3196 : basic_block cont_bb, basic_block body_bb)
3197 : {
3198 2593 : basic_block last_bb, bb, collapse_bb = NULL;
3199 2593 : int i;
3200 2593 : gimple_stmt_iterator gsi;
3201 2593 : edge e;
3202 2593 : tree t;
3203 2593 : gimple *stmt;
3204 :
3205 2593 : last_bb = cont_bb;
3206 9736 : for (i = fd->collapse - 1; i >= 0; i--)
3207 : {
3208 7143 : tree vtype = TREE_TYPE (fd->loops[i].v);
3209 :
3210 7143 : bb = create_empty_bb (last_bb);
3211 7143 : add_bb_to_loop (bb, last_bb->loop_father);
3212 7143 : gsi = gsi_start_bb (bb);
3213 :
3214 7143 : if (i < fd->collapse - 1)
3215 : {
3216 4550 : e = make_edge (last_bb, bb, EDGE_FALSE_VALUE);
3217 4550 : e->probability = profile_probability::guessed_always () / 8;
3218 :
3219 4550 : struct omp_for_data_loop *l = &fd->loops[i + 1];
3220 4550 : if (l->m1 == NULL_TREE || l->outer != 1)
3221 : {
3222 4342 : t = l->n1;
3223 4342 : if (l->m1)
3224 : {
3225 56 : if (POINTER_TYPE_P (TREE_TYPE (l->v)))
3226 3 : t = fold_build_pointer_plus (fd->loops[i + 1 - l->outer].v,
3227 : t);
3228 : else
3229 : {
3230 53 : tree t2
3231 53 : = fold_build2 (MULT_EXPR, TREE_TYPE (t),
3232 : fd->loops[i + 1 - l->outer].v, l->m1);
3233 53 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (t), t2, t);
3234 : }
3235 : }
3236 4342 : t = force_gimple_operand_gsi (&gsi, t,
3237 4342 : DECL_P (l->v)
3238 4342 : && TREE_ADDRESSABLE (l->v),
3239 : NULL_TREE, false,
3240 : GSI_CONTINUE_LINKING);
3241 4342 : stmt = gimple_build_assign (l->v, t);
3242 4342 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
3243 : }
3244 : }
3245 : else
3246 : collapse_bb = bb;
3247 :
3248 7143 : set_immediate_dominator (CDI_DOMINATORS, bb, last_bb);
3249 :
3250 7143 : if (POINTER_TYPE_P (vtype))
3251 902 : t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step);
3252 : else
3253 6241 : t = fold_build2 (PLUS_EXPR, vtype, fd->loops[i].v, fd->loops[i].step);
3254 7143 : t = force_gimple_operand_gsi (&gsi, t,
3255 7143 : DECL_P (fd->loops[i].v)
3256 7143 : && TREE_ADDRESSABLE (fd->loops[i].v),
3257 : NULL_TREE, false, GSI_CONTINUE_LINKING);
3258 7143 : stmt = gimple_build_assign (fd->loops[i].v, t);
3259 7143 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
3260 :
3261 7143 : if (fd->loops[i].non_rect_referenced)
3262 : {
3263 308 : basic_block update_bb = NULL, prev_bb = NULL;
3264 714 : for (int j = i + 1; j <= fd->last_nonrect; j++)
3265 406 : if (j - fd->loops[j].outer == i)
3266 : {
3267 308 : tree n1, n2;
3268 308 : struct omp_for_data_loop *l = &fd->loops[j];
3269 308 : basic_block this_bb = create_empty_bb (last_bb);
3270 308 : add_bb_to_loop (this_bb, last_bb->loop_father);
3271 308 : gimple_stmt_iterator gsi2 = gsi_start_bb (this_bb);
3272 308 : if (prev_bb)
3273 : {
3274 0 : e = make_edge (prev_bb, this_bb, EDGE_TRUE_VALUE);
3275 0 : e->probability
3276 0 : = profile_probability::guessed_always ().apply_scale (7,
3277 : 8);
3278 0 : set_immediate_dominator (CDI_DOMINATORS, this_bb, prev_bb);
3279 : }
3280 308 : if (l->m1)
3281 : {
3282 264 : if (POINTER_TYPE_P (TREE_TYPE (l->v)))
3283 26 : t = fold_build_pointer_plus (fd->loops[i].v, l->n1);
3284 : else
3285 : {
3286 238 : t = fold_build2 (MULT_EXPR, TREE_TYPE (l->m1), l->m1,
3287 : fd->loops[i].v);
3288 238 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (l->v),
3289 : t, l->n1);
3290 : }
3291 264 : n1 = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
3292 : false,
3293 : GSI_CONTINUE_LINKING);
3294 264 : stmt = gimple_build_assign (l->v, n1);
3295 264 : gsi_insert_after (&gsi2, stmt, GSI_CONTINUE_LINKING);
3296 264 : n1 = l->v;
3297 : }
3298 : else
3299 44 : n1 = force_gimple_operand_gsi (&gsi2, l->n1, true,
3300 : NULL_TREE, false,
3301 : GSI_CONTINUE_LINKING);
3302 308 : if (l->m2)
3303 : {
3304 204 : if (POINTER_TYPE_P (TREE_TYPE (l->v)))
3305 30 : t = fold_build_pointer_plus (fd->loops[i].v, l->n2);
3306 : else
3307 : {
3308 174 : t = fold_build2 (MULT_EXPR, TREE_TYPE (l->m2), l->m2,
3309 : fd->loops[i].v);
3310 174 : t = fold_build2 (PLUS_EXPR,
3311 : TREE_TYPE (nonrect_bounds[j]),
3312 : t, unshare_expr (l->n2));
3313 : }
3314 204 : n2 = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
3315 : false,
3316 : GSI_CONTINUE_LINKING);
3317 204 : stmt = gimple_build_assign (nonrect_bounds[j], n2);
3318 204 : gsi_insert_after (&gsi2, stmt, GSI_CONTINUE_LINKING);
3319 204 : n2 = nonrect_bounds[j];
3320 : }
3321 : else
3322 104 : n2 = force_gimple_operand_gsi (&gsi2, unshare_expr (l->n2),
3323 : true, NULL_TREE, false,
3324 : GSI_CONTINUE_LINKING);
3325 308 : gcond *cond_stmt
3326 308 : = gimple_build_cond (l->cond_code, n1, n2,
3327 : NULL_TREE, NULL_TREE);
3328 308 : gsi_insert_after (&gsi2, cond_stmt, GSI_CONTINUE_LINKING);
3329 308 : if (update_bb == NULL)
3330 308 : update_bb = this_bb;
3331 308 : e = make_edge (this_bb, bb, EDGE_FALSE_VALUE);
3332 308 : e->probability = profile_probability::guessed_always () / 8;
3333 308 : if (prev_bb == NULL)
3334 308 : set_immediate_dominator (CDI_DOMINATORS, this_bb, bb);
3335 308 : prev_bb = this_bb;
3336 : }
3337 308 : e = make_edge (prev_bb, body_bb, EDGE_TRUE_VALUE);
3338 308 : e->probability
3339 308 : = profile_probability::guessed_always ().apply_scale (7, 8);
3340 308 : body_bb = update_bb;
3341 : }
3342 :
3343 7143 : if (i > 0)
3344 : {
3345 4550 : if (fd->loops[i].m2)
3346 204 : t = nonrect_bounds[i];
3347 : else
3348 4346 : t = unshare_expr (fd->loops[i].n2);
3349 4550 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
3350 : false, GSI_CONTINUE_LINKING);
3351 4550 : tree v = fd->loops[i].v;
3352 4550 : if (DECL_P (v) && TREE_ADDRESSABLE (v))
3353 0 : v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE,
3354 : false, GSI_CONTINUE_LINKING);
3355 4550 : t = build2 (fd->loops[i].cond_code, boolean_type_node, v, t);
3356 4550 : stmt = gimple_build_cond_empty (t);
3357 4550 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
3358 4550 : if (walk_tree (gimple_cond_lhs_ptr (as_a <gcond *> (stmt)),
3359 : expand_omp_regimplify_p, NULL, NULL)
3360 4550 : || walk_tree (gimple_cond_rhs_ptr (as_a <gcond *> (stmt)),
3361 : expand_omp_regimplify_p, NULL, NULL))
3362 4 : gimple_regimplify_operands (stmt, &gsi);
3363 4550 : e = make_edge (bb, body_bb, EDGE_TRUE_VALUE);
3364 4550 : e->probability = profile_probability::guessed_always ().apply_scale (7, 8);
3365 : }
3366 : else
3367 2593 : make_edge (bb, body_bb, EDGE_FALLTHRU);
3368 7143 : set_immediate_dominator (CDI_DOMINATORS, bb, last_bb);
3369 7143 : last_bb = bb;
3370 : }
3371 :
3372 2593 : return collapse_bb;
3373 : }
3374 :
3375 : /* Expand #pragma omp ordered depend(source). */
3376 :
3377 : static void
3378 335 : expand_omp_ordered_source (gimple_stmt_iterator *gsi, struct omp_for_data *fd,
3379 : tree *counts, location_t loc)
3380 : {
3381 56 : enum built_in_function source_ix
3382 335 : = fd->iter_type == long_integer_type_node
3383 335 : ? BUILT_IN_GOMP_DOACROSS_POST : BUILT_IN_GOMP_DOACROSS_ULL_POST;
3384 335 : gimple *g
3385 335 : = gimple_build_call (builtin_decl_explicit (source_ix), 1,
3386 335 : build_fold_addr_expr (counts[fd->ordered]));
3387 335 : gimple_set_location (g, loc);
3388 335 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
3389 335 : }
3390 :
3391 : /* Expand a single depend from #pragma omp ordered depend(sink:...). */
3392 :
3393 : static void
3394 466 : expand_omp_ordered_sink (gimple_stmt_iterator *gsi, struct omp_for_data *fd,
3395 : tree *counts, tree c, location_t loc,
3396 : basic_block cont_bb)
3397 : {
3398 466 : auto_vec<tree, 10> args;
3399 73 : enum built_in_function sink_ix
3400 466 : = fd->iter_type == long_integer_type_node
3401 466 : ? BUILT_IN_GOMP_DOACROSS_WAIT : BUILT_IN_GOMP_DOACROSS_ULL_WAIT;
3402 466 : tree t, off, coff = NULL_TREE, deps = OMP_CLAUSE_DECL (c), cond = NULL_TREE;
3403 466 : int i;
3404 466 : gimple_stmt_iterator gsi2 = *gsi;
3405 466 : bool warned_step = false;
3406 :
3407 466 : if (deps == NULL)
3408 : {
3409 : /* Handle doacross(sink: omp_cur_iteration - 1). */
3410 52 : gsi_prev (&gsi2);
3411 52 : edge e1 = split_block (gsi_bb (gsi2), gsi_stmt (gsi2));
3412 52 : edge e2 = split_block_after_labels (e1->dest);
3413 52 : gsi2 = gsi_after_labels (e1->dest);
3414 52 : *gsi = gsi_last_bb (e1->src);
3415 52 : gimple_stmt_iterator gsi3 = *gsi;
3416 :
3417 52 : if (counts[fd->collapse - 1])
3418 : {
3419 8 : gcc_assert (fd->collapse == 1);
3420 8 : t = counts[fd->collapse - 1];
3421 : }
3422 44 : else if (fd->collapse > 1)
3423 28 : t = fd->loop.v;
3424 : else
3425 : {
3426 16 : t = fold_build2 (MINUS_EXPR, TREE_TYPE (fd->loops[0].v),
3427 : fd->loops[0].v, fd->loops[0].n1);
3428 16 : t = fold_convert (fd->iter_type, t);
3429 : }
3430 :
3431 52 : t = force_gimple_operand_gsi (gsi, t, true, NULL_TREE,
3432 : false, GSI_CONTINUE_LINKING);
3433 52 : gsi_insert_after (gsi, gimple_build_cond (NE_EXPR, t,
3434 52 : build_zero_cst (TREE_TYPE (t)),
3435 : NULL_TREE, NULL_TREE),
3436 : GSI_NEW_STMT);
3437 :
3438 52 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (t), t,
3439 : build_minus_one_cst (TREE_TYPE (t)));
3440 52 : t = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
3441 : true, GSI_SAME_STMT);
3442 52 : args.safe_push (t);
3443 281 : for (i = fd->collapse; i < fd->ordered; i++)
3444 : {
3445 229 : t = counts[fd->ordered + 2 + (i - fd->collapse)];
3446 229 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (t), t,
3447 : build_minus_one_cst (TREE_TYPE (t)));
3448 229 : t = fold_convert (fd->iter_type, t);
3449 229 : t = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
3450 : true, GSI_SAME_STMT);
3451 229 : args.safe_push (t);
3452 : }
3453 :
3454 52 : gimple *g = gimple_build_call_vec (builtin_decl_explicit (sink_ix),
3455 : args);
3456 52 : gimple_set_location (g, loc);
3457 52 : gsi_insert_before (&gsi2, g, GSI_SAME_STMT);
3458 :
3459 52 : edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE);
3460 52 : e3->probability = profile_probability::guessed_always () / 8;
3461 52 : e1->probability = e3->probability.invert ();
3462 52 : e1->flags = EDGE_TRUE_VALUE;
3463 52 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
3464 :
3465 52 : if (fd->ordered > fd->collapse && cont_bb)
3466 : {
3467 33 : if (counts[fd->ordered + 1] == NULL_TREE)
3468 33 : counts[fd->ordered + 1]
3469 33 : = create_tmp_var (boolean_type_node, ".first");
3470 :
3471 33 : edge e4;
3472 33 : if (gsi_end_p (gsi3))
3473 25 : e4 = split_block_after_labels (e1->src);
3474 : else
3475 : {
3476 8 : gsi_prev (&gsi3);
3477 8 : e4 = split_block (gsi_bb (gsi3), gsi_stmt (gsi3));
3478 : }
3479 33 : gsi3 = gsi_last_bb (e4->src);
3480 :
3481 33 : gsi_insert_after (&gsi3,
3482 33 : gimple_build_cond (NE_EXPR,
3483 33 : counts[fd->ordered + 1],
3484 : boolean_false_node,
3485 : NULL_TREE, NULL_TREE),
3486 : GSI_NEW_STMT);
3487 :
3488 33 : edge e5 = make_edge (e4->src, e2->dest, EDGE_FALSE_VALUE);
3489 33 : e4->probability = profile_probability::guessed_always () / 8;
3490 33 : e5->probability = e4->probability.invert ();
3491 33 : e4->flags = EDGE_TRUE_VALUE;
3492 33 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e4->src);
3493 : }
3494 :
3495 52 : *gsi = gsi_after_labels (e2->dest);
3496 52 : return;
3497 : }
3498 504 : for (i = 0; i < fd->ordered; i++)
3499 : {
3500 502 : tree step = NULL_TREE;
3501 502 : off = TREE_PURPOSE (deps);
3502 502 : if (TREE_CODE (off) == TRUNC_DIV_EXPR)
3503 : {
3504 83 : step = TREE_OPERAND (off, 1);
3505 83 : off = TREE_OPERAND (off, 0);
3506 : }
3507 502 : if (!integer_zerop (off))
3508 : {
3509 412 : gcc_assert (fd->loops[i].cond_code == LT_EXPR
3510 : || fd->loops[i].cond_code == GT_EXPR);
3511 412 : bool forward = fd->loops[i].cond_code == LT_EXPR;
3512 412 : if (step)
3513 : {
3514 : /* Non-simple Fortran DO loops. If step is variable,
3515 : we don't know at compile even the direction, so can't
3516 : warn. */
3517 83 : if (TREE_CODE (step) != INTEGER_CST)
3518 : break;
3519 0 : forward = tree_int_cst_sgn (step) != -1;
3520 : }
3521 329 : if (forward ^ OMP_CLAUSE_DOACROSS_SINK_NEGATIVE (deps))
3522 17 : warning_at (loc, OPT_Wopenmp,
3523 : "%qs clause with %<sink%> modifier "
3524 : "waiting for lexically later iteration",
3525 17 : OMP_CLAUSE_DOACROSS_DEPEND (c)
3526 : ? "depend" : "doacross");
3527 : break;
3528 : }
3529 90 : deps = TREE_CHAIN (deps);
3530 : }
3531 : /* If all offsets corresponding to the collapsed loops are zero,
3532 : this depend clause can be ignored. FIXME: but there is still a
3533 : flush needed. We need to emit one __sync_synchronize () for it
3534 : though (perhaps conditionally)? Solve this together with the
3535 : conservative dependence folding optimization.
3536 : if (i >= fd->collapse)
3537 : return; */
3538 :
3539 414 : deps = OMP_CLAUSE_DECL (c);
3540 414 : gsi_prev (&gsi2);
3541 414 : edge e1 = split_block (gsi_bb (gsi2), gsi_stmt (gsi2));
3542 414 : edge e2 = split_block_after_labels (e1->dest);
3543 :
3544 414 : gsi2 = gsi_after_labels (e1->dest);
3545 414 : *gsi = gsi_last_bb (e1->src);
3546 2806 : for (i = 0; i < fd->ordered; i++)
3547 : {
3548 2392 : tree itype = TREE_TYPE (fd->loops[i].v);
3549 2392 : tree step = NULL_TREE;
3550 2392 : tree orig_off = NULL_TREE;
3551 2392 : if (POINTER_TYPE_P (itype))
3552 9 : itype = sizetype;
3553 2392 : if (i)
3554 1978 : deps = TREE_CHAIN (deps);
3555 2392 : off = TREE_PURPOSE (deps);
3556 2392 : if (TREE_CODE (off) == TRUNC_DIV_EXPR)
3557 : {
3558 204 : step = TREE_OPERAND (off, 1);
3559 204 : off = TREE_OPERAND (off, 0);
3560 204 : gcc_assert (fd->loops[i].cond_code == LT_EXPR
3561 : && integer_onep (fd->loops[i].step)
3562 : && !POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v)));
3563 : }
3564 2392 : tree s = fold_convert_loc (loc, itype, step ? step : fd->loops[i].step);
3565 2392 : if (step)
3566 : {
3567 204 : off = fold_convert_loc (loc, itype, off);
3568 204 : orig_off = off;
3569 204 : off = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, off, s);
3570 : }
3571 :
3572 2392 : if (integer_zerop (off))
3573 1516 : t = boolean_true_node;
3574 : else
3575 : {
3576 876 : tree a;
3577 876 : tree co = fold_convert_loc (loc, itype, off);
3578 876 : if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v)))
3579 : {
3580 9 : if (OMP_CLAUSE_DOACROSS_SINK_NEGATIVE (deps))
3581 0 : co = fold_build1_loc (loc, NEGATE_EXPR, itype, co);
3582 9 : a = fold_build2_loc (loc, POINTER_PLUS_EXPR,
3583 9 : TREE_TYPE (fd->loops[i].v), fd->loops[i].v,
3584 : co);
3585 : }
3586 867 : else if (OMP_CLAUSE_DOACROSS_SINK_NEGATIVE (deps))
3587 640 : a = fold_build2_loc (loc, MINUS_EXPR, TREE_TYPE (fd->loops[i].v),
3588 : fd->loops[i].v, co);
3589 : else
3590 227 : a = fold_build2_loc (loc, PLUS_EXPR, TREE_TYPE (fd->loops[i].v),
3591 : fd->loops[i].v, co);
3592 876 : if (step)
3593 : {
3594 204 : tree t1, t2;
3595 204 : if (OMP_CLAUSE_DOACROSS_SINK_NEGATIVE (deps))
3596 149 : t1 = fold_build2_loc (loc, GE_EXPR, boolean_type_node, a,
3597 149 : fd->loops[i].n1);
3598 : else
3599 55 : t1 = fold_build2_loc (loc, LT_EXPR, boolean_type_node, a,
3600 55 : fd->loops[i].n2);
3601 204 : if (OMP_CLAUSE_DOACROSS_SINK_NEGATIVE (deps))
3602 149 : t2 = fold_build2_loc (loc, LT_EXPR, boolean_type_node, a,
3603 149 : fd->loops[i].n2);
3604 : else
3605 55 : t2 = fold_build2_loc (loc, GE_EXPR, boolean_type_node, a,
3606 55 : fd->loops[i].n1);
3607 204 : t = fold_build2_loc (loc, LT_EXPR, boolean_type_node,
3608 204 : step, build_int_cst (TREE_TYPE (step), 0));
3609 204 : if (TREE_CODE (step) != INTEGER_CST)
3610 : {
3611 204 : t1 = unshare_expr (t1);
3612 204 : t1 = force_gimple_operand_gsi (gsi, t1, true, NULL_TREE,
3613 : false, GSI_CONTINUE_LINKING);
3614 204 : t2 = unshare_expr (t2);
3615 204 : t2 = force_gimple_operand_gsi (gsi, t2, true, NULL_TREE,
3616 : false, GSI_CONTINUE_LINKING);
3617 : }
3618 204 : t = fold_build3_loc (loc, COND_EXPR, boolean_type_node,
3619 : t, t2, t1);
3620 : }
3621 672 : else if (fd->loops[i].cond_code == LT_EXPR)
3622 : {
3623 520 : if (OMP_CLAUSE_DOACROSS_SINK_NEGATIVE (deps))
3624 433 : t = fold_build2_loc (loc, GE_EXPR, boolean_type_node, a,
3625 : fd->loops[i].n1);
3626 : else
3627 87 : t = fold_build2_loc (loc, LT_EXPR, boolean_type_node, a,
3628 : fd->loops[i].n2);
3629 : }
3630 152 : else if (OMP_CLAUSE_DOACROSS_SINK_NEGATIVE (deps))
3631 58 : t = fold_build2_loc (loc, GT_EXPR, boolean_type_node, a,
3632 : fd->loops[i].n2);
3633 : else
3634 94 : t = fold_build2_loc (loc, LE_EXPR, boolean_type_node, a,
3635 : fd->loops[i].n1);
3636 : }
3637 2392 : if (cond)
3638 1978 : cond = fold_build2_loc (loc, BIT_AND_EXPR, boolean_type_node, cond, t);
3639 : else
3640 414 : cond = t;
3641 :
3642 2392 : off = fold_convert_loc (loc, itype, off);
3643 :
3644 2392 : if (step
3645 4411 : || (fd->loops[i].cond_code == LT_EXPR
3646 2188 : ? !integer_onep (fd->loops[i].step)
3647 169 : : !integer_minus_onep (fd->loops[i].step)))
3648 : {
3649 382 : if (step == NULL_TREE
3650 178 : && TYPE_UNSIGNED (itype)
3651 393 : && fd->loops[i].cond_code == GT_EXPR)
3652 9 : t = fold_build2_loc (loc, TRUNC_MOD_EXPR, itype, off,
3653 : fold_build1_loc (loc, NEGATE_EXPR, itype,
3654 : s));
3655 : else
3656 542 : t = fold_build2_loc (loc, TRUNC_MOD_EXPR, itype,
3657 : orig_off ? orig_off : off, s);
3658 382 : t = fold_build2_loc (loc, EQ_EXPR, boolean_type_node, t,
3659 : build_int_cst (itype, 0));
3660 382 : if (integer_zerop (t) && !warned_step)
3661 : {
3662 5 : warning_at (loc, OPT_Wopenmp,
3663 : "%qs clause with %<sink%> modifier refers to "
3664 : "iteration never in the iteration space",
3665 5 : OMP_CLAUSE_DOACROSS_DEPEND (c)
3666 : ? "depend" : "doacross");
3667 5 : warned_step = true;
3668 : }
3669 382 : cond = fold_build2_loc (loc, BIT_AND_EXPR, boolean_type_node,
3670 : cond, t);
3671 : }
3672 :
3673 2392 : if (i <= fd->collapse - 1 && fd->collapse > 1)
3674 322 : t = fd->loop.v;
3675 2070 : else if (counts[i])
3676 166 : t = counts[i];
3677 : else
3678 : {
3679 1904 : t = fold_build2_loc (loc, MINUS_EXPR, TREE_TYPE (fd->loops[i].v),
3680 1904 : fd->loops[i].v, fd->loops[i].n1);
3681 1904 : t = fold_convert_loc (loc, fd->iter_type, t);
3682 : }
3683 2392 : if (step)
3684 : /* We have divided off by step already earlier. */;
3685 2188 : else if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR)
3686 9 : off = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, off,
3687 : fold_build1_loc (loc, NEGATE_EXPR, itype,
3688 : s));
3689 : else
3690 2179 : off = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, off, s);
3691 2392 : if (OMP_CLAUSE_DOACROSS_SINK_NEGATIVE (deps))
3692 640 : off = fold_build1_loc (loc, NEGATE_EXPR, itype, off);
3693 2392 : off = fold_convert_loc (loc, fd->iter_type, off);
3694 2392 : if (i <= fd->collapse - 1 && fd->collapse > 1)
3695 : {
3696 322 : if (i)
3697 161 : off = fold_build2_loc (loc, PLUS_EXPR, fd->iter_type, coff,
3698 : off);
3699 322 : if (i < fd->collapse - 1)
3700 : {
3701 322 : coff = fold_build2_loc (loc, MULT_EXPR, fd->iter_type, off,
3702 161 : counts[i]);
3703 161 : continue;
3704 : }
3705 : }
3706 2231 : off = unshare_expr (off);
3707 2231 : t = fold_build2_loc (loc, PLUS_EXPR, fd->iter_type, t, off);
3708 2231 : t = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
3709 : true, GSI_SAME_STMT);
3710 2231 : args.safe_push (t);
3711 : }
3712 414 : gimple *g = gimple_build_call_vec (builtin_decl_explicit (sink_ix), args);
3713 414 : gimple_set_location (g, loc);
3714 414 : gsi_insert_before (&gsi2, g, GSI_SAME_STMT);
3715 :
3716 414 : cond = unshare_expr (cond);
3717 414 : cond = force_gimple_operand_gsi (gsi, cond, true, NULL_TREE, false,
3718 : GSI_CONTINUE_LINKING);
3719 414 : gsi_insert_after (gsi, gimple_build_cond_empty (cond), GSI_NEW_STMT);
3720 414 : edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE);
3721 414 : e3->probability = profile_probability::guessed_always () / 8;
3722 414 : e1->probability = e3->probability.invert ();
3723 414 : e1->flags = EDGE_TRUE_VALUE;
3724 414 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
3725 :
3726 414 : *gsi = gsi_after_labels (e2->dest);
3727 466 : }
3728 :
3729 : /* Expand all #pragma omp ordered depend(source) and
3730 : #pragma omp ordered depend(sink:...) constructs in the current
3731 : #pragma omp for ordered(n) region. */
3732 :
3733 : static void
3734 335 : expand_omp_ordered_source_sink (struct omp_region *region,
3735 : struct omp_for_data *fd, tree *counts,
3736 : basic_block cont_bb)
3737 : {
3738 335 : struct omp_region *inner;
3739 335 : int i;
3740 1563 : for (i = fd->collapse - 1; i < fd->ordered; i++)
3741 1228 : if (i == fd->collapse - 1 && fd->collapse > 1)
3742 147 : counts[i] = NULL_TREE;
3743 1081 : else if (i >= fd->collapse && !cont_bb)
3744 0 : counts[i] = build_zero_cst (fd->iter_type);
3745 2154 : else if (!POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v))
3746 2154 : && integer_onep (fd->loops[i].step))
3747 987 : counts[i] = NULL_TREE;
3748 : else
3749 94 : counts[i] = create_tmp_var (fd->iter_type, ".orditer");
3750 335 : tree atype
3751 335 : = build_array_type_nelts (fd->iter_type, fd->ordered - fd->collapse + 1);
3752 335 : counts[fd->ordered] = create_tmp_var (atype, ".orditera");
3753 335 : TREE_ADDRESSABLE (counts[fd->ordered]) = 1;
3754 335 : counts[fd->ordered + 1] = NULL_TREE;
3755 :
3756 1615 : for (inner = region->inner; inner; inner = inner->next)
3757 1280 : if (inner->type == GIMPLE_OMP_ORDERED)
3758 : {
3759 709 : gomp_ordered *ord_stmt = inner->ord_stmt;
3760 709 : gimple_stmt_iterator gsi = gsi_for_stmt (ord_stmt);
3761 709 : location_t loc = gimple_location (ord_stmt);
3762 709 : tree c;
3763 1175 : for (c = gimple_omp_ordered_clauses (ord_stmt);
3764 1175 : c; c = OMP_CLAUSE_CHAIN (c))
3765 801 : if (OMP_CLAUSE_DOACROSS_KIND (c) == OMP_CLAUSE_DOACROSS_SOURCE)
3766 : break;
3767 709 : if (c)
3768 335 : expand_omp_ordered_source (&gsi, fd, counts, loc);
3769 1510 : for (c = gimple_omp_ordered_clauses (ord_stmt);
3770 1510 : c; c = OMP_CLAUSE_CHAIN (c))
3771 801 : if (OMP_CLAUSE_DOACROSS_KIND (c) == OMP_CLAUSE_DOACROSS_SINK)
3772 466 : expand_omp_ordered_sink (&gsi, fd, counts, c, loc, cont_bb);
3773 709 : gsi_remove (&gsi, true);
3774 : }
3775 335 : }
3776 :
3777 : /* Wrap the body into fd->ordered - fd->collapse loops that aren't
3778 : collapsed. */
3779 :
3780 : static basic_block
3781 335 : expand_omp_for_ordered_loops (struct omp_for_data *fd, tree *counts,
3782 : basic_block cont_bb, basic_block body_bb,
3783 : basic_block l0_bb, bool ordered_lastprivate)
3784 : {
3785 335 : if (fd->ordered == fd->collapse)
3786 : return cont_bb;
3787 :
3788 171 : if (!cont_bb)
3789 : {
3790 0 : gimple_stmt_iterator gsi = gsi_after_labels (body_bb);
3791 0 : for (int i = fd->collapse; i < fd->ordered; i++)
3792 : {
3793 0 : tree type = TREE_TYPE (fd->loops[i].v);
3794 0 : tree n1 = fold_convert (type, fd->loops[i].n1);
3795 0 : expand_omp_build_assign (&gsi, fd->loops[i].v, n1);
3796 0 : tree aref = build4 (ARRAY_REF, fd->iter_type, counts[fd->ordered],
3797 0 : size_int (i - fd->collapse + 1),
3798 : NULL_TREE, NULL_TREE);
3799 0 : expand_omp_build_assign (&gsi, aref, build_zero_cst (fd->iter_type));
3800 : }
3801 0 : return NULL;
3802 : }
3803 :
3804 1064 : for (int i = fd->ordered - 1; i >= fd->collapse; i--)
3805 : {
3806 893 : tree t, type = TREE_TYPE (fd->loops[i].v);
3807 893 : gimple_stmt_iterator gsi = gsi_after_labels (body_bb);
3808 893 : if (counts[fd->ordered + 1] && i == fd->collapse)
3809 33 : expand_omp_build_assign (&gsi, counts[fd->ordered + 1],
3810 : boolean_true_node);
3811 893 : expand_omp_build_assign (&gsi, fd->loops[i].v,
3812 893 : fold_convert (type, fd->loops[i].n1));
3813 893 : if (counts[i])
3814 57 : expand_omp_build_assign (&gsi, counts[i],
3815 : build_zero_cst (fd->iter_type));
3816 893 : tree aref = build4 (ARRAY_REF, fd->iter_type, counts[fd->ordered],
3817 893 : size_int (i - fd->collapse + 1),
3818 : NULL_TREE, NULL_TREE);
3819 893 : expand_omp_build_assign (&gsi, aref, build_zero_cst (fd->iter_type));
3820 893 : if (!gsi_end_p (gsi))
3821 893 : gsi_prev (&gsi);
3822 : else
3823 0 : gsi = gsi_last_bb (body_bb);
3824 893 : edge e1 = split_block (body_bb, gsi_stmt (gsi));
3825 893 : basic_block new_body = e1->dest;
3826 893 : if (body_bb == cont_bb)
3827 0 : cont_bb = new_body;
3828 893 : edge e2 = NULL;
3829 893 : basic_block new_header;
3830 893 : if (EDGE_COUNT (cont_bb->preds) > 0)
3831 : {
3832 864 : gsi = gsi_last_bb (cont_bb);
3833 864 : if (POINTER_TYPE_P (type))
3834 0 : t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step);
3835 : else
3836 864 : t = fold_build2 (PLUS_EXPR, type, fd->loops[i].v,
3837 : fold_convert (type, fd->loops[i].step));
3838 864 : expand_omp_build_assign (&gsi, fd->loops[i].v, t);
3839 864 : if (counts[i])
3840 : {
3841 57 : t = fold_build2 (PLUS_EXPR, fd->iter_type, counts[i],
3842 : build_int_cst (fd->iter_type, 1));
3843 57 : expand_omp_build_assign (&gsi, counts[i], t);
3844 57 : t = counts[i];
3845 : }
3846 : else
3847 : {
3848 807 : t = fold_build2 (MINUS_EXPR, TREE_TYPE (fd->loops[i].v),
3849 : fd->loops[i].v, fd->loops[i].n1);
3850 807 : t = fold_convert (fd->iter_type, t);
3851 807 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
3852 : true, GSI_SAME_STMT);
3853 : }
3854 864 : aref = build4 (ARRAY_REF, fd->iter_type, counts[fd->ordered],
3855 864 : size_int (i - fd->collapse + 1),
3856 : NULL_TREE, NULL_TREE);
3857 864 : expand_omp_build_assign (&gsi, aref, t);
3858 864 : if (counts[fd->ordered + 1] && i == fd->ordered - 1)
3859 30 : expand_omp_build_assign (&gsi, counts[fd->ordered + 1],
3860 : boolean_false_node);
3861 864 : gsi_prev (&gsi);
3862 864 : e2 = split_block (cont_bb, gsi_stmt (gsi));
3863 864 : new_header = e2->dest;
3864 : }
3865 : else
3866 : new_header = cont_bb;
3867 893 : gsi = gsi_after_labels (new_header);
3868 893 : tree v = force_gimple_operand_gsi (&gsi, fd->loops[i].v, true, NULL_TREE,
3869 : true, GSI_SAME_STMT);
3870 893 : tree n2
3871 893 : = force_gimple_operand_gsi (&gsi, fold_convert (type, fd->loops[i].n2),
3872 : true, NULL_TREE, true, GSI_SAME_STMT);
3873 893 : t = build2 (fd->loops[i].cond_code, boolean_type_node, v, n2);
3874 893 : gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_NEW_STMT);
3875 893 : edge e3 = split_block (new_header, gsi_stmt (gsi));
3876 893 : cont_bb = e3->dest;
3877 893 : remove_edge (e1);
3878 893 : make_edge (body_bb, new_header, EDGE_FALLTHRU);
3879 893 : e3->flags = EDGE_FALSE_VALUE;
3880 893 : e3->probability = profile_probability::guessed_always () / 8;
3881 893 : e1 = make_edge (new_header, new_body, EDGE_TRUE_VALUE);
3882 893 : e1->probability = e3->probability.invert ();
3883 :
3884 893 : set_immediate_dominator (CDI_DOMINATORS, new_header, body_bb);
3885 893 : set_immediate_dominator (CDI_DOMINATORS, new_body, new_header);
3886 :
3887 893 : if (e2)
3888 : {
3889 864 : class loop *loop = alloc_loop ();
3890 864 : loop->header = new_header;
3891 864 : loop->latch = e2->src;
3892 864 : add_loop (loop, l0_bb->loop_father);
3893 : }
3894 : }
3895 :
3896 : /* If there are any lastprivate clauses and it is possible some loops
3897 : might have zero iterations, ensure all the decls are initialized,
3898 : otherwise we could crash evaluating C++ class iterators with lastprivate
3899 : clauses. */
3900 : bool need_inits = false;
3901 171 : for (int i = fd->collapse; ordered_lastprivate && i < fd->ordered; i++)
3902 0 : if (need_inits)
3903 : {
3904 0 : tree type = TREE_TYPE (fd->loops[i].v);
3905 0 : gimple_stmt_iterator gsi = gsi_after_labels (body_bb);
3906 0 : expand_omp_build_assign (&gsi, fd->loops[i].v,
3907 : fold_convert (type, fd->loops[i].n1));
3908 : }
3909 : else
3910 : {
3911 0 : tree type = TREE_TYPE (fd->loops[i].v);
3912 0 : tree this_cond = fold_build2 (fd->loops[i].cond_code,
3913 : boolean_type_node,
3914 : fold_convert (type, fd->loops[i].n1),
3915 : fold_convert (type, fd->loops[i].n2));
3916 0 : if (!integer_onep (this_cond))
3917 0 : need_inits = true;
3918 : }
3919 :
3920 : return cont_bb;
3921 : }
3922 :
3923 : /* A subroutine of expand_omp_for. Generate code for a parallel
3924 : loop with any schedule. Given parameters:
3925 :
3926 : for (V = N1; V cond N2; V += STEP) BODY;
3927 :
3928 : where COND is "<" or ">", we generate pseudocode
3929 :
3930 : more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
3931 : if (more) goto L0; else goto L3;
3932 : L0:
3933 : V = istart0;
3934 : iend = iend0;
3935 : L1:
3936 : BODY;
3937 : V += STEP;
3938 : if (V cond iend) goto L1; else goto L2;
3939 : L2:
3940 : if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
3941 : L3:
3942 :
3943 : If this is a combined omp parallel loop, instead of the call to
3944 : GOMP_loop_foo_start, we call GOMP_loop_foo_next.
3945 : If this is gimple_omp_for_combined_p loop, then instead of assigning
3946 : V and iend in L0 we assign the first two _looptemp_ clause decls of the
3947 : inner GIMPLE_OMP_FOR and V += STEP; and
3948 : if (V cond iend) goto L1; else goto L2; are removed.
3949 :
3950 : For collapsed loops, given parameters:
3951 : collapse(3)
3952 : for (V1 = N11; V1 cond1 N12; V1 += STEP1)
3953 : for (V2 = N21; V2 cond2 N22; V2 += STEP2)
3954 : for (V3 = N31; V3 cond3 N32; V3 += STEP3)
3955 : BODY;
3956 :
3957 : we generate pseudocode
3958 :
3959 : if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
3960 : if (cond3 is <)
3961 : adj = STEP3 - 1;
3962 : else
3963 : adj = STEP3 + 1;
3964 : count3 = (adj + N32 - N31) / STEP3;
3965 : if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
3966 : if (cond2 is <)
3967 : adj = STEP2 - 1;
3968 : else
3969 : adj = STEP2 + 1;
3970 : count2 = (adj + N22 - N21) / STEP2;
3971 : if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
3972 : if (cond1 is <)
3973 : adj = STEP1 - 1;
3974 : else
3975 : adj = STEP1 + 1;
3976 : count1 = (adj + N12 - N11) / STEP1;
3977 : count = count1 * count2 * count3;
3978 : goto Z1;
3979 : Z0:
3980 : count = 0;
3981 : Z1:
3982 : more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
3983 : if (more) goto L0; else goto L3;
3984 : L0:
3985 : V = istart0;
3986 : T = V;
3987 : V3 = N31 + (T % count3) * STEP3;
3988 : T = T / count3;
3989 : V2 = N21 + (T % count2) * STEP2;
3990 : T = T / count2;
3991 : V1 = N11 + T * STEP1;
3992 : iend = iend0;
3993 : L1:
3994 : BODY;
3995 : V += 1;
3996 : if (V < iend) goto L10; else goto L2;
3997 : L10:
3998 : V3 += STEP3;
3999 : if (V3 cond3 N32) goto L1; else goto L11;
4000 : L11:
4001 : V3 = N31;
4002 : V2 += STEP2;
4003 : if (V2 cond2 N22) goto L1; else goto L12;
4004 : L12:
4005 : V2 = N21;
4006 : V1 += STEP1;
4007 : goto L1;
4008 : L2:
4009 : if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
4010 : L3:
4011 :
4012 : */
4013 :
4014 : static void
4015 4134 : expand_omp_for_generic (struct omp_region *region,
4016 : struct omp_for_data *fd,
4017 : enum built_in_function start_fn,
4018 : enum built_in_function next_fn,
4019 : tree sched_arg,
4020 : gimple *inner_stmt)
4021 : {
4022 4134 : tree type, istart0, iend0, iend;
4023 4134 : tree t, vmain, vback, bias = NULL_TREE;
4024 4134 : basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, collapse_bb;
4025 4134 : basic_block l2_bb = NULL, l3_bb = NULL;
4026 4134 : gimple_stmt_iterator gsi;
4027 4134 : gassign *assign_stmt;
4028 4134 : bool in_combined_parallel = is_combined_parallel (region);
4029 4134 : bool broken_loop = region->cont == NULL;
4030 4134 : edge e, ne;
4031 4134 : tree *counts = NULL;
4032 4134 : int i;
4033 4134 : bool ordered_lastprivate = false;
4034 4134 : bool offload = is_in_offload_region (region);
4035 :
4036 4134 : gcc_assert (!broken_loop || !in_combined_parallel);
4037 4134 : gcc_assert (fd->iter_type == long_integer_type_node
4038 : || !in_combined_parallel);
4039 :
4040 4134 : entry_bb = region->entry;
4041 4134 : cont_bb = region->cont;
4042 4134 : collapse_bb = NULL;
4043 4134 : gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
4044 4134 : gcc_assert (broken_loop
4045 : || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
4046 4134 : l0_bb = split_edge (FALLTHRU_EDGE (entry_bb));
4047 4134 : l1_bb = single_succ (l0_bb);
4048 4134 : if (!broken_loop)
4049 : {
4050 3727 : l2_bb = create_empty_bb (cont_bb);
4051 3727 : gcc_assert (BRANCH_EDGE (cont_bb)->dest == l1_bb
4052 : || (single_succ_edge (BRANCH_EDGE (cont_bb)->dest)->dest
4053 : == l1_bb));
4054 3727 : gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
4055 : }
4056 : else
4057 : l2_bb = NULL;
4058 4134 : l3_bb = BRANCH_EDGE (entry_bb)->dest;
4059 4134 : exit_bb = region->exit;
4060 :
4061 4134 : gsi = gsi_last_nondebug_bb (entry_bb);
4062 :
4063 4134 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
4064 4134 : if (fd->ordered
4065 4134 : && omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
4066 : OMP_CLAUSE_LASTPRIVATE))
4067 4134 : ordered_lastprivate = false;
4068 4134 : tree reductions = NULL_TREE;
4069 4134 : tree mem = NULL_TREE, cond_var = NULL_TREE, condtemp = NULL_TREE;
4070 4134 : tree memv = NULL_TREE;
4071 4134 : if (fd->lastprivate_conditional)
4072 : {
4073 46 : tree c = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
4074 : OMP_CLAUSE__CONDTEMP_);
4075 46 : if (fd->have_pointer_condtemp)
4076 28 : condtemp = OMP_CLAUSE_DECL (c);
4077 46 : c = omp_find_clause (OMP_CLAUSE_CHAIN (c), OMP_CLAUSE__CONDTEMP_);
4078 46 : cond_var = OMP_CLAUSE_DECL (c);
4079 : }
4080 4134 : if (sched_arg)
4081 : {
4082 169 : if (fd->have_reductemp)
4083 : {
4084 149 : tree c = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
4085 : OMP_CLAUSE__REDUCTEMP_);
4086 149 : reductions = OMP_CLAUSE_DECL (c);
4087 149 : gcc_assert (TREE_CODE (reductions) == SSA_NAME);
4088 149 : gimple *g = SSA_NAME_DEF_STMT (reductions);
4089 149 : reductions = gimple_assign_rhs1 (g);
4090 149 : OMP_CLAUSE_DECL (c) = reductions;
4091 149 : entry_bb = gimple_bb (g);
4092 149 : edge e = split_block (entry_bb, g);
4093 149 : if (region->entry == entry_bb)
4094 8 : region->entry = e->dest;
4095 298 : gsi = gsi_last_bb (entry_bb);
4096 : }
4097 : else
4098 20 : reductions = null_pointer_node;
4099 169 : if (fd->have_pointer_condtemp)
4100 : {
4101 28 : tree type = TREE_TYPE (condtemp);
4102 28 : memv = create_tmp_var (type);
4103 28 : TREE_ADDRESSABLE (memv) = 1;
4104 28 : unsigned HOST_WIDE_INT sz
4105 28 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (type)));
4106 28 : sz *= fd->lastprivate_conditional;
4107 28 : expand_omp_build_assign (&gsi, memv, build_int_cst (type, sz),
4108 : false);
4109 28 : mem = build_fold_addr_expr (memv);
4110 : }
4111 : else
4112 141 : mem = null_pointer_node;
4113 : }
4114 4134 : if (fd->collapse > 1 || fd->ordered)
4115 : {
4116 1669 : int first_zero_iter1 = -1, first_zero_iter2 = -1;
4117 1669 : basic_block zero_iter1_bb = NULL, zero_iter2_bb = NULL, l2_dom_bb = NULL;
4118 :
4119 1669 : counts = XALLOCAVEC (tree, fd->ordered
4120 : ? fd->ordered + 2
4121 : + (fd->ordered - fd->collapse)
4122 : : fd->collapse);
4123 1669 : expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
4124 : zero_iter1_bb, first_zero_iter1,
4125 : zero_iter2_bb, first_zero_iter2, l2_dom_bb);
4126 :
4127 1669 : if (zero_iter1_bb)
4128 : {
4129 : /* Some counts[i] vars might be uninitialized if
4130 : some loop has zero iterations. But the body shouldn't
4131 : be executed in that case, so just avoid uninit warnings. */
4132 1913 : for (i = first_zero_iter1;
4133 1913 : i < (fd->ordered ? fd->ordered : fd->collapse); i++)
4134 1502 : if (SSA_VAR_P (counts[i]))
4135 950 : suppress_warning (counts[i], OPT_Wuninitialized);
4136 411 : gsi_prev (&gsi);
4137 411 : e = split_block (entry_bb, gsi_stmt (gsi));
4138 411 : entry_bb = e->dest;
4139 411 : make_edge (zero_iter1_bb, entry_bb, EDGE_FALLTHRU);
4140 411 : gsi = gsi_last_nondebug_bb (entry_bb);
4141 411 : set_immediate_dominator (CDI_DOMINATORS, entry_bb,
4142 : get_immediate_dominator (CDI_DOMINATORS,
4143 : zero_iter1_bb));
4144 : }
4145 1669 : if (zero_iter2_bb)
4146 : {
4147 : /* Some counts[i] vars might be uninitialized if
4148 : some loop has zero iterations. But the body shouldn't
4149 : be executed in that case, so just avoid uninit warnings. */
4150 394 : for (i = first_zero_iter2; i < fd->ordered; i++)
4151 276 : if (SSA_VAR_P (counts[i]))
4152 216 : suppress_warning (counts[i], OPT_Wuninitialized);
4153 118 : if (zero_iter1_bb)
4154 103 : make_edge (zero_iter2_bb, entry_bb, EDGE_FALLTHRU);
4155 : else
4156 : {
4157 15 : gsi_prev (&gsi);
4158 15 : e = split_block (entry_bb, gsi_stmt (gsi));
4159 15 : entry_bb = e->dest;
4160 15 : make_edge (zero_iter2_bb, entry_bb, EDGE_FALLTHRU);
4161 15 : gsi = gsi_last_nondebug_bb (entry_bb);
4162 15 : set_immediate_dominator (CDI_DOMINATORS, entry_bb,
4163 : get_immediate_dominator
4164 : (CDI_DOMINATORS, zero_iter2_bb));
4165 : }
4166 : }
4167 1669 : if (fd->collapse == 1)
4168 : {
4169 188 : counts[0] = fd->loop.n2;
4170 188 : fd->loop = fd->loops[0];
4171 : }
4172 : }
4173 :
4174 4134 : type = TREE_TYPE (fd->loop.v);
4175 4134 : istart0 = create_tmp_var (fd->iter_type, ".istart0");
4176 4134 : iend0 = create_tmp_var (fd->iter_type, ".iend0");
4177 4134 : TREE_ADDRESSABLE (istart0) = 1;
4178 4134 : TREE_ADDRESSABLE (iend0) = 1;
4179 :
4180 : /* See if we need to bias by LLONG_MIN. */
4181 4134 : if (fd->iter_type == long_long_unsigned_type_node
4182 778 : && (TREE_CODE (type) == INTEGER_TYPE || TREE_CODE (type) == BITINT_TYPE)
4183 530 : && !TYPE_UNSIGNED (type)
4184 4134 : && fd->ordered == 0)
4185 : {
4186 0 : tree n1, n2;
4187 :
4188 0 : if (fd->loop.cond_code == LT_EXPR)
4189 : {
4190 0 : n1 = fd->loop.n1;
4191 0 : n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step);
4192 : }
4193 : else
4194 : {
4195 0 : n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step);
4196 0 : n2 = fd->loop.n1;
4197 : }
4198 0 : if (TREE_CODE (n1) != INTEGER_CST
4199 0 : || TREE_CODE (n2) != INTEGER_CST
4200 0 : || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0)))
4201 0 : bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type));
4202 : }
4203 :
4204 4134 : gimple_stmt_iterator gsif = gsi;
4205 4134 : gsi_prev (&gsif);
4206 :
4207 4134 : tree arr = NULL_TREE;
4208 4134 : if (in_combined_parallel)
4209 : {
4210 1084 : gcc_assert (fd->ordered == 0);
4211 : /* In a combined parallel loop, emit a call to
4212 : GOMP_loop_foo_next. */
4213 1084 : t = build_call_expr (builtin_decl_explicit (next_fn), 2,
4214 : build_fold_addr_expr (istart0),
4215 : build_fold_addr_expr (iend0));
4216 : }
4217 : else
4218 : {
4219 3050 : tree t0, t1, t2, t3, t4;
4220 : /* If this is not a combined parallel loop, emit a call to
4221 : GOMP_loop_foo_start in ENTRY_BB. */
4222 3050 : t4 = build_fold_addr_expr (iend0);
4223 3050 : t3 = build_fold_addr_expr (istart0);
4224 3050 : if (fd->ordered)
4225 : {
4226 670 : t0 = build_int_cst (unsigned_type_node,
4227 335 : fd->ordered - fd->collapse + 1);
4228 335 : arr = create_tmp_var (build_array_type_nelts (fd->iter_type,
4229 335 : fd->ordered
4230 335 : - fd->collapse + 1),
4231 : ".omp_counts");
4232 335 : DECL_NAMELESS (arr) = 1;
4233 335 : TREE_ADDRESSABLE (arr) = 1;
4234 335 : TREE_STATIC (arr) = 1;
4235 335 : vec<constructor_elt, va_gc> *v;
4236 335 : vec_alloc (v, fd->ordered - fd->collapse + 1);
4237 335 : int idx;
4238 :
4239 1898 : for (idx = 0; idx < fd->ordered - fd->collapse + 1; idx++)
4240 : {
4241 1228 : tree c;
4242 1228 : if (idx == 0 && fd->collapse > 1)
4243 147 : c = fd->loop.n2;
4244 : else
4245 1081 : c = counts[idx + fd->collapse - 1];
4246 1228 : tree purpose = size_int (idx);
4247 1228 : CONSTRUCTOR_APPEND_ELT (v, purpose, c);
4248 1228 : if (TREE_CODE (c) != INTEGER_CST)
4249 433 : TREE_STATIC (arr) = 0;
4250 : }
4251 :
4252 335 : DECL_INITIAL (arr) = build_constructor (TREE_TYPE (arr), v);
4253 335 : if (!TREE_STATIC (arr))
4254 232 : force_gimple_operand_gsi (&gsi, build1 (DECL_EXPR,
4255 : void_type_node, arr),
4256 : true, NULL_TREE, true, GSI_SAME_STMT);
4257 335 : t1 = build_fold_addr_expr (arr);
4258 335 : t2 = NULL_TREE;
4259 : }
4260 : else
4261 : {
4262 2715 : t2 = fold_convert (fd->iter_type, fd->loop.step);
4263 2715 : t1 = fd->loop.n2;
4264 2715 : t0 = fd->loop.n1;
4265 2715 : if (gimple_omp_for_combined_into_p (fd->for_stmt))
4266 : {
4267 1152 : tree innerc
4268 1152 : = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
4269 : OMP_CLAUSE__LOOPTEMP_);
4270 1152 : gcc_assert (innerc);
4271 1152 : t0 = OMP_CLAUSE_DECL (innerc);
4272 1152 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
4273 : OMP_CLAUSE__LOOPTEMP_);
4274 1152 : gcc_assert (innerc);
4275 1152 : t1 = OMP_CLAUSE_DECL (innerc);
4276 : }
4277 5318 : if (POINTER_TYPE_P (TREE_TYPE (t0))
4278 2715 : && TYPE_PRECISION (TREE_TYPE (t0))
4279 112 : != TYPE_PRECISION (fd->iter_type))
4280 : {
4281 : /* Avoid casting pointers to integer of a different size. */
4282 0 : tree itype = signed_type_for (type);
4283 0 : t1 = fold_convert (fd->iter_type, fold_convert (itype, t1));
4284 0 : t0 = fold_convert (fd->iter_type, fold_convert (itype, t0));
4285 : }
4286 : else
4287 : {
4288 2715 : t1 = fold_convert (fd->iter_type, t1);
4289 2715 : t0 = fold_convert (fd->iter_type, t0);
4290 : }
4291 2715 : if (bias)
4292 : {
4293 0 : t1 = fold_build2 (PLUS_EXPR, fd->iter_type, t1, bias);
4294 0 : t0 = fold_build2 (PLUS_EXPR, fd->iter_type, t0, bias);
4295 : }
4296 : }
4297 3050 : if (fd->iter_type == long_integer_type_node || fd->ordered)
4298 : {
4299 2328 : if (fd->chunk_size)
4300 : {
4301 1579 : t = fold_convert (fd->iter_type, fd->chunk_size);
4302 1579 : t = omp_adjust_chunk_size (t, fd->simd_schedule, offload);
4303 1579 : if (sched_arg)
4304 : {
4305 135 : if (fd->ordered)
4306 36 : t = build_call_expr (builtin_decl_explicit (start_fn),
4307 : 8, t0, t1, sched_arg, t, t3, t4,
4308 : reductions, mem);
4309 : else
4310 99 : t = build_call_expr (builtin_decl_explicit (start_fn),
4311 : 9, t0, t1, t2, sched_arg, t, t3, t4,
4312 : reductions, mem);
4313 : }
4314 1444 : else if (fd->ordered)
4315 299 : t = build_call_expr (builtin_decl_explicit (start_fn),
4316 : 5, t0, t1, t, t3, t4);
4317 : else
4318 1145 : t = build_call_expr (builtin_decl_explicit (start_fn),
4319 : 6, t0, t1, t2, t, t3, t4);
4320 : }
4321 749 : else if (fd->ordered)
4322 0 : t = build_call_expr (builtin_decl_explicit (start_fn),
4323 : 4, t0, t1, t3, t4);
4324 : else
4325 749 : t = build_call_expr (builtin_decl_explicit (start_fn),
4326 : 5, t0, t1, t2, t3, t4);
4327 : }
4328 : else
4329 : {
4330 722 : tree t5;
4331 722 : tree c_bool_type;
4332 722 : tree bfn_decl;
4333 :
4334 : /* The GOMP_loop_ull_*start functions have additional boolean
4335 : argument, true for < loops and false for > loops.
4336 : In Fortran, the C bool type can be different from
4337 : boolean_type_node. */
4338 722 : bfn_decl = builtin_decl_explicit (start_fn);
4339 722 : c_bool_type = TREE_TYPE (TREE_TYPE (bfn_decl));
4340 722 : t5 = build_int_cst (c_bool_type,
4341 942 : fd->loop.cond_code == LT_EXPR ? 1 : 0);
4342 722 : if (fd->chunk_size)
4343 : {
4344 390 : tree bfn_decl = builtin_decl_explicit (start_fn);
4345 390 : t = fold_convert (fd->iter_type, fd->chunk_size);
4346 390 : t = omp_adjust_chunk_size (t, fd->simd_schedule, offload);
4347 390 : if (sched_arg)
4348 34 : t = build_call_expr (bfn_decl, 10, t5, t0, t1, t2, sched_arg,
4349 : t, t3, t4, reductions, mem);
4350 : else
4351 356 : t = build_call_expr (bfn_decl, 7, t5, t0, t1, t2, t, t3, t4);
4352 : }
4353 : else
4354 332 : t = build_call_expr (builtin_decl_explicit (start_fn),
4355 : 6, t5, t0, t1, t2, t3, t4);
4356 : }
4357 : }
4358 4134 : if (TREE_TYPE (t) != boolean_type_node)
4359 0 : t = fold_build2 (NE_EXPR, boolean_type_node,
4360 : t, build_int_cst (TREE_TYPE (t), 0));
4361 4134 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4362 : true, GSI_SAME_STMT);
4363 4134 : if (arr && !TREE_STATIC (arr))
4364 : {
4365 232 : tree clobber = build_clobber (TREE_TYPE (arr));
4366 232 : gsi_insert_before (&gsi, gimple_build_assign (arr, clobber),
4367 : GSI_SAME_STMT);
4368 : }
4369 4134 : if (fd->have_pointer_condtemp)
4370 28 : expand_omp_build_assign (&gsi, condtemp, memv, false);
4371 4134 : if (fd->have_reductemp)
4372 : {
4373 149 : gimple *g = gsi_stmt (gsi);
4374 149 : gsi_remove (&gsi, true);
4375 149 : release_ssa_name (gimple_assign_lhs (g));
4376 :
4377 149 : entry_bb = region->entry;
4378 149 : gsi = gsi_last_nondebug_bb (entry_bb);
4379 :
4380 149 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
4381 : }
4382 4134 : gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
4383 :
4384 : /* Remove the GIMPLE_OMP_FOR statement. */
4385 4134 : gsi_remove (&gsi, true);
4386 :
4387 4134 : if (gsi_end_p (gsif))
4388 1203 : gsif = gsi_after_labels (gsi_bb (gsif));
4389 4134 : gsi_next (&gsif);
4390 :
4391 : /* Iteration setup for sequential loop goes in L0_BB. */
4392 4134 : tree startvar = fd->loop.v;
4393 4134 : tree endvar = NULL_TREE;
4394 :
4395 4134 : if (gimple_omp_for_combined_p (fd->for_stmt))
4396 : {
4397 1389 : gcc_assert (gimple_code (inner_stmt) == GIMPLE_OMP_FOR
4398 : && gimple_omp_for_kind (inner_stmt)
4399 : == GF_OMP_FOR_KIND_SIMD);
4400 1389 : tree innerc = omp_find_clause (gimple_omp_for_clauses (inner_stmt),
4401 : OMP_CLAUSE__LOOPTEMP_);
4402 1389 : gcc_assert (innerc);
4403 1389 : startvar = OMP_CLAUSE_DECL (innerc);
4404 1389 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
4405 : OMP_CLAUSE__LOOPTEMP_);
4406 1389 : gcc_assert (innerc);
4407 1389 : endvar = OMP_CLAUSE_DECL (innerc);
4408 : }
4409 :
4410 4134 : gsi = gsi_start_bb (l0_bb);
4411 4134 : t = istart0;
4412 4134 : if (fd->ordered && fd->collapse == 1)
4413 188 : t = fold_build2 (MULT_EXPR, fd->iter_type, t,
4414 : fold_convert (fd->iter_type, fd->loop.step));
4415 3946 : else if (bias)
4416 0 : t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
4417 4134 : if (fd->ordered && fd->collapse == 1)
4418 : {
4419 188 : if (POINTER_TYPE_P (TREE_TYPE (startvar)))
4420 8 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (startvar),
4421 : fd->loop.n1, fold_convert (sizetype, t));
4422 : else
4423 : {
4424 180 : t = fold_convert (TREE_TYPE (startvar), t);
4425 180 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (startvar),
4426 : fd->loop.n1, t);
4427 : }
4428 : }
4429 : else
4430 : {
4431 3946 : if (POINTER_TYPE_P (TREE_TYPE (startvar)))
4432 240 : t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t);
4433 3946 : t = fold_convert (TREE_TYPE (startvar), t);
4434 : }
4435 4134 : t = force_gimple_operand_gsi (&gsi, t,
4436 4134 : DECL_P (startvar)
4437 4121 : && TREE_ADDRESSABLE (startvar),
4438 : NULL_TREE, false, GSI_CONTINUE_LINKING);
4439 4134 : assign_stmt = gimple_build_assign (startvar, t);
4440 4134 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
4441 4134 : if (cond_var)
4442 : {
4443 46 : tree itype = TREE_TYPE (cond_var);
4444 : /* For lastprivate(conditional:) itervar, we need some iteration
4445 : counter that starts at unsigned non-zero and increases.
4446 : Prefer as few IVs as possible, so if we can use startvar
4447 : itself, use that, or startvar + constant (those would be
4448 : incremented with step), and as last resort use the s0 + 1
4449 : incremented by 1. */
4450 46 : if ((fd->ordered && fd->collapse == 1)
4451 46 : || bias
4452 46 : || POINTER_TYPE_P (type)
4453 46 : || TREE_CODE (fd->loop.n1) != INTEGER_CST
4454 40 : || fd->loop.cond_code != LT_EXPR)
4455 6 : t = fold_build2 (PLUS_EXPR, itype, fold_convert (itype, istart0),
4456 : build_int_cst (itype, 1));
4457 40 : else if (tree_int_cst_sgn (fd->loop.n1) == 1)
4458 9 : t = fold_convert (itype, t);
4459 : else
4460 : {
4461 31 : tree c = fold_convert (itype, fd->loop.n1);
4462 31 : c = fold_build2 (MINUS_EXPR, itype, build_int_cst (itype, 1), c);
4463 31 : t = fold_build2 (PLUS_EXPR, itype, fold_convert (itype, t), c);
4464 : }
4465 46 : t = force_gimple_operand_gsi (&gsi, t, false,
4466 : NULL_TREE, false, GSI_CONTINUE_LINKING);
4467 46 : assign_stmt = gimple_build_assign (cond_var, t);
4468 46 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
4469 : }
4470 :
4471 4134 : t = iend0;
4472 4134 : if (fd->ordered && fd->collapse == 1)
4473 188 : t = fold_build2 (MULT_EXPR, fd->iter_type, t,
4474 : fold_convert (fd->iter_type, fd->loop.step));
4475 3946 : else if (bias)
4476 0 : t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
4477 4134 : if (fd->ordered && fd->collapse == 1)
4478 : {
4479 188 : if (POINTER_TYPE_P (TREE_TYPE (startvar)))
4480 8 : t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (startvar),
4481 : fd->loop.n1, fold_convert (sizetype, t));
4482 : else
4483 : {
4484 180 : t = fold_convert (TREE_TYPE (startvar), t);
4485 180 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (startvar),
4486 : fd->loop.n1, t);
4487 : }
4488 : }
4489 : else
4490 : {
4491 3946 : if (POINTER_TYPE_P (TREE_TYPE (startvar)))
4492 240 : t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t);
4493 3946 : t = fold_convert (TREE_TYPE (startvar), t);
4494 : }
4495 4134 : iend = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4496 : false, GSI_CONTINUE_LINKING);
4497 4134 : if (endvar)
4498 : {
4499 1389 : assign_stmt = gimple_build_assign (endvar, iend);
4500 1389 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
4501 1389 : if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (iend)))
4502 857 : assign_stmt = gimple_build_assign (fd->loop.v, iend);
4503 : else
4504 532 : assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, iend);
4505 1389 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
4506 : }
4507 : /* Handle linear clause adjustments. */
4508 4134 : tree itercnt = NULL_TREE;
4509 4134 : if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_FOR)
4510 27021 : for (tree c = gimple_omp_for_clauses (fd->for_stmt);
4511 27021 : c; c = OMP_CLAUSE_CHAIN (c))
4512 22887 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
4513 22887 : && !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
4514 : {
4515 72 : tree d = OMP_CLAUSE_DECL (c);
4516 72 : tree t = d, a, dest;
4517 72 : if (omp_privatize_by_reference (t))
4518 2 : t = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), t);
4519 72 : tree type = TREE_TYPE (t);
4520 72 : if (POINTER_TYPE_P (type))
4521 0 : type = sizetype;
4522 72 : dest = unshare_expr (t);
4523 72 : tree v = create_tmp_var (TREE_TYPE (t), NULL);
4524 72 : expand_omp_build_assign (&gsif, v, t);
4525 72 : if (itercnt == NULL_TREE)
4526 : {
4527 72 : itercnt = startvar;
4528 72 : tree n1 = fd->loop.n1;
4529 72 : if (POINTER_TYPE_P (TREE_TYPE (itercnt)))
4530 : {
4531 0 : itercnt
4532 0 : = fold_convert (signed_type_for (TREE_TYPE (itercnt)),
4533 : itercnt);
4534 0 : n1 = fold_convert (TREE_TYPE (itercnt), n1);
4535 : }
4536 72 : itercnt = fold_build2 (MINUS_EXPR, TREE_TYPE (itercnt),
4537 : itercnt, n1);
4538 72 : itercnt = fold_build2 (EXACT_DIV_EXPR, TREE_TYPE (itercnt),
4539 : itercnt, fd->loop.step);
4540 72 : itercnt = force_gimple_operand_gsi (&gsi, itercnt, true,
4541 : NULL_TREE, false,
4542 : GSI_CONTINUE_LINKING);
4543 : }
4544 72 : a = fold_build2 (MULT_EXPR, type,
4545 : fold_convert (type, itercnt),
4546 : fold_convert (type, OMP_CLAUSE_LINEAR_STEP (c)));
4547 72 : t = fold_build2 (type == TREE_TYPE (t) ? PLUS_EXPR
4548 : : POINTER_PLUS_EXPR, TREE_TYPE (t), v, a);
4549 72 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4550 : false, GSI_CONTINUE_LINKING);
4551 72 : expand_omp_build_assign (&gsi, dest, t, true);
4552 : }
4553 4134 : if (fd->collapse > 1)
4554 1481 : expand_omp_for_init_vars (fd, &gsi, counts, NULL, inner_stmt, startvar);
4555 :
4556 4134 : if (fd->ordered)
4557 : {
4558 : /* Until now, counts array contained number of iterations or
4559 : variable containing it for ith loop. From now on, we usually need
4560 : those counts only for collapsed loops, and only for the 2nd
4561 : till the last collapsed one. Move those one element earlier,
4562 : we'll use counts[fd->collapse - 1] for the first source/sink
4563 : iteration counter and so on and counts[fd->ordered]
4564 : as the array holding the current counter values for
4565 : depend(source). For doacross(sink:omp_cur_iteration - 1) we need
4566 : the counts from fd->collapse to fd->ordered - 1; make a copy of
4567 : those to counts[fd->ordered + 2] and onwards.
4568 : counts[fd->ordered + 1] can be a flag whether it is the first
4569 : iteration with a new collapsed counter (used only if
4570 : fd->ordered > fd->collapse). */
4571 335 : if (fd->ordered > fd->collapse)
4572 171 : memcpy (counts + fd->ordered + 2, counts + fd->collapse,
4573 171 : (fd->ordered - fd->collapse) * sizeof (counts[0]));
4574 335 : if (fd->collapse > 1)
4575 147 : memmove (counts, counts + 1, (fd->collapse - 1) * sizeof (counts[0]));
4576 335 : if (broken_loop)
4577 : {
4578 : int i;
4579 41 : for (i = fd->collapse; i < fd->ordered; i++)
4580 : {
4581 33 : tree type = TREE_TYPE (fd->loops[i].v);
4582 33 : tree this_cond
4583 33 : = fold_build2 (fd->loops[i].cond_code, boolean_type_node,
4584 : fold_convert (type, fd->loops[i].n1),
4585 : fold_convert (type, fd->loops[i].n2));
4586 33 : if (!integer_onep (this_cond))
4587 : break;
4588 : }
4589 37 : if (i < fd->ordered)
4590 : {
4591 29 : if (entry_bb->loop_father != l0_bb->loop_father)
4592 : {
4593 4 : remove_bb_from_loops (l0_bb);
4594 4 : add_bb_to_loop (l0_bb, entry_bb->loop_father);
4595 4 : gcc_assert (single_succ (l0_bb) == l1_bb);
4596 : }
4597 29 : cont_bb
4598 29 : = create_empty_bb (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb);
4599 29 : add_bb_to_loop (cont_bb, l0_bb->loop_father);
4600 29 : gimple_stmt_iterator gsi = gsi_after_labels (cont_bb);
4601 29 : gimple *g = gimple_build_omp_continue (fd->loop.v, fd->loop.v);
4602 29 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
4603 29 : make_edge (cont_bb, l3_bb, EDGE_FALLTHRU);
4604 29 : make_edge (cont_bb, l1_bb, 0);
4605 29 : l2_bb = create_empty_bb (cont_bb);
4606 29 : broken_loop = false;
4607 : }
4608 : }
4609 335 : expand_omp_ordered_source_sink (region, fd, counts, cont_bb);
4610 335 : cont_bb = expand_omp_for_ordered_loops (fd, counts, cont_bb, l1_bb,
4611 : l0_bb, ordered_lastprivate);
4612 335 : if (counts[fd->collapse - 1])
4613 : {
4614 37 : gcc_assert (fd->collapse == 1);
4615 37 : gsi = gsi_last_bb (l0_bb);
4616 37 : expand_omp_build_assign (&gsi, counts[fd->collapse - 1],
4617 : istart0, true);
4618 37 : if (cont_bb)
4619 : {
4620 33 : gsi = gsi_last_bb (cont_bb);
4621 33 : t = fold_build2 (PLUS_EXPR, fd->iter_type,
4622 : counts[fd->collapse - 1],
4623 : build_int_cst (fd->iter_type, 1));
4624 33 : expand_omp_build_assign (&gsi, counts[fd->collapse - 1], t);
4625 66 : tree aref = build4 (ARRAY_REF, fd->iter_type,
4626 33 : counts[fd->ordered], size_zero_node,
4627 : NULL_TREE, NULL_TREE);
4628 33 : expand_omp_build_assign (&gsi, aref, counts[fd->collapse - 1]);
4629 : }
4630 37 : t = counts[fd->collapse - 1];
4631 : }
4632 298 : else if (fd->collapse > 1)
4633 147 : t = fd->loop.v;
4634 : else
4635 : {
4636 151 : t = fold_build2 (MINUS_EXPR, TREE_TYPE (fd->loops[0].v),
4637 : fd->loops[0].v, fd->loops[0].n1);
4638 151 : t = fold_convert (fd->iter_type, t);
4639 : }
4640 335 : gsi = gsi_last_bb (l0_bb);
4641 335 : tree aref = build4 (ARRAY_REF, fd->iter_type, counts[fd->ordered],
4642 : size_zero_node, NULL_TREE, NULL_TREE);
4643 335 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4644 : false, GSI_CONTINUE_LINKING);
4645 335 : expand_omp_build_assign (&gsi, aref, t, true);
4646 : }
4647 :
4648 4134 : if (!broken_loop)
4649 : {
4650 : /* Code to control the increment and predicate for the sequential
4651 : loop goes in the CONT_BB. */
4652 3756 : gsi = gsi_last_nondebug_bb (cont_bb);
4653 3756 : gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
4654 3756 : gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE);
4655 3756 : vmain = gimple_omp_continue_control_use (cont_stmt);
4656 3756 : vback = gimple_omp_continue_control_def (cont_stmt);
4657 :
4658 3756 : if (cond_var)
4659 : {
4660 46 : tree itype = TREE_TYPE (cond_var);
4661 46 : tree t2;
4662 46 : if ((fd->ordered && fd->collapse == 1)
4663 46 : || bias
4664 46 : || POINTER_TYPE_P (type)
4665 46 : || TREE_CODE (fd->loop.n1) != INTEGER_CST
4666 40 : || fd->loop.cond_code != LT_EXPR)
4667 6 : t2 = build_int_cst (itype, 1);
4668 : else
4669 40 : t2 = fold_convert (itype, fd->loop.step);
4670 46 : t2 = fold_build2 (PLUS_EXPR, itype, cond_var, t2);
4671 46 : t2 = force_gimple_operand_gsi (&gsi, t2, false,
4672 : NULL_TREE, true, GSI_SAME_STMT);
4673 46 : assign_stmt = gimple_build_assign (cond_var, t2);
4674 46 : gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
4675 : }
4676 :
4677 3756 : if (!gimple_omp_for_combined_p (fd->for_stmt))
4678 : {
4679 2367 : if (POINTER_TYPE_P (type))
4680 160 : t = fold_build_pointer_plus (vmain, fd->loop.step);
4681 : else
4682 2207 : t = fold_build2 (PLUS_EXPR, type, vmain, fd->loop.step);
4683 2367 : t = force_gimple_operand_gsi (&gsi, t,
4684 2367 : DECL_P (vback)
4685 2354 : && TREE_ADDRESSABLE (vback),
4686 : NULL_TREE, true, GSI_SAME_STMT);
4687 2367 : assign_stmt = gimple_build_assign (vback, t);
4688 2367 : gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
4689 :
4690 2367 : if (fd->ordered && counts[fd->collapse - 1] == NULL_TREE)
4691 : {
4692 294 : tree tem;
4693 294 : if (fd->collapse > 1)
4694 143 : tem = fd->loop.v;
4695 : else
4696 : {
4697 151 : tem = fold_build2 (MINUS_EXPR, TREE_TYPE (fd->loops[0].v),
4698 : fd->loops[0].v, fd->loops[0].n1);
4699 151 : tem = fold_convert (fd->iter_type, tem);
4700 : }
4701 588 : tree aref = build4 (ARRAY_REF, fd->iter_type,
4702 294 : counts[fd->ordered], size_zero_node,
4703 : NULL_TREE, NULL_TREE);
4704 294 : tem = force_gimple_operand_gsi (&gsi, tem, true, NULL_TREE,
4705 : true, GSI_SAME_STMT);
4706 294 : expand_omp_build_assign (&gsi, aref, tem);
4707 : }
4708 :
4709 2367 : t = build2 (fd->loop.cond_code, boolean_type_node,
4710 2367 : DECL_P (vback) && TREE_ADDRESSABLE (vback) ? t : vback,
4711 : iend);
4712 2367 : gcond *cond_stmt = gimple_build_cond_empty (t);
4713 2367 : gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
4714 : }
4715 :
4716 : /* Remove GIMPLE_OMP_CONTINUE. */
4717 3756 : gsi_remove (&gsi, true);
4718 :
4719 3756 : if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
4720 681 : collapse_bb = extract_omp_for_update_vars (fd, NULL, cont_bb, l1_bb);
4721 :
4722 : /* Emit code to get the next parallel iteration in L2_BB. */
4723 3756 : gsi = gsi_start_bb (l2_bb);
4724 :
4725 3756 : t = build_call_expr (builtin_decl_explicit (next_fn), 2,
4726 : build_fold_addr_expr (istart0),
4727 : build_fold_addr_expr (iend0));
4728 3756 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4729 : false, GSI_CONTINUE_LINKING);
4730 3756 : if (TREE_TYPE (t) != boolean_type_node)
4731 0 : t = fold_build2 (NE_EXPR, boolean_type_node,
4732 : t, build_int_cst (TREE_TYPE (t), 0));
4733 3756 : gcond *cond_stmt = gimple_build_cond_empty (t);
4734 3756 : gsi_insert_after (&gsi, cond_stmt, GSI_CONTINUE_LINKING);
4735 : }
4736 :
4737 : /* Add the loop cleanup function. */
4738 4134 : gsi = gsi_last_nondebug_bb (exit_bb);
4739 4134 : if (gimple_omp_return_nowait_p (gsi_stmt (gsi)))
4740 3047 : t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT);
4741 1087 : else if (gimple_omp_return_lhs (gsi_stmt (gsi)))
4742 2 : t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL);
4743 : else
4744 1085 : t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END);
4745 4134 : gcall *call_stmt = gimple_build_call (t, 0);
4746 4134 : if (fd->ordered)
4747 : {
4748 335 : tree arr = counts[fd->ordered];
4749 335 : tree clobber = build_clobber (TREE_TYPE (arr));
4750 335 : gsi_insert_after (&gsi, gimple_build_assign (arr, clobber),
4751 : GSI_SAME_STMT);
4752 : }
4753 4134 : if (gimple_omp_return_lhs (gsi_stmt (gsi)))
4754 : {
4755 2 : gimple_call_set_lhs (call_stmt, gimple_omp_return_lhs (gsi_stmt (gsi)));
4756 2 : if (fd->have_reductemp)
4757 : {
4758 0 : gimple *g = gimple_build_assign (reductions, NOP_EXPR,
4759 : gimple_call_lhs (call_stmt));
4760 0 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
4761 : }
4762 : }
4763 4134 : gsi_insert_after (&gsi, call_stmt, GSI_SAME_STMT);
4764 4134 : gsi_remove (&gsi, true);
4765 :
4766 : /* Connect the new blocks. */
4767 4134 : find_edge (entry_bb, l0_bb)->flags = EDGE_TRUE_VALUE;
4768 4134 : find_edge (entry_bb, l3_bb)->flags = EDGE_FALSE_VALUE;
4769 :
4770 4134 : if (!broken_loop)
4771 : {
4772 3756 : gimple_seq phis;
4773 :
4774 3756 : e = find_edge (cont_bb, l3_bb);
4775 3756 : ne = make_edge (l2_bb, l3_bb, EDGE_FALSE_VALUE);
4776 :
4777 3756 : phis = phi_nodes (l3_bb);
4778 3778 : for (gsi = gsi_start (phis); !gsi_end_p (gsi); gsi_next (&gsi))
4779 : {
4780 11 : gimple *phi = gsi_stmt (gsi);
4781 11 : SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, ne),
4782 : PHI_ARG_DEF_FROM_EDGE (phi, e));
4783 : }
4784 3756 : remove_edge (e);
4785 :
4786 3756 : make_edge (cont_bb, l2_bb, EDGE_FALSE_VALUE);
4787 3756 : e = find_edge (cont_bb, l1_bb);
4788 3756 : if (e == NULL)
4789 : {
4790 13 : e = BRANCH_EDGE (cont_bb);
4791 13 : gcc_assert (single_succ (e->dest) == l1_bb);
4792 : }
4793 3756 : if (gimple_omp_for_combined_p (fd->for_stmt))
4794 : {
4795 1389 : remove_edge (e);
4796 1389 : e = NULL;
4797 : }
4798 2367 : else if (fd->collapse > 1)
4799 : {
4800 681 : remove_edge (e);
4801 681 : e = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
4802 : }
4803 : else
4804 1686 : e->flags = EDGE_TRUE_VALUE;
4805 3756 : if (e)
4806 : {
4807 2367 : e->probability = profile_probability::guessed_always ().apply_scale (7, 8);
4808 2367 : find_edge (cont_bb, l2_bb)->probability = e->probability.invert ();
4809 : }
4810 : else
4811 : {
4812 1389 : e = find_edge (cont_bb, l2_bb);
4813 1389 : e->flags = EDGE_FALLTHRU;
4814 : }
4815 3756 : make_edge (l2_bb, l0_bb, EDGE_TRUE_VALUE);
4816 :
4817 3756 : if (gimple_in_ssa_p (cfun))
4818 : {
4819 : /* Add phis to the outer loop that connect to the phis in the inner,
4820 : original loop, and move the loop entry value of the inner phi to
4821 : the loop entry value of the outer phi. */
4822 13 : gphi_iterator psi;
4823 24 : for (psi = gsi_start_phis (l3_bb); !gsi_end_p (psi); gsi_next (&psi))
4824 : {
4825 11 : location_t locus;
4826 11 : gphi *nphi;
4827 11 : gphi *exit_phi = psi.phi ();
4828 :
4829 22 : if (virtual_operand_p (gimple_phi_result (exit_phi)))
4830 6 : continue;
4831 :
4832 5 : edge l2_to_l3 = find_edge (l2_bb, l3_bb);
4833 5 : tree exit_res = PHI_ARG_DEF_FROM_EDGE (exit_phi, l2_to_l3);
4834 :
4835 5 : basic_block latch = BRANCH_EDGE (cont_bb)->dest;
4836 5 : edge latch_to_l1 = find_edge (latch, l1_bb);
4837 5 : gphi *inner_phi
4838 5 : = find_phi_with_arg_on_edge (exit_res, latch_to_l1);
4839 :
4840 5 : tree t = gimple_phi_result (exit_phi);
4841 5 : tree new_res = copy_ssa_name (t, NULL);
4842 5 : nphi = create_phi_node (new_res, l0_bb);
4843 :
4844 5 : edge l0_to_l1 = find_edge (l0_bb, l1_bb);
4845 5 : t = PHI_ARG_DEF_FROM_EDGE (inner_phi, l0_to_l1);
4846 5 : locus = gimple_phi_arg_location_from_edge (inner_phi, l0_to_l1);
4847 5 : edge entry_to_l0 = find_edge (entry_bb, l0_bb);
4848 5 : add_phi_arg (nphi, t, entry_to_l0, locus);
4849 :
4850 5 : edge l2_to_l0 = find_edge (l2_bb, l0_bb);
4851 5 : add_phi_arg (nphi, exit_res, l2_to_l0, UNKNOWN_LOCATION);
4852 :
4853 5 : add_phi_arg (inner_phi, new_res, l0_to_l1, UNKNOWN_LOCATION);
4854 : }
4855 : }
4856 :
4857 3756 : set_immediate_dominator (CDI_DOMINATORS, l2_bb,
4858 : recompute_dominator (CDI_DOMINATORS, l2_bb));
4859 3756 : set_immediate_dominator (CDI_DOMINATORS, l3_bb,
4860 : recompute_dominator (CDI_DOMINATORS, l3_bb));
4861 3756 : set_immediate_dominator (CDI_DOMINATORS, l0_bb,
4862 : recompute_dominator (CDI_DOMINATORS, l0_bb));
4863 3756 : set_immediate_dominator (CDI_DOMINATORS, l1_bb,
4864 : recompute_dominator (CDI_DOMINATORS, l1_bb));
4865 :
4866 : /* We enter expand_omp_for_generic with a loop. This original loop may
4867 : have its own loop struct, or it may be part of an outer loop struct
4868 : (which may be the fake loop). */
4869 3756 : class loop *outer_loop = entry_bb->loop_father;
4870 3756 : bool orig_loop_has_loop_struct = l1_bb->loop_father != outer_loop;
4871 :
4872 3756 : add_bb_to_loop (l2_bb, outer_loop);
4873 :
4874 : /* We've added a new loop around the original loop. Allocate the
4875 : corresponding loop struct. */
4876 3756 : class loop *new_loop = alloc_loop ();
4877 3756 : new_loop->header = l0_bb;
4878 3756 : new_loop->latch = l2_bb;
4879 3756 : add_loop (new_loop, outer_loop);
4880 :
4881 : /* Allocate a loop structure for the original loop unless we already
4882 : had one. */
4883 3756 : if (!orig_loop_has_loop_struct
4884 3756 : && !gimple_omp_for_combined_p (fd->for_stmt))
4885 : {
4886 2350 : class loop *orig_loop = alloc_loop ();
4887 2350 : orig_loop->header = l1_bb;
4888 : /* The loop may have multiple latches. */
4889 2350 : add_loop (orig_loop, new_loop);
4890 : }
4891 : }
4892 4134 : }
4893 :
4894 : /* Helper function for expand_omp_for_static_nochunk. If PTR is NULL,
4895 : compute needed allocation size. If !ALLOC of team allocations,
4896 : if ALLOC of thread allocation. SZ is the initial needed size for
4897 : other purposes, ALLOC_ALIGN guaranteed alignment of allocation in bytes,
4898 : CNT number of elements of each array, for !ALLOC this is
4899 : omp_get_num_threads (), for ALLOC number of iterations handled by the
4900 : current thread. If PTR is non-NULL, it is the start of the allocation
4901 : and this routine shall assign to OMP_CLAUSE_DECL (c) of those _scantemp_
4902 : clauses pointers to the corresponding arrays. */
4903 :
4904 : static tree
4905 740 : expand_omp_scantemp_alloc (tree clauses, tree ptr, unsigned HOST_WIDE_INT sz,
4906 : unsigned HOST_WIDE_INT alloc_align, tree cnt,
4907 : gimple_stmt_iterator *gsi, bool alloc)
4908 : {
4909 740 : tree eltsz = NULL_TREE;
4910 740 : unsigned HOST_WIDE_INT preval = 0;
4911 740 : if (ptr && sz)
4912 5 : ptr = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (ptr),
4913 : ptr, size_int (sz));
4914 6088 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
4915 5348 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__SCANTEMP_
4916 3216 : && !OMP_CLAUSE__SCANTEMP__CONTROL (c)
4917 7084 : && (!OMP_CLAUSE__SCANTEMP__ALLOC (c)) != alloc)
4918 : {
4919 868 : tree pointee_type = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
4920 868 : unsigned HOST_WIDE_INT al = TYPE_ALIGN_UNIT (pointee_type);
4921 868 : if (tree_fits_uhwi_p (TYPE_SIZE_UNIT (pointee_type)))
4922 : {
4923 868 : unsigned HOST_WIDE_INT szl
4924 868 : = tree_to_uhwi (TYPE_SIZE_UNIT (pointee_type));
4925 868 : szl = least_bit_hwi (szl);
4926 868 : if (szl)
4927 868 : al = MIN (al, szl);
4928 : }
4929 868 : if (ptr == NULL_TREE)
4930 : {
4931 434 : if (eltsz == NULL_TREE)
4932 370 : eltsz = TYPE_SIZE_UNIT (pointee_type);
4933 : else
4934 64 : eltsz = size_binop (PLUS_EXPR, eltsz,
4935 : TYPE_SIZE_UNIT (pointee_type));
4936 : }
4937 868 : if (preval == 0 && al <= alloc_align)
4938 : {
4939 740 : unsigned HOST_WIDE_INT diff = ROUND_UP (sz, al) - sz;
4940 740 : sz += diff;
4941 740 : if (diff && ptr)
4942 0 : ptr = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (ptr),
4943 : ptr, size_int (diff));
4944 : }
4945 128 : else if (al > preval)
4946 : {
4947 32 : if (ptr)
4948 : {
4949 16 : ptr = fold_convert (pointer_sized_int_node, ptr);
4950 16 : ptr = fold_build2 (PLUS_EXPR, pointer_sized_int_node, ptr,
4951 : build_int_cst (pointer_sized_int_node,
4952 : al - 1));
4953 16 : ptr = fold_build2 (BIT_AND_EXPR, pointer_sized_int_node, ptr,
4954 : build_int_cst (pointer_sized_int_node,
4955 : -(HOST_WIDE_INT) al));
4956 16 : ptr = fold_convert (ptr_type_node, ptr);
4957 : }
4958 : else
4959 16 : sz += al - 1;
4960 : }
4961 868 : if (tree_fits_uhwi_p (TYPE_SIZE_UNIT (pointee_type)))
4962 : preval = al;
4963 : else
4964 0 : preval = 1;
4965 868 : if (ptr)
4966 : {
4967 434 : expand_omp_build_assign (gsi, OMP_CLAUSE_DECL (c), ptr, false);
4968 434 : ptr = OMP_CLAUSE_DECL (c);
4969 434 : ptr = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (ptr), ptr,
4970 : size_binop (MULT_EXPR, cnt,
4971 : TYPE_SIZE_UNIT (pointee_type)));
4972 : }
4973 : }
4974 :
4975 740 : if (ptr == NULL_TREE)
4976 : {
4977 370 : eltsz = size_binop (MULT_EXPR, eltsz, cnt);
4978 370 : if (sz)
4979 13 : eltsz = size_binop (PLUS_EXPR, eltsz, size_int (sz));
4980 : return eltsz;
4981 : }
4982 : else
4983 : return ptr;
4984 : }
4985 :
4986 : /* Return the last _looptemp_ clause if one has been created for
4987 : lastprivate on distribute parallel for{, simd} or taskloop.
4988 : FD is the loop data and INNERC should be the second _looptemp_
4989 : clause (the one holding the end of the range).
4990 : This is followed by collapse - 1 _looptemp_ clauses for the
4991 : counts[1] and up, and for triangular loops followed by 4
4992 : further _looptemp_ clauses (one for counts[0], one first_inner_iterations,
4993 : one factor and one adjn1). After this there is optionally one
4994 : _looptemp_ clause that this function returns. */
4995 :
4996 : static tree
4997 1634 : find_lastprivate_looptemp (struct omp_for_data *fd, tree innerc)
4998 : {
4999 1634 : gcc_assert (innerc);
5000 1634 : int count = fd->collapse - 1;
5001 1634 : if (fd->non_rect
5002 24 : && fd->last_nonrect == fd->first_nonrect + 1
5003 1646 : && !TYPE_UNSIGNED (TREE_TYPE (fd->loops[fd->last_nonrect].v)))
5004 12 : count += 4;
5005 4811 : for (int i = 0; i < count; i++)
5006 : {
5007 3177 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
5008 : OMP_CLAUSE__LOOPTEMP_);
5009 3177 : gcc_assert (innerc);
5010 : }
5011 1634 : return omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
5012 1634 : OMP_CLAUSE__LOOPTEMP_);
5013 : }
5014 :
5015 : /* A subroutine of expand_omp_for. Generate code for a parallel
5016 : loop with static schedule and no specified chunk size. Given
5017 : parameters:
5018 :
5019 : for (V = N1; V cond N2; V += STEP) BODY;
5020 :
5021 : where COND is "<" or ">", we generate pseudocode
5022 :
5023 : if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5024 : if (cond is <)
5025 : adj = STEP - 1;
5026 : else
5027 : adj = STEP + 1;
5028 : if ((__typeof (V)) -1 > 0 && cond is >)
5029 : n = -(adj + N2 - N1) / -STEP;
5030 : else
5031 : n = (adj + N2 - N1) / STEP;
5032 : q = n / nthreads;
5033 : tt = n % nthreads;
5034 : if (threadid < tt) goto L3; else goto L4;
5035 : L3:
5036 : tt = 0;
5037 : q = q + 1;
5038 : L4:
5039 : s0 = q * threadid + tt;
5040 : e0 = s0 + q;
5041 : V = s0 * STEP + N1;
5042 : if (s0 >= e0) goto L2; else goto L0;
5043 : L0:
5044 : e = e0 * STEP + N1;
5045 : L1:
5046 : BODY;
5047 : V += STEP;
5048 : if (V cond e) goto L1;
5049 : L2:
5050 : */
5051 :
5052 : static void
5053 14067 : expand_omp_for_static_nochunk (struct omp_region *region,
5054 : struct omp_for_data *fd,
5055 : gimple *inner_stmt)
5056 : {
5057 14067 : tree n, q, s0, e0, e, t, tt, nthreads = NULL_TREE, threadid;
5058 14067 : tree type, itype, vmain, vback;
5059 14067 : basic_block entry_bb, second_bb, third_bb, exit_bb, seq_start_bb;
5060 14067 : basic_block body_bb, cont_bb, collapse_bb = NULL;
5061 14067 : basic_block fin_bb, fourth_bb = NULL, fifth_bb = NULL, sixth_bb = NULL;
5062 14067 : basic_block exit1_bb = NULL, exit2_bb = NULL, exit3_bb = NULL;
5063 14067 : gimple_stmt_iterator gsi, gsip;
5064 14067 : edge ep;
5065 14067 : bool broken_loop = region->cont == NULL;
5066 14067 : tree *counts = NULL;
5067 14067 : tree n1, n2, step;
5068 14067 : tree reductions = NULL_TREE;
5069 14067 : tree cond_var = NULL_TREE, condtemp = NULL_TREE;
5070 :
5071 14067 : itype = type = TREE_TYPE (fd->loop.v);
5072 14067 : if (POINTER_TYPE_P (type))
5073 606 : itype = signed_type_for (type);
5074 :
5075 14067 : entry_bb = region->entry;
5076 14067 : cont_bb = region->cont;
5077 14067 : gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
5078 14067 : fin_bb = BRANCH_EDGE (entry_bb)->dest;
5079 14067 : gcc_assert (broken_loop
5080 : || (fin_bb == FALLTHRU_EDGE (cont_bb)->dest));
5081 14067 : seq_start_bb = split_edge (FALLTHRU_EDGE (entry_bb));
5082 14067 : body_bb = single_succ (seq_start_bb);
5083 14067 : if (!broken_loop)
5084 : {
5085 13571 : gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb
5086 : || single_succ (BRANCH_EDGE (cont_bb)->dest) == body_bb);
5087 13571 : gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
5088 : }
5089 14067 : exit_bb = region->exit;
5090 :
5091 : /* Iteration space partitioning goes in ENTRY_BB. */
5092 14067 : gsi = gsi_last_nondebug_bb (entry_bb);
5093 14067 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5094 14067 : gsip = gsi;
5095 14067 : gsi_prev (&gsip);
5096 :
5097 14067 : if (fd->collapse > 1)
5098 : {
5099 3921 : int first_zero_iter = -1, dummy = -1;
5100 3921 : basic_block l2_dom_bb = NULL, dummy_bb = NULL;
5101 :
5102 3921 : counts = XALLOCAVEC (tree, fd->collapse);
5103 3921 : expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
5104 : fin_bb, first_zero_iter,
5105 : dummy_bb, dummy, l2_dom_bb);
5106 3921 : t = NULL_TREE;
5107 : }
5108 10146 : else if (gimple_omp_for_combined_into_p (fd->for_stmt))
5109 2031 : t = integer_one_node;
5110 : else
5111 8115 : t = fold_binary (fd->loop.cond_code, boolean_type_node,
5112 : fold_convert (type, fd->loop.n1),
5113 : fold_convert (type, fd->loop.n2));
5114 14067 : if (fd->collapse == 1
5115 10146 : && TYPE_UNSIGNED (type)
5116 16080 : && (t == NULL_TREE || !integer_onep (t)))
5117 : {
5118 523 : n1 = fold_convert (type, unshare_expr (fd->loop.n1));
5119 523 : n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE,
5120 : true, GSI_SAME_STMT);
5121 523 : n2 = fold_convert (type, unshare_expr (fd->loop.n2));
5122 523 : n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
5123 : true, GSI_SAME_STMT);
5124 523 : gcond *cond_stmt = expand_omp_build_cond (&gsi, fd->loop.cond_code,
5125 : n1, n2);
5126 523 : ep = split_block (entry_bb, cond_stmt);
5127 523 : ep->flags = EDGE_TRUE_VALUE;
5128 523 : entry_bb = ep->dest;
5129 523 : ep->probability = profile_probability::very_likely ();
5130 523 : ep = make_edge (ep->src, fin_bb, EDGE_FALSE_VALUE);
5131 523 : ep->probability = profile_probability::very_unlikely ();
5132 523 : if (gimple_in_ssa_p (cfun))
5133 : {
5134 22 : int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx;
5135 22 : for (gphi_iterator gpi = gsi_start_phis (fin_bb);
5136 44 : !gsi_end_p (gpi); gsi_next (&gpi))
5137 : {
5138 22 : gphi *phi = gpi.phi ();
5139 22 : add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
5140 : ep, UNKNOWN_LOCATION);
5141 : }
5142 : }
5143 1046 : gsi = gsi_last_bb (entry_bb);
5144 : }
5145 :
5146 14067 : if (fd->lastprivate_conditional)
5147 : {
5148 108 : tree clauses = gimple_omp_for_clauses (fd->for_stmt);
5149 108 : tree c = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_);
5150 108 : if (fd->have_pointer_condtemp)
5151 38 : condtemp = OMP_CLAUSE_DECL (c);
5152 108 : c = omp_find_clause (OMP_CLAUSE_CHAIN (c), OMP_CLAUSE__CONDTEMP_);
5153 108 : cond_var = OMP_CLAUSE_DECL (c);
5154 : }
5155 14067 : if (fd->have_reductemp
5156 : /* For scan, we don't want to reinitialize condtemp before the
5157 : second loop. */
5158 14014 : || (fd->have_pointer_condtemp && !fd->have_scantemp)
5159 13996 : || fd->have_nonctrl_scantemp)
5160 : {
5161 256 : tree t1 = build_int_cst (long_integer_type_node, 0);
5162 256 : tree t2 = build_int_cst (long_integer_type_node, 1);
5163 256 : tree t3 = build_int_cstu (long_integer_type_node,
5164 : (HOST_WIDE_INT_1U << 31) + 1);
5165 256 : tree clauses = gimple_omp_for_clauses (fd->for_stmt);
5166 256 : gimple_stmt_iterator gsi2 = gsi_none ();
5167 256 : gimple *g = NULL;
5168 256 : tree mem = null_pointer_node, memv = NULL_TREE;
5169 256 : unsigned HOST_WIDE_INT condtemp_sz = 0;
5170 256 : unsigned HOST_WIDE_INT alloc_align = 0;
5171 256 : if (fd->have_reductemp)
5172 : {
5173 53 : gcc_assert (!fd->have_nonctrl_scantemp);
5174 53 : tree c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
5175 53 : reductions = OMP_CLAUSE_DECL (c);
5176 53 : gcc_assert (TREE_CODE (reductions) == SSA_NAME);
5177 53 : g = SSA_NAME_DEF_STMT (reductions);
5178 53 : reductions = gimple_assign_rhs1 (g);
5179 53 : OMP_CLAUSE_DECL (c) = reductions;
5180 53 : gsi2 = gsi_for_stmt (g);
5181 : }
5182 : else
5183 : {
5184 203 : if (gsi_end_p (gsip))
5185 0 : gsi2 = gsi_after_labels (region->entry);
5186 : else
5187 203 : gsi2 = gsip;
5188 : reductions = null_pointer_node;
5189 : }
5190 256 : if (fd->have_pointer_condtemp || fd->have_nonctrl_scantemp)
5191 : {
5192 213 : tree type;
5193 213 : if (fd->have_pointer_condtemp)
5194 33 : type = TREE_TYPE (condtemp);
5195 : else
5196 180 : type = ptr_type_node;
5197 213 : memv = create_tmp_var (type);
5198 213 : TREE_ADDRESSABLE (memv) = 1;
5199 213 : unsigned HOST_WIDE_INT sz = 0;
5200 213 : tree size = NULL_TREE;
5201 213 : if (fd->have_pointer_condtemp)
5202 : {
5203 33 : sz = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (type)));
5204 33 : sz *= fd->lastprivate_conditional;
5205 33 : condtemp_sz = sz;
5206 : }
5207 213 : if (fd->have_nonctrl_scantemp)
5208 : {
5209 185 : nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
5210 185 : gimple *g = gimple_build_call (nthreads, 0);
5211 185 : nthreads = create_tmp_var (integer_type_node);
5212 185 : gimple_call_set_lhs (g, nthreads);
5213 185 : gsi_insert_before (&gsi2, g, GSI_SAME_STMT);
5214 185 : nthreads = fold_convert (sizetype, nthreads);
5215 185 : alloc_align = TYPE_ALIGN_UNIT (long_long_integer_type_node);
5216 185 : size = expand_omp_scantemp_alloc (clauses, NULL_TREE, sz,
5217 : alloc_align, nthreads, NULL,
5218 : false);
5219 185 : size = fold_convert (type, size);
5220 : }
5221 : else
5222 28 : size = build_int_cst (type, sz);
5223 213 : expand_omp_build_assign (&gsi2, memv, size, false);
5224 213 : mem = build_fold_addr_expr (memv);
5225 : }
5226 256 : tree t
5227 256 : = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_LOOP_START),
5228 : 9, t1, t2, t2, t3, t1, null_pointer_node,
5229 : null_pointer_node, reductions, mem);
5230 256 : force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
5231 : true, GSI_SAME_STMT);
5232 256 : if (fd->have_pointer_condtemp)
5233 33 : expand_omp_build_assign (&gsi2, condtemp, memv, false);
5234 256 : if (fd->have_nonctrl_scantemp)
5235 : {
5236 185 : tree ptr = fd->have_pointer_condtemp ? condtemp : memv;
5237 185 : expand_omp_scantemp_alloc (clauses, ptr, condtemp_sz,
5238 : alloc_align, nthreads, &gsi2, false);
5239 : }
5240 256 : if (fd->have_reductemp)
5241 : {
5242 53 : gsi_remove (&gsi2, true);
5243 53 : release_ssa_name (gimple_assign_lhs (g));
5244 : }
5245 : }
5246 :
5247 14067 : n1 = fd->loop.n1;
5248 14067 : n2 = fd->loop.n2;
5249 14067 : step = fd->loop.step;
5250 14067 : if (gimple_omp_for_combined_into_p (fd->for_stmt))
5251 : {
5252 3075 : tree innerc = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
5253 : OMP_CLAUSE__LOOPTEMP_);
5254 3075 : gcc_assert (innerc);
5255 3075 : n1 = OMP_CLAUSE_DECL (innerc);
5256 3075 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
5257 : OMP_CLAUSE__LOOPTEMP_);
5258 3075 : gcc_assert (innerc);
5259 3075 : n2 = OMP_CLAUSE_DECL (innerc);
5260 : }
5261 14067 : n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
5262 : true, NULL_TREE, true, GSI_SAME_STMT);
5263 14067 : n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
5264 : true, NULL_TREE, true, GSI_SAME_STMT);
5265 14067 : step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
5266 : true, NULL_TREE, true, GSI_SAME_STMT);
5267 :
5268 15086 : t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
5269 14067 : t = fold_build2 (PLUS_EXPR, itype, step, t);
5270 14067 : t = fold_build2 (PLUS_EXPR, itype, t, n2);
5271 14067 : t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1));
5272 14067 : if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
5273 414 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
5274 : fold_build1 (NEGATE_EXPR, itype, t),
5275 : fold_build1 (NEGATE_EXPR, itype, step));
5276 : else
5277 13653 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
5278 14067 : t = fold_convert (itype, t);
5279 14067 : n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
5280 :
5281 : /* When GOMP_loop_end (or one of its variants) is emitted (e.g. with the
5282 : inscan modifier), which already implies the end of the scope, _start
5283 : variants of GOMP builtin calls have to be used and _end can be skipped. */
5284 28134 : bool has_gomp_loop_end = fd->have_reductemp
5285 14067 : || ((fd->have_pointer_condtemp || fd->have_scantemp)
5286 388 : && !fd->have_nonctrl_scantemp);
5287 14067 : {
5288 : /* Fetch the thread/team id and the number of threads/teams in a single
5289 : call to GOMP_loop_static_worksharing or
5290 : GOMP_distribute_static_worksharing. The helper returns both values packed
5291 : into one complex int, with the id as the imaginary part and the count as
5292 : the real part. Returning (rather than writing through pointers) keeps
5293 : both values as plain SSA names, which lets later passes - notably IPA-CP
5294 : propagating constants into the outlined kernel - reason about them.
5295 : Also pass the total number of iterations for OMPT. */
5296 14067 : tree decl;
5297 14067 : switch (gimple_omp_for_kind (fd->for_stmt))
5298 : {
5299 9494 : case GF_OMP_FOR_KIND_FOR:
5300 18988 : decl = builtin_decl_explicit (
5301 9494 : flag_openmp_ompt || has_gomp_loop_end
5302 : ? BUILT_IN_GOMP_LOOP_STATIC_WORKSHARING_START
5303 : : BUILT_IN_GOMP_LOOP_STATIC_WORKSHARING);
5304 9494 : break;
5305 4573 : case GF_OMP_FOR_KIND_DISTRIBUTE:
5306 9146 : decl = builtin_decl_explicit (
5307 4573 : flag_openmp_ompt || has_gomp_loop_end
5308 : ? BUILT_IN_GOMP_DISTRIBUTE_STATIC_WORKSHARING_START
5309 : : BUILT_IN_GOMP_DISTRIBUTE_STATIC_WORKSHARING);
5310 4573 : break;
5311 0 : default:
5312 0 : gcc_unreachable ();
5313 : }
5314 14067 : tree n_ull = fold_convert (long_long_unsigned_type_node, n);
5315 14067 : tree packed = build_call_expr (decl, 1, n_ull);
5316 14067 : packed = force_gimple_operand_gsi (&gsi, packed, true, NULL_TREE,
5317 : true, GSI_SAME_STMT);
5318 14067 : threadid = fold_build1 (IMAGPART_EXPR, integer_type_node, packed);
5319 14067 : threadid = fold_convert (itype, threadid);
5320 14067 : threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE,
5321 : true, GSI_SAME_STMT);
5322 14067 : nthreads = fold_build1 (REALPART_EXPR, integer_type_node, packed);
5323 14067 : nthreads = fold_convert (itype, nthreads);
5324 14067 : nthreads = force_gimple_operand_gsi (&gsi, nthreads, true, NULL_TREE,
5325 : true, GSI_SAME_STMT);
5326 : }
5327 :
5328 14067 : q = create_tmp_reg (itype, "q");
5329 14067 : t = fold_build2 (TRUNC_DIV_EXPR, itype, n, nthreads);
5330 14067 : t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
5331 14067 : gsi_insert_before (&gsi, gimple_build_assign (q, t), GSI_SAME_STMT);
5332 :
5333 14067 : tt = create_tmp_reg (itype, "tt");
5334 14067 : t = fold_build2 (TRUNC_MOD_EXPR, itype, n, nthreads);
5335 14067 : t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
5336 14067 : gsi_insert_before (&gsi, gimple_build_assign (tt, t), GSI_SAME_STMT);
5337 :
5338 14067 : t = build2 (LT_EXPR, boolean_type_node, threadid, tt);
5339 14067 : gcond *cond_stmt = gimple_build_cond_empty (t);
5340 14067 : gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
5341 :
5342 14067 : second_bb = split_block (entry_bb, cond_stmt)->dest;
5343 14067 : gsi = gsi_last_nondebug_bb (second_bb);
5344 14067 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5345 :
5346 14067 : gsi_insert_before (&gsi, gimple_build_assign (tt, build_int_cst (itype, 0)),
5347 : GSI_SAME_STMT);
5348 14067 : gassign *assign_stmt
5349 14067 : = gimple_build_assign (q, PLUS_EXPR, q, build_int_cst (itype, 1));
5350 14067 : gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
5351 :
5352 14067 : third_bb = split_block (second_bb, assign_stmt)->dest;
5353 14067 : gsi = gsi_last_nondebug_bb (third_bb);
5354 14067 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5355 :
5356 14067 : if (fd->have_nonctrl_scantemp)
5357 : {
5358 185 : tree clauses = gimple_omp_for_clauses (fd->for_stmt);
5359 185 : tree controlp = NULL_TREE, controlb = NULL_TREE;
5360 804 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
5361 804 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__SCANTEMP_
5362 804 : && OMP_CLAUSE__SCANTEMP__CONTROL (c))
5363 : {
5364 370 : if (TREE_TYPE (OMP_CLAUSE_DECL (c)) == boolean_type_node)
5365 185 : controlb = OMP_CLAUSE_DECL (c);
5366 : else
5367 185 : controlp = OMP_CLAUSE_DECL (c);
5368 370 : if (controlb && controlp)
5369 : break;
5370 : }
5371 185 : gcc_assert (controlp && controlb);
5372 185 : tree cnt = create_tmp_var (sizetype);
5373 185 : gimple *g = gimple_build_assign (cnt, NOP_EXPR, q);
5374 185 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5375 185 : unsigned HOST_WIDE_INT alloc_align = TYPE_ALIGN_UNIT (ptr_type_node);
5376 185 : tree sz = expand_omp_scantemp_alloc (clauses, NULL_TREE, 0,
5377 : alloc_align, cnt, NULL, true);
5378 185 : tree size = create_tmp_var (sizetype);
5379 185 : expand_omp_build_assign (&gsi, size, sz, false);
5380 185 : tree cmp = fold_build2 (GT_EXPR, boolean_type_node,
5381 : size, size_int (16384));
5382 185 : expand_omp_build_assign (&gsi, controlb, cmp);
5383 185 : g = gimple_build_cond (NE_EXPR, controlb, boolean_false_node,
5384 : NULL_TREE, NULL_TREE);
5385 185 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5386 185 : fourth_bb = split_block (third_bb, g)->dest;
5387 185 : gsi = gsi_last_nondebug_bb (fourth_bb);
5388 : /* FIXME: Once we have allocators, this should use allocator. */
5389 185 : g = gimple_build_call (builtin_decl_explicit (BUILT_IN_MALLOC), 1, size);
5390 185 : gimple_call_set_lhs (g, controlp);
5391 185 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5392 185 : expand_omp_scantemp_alloc (clauses, controlp, 0, alloc_align, cnt,
5393 : &gsi, true);
5394 185 : gsi_prev (&gsi);
5395 185 : g = gsi_stmt (gsi);
5396 185 : fifth_bb = split_block (fourth_bb, g)->dest;
5397 185 : gsi = gsi_last_nondebug_bb (fifth_bb);
5398 :
5399 370 : g = gimple_build_call (builtin_decl_implicit (BUILT_IN_STACK_SAVE), 0);
5400 185 : gimple_call_set_lhs (g, controlp);
5401 185 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5402 185 : tree alloca_decl = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
5403 1522 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
5404 1337 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__SCANTEMP_
5405 1337 : && OMP_CLAUSE__SCANTEMP__ALLOC (c))
5406 : {
5407 217 : tree tmp = create_tmp_var (sizetype);
5408 217 : tree pointee_type = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
5409 217 : g = gimple_build_assign (tmp, MULT_EXPR, cnt,
5410 217 : TYPE_SIZE_UNIT (pointee_type));
5411 217 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5412 217 : g = gimple_build_call (alloca_decl, 2, tmp,
5413 217 : size_int (TYPE_ALIGN (pointee_type)));
5414 217 : gimple_call_set_lhs (g, OMP_CLAUSE_DECL (c));
5415 217 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5416 : }
5417 :
5418 185 : sixth_bb = split_block (fifth_bb, g)->dest;
5419 185 : gsi = gsi_last_nondebug_bb (sixth_bb);
5420 : }
5421 :
5422 14067 : t = build2 (MULT_EXPR, itype, q, threadid);
5423 14067 : t = build2 (PLUS_EXPR, itype, t, tt);
5424 14067 : s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
5425 :
5426 14067 : t = fold_build2 (PLUS_EXPR, itype, s0, q);
5427 14067 : e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
5428 :
5429 14067 : t = build2 (GE_EXPR, boolean_type_node, s0, e0);
5430 14067 : gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
5431 :
5432 : /* Remove the GIMPLE_OMP_FOR statement. */
5433 14067 : gsi_remove (&gsi, true);
5434 :
5435 : /* Setup code for sequential iteration goes in SEQ_START_BB. */
5436 14067 : gsi = gsi_start_bb (seq_start_bb);
5437 :
5438 14067 : tree startvar = fd->loop.v;
5439 14067 : tree endvar = NULL_TREE;
5440 :
5441 14067 : if (gimple_omp_for_combined_p (fd->for_stmt))
5442 : {
5443 7112 : tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
5444 10668 : ? gimple_omp_parallel_clauses (inner_stmt)
5445 3556 : : gimple_omp_for_clauses (inner_stmt);
5446 7112 : tree innerc = omp_find_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
5447 7112 : gcc_assert (innerc);
5448 7112 : startvar = OMP_CLAUSE_DECL (innerc);
5449 7112 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
5450 : OMP_CLAUSE__LOOPTEMP_);
5451 7112 : gcc_assert (innerc);
5452 7112 : endvar = OMP_CLAUSE_DECL (innerc);
5453 2311 : if (fd->collapse > 1 && TREE_CODE (fd->loop.n2) != INTEGER_CST
5454 8434 : && gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
5455 : {
5456 829 : innerc = find_lastprivate_looptemp (fd, innerc);
5457 829 : if (innerc)
5458 : {
5459 : /* If needed (distribute parallel for with lastprivate),
5460 : propagate down the total number of iterations. */
5461 375 : tree t = fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc)),
5462 : fd->loop.n2);
5463 375 : t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, false,
5464 : GSI_CONTINUE_LINKING);
5465 375 : assign_stmt = gimple_build_assign (OMP_CLAUSE_DECL (innerc), t);
5466 375 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
5467 : }
5468 : }
5469 : }
5470 14067 : t = fold_convert (itype, s0);
5471 14067 : t = fold_build2 (MULT_EXPR, itype, t, step);
5472 14067 : if (POINTER_TYPE_P (type))
5473 : {
5474 606 : t = fold_build_pointer_plus (n1, t);
5475 670 : if (!POINTER_TYPE_P (TREE_TYPE (startvar))
5476 670 : && TYPE_PRECISION (TREE_TYPE (startvar)) > TYPE_PRECISION (type))
5477 0 : t = fold_convert (signed_type_for (type), t);
5478 : }
5479 : else
5480 13461 : t = fold_build2 (PLUS_EXPR, type, t, n1);
5481 14067 : t = fold_convert (TREE_TYPE (startvar), t);
5482 14067 : t = force_gimple_operand_gsi (&gsi, t,
5483 14067 : DECL_P (startvar)
5484 13897 : && TREE_ADDRESSABLE (startvar),
5485 : NULL_TREE, false, GSI_CONTINUE_LINKING);
5486 14067 : assign_stmt = gimple_build_assign (startvar, t);
5487 14067 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
5488 14067 : if (cond_var)
5489 : {
5490 108 : tree itype = TREE_TYPE (cond_var);
5491 : /* For lastprivate(conditional:) itervar, we need some iteration
5492 : counter that starts at unsigned non-zero and increases.
5493 : Prefer as few IVs as possible, so if we can use startvar
5494 : itself, use that, or startvar + constant (those would be
5495 : incremented with step), and as last resort use the s0 + 1
5496 : incremented by 1. */
5497 108 : if (POINTER_TYPE_P (type)
5498 108 : || TREE_CODE (n1) != INTEGER_CST
5499 102 : || fd->loop.cond_code != LT_EXPR)
5500 6 : t = fold_build2 (PLUS_EXPR, itype, fold_convert (itype, s0),
5501 : build_int_cst (itype, 1));
5502 102 : else if (tree_int_cst_sgn (n1) == 1)
5503 12 : t = fold_convert (itype, t);
5504 : else
5505 : {
5506 90 : tree c = fold_convert (itype, n1);
5507 90 : c = fold_build2 (MINUS_EXPR, itype, build_int_cst (itype, 1), c);
5508 90 : t = fold_build2 (PLUS_EXPR, itype, fold_convert (itype, t), c);
5509 : }
5510 108 : t = force_gimple_operand_gsi (&gsi, t, false,
5511 : NULL_TREE, false, GSI_CONTINUE_LINKING);
5512 108 : assign_stmt = gimple_build_assign (cond_var, t);
5513 108 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
5514 : }
5515 :
5516 14067 : t = fold_convert (itype, e0);
5517 14067 : t = fold_build2 (MULT_EXPR, itype, t, step);
5518 14067 : if (POINTER_TYPE_P (type))
5519 : {
5520 606 : t = fold_build_pointer_plus (n1, t);
5521 670 : if (!POINTER_TYPE_P (TREE_TYPE (startvar))
5522 670 : && TYPE_PRECISION (TREE_TYPE (startvar)) > TYPE_PRECISION (type))
5523 0 : t = fold_convert (signed_type_for (type), t);
5524 : }
5525 : else
5526 13461 : t = fold_build2 (PLUS_EXPR, type, t, n1);
5527 14067 : t = fold_convert (TREE_TYPE (startvar), t);
5528 14067 : e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5529 : false, GSI_CONTINUE_LINKING);
5530 14067 : if (endvar)
5531 : {
5532 7112 : assign_stmt = gimple_build_assign (endvar, e);
5533 7112 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
5534 7112 : if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e)))
5535 6343 : assign_stmt = gimple_build_assign (fd->loop.v, e);
5536 : else
5537 769 : assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e);
5538 7112 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
5539 : }
5540 : /* Handle linear clause adjustments. */
5541 14067 : tree itercnt = NULL_TREE;
5542 14067 : tree *nonrect_bounds = NULL;
5543 14067 : if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_FOR)
5544 45461 : for (tree c = gimple_omp_for_clauses (fd->for_stmt);
5545 45461 : c; c = OMP_CLAUSE_CHAIN (c))
5546 35967 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
5547 35967 : && !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
5548 : {
5549 87 : tree d = OMP_CLAUSE_DECL (c);
5550 87 : tree t = d, a, dest;
5551 87 : if (omp_privatize_by_reference (t))
5552 13 : t = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), t);
5553 87 : if (itercnt == NULL_TREE)
5554 : {
5555 87 : if (gimple_omp_for_combined_into_p (fd->for_stmt))
5556 : {
5557 0 : itercnt = fold_build2 (MINUS_EXPR, itype,
5558 : fold_convert (itype, n1),
5559 : fold_convert (itype, fd->loop.n1));
5560 0 : itercnt = fold_build2 (EXACT_DIV_EXPR, itype, itercnt, step);
5561 0 : itercnt = fold_build2 (PLUS_EXPR, itype, itercnt, s0);
5562 0 : itercnt = force_gimple_operand_gsi (&gsi, itercnt, true,
5563 : NULL_TREE, false,
5564 : GSI_CONTINUE_LINKING);
5565 : }
5566 : else
5567 : itercnt = s0;
5568 : }
5569 87 : tree type = TREE_TYPE (t);
5570 87 : if (POINTER_TYPE_P (type))
5571 0 : type = sizetype;
5572 87 : a = fold_build2 (MULT_EXPR, type,
5573 : fold_convert (type, itercnt),
5574 : fold_convert (type, OMP_CLAUSE_LINEAR_STEP (c)));
5575 87 : dest = unshare_expr (t);
5576 87 : t = fold_build2 (type == TREE_TYPE (t) ? PLUS_EXPR
5577 : : POINTER_PLUS_EXPR, TREE_TYPE (t), t, a);
5578 87 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5579 : false, GSI_CONTINUE_LINKING);
5580 87 : expand_omp_build_assign (&gsi, dest, t, true);
5581 : }
5582 14067 : if (fd->collapse > 1)
5583 : {
5584 3921 : if (fd->non_rect)
5585 : {
5586 376 : nonrect_bounds = XALLOCAVEC (tree, fd->last_nonrect + 1);
5587 376 : memset (nonrect_bounds, 0, sizeof (tree) * (fd->last_nonrect + 1));
5588 : }
5589 3921 : expand_omp_for_init_vars (fd, &gsi, counts, nonrect_bounds, inner_stmt,
5590 : startvar);
5591 : }
5592 :
5593 14067 : if (!broken_loop)
5594 : {
5595 : /* The code controlling the sequential loop replaces the
5596 : GIMPLE_OMP_CONTINUE. */
5597 13571 : gsi = gsi_last_nondebug_bb (cont_bb);
5598 13571 : gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
5599 13571 : gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE);
5600 13571 : vmain = gimple_omp_continue_control_use (cont_stmt);
5601 13571 : vback = gimple_omp_continue_control_def (cont_stmt);
5602 :
5603 13571 : if (cond_var)
5604 : {
5605 108 : tree itype = TREE_TYPE (cond_var);
5606 108 : tree t2;
5607 108 : if (POINTER_TYPE_P (type)
5608 108 : || TREE_CODE (n1) != INTEGER_CST
5609 102 : || fd->loop.cond_code != LT_EXPR)
5610 6 : t2 = build_int_cst (itype, 1);
5611 : else
5612 102 : t2 = fold_convert (itype, step);
5613 108 : t2 = fold_build2 (PLUS_EXPR, itype, cond_var, t2);
5614 108 : t2 = force_gimple_operand_gsi (&gsi, t2, false,
5615 : NULL_TREE, true, GSI_SAME_STMT);
5616 108 : assign_stmt = gimple_build_assign (cond_var, t2);
5617 108 : gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
5618 : }
5619 :
5620 13571 : if (!gimple_omp_for_combined_p (fd->for_stmt))
5621 : {
5622 6459 : if (POINTER_TYPE_P (type))
5623 316 : t = fold_build_pointer_plus (vmain, step);
5624 : else
5625 6143 : t = fold_build2 (PLUS_EXPR, type, vmain, step);
5626 6459 : t = force_gimple_operand_gsi (&gsi, t,
5627 6459 : DECL_P (vback)
5628 6289 : && TREE_ADDRESSABLE (vback),
5629 : NULL_TREE, true, GSI_SAME_STMT);
5630 6459 : assign_stmt = gimple_build_assign (vback, t);
5631 6459 : gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
5632 :
5633 6459 : t = build2 (fd->loop.cond_code, boolean_type_node,
5634 6459 : DECL_P (vback) && TREE_ADDRESSABLE (vback)
5635 : ? t : vback, e);
5636 6459 : gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
5637 : }
5638 :
5639 : /* Remove the GIMPLE_OMP_CONTINUE statement. */
5640 13571 : gsi_remove (&gsi, true);
5641 :
5642 13571 : if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
5643 1382 : collapse_bb = extract_omp_for_update_vars (fd, nonrect_bounds,
5644 : cont_bb, body_bb);
5645 : }
5646 :
5647 14067 : if (flag_openmp_ompt == OMP_OMPT_LEVEL_EXTENDED)
5648 : {
5649 : /* Insert call to GOMP_*_static_worksharing_dispatch at the end of
5650 : seq_start_bb. */
5651 8 : gsi = gsi_last_nondebug_bb (seq_start_bb);
5652 8 : tree decl;
5653 8 : switch (gimple_omp_for_kind (fd->for_stmt))
5654 : {
5655 4 : case GF_OMP_FOR_KIND_FOR:
5656 4 : decl = builtin_decl_explicit (
5657 : BUILT_IN_GOMP_LOOP_STATIC_WORKSHARING_DISPATCH);
5658 4 : break;
5659 4 : case GF_OMP_FOR_KIND_DISTRIBUTE:
5660 4 : decl = builtin_decl_explicit (
5661 : BUILT_IN_GOMP_DISTRIBUTE_STATIC_WORKSHARING_DISPATCH);
5662 4 : break;
5663 0 : default:
5664 0 : gcc_unreachable ();
5665 : }
5666 8 : tree s0_ull = fold_convert (long_long_unsigned_type_node, s0);
5667 8 : tree q_ull = fold_convert (long_long_unsigned_type_node, q);
5668 8 : tree call = build_call_expr (decl, 2, s0_ull, q_ull);
5669 8 : force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true,
5670 : GSI_SAME_STMT);
5671 : }
5672 :
5673 14067 : gsi = gsi_last_nondebug_bb (exit_bb);
5674 14067 : if (flag_openmp_ompt && !has_gomp_loop_end)
5675 : {
5676 : /* Insert call to GOMP_*_static_worksharing_end at the end of exit_bb.
5677 : */
5678 20 : tree decl;
5679 20 : switch (gimple_omp_for_kind (fd->for_stmt))
5680 : {
5681 12 : case GF_OMP_FOR_KIND_FOR:
5682 12 : decl
5683 12 : = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_STATIC_WORKSHARING_END);
5684 12 : break;
5685 8 : case GF_OMP_FOR_KIND_DISTRIBUTE:
5686 8 : decl = builtin_decl_explicit (
5687 : BUILT_IN_GOMP_DISTRIBUTE_STATIC_WORKSHARING_END);
5688 8 : break;
5689 0 : default:
5690 0 : gcc_unreachable ();
5691 : }
5692 20 : gcall *g = gimple_build_call (decl, 0);
5693 20 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
5694 : }
5695 :
5696 : /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
5697 14067 : if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
5698 : {
5699 2399 : t = gimple_omp_return_lhs (gsi_stmt (gsi));
5700 2399 : if (fd->have_reductemp
5701 2346 : || ((fd->have_pointer_condtemp || fd->have_scantemp)
5702 63 : && !fd->have_nonctrl_scantemp))
5703 : {
5704 116 : tree fn;
5705 116 : if (t)
5706 4 : fn = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL);
5707 : else
5708 112 : fn = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END);
5709 116 : gcall *g = gimple_build_call (fn, 0);
5710 116 : if (t)
5711 : {
5712 4 : gimple_call_set_lhs (g, t);
5713 4 : if (fd->have_reductemp)
5714 4 : gsi_insert_after (&gsi, gimple_build_assign (reductions,
5715 : NOP_EXPR, t),
5716 : GSI_SAME_STMT);
5717 : }
5718 116 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
5719 116 : }
5720 : else
5721 2283 : gsi_insert_after (&gsi,
5722 : omp_build_barrier (t,
5723 : GOMP_BARRIER_IMPLICIT_WORKSHARE),
5724 : GSI_SAME_STMT);
5725 : }
5726 11668 : else if ((fd->have_pointer_condtemp || fd->have_scantemp)
5727 325 : && !fd->have_nonctrl_scantemp)
5728 : {
5729 140 : tree fn = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT);
5730 140 : gcall *g = gimple_build_call (fn, 0);
5731 140 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
5732 : }
5733 14067 : if (fd->have_scantemp && !fd->have_nonctrl_scantemp)
5734 : {
5735 185 : tree clauses = gimple_omp_for_clauses (fd->for_stmt);
5736 185 : tree controlp = NULL_TREE, controlb = NULL_TREE;
5737 370 : for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
5738 370 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__SCANTEMP_
5739 370 : && OMP_CLAUSE__SCANTEMP__CONTROL (c))
5740 : {
5741 370 : if (TREE_TYPE (OMP_CLAUSE_DECL (c)) == boolean_type_node)
5742 185 : controlb = OMP_CLAUSE_DECL (c);
5743 : else
5744 185 : controlp = OMP_CLAUSE_DECL (c);
5745 370 : if (controlb && controlp)
5746 : break;
5747 : }
5748 185 : gcc_assert (controlp && controlb);
5749 185 : gimple *g = gimple_build_cond (NE_EXPR, controlb, boolean_false_node,
5750 : NULL_TREE, NULL_TREE);
5751 185 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5752 185 : exit1_bb = split_block (exit_bb, g)->dest;
5753 185 : gsi = gsi_after_labels (exit1_bb);
5754 185 : g = gimple_build_call (builtin_decl_explicit (BUILT_IN_FREE), 1,
5755 : controlp);
5756 185 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5757 185 : exit2_bb = split_block (exit1_bb, g)->dest;
5758 185 : gsi = gsi_after_labels (exit2_bb);
5759 370 : g = gimple_build_call (builtin_decl_implicit (BUILT_IN_STACK_RESTORE), 1,
5760 : controlp);
5761 185 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5762 185 : exit3_bb = split_block (exit2_bb, g)->dest;
5763 185 : gsi = gsi_after_labels (exit3_bb);
5764 : }
5765 :
5766 14067 : gsi_remove (&gsi, true);
5767 :
5768 : /* Connect all the blocks. */
5769 14067 : ep = make_edge (entry_bb, third_bb, EDGE_FALSE_VALUE);
5770 14067 : ep->probability = profile_probability::guessed_always ().apply_scale (3, 4);
5771 14067 : ep = find_edge (entry_bb, second_bb);
5772 14067 : ep->flags = EDGE_TRUE_VALUE;
5773 14067 : ep->probability = profile_probability::guessed_always () / 4;
5774 14067 : if (fourth_bb)
5775 : {
5776 185 : ep = make_edge (third_bb, fifth_bb, EDGE_FALSE_VALUE);
5777 185 : ep->probability = profile_probability::guessed_always () / 2;
5778 185 : ep = find_edge (third_bb, fourth_bb);
5779 185 : ep->flags = EDGE_TRUE_VALUE;
5780 185 : ep->probability = profile_probability::guessed_always () / 2;
5781 185 : ep = find_edge (fourth_bb, fifth_bb);
5782 185 : redirect_edge_and_branch (ep, sixth_bb);
5783 : }
5784 : else
5785 : sixth_bb = third_bb;
5786 14067 : find_edge (sixth_bb, seq_start_bb)->flags = EDGE_FALSE_VALUE;
5787 14067 : find_edge (sixth_bb, fin_bb)->flags = EDGE_TRUE_VALUE;
5788 14067 : if (exit1_bb)
5789 : {
5790 185 : ep = make_edge (exit_bb, exit2_bb, EDGE_FALSE_VALUE);
5791 185 : ep->probability = profile_probability::guessed_always () / 2;
5792 185 : ep = find_edge (exit_bb, exit1_bb);
5793 185 : ep->flags = EDGE_TRUE_VALUE;
5794 185 : ep->probability = profile_probability::guessed_always () / 2;
5795 185 : ep = find_edge (exit1_bb, exit2_bb);
5796 185 : redirect_edge_and_branch (ep, exit3_bb);
5797 : }
5798 :
5799 14067 : if (!broken_loop)
5800 : {
5801 13571 : ep = find_edge (cont_bb, body_bb);
5802 13571 : if (ep == NULL)
5803 : {
5804 170 : ep = BRANCH_EDGE (cont_bb);
5805 170 : gcc_assert (single_succ (ep->dest) == body_bb);
5806 : }
5807 13571 : if (gimple_omp_for_combined_p (fd->for_stmt))
5808 : {
5809 7112 : remove_edge (ep);
5810 7112 : ep = NULL;
5811 : }
5812 6459 : else if (fd->collapse > 1)
5813 : {
5814 1382 : remove_edge (ep);
5815 1382 : ep = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
5816 : }
5817 : else
5818 5077 : ep->flags = EDGE_TRUE_VALUE;
5819 20683 : find_edge (cont_bb, fin_bb)->flags
5820 14953 : = ep ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
5821 : }
5822 :
5823 14067 : set_immediate_dominator (CDI_DOMINATORS, second_bb, entry_bb);
5824 14067 : set_immediate_dominator (CDI_DOMINATORS, third_bb, entry_bb);
5825 14067 : if (fourth_bb)
5826 : {
5827 185 : set_immediate_dominator (CDI_DOMINATORS, fifth_bb, third_bb);
5828 185 : set_immediate_dominator (CDI_DOMINATORS, sixth_bb, third_bb);
5829 : }
5830 14067 : set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, sixth_bb);
5831 :
5832 14067 : set_immediate_dominator (CDI_DOMINATORS, body_bb,
5833 : recompute_dominator (CDI_DOMINATORS, body_bb));
5834 14067 : set_immediate_dominator (CDI_DOMINATORS, fin_bb,
5835 : recompute_dominator (CDI_DOMINATORS, fin_bb));
5836 14067 : if (exit1_bb)
5837 : {
5838 185 : set_immediate_dominator (CDI_DOMINATORS, exit2_bb, exit_bb);
5839 185 : set_immediate_dominator (CDI_DOMINATORS, exit3_bb, exit_bb);
5840 : }
5841 :
5842 14067 : class loop *loop = body_bb->loop_father;
5843 14067 : if (loop != entry_bb->loop_father)
5844 : {
5845 332 : gcc_assert (broken_loop || loop->header == body_bb);
5846 170 : gcc_assert (broken_loop
5847 : || loop->latch == region->cont
5848 : || single_pred (loop->latch) == region->cont);
5849 332 : return;
5850 : }
5851 :
5852 13735 : if (!broken_loop && !gimple_omp_for_combined_p (fd->for_stmt))
5853 : {
5854 6289 : loop = alloc_loop ();
5855 6289 : loop->header = body_bb;
5856 6289 : if (collapse_bb == NULL)
5857 4907 : loop->latch = cont_bb;
5858 6289 : add_loop (loop, body_bb->loop_father);
5859 : }
5860 : }
5861 :
5862 : /* Return phi in E->DEST with ARG on edge E. */
5863 :
5864 : static gphi *
5865 14 : find_phi_with_arg_on_edge (tree arg, edge e)
5866 : {
5867 14 : basic_block bb = e->dest;
5868 :
5869 14 : for (gphi_iterator gpi = gsi_start_phis (bb);
5870 14 : !gsi_end_p (gpi);
5871 0 : gsi_next (&gpi))
5872 : {
5873 14 : gphi *phi = gpi.phi ();
5874 14 : if (PHI_ARG_DEF_FROM_EDGE (phi, e) == arg)
5875 14 : return phi;
5876 : }
5877 :
5878 0 : return NULL;
5879 : }
5880 :
5881 : /* A subroutine of expand_omp_for. Generate code for a parallel
5882 : loop with static schedule and a specified chunk size. Given
5883 : parameters:
5884 :
5885 : for (V = N1; V cond N2; V += STEP) BODY;
5886 :
5887 : where COND is "<" or ">", we generate pseudocode
5888 :
5889 : if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5890 : if (cond is <)
5891 : adj = STEP - 1;
5892 : else
5893 : adj = STEP + 1;
5894 : if ((__typeof (V)) -1 > 0 && cond is >)
5895 : n = -(adj + N2 - N1) / -STEP;
5896 : else
5897 : n = (adj + N2 - N1) / STEP;
5898 : trip = 0;
5899 : V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
5900 : here so that V is defined
5901 : if the loop is not entered
5902 : L0:
5903 : s0 = (trip * nthreads + threadid) * CHUNK;
5904 : e0 = min (s0 + CHUNK, n);
5905 : if (s0 < n) goto L1; else goto L4;
5906 : L1:
5907 : V = s0 * STEP + N1;
5908 : e = e0 * STEP + N1;
5909 : L2:
5910 : BODY;
5911 : V += STEP;
5912 : if (V cond e) goto L2; else goto L3;
5913 : L3:
5914 : trip += 1;
5915 : goto L0;
5916 : L4:
5917 : */
5918 :
5919 : static void
5920 5855 : expand_omp_for_static_chunk (struct omp_region *region,
5921 : struct omp_for_data *fd, gimple *inner_stmt)
5922 : {
5923 5855 : tree n, s0, e0, e, t;
5924 5855 : tree trip_var, trip_init, trip_main, trip_back, nthreads, threadid;
5925 5855 : tree type, itype, vmain, vback, vextra;
5926 5855 : basic_block entry_bb, exit_bb, body_bb, seq_start_bb, iter_part_bb;
5927 5855 : basic_block trip_update_bb = NULL, cont_bb, collapse_bb = NULL, fin_bb;
5928 5855 : gimple_stmt_iterator gsi, gsip;
5929 5855 : edge se;
5930 5855 : bool broken_loop = region->cont == NULL;
5931 5855 : tree *counts = NULL;
5932 5855 : tree n1, n2, step;
5933 5855 : tree reductions = NULL_TREE;
5934 5855 : tree cond_var = NULL_TREE, condtemp = NULL_TREE;
5935 :
5936 5855 : itype = type = TREE_TYPE (fd->loop.v);
5937 5855 : if (POINTER_TYPE_P (type))
5938 316 : itype = signed_type_for (type);
5939 :
5940 5855 : entry_bb = region->entry;
5941 5855 : se = split_block (entry_bb, last_nondebug_stmt (entry_bb));
5942 5855 : entry_bb = se->src;
5943 5855 : iter_part_bb = se->dest;
5944 5855 : cont_bb = region->cont;
5945 5855 : gcc_assert (EDGE_COUNT (iter_part_bb->succs) == 2);
5946 5855 : fin_bb = BRANCH_EDGE (iter_part_bb)->dest;
5947 5855 : gcc_assert (broken_loop
5948 : || fin_bb == FALLTHRU_EDGE (cont_bb)->dest);
5949 5855 : seq_start_bb = split_edge (FALLTHRU_EDGE (iter_part_bb));
5950 5855 : body_bb = single_succ (seq_start_bb);
5951 5855 : if (!broken_loop)
5952 : {
5953 5598 : gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb
5954 : || single_succ (BRANCH_EDGE (cont_bb)->dest) == body_bb);
5955 5598 : gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
5956 5598 : trip_update_bb = split_edge (FALLTHRU_EDGE (cont_bb));
5957 : }
5958 5855 : exit_bb = region->exit;
5959 :
5960 : /* Trip and adjustment setup goes in ENTRY_BB. */
5961 5855 : gsi = gsi_last_nondebug_bb (entry_bb);
5962 5855 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5963 5855 : gsip = gsi;
5964 5855 : gsi_prev (&gsip);
5965 :
5966 5855 : if (fd->collapse > 1)
5967 : {
5968 2232 : int first_zero_iter = -1, dummy = -1;
5969 2232 : basic_block l2_dom_bb = NULL, dummy_bb = NULL;
5970 :
5971 2232 : counts = XALLOCAVEC (tree, fd->collapse);
5972 2232 : expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
5973 : fin_bb, first_zero_iter,
5974 : dummy_bb, dummy, l2_dom_bb);
5975 2232 : t = NULL_TREE;
5976 : }
5977 3623 : else if (gimple_omp_for_combined_into_p (fd->for_stmt))
5978 813 : t = integer_one_node;
5979 : else
5980 2810 : t = fold_binary (fd->loop.cond_code, boolean_type_node,
5981 : fold_convert (type, fd->loop.n1),
5982 : fold_convert (type, fd->loop.n2));
5983 5855 : if (fd->collapse == 1
5984 3623 : && TYPE_UNSIGNED (type)
5985 7042 : && (t == NULL_TREE || !integer_onep (t)))
5986 : {
5987 236 : n1 = fold_convert (type, unshare_expr (fd->loop.n1));
5988 236 : n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE,
5989 : true, GSI_SAME_STMT);
5990 236 : n2 = fold_convert (type, unshare_expr (fd->loop.n2));
5991 236 : n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
5992 : true, GSI_SAME_STMT);
5993 236 : gcond *cond_stmt = expand_omp_build_cond (&gsi, fd->loop.cond_code,
5994 : n1, n2);
5995 236 : se = split_block (entry_bb, cond_stmt);
5996 236 : se->flags = EDGE_TRUE_VALUE;
5997 236 : entry_bb = se->dest;
5998 236 : se->probability = profile_probability::very_likely ();
5999 236 : se = make_edge (se->src, fin_bb, EDGE_FALSE_VALUE);
6000 236 : se->probability = profile_probability::very_unlikely ();
6001 236 : if (gimple_in_ssa_p (cfun))
6002 : {
6003 1 : int dest_idx = find_edge (iter_part_bb, fin_bb)->dest_idx;
6004 1 : for (gphi_iterator gpi = gsi_start_phis (fin_bb);
6005 2 : !gsi_end_p (gpi); gsi_next (&gpi))
6006 : {
6007 1 : gphi *phi = gpi.phi ();
6008 1 : add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
6009 : se, UNKNOWN_LOCATION);
6010 : }
6011 : }
6012 472 : gsi = gsi_last_bb (entry_bb);
6013 : }
6014 :
6015 5855 : if (fd->lastprivate_conditional)
6016 : {
6017 49 : tree clauses = gimple_omp_for_clauses (fd->for_stmt);
6018 49 : tree c = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_);
6019 49 : if (fd->have_pointer_condtemp)
6020 29 : condtemp = OMP_CLAUSE_DECL (c);
6021 49 : c = omp_find_clause (OMP_CLAUSE_CHAIN (c), OMP_CLAUSE__CONDTEMP_);
6022 49 : cond_var = OMP_CLAUSE_DECL (c);
6023 : }
6024 5855 : if (fd->have_reductemp || fd->have_pointer_condtemp)
6025 : {
6026 44 : tree t1 = build_int_cst (long_integer_type_node, 0);
6027 44 : tree t2 = build_int_cst (long_integer_type_node, 1);
6028 44 : tree t3 = build_int_cstu (long_integer_type_node,
6029 : (HOST_WIDE_INT_1U << 31) + 1);
6030 44 : tree clauses = gimple_omp_for_clauses (fd->for_stmt);
6031 44 : gimple_stmt_iterator gsi2 = gsi_none ();
6032 44 : gimple *g = NULL;
6033 44 : tree mem = null_pointer_node, memv = NULL_TREE;
6034 44 : if (fd->have_reductemp)
6035 : {
6036 25 : tree c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
6037 25 : reductions = OMP_CLAUSE_DECL (c);
6038 25 : gcc_assert (TREE_CODE (reductions) == SSA_NAME);
6039 25 : g = SSA_NAME_DEF_STMT (reductions);
6040 25 : reductions = gimple_assign_rhs1 (g);
6041 25 : OMP_CLAUSE_DECL (c) = reductions;
6042 25 : gsi2 = gsi_for_stmt (g);
6043 : }
6044 : else
6045 : {
6046 19 : if (gsi_end_p (gsip))
6047 0 : gsi2 = gsi_after_labels (region->entry);
6048 : else
6049 19 : gsi2 = gsip;
6050 : reductions = null_pointer_node;
6051 : }
6052 44 : if (fd->have_pointer_condtemp)
6053 : {
6054 29 : tree type = TREE_TYPE (condtemp);
6055 29 : memv = create_tmp_var (type);
6056 29 : TREE_ADDRESSABLE (memv) = 1;
6057 29 : unsigned HOST_WIDE_INT sz
6058 29 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (type)));
6059 29 : sz *= fd->lastprivate_conditional;
6060 29 : expand_omp_build_assign (&gsi2, memv, build_int_cst (type, sz),
6061 : false);
6062 29 : mem = build_fold_addr_expr (memv);
6063 : }
6064 44 : tree t
6065 44 : = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_LOOP_START),
6066 : 9, t1, t2, t2, t3, t1, null_pointer_node,
6067 : null_pointer_node, reductions, mem);
6068 44 : force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
6069 : true, GSI_SAME_STMT);
6070 44 : if (fd->have_pointer_condtemp)
6071 29 : expand_omp_build_assign (&gsi2, condtemp, memv, false);
6072 44 : if (fd->have_reductemp)
6073 : {
6074 25 : gsi_remove (&gsi2, true);
6075 25 : release_ssa_name (gimple_assign_lhs (g));
6076 : }
6077 : }
6078 :
6079 5855 : n1 = fd->loop.n1;
6080 5855 : n2 = fd->loop.n2;
6081 5855 : step = fd->loop.step;
6082 5855 : if (gimple_omp_for_combined_into_p (fd->for_stmt))
6083 : {
6084 1301 : tree innerc = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
6085 : OMP_CLAUSE__LOOPTEMP_);
6086 1301 : gcc_assert (innerc);
6087 1301 : n1 = OMP_CLAUSE_DECL (innerc);
6088 1301 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
6089 : OMP_CLAUSE__LOOPTEMP_);
6090 1301 : gcc_assert (innerc);
6091 1301 : n2 = OMP_CLAUSE_DECL (innerc);
6092 : }
6093 5855 : n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
6094 : true, NULL_TREE, true, GSI_SAME_STMT);
6095 5855 : n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
6096 : true, NULL_TREE, true, GSI_SAME_STMT);
6097 5855 : step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
6098 : true, NULL_TREE, true, GSI_SAME_STMT);
6099 5855 : tree chunk_size = fold_convert (itype, fd->chunk_size);
6100 5855 : chunk_size = omp_adjust_chunk_size (chunk_size, fd->simd_schedule,
6101 : is_in_offload_region (region));
6102 5855 : chunk_size
6103 5855 : = force_gimple_operand_gsi (&gsi, chunk_size, true, NULL_TREE, true,
6104 : GSI_SAME_STMT);
6105 :
6106 6461 : t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
6107 5855 : t = fold_build2 (PLUS_EXPR, itype, step, t);
6108 5855 : t = fold_build2 (PLUS_EXPR, itype, t, n2);
6109 5855 : t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1));
6110 5855 : if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
6111 284 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
6112 : fold_build1 (NEGATE_EXPR, itype, t),
6113 : fold_build1 (NEGATE_EXPR, itype, step));
6114 : else
6115 5571 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
6116 5855 : t = fold_convert (itype, t);
6117 5855 : n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6118 : true, GSI_SAME_STMT);
6119 :
6120 : /* When GOMP_loop_end (or one of its variants) is emitted (e.g. with the
6121 : inscan modifier), which already implies the end of the scope, _start
6122 : variants of GOMP builtin calls have to be used and _end can be skipped. */
6123 5855 : bool has_gomp_loop_end = fd->have_reductemp || fd->have_pointer_condtemp;
6124 5855 : {
6125 : /* Fetch the thread/team id and the number of threads/teams in a single
6126 : call to GOMP_loop_static_worksharing or
6127 : GOMP_distribute_static_worksharing. The helper returns both values packed
6128 : into one complex int, with the id as the imaginary part and the count as
6129 : the real part. Returning (rather than writing through pointers) keeps
6130 : both values as plain SSA names, which lets later passes - notably IPA-CP
6131 : propagating constants into the outlined kernel - reason about them.
6132 : Also pass the total number of iterations for OMPT. */
6133 5855 : tree decl;
6134 5855 : switch (gimple_omp_for_kind (fd->for_stmt))
6135 : {
6136 2385 : case GF_OMP_FOR_KIND_FOR:
6137 4770 : decl = builtin_decl_explicit (
6138 2385 : flag_openmp_ompt || has_gomp_loop_end
6139 : ? BUILT_IN_GOMP_LOOP_STATIC_WORKSHARING_START
6140 : : BUILT_IN_GOMP_LOOP_STATIC_WORKSHARING);
6141 2385 : break;
6142 3470 : case GF_OMP_FOR_KIND_DISTRIBUTE:
6143 6940 : decl = builtin_decl_explicit (
6144 3470 : flag_openmp_ompt || has_gomp_loop_end
6145 : ? BUILT_IN_GOMP_DISTRIBUTE_STATIC_WORKSHARING_START
6146 : : BUILT_IN_GOMP_DISTRIBUTE_STATIC_WORKSHARING);
6147 3470 : break;
6148 0 : default:
6149 0 : gcc_unreachable ();
6150 : }
6151 5855 : tree n_ull = fold_convert (long_long_unsigned_type_node, n);
6152 5855 : tree packed = build_call_expr (decl, 1, n_ull);
6153 5855 : packed = force_gimple_operand_gsi (&gsi, packed, true, NULL_TREE, true,
6154 : GSI_SAME_STMT);
6155 5855 : threadid = fold_build1 (IMAGPART_EXPR, integer_type_node, packed);
6156 5855 : threadid = fold_convert (itype, threadid);
6157 5855 : threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE, true,
6158 : GSI_SAME_STMT);
6159 5855 : nthreads = fold_build1 (REALPART_EXPR, integer_type_node, packed);
6160 5855 : nthreads = fold_convert (itype, nthreads);
6161 5855 : nthreads = force_gimple_operand_gsi (&gsi, nthreads, true, NULL_TREE, true,
6162 : GSI_SAME_STMT);
6163 : }
6164 :
6165 5855 : trip_var = create_tmp_reg (itype, ".trip");
6166 5855 : if (gimple_in_ssa_p (cfun))
6167 : {
6168 12 : trip_init = make_ssa_name (trip_var);
6169 12 : trip_main = make_ssa_name (trip_var);
6170 12 : trip_back = make_ssa_name (trip_var);
6171 : }
6172 : else
6173 : {
6174 : trip_init = trip_var;
6175 : trip_main = trip_var;
6176 : trip_back = trip_var;
6177 : }
6178 :
6179 5855 : gassign *assign_stmt
6180 5855 : = gimple_build_assign (trip_init, build_int_cst (itype, 0));
6181 5855 : gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
6182 :
6183 5855 : t = fold_build2 (MULT_EXPR, itype, threadid, chunk_size);
6184 5855 : t = fold_build2 (MULT_EXPR, itype, t, step);
6185 5855 : if (POINTER_TYPE_P (type))
6186 316 : t = fold_build_pointer_plus (n1, t);
6187 : else
6188 5539 : t = fold_build2 (PLUS_EXPR, type, t, n1);
6189 5855 : vextra = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6190 : true, GSI_SAME_STMT);
6191 :
6192 : /* Remove the GIMPLE_OMP_FOR. */
6193 5855 : gsi_remove (&gsi, true);
6194 :
6195 5855 : gimple_stmt_iterator gsif = gsi;
6196 :
6197 : /* Iteration space partitioning goes in ITER_PART_BB. */
6198 5855 : gsi = gsi_last_bb (iter_part_bb);
6199 :
6200 5855 : t = fold_build2 (MULT_EXPR, itype, trip_main, nthreads);
6201 5855 : t = fold_build2 (PLUS_EXPR, itype, t, threadid);
6202 5855 : t = fold_build2 (MULT_EXPR, itype, t, chunk_size);
6203 5855 : s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6204 : false, GSI_CONTINUE_LINKING);
6205 :
6206 5855 : t = fold_build2 (PLUS_EXPR, itype, s0, chunk_size);
6207 5855 : t = fold_build2 (MIN_EXPR, itype, t, n);
6208 5855 : e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6209 : false, GSI_CONTINUE_LINKING);
6210 :
6211 5855 : t = build2 (LT_EXPR, boolean_type_node, s0, n);
6212 5855 : gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_CONTINUE_LINKING);
6213 :
6214 : /* Setup code for sequential iteration goes in SEQ_START_BB. */
6215 5855 : gsi = gsi_start_bb (seq_start_bb);
6216 :
6217 5855 : tree startvar = fd->loop.v;
6218 5855 : tree endvar = NULL_TREE;
6219 :
6220 5855 : if (gimple_omp_for_combined_p (fd->for_stmt))
6221 : {
6222 4202 : tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
6223 6943 : ? gimple_omp_parallel_clauses (inner_stmt)
6224 1461 : : gimple_omp_for_clauses (inner_stmt);
6225 4202 : tree innerc = omp_find_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
6226 4202 : gcc_assert (innerc);
6227 4202 : startvar = OMP_CLAUSE_DECL (innerc);
6228 4202 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
6229 : OMP_CLAUSE__LOOPTEMP_);
6230 4202 : gcc_assert (innerc);
6231 4202 : endvar = OMP_CLAUSE_DECL (innerc);
6232 1663 : if (fd->collapse > 1 && TREE_CODE (fd->loop.n2) != INTEGER_CST
6233 5087 : && gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
6234 : {
6235 704 : innerc = find_lastprivate_looptemp (fd, innerc);
6236 704 : if (innerc)
6237 : {
6238 : /* If needed (distribute parallel for with lastprivate),
6239 : propagate down the total number of iterations. */
6240 320 : tree t = fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc)),
6241 : fd->loop.n2);
6242 320 : t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, false,
6243 : GSI_CONTINUE_LINKING);
6244 320 : assign_stmt = gimple_build_assign (OMP_CLAUSE_DECL (innerc), t);
6245 320 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6246 : }
6247 : }
6248 : }
6249 :
6250 5855 : t = fold_convert (itype, s0);
6251 5855 : t = fold_build2 (MULT_EXPR, itype, t, step);
6252 5855 : if (POINTER_TYPE_P (type))
6253 : {
6254 316 : t = fold_build_pointer_plus (n1, t);
6255 380 : if (!POINTER_TYPE_P (TREE_TYPE (startvar))
6256 380 : && TYPE_PRECISION (TREE_TYPE (startvar)) > TYPE_PRECISION (type))
6257 0 : t = fold_convert (signed_type_for (type), t);
6258 : }
6259 : else
6260 5539 : t = fold_build2 (PLUS_EXPR, type, t, n1);
6261 5855 : t = fold_convert (TREE_TYPE (startvar), t);
6262 5855 : t = force_gimple_operand_gsi (&gsi, t,
6263 5855 : DECL_P (startvar)
6264 5843 : && TREE_ADDRESSABLE (startvar),
6265 : NULL_TREE, false, GSI_CONTINUE_LINKING);
6266 5855 : assign_stmt = gimple_build_assign (startvar, t);
6267 5855 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6268 5855 : if (cond_var)
6269 : {
6270 49 : tree itype = TREE_TYPE (cond_var);
6271 : /* For lastprivate(conditional:) itervar, we need some iteration
6272 : counter that starts at unsigned non-zero and increases.
6273 : Prefer as few IVs as possible, so if we can use startvar
6274 : itself, use that, or startvar + constant (those would be
6275 : incremented with step), and as last resort use the s0 + 1
6276 : incremented by 1. */
6277 49 : if (POINTER_TYPE_P (type)
6278 49 : || TREE_CODE (n1) != INTEGER_CST
6279 43 : || fd->loop.cond_code != LT_EXPR)
6280 6 : t = fold_build2 (PLUS_EXPR, itype, fold_convert (itype, s0),
6281 : build_int_cst (itype, 1));
6282 43 : else if (tree_int_cst_sgn (n1) == 1)
6283 21 : t = fold_convert (itype, t);
6284 : else
6285 : {
6286 22 : tree c = fold_convert (itype, n1);
6287 22 : c = fold_build2 (MINUS_EXPR, itype, build_int_cst (itype, 1), c);
6288 22 : t = fold_build2 (PLUS_EXPR, itype, fold_convert (itype, t), c);
6289 : }
6290 49 : t = force_gimple_operand_gsi (&gsi, t, false,
6291 : NULL_TREE, false, GSI_CONTINUE_LINKING);
6292 49 : assign_stmt = gimple_build_assign (cond_var, t);
6293 49 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6294 : }
6295 :
6296 5855 : t = fold_convert (itype, e0);
6297 5855 : t = fold_build2 (MULT_EXPR, itype, t, step);
6298 5855 : if (POINTER_TYPE_P (type))
6299 : {
6300 316 : t = fold_build_pointer_plus (n1, t);
6301 380 : if (!POINTER_TYPE_P (TREE_TYPE (startvar))
6302 380 : && TYPE_PRECISION (TREE_TYPE (startvar)) > TYPE_PRECISION (type))
6303 0 : t = fold_convert (signed_type_for (type), t);
6304 : }
6305 : else
6306 5539 : t = fold_build2 (PLUS_EXPR, type, t, n1);
6307 5855 : t = fold_convert (TREE_TYPE (startvar), t);
6308 5855 : e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6309 : false, GSI_CONTINUE_LINKING);
6310 5855 : if (endvar)
6311 : {
6312 4202 : assign_stmt = gimple_build_assign (endvar, e);
6313 4202 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6314 4202 : if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e)))
6315 3434 : assign_stmt = gimple_build_assign (fd->loop.v, e);
6316 : else
6317 768 : assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e);
6318 4202 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6319 : }
6320 : /* Handle linear clause adjustments. */
6321 5855 : tree itercnt = NULL_TREE, itercntbias = NULL_TREE;
6322 5855 : if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_FOR)
6323 15866 : for (tree c = gimple_omp_for_clauses (fd->for_stmt);
6324 15866 : c; c = OMP_CLAUSE_CHAIN (c))
6325 13481 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
6326 13481 : && !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
6327 : {
6328 68 : tree d = OMP_CLAUSE_DECL (c);
6329 68 : tree t = d, a, dest;
6330 68 : if (omp_privatize_by_reference (t))
6331 3 : t = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), t);
6332 68 : tree type = TREE_TYPE (t);
6333 68 : if (POINTER_TYPE_P (type))
6334 0 : type = sizetype;
6335 68 : dest = unshare_expr (t);
6336 68 : tree v = create_tmp_var (TREE_TYPE (t), NULL);
6337 68 : expand_omp_build_assign (&gsif, v, t);
6338 68 : if (itercnt == NULL_TREE)
6339 : {
6340 68 : if (gimple_omp_for_combined_into_p (fd->for_stmt))
6341 : {
6342 0 : itercntbias
6343 0 : = fold_build2 (MINUS_EXPR, itype, fold_convert (itype, n1),
6344 : fold_convert (itype, fd->loop.n1));
6345 0 : itercntbias = fold_build2 (EXACT_DIV_EXPR, itype,
6346 : itercntbias, step);
6347 0 : itercntbias
6348 0 : = force_gimple_operand_gsi (&gsif, itercntbias, true,
6349 : NULL_TREE, true,
6350 : GSI_SAME_STMT);
6351 0 : itercnt = fold_build2 (PLUS_EXPR, itype, itercntbias, s0);
6352 0 : itercnt = force_gimple_operand_gsi (&gsi, itercnt, true,
6353 : NULL_TREE, false,
6354 : GSI_CONTINUE_LINKING);
6355 : }
6356 : else
6357 : itercnt = s0;
6358 : }
6359 68 : a = fold_build2 (MULT_EXPR, type,
6360 : fold_convert (type, itercnt),
6361 : fold_convert (type, OMP_CLAUSE_LINEAR_STEP (c)));
6362 68 : t = fold_build2 (type == TREE_TYPE (t) ? PLUS_EXPR
6363 : : POINTER_PLUS_EXPR, TREE_TYPE (t), v, a);
6364 68 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6365 : false, GSI_CONTINUE_LINKING);
6366 68 : expand_omp_build_assign (&gsi, dest, t, true);
6367 : }
6368 5855 : if (fd->collapse > 1)
6369 2232 : expand_omp_for_init_vars (fd, &gsi, counts, NULL, inner_stmt, startvar);
6370 :
6371 5855 : if (!broken_loop)
6372 : {
6373 : /* The code controlling the sequential loop goes in CONT_BB,
6374 : replacing the GIMPLE_OMP_CONTINUE. */
6375 5598 : gsi = gsi_last_nondebug_bb (cont_bb);
6376 5598 : gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
6377 5598 : vmain = gimple_omp_continue_control_use (cont_stmt);
6378 5598 : vback = gimple_omp_continue_control_def (cont_stmt);
6379 :
6380 5598 : if (cond_var)
6381 : {
6382 49 : tree itype = TREE_TYPE (cond_var);
6383 49 : tree t2;
6384 49 : if (POINTER_TYPE_P (type)
6385 49 : || TREE_CODE (n1) != INTEGER_CST
6386 43 : || fd->loop.cond_code != LT_EXPR)
6387 6 : t2 = build_int_cst (itype, 1);
6388 : else
6389 43 : t2 = fold_convert (itype, step);
6390 49 : t2 = fold_build2 (PLUS_EXPR, itype, cond_var, t2);
6391 49 : t2 = force_gimple_operand_gsi (&gsi, t2, false,
6392 : NULL_TREE, true, GSI_SAME_STMT);
6393 49 : assign_stmt = gimple_build_assign (cond_var, t2);
6394 49 : gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
6395 : }
6396 :
6397 5598 : if (!gimple_omp_for_combined_p (fd->for_stmt))
6398 : {
6399 1396 : if (POINTER_TYPE_P (type))
6400 96 : t = fold_build_pointer_plus (vmain, step);
6401 : else
6402 1300 : t = fold_build2 (PLUS_EXPR, type, vmain, step);
6403 1396 : t = force_gimple_operand_gsi (&gsi, t,
6404 1396 : DECL_P (vback)
6405 1384 : && TREE_ADDRESSABLE (vback), NULL_TREE,
6406 : true, GSI_SAME_STMT);
6407 1396 : assign_stmt = gimple_build_assign (vback, t);
6408 1396 : gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
6409 :
6410 1396 : if (tree_int_cst_equal (fd->chunk_size, integer_one_node))
6411 94 : t = build2 (EQ_EXPR, boolean_type_node,
6412 : build_int_cst (itype, 0),
6413 : build_int_cst (itype, 1));
6414 : else
6415 1302 : t = build2 (fd->loop.cond_code, boolean_type_node,
6416 1302 : DECL_P (vback) && TREE_ADDRESSABLE (vback)
6417 : ? t : vback, e);
6418 1396 : gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
6419 : }
6420 :
6421 : /* Remove GIMPLE_OMP_CONTINUE. */
6422 5598 : gsi_remove (&gsi, true);
6423 :
6424 5598 : if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
6425 449 : collapse_bb = extract_omp_for_update_vars (fd, NULL, cont_bb, body_bb);
6426 :
6427 : /* Trip update code goes into TRIP_UPDATE_BB. */
6428 5598 : gsi = gsi_start_bb (trip_update_bb);
6429 :
6430 5598 : t = build_int_cst (itype, 1);
6431 5598 : t = build2 (PLUS_EXPR, itype, trip_main, t);
6432 5598 : assign_stmt = gimple_build_assign (trip_back, t);
6433 5598 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6434 : }
6435 :
6436 5855 : gsi = gsi_last_nondebug_bb (exit_bb);
6437 5855 : if (flag_openmp_ompt && !has_gomp_loop_end)
6438 : {
6439 : /* Insert call to GOMP_*_static_worksharing_end at the end of exit_bb.
6440 : */
6441 16 : tree decl;
6442 16 : switch (gimple_omp_for_kind (fd->for_stmt))
6443 : {
6444 8 : case GF_OMP_FOR_KIND_FOR:
6445 8 : decl
6446 8 : = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_STATIC_WORKSHARING_END);
6447 8 : break;
6448 8 : case GF_OMP_FOR_KIND_DISTRIBUTE:
6449 8 : decl = builtin_decl_explicit (
6450 : BUILT_IN_GOMP_DISTRIBUTE_STATIC_WORKSHARING_END);
6451 8 : break;
6452 0 : default:
6453 0 : gcc_unreachable ();
6454 : }
6455 16 : gcall *g = gimple_build_call (decl, 0);
6456 16 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
6457 : }
6458 :
6459 : /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6460 5855 : if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
6461 : {
6462 320 : t = gimple_omp_return_lhs (gsi_stmt (gsi));
6463 320 : if (fd->have_reductemp || fd->have_pointer_condtemp)
6464 : {
6465 35 : tree fn;
6466 35 : if (t)
6467 0 : fn = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL);
6468 : else
6469 35 : fn = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END);
6470 35 : gcall *g = gimple_build_call (fn, 0);
6471 35 : if (t)
6472 : {
6473 0 : gimple_call_set_lhs (g, t);
6474 0 : if (fd->have_reductemp)
6475 0 : gsi_insert_after (&gsi, gimple_build_assign (reductions,
6476 : NOP_EXPR, t),
6477 : GSI_SAME_STMT);
6478 : }
6479 35 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
6480 35 : }
6481 : else
6482 285 : gsi_insert_after (&gsi,
6483 : omp_build_barrier (t,
6484 : GOMP_BARRIER_IMPLICIT_WORKSHARE),
6485 : GSI_SAME_STMT);
6486 : }
6487 5535 : else if (fd->have_pointer_condtemp)
6488 : {
6489 9 : tree fn = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT);
6490 9 : gcall *g = gimple_build_call (fn, 0);
6491 9 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
6492 : }
6493 5855 : gsi_remove (&gsi, true);
6494 :
6495 5855 : if (flag_openmp_ompt == OMP_OMPT_LEVEL_EXTENDED)
6496 : {
6497 : /* Insert call to GOMP_*_static_worksharing_dispatch at the end of
6498 : seq_start_bb. */
6499 8 : gsi = gsi_last_nondebug_bb (seq_start_bb);
6500 8 : tree decl;
6501 8 : switch (gimple_omp_for_kind (fd->for_stmt))
6502 : {
6503 4 : case GF_OMP_FOR_KIND_FOR:
6504 4 : decl = builtin_decl_explicit (
6505 : BUILT_IN_GOMP_LOOP_STATIC_WORKSHARING_DISPATCH);
6506 4 : break;
6507 4 : case GF_OMP_FOR_KIND_DISTRIBUTE:
6508 4 : decl = builtin_decl_explicit (
6509 : BUILT_IN_GOMP_DISTRIBUTE_STATIC_WORKSHARING_DISPATCH);
6510 4 : break;
6511 0 : default:
6512 0 : gcc_unreachable ();
6513 : }
6514 8 : tree s0_ull = fold_convert (long_long_unsigned_type_node, s0);
6515 8 : tree iterations = fold_build2 (MINUS_EXPR, itype, e0, s0);
6516 8 : iterations = fold_convert (long_long_unsigned_type_node, iterations);
6517 8 : tree call = build_call_expr (decl, 2, s0_ull, iterations);
6518 8 : force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true,
6519 : GSI_SAME_STMT);
6520 : }
6521 :
6522 : /* Connect the new blocks. */
6523 5855 : find_edge (iter_part_bb, seq_start_bb)->flags = EDGE_TRUE_VALUE;
6524 5855 : find_edge (iter_part_bb, fin_bb)->flags = EDGE_FALSE_VALUE;
6525 :
6526 5855 : if (!broken_loop)
6527 : {
6528 5598 : se = find_edge (cont_bb, body_bb);
6529 5598 : if (se == NULL)
6530 : {
6531 12 : se = BRANCH_EDGE (cont_bb);
6532 12 : gcc_assert (single_succ (se->dest) == body_bb);
6533 : }
6534 5598 : if (gimple_omp_for_combined_p (fd->for_stmt))
6535 : {
6536 4202 : remove_edge (se);
6537 4202 : se = NULL;
6538 : }
6539 1396 : else if (fd->collapse > 1)
6540 : {
6541 449 : remove_edge (se);
6542 449 : se = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
6543 : }
6544 : else
6545 947 : se->flags = EDGE_TRUE_VALUE;
6546 9800 : find_edge (cont_bb, trip_update_bb)->flags
6547 449 : = se ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
6548 :
6549 5598 : redirect_edge_and_branch (single_succ_edge (trip_update_bb),
6550 : iter_part_bb);
6551 : }
6552 :
6553 5855 : if (gimple_in_ssa_p (cfun))
6554 : {
6555 12 : gphi_iterator psi;
6556 12 : gphi *phi;
6557 12 : edge re, ene;
6558 12 : edge_var_map *vm;
6559 12 : size_t i;
6560 :
6561 12 : gcc_assert (fd->collapse == 1 && !broken_loop);
6562 :
6563 : /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6564 : remove arguments of the phi nodes in fin_bb. We need to create
6565 : appropriate phi nodes in iter_part_bb instead. */
6566 12 : se = find_edge (iter_part_bb, fin_bb);
6567 12 : re = single_succ_edge (trip_update_bb);
6568 12 : vec<edge_var_map> *head = redirect_edge_var_map_vector (re);
6569 12 : ene = single_succ_edge (entry_bb);
6570 :
6571 12 : psi = gsi_start_phis (fin_bb);
6572 33 : for (i = 0; !gsi_end_p (psi) && head->iterate (i, &vm);
6573 9 : gsi_next (&psi), ++i)
6574 : {
6575 9 : gphi *nphi;
6576 9 : location_t locus;
6577 :
6578 9 : phi = psi.phi ();
6579 9 : if (operand_equal_p (gimple_phi_arg_def (phi, 0),
6580 9 : redirect_edge_var_map_def (vm), 0))
6581 0 : continue;
6582 :
6583 9 : t = gimple_phi_result (phi);
6584 9 : gcc_assert (t == redirect_edge_var_map_result (vm));
6585 :
6586 9 : if (!single_pred_p (fin_bb))
6587 1 : t = copy_ssa_name (t, phi);
6588 :
6589 9 : nphi = create_phi_node (t, iter_part_bb);
6590 :
6591 9 : t = PHI_ARG_DEF_FROM_EDGE (phi, se);
6592 9 : locus = gimple_phi_arg_location_from_edge (phi, se);
6593 :
6594 : /* A special case -- fd->loop.v is not yet computed in
6595 : iter_part_bb, we need to use vextra instead. */
6596 9 : if (t == fd->loop.v)
6597 0 : t = vextra;
6598 9 : add_phi_arg (nphi, t, ene, locus);
6599 9 : locus = redirect_edge_var_map_location (vm);
6600 9 : tree back_arg = redirect_edge_var_map_def (vm);
6601 9 : add_phi_arg (nphi, back_arg, re, locus);
6602 9 : edge ce = find_edge (cont_bb, body_bb);
6603 9 : if (ce == NULL)
6604 : {
6605 9 : ce = BRANCH_EDGE (cont_bb);
6606 9 : gcc_assert (single_succ (ce->dest) == body_bb);
6607 : ce = single_succ_edge (ce->dest);
6608 : }
6609 9 : gphi *inner_loop_phi = find_phi_with_arg_on_edge (back_arg, ce);
6610 9 : gcc_assert (inner_loop_phi != NULL);
6611 9 : add_phi_arg (inner_loop_phi, gimple_phi_result (nphi),
6612 : find_edge (seq_start_bb, body_bb), locus);
6613 :
6614 9 : if (!single_pred_p (fin_bb))
6615 1 : add_phi_arg (phi, gimple_phi_result (nphi), se, locus);
6616 : }
6617 21 : gcc_assert (gsi_end_p (psi) && (head == NULL || i == head->length ()));
6618 12 : redirect_edge_var_map_clear (re);
6619 12 : if (single_pred_p (fin_bb))
6620 27 : while (1)
6621 : {
6622 19 : psi = gsi_start_phis (fin_bb);
6623 19 : if (gsi_end_p (psi))
6624 : break;
6625 8 : remove_phi_node (&psi, false);
6626 : }
6627 :
6628 : /* Make phi node for trip. */
6629 12 : phi = create_phi_node (trip_main, iter_part_bb);
6630 12 : add_phi_arg (phi, trip_back, single_succ_edge (trip_update_bb),
6631 : UNKNOWN_LOCATION);
6632 12 : add_phi_arg (phi, trip_init, single_succ_edge (entry_bb),
6633 : UNKNOWN_LOCATION);
6634 : }
6635 :
6636 5855 : if (!broken_loop)
6637 5598 : set_immediate_dominator (CDI_DOMINATORS, trip_update_bb, cont_bb);
6638 5855 : set_immediate_dominator (CDI_DOMINATORS, iter_part_bb,
6639 : recompute_dominator (CDI_DOMINATORS, iter_part_bb));
6640 5855 : set_immediate_dominator (CDI_DOMINATORS, fin_bb,
6641 : recompute_dominator (CDI_DOMINATORS, fin_bb));
6642 5855 : set_immediate_dominator (CDI_DOMINATORS, seq_start_bb,
6643 : recompute_dominator (CDI_DOMINATORS, seq_start_bb));
6644 5855 : set_immediate_dominator (CDI_DOMINATORS, body_bb,
6645 : recompute_dominator (CDI_DOMINATORS, body_bb));
6646 :
6647 5855 : if (!broken_loop)
6648 : {
6649 5598 : class loop *loop = body_bb->loop_father;
6650 5598 : class loop *trip_loop = alloc_loop ();
6651 5598 : trip_loop->header = iter_part_bb;
6652 5598 : trip_loop->latch = trip_update_bb;
6653 5598 : add_loop (trip_loop, iter_part_bb->loop_father);
6654 :
6655 5598 : if (loop != entry_bb->loop_father)
6656 : {
6657 12 : gcc_assert (loop->header == body_bb);
6658 12 : gcc_assert (loop->latch == region->cont
6659 : || single_pred (loop->latch) == region->cont);
6660 12 : trip_loop->inner = loop;
6661 12 : return;
6662 : }
6663 :
6664 5586 : if (!gimple_omp_for_combined_p (fd->for_stmt))
6665 : {
6666 1384 : loop = alloc_loop ();
6667 1384 : loop->header = body_bb;
6668 1384 : if (collapse_bb == NULL)
6669 935 : loop->latch = cont_bb;
6670 1384 : add_loop (loop, trip_loop);
6671 : }
6672 : }
6673 : }
6674 :
6675 : /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
6676 : loop. Given parameters:
6677 :
6678 : for (V = N1; V cond N2; V += STEP) BODY;
6679 :
6680 : where COND is "<" or ">", we generate pseudocode
6681 :
6682 : V = N1;
6683 : goto L1;
6684 : L0:
6685 : BODY;
6686 : V += STEP;
6687 : L1:
6688 : if (V cond N2) goto L0; else goto L2;
6689 : L2:
6690 :
6691 : For collapsed loops, emit the outer loops as scalar
6692 : and only try to vectorize the innermost loop. */
6693 :
6694 : static void
6695 9374 : expand_omp_simd (struct omp_region *region, struct omp_for_data *fd)
6696 : {
6697 9374 : tree type, t;
6698 9374 : basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, l2_bb, l2_dom_bb;
6699 9374 : gimple_stmt_iterator gsi;
6700 9374 : gimple *stmt;
6701 9374 : gcond *cond_stmt;
6702 9374 : bool broken_loop = region->cont == NULL;
6703 9374 : edge e, ne;
6704 9374 : tree *counts = NULL;
6705 9374 : int i;
6706 9374 : int safelen_int = INT_MAX;
6707 9374 : bool dont_vectorize = false;
6708 9374 : tree safelen = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
6709 : OMP_CLAUSE_SAFELEN);
6710 9374 : tree simduid = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
6711 : OMP_CLAUSE__SIMDUID_);
6712 9374 : tree ifc = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
6713 : OMP_CLAUSE_IF);
6714 9374 : tree simdlen = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
6715 : OMP_CLAUSE_SIMDLEN);
6716 9374 : tree condtemp = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
6717 : OMP_CLAUSE__CONDTEMP_);
6718 9374 : tree n1, n2;
6719 9451 : tree cond_var = condtemp ? OMP_CLAUSE_DECL (condtemp) : NULL_TREE;
6720 :
6721 9374 : if (safelen)
6722 : {
6723 5367 : poly_uint64 val;
6724 5367 : safelen = OMP_CLAUSE_SAFELEN_EXPR (safelen);
6725 5367 : if (!poly_int_tree_p (safelen, &val))
6726 : safelen_int = 0;
6727 : else
6728 5352 : safelen_int = MIN (constant_lower_bound (val), INT_MAX);
6729 5352 : if (safelen_int == 1)
6730 1805 : safelen_int = 0;
6731 : }
6732 759 : if ((ifc && integer_zerop (OMP_CLAUSE_IF_EXPR (ifc)))
6733 10009 : || (simdlen && integer_onep (OMP_CLAUSE_SIMDLEN_EXPR (simdlen))))
6734 : {
6735 : safelen_int = 0;
6736 : dont_vectorize = true;
6737 : }
6738 9374 : type = TREE_TYPE (fd->loop.v);
6739 9374 : entry_bb = region->entry;
6740 9374 : cont_bb = region->cont;
6741 9374 : gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
6742 9374 : gcc_assert (broken_loop
6743 : || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
6744 9374 : l0_bb = FALLTHRU_EDGE (entry_bb)->dest;
6745 9374 : if (!broken_loop)
6746 : {
6747 8302 : gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb);
6748 8302 : gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
6749 8302 : l1_bb = split_block (cont_bb, last_nondebug_stmt (cont_bb))->dest;
6750 8302 : l2_bb = BRANCH_EDGE (entry_bb)->dest;
6751 : }
6752 : else
6753 : {
6754 1072 : BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL;
6755 1072 : l1_bb = split_edge (BRANCH_EDGE (entry_bb));
6756 1072 : l2_bb = single_succ (l1_bb);
6757 : }
6758 9374 : exit_bb = region->exit;
6759 9374 : l2_dom_bb = NULL;
6760 :
6761 9374 : gsi = gsi_last_nondebug_bb (entry_bb);
6762 :
6763 9374 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
6764 : /* Not needed in SSA form right now. */
6765 9374 : gcc_assert (!gimple_in_ssa_p (cfun));
6766 9374 : if (fd->collapse > 1
6767 9374 : && (gimple_omp_for_combined_into_p (fd->for_stmt)
6768 496 : || broken_loop))
6769 : {
6770 2277 : int first_zero_iter = -1, dummy = -1;
6771 2277 : basic_block zero_iter_bb = l2_bb, dummy_bb = NULL;
6772 :
6773 2277 : counts = XALLOCAVEC (tree, fd->collapse);
6774 2277 : expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
6775 : zero_iter_bb, first_zero_iter,
6776 : dummy_bb, dummy, l2_dom_bb);
6777 : }
6778 9374 : if (l2_dom_bb == NULL)
6779 9366 : l2_dom_bb = l1_bb;
6780 :
6781 9374 : n1 = fd->loop.n1;
6782 9374 : n2 = fd->loop.n2;
6783 9374 : if (gimple_omp_for_combined_into_p (fd->for_stmt))
6784 : {
6785 7023 : tree innerc = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
6786 : OMP_CLAUSE__LOOPTEMP_);
6787 7023 : gcc_assert (innerc);
6788 7023 : n1 = OMP_CLAUSE_DECL (innerc);
6789 7023 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
6790 : OMP_CLAUSE__LOOPTEMP_);
6791 7023 : gcc_assert (innerc);
6792 7023 : n2 = OMP_CLAUSE_DECL (innerc);
6793 : }
6794 9374 : tree step = fd->loop.step;
6795 9374 : tree orig_step = step; /* May be different from step if is_simt. */
6796 :
6797 9374 : bool is_simt = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
6798 9374 : OMP_CLAUSE__SIMT_);
6799 9374 : if (is_simt)
6800 : {
6801 0 : cfun->curr_properties &= ~PROP_gimple_lomp_dev;
6802 0 : is_simt = safelen_int > 1;
6803 : }
6804 9374 : tree simt_lane = NULL_TREE, simt_maxlane = NULL_TREE;
6805 9374 : if (is_simt)
6806 : {
6807 0 : simt_lane = create_tmp_var (unsigned_type_node);
6808 0 : gimple *g = gimple_build_call_internal (IFN_GOMP_SIMT_LANE, 0);
6809 0 : gimple_call_set_lhs (g, simt_lane);
6810 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
6811 0 : tree offset = fold_build2 (MULT_EXPR, TREE_TYPE (step), step,
6812 : fold_convert (TREE_TYPE (step), simt_lane));
6813 0 : n1 = fold_convert (type, n1);
6814 0 : if (POINTER_TYPE_P (type))
6815 0 : n1 = fold_build_pointer_plus (n1, offset);
6816 : else
6817 0 : n1 = fold_build2 (PLUS_EXPR, type, n1, fold_convert (type, offset));
6818 :
6819 : /* Collapsed loops not handled for SIMT yet: limit to one lane only. */
6820 0 : if (fd->collapse > 1)
6821 0 : simt_maxlane = build_one_cst (unsigned_type_node);
6822 0 : else if (safelen_int < omp_max_simt_vf ())
6823 0 : simt_maxlane = build_int_cst (unsigned_type_node, safelen_int);
6824 0 : tree vf
6825 0 : = build_call_expr_internal_loc (UNKNOWN_LOCATION, IFN_GOMP_SIMT_VF,
6826 : unsigned_type_node, 0);
6827 0 : if (simt_maxlane)
6828 0 : vf = fold_build2 (MIN_EXPR, unsigned_type_node, vf, simt_maxlane);
6829 0 : vf = fold_convert (TREE_TYPE (step), vf);
6830 0 : step = fold_build2 (MULT_EXPR, TREE_TYPE (step), step, vf);
6831 : }
6832 :
6833 9374 : tree n2var = NULL_TREE;
6834 9374 : tree n2v = NULL_TREE;
6835 9374 : tree *nonrect_bounds = NULL;
6836 9374 : tree min_arg1 = NULL_TREE, min_arg2 = NULL_TREE;
6837 9374 : if (fd->collapse > 1)
6838 : {
6839 2753 : if (broken_loop || gimple_omp_for_combined_into_p (fd->for_stmt))
6840 : {
6841 2277 : if (fd->non_rect)
6842 : {
6843 99 : nonrect_bounds = XALLOCAVEC (tree, fd->last_nonrect + 1);
6844 99 : memset (nonrect_bounds, 0,
6845 : sizeof (tree) * (fd->last_nonrect + 1));
6846 : }
6847 2277 : expand_omp_build_assign (&gsi, fd->loop.v, fold_convert (type, n1));
6848 2277 : gcc_assert (entry_bb == gsi_bb (gsi));
6849 2277 : gcc_assert (fd->for_stmt == gsi_stmt (gsi));
6850 2277 : gsi_prev (&gsi);
6851 2277 : entry_bb = split_block (entry_bb, gsi_stmt (gsi))->dest;
6852 2277 : expand_omp_for_init_vars (fd, &gsi, counts, nonrect_bounds,
6853 : NULL, n1);
6854 2277 : gsi = gsi_for_stmt (fd->for_stmt);
6855 : }
6856 2753 : if (broken_loop)
6857 : ;
6858 2217 : else if (gimple_omp_for_combined_into_p (fd->for_stmt))
6859 : {
6860 : /* Compute in n2var the limit for the first innermost loop,
6861 : i.e. fd->loop.v + MIN (n2 - fd->loop.v, cnt)
6862 : where cnt is how many iterations would the loop have if
6863 : all further iterations were assigned to the current task. */
6864 1741 : n2var = create_tmp_var (type);
6865 1741 : i = fd->collapse - 1;
6866 1741 : tree itype = TREE_TYPE (fd->loops[i].v);
6867 1741 : if (POINTER_TYPE_P (itype))
6868 266 : itype = signed_type_for (itype);
6869 1741 : t = build_int_cst (itype, (fd->loops[i].cond_code == LT_EXPR
6870 2511 : ? -1 : 1));
6871 1741 : t = fold_build2 (PLUS_EXPR, itype,
6872 : fold_convert (itype, fd->loops[i].step), t);
6873 1741 : t = fold_build2 (PLUS_EXPR, itype, t,
6874 : fold_convert (itype, fd->loops[i].n2));
6875 1741 : if (fd->loops[i].m2)
6876 : {
6877 26 : tree t2 = fold_convert (itype,
6878 : fd->loops[i - fd->loops[i].outer].v);
6879 26 : tree t3 = fold_convert (itype, fd->loops[i].m2);
6880 26 : t2 = fold_build2 (MULT_EXPR, TREE_TYPE (t), t2, t3);
6881 26 : t = fold_build2 (PLUS_EXPR, itype, t, t2);
6882 : }
6883 1741 : t = fold_build2 (MINUS_EXPR, itype, t,
6884 : fold_convert (itype, fd->loops[i].v));
6885 1741 : if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR)
6886 256 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
6887 : fold_build1 (NEGATE_EXPR, itype, t),
6888 : fold_build1 (NEGATE_EXPR, itype,
6889 : fold_convert (itype,
6890 : fd->loops[i].step)));
6891 : else
6892 1485 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t,
6893 : fold_convert (itype, fd->loops[i].step));
6894 1741 : t = fold_convert (type, t);
6895 1741 : tree t2 = fold_build2 (MINUS_EXPR, type, n2, n1);
6896 1741 : min_arg1 = create_tmp_var (type);
6897 1741 : expand_omp_build_assign (&gsi, min_arg1, t2);
6898 1741 : min_arg2 = create_tmp_var (type);
6899 1741 : expand_omp_build_assign (&gsi, min_arg2, t);
6900 : }
6901 : else
6902 : {
6903 476 : if (TREE_CODE (n2) == INTEGER_CST)
6904 : {
6905 : /* Indicate for lastprivate handling that at least one iteration
6906 : has been performed, without wasting runtime. */
6907 138 : if (integer_nonzerop (n2))
6908 130 : expand_omp_build_assign (&gsi, fd->loop.v,
6909 : fold_convert (type, n2));
6910 : else
6911 : /* Indicate that no iteration has been performed. */
6912 8 : expand_omp_build_assign (&gsi, fd->loop.v,
6913 : build_one_cst (type));
6914 : }
6915 : else
6916 : {
6917 338 : expand_omp_build_assign (&gsi, fd->loop.v,
6918 : build_zero_cst (type));
6919 338 : expand_omp_build_assign (&gsi, n2, build_one_cst (type));
6920 : }
6921 476 : for (i = 0; i < fd->collapse; i++)
6922 : {
6923 476 : t = fold_convert (TREE_TYPE (fd->loops[i].v), fd->loops[i].n1);
6924 476 : if (fd->loops[i].m1)
6925 : {
6926 0 : tree t2
6927 0 : = fold_convert (TREE_TYPE (t),
6928 : fd->loops[i - fd->loops[i].outer].v);
6929 0 : tree t3 = fold_convert (TREE_TYPE (t), fd->loops[i].m1);
6930 0 : t2 = fold_build2 (MULT_EXPR, TREE_TYPE (t), t2, t3);
6931 0 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (t), t, t2);
6932 : }
6933 476 : expand_omp_build_assign (&gsi, fd->loops[i].v, t);
6934 : /* For normal non-combined collapsed loops just initialize
6935 : the outermost iterator in the entry_bb. */
6936 476 : if (!broken_loop)
6937 : break;
6938 : }
6939 : }
6940 : }
6941 : else
6942 6621 : expand_omp_build_assign (&gsi, fd->loop.v, fold_convert (type, n1));
6943 9374 : tree altv = NULL_TREE, altn2 = NULL_TREE;
6944 9374 : if (fd->collapse == 1
6945 6621 : && !broken_loop
6946 6085 : && TREE_CODE (orig_step) != INTEGER_CST)
6947 : {
6948 : /* The vectorizer currently punts on loops with non-constant steps
6949 : for the main IV (can't compute number of iterations and gives up
6950 : because of that). As for OpenMP loops it is always possible to
6951 : compute the number of iterations upfront, use an alternate IV
6952 : as the loop iterator:
6953 : altn2 = n1 < n2 ? (n2 - n1 + step - 1) / step : 0;
6954 : for (i = n1, altv = 0; altv < altn2; altv++, i += step) */
6955 176 : altv = create_tmp_var (unsigned_type_for (TREE_TYPE (fd->loops[0].v)));
6956 176 : expand_omp_build_assign (&gsi, altv, build_zero_cst (TREE_TYPE (altv)));
6957 176 : tree itype = TREE_TYPE (fd->loop.v);
6958 176 : if (POINTER_TYPE_P (itype))
6959 0 : itype = signed_type_for (itype);
6960 334 : t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
6961 176 : t = fold_build2 (PLUS_EXPR, itype,
6962 : fold_convert (itype, step), t);
6963 176 : t = fold_build2 (PLUS_EXPR, itype, t, fold_convert (itype, n2));
6964 176 : t = fold_build2 (MINUS_EXPR, itype, t,
6965 : fold_convert (itype, fd->loop.v));
6966 176 : if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
6967 0 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
6968 : fold_build1 (NEGATE_EXPR, itype, t),
6969 : fold_build1 (NEGATE_EXPR, itype,
6970 : fold_convert (itype, step)));
6971 : else
6972 176 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t,
6973 : fold_convert (itype, step));
6974 176 : t = fold_convert (TREE_TYPE (altv), t);
6975 176 : altn2 = create_tmp_var (TREE_TYPE (altv));
6976 176 : expand_omp_build_assign (&gsi, altn2, t);
6977 176 : tree t2 = fold_convert (TREE_TYPE (fd->loop.v), n2);
6978 176 : t2 = fold_build2 (fd->loop.cond_code, boolean_type_node, fd->loop.v, t2);
6979 176 : t2 = force_gimple_operand_gsi (&gsi, t2, true, NULL_TREE,
6980 : true, GSI_SAME_STMT);
6981 176 : gassign *g = gimple_build_assign (altn2, COND_EXPR, t2, altn2,
6982 176 : build_zero_cst (TREE_TYPE (altv)));
6983 176 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
6984 176 : }
6985 9198 : else if (fd->collapse > 1
6986 2753 : && !broken_loop
6987 2217 : && !gimple_omp_for_combined_into_p (fd->for_stmt)
6988 9674 : && TREE_CODE (fd->loops[fd->collapse - 1].step) != INTEGER_CST)
6989 : {
6990 22 : altv = create_tmp_var (unsigned_type_for (TREE_TYPE (fd->loops[0].v)));
6991 22 : altn2 = create_tmp_var (TREE_TYPE (altv));
6992 : }
6993 9374 : if (cond_var)
6994 : {
6995 77 : if (POINTER_TYPE_P (type)
6996 77 : || TREE_CODE (n1) != INTEGER_CST
6997 15 : || fd->loop.cond_code != LT_EXPR
6998 92 : || tree_int_cst_sgn (n1) != 1)
6999 72 : expand_omp_build_assign (&gsi, cond_var,
7000 72 : build_one_cst (TREE_TYPE (cond_var)));
7001 : else
7002 5 : expand_omp_build_assign (&gsi, cond_var,
7003 5 : fold_convert (TREE_TYPE (cond_var), n1));
7004 : }
7005 :
7006 : /* Remove the GIMPLE_OMP_FOR statement. */
7007 9374 : gsi_remove (&gsi, true);
7008 :
7009 9374 : if (!broken_loop)
7010 : {
7011 : /* Code to control the increment goes in the CONT_BB. */
7012 8302 : gsi = gsi_last_nondebug_bb (cont_bb);
7013 8302 : stmt = gsi_stmt (gsi);
7014 8302 : gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
7015 :
7016 8302 : if (fd->collapse == 1
7017 8302 : || gimple_omp_for_combined_into_p (fd->for_stmt))
7018 : {
7019 7826 : if (POINTER_TYPE_P (type))
7020 286 : t = fold_build_pointer_plus (fd->loop.v, step);
7021 : else
7022 7540 : t = fold_build2 (PLUS_EXPR, type, fd->loop.v, step);
7023 7826 : expand_omp_build_assign (&gsi, fd->loop.v, t);
7024 : }
7025 476 : else if (TREE_CODE (n2) != INTEGER_CST)
7026 338 : expand_omp_build_assign (&gsi, fd->loop.v, build_one_cst (type));
7027 8302 : if (altv)
7028 : {
7029 198 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (altv), altv,
7030 : build_one_cst (TREE_TYPE (altv)));
7031 198 : expand_omp_build_assign (&gsi, altv, t);
7032 : }
7033 :
7034 8302 : if (fd->collapse > 1)
7035 : {
7036 2217 : i = fd->collapse - 1;
7037 2217 : if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v)))
7038 280 : t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step);
7039 : else
7040 : {
7041 1937 : t = fold_convert (TREE_TYPE (fd->loops[i].v),
7042 : fd->loops[i].step);
7043 1937 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loops[i].v),
7044 : fd->loops[i].v, t);
7045 : }
7046 2217 : expand_omp_build_assign (&gsi, fd->loops[i].v, t);
7047 : }
7048 8302 : if (cond_var)
7049 : {
7050 77 : if (POINTER_TYPE_P (type)
7051 77 : || TREE_CODE (n1) != INTEGER_CST
7052 15 : || fd->loop.cond_code != LT_EXPR
7053 92 : || tree_int_cst_sgn (n1) != 1)
7054 72 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (cond_var), cond_var,
7055 : build_one_cst (TREE_TYPE (cond_var)));
7056 : else
7057 5 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (cond_var), cond_var,
7058 : fold_convert (TREE_TYPE (cond_var), step));
7059 77 : expand_omp_build_assign (&gsi, cond_var, t);
7060 : }
7061 :
7062 : /* Remove GIMPLE_OMP_CONTINUE. */
7063 8302 : gsi_remove (&gsi, true);
7064 : }
7065 :
7066 : /* Emit the condition in L1_BB. */
7067 9374 : gsi = gsi_start_bb (l1_bb);
7068 :
7069 9374 : if (altv)
7070 198 : t = build2 (LT_EXPR, boolean_type_node, altv, altn2);
7071 9176 : else if (fd->collapse > 1
7072 2731 : && !gimple_omp_for_combined_into_p (fd->for_stmt)
7073 9650 : && !broken_loop)
7074 : {
7075 454 : i = fd->collapse - 1;
7076 454 : tree itype = TREE_TYPE (fd->loops[i].v);
7077 454 : if (fd->loops[i].m2)
7078 203 : t = n2v = create_tmp_var (itype);
7079 : else
7080 251 : t = fold_convert (itype, unshare_expr (fd->loops[i].n2));
7081 454 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7082 : false, GSI_CONTINUE_LINKING);
7083 454 : tree v = fd->loops[i].v;
7084 454 : if (DECL_P (v) && TREE_ADDRESSABLE (v))
7085 0 : v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE,
7086 : false, GSI_CONTINUE_LINKING);
7087 454 : t = build2 (fd->loops[i].cond_code, boolean_type_node, v, t);
7088 : }
7089 : else
7090 : {
7091 8722 : if (fd->collapse > 1 && !broken_loop)
7092 : t = n2var;
7093 : else
7094 6981 : t = fold_convert (type, unshare_expr (n2));
7095 8722 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7096 : false, GSI_CONTINUE_LINKING);
7097 8722 : tree v = fd->loop.v;
7098 8722 : if (DECL_P (v) && TREE_ADDRESSABLE (v))
7099 0 : v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE,
7100 : false, GSI_CONTINUE_LINKING);
7101 8722 : t = build2 (fd->loop.cond_code, boolean_type_node, v, t);
7102 : }
7103 9374 : cond_stmt = gimple_build_cond_empty (t);
7104 9374 : gsi_insert_after (&gsi, cond_stmt, GSI_CONTINUE_LINKING);
7105 9374 : if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), expand_omp_regimplify_p,
7106 : NULL, NULL)
7107 9374 : || walk_tree (gimple_cond_rhs_ptr (cond_stmt), expand_omp_regimplify_p,
7108 : NULL, NULL))
7109 : {
7110 0 : gsi = gsi_for_stmt (cond_stmt);
7111 0 : gimple_regimplify_operands (cond_stmt, &gsi);
7112 : }
7113 :
7114 : /* Add 'V -= STEP * (SIMT_VF - 1)' after the loop. */
7115 9374 : if (is_simt)
7116 : {
7117 0 : gsi = gsi_start_bb (l2_bb);
7118 0 : step = fold_build2 (MINUS_EXPR, TREE_TYPE (step), orig_step, step);
7119 0 : if (POINTER_TYPE_P (type))
7120 0 : t = fold_build_pointer_plus (fd->loop.v, step);
7121 : else
7122 0 : t = fold_build2 (PLUS_EXPR, type, fd->loop.v, step);
7123 0 : expand_omp_build_assign (&gsi, fd->loop.v, t);
7124 : }
7125 :
7126 : /* Remove GIMPLE_OMP_RETURN. */
7127 9374 : gsi = gsi_last_nondebug_bb (exit_bb);
7128 9374 : gsi_remove (&gsi, true);
7129 :
7130 : /* Connect the new blocks. */
7131 9374 : remove_edge (FALLTHRU_EDGE (entry_bb));
7132 :
7133 9374 : if (!broken_loop)
7134 : {
7135 8302 : remove_edge (BRANCH_EDGE (entry_bb));
7136 8302 : make_edge (entry_bb, l1_bb, EDGE_FALLTHRU);
7137 :
7138 8302 : e = BRANCH_EDGE (l1_bb);
7139 8302 : ne = FALLTHRU_EDGE (l1_bb);
7140 8302 : e->flags = EDGE_TRUE_VALUE;
7141 : }
7142 : else
7143 : {
7144 1072 : single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
7145 :
7146 1072 : ne = single_succ_edge (l1_bb);
7147 1072 : e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE);
7148 :
7149 : }
7150 9374 : ne->flags = EDGE_FALSE_VALUE;
7151 9374 : e->probability = profile_probability::guessed_always ().apply_scale (7, 8);
7152 9374 : ne->probability = e->probability.invert ();
7153 :
7154 9374 : set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb);
7155 9374 : set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb);
7156 :
7157 9374 : if (simt_maxlane)
7158 : {
7159 0 : cond_stmt = gimple_build_cond (LT_EXPR, simt_lane, simt_maxlane,
7160 : NULL_TREE, NULL_TREE);
7161 0 : gsi = gsi_last_bb (entry_bb);
7162 0 : gsi_insert_after (&gsi, cond_stmt, GSI_NEW_STMT);
7163 0 : make_edge (entry_bb, l2_bb, EDGE_FALSE_VALUE);
7164 0 : FALLTHRU_EDGE (entry_bb)->flags = EDGE_TRUE_VALUE;
7165 0 : FALLTHRU_EDGE (entry_bb)->probability
7166 0 : = profile_probability::guessed_always ().apply_scale (7, 8);
7167 0 : BRANCH_EDGE (entry_bb)->probability
7168 0 : = FALLTHRU_EDGE (entry_bb)->probability.invert ();
7169 0 : l2_dom_bb = entry_bb;
7170 : }
7171 9374 : set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb);
7172 :
7173 9374 : if (!broken_loop && fd->collapse > 1)
7174 : {
7175 2217 : basic_block last_bb = l1_bb;
7176 2217 : basic_block init_bb = NULL;
7177 6108 : for (i = fd->collapse - 2; i >= 0; i--)
7178 : {
7179 3891 : tree nextn2v = NULL_TREE;
7180 3891 : if (EDGE_SUCC (last_bb, 0)->flags & EDGE_FALSE_VALUE)
7181 : e = EDGE_SUCC (last_bb, 0);
7182 : else
7183 2217 : e = EDGE_SUCC (last_bb, 1);
7184 3891 : basic_block bb = split_edge (e);
7185 3891 : if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v)))
7186 553 : t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step);
7187 : else
7188 : {
7189 3338 : t = fold_convert (TREE_TYPE (fd->loops[i].v),
7190 : fd->loops[i].step);
7191 3338 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loops[i].v),
7192 : fd->loops[i].v, t);
7193 : }
7194 3891 : gsi = gsi_after_labels (bb);
7195 3891 : expand_omp_build_assign (&gsi, fd->loops[i].v, t);
7196 :
7197 3891 : bb = split_block (bb, last_nondebug_stmt (bb))->dest;
7198 3891 : gsi = gsi_start_bb (bb);
7199 3891 : tree itype = TREE_TYPE (fd->loops[i].v);
7200 3891 : if (fd->loops[i].m2)
7201 0 : t = nextn2v = create_tmp_var (itype);
7202 : else
7203 3891 : t = fold_convert (itype, unshare_expr (fd->loops[i].n2));
7204 3891 : t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7205 : false, GSI_CONTINUE_LINKING);
7206 3891 : tree v = fd->loops[i].v;
7207 3891 : if (DECL_P (v) && TREE_ADDRESSABLE (v))
7208 0 : v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE,
7209 : false, GSI_CONTINUE_LINKING);
7210 3891 : t = build2 (fd->loops[i].cond_code, boolean_type_node, v, t);
7211 3891 : cond_stmt = gimple_build_cond_empty (t);
7212 3891 : gsi_insert_after (&gsi, cond_stmt, GSI_CONTINUE_LINKING);
7213 3891 : if (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
7214 : expand_omp_regimplify_p, NULL, NULL)
7215 3891 : || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
7216 : expand_omp_regimplify_p, NULL, NULL))
7217 : {
7218 6 : gsi = gsi_for_stmt (cond_stmt);
7219 6 : gimple_regimplify_operands (cond_stmt, &gsi);
7220 : }
7221 3891 : ne = single_succ_edge (bb);
7222 3891 : ne->flags = EDGE_FALSE_VALUE;
7223 :
7224 3891 : init_bb = create_empty_bb (bb);
7225 3891 : set_immediate_dominator (CDI_DOMINATORS, init_bb, bb);
7226 3891 : add_bb_to_loop (init_bb, bb->loop_father);
7227 3891 : e = make_edge (bb, init_bb, EDGE_TRUE_VALUE);
7228 3891 : e->probability
7229 3891 : = profile_probability::guessed_always ().apply_scale (7, 8);
7230 3891 : ne->probability = e->probability.invert ();
7231 :
7232 3891 : gsi = gsi_after_labels (init_bb);
7233 3891 : if (fd->loops[i + 1].m1)
7234 : {
7235 242 : tree t2 = fold_convert (TREE_TYPE (fd->loops[i + 1].v),
7236 : fd->loops[i + 1
7237 : - fd->loops[i + 1].outer].v);
7238 242 : if (POINTER_TYPE_P (TREE_TYPE (t2)))
7239 6 : t = fold_build_pointer_plus (t2, fd->loops[i + 1].n1);
7240 : else
7241 : {
7242 236 : t = fold_convert (TREE_TYPE (fd->loops[i + 1].v),
7243 : fd->loops[i + 1].n1);
7244 236 : tree t3 = fold_convert (TREE_TYPE (t), fd->loops[i + 1].m1);
7245 236 : t2 = fold_build2 (MULT_EXPR, TREE_TYPE (t), t2, t3);
7246 236 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (t), t, t2);
7247 : }
7248 : }
7249 : else
7250 3649 : t = fold_convert (TREE_TYPE (fd->loops[i + 1].v),
7251 : fd->loops[i + 1].n1);
7252 3891 : expand_omp_build_assign (&gsi, fd->loops[i + 1].v, t);
7253 3891 : if (fd->loops[i + 1].m2)
7254 : {
7255 241 : if (i + 2 == fd->collapse && (n2var || altv))
7256 : {
7257 38 : gcc_assert (n2v == NULL_TREE);
7258 38 : n2v = create_tmp_var (TREE_TYPE (fd->loops[i + 1].v));
7259 : }
7260 241 : tree t2 = fold_convert (TREE_TYPE (fd->loops[i + 1].v),
7261 : fd->loops[i + 1
7262 : - fd->loops[i + 1].outer].v);
7263 241 : if (POINTER_TYPE_P (TREE_TYPE (t2)))
7264 6 : t = fold_build_pointer_plus (t2, fd->loops[i + 1].n2);
7265 : else
7266 : {
7267 235 : t = fold_convert (TREE_TYPE (fd->loops[i + 1].v),
7268 : fd->loops[i + 1].n2);
7269 235 : tree t3 = fold_convert (TREE_TYPE (t), fd->loops[i + 1].m2);
7270 235 : t2 = fold_build2 (MULT_EXPR, TREE_TYPE (t), t2, t3);
7271 235 : t = fold_build2 (PLUS_EXPR, TREE_TYPE (t), t, t2);
7272 : }
7273 241 : expand_omp_build_assign (&gsi, n2v, t);
7274 : }
7275 3891 : if (i + 2 == fd->collapse && n2var)
7276 : {
7277 : /* For composite simd, n2 is the first iteration the current
7278 : task shouldn't already handle, so we effectively want to use
7279 : for (V3 = N31; V < N2 && V3 < N32; V++, V3 += STEP3)
7280 : as the vectorized loop. Except the vectorizer will not
7281 : vectorize that, so instead compute N2VAR as
7282 : N2VAR = V + MIN (N2 - V, COUNTS3) and use
7283 : for (V3 = N31; V < N2VAR; V++, V3 += STEP3)
7284 : as the loop to vectorize. */
7285 1741 : tree t2 = fold_build2 (MINUS_EXPR, type, n2, fd->loop.v);
7286 1741 : if (fd->loops[i + 1].m1 || fd->loops[i + 1].m2)
7287 : {
7288 99 : tree itype = TREE_TYPE (fd->loops[i].v);
7289 99 : if (POINTER_TYPE_P (itype))
7290 4 : itype = signed_type_for (itype);
7291 99 : t = build_int_cst (itype, (fd->loops[i + 1].cond_code
7292 99 : == LT_EXPR ? -1 : 1));
7293 99 : t = fold_build2 (PLUS_EXPR, itype,
7294 : fold_convert (itype,
7295 : fd->loops[i + 1].step), t);
7296 99 : if (fd->loops[i + 1].m2 == NULL_TREE)
7297 73 : t = fold_build2 (PLUS_EXPR, itype, t,
7298 : fold_convert (itype,
7299 : fd->loops[i + 1].n2));
7300 26 : else if (POINTER_TYPE_P (TREE_TYPE (n2v)))
7301 : {
7302 4 : t = fold_build_pointer_plus (n2v, t);
7303 4 : t = fold_convert (itype, t);
7304 : }
7305 : else
7306 22 : t = fold_build2 (PLUS_EXPR, itype, t, n2v);
7307 99 : t = fold_build2 (MINUS_EXPR, itype, t,
7308 : fold_convert (itype, fd->loops[i + 1].v));
7309 99 : tree step = fold_convert (itype, fd->loops[i + 1].step);
7310 99 : if (TYPE_UNSIGNED (itype)
7311 99 : && fd->loops[i + 1].cond_code == GT_EXPR)
7312 0 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
7313 : fold_build1 (NEGATE_EXPR, itype, t),
7314 : fold_build1 (NEGATE_EXPR, itype, step));
7315 : else
7316 99 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
7317 99 : t = fold_convert (type, t);
7318 99 : }
7319 : else
7320 1642 : t = counts[i + 1];
7321 1741 : expand_omp_build_assign (&gsi, min_arg1, t2);
7322 1741 : expand_omp_build_assign (&gsi, min_arg2, t);
7323 1741 : e = split_block (init_bb, last_nondebug_stmt (init_bb));
7324 1741 : gsi = gsi_after_labels (e->dest);
7325 1741 : init_bb = e->dest;
7326 1741 : remove_edge (FALLTHRU_EDGE (entry_bb));
7327 1741 : make_edge (entry_bb, init_bb, EDGE_FALLTHRU);
7328 1741 : set_immediate_dominator (CDI_DOMINATORS, init_bb, entry_bb);
7329 1741 : set_immediate_dominator (CDI_DOMINATORS, l1_bb, init_bb);
7330 1741 : t = fold_build2 (MIN_EXPR, type, min_arg1, min_arg2);
7331 1741 : t = fold_build2 (PLUS_EXPR, type, fd->loop.v, t);
7332 1741 : expand_omp_build_assign (&gsi, n2var, t);
7333 : }
7334 3891 : if (i + 2 == fd->collapse && altv)
7335 : {
7336 : /* The vectorizer currently punts on loops with non-constant
7337 : steps for the main IV (can't compute number of iterations
7338 : and gives up because of that). As for OpenMP loops it is
7339 : always possible to compute the number of iterations upfront,
7340 : use an alternate IV as the loop iterator. */
7341 22 : expand_omp_build_assign (&gsi, altv,
7342 22 : build_zero_cst (TREE_TYPE (altv)));
7343 22 : tree itype = TREE_TYPE (fd->loops[i + 1].v);
7344 22 : if (POINTER_TYPE_P (itype))
7345 0 : itype = signed_type_for (itype);
7346 22 : t = build_int_cst (itype, (fd->loops[i + 1].cond_code == LT_EXPR
7347 26 : ? -1 : 1));
7348 22 : t = fold_build2 (PLUS_EXPR, itype,
7349 : fold_convert (itype, fd->loops[i + 1].step), t);
7350 22 : t = fold_build2 (PLUS_EXPR, itype, t,
7351 : fold_convert (itype,
7352 : fd->loops[i + 1].m2
7353 : ? n2v : fd->loops[i + 1].n2));
7354 22 : t = fold_build2 (MINUS_EXPR, itype, t,
7355 : fold_convert (itype, fd->loops[i + 1].v));
7356 22 : tree step = fold_convert (itype, fd->loops[i + 1].step);
7357 22 : if (TYPE_UNSIGNED (itype)
7358 22 : && fd->loops[i + 1].cond_code == GT_EXPR)
7359 0 : t = fold_build2 (TRUNC_DIV_EXPR, itype,
7360 : fold_build1 (NEGATE_EXPR, itype, t),
7361 : fold_build1 (NEGATE_EXPR, itype, step));
7362 : else
7363 22 : t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
7364 22 : t = fold_convert (TREE_TYPE (altv), t);
7365 22 : expand_omp_build_assign (&gsi, altn2, t);
7366 22 : tree t2 = fold_convert (TREE_TYPE (fd->loops[i + 1].v),
7367 : fd->loops[i + 1].m2
7368 : ? n2v : fd->loops[i + 1].n2);
7369 22 : t2 = fold_build2 (fd->loops[i + 1].cond_code, boolean_type_node,
7370 : fd->loops[i + 1].v, t2);
7371 22 : t2 = force_gimple_operand_gsi (&gsi, t2, true, NULL_TREE,
7372 : true, GSI_SAME_STMT);
7373 22 : gassign *g
7374 22 : = gimple_build_assign (altn2, COND_EXPR, t2, altn2,
7375 22 : build_zero_cst (TREE_TYPE (altv)));
7376 22 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7377 : }
7378 3891 : n2v = nextn2v;
7379 :
7380 3891 : make_edge (init_bb, last_bb, EDGE_FALLTHRU);
7381 3891 : if (!gimple_omp_for_combined_into_p (fd->for_stmt))
7382 : {
7383 787 : e = find_edge (entry_bb, last_bb);
7384 787 : redirect_edge_succ (e, bb);
7385 787 : set_immediate_dominator (CDI_DOMINATORS, bb, entry_bb);
7386 787 : set_immediate_dominator (CDI_DOMINATORS, last_bb, init_bb);
7387 : }
7388 :
7389 3891 : last_bb = bb;
7390 : }
7391 : }
7392 8302 : if (!broken_loop)
7393 : {
7394 8302 : class loop *loop = alloc_loop ();
7395 8302 : loop->header = l1_bb;
7396 8302 : loop->latch = cont_bb;
7397 8302 : add_loop (loop, l1_bb->loop_father);
7398 8302 : loop->safelen = safelen_int;
7399 8302 : if (simduid)
7400 : {
7401 3473 : loop->simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
7402 3473 : cfun->has_simduid_loops = true;
7403 : }
7404 : /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
7405 : the loop. */
7406 8302 : if ((flag_tree_loop_vectorize
7407 3316 : || !OPTION_SET_P (flag_tree_loop_vectorize))
7408 8302 : && flag_tree_loop_optimize
7409 8301 : && loop->safelen > 1)
7410 : {
7411 6498 : loop->force_vectorize = true;
7412 6498 : if (simdlen && tree_fits_uhwi_p (OMP_CLAUSE_SIMDLEN_EXPR (simdlen)))
7413 : {
7414 60 : unsigned HOST_WIDE_INT v
7415 60 : = tree_to_uhwi (OMP_CLAUSE_SIMDLEN_EXPR (simdlen));
7416 60 : if (v < INT_MAX && v <= (unsigned HOST_WIDE_INT) loop->safelen)
7417 60 : loop->simdlen = v;
7418 : }
7419 6498 : cfun->has_force_vectorize_loops = true;
7420 6498 : }
7421 1804 : else if (dont_vectorize)
7422 234 : loop->dont_vectorize = true;
7423 : }
7424 1072 : else if (simduid)
7425 530 : cfun->has_simduid_loops = true;
7426 9374 : }
7427 :
7428 : /* Taskloop construct is represented after gimplification with
7429 : two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
7430 : in between them. This routine expands the outer GIMPLE_OMP_FOR,
7431 : which should just compute all the needed loop temporaries
7432 : for GIMPLE_OMP_TASK. */
7433 :
7434 : static void
7435 1330 : expand_omp_taskloop_for_outer (struct omp_region *region,
7436 : struct omp_for_data *fd,
7437 : gimple *inner_stmt)
7438 : {
7439 1330 : tree type, bias = NULL_TREE;
7440 1330 : basic_block entry_bb, cont_bb, exit_bb;
7441 1330 : gimple_stmt_iterator gsi;
7442 1330 : gassign *assign_stmt;
7443 1330 : tree *counts = NULL;
7444 1330 : int i;
7445 :
7446 1330 : gcc_assert (inner_stmt);
7447 1330 : gcc_assert (region->cont);
7448 1330 : gcc_assert (gimple_code (inner_stmt) == GIMPLE_OMP_TASK
7449 : && gimple_omp_task_taskloop_p (inner_stmt));
7450 1330 : type = TREE_TYPE (fd->loop.v);
7451 :
7452 : /* See if we need to bias by LLONG_MIN. */
7453 1330 : if (fd->iter_type == long_long_unsigned_type_node
7454 42 : && (TREE_CODE (type) == INTEGER_TYPE || TREE_CODE (type) == BITINT_TYPE)
7455 1358 : && !TYPE_UNSIGNED (type))
7456 : {
7457 0 : tree n1, n2;
7458 :
7459 0 : if (fd->loop.cond_code == LT_EXPR)
7460 : {
7461 0 : n1 = fd->loop.n1;
7462 0 : n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step);
7463 : }
7464 : else
7465 : {
7466 0 : n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step);
7467 0 : n2 = fd->loop.n1;
7468 : }
7469 0 : if (TREE_CODE (n1) != INTEGER_CST
7470 0 : || TREE_CODE (n2) != INTEGER_CST
7471 0 : || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0)))
7472 0 : bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type));
7473 : }
7474 :
7475 1330 : entry_bb = region->entry;
7476 1330 : cont_bb = region->cont;
7477 1330 : gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
7478 1330 : gcc_assert (BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
7479 1330 : exit_bb = region->exit;
7480 :
7481 1330 : gsi = gsi_last_nondebug_bb (entry_bb);
7482 1330 : gimple *for_stmt = gsi_stmt (gsi);
7483 1330 : gcc_assert (gimple_code (for_stmt) == GIMPLE_OMP_FOR);
7484 1330 : if (fd->collapse > 1)
7485 : {
7486 181 : int first_zero_iter = -1, dummy = -1;
7487 181 : basic_block zero_iter_bb = NULL, dummy_bb = NULL, l2_dom_bb = NULL;
7488 :
7489 181 : counts = XALLOCAVEC (tree, fd->collapse);
7490 181 : expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
7491 : zero_iter_bb, first_zero_iter,
7492 : dummy_bb, dummy, l2_dom_bb);
7493 :
7494 181 : if (zero_iter_bb)
7495 : {
7496 : /* Some counts[i] vars might be uninitialized if
7497 : some loop has zero iterations. But the body shouldn't
7498 : be executed in that case, so just avoid uninit warnings. */
7499 336 : for (i = first_zero_iter; i < fd->collapse; i++)
7500 239 : if (SSA_VAR_P (counts[i]))
7501 226 : suppress_warning (counts[i], OPT_Wuninitialized);
7502 97 : gsi_prev (&gsi);
7503 97 : edge e = split_block (entry_bb, gsi_stmt (gsi));
7504 97 : entry_bb = e->dest;
7505 97 : make_edge (zero_iter_bb, entry_bb, EDGE_FALLTHRU);
7506 97 : gsi = gsi_last_bb (entry_bb);
7507 97 : set_immediate_dominator (CDI_DOMINATORS, entry_bb,
7508 : get_immediate_dominator (CDI_DOMINATORS,
7509 : zero_iter_bb));
7510 : }
7511 : }
7512 :
7513 1330 : tree t0, t1;
7514 1330 : t1 = fd->loop.n2;
7515 1330 : t0 = fd->loop.n1;
7516 2646 : if (POINTER_TYPE_P (TREE_TYPE (t0))
7517 1330 : && TYPE_PRECISION (TREE_TYPE (t0))
7518 14 : != TYPE_PRECISION (fd->iter_type))
7519 : {
7520 : /* Avoid casting pointers to integer of a different size. */
7521 0 : tree itype = signed_type_for (type);
7522 0 : t1 = fold_convert (fd->iter_type, fold_convert (itype, t1));
7523 0 : t0 = fold_convert (fd->iter_type, fold_convert (itype, t0));
7524 : }
7525 : else
7526 : {
7527 1330 : t1 = fold_convert (fd->iter_type, t1);
7528 1330 : t0 = fold_convert (fd->iter_type, t0);
7529 : }
7530 1330 : if (bias)
7531 : {
7532 0 : t1 = fold_build2 (PLUS_EXPR, fd->iter_type, t1, bias);
7533 0 : t0 = fold_build2 (PLUS_EXPR, fd->iter_type, t0, bias);
7534 : }
7535 :
7536 1330 : tree innerc = omp_find_clause (gimple_omp_task_clauses (inner_stmt),
7537 : OMP_CLAUSE__LOOPTEMP_);
7538 1330 : gcc_assert (innerc);
7539 1330 : tree startvar = OMP_CLAUSE_DECL (innerc);
7540 1330 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc), OMP_CLAUSE__LOOPTEMP_);
7541 1330 : gcc_assert (innerc);
7542 1330 : tree endvar = OMP_CLAUSE_DECL (innerc);
7543 1330 : if (fd->collapse > 1 && TREE_CODE (fd->loop.n2) != INTEGER_CST)
7544 : {
7545 101 : innerc = find_lastprivate_looptemp (fd, innerc);
7546 101 : if (innerc)
7547 : {
7548 : /* If needed (inner taskloop has lastprivate clause), propagate
7549 : down the total number of iterations. */
7550 31 : tree t = force_gimple_operand_gsi (&gsi, fd->loop.n2, false,
7551 : NULL_TREE, false,
7552 : GSI_CONTINUE_LINKING);
7553 31 : assign_stmt = gimple_build_assign (OMP_CLAUSE_DECL (innerc), t);
7554 31 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7555 : }
7556 : }
7557 :
7558 1330 : t0 = force_gimple_operand_gsi (&gsi, t0, false, NULL_TREE, false,
7559 : GSI_CONTINUE_LINKING);
7560 1330 : assign_stmt = gimple_build_assign (startvar, t0);
7561 1330 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7562 :
7563 1330 : t1 = force_gimple_operand_gsi (&gsi, t1, false, NULL_TREE, false,
7564 : GSI_CONTINUE_LINKING);
7565 1330 : assign_stmt = gimple_build_assign (endvar, t1);
7566 1330 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7567 1330 : if (fd->collapse > 1)
7568 181 : expand_omp_for_init_vars (fd, &gsi, counts, NULL, inner_stmt, startvar);
7569 :
7570 : /* Remove the GIMPLE_OMP_FOR statement. */
7571 1330 : gsi = gsi_for_stmt (for_stmt);
7572 1330 : gsi_remove (&gsi, true);
7573 :
7574 1330 : gsi = gsi_last_nondebug_bb (cont_bb);
7575 1330 : gsi_remove (&gsi, true);
7576 :
7577 1330 : gsi = gsi_last_nondebug_bb (exit_bb);
7578 1330 : gsi_remove (&gsi, true);
7579 :
7580 1330 : FALLTHRU_EDGE (entry_bb)->probability = profile_probability::always ();
7581 1330 : remove_edge (BRANCH_EDGE (entry_bb));
7582 1330 : FALLTHRU_EDGE (cont_bb)->probability = profile_probability::always ();
7583 1330 : remove_edge (BRANCH_EDGE (cont_bb));
7584 1330 : set_immediate_dominator (CDI_DOMINATORS, exit_bb, cont_bb);
7585 1330 : set_immediate_dominator (CDI_DOMINATORS, region->entry,
7586 : recompute_dominator (CDI_DOMINATORS, region->entry));
7587 1330 : }
7588 :
7589 : /* Taskloop construct is represented after gimplification with
7590 : two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
7591 : in between them. This routine expands the inner GIMPLE_OMP_FOR.
7592 : GOMP_taskloop{,_ull} function arranges for each task to be given just
7593 : a single range of iterations. */
7594 :
7595 : static void
7596 1330 : expand_omp_taskloop_for_inner (struct omp_region *region,
7597 : struct omp_for_data *fd,
7598 : gimple *inner_stmt)
7599 : {
7600 1330 : tree e, t, type, itype, vmain, vback, bias = NULL_TREE;
7601 1330 : basic_block entry_bb, exit_bb, body_bb, cont_bb, collapse_bb = NULL;
7602 1330 : basic_block fin_bb;
7603 1330 : gimple_stmt_iterator gsi;
7604 1330 : edge ep;
7605 1330 : bool broken_loop = region->cont == NULL;
7606 1330 : tree *counts = NULL;
7607 1330 : tree n1, n2, step;
7608 :
7609 1330 : itype = type = TREE_TYPE (fd->loop.v);
7610 1330 : if (POINTER_TYPE_P (type))
7611 14 : itype = signed_type_for (type);
7612 :
7613 : /* See if we need to bias by LLONG_MIN. */
7614 1330 : if (fd->iter_type == long_long_unsigned_type_node
7615 42 : && (TREE_CODE (type) == INTEGER_TYPE || TREE_CODE (type) == BITINT_TYPE)
7616 1358 : && !TYPE_UNSIGNED (type))
7617 : {
7618 0 : tree n1, n2;
7619 :
7620 0 : if (fd->loop.cond_code == LT_EXPR)
7621 : {
7622 0 : n1 = fd->loop.n1;
7623 0 : n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step);
7624 : }
7625 : else
7626 : {
7627 0 : n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step);
7628 0 : n2 = fd->loop.n1;
7629 : }
7630 0 : if (TREE_CODE (n1) != INTEGER_CST
7631 0 : || TREE_CODE (n2) != INTEGER_CST
7632 0 : || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0)))
7633 0 : bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type));
7634 : }
7635 :
7636 1330 : entry_bb = region->entry;
7637 1330 : cont_bb = region->cont;
7638 1330 : gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
7639 1330 : fin_bb = BRANCH_EDGE (entry_bb)->dest;
7640 1330 : gcc_assert (broken_loop
7641 : || (fin_bb == FALLTHRU_EDGE (cont_bb)->dest));
7642 1330 : body_bb = FALLTHRU_EDGE (entry_bb)->dest;
7643 1330 : if (!broken_loop)
7644 : {
7645 1314 : gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
7646 1314 : gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
7647 : }
7648 1330 : exit_bb = region->exit;
7649 :
7650 : /* Iteration space partitioning goes in ENTRY_BB. */
7651 1330 : gsi = gsi_last_nondebug_bb (entry_bb);
7652 1330 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
7653 :
7654 1330 : if (fd->collapse > 1)
7655 : {
7656 181 : int first_zero_iter = -1, dummy = -1;
7657 181 : basic_block l2_dom_bb = NULL, dummy_bb = NULL;
7658 :
7659 181 : counts = XALLOCAVEC (tree, fd->collapse);
7660 181 : expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
7661 : fin_bb, first_zero_iter,
7662 : dummy_bb, dummy, l2_dom_bb);
7663 181 : t = NULL_TREE;
7664 : }
7665 : else
7666 1330 : t = integer_one_node;
7667 :
7668 1330 : step = fd->loop.step;
7669 1330 : tree innerc = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
7670 : OMP_CLAUSE__LOOPTEMP_);
7671 1330 : gcc_assert (innerc);
7672 1330 : n1 = OMP_CLAUSE_DECL (innerc);
7673 1330 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc), OMP_CLAUSE__LOOPTEMP_);
7674 1330 : gcc_assert (innerc);
7675 1330 : n2 = OMP_CLAUSE_DECL (innerc);
7676 1330 : if (bias)
7677 : {
7678 0 : n1 = fold_build2 (PLUS_EXPR, fd->iter_type, n1, bias);
7679 0 : n2 = fold_build2 (PLUS_EXPR, fd->iter_type, n2, bias);
7680 : }
7681 1330 : n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
7682 : true, NULL_TREE, true, GSI_SAME_STMT);
7683 1330 : n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
7684 : true, NULL_TREE, true, GSI_SAME_STMT);
7685 1330 : step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
7686 : true, NULL_TREE, true, GSI_SAME_STMT);
7687 :
7688 1330 : tree startvar = fd->loop.v;
7689 1330 : tree endvar = NULL_TREE;
7690 :
7691 1330 : if (gimple_omp_for_combined_p (fd->for_stmt))
7692 : {
7693 617 : tree clauses = gimple_omp_for_clauses (inner_stmt);
7694 617 : tree innerc = omp_find_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
7695 617 : gcc_assert (innerc);
7696 617 : startvar = OMP_CLAUSE_DECL (innerc);
7697 617 : innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
7698 : OMP_CLAUSE__LOOPTEMP_);
7699 617 : gcc_assert (innerc);
7700 617 : endvar = OMP_CLAUSE_DECL (innerc);
7701 : }
7702 1330 : t = fold_convert (TREE_TYPE (startvar), n1);
7703 1330 : t = force_gimple_operand_gsi (&gsi, t,
7704 1330 : DECL_P (startvar)
7705 1330 : && TREE_ADDRESSABLE (startvar),
7706 : NULL_TREE, false, GSI_CONTINUE_LINKING);
7707 1330 : gimple *assign_stmt = gimple_build_assign (startvar, t);
7708 1330 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7709 :
7710 1330 : t = fold_convert (TREE_TYPE (startvar), n2);
7711 1330 : e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7712 : false, GSI_CONTINUE_LINKING);
7713 1330 : if (endvar)
7714 : {
7715 617 : assign_stmt = gimple_build_assign (endvar, e);
7716 617 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7717 617 : if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e)))
7718 529 : assign_stmt = gimple_build_assign (fd->loop.v, e);
7719 : else
7720 88 : assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e);
7721 617 : gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7722 : }
7723 :
7724 1330 : tree *nonrect_bounds = NULL;
7725 1330 : if (fd->collapse > 1)
7726 : {
7727 181 : if (fd->non_rect)
7728 : {
7729 19 : nonrect_bounds = XALLOCAVEC (tree, fd->last_nonrect + 1);
7730 19 : memset (nonrect_bounds, 0, sizeof (tree) * (fd->last_nonrect + 1));
7731 : }
7732 181 : gcc_assert (gsi_bb (gsi) == entry_bb);
7733 181 : expand_omp_for_init_vars (fd, &gsi, counts, nonrect_bounds, inner_stmt,
7734 : startvar);
7735 181 : entry_bb = gsi_bb (gsi);
7736 : }
7737 :
7738 1330 : if (!broken_loop)
7739 : {
7740 : /* The code controlling the sequential loop replaces the
7741 : GIMPLE_OMP_CONTINUE. */
7742 1314 : gsi = gsi_last_nondebug_bb (cont_bb);
7743 1314 : gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
7744 1314 : gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE);
7745 1314 : vmain = gimple_omp_continue_control_use (cont_stmt);
7746 1314 : vback = gimple_omp_continue_control_def (cont_stmt);
7747 :
7748 1314 : if (!gimple_omp_for_combined_p (fd->for_stmt))
7749 : {
7750 697 : if (POINTER_TYPE_P (type))
7751 8 : t = fold_build_pointer_plus (vmain, step);
7752 : else
7753 689 : t = fold_build2 (PLUS_EXPR, type, vmain, step);
7754 697 : t = force_gimple_operand_gsi (&gsi, t,
7755 697 : DECL_P (vback)
7756 697 : && TREE_ADDRESSABLE (vback),
7757 : NULL_TREE, true, GSI_SAME_STMT);
7758 697 : assign_stmt = gimple_build_assign (vback, t);
7759 697 : gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
7760 :
7761 697 : t = build2 (fd->loop.cond_code, boolean_type_node,
7762 697 : DECL_P (vback) && TREE_ADDRESSABLE (vback)
7763 : ? t : vback, e);
7764 697 : gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
7765 : }
7766 :
7767 : /* Remove the GIMPLE_OMP_CONTINUE statement. */
7768 1314 : gsi_remove (&gsi, true);
7769 :
7770 1314 : if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
7771 81 : collapse_bb = extract_omp_for_update_vars (fd, nonrect_bounds,
7772 : cont_bb, body_bb);
7773 : }
7774 :
7775 : /* Remove the GIMPLE_OMP_FOR statement. */
7776 1330 : gsi = gsi_for_stmt (fd->for_stmt);
7777 1330 : gsi_remove (&gsi, true);
7778 :
7779 : /* Remove the GIMPLE_OMP_RETURN statement. */
7780 1330 : gsi = gsi_last_nondebug_bb (exit_bb);
7781 1330 : gsi_remove (&gsi, true);
7782 :
7783 1330 : FALLTHRU_EDGE (entry_bb)->probability = profile_probability::always ();
7784 1330 : if (!broken_loop)
7785 1314 : remove_edge (BRANCH_EDGE (entry_bb));
7786 : else
7787 : {
7788 16 : remove_edge_and_dominated_blocks (BRANCH_EDGE (entry_bb));
7789 16 : region->outer->cont = NULL;
7790 : }
7791 :
7792 : /* Connect all the blocks. */
7793 1330 : if (!broken_loop)
7794 : {
7795 1314 : ep = find_edge (cont_bb, body_bb);
7796 1314 : if (gimple_omp_for_combined_p (fd->for_stmt))
7797 : {
7798 617 : remove_edge (ep);
7799 617 : ep = NULL;
7800 : }
7801 697 : else if (fd->collapse > 1)
7802 : {
7803 81 : remove_edge (ep);
7804 81 : ep = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
7805 : }
7806 : else
7807 616 : ep->flags = EDGE_TRUE_VALUE;
7808 1931 : find_edge (cont_bb, fin_bb)->flags
7809 1395 : = ep ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
7810 : }
7811 :
7812 1330 : set_immediate_dominator (CDI_DOMINATORS, body_bb,
7813 : recompute_dominator (CDI_DOMINATORS, body_bb));
7814 1330 : if (!broken_loop)
7815 1314 : set_immediate_dominator (CDI_DOMINATORS, fin_bb,
7816 : recompute_dominator (CDI_DOMINATORS, fin_bb));
7817 :
7818 1314 : if (!broken_loop && !gimple_omp_for_combined_p (fd->for_stmt))
7819 : {
7820 697 : class loop *loop = alloc_loop ();
7821 697 : loop->header = body_bb;
7822 697 : if (collapse_bb == NULL)
7823 616 : loop->latch = cont_bb;
7824 697 : add_loop (loop, body_bb->loop_father);
7825 : }
7826 1330 : }
7827 :
7828 : /* A subroutine of expand_omp_for. Generate code for an OpenACC
7829 : partitioned loop. The lowering here is abstracted, in that the
7830 : loop parameters are passed through internal functions, which are
7831 : further lowered by oacc_device_lower, once we get to the target
7832 : compiler. The loop is of the form:
7833 :
7834 : for (V = B; V LTGT E; V += S) {BODY}
7835 :
7836 : where LTGT is < or >. We may have a specified chunking size, CHUNKING
7837 : (constant 0 for no chunking) and we will have a GWV partitioning
7838 : mask, specifying dimensions over which the loop is to be
7839 : partitioned (see note below). We generate code that looks like
7840 : (this ignores tiling):
7841 :
7842 : <entry_bb> [incoming FALL->body, BRANCH->exit]
7843 : typedef signedintify (typeof (V)) T; // underlying signed integral type
7844 : T range = E - B;
7845 : T chunk_no = 0;
7846 : T DIR = LTGT == '<' ? +1 : -1;
7847 : T chunk_max = GOACC_LOOP_CHUNK (dir, range, S, CHUNK_SIZE, GWV);
7848 : T step = GOACC_LOOP_STEP (dir, range, S, CHUNK_SIZE, GWV);
7849 :
7850 : <head_bb> [created by splitting end of entry_bb]
7851 : T offset = GOACC_LOOP_OFFSET (dir, range, S, CHUNK_SIZE, GWV, chunk_no);
7852 : T bound = GOACC_LOOP_BOUND (dir, range, S, CHUNK_SIZE, GWV, offset);
7853 : if (!(offset LTGT bound)) goto bottom_bb;
7854 :
7855 : <body_bb> [incoming]
7856 : V = B + offset;
7857 : {BODY}
7858 :
7859 : <cont_bb> [incoming, may == body_bb FALL->exit_bb, BRANCH->body_bb]
7860 : offset += step;
7861 : if (offset LTGT bound) goto body_bb; [*]
7862 :
7863 : <bottom_bb> [created by splitting start of exit_bb] insert BRANCH->head_bb
7864 : chunk_no++;
7865 : if (chunk < chunk_max) goto head_bb;
7866 :
7867 : <exit_bb> [incoming]
7868 : V = B + ((range -/+ 1) / S +/- 1) * S [*]
7869 :
7870 : [*] Needed if V live at end of loop. */
7871 :
7872 : static void
7873 11711 : expand_oacc_for (struct omp_region *region, struct omp_for_data *fd)
7874 : {
7875 11711 : bool is_oacc_kernels_parallelized
7876 11711 : = (lookup_attribute ("oacc kernels parallelized",
7877 11711 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
7878 11711 : {
7879 11711 : bool is_oacc_kernels
7880 11711 : = (lookup_attribute ("oacc kernels",
7881 11711 : DECL_ATTRIBUTES (current_function_decl)) != NULL);
7882 11711 : if (is_oacc_kernels_parallelized)
7883 386 : gcc_checking_assert (is_oacc_kernels);
7884 : }
7885 23422 : gcc_assert (gimple_in_ssa_p (cfun) == is_oacc_kernels_parallelized);
7886 : /* In the following, some of the 'gimple_in_ssa_p (cfun)' conditionals are
7887 : for SSA specifics, and some are for 'parloops' OpenACC
7888 : 'kernels'-parallelized specifics. */
7889 :
7890 11711 : tree v = fd->loop.v;
7891 11711 : enum tree_code cond_code = fd->loop.cond_code;
7892 11711 : enum tree_code plus_code = PLUS_EXPR;
7893 :
7894 11711 : tree chunk_size = integer_minus_one_node;
7895 11711 : tree gwv = integer_zero_node;
7896 11711 : tree iter_type = TREE_TYPE (v);
7897 11711 : tree diff_type = iter_type;
7898 11711 : tree plus_type = iter_type;
7899 11711 : struct oacc_collapse *counts = NULL;
7900 :
7901 11711 : gcc_checking_assert (gimple_omp_for_kind (fd->for_stmt)
7902 : == GF_OMP_FOR_KIND_OACC_LOOP);
7903 11711 : gcc_assert (!gimple_omp_for_combined_into_p (fd->for_stmt));
7904 11711 : gcc_assert (cond_code == LT_EXPR || cond_code == GT_EXPR);
7905 :
7906 11711 : if (POINTER_TYPE_P (iter_type))
7907 : {
7908 52 : plus_code = POINTER_PLUS_EXPR;
7909 52 : plus_type = sizetype;
7910 : }
7911 24124 : for (int ix = fd->collapse; ix--;)
7912 : {
7913 12413 : tree diff_type2 = TREE_TYPE (fd->loops[ix].step);
7914 12413 : if (TYPE_PRECISION (diff_type) < TYPE_PRECISION (diff_type2))
7915 0 : diff_type = diff_type2;
7916 : }
7917 11711 : if (POINTER_TYPE_P (diff_type) || TYPE_UNSIGNED (diff_type))
7918 1002 : diff_type = signed_type_for (diff_type);
7919 11711 : if (TYPE_PRECISION (diff_type) < TYPE_PRECISION (integer_type_node))
7920 23 : diff_type = integer_type_node;
7921 :
7922 11711 : basic_block entry_bb = region->entry; /* BB ending in OMP_FOR */
7923 11711 : basic_block exit_bb = region->exit; /* BB ending in OMP_RETURN */
7924 11711 : basic_block cont_bb = region->cont; /* BB ending in OMP_CONTINUE */
7925 11711 : basic_block bottom_bb = NULL;
7926 :
7927 : /* entry_bb has two successors; the branch edge is to the exit
7928 : block (or to finalization blocks preceding it), fallthrough edge
7929 : to body. */
7930 11711 : gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
7931 :
7932 : /* If cont_bb non-NULL, it has 2 successors. The branch successor is
7933 : body_bb, or to a block whose only successor is the body_bb. Its
7934 : fallthrough successor is the final block (same as the branch
7935 : successor of the entry_bb), possibly via finalization blocks. */
7936 11711 : if (cont_bb)
7937 : {
7938 11672 : basic_block body_bb = FALLTHRU_EDGE (entry_bb)->dest;
7939 11672 : basic_block bed = BRANCH_EDGE (cont_bb)->dest;
7940 :
7941 11672 : gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
7942 11672 : gcc_assert (bed == body_bb || single_succ_edge (bed)->dest == body_bb);
7943 : }
7944 : else
7945 39 : gcc_assert (!gimple_in_ssa_p (cfun));
7946 :
7947 11711 : tree chunk_no;
7948 11711 : tree chunk_max = NULL_TREE;
7949 11711 : tree bound, offset;
7950 11711 : tree step = create_tmp_var (diff_type, ".step");
7951 11711 : bool up = cond_code == LT_EXPR;
7952 11871 : tree dir = build_int_cst (diff_type, up ? +1 : -1);
7953 11711 : bool chunking = !gimple_in_ssa_p (cfun);
7954 11711 : bool negating;
7955 :
7956 : /* Tiling vars. */
7957 11711 : tree tile_size = NULL_TREE;
7958 11711 : tree element_s = NULL_TREE;
7959 11711 : tree e_bound = NULL_TREE, e_offset = NULL_TREE, e_step = NULL_TREE;
7960 11711 : basic_block elem_body_bb = NULL;
7961 11711 : basic_block elem_cont_bb = NULL;
7962 :
7963 : /* SSA instances. */
7964 11711 : tree offset_incr = NULL_TREE;
7965 11711 : tree offset_init = NULL_TREE;
7966 :
7967 11711 : gimple_stmt_iterator gsi;
7968 11711 : gassign *ass;
7969 11711 : gcall *call;
7970 11711 : gimple *stmt;
7971 11711 : tree expr;
7972 11711 : location_t loc;
7973 11711 : edge split, be, fte;
7974 :
7975 : /* Split the end of entry_bb to create head_bb. */
7976 11711 : split = split_block (entry_bb, last_nondebug_stmt (entry_bb));
7977 11711 : basic_block head_bb = split->dest;
7978 11711 : entry_bb = split->src;
7979 :
7980 : /* Chunk setup goes at end of entry_bb, replacing the omp_for. */
7981 11711 : gsi = gsi_last_nondebug_bb (entry_bb);
7982 11711 : gomp_for *for_stmt = as_a <gomp_for *> (gsi_stmt (gsi));
7983 11711 : loc = gimple_location (for_stmt);
7984 :
7985 11711 : if (gimple_in_ssa_p (cfun))
7986 : {
7987 386 : offset_init = gimple_omp_for_index (for_stmt, 0);
7988 386 : gcc_assert (integer_zerop (fd->loop.n1));
7989 : /* The SSA parallelizer does gang parallelism. */
7990 386 : gwv = build_int_cst (integer_type_node, GOMP_DIM_MASK (GOMP_DIM_GANG));
7991 : }
7992 :
7993 11711 : if (fd->collapse > 1 || fd->tiling)
7994 : {
7995 1188 : gcc_assert (!gimple_in_ssa_p (cfun) && up);
7996 594 : counts = XALLOCAVEC (struct oacc_collapse, fd->collapse);
7997 594 : tree total = expand_oacc_collapse_init (fd, &gsi, counts, diff_type,
7998 594 : TREE_TYPE (fd->loop.n2), loc);
7999 :
8000 594 : if (SSA_VAR_P (fd->loop.n2))
8001 : {
8002 107 : total = force_gimple_operand_gsi (&gsi, total, false, NULL_TREE,
8003 : true, GSI_SAME_STMT);
8004 107 : ass = gimple_build_assign (fd->loop.n2, total);
8005 107 : gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
8006 : }
8007 : }
8008 :
8009 11711 : tree b = fd->loop.n1;
8010 11711 : tree e = fd->loop.n2;
8011 11711 : tree s = fd->loop.step;
8012 :
8013 11711 : b = force_gimple_operand_gsi (&gsi, b, true, NULL_TREE, true, GSI_SAME_STMT);
8014 11711 : e = force_gimple_operand_gsi (&gsi, e, true, NULL_TREE, true, GSI_SAME_STMT);
8015 :
8016 : /* Convert the step, avoiding possible unsigned->signed overflow. */
8017 11711 : negating = !up && TYPE_UNSIGNED (TREE_TYPE (s));
8018 32 : if (negating)
8019 32 : s = fold_build1 (NEGATE_EXPR, TREE_TYPE (s), s);
8020 11711 : s = fold_convert (diff_type, s);
8021 11711 : if (negating)
8022 32 : s = fold_build1 (NEGATE_EXPR, diff_type, s);
8023 11711 : s = force_gimple_operand_gsi (&gsi, s, true, NULL_TREE, true, GSI_SAME_STMT);
8024 :
8025 11711 : if (!chunking)
8026 386 : chunk_size = integer_zero_node;
8027 11711 : expr = fold_convert (diff_type, chunk_size);
8028 11711 : chunk_size = force_gimple_operand_gsi (&gsi, expr, true,
8029 : NULL_TREE, true, GSI_SAME_STMT);
8030 :
8031 11711 : if (fd->tiling)
8032 : {
8033 : /* Determine the tile size and element step,
8034 : modify the outer loop step size. */
8035 177 : tile_size = create_tmp_var (diff_type, ".tile_size");
8036 177 : expr = build_int_cst (diff_type, 1);
8037 461 : for (int ix = 0; ix < fd->collapse; ix++)
8038 284 : expr = fold_build2 (MULT_EXPR, diff_type, counts[ix].tile, expr);
8039 177 : expr = force_gimple_operand_gsi (&gsi, expr, true,
8040 : NULL_TREE, true, GSI_SAME_STMT);
8041 177 : ass = gimple_build_assign (tile_size, expr);
8042 177 : gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
8043 :
8044 177 : element_s = create_tmp_var (diff_type, ".element_s");
8045 177 : ass = gimple_build_assign (element_s, s);
8046 177 : gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
8047 :
8048 177 : expr = fold_build2 (MULT_EXPR, diff_type, s, tile_size);
8049 177 : s = force_gimple_operand_gsi (&gsi, expr, true,
8050 : NULL_TREE, true, GSI_SAME_STMT);
8051 : }
8052 :
8053 : /* Determine the range, avoiding possible unsigned->signed overflow. */
8054 11711 : negating = !up && TYPE_UNSIGNED (iter_type);
8055 23374 : expr = fold_build2 (MINUS_EXPR, plus_type,
8056 : fold_convert (plus_type, negating ? b : e),
8057 : fold_convert (plus_type, negating ? e : b));
8058 11711 : expr = fold_convert (diff_type, expr);
8059 11711 : if (negating)
8060 48 : expr = fold_build1 (NEGATE_EXPR, diff_type, expr);
8061 11711 : tree range = force_gimple_operand_gsi (&gsi, expr, true,
8062 : NULL_TREE, true, GSI_SAME_STMT);
8063 :
8064 11711 : chunk_no = build_int_cst (diff_type, 0);
8065 11711 : if (chunking)
8066 : {
8067 11325 : gcc_assert (!gimple_in_ssa_p (cfun));
8068 :
8069 11325 : expr = chunk_no;
8070 11325 : chunk_max = create_tmp_var (diff_type, ".chunk_max");
8071 11325 : chunk_no = create_tmp_var (diff_type, ".chunk_no");
8072 :
8073 11325 : ass = gimple_build_assign (chunk_no, expr);
8074 11325 : gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
8075 :
8076 11325 : call = gimple_build_call_internal (IFN_GOACC_LOOP, 6,
8077 : build_int_cst (integer_type_node,
8078 : IFN_GOACC_LOOP_CHUNKS),
8079 : dir, range, s, chunk_size, gwv);
8080 11325 : gimple_call_set_lhs (call, chunk_max);
8081 11325 : gimple_set_location (call, loc);
8082 11325 : gsi_insert_before (&gsi, call, GSI_SAME_STMT);
8083 : }
8084 : else
8085 : chunk_size = chunk_no;
8086 :
8087 11711 : call = gimple_build_call_internal (IFN_GOACC_LOOP, 6,
8088 : build_int_cst (integer_type_node,
8089 : IFN_GOACC_LOOP_STEP),
8090 : dir, range, s, chunk_size, gwv);
8091 11711 : gimple_call_set_lhs (call, step);
8092 11711 : gimple_set_location (call, loc);
8093 11711 : gsi_insert_before (&gsi, call, GSI_SAME_STMT);
8094 :
8095 : /* Remove the GIMPLE_OMP_FOR. */
8096 11711 : gsi_remove (&gsi, true);
8097 :
8098 : /* Fixup edges from head_bb. */
8099 11711 : be = BRANCH_EDGE (head_bb);
8100 11711 : fte = FALLTHRU_EDGE (head_bb);
8101 11711 : be->flags |= EDGE_FALSE_VALUE;
8102 11711 : fte->flags ^= EDGE_FALLTHRU | EDGE_TRUE_VALUE;
8103 :
8104 11711 : basic_block body_bb = fte->dest;
8105 :
8106 11711 : if (gimple_in_ssa_p (cfun))
8107 : {
8108 386 : gsi = gsi_last_nondebug_bb (cont_bb);
8109 386 : gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
8110 :
8111 386 : offset = gimple_omp_continue_control_use (cont_stmt);
8112 386 : offset_incr = gimple_omp_continue_control_def (cont_stmt);
8113 : }
8114 : else
8115 : {
8116 11325 : offset = create_tmp_var (diff_type, ".offset");
8117 11325 : offset_init = offset_incr = offset;
8118 : }
8119 11711 : bound = create_tmp_var (TREE_TYPE (offset), ".bound");
8120 :
8121 : /* Loop offset & bound go into head_bb. */
8122 11711 : gsi = gsi_start_bb (head_bb);
8123 :
8124 11711 : call = gimple_build_call_internal (IFN_GOACC_LOOP, 7,
8125 : build_int_cst (integer_type_node,
8126 : IFN_GOACC_LOOP_OFFSET),
8127 : dir, range, s,
8128 : chunk_size, gwv, chunk_no);
8129 11711 : gimple_call_set_lhs (call, offset_init);
8130 11711 : gimple_set_location (call, loc);
8131 11711 : gsi_insert_after (&gsi, call, GSI_CONTINUE_LINKING);
8132 :
8133 11711 : call = gimple_build_call_internal (IFN_GOACC_LOOP, 7,
8134 : build_int_cst (integer_type_node,
8135 : IFN_GOACC_LOOP_BOUND),
8136 : dir, range, s,
8137 : chunk_size, gwv, offset_init);
8138 11711 : gimple_call_set_lhs (call, bound);
8139 11711 : gimple_set_location (call, loc);
8140 11711 : gsi_insert_after (&gsi, call, GSI_CONTINUE_LINKING);
8141 :
8142 11711 : expr = build2 (cond_code, boolean_type_node, offset_init, bound);
8143 11711 : gsi_insert_after (&gsi, gimple_build_cond_empty (expr),
8144 : GSI_CONTINUE_LINKING);
8145 :
8146 : /* V assignment goes into body_bb. */
8147 11711 : if (!gimple_in_ssa_p (cfun))
8148 : {
8149 11325 : gsi = gsi_start_bb (body_bb);
8150 :
8151 11325 : expr = build2 (plus_code, iter_type, b,
8152 : fold_convert (plus_type, offset));
8153 11325 : expr = force_gimple_operand_gsi (&gsi, expr, false, NULL_TREE,
8154 : true, GSI_SAME_STMT);
8155 11325 : ass = gimple_build_assign (v, expr);
8156 11325 : gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
8157 :
8158 11325 : if (fd->collapse > 1 || fd->tiling)
8159 594 : expand_oacc_collapse_vars (fd, false, &gsi, counts, v, diff_type);
8160 :
8161 11325 : if (fd->tiling)
8162 : {
8163 : /* Determine the range of the element loop -- usually simply
8164 : the tile_size, but could be smaller if the final
8165 : iteration of the outer loop is a partial tile. */
8166 177 : tree e_range = create_tmp_var (diff_type, ".e_range");
8167 :
8168 177 : expr = build2 (MIN_EXPR, diff_type,
8169 : build2 (MINUS_EXPR, diff_type, bound, offset),
8170 : build2 (MULT_EXPR, diff_type, tile_size,
8171 : element_s));
8172 177 : expr = force_gimple_operand_gsi (&gsi, expr, false, NULL_TREE,
8173 : true, GSI_SAME_STMT);
8174 177 : ass = gimple_build_assign (e_range, expr);
8175 177 : gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
8176 :
8177 : /* Determine bound, offset & step of inner loop. */
8178 177 : e_bound = create_tmp_var (diff_type, ".e_bound");
8179 177 : e_offset = create_tmp_var (diff_type, ".e_offset");
8180 177 : e_step = create_tmp_var (diff_type, ".e_step");
8181 :
8182 : /* Mark these as element loops. */
8183 177 : tree t, e_gwv = integer_minus_one_node;
8184 177 : tree chunk = build_int_cst (diff_type, 0); /* Never chunked. */
8185 :
8186 177 : t = build_int_cst (integer_type_node, IFN_GOACC_LOOP_OFFSET);
8187 177 : call = gimple_build_call_internal (IFN_GOACC_LOOP, 7, t, dir, e_range,
8188 : element_s, chunk, e_gwv, chunk);
8189 177 : gimple_call_set_lhs (call, e_offset);
8190 177 : gimple_set_location (call, loc);
8191 177 : gsi_insert_before (&gsi, call, GSI_SAME_STMT);
8192 :
8193 177 : t = build_int_cst (integer_type_node, IFN_GOACC_LOOP_BOUND);
8194 177 : call = gimple_build_call_internal (IFN_GOACC_LOOP, 7, t, dir, e_range,
8195 : element_s, chunk, e_gwv, e_offset);
8196 177 : gimple_call_set_lhs (call, e_bound);
8197 177 : gimple_set_location (call, loc);
8198 177 : gsi_insert_before (&gsi, call, GSI_SAME_STMT);
8199 :
8200 177 : t = build_int_cst (integer_type_node, IFN_GOACC_LOOP_STEP);
8201 177 : call = gimple_build_call_internal (IFN_GOACC_LOOP, 6, t, dir, e_range,
8202 : element_s, chunk, e_gwv);
8203 177 : gimple_call_set_lhs (call, e_step);
8204 177 : gimple_set_location (call, loc);
8205 177 : gsi_insert_before (&gsi, call, GSI_SAME_STMT);
8206 :
8207 : /* Add test and split block. */
8208 177 : expr = build2 (cond_code, boolean_type_node, e_offset, e_bound);
8209 177 : stmt = gimple_build_cond_empty (expr);
8210 177 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
8211 177 : split = split_block (body_bb, stmt);
8212 177 : elem_body_bb = split->dest;
8213 177 : if (cont_bb == body_bb)
8214 146 : cont_bb = elem_body_bb;
8215 177 : body_bb = split->src;
8216 :
8217 177 : split->flags ^= EDGE_FALLTHRU | EDGE_TRUE_VALUE;
8218 :
8219 : /* Add a dummy exit for the tiled block when cont_bb is missing. */
8220 177 : if (cont_bb == NULL)
8221 : {
8222 5 : edge e = make_edge (body_bb, exit_bb, EDGE_FALSE_VALUE);
8223 5 : e->probability = profile_probability::even ();
8224 5 : split->probability = profile_probability::even ();
8225 : }
8226 :
8227 : /* Initialize the user's loop vars. */
8228 177 : gsi = gsi_start_bb (elem_body_bb);
8229 177 : expand_oacc_collapse_vars (fd, true, &gsi, counts, e_offset,
8230 : diff_type);
8231 : }
8232 : }
8233 :
8234 : /* Loop increment goes into cont_bb. If this is not a loop, we
8235 : will have spawned threads as if it was, and each one will
8236 : execute one iteration. The specification is not explicit about
8237 : whether such constructs are ill-formed or not, and they can
8238 : occur, especially when noreturn routines are involved. */
8239 11711 : if (cont_bb)
8240 : {
8241 11672 : gsi = gsi_last_nondebug_bb (cont_bb);
8242 11672 : gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
8243 11672 : loc = gimple_location (cont_stmt);
8244 :
8245 11672 : if (fd->tiling)
8246 : {
8247 : /* Insert element loop increment and test. */
8248 172 : expr = build2 (PLUS_EXPR, diff_type, e_offset, e_step);
8249 172 : expr = force_gimple_operand_gsi (&gsi, expr, false, NULL_TREE,
8250 : true, GSI_SAME_STMT);
8251 172 : ass = gimple_build_assign (e_offset, expr);
8252 172 : gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
8253 172 : expr = build2 (cond_code, boolean_type_node, e_offset, e_bound);
8254 :
8255 172 : stmt = gimple_build_cond_empty (expr);
8256 172 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
8257 172 : split = split_block (cont_bb, stmt);
8258 172 : elem_cont_bb = split->src;
8259 172 : cont_bb = split->dest;
8260 :
8261 172 : split->flags ^= EDGE_FALLTHRU | EDGE_FALSE_VALUE;
8262 172 : split->probability = profile_probability::unlikely ().guessed ();
8263 172 : edge latch_edge
8264 172 : = make_edge (elem_cont_bb, elem_body_bb, EDGE_TRUE_VALUE);
8265 172 : latch_edge->probability = profile_probability::likely ().guessed ();
8266 :
8267 172 : edge skip_edge = make_edge (body_bb, cont_bb, EDGE_FALSE_VALUE);
8268 172 : skip_edge->probability = profile_probability::unlikely ().guessed ();
8269 172 : edge loop_entry_edge = EDGE_SUCC (body_bb, 1 - skip_edge->dest_idx);
8270 172 : loop_entry_edge->probability
8271 172 : = profile_probability::likely ().guessed ();
8272 :
8273 172 : gsi = gsi_for_stmt (cont_stmt);
8274 : }
8275 :
8276 : /* Increment offset. */
8277 11672 : if (gimple_in_ssa_p (cfun))
8278 386 : expr = build2 (plus_code, iter_type, offset,
8279 : fold_convert (plus_type, step));
8280 : else
8281 11286 : expr = build2 (PLUS_EXPR, diff_type, offset, step);
8282 11672 : expr = force_gimple_operand_gsi (&gsi, expr, false, NULL_TREE,
8283 : true, GSI_SAME_STMT);
8284 11672 : ass = gimple_build_assign (offset_incr, expr);
8285 11672 : gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
8286 11672 : expr = build2 (cond_code, boolean_type_node, offset_incr, bound);
8287 11672 : gsi_insert_before (&gsi, gimple_build_cond_empty (expr), GSI_SAME_STMT);
8288 :
8289 : /* Remove the GIMPLE_OMP_CONTINUE. */
8290 11672 : gsi_remove (&gsi, true);
8291 :
8292 : /* Fixup edges from cont_bb. */
8293 11672 : be = BRANCH_EDGE (cont_bb);
8294 11672 : fte = FALLTHRU_EDGE (cont_bb);
8295 11672 : be->flags |= EDGE_TRUE_VALUE;
8296 11672 : fte->flags ^= EDGE_FALLTHRU | EDGE_FALSE_VALUE;
8297 :
8298 11672 : if (chunking)
8299 : {
8300 : /* Split the beginning of exit_bb to make bottom_bb. We
8301 : need to insert a nop at the start, because splitting is
8302 : after a stmt, not before. */
8303 11286 : gsi = gsi_start_bb (exit_bb);
8304 11286 : stmt = gimple_build_nop ();
8305 11286 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
8306 11286 : split = split_block (exit_bb, stmt);
8307 11286 : bottom_bb = split->src;
8308 11286 : exit_bb = split->dest;
8309 11286 : gsi = gsi_last_bb (bottom_bb);
8310 :
8311 : /* Chunk increment and test goes into bottom_bb. */
8312 11286 : expr = build2 (PLUS_EXPR, diff_type, chunk_no,
8313 : build_int_cst (diff_type, 1));
8314 11286 : ass = gimple_build_assign (chunk_no, expr);
8315 11286 : gsi_insert_after (&gsi, ass, GSI_CONTINUE_LINKING);
8316 :
8317 : /* Chunk test at end of bottom_bb. */
8318 11286 : expr = build2 (LT_EXPR, boolean_type_node, chunk_no, chunk_max);
8319 11286 : gsi_insert_after (&gsi, gimple_build_cond_empty (expr),
8320 : GSI_CONTINUE_LINKING);
8321 :
8322 : /* Fixup edges from bottom_bb. */
8323 11286 : split->flags ^= EDGE_FALLTHRU | EDGE_FALSE_VALUE;
8324 11286 : split->probability = profile_probability::unlikely ().guessed ();
8325 11286 : edge latch_edge = make_edge (bottom_bb, head_bb, EDGE_TRUE_VALUE);
8326 11286 : latch_edge->probability = profile_probability::likely ().guessed ();
8327 : }
8328 : }
8329 :
8330 11711 : gsi = gsi_last_nondebug_bb (exit_bb);
8331 11711 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
8332 11711 : loc = gimple_location (gsi_stmt (gsi));
8333 :
8334 11711 : if (!gimple_in_ssa_p (cfun))
8335 : {
8336 : /* Insert the final value of V, in case it is live. This is the
8337 : value for the only thread that survives past the join. */
8338 11325 : expr = fold_build2 (MINUS_EXPR, diff_type, range, dir);
8339 11325 : expr = fold_build2 (PLUS_EXPR, diff_type, expr, s);
8340 11325 : expr = fold_build2 (TRUNC_DIV_EXPR, diff_type, expr, s);
8341 11325 : expr = fold_build2 (MULT_EXPR, diff_type, expr, s);
8342 11325 : expr = build2 (plus_code, iter_type, b, fold_convert (plus_type, expr));
8343 11325 : expr = force_gimple_operand_gsi (&gsi, expr, false, NULL_TREE,
8344 : true, GSI_SAME_STMT);
8345 11325 : ass = gimple_build_assign (v, expr);
8346 11325 : gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
8347 : }
8348 :
8349 : /* Remove the OMP_RETURN. */
8350 11711 : gsi_remove (&gsi, true);
8351 :
8352 11711 : if (cont_bb)
8353 : {
8354 : /* We now have one, two or three nested loops. Update the loop
8355 : structures. */
8356 11672 : class loop *parent = entry_bb->loop_father;
8357 11672 : class loop *body = body_bb->loop_father;
8358 :
8359 11672 : if (chunking)
8360 : {
8361 11286 : class loop *chunk_loop = alloc_loop ();
8362 11286 : chunk_loop->header = head_bb;
8363 11286 : chunk_loop->latch = bottom_bb;
8364 11286 : add_loop (chunk_loop, parent);
8365 11286 : parent = chunk_loop;
8366 : }
8367 386 : else if (parent != body)
8368 : {
8369 386 : gcc_assert (body->header == body_bb);
8370 386 : gcc_assert (body->latch == cont_bb
8371 : || single_pred (body->latch) == cont_bb);
8372 : parent = NULL;
8373 : }
8374 :
8375 11286 : if (parent)
8376 : {
8377 11286 : class loop *body_loop = alloc_loop ();
8378 11286 : body_loop->header = body_bb;
8379 11286 : body_loop->latch = cont_bb;
8380 11286 : add_loop (body_loop, parent);
8381 :
8382 11286 : if (fd->tiling)
8383 : {
8384 : /* Insert tiling's element loop. */
8385 172 : class loop *inner_loop = alloc_loop ();
8386 172 : inner_loop->header = elem_body_bb;
8387 172 : inner_loop->latch = elem_cont_bb;
8388 172 : add_loop (inner_loop, body_loop);
8389 : }
8390 : }
8391 : }
8392 11711 : }
8393 :
8394 : /* Expand the OMP loop defined by REGION. */
8395 :
8396 : static void
8397 47801 : expand_omp_for (struct omp_region *region, gimple *inner_stmt)
8398 : {
8399 47801 : struct omp_for_data fd;
8400 47801 : struct omp_for_data_loop *loops;
8401 :
8402 47801 : loops = XALLOCAVEC (struct omp_for_data_loop,
8403 : gimple_omp_for_collapse
8404 : (last_nondebug_stmt (region->entry)));
8405 47801 : omp_extract_for_data (as_a <gomp_for *> (last_nondebug_stmt (region->entry)),
8406 : &fd, loops);
8407 47801 : region->sched_kind = fd.sched_kind;
8408 47801 : region->sched_modifiers = fd.sched_modifiers;
8409 47801 : region->has_lastprivate_conditional = fd.lastprivate_conditional != 0;
8410 47801 : if (fd.non_rect && !gimple_omp_for_combined_into_p (fd.for_stmt))
8411 : {
8412 2344 : for (int i = fd.first_nonrect; i <= fd.last_nonrect; i++)
8413 1644 : if ((loops[i].m1 || loops[i].m2)
8414 731 : && (loops[i].m1 == NULL_TREE
8415 506 : || TREE_CODE (loops[i].m1) == INTEGER_CST)
8416 612 : && (loops[i].m2 == NULL_TREE
8417 343 : || TREE_CODE (loops[i].m2) == INTEGER_CST)
8418 596 : && TREE_CODE (loops[i].step) == INTEGER_CST
8419 586 : && TREE_CODE (loops[i - loops[i].outer].step) == INTEGER_CST)
8420 : {
8421 586 : tree t;
8422 586 : tree itype = TREE_TYPE (loops[i].v);
8423 586 : if (loops[i].m1 && loops[i].m2)
8424 108 : t = fold_build2 (MINUS_EXPR, itype, loops[i].m2, loops[i].m1);
8425 478 : else if (loops[i].m1)
8426 267 : t = fold_build1 (NEGATE_EXPR, itype, loops[i].m1);
8427 : else
8428 : t = loops[i].m2;
8429 586 : t = fold_build2 (MULT_EXPR, itype, t,
8430 : fold_convert (itype,
8431 : loops[i - loops[i].outer].step));
8432 586 : if (TYPE_UNSIGNED (itype) && loops[i].cond_code == GT_EXPR)
8433 3 : t = fold_build2 (TRUNC_MOD_EXPR, itype,
8434 : fold_build1 (NEGATE_EXPR, itype, t),
8435 : fold_build1 (NEGATE_EXPR, itype,
8436 : fold_convert (itype,
8437 : loops[i].step)));
8438 : else
8439 583 : t = fold_build2 (TRUNC_MOD_EXPR, itype, t,
8440 : fold_convert (itype, loops[i].step));
8441 586 : if (integer_nonzerop (t))
8442 8 : error_at (gimple_location (fd.for_stmt),
8443 : "invalid OpenMP non-rectangular loop step; "
8444 : "%<(%E - %E) * %E%> is not a multiple of loop %d "
8445 : "step %qE",
8446 8 : loops[i].m2 ? loops[i].m2 : integer_zero_node,
8447 8 : loops[i].m1 ? loops[i].m1 : integer_zero_node,
8448 8 : loops[i - loops[i].outer].step, i + 1,
8449 : loops[i].step);
8450 : }
8451 : }
8452 :
8453 47801 : gcc_assert (EDGE_COUNT (region->entry->succs) == 2);
8454 47801 : BRANCH_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
8455 47801 : FALLTHRU_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
8456 47801 : if (region->cont)
8457 : {
8458 45514 : gcc_assert (EDGE_COUNT (region->cont->succs) == 2);
8459 45514 : BRANCH_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
8460 45514 : FALLTHRU_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
8461 : }
8462 : else
8463 : /* If there isn't a continue then this is a degenerate case where
8464 : the introduction of abnormal edges during lowering will prevent
8465 : original loops from being detected. Fix that up. */
8466 2287 : loops_state_set (LOOPS_NEED_FIXUP);
8467 :
8468 47801 : if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_SIMD)
8469 9374 : expand_omp_simd (region, &fd);
8470 38427 : else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
8471 : {
8472 11711 : gcc_assert (!inner_stmt && !fd.non_rect);
8473 11711 : expand_oacc_for (region, &fd);
8474 : }
8475 26716 : else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_TASKLOOP)
8476 : {
8477 2660 : if (gimple_omp_for_combined_into_p (fd.for_stmt))
8478 1330 : expand_omp_taskloop_for_inner (region, &fd, inner_stmt);
8479 : else
8480 1330 : expand_omp_taskloop_for_outer (region, &fd, inner_stmt);
8481 : }
8482 24056 : else if (fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
8483 20374 : && !fd.have_ordered)
8484 : {
8485 19922 : if (fd.chunk_size == NULL)
8486 14067 : expand_omp_for_static_nochunk (region, &fd, inner_stmt);
8487 : else
8488 5855 : expand_omp_for_static_chunk (region, &fd, inner_stmt);
8489 : }
8490 : else
8491 : {
8492 4134 : int fn_index, start_ix, next_ix;
8493 4134 : unsigned HOST_WIDE_INT sched = 0;
8494 4134 : tree sched_arg = NULL_TREE;
8495 :
8496 4134 : gcc_assert (gimple_omp_for_kind (fd.for_stmt)
8497 : == GF_OMP_FOR_KIND_FOR && !fd.non_rect);
8498 4134 : if (fd.chunk_size == NULL
8499 1665 : && fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
8500 0 : fd.chunk_size = integer_zero_node;
8501 4134 : switch (fd.sched_kind)
8502 : {
8503 1665 : case OMP_CLAUSE_SCHEDULE_RUNTIME:
8504 1665 : if ((fd.sched_modifiers & OMP_CLAUSE_SCHEDULE_NONMONOTONIC) != 0
8505 28 : && fd.lastprivate_conditional == 0)
8506 : {
8507 28 : gcc_assert (!fd.have_ordered);
8508 : fn_index = 6;
8509 : sched = 4;
8510 : }
8511 1637 : else if ((fd.sched_modifiers & OMP_CLAUSE_SCHEDULE_MONOTONIC) == 0
8512 1605 : && !fd.have_ordered
8513 1561 : && fd.lastprivate_conditional == 0)
8514 : fn_index = 7;
8515 : else
8516 : {
8517 87 : fn_index = 3;
8518 87 : sched = (HOST_WIDE_INT_1U << 31);
8519 : }
8520 : break;
8521 2017 : case OMP_CLAUSE_SCHEDULE_DYNAMIC:
8522 2017 : case OMP_CLAUSE_SCHEDULE_GUIDED:
8523 2017 : if ((fd.sched_modifiers & OMP_CLAUSE_SCHEDULE_MONOTONIC) == 0
8524 1961 : && !fd.have_ordered
8525 1798 : && fd.lastprivate_conditional == 0)
8526 : {
8527 1771 : fn_index = 3 + fd.sched_kind;
8528 1771 : sched = (fd.sched_kind == OMP_CLAUSE_SCHEDULE_GUIDED) + 2;
8529 : break;
8530 : }
8531 246 : fn_index = fd.sched_kind;
8532 246 : sched = (fd.sched_kind == OMP_CLAUSE_SCHEDULE_GUIDED) + 2;
8533 246 : sched += (HOST_WIDE_INT_1U << 31);
8534 246 : break;
8535 452 : case OMP_CLAUSE_SCHEDULE_STATIC:
8536 452 : gcc_assert (fd.have_ordered);
8537 : fn_index = 0;
8538 : sched = (HOST_WIDE_INT_1U << 31) + 1;
8539 : break;
8540 0 : default:
8541 0 : gcc_unreachable ();
8542 : }
8543 4134 : if (!fd.ordered)
8544 3799 : fn_index += fd.have_ordered * 8;
8545 4134 : if (fd.ordered)
8546 335 : start_ix = ((int)BUILT_IN_GOMP_LOOP_DOACROSS_STATIC_START) + fn_index;
8547 : else
8548 3799 : start_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_START) + fn_index;
8549 4134 : next_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT) + fn_index;
8550 4134 : if (fd.have_reductemp || fd.have_pointer_condtemp)
8551 : {
8552 169 : if (fd.ordered)
8553 : start_ix = (int)BUILT_IN_GOMP_LOOP_DOACROSS_START;
8554 133 : else if (fd.have_ordered)
8555 : start_ix = (int)BUILT_IN_GOMP_LOOP_ORDERED_START;
8556 : else
8557 95 : start_ix = (int)BUILT_IN_GOMP_LOOP_START;
8558 169 : sched_arg = build_int_cstu (long_integer_type_node, sched);
8559 169 : if (!fd.chunk_size)
8560 38 : fd.chunk_size = integer_zero_node;
8561 : }
8562 4134 : if (fd.iter_type == long_long_unsigned_type_node)
8563 : {
8564 778 : start_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
8565 : - (int)BUILT_IN_GOMP_LOOP_STATIC_START);
8566 778 : next_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
8567 : - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT);
8568 : }
8569 4134 : expand_omp_for_generic (region, &fd, (enum built_in_function) start_ix,
8570 : (enum built_in_function) next_ix, sched_arg,
8571 : inner_stmt);
8572 : }
8573 47801 : }
8574 :
8575 : /* Expand code for an OpenMP sections directive. In pseudo code, we generate
8576 :
8577 : v = GOMP_sections_start (n);
8578 : L0:
8579 : switch (v)
8580 : {
8581 : case 0:
8582 : goto L2;
8583 : case 1:
8584 : section 1;
8585 : goto L1;
8586 : case 2:
8587 : ...
8588 : case n:
8589 : ...
8590 : default:
8591 : abort ();
8592 : }
8593 : L1:
8594 : v = GOMP_sections_next ();
8595 : goto L0;
8596 : L2:
8597 : reduction;
8598 :
8599 : If this is a combined parallel sections, replace the call to
8600 : GOMP_sections_start with call to GOMP_sections_next. */
8601 :
8602 : static void
8603 386 : expand_omp_sections (struct omp_region *region)
8604 : {
8605 386 : tree t, u, vin = NULL, vmain, vnext, l2;
8606 386 : unsigned len;
8607 386 : basic_block entry_bb, l0_bb, l1_bb, l2_bb, default_bb;
8608 386 : gimple_stmt_iterator si, switch_si;
8609 386 : gomp_sections *sections_stmt;
8610 386 : gimple *stmt;
8611 386 : gomp_continue *cont;
8612 386 : edge_iterator ei;
8613 386 : edge e;
8614 386 : struct omp_region *inner;
8615 386 : unsigned i, casei;
8616 386 : bool exit_reachable = region->cont != NULL;
8617 :
8618 386 : gcc_assert (region->exit != NULL);
8619 386 : entry_bb = region->entry;
8620 386 : l0_bb = single_succ (entry_bb);
8621 386 : l1_bb = region->cont;
8622 386 : l2_bb = region->exit;
8623 735 : if (single_pred_p (l2_bb) && single_pred (l2_bb) == l0_bb)
8624 322 : l2 = gimple_block_label (l2_bb);
8625 : else
8626 : {
8627 : /* This can happen if there are reductions. */
8628 64 : len = EDGE_COUNT (l0_bb->succs);
8629 64 : gcc_assert (len > 0);
8630 64 : e = EDGE_SUCC (l0_bb, len - 1);
8631 64 : si = gsi_last_nondebug_bb (e->dest);
8632 64 : l2 = NULL_TREE;
8633 64 : if (gsi_end_p (si)
8634 64 : || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
8635 64 : l2 = gimple_block_label (e->dest);
8636 : else
8637 0 : FOR_EACH_EDGE (e, ei, l0_bb->succs)
8638 : {
8639 0 : si = gsi_last_nondebug_bb (e->dest);
8640 0 : if (gsi_end_p (si)
8641 0 : || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
8642 : {
8643 0 : l2 = gimple_block_label (e->dest);
8644 0 : break;
8645 : }
8646 : }
8647 : }
8648 386 : if (exit_reachable)
8649 340 : default_bb = create_empty_bb (l1_bb->prev_bb);
8650 : else
8651 46 : default_bb = create_empty_bb (l0_bb);
8652 :
8653 : /* We will build a switch() with enough cases for all the
8654 : GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
8655 : and a default case to abort if something goes wrong. */
8656 386 : len = EDGE_COUNT (l0_bb->succs);
8657 :
8658 : /* Use vec::quick_push on label_vec throughout, since we know the size
8659 : in advance. */
8660 386 : auto_vec<tree> label_vec (len);
8661 :
8662 : /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
8663 : GIMPLE_OMP_SECTIONS statement. */
8664 386 : si = gsi_last_nondebug_bb (entry_bb);
8665 386 : sections_stmt = as_a <gomp_sections *> (gsi_stmt (si));
8666 386 : gcc_assert (gimple_code (sections_stmt) == GIMPLE_OMP_SECTIONS);
8667 386 : vin = gimple_omp_sections_control (sections_stmt);
8668 386 : tree clauses = gimple_omp_sections_clauses (sections_stmt);
8669 386 : tree reductmp = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
8670 386 : tree condtmp = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_);
8671 386 : tree cond_var = NULL_TREE;
8672 386 : if (reductmp || condtmp)
8673 : {
8674 18 : tree reductions = null_pointer_node, mem = null_pointer_node;
8675 18 : tree memv = NULL_TREE, condtemp = NULL_TREE;
8676 18 : gimple_stmt_iterator gsi = gsi_none ();
8677 18 : gimple *g = NULL;
8678 18 : if (reductmp)
8679 : {
8680 8 : reductions = OMP_CLAUSE_DECL (reductmp);
8681 8 : gcc_assert (TREE_CODE (reductions) == SSA_NAME);
8682 8 : g = SSA_NAME_DEF_STMT (reductions);
8683 8 : reductions = gimple_assign_rhs1 (g);
8684 8 : OMP_CLAUSE_DECL (reductmp) = reductions;
8685 8 : gsi = gsi_for_stmt (g);
8686 : }
8687 : else
8688 10 : gsi = si;
8689 18 : if (condtmp)
8690 : {
8691 12 : condtemp = OMP_CLAUSE_DECL (condtmp);
8692 12 : tree c = omp_find_clause (OMP_CLAUSE_CHAIN (condtmp),
8693 : OMP_CLAUSE__CONDTEMP_);
8694 12 : cond_var = OMP_CLAUSE_DECL (c);
8695 12 : tree type = TREE_TYPE (condtemp);
8696 12 : memv = create_tmp_var (type);
8697 12 : TREE_ADDRESSABLE (memv) = 1;
8698 12 : unsigned cnt = 0;
8699 74 : for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
8700 62 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
8701 62 : && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
8702 24 : ++cnt;
8703 12 : unsigned HOST_WIDE_INT sz
8704 12 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (type))) * cnt;
8705 12 : expand_omp_build_assign (&gsi, memv, build_int_cst (type, sz),
8706 : false);
8707 12 : mem = build_fold_addr_expr (memv);
8708 : }
8709 18 : t = build_int_cst (unsigned_type_node, len - 1);
8710 18 : u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS2_START);
8711 18 : stmt = gimple_build_call (u, 3, t, reductions, mem);
8712 18 : gimple_call_set_lhs (stmt, vin);
8713 18 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
8714 18 : if (condtmp)
8715 : {
8716 12 : expand_omp_build_assign (&gsi, condtemp, memv, false);
8717 12 : tree t = build2 (PLUS_EXPR, TREE_TYPE (cond_var),
8718 12 : vin, build_one_cst (TREE_TYPE (cond_var)));
8719 12 : expand_omp_build_assign (&gsi, cond_var, t, false);
8720 : }
8721 18 : if (reductmp)
8722 : {
8723 8 : gsi_remove (&gsi, true);
8724 8 : release_ssa_name (gimple_assign_lhs (g));
8725 : }
8726 : }
8727 368 : else if (!is_combined_parallel (region))
8728 : {
8729 : /* If we are not inside a combined parallel+sections region,
8730 : call GOMP_sections_start. */
8731 252 : t = build_int_cst (unsigned_type_node, len - 1);
8732 252 : u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START);
8733 252 : stmt = gimple_build_call (u, 1, t);
8734 : }
8735 : else
8736 : {
8737 : /* Otherwise, call GOMP_sections_next. */
8738 116 : u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
8739 116 : stmt = gimple_build_call (u, 0);
8740 : }
8741 386 : if (!reductmp && !condtmp)
8742 : {
8743 368 : gimple_call_set_lhs (stmt, vin);
8744 368 : gsi_insert_after (&si, stmt, GSI_SAME_STMT);
8745 : }
8746 386 : gsi_remove (&si, true);
8747 :
8748 : /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
8749 : L0_BB. */
8750 386 : switch_si = gsi_last_nondebug_bb (l0_bb);
8751 386 : gcc_assert (gimple_code (gsi_stmt (switch_si)) == GIMPLE_OMP_SECTIONS_SWITCH);
8752 386 : if (exit_reachable)
8753 : {
8754 340 : cont = as_a <gomp_continue *> (last_nondebug_stmt (l1_bb));
8755 340 : gcc_assert (gimple_code (cont) == GIMPLE_OMP_CONTINUE);
8756 340 : vmain = gimple_omp_continue_control_use (cont);
8757 340 : vnext = gimple_omp_continue_control_def (cont);
8758 : }
8759 : else
8760 : {
8761 : vmain = vin;
8762 : vnext = NULL_TREE;
8763 : }
8764 :
8765 386 : t = build_case_label (build_int_cst (unsigned_type_node, 0), NULL, l2);
8766 386 : label_vec.quick_push (t);
8767 386 : i = 1;
8768 :
8769 : /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
8770 386 : for (inner = region->inner, casei = 1;
8771 1276 : inner;
8772 890 : inner = inner->next, i++, casei++)
8773 : {
8774 890 : basic_block s_entry_bb, s_exit_bb;
8775 :
8776 : /* Skip optional reduction region. */
8777 890 : if (inner->type == GIMPLE_OMP_ATOMIC_LOAD)
8778 : {
8779 27 : --i;
8780 27 : --casei;
8781 27 : continue;
8782 : }
8783 :
8784 863 : s_entry_bb = inner->entry;
8785 863 : s_exit_bb = inner->exit;
8786 :
8787 863 : t = gimple_block_label (s_entry_bb);
8788 863 : u = build_int_cst (unsigned_type_node, casei);
8789 863 : u = build_case_label (u, NULL, t);
8790 863 : label_vec.quick_push (u);
8791 :
8792 863 : si = gsi_last_nondebug_bb (s_entry_bb);
8793 863 : gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SECTION);
8794 863 : gcc_assert (i < len || gimple_omp_section_last_p (gsi_stmt (si)));
8795 863 : gsi_remove (&si, true);
8796 863 : single_succ_edge (s_entry_bb)->flags = EDGE_FALLTHRU;
8797 :
8798 863 : if (s_exit_bb == NULL)
8799 122 : continue;
8800 :
8801 741 : si = gsi_last_nondebug_bb (s_exit_bb);
8802 741 : gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
8803 741 : gsi_remove (&si, true);
8804 :
8805 741 : single_succ_edge (s_exit_bb)->flags = EDGE_FALLTHRU;
8806 : }
8807 :
8808 : /* Error handling code goes in DEFAULT_BB. */
8809 386 : t = gimple_block_label (default_bb);
8810 386 : u = build_case_label (NULL, NULL, t);
8811 386 : make_edge (l0_bb, default_bb, 0);
8812 386 : add_bb_to_loop (default_bb, current_loops->tree_root);
8813 :
8814 386 : stmt = gimple_build_switch (vmain, u, label_vec);
8815 386 : gsi_insert_after (&switch_si, stmt, GSI_SAME_STMT);
8816 386 : gsi_remove (&switch_si, true);
8817 :
8818 386 : si = gsi_start_bb (default_bb);
8819 386 : stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP), 0);
8820 386 : gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING);
8821 :
8822 386 : if (exit_reachable)
8823 : {
8824 340 : tree bfn_decl;
8825 :
8826 : /* Code to get the next section goes in L1_BB. */
8827 340 : si = gsi_last_nondebug_bb (l1_bb);
8828 340 : gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CONTINUE);
8829 :
8830 340 : bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
8831 340 : stmt = gimple_build_call (bfn_decl, 0);
8832 340 : gimple_call_set_lhs (stmt, vnext);
8833 340 : gsi_insert_before (&si, stmt, GSI_SAME_STMT);
8834 340 : if (cond_var)
8835 : {
8836 12 : tree t = build2 (PLUS_EXPR, TREE_TYPE (cond_var),
8837 12 : vnext, build_one_cst (TREE_TYPE (cond_var)));
8838 12 : expand_omp_build_assign (&si, cond_var, t, false);
8839 : }
8840 340 : gsi_remove (&si, true);
8841 :
8842 340 : single_succ_edge (l1_bb)->flags = EDGE_FALLTHRU;
8843 : }
8844 :
8845 : /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
8846 386 : si = gsi_last_nondebug_bb (l2_bb);
8847 386 : if (gimple_omp_return_nowait_p (gsi_stmt (si)))
8848 255 : t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT);
8849 131 : else if (gimple_omp_return_lhs (gsi_stmt (si)))
8850 0 : t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL);
8851 : else
8852 131 : t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END);
8853 386 : stmt = gimple_build_call (t, 0);
8854 386 : if (gimple_omp_return_lhs (gsi_stmt (si)))
8855 0 : gimple_call_set_lhs (stmt, gimple_omp_return_lhs (gsi_stmt (si)));
8856 386 : gsi_insert_after (&si, stmt, GSI_SAME_STMT);
8857 386 : gsi_remove (&si, true);
8858 :
8859 386 : set_immediate_dominator (CDI_DOMINATORS, default_bb, l0_bb);
8860 386 : }
8861 :
8862 : /* Expand code for an OpenMP single or scope directive. We've already expanded
8863 : much of the code, here we simply place the GOMP_barrier call. */
8864 :
8865 : static void
8866 1288 : expand_omp_single (struct omp_region *region)
8867 : {
8868 1288 : basic_block entry_bb, exit_bb;
8869 1288 : gimple_stmt_iterator si;
8870 :
8871 1288 : entry_bb = region->entry;
8872 1288 : exit_bb = region->exit;
8873 :
8874 1288 : si = gsi_last_nondebug_bb (entry_bb);
8875 1288 : enum gimple_code code = gimple_code (gsi_stmt (si));
8876 1288 : gcc_assert (code == GIMPLE_OMP_SINGLE || code == GIMPLE_OMP_SCOPE);
8877 1288 : gsi_remove (&si, true);
8878 1288 : single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
8879 :
8880 1288 : if (exit_bb == NULL)
8881 : {
8882 8 : gcc_assert (code == GIMPLE_OMP_SCOPE);
8883 8 : return;
8884 : }
8885 :
8886 1280 : si = gsi_last_nondebug_bb (exit_bb);
8887 1280 : if (!gimple_omp_return_nowait_p (gsi_stmt (si)))
8888 : {
8889 1001 : tree t = gimple_omp_return_lhs (gsi_stmt (si));
8890 1001 : gsi_insert_after (&si,
8891 : omp_build_barrier (t, GOMP_BARRIER_IMPLICIT_WORKSHARE),
8892 : GSI_SAME_STMT);
8893 : }
8894 1280 : gsi_remove (&si, true);
8895 1280 : single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
8896 : }
8897 :
8898 : /* Generic expansion for OpenMP synchronization directives: master,
8899 : ordered and critical. All we need to do here is remove the entry
8900 : and exit markers for REGION. */
8901 :
8902 : static void
8903 10872 : expand_omp_synch (struct omp_region *region)
8904 : {
8905 10872 : basic_block entry_bb, exit_bb;
8906 10872 : gimple_stmt_iterator si;
8907 :
8908 10872 : entry_bb = region->entry;
8909 10872 : exit_bb = region->exit;
8910 :
8911 10872 : si = gsi_last_nondebug_bb (entry_bb);
8912 10872 : gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE
8913 : || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_MASTER
8914 : || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_MASKED
8915 : || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TASKGROUP
8916 : || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ORDERED
8917 : || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CRITICAL
8918 : || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TEAMS);
8919 10872 : if (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TEAMS
8920 10872 : && gimple_omp_teams_host (as_a <gomp_teams *> (gsi_stmt (si))))
8921 : {
8922 2572 : expand_omp_taskreg (region);
8923 2572 : return;
8924 : }
8925 8300 : gsi_remove (&si, true);
8926 8300 : single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
8927 :
8928 8300 : if (exit_bb)
8929 : {
8930 7742 : si = gsi_last_nondebug_bb (exit_bb);
8931 7742 : gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
8932 7742 : gsi_remove (&si, true);
8933 7742 : single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
8934 : }
8935 : }
8936 :
8937 : /* Translate enum omp_memory_order to enum memmodel for the embedded
8938 : fail clause in there. */
8939 :
8940 : static enum memmodel
8941 2936 : omp_memory_order_to_fail_memmodel (enum omp_memory_order mo)
8942 : {
8943 2936 : switch (mo & OMP_FAIL_MEMORY_ORDER_MASK)
8944 : {
8945 2665 : case OMP_FAIL_MEMORY_ORDER_UNSPECIFIED:
8946 2665 : switch (mo & OMP_MEMORY_ORDER_MASK)
8947 : {
8948 : case OMP_MEMORY_ORDER_RELAXED: return MEMMODEL_RELAXED;
8949 : case OMP_MEMORY_ORDER_ACQUIRE: return MEMMODEL_ACQUIRE;
8950 : case OMP_MEMORY_ORDER_RELEASE: return MEMMODEL_RELAXED;
8951 : case OMP_MEMORY_ORDER_ACQ_REL: return MEMMODEL_ACQUIRE;
8952 : case OMP_MEMORY_ORDER_SEQ_CST: return MEMMODEL_SEQ_CST;
8953 0 : default: break;
8954 : }
8955 0 : gcc_unreachable ();
8956 : case OMP_FAIL_MEMORY_ORDER_RELAXED: return MEMMODEL_RELAXED;
8957 : case OMP_FAIL_MEMORY_ORDER_ACQUIRE: return MEMMODEL_ACQUIRE;
8958 : case OMP_FAIL_MEMORY_ORDER_SEQ_CST: return MEMMODEL_SEQ_CST;
8959 0 : default: gcc_unreachable ();
8960 : }
8961 : }
8962 :
8963 : /* Translate enum omp_memory_order to enum memmodel. The two enums
8964 : are using different numbers so that OMP_MEMORY_ORDER_UNSPECIFIED
8965 : is 0 and omp_memory_order has the fail mode encoded in it too. */
8966 :
8967 : static enum memmodel
8968 10060 : omp_memory_order_to_memmodel (enum omp_memory_order mo)
8969 : {
8970 10060 : enum memmodel ret, fail_ret;
8971 10060 : switch (mo & OMP_MEMORY_ORDER_MASK)
8972 : {
8973 : case OMP_MEMORY_ORDER_RELAXED: ret = MEMMODEL_RELAXED; break;
8974 : case OMP_MEMORY_ORDER_ACQUIRE: ret = MEMMODEL_ACQUIRE; break;
8975 : case OMP_MEMORY_ORDER_RELEASE: ret = MEMMODEL_RELEASE; break;
8976 : case OMP_MEMORY_ORDER_ACQ_REL: ret = MEMMODEL_ACQ_REL; break;
8977 : case OMP_MEMORY_ORDER_SEQ_CST: ret = MEMMODEL_SEQ_CST; break;
8978 0 : default: gcc_unreachable ();
8979 : }
8980 : /* If we drop the -Winvalid-memory-model warning for C++17 P0418R2,
8981 : we can just return ret here unconditionally. Otherwise, work around
8982 : it here and make sure fail memmodel is not stronger. */
8983 10060 : if ((mo & OMP_FAIL_MEMORY_ORDER_MASK) == OMP_FAIL_MEMORY_ORDER_UNSPECIFIED)
8984 : return ret;
8985 142 : fail_ret = omp_memory_order_to_fail_memmodel (mo);
8986 142 : if (fail_ret > ret)
8987 13 : return fail_ret;
8988 : return ret;
8989 : }
8990 :
8991 : /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8992 : operation as a normal volatile load. */
8993 :
8994 : static bool
8995 1183 : expand_omp_atomic_load (basic_block load_bb, tree addr,
8996 : tree loaded_val, int index)
8997 : {
8998 1183 : enum built_in_function tmpbase;
8999 1183 : gimple_stmt_iterator gsi;
9000 1183 : basic_block store_bb;
9001 1183 : location_t loc;
9002 1183 : gimple *stmt;
9003 1183 : tree decl, type, itype;
9004 :
9005 1183 : gsi = gsi_last_nondebug_bb (load_bb);
9006 1183 : stmt = gsi_stmt (gsi);
9007 1183 : gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
9008 1183 : loc = gimple_location (stmt);
9009 :
9010 : /* ??? If the target does not implement atomic_load_optab[mode], and mode
9011 : is smaller than word size, then expand_atomic_load assumes that the load
9012 : is atomic. We could avoid the builtin entirely in this case. */
9013 :
9014 1183 : tmpbase = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1);
9015 1183 : decl = builtin_decl_explicit (tmpbase);
9016 1183 : if (decl == NULL_TREE)
9017 : return false;
9018 :
9019 1183 : type = TREE_TYPE (loaded_val);
9020 1183 : itype = TREE_TYPE (TREE_TYPE (decl));
9021 :
9022 1183 : enum omp_memory_order omo = gimple_omp_atomic_memory_order (stmt);
9023 1183 : tree mo = build_int_cst (integer_type_node,
9024 1183 : omp_memory_order_to_memmodel (omo));
9025 1183 : gcall *call = gimple_build_call (decl, 2, addr, mo);
9026 1183 : gimple_set_location (call, loc);
9027 2366 : gimple_set_vuse (call, gimple_vuse (stmt));
9028 1183 : gimple *repl;
9029 1183 : if (!useless_type_conversion_p (type, itype))
9030 : {
9031 1181 : tree lhs = make_ssa_name (itype);
9032 1181 : gimple_call_set_lhs (call, lhs);
9033 1181 : gsi_insert_before (&gsi, call, GSI_SAME_STMT);
9034 1181 : repl = gimple_build_assign (loaded_val,
9035 : build1 (VIEW_CONVERT_EXPR, type, lhs));
9036 1181 : gimple_set_location (repl, loc);
9037 : }
9038 : else
9039 : {
9040 2 : gimple_call_set_lhs (call, loaded_val);
9041 : repl = call;
9042 : }
9043 1183 : gsi_replace (&gsi, repl, true);
9044 :
9045 1183 : store_bb = single_succ (load_bb);
9046 1183 : gsi = gsi_last_nondebug_bb (store_bb);
9047 1183 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
9048 1183 : gsi_remove (&gsi, true);
9049 :
9050 1183 : return true;
9051 : }
9052 :
9053 : /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
9054 : operation as a normal volatile store. */
9055 :
9056 : static bool
9057 856 : expand_omp_atomic_store (basic_block load_bb, tree addr,
9058 : tree loaded_val, tree stored_val, int index)
9059 : {
9060 856 : enum built_in_function tmpbase;
9061 856 : gimple_stmt_iterator gsi;
9062 856 : basic_block store_bb = single_succ (load_bb);
9063 856 : location_t loc;
9064 856 : gimple *stmt;
9065 856 : tree decl, type, itype;
9066 856 : machine_mode imode;
9067 856 : bool exchange;
9068 :
9069 856 : gsi = gsi_last_nondebug_bb (load_bb);
9070 856 : stmt = gsi_stmt (gsi);
9071 856 : gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
9072 :
9073 : /* If the load value is needed, then this isn't a store but an exchange. */
9074 856 : exchange = gimple_omp_atomic_need_value_p (stmt);
9075 :
9076 856 : gsi = gsi_last_nondebug_bb (store_bb);
9077 856 : stmt = gsi_stmt (gsi);
9078 856 : gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE);
9079 856 : loc = gimple_location (stmt);
9080 :
9081 : /* ??? If the target does not implement atomic_store_optab[mode], and mode
9082 : is smaller than word size, then expand_atomic_store assumes that the store
9083 : is atomic. We could avoid the builtin entirely in this case. */
9084 :
9085 856 : tmpbase = (exchange ? BUILT_IN_ATOMIC_EXCHANGE_N : BUILT_IN_ATOMIC_STORE_N);
9086 856 : tmpbase = (enum built_in_function) ((int) tmpbase + index + 1);
9087 856 : decl = builtin_decl_explicit (tmpbase);
9088 856 : if (decl == NULL_TREE)
9089 : return false;
9090 :
9091 856 : type = TREE_TYPE (stored_val);
9092 :
9093 : /* Dig out the type of the function's second argument. */
9094 856 : itype = TREE_TYPE (decl);
9095 856 : itype = TYPE_ARG_TYPES (itype);
9096 856 : itype = TREE_CHAIN (itype);
9097 856 : itype = TREE_VALUE (itype);
9098 856 : imode = TYPE_MODE (itype);
9099 :
9100 856 : if (exchange && !can_atomic_exchange_p (imode, true))
9101 : return false;
9102 :
9103 856 : if (!useless_type_conversion_p (itype, type))
9104 856 : stored_val = fold_build1_loc (loc, VIEW_CONVERT_EXPR, itype, stored_val);
9105 856 : enum omp_memory_order omo = gimple_omp_atomic_memory_order (stmt);
9106 856 : tree mo = build_int_cst (integer_type_node,
9107 856 : omp_memory_order_to_memmodel (omo));
9108 856 : stored_val = force_gimple_operand_gsi (&gsi, stored_val, true, NULL_TREE,
9109 : true, GSI_SAME_STMT);
9110 856 : gcall *call = gimple_build_call (decl, 3, addr, stored_val, mo);
9111 856 : gimple_set_location (call, loc);
9112 1712 : gimple_set_vuse (call, gimple_vuse (stmt));
9113 1712 : gimple_set_vdef (call, gimple_vdef (stmt));
9114 :
9115 856 : gimple *repl = call;
9116 856 : if (exchange)
9117 : {
9118 88 : if (!useless_type_conversion_p (type, itype))
9119 : {
9120 88 : tree lhs = make_ssa_name (itype);
9121 88 : gimple_call_set_lhs (call, lhs);
9122 88 : gsi_insert_before (&gsi, call, GSI_SAME_STMT);
9123 88 : repl = gimple_build_assign (loaded_val,
9124 : build1 (VIEW_CONVERT_EXPR, type, lhs));
9125 88 : gimple_set_location (repl, loc);
9126 : }
9127 : else
9128 0 : gimple_call_set_lhs (call, loaded_val);
9129 : }
9130 856 : gsi_replace (&gsi, repl, true);
9131 :
9132 : /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
9133 856 : gsi = gsi_last_nondebug_bb (load_bb);
9134 856 : gsi_remove (&gsi, true);
9135 :
9136 856 : return true;
9137 : }
9138 :
9139 : /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
9140 : operation as a __atomic_fetch_op builtin. INDEX is log2 of the
9141 : size of the data type, and thus usable to find the index of the builtin
9142 : decl. Returns false if the expression is not of the proper form. */
9143 :
9144 : static bool
9145 5892 : expand_omp_atomic_fetch_op (basic_block load_bb,
9146 : tree addr, tree loaded_val,
9147 : tree stored_val, int index)
9148 : {
9149 5892 : enum built_in_function oldbase, newbase, tmpbase;
9150 5892 : tree decl, itype, call;
9151 5892 : tree lhs, rhs;
9152 5892 : basic_block store_bb = single_succ (load_bb);
9153 5892 : gimple_stmt_iterator gsi;
9154 5892 : gimple *stmt;
9155 5892 : location_t loc;
9156 5892 : enum tree_code code;
9157 5892 : bool need_old, need_new;
9158 5892 : machine_mode imode;
9159 :
9160 : /* We expect to find the following sequences:
9161 :
9162 : load_bb:
9163 : GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
9164 :
9165 : store_bb:
9166 : val = tmp OP something; (or: something OP tmp)
9167 : GIMPLE_OMP_STORE (val)
9168 :
9169 : ???FIXME: Allow a more flexible sequence.
9170 : Perhaps use data flow to pick the statements.
9171 :
9172 : */
9173 :
9174 5892 : gsi = gsi_after_labels (store_bb);
9175 5892 : stmt = gsi_stmt (gsi);
9176 5892 : if (is_gimple_debug (stmt))
9177 : {
9178 0 : gsi_next_nondebug (&gsi);
9179 0 : if (gsi_end_p (gsi))
9180 : return false;
9181 5892 : stmt = gsi_stmt (gsi);
9182 : }
9183 5892 : loc = gimple_location (stmt);
9184 5892 : if (!is_gimple_assign (stmt))
9185 : return false;
9186 5892 : gsi_next_nondebug (&gsi);
9187 5892 : if (gimple_code (gsi_stmt (gsi)) != GIMPLE_OMP_ATOMIC_STORE)
9188 : return false;
9189 5227 : need_new = gimple_omp_atomic_need_value_p (gsi_stmt (gsi));
9190 5227 : need_old = gimple_omp_atomic_need_value_p (last_nondebug_stmt (load_bb));
9191 5227 : enum omp_memory_order omo
9192 5227 : = gimple_omp_atomic_memory_order (last_nondebug_stmt (load_bb));
9193 5227 : enum memmodel mo = omp_memory_order_to_memmodel (omo);
9194 5227 : gcc_checking_assert (!need_old || !need_new);
9195 :
9196 5227 : if (!operand_equal_p (gimple_assign_lhs (stmt), stored_val, 0))
9197 : return false;
9198 :
9199 : /* Check for one of the supported fetch-op operations. */
9200 5227 : code = gimple_assign_rhs_code (stmt);
9201 5227 : switch (code)
9202 : {
9203 : case PLUS_EXPR:
9204 : case POINTER_PLUS_EXPR:
9205 : oldbase = BUILT_IN_ATOMIC_FETCH_ADD_N;
9206 : newbase = BUILT_IN_ATOMIC_ADD_FETCH_N;
9207 : break;
9208 79 : case MINUS_EXPR:
9209 79 : oldbase = BUILT_IN_ATOMIC_FETCH_SUB_N;
9210 79 : newbase = BUILT_IN_ATOMIC_SUB_FETCH_N;
9211 79 : break;
9212 112 : case BIT_AND_EXPR:
9213 112 : oldbase = BUILT_IN_ATOMIC_FETCH_AND_N;
9214 112 : newbase = BUILT_IN_ATOMIC_AND_FETCH_N;
9215 112 : break;
9216 227 : case BIT_IOR_EXPR:
9217 227 : oldbase = BUILT_IN_ATOMIC_FETCH_OR_N;
9218 227 : newbase = BUILT_IN_ATOMIC_OR_FETCH_N;
9219 227 : break;
9220 111 : case BIT_XOR_EXPR:
9221 111 : oldbase = BUILT_IN_ATOMIC_FETCH_XOR_N;
9222 111 : newbase = BUILT_IN_ATOMIC_XOR_FETCH_N;
9223 111 : break;
9224 : default:
9225 : return false;
9226 : }
9227 :
9228 : /* Make sure the expression is of the proper form. */
9229 4588 : if (operand_equal_p (gimple_assign_rhs1 (stmt), loaded_val, 0))
9230 4406 : rhs = gimple_assign_rhs2 (stmt);
9231 182 : else if (commutative_tree_code (gimple_assign_rhs_code (stmt))
9232 182 : && operand_equal_p (gimple_assign_rhs2 (stmt), loaded_val, 0))
9233 139 : rhs = gimple_assign_rhs1 (stmt);
9234 : else
9235 : return false;
9236 :
9237 9090 : tmpbase = ((enum built_in_function)
9238 4545 : ((need_new ? newbase : oldbase) + index + 1));
9239 4545 : decl = builtin_decl_explicit (tmpbase);
9240 4545 : if (decl == NULL_TREE)
9241 : return false;
9242 4545 : itype = TREE_TYPE (TREE_TYPE (decl));
9243 4545 : imode = TYPE_MODE (itype);
9244 :
9245 : /* We could test all of the various optabs involved, but the fact of the
9246 : matter is that (with the exception of i486 vs i586 and xadd) all targets
9247 : that support any atomic operation optab also implements compare-and-swap.
9248 : Let optabs.cc take care of expanding any compare-and-swap loop. */
9249 4545 : if (!can_compare_and_swap_p (imode, true) || !can_atomic_load_p (imode))
9250 : return false;
9251 :
9252 4541 : gsi = gsi_last_nondebug_bb (load_bb);
9253 4541 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_LOAD);
9254 :
9255 : /* OpenMP does not imply any barrier-like semantics on its atomic ops.
9256 : It only requires that the operation happen atomically. Thus we can
9257 : use the RELAXED memory model. */
9258 4541 : call = build_call_expr_loc (loc, decl, 3, addr,
9259 : fold_convert_loc (loc, itype, rhs),
9260 4541 : build_int_cst (NULL, mo));
9261 :
9262 4541 : if (need_old || need_new)
9263 : {
9264 552 : lhs = need_old ? loaded_val : stored_val;
9265 552 : call = fold_convert_loc (loc, TREE_TYPE (lhs), call);
9266 552 : call = build2_loc (loc, MODIFY_EXPR, void_type_node, lhs, call);
9267 : }
9268 : else
9269 3989 : call = fold_convert_loc (loc, void_type_node, call);
9270 4541 : force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
9271 4541 : gsi_remove (&gsi, true);
9272 :
9273 4541 : gsi = gsi_last_nondebug_bb (store_bb);
9274 4541 : gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
9275 4541 : gsi_remove (&gsi, true);
9276 4541 : gsi = gsi_last_nondebug_bb (store_bb);
9277 4541 : stmt = gsi_stmt (gsi);
9278 4541 : gsi_remove (&gsi, true);
9279 :
9280 4541 : if (gimple_in_ssa_p (cfun))
9281 29 : release_defs (stmt);
9282 :
9283 : return true;
9284 : }
9285 :
9286 : /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
9287 : compare and exchange as an ATOMIC_COMPARE_EXCHANGE internal function.
9288 : Returns false if the expression is not of the proper form. */
9289 :
9290 : static bool
9291 2217 : expand_omp_atomic_cas (basic_block load_bb, tree addr,
9292 : tree loaded_val, tree stored_val, int index)
9293 : {
9294 : /* We expect to find the following sequences:
9295 :
9296 : load_bb:
9297 : GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
9298 :
9299 : store_bb:
9300 : val = tmp == e ? d : tmp;
9301 : GIMPLE_OMP_ATOMIC_STORE (val)
9302 :
9303 : or in store_bb instead:
9304 : tmp2 = tmp == e;
9305 : val = tmp2 ? d : tmp;
9306 : GIMPLE_OMP_ATOMIC_STORE (val)
9307 :
9308 : or:
9309 : tmp3 = VIEW_CONVERT_EXPR<integral_type>(tmp);
9310 : val = e == tmp3 ? d : tmp;
9311 : GIMPLE_OMP_ATOMIC_STORE (val)
9312 :
9313 : etc. */
9314 :
9315 :
9316 2217 : basic_block store_bb = single_succ (load_bb);
9317 2217 : gimple_stmt_iterator gsi = gsi_last_nondebug_bb (store_bb);
9318 2217 : gimple *store_stmt = gsi_stmt (gsi);
9319 2217 : if (!store_stmt || gimple_code (store_stmt) != GIMPLE_OMP_ATOMIC_STORE)
9320 : return false;
9321 2217 : gsi_prev_nondebug (&gsi);
9322 2217 : if (gsi_end_p (gsi))
9323 : return false;
9324 2211 : gimple *condexpr_stmt = gsi_stmt (gsi);
9325 2211 : if (!is_gimple_assign (condexpr_stmt)
9326 2211 : || gimple_assign_rhs_code (condexpr_stmt) != COND_EXPR)
9327 : return false;
9328 408 : if (!operand_equal_p (gimple_assign_lhs (condexpr_stmt), stored_val, 0))
9329 : return false;
9330 408 : gimple *cond_stmt = NULL;
9331 408 : gimple *vce_stmt = NULL;
9332 408 : gsi_prev_nondebug (&gsi);
9333 408 : if (!gsi_end_p (gsi))
9334 : {
9335 408 : cond_stmt = gsi_stmt (gsi);
9336 408 : if (!is_gimple_assign (cond_stmt))
9337 : return false;
9338 408 : if (gimple_assign_rhs_code (cond_stmt) == EQ_EXPR)
9339 : {
9340 323 : gsi_prev_nondebug (&gsi);
9341 323 : if (!gsi_end_p (gsi))
9342 : {
9343 89 : vce_stmt = gsi_stmt (gsi);
9344 89 : if (!is_gimple_assign (vce_stmt)
9345 89 : || gimple_assign_rhs_code (vce_stmt) != VIEW_CONVERT_EXPR)
9346 : return false;
9347 : }
9348 : }
9349 85 : else if (gimple_assign_rhs_code (cond_stmt) == VIEW_CONVERT_EXPR)
9350 : std::swap (vce_stmt, cond_stmt);
9351 : else
9352 : return false;
9353 69 : if (vce_stmt)
9354 : {
9355 69 : tree vce_rhs = gimple_assign_rhs1 (vce_stmt);
9356 69 : if (TREE_CODE (vce_rhs) != VIEW_CONVERT_EXPR
9357 69 : || !operand_equal_p (TREE_OPERAND (vce_rhs, 0), loaded_val))
9358 : return false;
9359 114 : if (!INTEGRAL_TYPE_P (TREE_TYPE (vce_rhs))
9360 57 : || !SCALAR_FLOAT_TYPE_P (TREE_TYPE (loaded_val))
9361 114 : || !tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vce_rhs)),
9362 57 : TYPE_SIZE (TREE_TYPE (loaded_val))))
9363 : return false;
9364 57 : gsi_prev_nondebug (&gsi);
9365 57 : if (!gsi_end_p (gsi))
9366 : return false;
9367 : }
9368 : }
9369 291 : tree cond = gimple_assign_rhs1 (condexpr_stmt);
9370 291 : tree cond_op1, cond_op2;
9371 291 : if (cond_stmt)
9372 : {
9373 : /* We should now always get a separate cond_stmt. */
9374 291 : if (!operand_equal_p (cond, gimple_assign_lhs (cond_stmt)))
9375 : return false;
9376 291 : cond_op1 = gimple_assign_rhs1 (cond_stmt);
9377 291 : cond_op2 = gimple_assign_rhs2 (cond_stmt);
9378 : }
9379 0 : else if (TREE_CODE (cond) != EQ_EXPR && TREE_CODE (cond) != NE_EXPR)
9380 : return false;
9381 : else
9382 : {
9383 0 : cond_op1 = TREE_OPERAND (cond, 0);
9384 0 : cond_op2 = TREE_OPERAND (cond, 1);
9385 : }
9386 291 : tree d;
9387 291 : if (TREE_CODE (cond) == NE_EXPR)
9388 : {
9389 0 : if (!operand_equal_p (gimple_assign_rhs2 (condexpr_stmt), loaded_val))
9390 : return false;
9391 0 : d = gimple_assign_rhs3 (condexpr_stmt);
9392 : }
9393 291 : else if (!operand_equal_p (gimple_assign_rhs3 (condexpr_stmt), loaded_val))
9394 : return false;
9395 : else
9396 291 : d = gimple_assign_rhs2 (condexpr_stmt);
9397 291 : tree e = vce_stmt ? gimple_assign_lhs (vce_stmt) : loaded_val;
9398 291 : if (operand_equal_p (e, cond_op1))
9399 : e = cond_op2;
9400 21 : else if (operand_equal_p (e, cond_op2))
9401 : e = cond_op1;
9402 : else
9403 : return false;
9404 :
9405 291 : location_t loc = gimple_location (store_stmt);
9406 291 : gimple *load_stmt = last_nondebug_stmt (load_bb);
9407 291 : bool need_new = gimple_omp_atomic_need_value_p (store_stmt);
9408 291 : bool need_old = gimple_omp_atomic_need_value_p (load_stmt);
9409 291 : bool weak = gimple_omp_atomic_weak_p (load_stmt);
9410 291 : enum omp_memory_order omo = gimple_omp_atomic_memory_order (load_stmt);
9411 291 : tree mo = build_int_cst (NULL, omp_memory_order_to_memmodel (omo));
9412 291 : tree fmo = build_int_cst (NULL, omp_memory_order_to_fail_memmodel (omo));
9413 291 : gcc_checking_assert (!need_old || !need_new);
9414 :
9415 291 : enum built_in_function fncode
9416 : = (enum built_in_function) ((int) BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
9417 291 : + index + 1);
9418 291 : tree cmpxchg = builtin_decl_explicit (fncode);
9419 291 : if (cmpxchg == NULL_TREE)
9420 : return false;
9421 291 : tree itype = TREE_TYPE (TREE_TYPE (cmpxchg));
9422 :
9423 291 : if (!can_compare_and_swap_p (TYPE_MODE (itype), true)
9424 291 : || !can_atomic_load_p (TYPE_MODE (itype)))
9425 : return false;
9426 :
9427 290 : tree type = TYPE_MAIN_VARIANT (TREE_TYPE (loaded_val));
9428 290 : if (SCALAR_FLOAT_TYPE_P (type) && !vce_stmt)
9429 : return false;
9430 :
9431 289 : gsi = gsi_for_stmt (store_stmt);
9432 289 : if (!useless_type_conversion_p (itype, TREE_TYPE (e)))
9433 : {
9434 232 : tree ne = create_tmp_reg (itype);
9435 232 : gimple *g = gimple_build_assign (ne, NOP_EXPR, e);
9436 232 : gimple_set_location (g, loc);
9437 232 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9438 232 : e = ne;
9439 : }
9440 289 : if (!useless_type_conversion_p (itype, TREE_TYPE (d)))
9441 : {
9442 289 : tree nd = create_tmp_reg (itype);
9443 289 : enum tree_code code;
9444 289 : if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (d)))
9445 : {
9446 57 : code = VIEW_CONVERT_EXPR;
9447 57 : d = build1 (VIEW_CONVERT_EXPR, itype, d);
9448 : }
9449 : else
9450 : code = NOP_EXPR;
9451 289 : gimple *g = gimple_build_assign (nd, code, d);
9452 289 : gimple_set_location (g, loc);
9453 289 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9454 289 : d = nd;
9455 : }
9456 :
9457 289 : tree ctype = build_complex_type (itype);
9458 289 : int flag = int_size_in_bytes (itype) + (weak ? 256 : 0);
9459 289 : gimple *g
9460 289 : = gimple_build_call_internal (IFN_ATOMIC_COMPARE_EXCHANGE, 6, addr, e, d,
9461 289 : build_int_cst (integer_type_node, flag),
9462 : mo, fmo);
9463 289 : tree cres = create_tmp_reg (ctype);
9464 289 : gimple_call_set_lhs (g, cres);
9465 289 : gimple_set_location (g, loc);
9466 289 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9467 :
9468 289 : if (cond_stmt || need_old || need_new)
9469 : {
9470 289 : tree im = create_tmp_reg (itype);
9471 289 : g = gimple_build_assign (im, IMAGPART_EXPR,
9472 : build1 (IMAGPART_EXPR, itype, cres));
9473 289 : gimple_set_location (g, loc);
9474 289 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9475 :
9476 289 : tree re = NULL_TREE;
9477 289 : if (need_old || need_new)
9478 : {
9479 199 : re = create_tmp_reg (itype);
9480 199 : g = gimple_build_assign (re, REALPART_EXPR,
9481 : build1 (REALPART_EXPR, itype, cres));
9482 199 : gimple_set_location (g, loc);
9483 199 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9484 : }
9485 :
9486 289 : if (cond_stmt)
9487 : {
9488 289 : g = gimple_build_assign (cond, NOP_EXPR, im);
9489 289 : gimple_set_location (g, loc);
9490 289 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9491 : }
9492 :
9493 289 : if (need_new)
9494 : {
9495 44 : g = gimple_build_assign (create_tmp_reg (itype), COND_EXPR,
9496 : cond_stmt
9497 0 : ? cond : build2 (NE_EXPR, boolean_type_node,
9498 : im, build_zero_cst (itype)),
9499 : d, re);
9500 44 : gimple_set_location (g, loc);
9501 44 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9502 44 : re = gimple_assign_lhs (g);
9503 : }
9504 :
9505 289 : if (need_old || need_new)
9506 : {
9507 199 : tree v = need_old ? loaded_val : stored_val;
9508 199 : enum tree_code code;
9509 199 : if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (v)))
9510 : {
9511 38 : code = VIEW_CONVERT_EXPR;
9512 38 : re = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (v), re);
9513 : }
9514 161 : else if (!useless_type_conversion_p (TREE_TYPE (v), itype))
9515 : code = NOP_EXPR;
9516 : else
9517 0 : code = TREE_CODE (re);
9518 199 : g = gimple_build_assign (v, code, re);
9519 199 : gimple_set_location (g, loc);
9520 199 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9521 : }
9522 : }
9523 :
9524 289 : gsi_remove (&gsi, true);
9525 289 : gsi = gsi_for_stmt (load_stmt);
9526 289 : gsi_remove (&gsi, true);
9527 289 : gsi = gsi_for_stmt (condexpr_stmt);
9528 289 : gsi_remove (&gsi, true);
9529 289 : if (cond_stmt)
9530 : {
9531 289 : gsi = gsi_for_stmt (cond_stmt);
9532 289 : gsi_remove (&gsi, true);
9533 : }
9534 289 : if (vce_stmt)
9535 : {
9536 57 : gsi = gsi_for_stmt (vce_stmt);
9537 57 : gsi_remove (&gsi, true);
9538 : }
9539 :
9540 : return true;
9541 : }
9542 :
9543 : /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
9544 :
9545 : oldval = *addr;
9546 : repeat:
9547 : newval = rhs; // with oldval replacing *addr in rhs
9548 : oldval = __sync_val_compare_and_swap (addr, oldval, newval);
9549 : if (oldval != newval)
9550 : goto repeat;
9551 :
9552 : INDEX is log2 of the size of the data type, and thus usable to find the
9553 : index of the builtin decl. */
9554 :
9555 : static bool
9556 2586 : expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb,
9557 : tree addr, tree loaded_val, tree stored_val,
9558 : int index)
9559 : {
9560 2586 : tree loadedi, storedi, initial, new_storedi, old_vali;
9561 2586 : tree type, itype, cmpxchg, iaddr, atype;
9562 2586 : gimple_stmt_iterator si;
9563 2586 : basic_block loop_header = single_succ (load_bb);
9564 2586 : gimple *phi, *stmt;
9565 2586 : edge e;
9566 2586 : enum built_in_function fncode;
9567 :
9568 2586 : fncode = (enum built_in_function)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
9569 2586 : + index + 1);
9570 2586 : cmpxchg = builtin_decl_explicit (fncode);
9571 2586 : if (cmpxchg == NULL_TREE)
9572 : return false;
9573 2586 : type = TYPE_MAIN_VARIANT (TREE_TYPE (loaded_val));
9574 2586 : atype = type;
9575 2586 : itype = TREE_TYPE (TREE_TYPE (cmpxchg));
9576 :
9577 2586 : if (!can_compare_and_swap_p (TYPE_MODE (itype), true)
9578 2586 : || !can_atomic_load_p (TYPE_MODE (itype)))
9579 : return false;
9580 :
9581 : /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
9582 2503 : si = gsi_last_nondebug_bb (load_bb);
9583 2503 : gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
9584 2503 : location_t loc = gimple_location (gsi_stmt (si));
9585 2503 : enum omp_memory_order omo = gimple_omp_atomic_memory_order (gsi_stmt (si));
9586 2503 : tree mo = build_int_cst (NULL, omp_memory_order_to_memmodel (omo));
9587 2503 : tree fmo = build_int_cst (NULL, omp_memory_order_to_fail_memmodel (omo));
9588 :
9589 : /* For floating-point values, we'll need to view-convert them to integers
9590 : so that we can perform the atomic compare and swap. Simplify the
9591 : following code by always setting up the "i"ntegral variables. */
9592 2503 : if (!INTEGRAL_TYPE_P (type) && !POINTER_TYPE_P (type))
9593 : {
9594 845 : tree iaddr_val;
9595 :
9596 845 : iaddr = create_tmp_reg (build_pointer_type_for_mode (itype, ptr_mode,
9597 : true));
9598 845 : atype = itype;
9599 845 : iaddr_val
9600 845 : = force_gimple_operand_gsi (&si,
9601 845 : fold_convert (TREE_TYPE (iaddr), addr),
9602 : false, NULL_TREE, true, GSI_SAME_STMT);
9603 845 : stmt = gimple_build_assign (iaddr, iaddr_val);
9604 845 : gsi_insert_before (&si, stmt, GSI_SAME_STMT);
9605 845 : loadedi = create_tmp_var (itype);
9606 845 : if (gimple_in_ssa_p (cfun))
9607 15 : loadedi = make_ssa_name (loadedi);
9608 : }
9609 : else
9610 : {
9611 : iaddr = addr;
9612 : loadedi = loaded_val;
9613 : }
9614 :
9615 2503 : fncode = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1);
9616 2503 : tree loaddecl = builtin_decl_explicit (fncode);
9617 2503 : if (loaddecl)
9618 2503 : initial
9619 2503 : = fold_convert (atype,
9620 : build_call_expr (loaddecl, 2, iaddr,
9621 : build_int_cst (NULL_TREE,
9622 : MEMMODEL_RELAXED)));
9623 : else
9624 : {
9625 0 : tree off
9626 0 : = build_int_cst (build_pointer_type_for_mode (atype, ptr_mode,
9627 : true), 0);
9628 0 : initial = build2 (MEM_REF, atype, iaddr, off);
9629 : }
9630 :
9631 2503 : initial
9632 2503 : = force_gimple_operand_gsi (&si, initial, true, NULL_TREE, true,
9633 : GSI_SAME_STMT);
9634 :
9635 : /* Move the value to the LOADEDI temporary. */
9636 2503 : if (gimple_in_ssa_p (cfun))
9637 : {
9638 39 : gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header)));
9639 39 : phi = create_phi_node (loadedi, loop_header);
9640 39 : SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, single_succ_edge (load_bb)),
9641 : initial);
9642 : }
9643 : else
9644 2464 : gsi_insert_before (&si,
9645 2464 : gimple_build_assign (loadedi, initial),
9646 : GSI_SAME_STMT);
9647 2503 : if (loadedi != loaded_val)
9648 : {
9649 845 : gimple_stmt_iterator gsi2;
9650 845 : tree x;
9651 :
9652 845 : x = build1 (VIEW_CONVERT_EXPR, type, loadedi);
9653 845 : gsi2 = gsi_start_bb (loop_header);
9654 845 : if (gimple_in_ssa_p (cfun))
9655 : {
9656 15 : gassign *stmt;
9657 15 : x = force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
9658 : true, GSI_SAME_STMT);
9659 15 : stmt = gimple_build_assign (loaded_val, x);
9660 15 : gsi_insert_before (&gsi2, stmt, GSI_SAME_STMT);
9661 : }
9662 : else
9663 : {
9664 830 : x = build2 (MODIFY_EXPR, TREE_TYPE (loaded_val), loaded_val, x);
9665 830 : force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
9666 : true, GSI_SAME_STMT);
9667 : }
9668 : }
9669 2503 : gsi_remove (&si, true);
9670 :
9671 2503 : si = gsi_last_nondebug_bb (store_bb);
9672 2503 : gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
9673 :
9674 2503 : if (iaddr == addr)
9675 : storedi = stored_val;
9676 : else
9677 845 : storedi
9678 845 : = force_gimple_operand_gsi (&si,
9679 : build1 (VIEW_CONVERT_EXPR, itype,
9680 : stored_val), true, NULL_TREE, true,
9681 : GSI_SAME_STMT);
9682 :
9683 : /* Build the compare&swap statement. */
9684 2503 : tree ctype = build_complex_type (itype);
9685 2503 : int flag = int_size_in_bytes (itype);
9686 2503 : new_storedi = build_call_expr_internal_loc (loc, IFN_ATOMIC_COMPARE_EXCHANGE,
9687 : ctype, 6, iaddr, loadedi,
9688 : storedi,
9689 : build_int_cst (integer_type_node,
9690 2503 : flag),
9691 : mo, fmo);
9692 2503 : new_storedi = build1 (REALPART_EXPR, itype, new_storedi);
9693 2503 : new_storedi = force_gimple_operand_gsi (&si,
9694 2503 : fold_convert (TREE_TYPE (loadedi),
9695 : new_storedi),
9696 : true, NULL_TREE,
9697 : true, GSI_SAME_STMT);
9698 :
9699 2503 : if (gimple_in_ssa_p (cfun))
9700 : old_vali = loadedi;
9701 : else
9702 : {
9703 2464 : old_vali = create_tmp_var (TREE_TYPE (loadedi));
9704 2464 : stmt = gimple_build_assign (old_vali, loadedi);
9705 2464 : gsi_insert_before (&si, stmt, GSI_SAME_STMT);
9706 :
9707 2464 : stmt = gimple_build_assign (loadedi, new_storedi);
9708 2464 : gsi_insert_before (&si, stmt, GSI_SAME_STMT);
9709 : }
9710 :
9711 : /* Note that we always perform the comparison as an integer, even for
9712 : floating point. This allows the atomic operation to properly
9713 : succeed even with NaNs and -0.0. */
9714 2503 : tree ne = build2 (NE_EXPR, boolean_type_node, new_storedi, old_vali);
9715 2503 : stmt = gimple_build_cond_empty (ne);
9716 2503 : gsi_insert_before (&si, stmt, GSI_SAME_STMT);
9717 :
9718 : /* Update cfg. */
9719 2503 : e = single_succ_edge (store_bb);
9720 2503 : e->flags &= ~EDGE_FALLTHRU;
9721 2503 : e->flags |= EDGE_FALSE_VALUE;
9722 : /* Expect no looping. */
9723 2503 : e->probability = profile_probability::guessed_always ();
9724 :
9725 2503 : e = make_edge (store_bb, loop_header, EDGE_TRUE_VALUE);
9726 2503 : e->probability = profile_probability::guessed_never ();
9727 :
9728 : /* Copy the new value to loadedi (we already did that before the condition
9729 : if we are not in SSA). */
9730 2503 : if (gimple_in_ssa_p (cfun))
9731 : {
9732 39 : phi = gimple_seq_first_stmt (phi_nodes (loop_header));
9733 39 : SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e), new_storedi);
9734 : }
9735 :
9736 : /* Remove GIMPLE_OMP_ATOMIC_STORE. */
9737 2503 : stmt = gsi_stmt (si);
9738 2503 : gsi_remove (&si, true);
9739 2503 : if (gimple_in_ssa_p (cfun))
9740 39 : release_defs (stmt);
9741 :
9742 2503 : class loop *loop = alloc_loop ();
9743 2503 : loop->header = loop_header;
9744 2503 : loop->latch = store_bb;
9745 2503 : add_loop (loop, loop_header->loop_father);
9746 :
9747 2503 : return true;
9748 : }
9749 :
9750 : /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
9751 :
9752 : GOMP_atomic_start ();
9753 : *addr = rhs;
9754 : GOMP_atomic_end ();
9755 :
9756 : The result is not globally atomic, but works so long as all parallel
9757 : references are within #pragma omp atomic directives. According to
9758 : responses received from omp@openmp.org, appears to be within spec.
9759 : Which makes sense, since that's how several other compilers handle
9760 : this situation as well.
9761 : LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
9762 : expanding. STORED_VAL is the operand of the matching
9763 : GIMPLE_OMP_ATOMIC_STORE.
9764 :
9765 : We replace
9766 : GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
9767 : loaded_val = *addr;
9768 :
9769 : and replace
9770 : GIMPLE_OMP_ATOMIC_STORE (stored_val) with
9771 : *addr = stored_val;
9772 : */
9773 :
9774 : static bool
9775 293 : expand_omp_atomic_mutex (basic_block load_bb, basic_block store_bb,
9776 : tree addr, tree loaded_val, tree stored_val)
9777 : {
9778 293 : gimple_stmt_iterator si;
9779 293 : gassign *stmt;
9780 293 : tree t;
9781 :
9782 293 : si = gsi_last_nondebug_bb (load_bb);
9783 293 : gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
9784 :
9785 293 : t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START);
9786 293 : t = build_call_expr (t, 0);
9787 293 : force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
9788 :
9789 293 : tree mem = build_simple_mem_ref (addr);
9790 293 : TREE_TYPE (mem) = TREE_TYPE (loaded_val);
9791 293 : TREE_OPERAND (mem, 1)
9792 293 : = fold_convert (build_pointer_type_for_mode (TREE_TYPE (mem), ptr_mode,
9793 : true),
9794 : TREE_OPERAND (mem, 1));
9795 293 : stmt = gimple_build_assign (loaded_val, mem);
9796 293 : gsi_insert_before (&si, stmt, GSI_SAME_STMT);
9797 293 : gsi_remove (&si, true);
9798 :
9799 293 : si = gsi_last_nondebug_bb (store_bb);
9800 293 : gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
9801 :
9802 293 : stmt = gimple_build_assign (unshare_expr (mem), stored_val);
9803 586 : gimple_set_vuse (stmt, gimple_vuse (gsi_stmt (si)));
9804 586 : gimple_set_vdef (stmt, gimple_vdef (gsi_stmt (si)));
9805 293 : gsi_insert_before (&si, stmt, GSI_SAME_STMT);
9806 :
9807 293 : t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END);
9808 293 : t = build_call_expr (t, 0);
9809 293 : force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
9810 293 : gsi_remove (&si, true);
9811 293 : return true;
9812 : }
9813 :
9814 : /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
9815 : using expand_omp_atomic_fetch_op. If it failed, we try to
9816 : call expand_omp_atomic_pipeline, and if it fails too, the
9817 : ultimate fallback is wrapping the operation in a mutex
9818 : (expand_omp_atomic_mutex). REGION is the atomic region built
9819 : by build_omp_regions_1(). */
9820 :
9821 : static void
9822 9665 : expand_omp_atomic (struct omp_region *region)
9823 : {
9824 9665 : basic_block load_bb = region->entry, store_bb = region->exit;
9825 9665 : gomp_atomic_load *load
9826 9665 : = as_a <gomp_atomic_load *> (last_nondebug_stmt (load_bb));
9827 9665 : gomp_atomic_store *store
9828 9665 : = as_a <gomp_atomic_store *> (last_nondebug_stmt (store_bb));
9829 9665 : tree loaded_val = gimple_omp_atomic_load_lhs (load);
9830 9665 : tree addr = gimple_omp_atomic_load_rhs (load);
9831 9665 : tree stored_val = gimple_omp_atomic_store_val (store);
9832 9665 : tree type = TYPE_MAIN_VARIANT (TREE_TYPE (loaded_val));
9833 9665 : HOST_WIDE_INT index;
9834 :
9835 : /* Make sure the type is one of the supported sizes. */
9836 9665 : index = tree_to_uhwi (TYPE_SIZE_UNIT (type));
9837 9665 : index = exact_log2 (index);
9838 9631 : if (index >= 0 && index <= 4)
9839 : {
9840 9631 : unsigned int align = TYPE_ALIGN_UNIT (type);
9841 :
9842 : /* __sync builtins require strict data alignment. */
9843 19262 : if (exact_log2 (align) >= index)
9844 : {
9845 : /* Atomic load. */
9846 9455 : scalar_mode smode;
9847 9455 : if (loaded_val == stored_val
9848 1187 : && (is_int_mode (TYPE_MODE (type), &smode)
9849 1337 : || is_float_mode (TYPE_MODE (type), &smode))
9850 1187 : && GET_MODE_BITSIZE (smode) <= BITS_PER_WORD
9851 10638 : && expand_omp_atomic_load (load_bb, addr, loaded_val, index))
9852 9665 : return;
9853 :
9854 : /* Atomic store. */
9855 8272 : if ((is_int_mode (TYPE_MODE (type), &smode)
9856 9317 : || is_float_mode (TYPE_MODE (type), &smode))
9857 8272 : && GET_MODE_BITSIZE (smode) <= BITS_PER_WORD
9858 8189 : && store_bb == single_succ (load_bb)
9859 7574 : && first_stmt (store_bb) == store
9860 856 : && expand_omp_atomic_store (load_bb, addr, loaded_val,
9861 : stored_val, index))
9862 : return;
9863 :
9864 : /* When possible, use specialized atomic update functions. */
9865 985 : if ((INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type))
9866 6439 : && store_bb == single_succ (load_bb)
9867 13308 : && expand_omp_atomic_fetch_op (load_bb, addr,
9868 : loaded_val, stored_val, index))
9869 : return;
9870 :
9871 : /* When possible, use ATOMIC_COMPARE_EXCHANGE ifn without a loop. */
9872 2875 : if (store_bb == single_succ (load_bb)
9873 2256 : && !gimple_in_ssa_p (cfun)
9874 5092 : && expand_omp_atomic_cas (load_bb, addr, loaded_val, stored_val,
9875 : index))
9876 : return;
9877 :
9878 : /* If we don't have specialized __sync builtins, try and implement
9879 : as a compare and swap loop. */
9880 2586 : if (expand_omp_atomic_pipeline (load_bb, store_bb, addr,
9881 : loaded_val, stored_val, index))
9882 : return;
9883 : }
9884 : }
9885 :
9886 : /* The ultimate fallback is wrapping the operation in a mutex. */
9887 293 : expand_omp_atomic_mutex (load_bb, store_bb, addr, loaded_val, stored_val);
9888 : }
9889 :
9890 : /* Mark the loops inside the kernels region starting at REGION_ENTRY and ending
9891 : at REGION_EXIT. */
9892 :
9893 : static void
9894 1651 : mark_loops_in_oacc_kernels_region (basic_block region_entry,
9895 : basic_block region_exit)
9896 : {
9897 1651 : class loop *outer = region_entry->loop_father;
9898 1651 : gcc_assert (region_exit == NULL || outer == region_exit->loop_father);
9899 :
9900 : /* Don't parallelize the kernels region if it contains more than one outer
9901 : loop. */
9902 1651 : unsigned int nr_outer_loops = 0;
9903 1651 : class loop *single_outer = NULL;
9904 33062 : for (class loop *loop = outer->inner; loop != NULL; loop = loop->next)
9905 : {
9906 31411 : gcc_assert (loop_outer (loop) == outer);
9907 :
9908 31411 : if (!dominated_by_p (CDI_DOMINATORS, loop->header, region_entry))
9909 23379 : continue;
9910 :
9911 14588 : if (region_exit != NULL
9912 8032 : && dominated_by_p (CDI_DOMINATORS, loop->header, region_exit))
9913 6556 : continue;
9914 :
9915 1476 : nr_outer_loops++;
9916 1476 : single_outer = loop;
9917 : }
9918 1651 : if (nr_outer_loops != 1)
9919 : return;
9920 :
9921 1336 : for (class loop *loop = single_outer->inner;
9922 2293 : loop != NULL;
9923 957 : loop = loop->inner)
9924 1045 : if (loop->next)
9925 : return;
9926 :
9927 : /* Mark the loops in the region. */
9928 3269 : for (class loop *loop = single_outer; loop != NULL; loop = loop->inner)
9929 2021 : loop->in_oacc_kernels_region = true;
9930 : }
9931 :
9932 : /* Build target argument identifier from the DEVICE identifier, value
9933 : identifier ID and whether the element also has a SUBSEQUENT_PARAM. */
9934 :
9935 : static tree
9936 23580 : get_target_argument_identifier_1 (int device, bool subseqent_param, int id)
9937 : {
9938 23580 : tree t = build_int_cst (integer_type_node, device);
9939 23580 : if (subseqent_param)
9940 672 : t = fold_build2 (BIT_IOR_EXPR, integer_type_node, t,
9941 : build_int_cst (integer_type_node,
9942 : GOMP_TARGET_ARG_SUBSEQUENT_PARAM));
9943 23580 : t = fold_build2 (BIT_IOR_EXPR, integer_type_node, t,
9944 : build_int_cst (integer_type_node, id));
9945 23580 : return t;
9946 : }
9947 :
9948 : /* Like above but return it in type that can be directly stored as an element
9949 : of the argument array. */
9950 :
9951 : static tree
9952 672 : get_target_argument_identifier (int device, bool subseqent_param, int id)
9953 : {
9954 672 : tree t = get_target_argument_identifier_1 (device, subseqent_param, id);
9955 672 : return fold_convert (ptr_type_node, t);
9956 : }
9957 :
9958 : /* Return a target argument consisting of DEVICE identifier, value identifier
9959 : ID, and the actual VALUE. */
9960 :
9961 : static tree
9962 22908 : get_target_argument_value (gimple_stmt_iterator *gsi, int device, int id,
9963 : tree value)
9964 : {
9965 22908 : tree t = fold_build2 (LSHIFT_EXPR, integer_type_node,
9966 : fold_convert (integer_type_node, value),
9967 : build_int_cst (unsigned_type_node,
9968 : GOMP_TARGET_ARG_VALUE_SHIFT));
9969 22908 : t = fold_build2 (BIT_IOR_EXPR, integer_type_node, t,
9970 : get_target_argument_identifier_1 (device, false, id));
9971 22908 : t = fold_convert (ptr_type_node, t);
9972 22908 : return force_gimple_operand_gsi (gsi, t, true, NULL, true, GSI_SAME_STMT);
9973 : }
9974 :
9975 : /* If VALUE is an integer constant greater than -2^15 and smaller than 2^15,
9976 : push one argument to ARGS with both the DEVICE, ID and VALUE embedded in it,
9977 : otherwise push an identifier (with DEVICE and ID) and the VALUE in two
9978 : arguments. */
9979 :
9980 : static void
9981 23580 : push_target_argument_according_to_value (gimple_stmt_iterator *gsi, int device,
9982 : int id, tree value, vec <tree> *args)
9983 : {
9984 23580 : if (tree_fits_shwi_p (value)
9985 22908 : && tree_to_shwi (value) > -(1 << 15)
9986 22908 : && tree_to_shwi (value) < (1 << 15))
9987 22908 : args->quick_push (get_target_argument_value (gsi, device, id, value));
9988 : else
9989 : {
9990 672 : args->quick_push (get_target_argument_identifier (device, true, id));
9991 672 : value = fold_convert (ptr_type_node, value);
9992 672 : value = force_gimple_operand_gsi (gsi, value, true, NULL, true,
9993 : GSI_SAME_STMT);
9994 672 : args->quick_push (value);
9995 : }
9996 23580 : }
9997 :
9998 : /* Create an array of arguments that is then passed to GOMP_target. */
9999 :
10000 : static tree
10001 11790 : get_target_arguments (gimple_stmt_iterator *gsi, gomp_target *tgt_stmt)
10002 : {
10003 11790 : auto_vec <tree, 6> args;
10004 11790 : tree clauses = gimple_omp_target_clauses (tgt_stmt);
10005 11790 : tree num_teams = omp_find_clause (clauses, OMP_CLAUSE_NUM_TEAMS);
10006 11790 : tree thread_limit = omp_find_clause (clauses, OMP_CLAUSE_THREAD_LIMIT);
10007 11790 : tree message = omp_find_clause (clauses, OMP_CLAUSE_MESSAGE);
10008 11790 : if (message)
10009 : /* FIXME: error termination checking also requires to check
10010 : OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR not only UPPER, i.e. it
10011 : needs to be passed to the runtime! Note that both target and
10012 : a nested teams can have a 'message' clause - and both take a
10013 : thread_limit clause. */
10014 1 : sorry_at (OMP_CLAUSE_LOCATION (message), "%<message%> clause");
10015 11790 : if (num_teams)
10016 : {
10017 11790 : tree expr = OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (num_teams);
10018 : /* FIXME: Update omp-low.cc's lower_omp_teams when implementing. */
10019 11790 : if (TREE_CODE (expr) == TREE_LIST
10020 0 : && OMP_CLAUSE_NUM_TEAMS_DIMS (num_teams)
10021 11790 : && TREE_CHAIN (expr))
10022 : {
10023 0 : sorry_at (OMP_CLAUSE_LOCATION (num_teams),
10024 : "%<num_teams%> clause with %<dims%> modifier");
10025 0 : num_teams = TREE_VALUE (expr);
10026 : }
10027 11790 : else if (TREE_CODE (expr) == TREE_LIST)
10028 0 : num_teams = (TREE_CHAIN (expr)
10029 0 : ? TREE_VALUE (TREE_CHAIN (expr)) : TREE_VALUE (expr));
10030 : else
10031 : num_teams = expr;
10032 : }
10033 : else
10034 0 : num_teams = integer_minus_one_node;
10035 11790 : push_target_argument_according_to_value (gsi, GOMP_TARGET_ARG_DEVICE_ALL,
10036 : GOMP_TARGET_ARG_NUM_TEAMS,
10037 : num_teams, &args);
10038 11790 : if (thread_limit)
10039 : {
10040 11790 : tree expr = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit);
10041 11790 : if (TREE_CODE (expr) == TREE_LIST)
10042 : {
10043 43 : if (OMP_CLAUSE_THREAD_LIMIT_DIMS (thread_limit)
10044 43 : && (TREE_CHAIN (expr)
10045 1 : || OMP_CLAUSE_THREAD_LIMIT_STRICT (thread_limit)))
10046 1 : sorry_at (OMP_CLAUSE_LOCATION (thread_limit),
10047 : "%<thread_limit%> clause with %<dims%> modifier");
10048 42 : else if (OMP_CLAUSE_THREAD_LIMIT_STRICT (thread_limit))
10049 0 : sorry_at (OMP_CLAUSE_LOCATION (thread_limit),
10050 : "%<thread_limit%> clause with %<strict%> modifier");
10051 43 : expr = TREE_VALUE (expr);
10052 : }
10053 11747 : else if (OMP_CLAUSE_THREAD_LIMIT_STRICT (thread_limit))
10054 0 : sorry_at (OMP_CLAUSE_LOCATION (thread_limit),
10055 : "%<thread_limit%> clause with %<strict%> modifier");
10056 : thread_limit = expr;
10057 : }
10058 : else
10059 0 : thread_limit = integer_minus_one_node;
10060 11790 : push_target_argument_according_to_value (gsi, GOMP_TARGET_ARG_DEVICE_ALL,
10061 : GOMP_TARGET_ARG_THREAD_LIMIT,
10062 : thread_limit, &args);
10063 :
10064 : /* Produce more, perhaps device specific, arguments here. */
10065 :
10066 23580 : tree argarray = create_tmp_var (build_array_type_nelts (ptr_type_node,
10067 11790 : args.length () + 1),
10068 : ".omp_target_args");
10069 36042 : for (unsigned i = 0; i < args.length (); i++)
10070 : {
10071 24252 : tree ref = build4 (ARRAY_REF, ptr_type_node, argarray,
10072 24252 : build_int_cst (integer_type_node, i),
10073 : NULL_TREE, NULL_TREE);
10074 24252 : gsi_insert_before (gsi, gimple_build_assign (ref, args[i]),
10075 : GSI_SAME_STMT);
10076 : }
10077 23580 : tree ref = build4 (ARRAY_REF, ptr_type_node, argarray,
10078 11790 : build_int_cst (integer_type_node, args.length ()),
10079 : NULL_TREE, NULL_TREE);
10080 11790 : gsi_insert_before (gsi, gimple_build_assign (ref, null_pointer_node),
10081 : GSI_SAME_STMT);
10082 11790 : TREE_ADDRESSABLE (argarray) = 1;
10083 11790 : return build_fold_addr_expr (argarray);
10084 11790 : }
10085 :
10086 : /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
10087 :
10088 : static void
10089 37153 : expand_omp_target (struct omp_region *region)
10090 : {
10091 37153 : basic_block entry_bb, exit_bb, new_bb;
10092 37153 : struct function *child_cfun;
10093 37153 : tree child_fn, child_fn2, block, t, c;
10094 37153 : gimple_stmt_iterator gsi;
10095 37153 : gomp_target *entry_stmt;
10096 37153 : gimple *stmt;
10097 37153 : edge e;
10098 37153 : bool offloaded;
10099 37153 : int target_kind;
10100 :
10101 37153 : entry_stmt = as_a <gomp_target *> (last_nondebug_stmt (region->entry));
10102 37153 : target_kind = gimple_omp_target_kind (entry_stmt);
10103 37153 : new_bb = region->entry;
10104 :
10105 37153 : offloaded = is_gimple_omp_offloaded (entry_stmt);
10106 37153 : switch (target_kind)
10107 : {
10108 37153 : case GF_OMP_TARGET_KIND_REGION:
10109 37153 : case GF_OMP_TARGET_KIND_UPDATE:
10110 37153 : case GF_OMP_TARGET_KIND_ENTER_DATA:
10111 37153 : case GF_OMP_TARGET_KIND_EXIT_DATA:
10112 37153 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
10113 37153 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
10114 37153 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
10115 37153 : case GF_OMP_TARGET_KIND_OACC_UPDATE:
10116 37153 : case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
10117 37153 : case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
10118 37153 : case GF_OMP_TARGET_KIND_OACC_DECLARE:
10119 37153 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
10120 37153 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
10121 37153 : case GF_OMP_TARGET_KIND_DATA:
10122 37153 : case GF_OMP_TARGET_KIND_OACC_DATA:
10123 37153 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
10124 37153 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
10125 37153 : break;
10126 0 : default:
10127 0 : gcc_unreachable ();
10128 : }
10129 :
10130 37153 : tree clauses = gimple_omp_target_clauses (entry_stmt);
10131 :
10132 37153 : bool is_ancestor = false;
10133 37153 : bool is_host_only = false;
10134 37153 : child_fn = child_fn2 = NULL_TREE;
10135 37153 : child_cfun = NULL;
10136 37153 : if (offloaded)
10137 : {
10138 21185 : c = omp_find_clause (clauses, OMP_CLAUSE_DEVICE);
10139 21185 : if (ENABLE_OFFLOADING && c)
10140 : is_ancestor = OMP_CLAUSE_DEVICE_ANCESTOR (c);
10141 21185 : c = omp_find_clause (clauses, OMP_CLAUSE_DEVICE_TYPE);
10142 21185 : if (c && OMP_CLAUSE_DEVICE_TYPE_KIND (c) == OMP_CLAUSE_DEVICE_TYPE_HOST)
10143 : is_host_only = true;
10144 21185 : child_fn = gimple_omp_target_child_fn (entry_stmt);
10145 21185 : child_cfun = DECL_STRUCT_FUNCTION (child_fn);
10146 : }
10147 :
10148 : /* Supported by expand_omp_taskreg, but not here. */
10149 21185 : if (child_cfun != NULL)
10150 21185 : gcc_checking_assert (!child_cfun->cfg);
10151 37153 : gcc_checking_assert (!gimple_in_ssa_p (cfun));
10152 :
10153 37153 : entry_bb = region->entry;
10154 37153 : exit_bb = region->exit;
10155 :
10156 37153 : if (target_kind == GF_OMP_TARGET_KIND_OACC_KERNELS)
10157 1651 : mark_loops_in_oacc_kernels_region (region->entry, region->exit);
10158 :
10159 : /* Going on, all OpenACC compute constructs are mapped to
10160 : 'BUILT_IN_GOACC_PARALLEL', and get their compute regions outlined.
10161 : To distinguish between them, we attach attributes. */
10162 37153 : switch (target_kind)
10163 : {
10164 6825 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
10165 6825 : DECL_ATTRIBUTES (child_fn)
10166 6825 : = tree_cons (get_identifier ("oacc parallel"),
10167 6825 : NULL_TREE, DECL_ATTRIBUTES (child_fn));
10168 6825 : break;
10169 1651 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
10170 1651 : DECL_ATTRIBUTES (child_fn)
10171 1651 : = tree_cons (get_identifier ("oacc kernels"),
10172 1651 : NULL_TREE, DECL_ATTRIBUTES (child_fn));
10173 1651 : break;
10174 756 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
10175 756 : DECL_ATTRIBUTES (child_fn)
10176 756 : = tree_cons (get_identifier ("oacc serial"),
10177 756 : NULL_TREE, DECL_ATTRIBUTES (child_fn));
10178 756 : break;
10179 54 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
10180 54 : DECL_ATTRIBUTES (child_fn)
10181 54 : = tree_cons (get_identifier ("oacc parallel_kernels_parallelized"),
10182 54 : NULL_TREE, DECL_ATTRIBUTES (child_fn));
10183 54 : break;
10184 109 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
10185 109 : DECL_ATTRIBUTES (child_fn)
10186 109 : = tree_cons (get_identifier ("oacc parallel_kernels_gang_single"),
10187 109 : NULL_TREE, DECL_ATTRIBUTES (child_fn));
10188 109 : break;
10189 27758 : default:
10190 : /* Make sure we don't miss any. */
10191 27758 : gcc_checking_assert (!(is_gimple_omp_oacc (entry_stmt)
10192 : && is_gimple_omp_offloaded (entry_stmt)));
10193 : break;
10194 : }
10195 :
10196 37153 : if (offloaded)
10197 : {
10198 21185 : unsigned srcidx, dstidx, num;
10199 :
10200 : /* If the offloading region needs data sent from the parent
10201 : function, then the very first statement (except possible
10202 : tree profile counter updates) of the offloading body
10203 : is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
10204 : &.OMP_DATA_O is passed as an argument to the child function,
10205 : we need to replace it with the argument as seen by the child
10206 : function.
10207 :
10208 : In most cases, this will end up being the identity assignment
10209 : .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
10210 : a function call that has been inlined, the original PARM_DECL
10211 : .OMP_DATA_I may have been converted into a different local
10212 : variable. In which case, we need to keep the assignment. */
10213 21185 : tree data_arg = gimple_omp_target_data_arg (entry_stmt);
10214 21185 : if (data_arg)
10215 : {
10216 16531 : basic_block entry_succ_bb = single_succ (entry_bb);
10217 16531 : gimple_stmt_iterator gsi;
10218 16531 : tree arg;
10219 16531 : gimple *tgtcopy_stmt = NULL;
10220 16531 : tree sender = TREE_VEC_ELT (data_arg, 0);
10221 :
10222 33062 : for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
10223 : {
10224 16531 : gcc_assert (!gsi_end_p (gsi));
10225 16531 : stmt = gsi_stmt (gsi);
10226 16531 : if (gimple_code (stmt) != GIMPLE_ASSIGN)
10227 0 : continue;
10228 :
10229 16531 : if (gimple_num_ops (stmt) == 2)
10230 : {
10231 16531 : tree arg = gimple_assign_rhs1 (stmt);
10232 :
10233 : /* We're ignoring the subcode because we're
10234 : effectively doing a STRIP_NOPS. */
10235 :
10236 16531 : if ((TREE_CODE (arg) == ADDR_EXPR
10237 16424 : && TREE_OPERAND (arg, 0) == sender)
10238 16531 : || arg == sender)
10239 : {
10240 16531 : tgtcopy_stmt = stmt;
10241 16531 : break;
10242 : }
10243 : }
10244 0 : }
10245 :
10246 16531 : gcc_assert (tgtcopy_stmt != NULL);
10247 16531 : arg = DECL_ARGUMENTS (child_fn);
10248 :
10249 16531 : gcc_assert (gimple_assign_lhs (tgtcopy_stmt) == arg);
10250 16531 : gsi_remove (&gsi, true);
10251 : }
10252 :
10253 : /* Declare local variables needed in CHILD_CFUN. */
10254 21185 : block = DECL_INITIAL (child_fn);
10255 21185 : BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
10256 : /* The gimplifier could record temporaries in the offloading block
10257 : rather than in containing function's local_decls chain,
10258 : which would mean cgraph missed finalizing them. Do it now. */
10259 158408 : for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
10260 137223 : if (VAR_P (t) && TREE_STATIC (t) && !DECL_EXTERNAL (t))
10261 0 : varpool_node::finalize_decl (t);
10262 21185 : DECL_SAVED_TREE (child_fn) = NULL;
10263 : /* We'll create a CFG for child_fn, so no gimple body is needed. */
10264 21185 : gimple_set_body (child_fn, NULL);
10265 21185 : TREE_USED (block) = 1;
10266 :
10267 : /* Reset DECL_CONTEXT on function arguments. */
10268 42370 : for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
10269 21185 : DECL_CONTEXT (t) = child_fn;
10270 :
10271 : /* Split ENTRY_BB at GIMPLE_*,
10272 : so that it can be moved to the child function. */
10273 21185 : gsi = gsi_last_nondebug_bb (entry_bb);
10274 21185 : stmt = gsi_stmt (gsi);
10275 21185 : gcc_assert (stmt
10276 : && gimple_code (stmt) == gimple_code (entry_stmt));
10277 21185 : e = split_block (entry_bb, stmt);
10278 21185 : gsi_remove (&gsi, true);
10279 21185 : entry_bb = e->dest;
10280 21185 : single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
10281 :
10282 : /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
10283 21185 : if (exit_bb)
10284 : {
10285 21109 : gsi = gsi_last_nondebug_bb (exit_bb);
10286 21109 : gcc_assert (!gsi_end_p (gsi)
10287 : && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
10288 21109 : stmt = gimple_build_return (NULL);
10289 21109 : gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
10290 21109 : gsi_remove (&gsi, true);
10291 : }
10292 :
10293 : /* Move the offloading region into CHILD_CFUN. */
10294 :
10295 21185 : block = gimple_block (entry_stmt);
10296 :
10297 21185 : new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
10298 21185 : if (exit_bb)
10299 21109 : single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
10300 : /* When the OMP expansion process cannot guarantee an up-to-date
10301 : loop tree arrange for the child function to fixup loops. */
10302 21185 : if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
10303 21185 : child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
10304 :
10305 : /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
10306 21185 : num = vec_safe_length (child_cfun->local_decls);
10307 650156 : for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
10308 : {
10309 628971 : t = (*child_cfun->local_decls)[srcidx];
10310 628971 : if (DECL_CONTEXT (t) == cfun->decl)
10311 137223 : continue;
10312 491748 : if (srcidx != dstidx)
10313 476774 : (*child_cfun->local_decls)[dstidx] = t;
10314 491748 : dstidx++;
10315 : }
10316 21185 : if (dstidx != num)
10317 17835 : vec_safe_truncate (child_cfun->local_decls, dstidx);
10318 :
10319 : /* Inform the callgraph about the new function. */
10320 21185 : child_cfun->curr_properties = cfun->curr_properties;
10321 21185 : child_cfun->has_simduid_loops |= cfun->has_simduid_loops;
10322 21185 : child_cfun->has_force_vectorize_loops |= cfun->has_force_vectorize_loops;
10323 21185 : cgraph_node *node = cgraph_node::get_create (child_fn);
10324 21185 : node->parallelized_function = 1;
10325 42370 : node->has_omp_variant_constructs
10326 21185 : |= cgraph_node::get (cfun->decl)->has_omp_variant_constructs;
10327 21185 : cgraph_node::add_new_function (child_fn, true);
10328 :
10329 : /* Add the new function to the offload table. */
10330 21185 : if (ENABLE_OFFLOADING)
10331 : {
10332 : if (in_lto_p)
10333 : DECL_PRESERVE_P (child_fn) = 1;
10334 : if (!is_ancestor && !is_host_only)
10335 : vec_safe_push (offload_funcs, child_fn);
10336 : }
10337 :
10338 21185 : bool need_asm = DECL_ASSEMBLER_NAME_SET_P (current_function_decl)
10339 21185 : && !DECL_ASSEMBLER_NAME_SET_P (child_fn);
10340 :
10341 : /* Fix the callgraph edges for child_cfun. Those for cfun will be
10342 : fixed in a following pass. */
10343 21185 : push_cfun (child_cfun);
10344 21185 : if (need_asm)
10345 21179 : assign_assembler_name_if_needed (child_fn);
10346 21185 : cgraph_edge::rebuild_edges ();
10347 :
10348 : /* Some EH regions might become dead, see PR34608. If
10349 : pass_cleanup_cfg isn't the first pass to happen with the
10350 : new child, these dead EH edges might cause problems.
10351 : Clean them up now. */
10352 21185 : if (flag_exceptions)
10353 : {
10354 8304 : basic_block bb;
10355 8304 : bool changed = false;
10356 :
10357 119073 : FOR_EACH_BB_FN (bb, cfun)
10358 110769 : changed |= gimple_purge_dead_eh_edges (bb);
10359 8304 : if (changed)
10360 0 : cleanup_tree_cfg ();
10361 : }
10362 21185 : if (flag_checking && !loops_state_satisfies_p (LOOPS_NEED_FIXUP))
10363 0 : verify_loop_structure ();
10364 21185 : pop_cfun ();
10365 :
10366 21185 : if (dump_file && !gimple_in_ssa_p (cfun))
10367 : {
10368 37 : omp_any_child_fn_dumped = true;
10369 37 : dump_function_header (dump_file, child_fn, dump_flags);
10370 37 : dump_function_to_file (child_fn, dump_file, dump_flags);
10371 : }
10372 :
10373 21185 : adjust_context_and_scope (region, gimple_block (entry_stmt), child_fn);
10374 :
10375 : /* Handle the case that an inner ancestor:1 target is called by an outer
10376 : target region. */
10377 21185 : if (is_ancestor)
10378 : {
10379 : cgraph_node *fn2_node;
10380 : child_fn2 = build_decl (DECL_SOURCE_LOCATION (child_fn),
10381 : FUNCTION_DECL,
10382 : clone_function_name (child_fn, "nohost"),
10383 : TREE_TYPE (child_fn));
10384 : if (in_lto_p)
10385 : DECL_PRESERVE_P (child_fn2) = 1;
10386 : TREE_STATIC (child_fn2) = 1;
10387 : DECL_ARTIFICIAL (child_fn2) = 1;
10388 : DECL_IGNORED_P (child_fn2) = 0;
10389 : TREE_PUBLIC (child_fn2) = 0;
10390 : DECL_UNINLINABLE (child_fn2) = 1;
10391 : DECL_EXTERNAL (child_fn2) = 0;
10392 : DECL_CONTEXT (child_fn2) = DECL_CONTEXT (child_fn);
10393 : DECL_INITIAL (child_fn2) = make_node (BLOCK);
10394 : BLOCK_SUPERCONTEXT (DECL_INITIAL (child_fn2)) = child_fn2;
10395 : DECL_ATTRIBUTES (child_fn)
10396 : = remove_attribute ("omp target entrypoint",
10397 : DECL_ATTRIBUTES (child_fn));
10398 : DECL_ATTRIBUTES (child_fn2)
10399 : = tree_cons (get_identifier ("omp target device_ancestor_nohost"),
10400 : NULL_TREE, copy_list (DECL_ATTRIBUTES (child_fn)));
10401 : DECL_ATTRIBUTES (child_fn)
10402 : = tree_cons (get_identifier ("omp target device_ancestor_host"),
10403 : NULL_TREE, DECL_ATTRIBUTES (child_fn));
10404 : DECL_FUNCTION_SPECIFIC_OPTIMIZATION (child_fn2)
10405 : = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (current_function_decl);
10406 : DECL_FUNCTION_SPECIFIC_TARGET (child_fn2)
10407 : = DECL_FUNCTION_SPECIFIC_TARGET (current_function_decl);
10408 : DECL_FUNCTION_VERSIONED (child_fn2)
10409 : = DECL_FUNCTION_VERSIONED (current_function_decl);
10410 :
10411 : fn2_node = cgraph_node::get_create (child_fn2);
10412 : fn2_node->offloadable = 1;
10413 : fn2_node->force_output = 1;
10414 : node->offloadable = 0;
10415 :
10416 : /* Enable pass_omp_device_lower pass. */
10417 : fn2_node = cgraph_node::get (DECL_CONTEXT (child_fn));
10418 : fn2_node->has_omp_variant_constructs = 1;
10419 :
10420 : t = build_decl (DECL_SOURCE_LOCATION (child_fn),
10421 : RESULT_DECL, NULL_TREE, void_type_node);
10422 : DECL_ARTIFICIAL (t) = 1;
10423 : DECL_IGNORED_P (t) = 1;
10424 : DECL_CONTEXT (t) = child_fn2;
10425 : DECL_RESULT (child_fn2) = t;
10426 : DECL_SAVED_TREE (child_fn2) = build1 (RETURN_EXPR,
10427 : void_type_node, NULL);
10428 : tree tmp = DECL_ARGUMENTS (child_fn);
10429 : t = build_decl (DECL_SOURCE_LOCATION (child_fn), PARM_DECL,
10430 : DECL_NAME (tmp), TREE_TYPE (tmp));
10431 : DECL_ARTIFICIAL (t) = 1;
10432 : DECL_NAMELESS (t) = 1;
10433 : DECL_ARG_TYPE (t) = ptr_type_node;
10434 : DECL_CONTEXT (t) = current_function_decl;
10435 : TREE_USED (t) = 1;
10436 : TREE_READONLY (t) = 1;
10437 : DECL_ARGUMENTS (child_fn2) = t;
10438 : gcc_assert (TREE_CHAIN (tmp) == NULL_TREE);
10439 :
10440 : gimplify_function_tree (child_fn2);
10441 : cgraph_node::add_new_function (child_fn2, true);
10442 :
10443 : vec_safe_push (offload_funcs, child_fn2);
10444 : if (dump_file && !gimple_in_ssa_p (cfun))
10445 : {
10446 : dump_function_header (dump_file, child_fn2, dump_flags);
10447 : dump_function_to_file (child_fn2, dump_file, dump_flags);
10448 : }
10449 : }
10450 : }
10451 :
10452 : /* Emit a library call to launch the offloading region, or do data
10453 : transfers. */
10454 37153 : tree t1, t2, t3, t4, depend;
10455 37153 : enum built_in_function start_ix;
10456 37153 : unsigned int flags_i = 0;
10457 :
10458 37153 : switch (gimple_omp_target_kind (entry_stmt))
10459 : {
10460 : case GF_OMP_TARGET_KIND_REGION:
10461 : start_ix = BUILT_IN_GOMP_TARGET;
10462 : break;
10463 : case GF_OMP_TARGET_KIND_DATA:
10464 : start_ix = BUILT_IN_GOMP_TARGET_DATA;
10465 : break;
10466 : case GF_OMP_TARGET_KIND_UPDATE:
10467 : start_ix = BUILT_IN_GOMP_TARGET_UPDATE;
10468 : break;
10469 : case GF_OMP_TARGET_KIND_ENTER_DATA:
10470 : start_ix = BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA;
10471 : break;
10472 : case GF_OMP_TARGET_KIND_EXIT_DATA:
10473 : start_ix = BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA;
10474 : flags_i |= GOMP_TARGET_FLAG_EXIT_DATA;
10475 : break;
10476 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
10477 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
10478 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
10479 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
10480 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
10481 : start_ix = BUILT_IN_GOACC_PARALLEL;
10482 : break;
10483 : case GF_OMP_TARGET_KIND_OACC_DATA:
10484 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
10485 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
10486 : start_ix = BUILT_IN_GOACC_DATA_START;
10487 : break;
10488 : case GF_OMP_TARGET_KIND_OACC_UPDATE:
10489 : start_ix = BUILT_IN_GOACC_UPDATE;
10490 : break;
10491 : case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
10492 : start_ix = BUILT_IN_GOACC_ENTER_DATA;
10493 : break;
10494 : case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
10495 : start_ix = BUILT_IN_GOACC_EXIT_DATA;
10496 : break;
10497 : case GF_OMP_TARGET_KIND_OACC_DECLARE:
10498 : start_ix = BUILT_IN_GOACC_DECLARE;
10499 : break;
10500 0 : default:
10501 0 : gcc_unreachable ();
10502 : }
10503 :
10504 37153 : tree device = NULL_TREE;
10505 37153 : location_t device_loc = UNKNOWN_LOCATION;
10506 37153 : tree goacc_flags = NULL_TREE;
10507 37153 : bool need_device_adjustment = false;
10508 37153 : gimple_stmt_iterator adj_gsi;
10509 37153 : if (is_gimple_omp_oacc (entry_stmt))
10510 : {
10511 : /* By default, no GOACC_FLAGs are set. */
10512 14785 : goacc_flags = integer_zero_node;
10513 : }
10514 : else
10515 : {
10516 22368 : if (is_host_only)
10517 6 : device = build_int_cst (integer_type_node,
10518 : GOMP_DEVICE_HOST_FALLBACK - 1);
10519 22362 : else if ((c = omp_find_clause (clauses, OMP_CLAUSE_DEVICE)) != NULL_TREE)
10520 : {
10521 928 : device = OMP_CLAUSE_DEVICE_ID (c);
10522 : /* Ensure 'device' is of the correct type. */
10523 928 : device = fold_convert_loc (device_loc, integer_type_node, device);
10524 928 : if (TREE_CODE (device) == INTEGER_CST)
10525 : {
10526 54 : if (wi::to_wide (device) == GOMP_DEVICE_ICV)
10527 2 : device = build_int_cst (integer_type_node,
10528 : GOMP_DEVICE_HOST_FALLBACK);
10529 52 : else if (wi::to_wide (device) == GOMP_DEVICE_HOST_FALLBACK)
10530 0 : device = build_int_cst (integer_type_node,
10531 : GOMP_DEVICE_HOST_FALLBACK - 1);
10532 : }
10533 : else
10534 : need_device_adjustment = true;
10535 928 : device_loc = OMP_CLAUSE_LOCATION (c);
10536 928 : if (OMP_CLAUSE_DEVICE_ANCESTOR (c))
10537 41 : device = build_int_cst (integer_type_node,
10538 : GOMP_DEVICE_HOST_FALLBACK);
10539 : }
10540 : else
10541 : {
10542 : /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
10543 : library choose). */
10544 21434 : device = build_int_cst (integer_type_node, GOMP_DEVICE_ICV);
10545 21434 : device_loc = gimple_location (entry_stmt);
10546 : }
10547 :
10548 22368 : c = omp_find_clause (clauses, OMP_CLAUSE_NOWAIT);
10549 : /* FIXME: in_reduction(...) nowait is unimplemented yet, pretend
10550 : nowait doesn't appear. */
10551 22368 : if (c && omp_find_clause (clauses, OMP_CLAUSE_IN_REDUCTION))
10552 : c = NULL;
10553 22104 : if (c)
10554 137 : flags_i |= GOMP_TARGET_FLAG_NOWAIT;
10555 : }
10556 :
10557 : /* By default, there is no conditional. */
10558 37153 : tree cond = NULL_TREE;
10559 37153 : c = omp_find_clause (clauses, OMP_CLAUSE_IF);
10560 37153 : if (c)
10561 1741 : cond = OMP_CLAUSE_IF_EXPR (c);
10562 : /* If we found the clause 'if (cond)', build:
10563 : OpenACC: goacc_flags = (cond ? goacc_flags
10564 : : goacc_flags | GOACC_FLAG_HOST_FALLBACK)
10565 : OpenMP: device = (cond ? device : GOMP_DEVICE_HOST_FALLBACK) */
10566 1741 : if (cond)
10567 : {
10568 1741 : tree *tp;
10569 1741 : if (is_gimple_omp_oacc (entry_stmt))
10570 : tp = &goacc_flags;
10571 : else
10572 666 : tp = &device;
10573 :
10574 1741 : cond = gimple_boolify (cond);
10575 :
10576 1741 : basic_block cond_bb, then_bb, else_bb;
10577 1741 : edge e;
10578 1741 : tree tmp_var = create_tmp_var (TREE_TYPE (*tp));
10579 1741 : if (offloaded)
10580 806 : e = split_block_after_labels (new_bb);
10581 : else
10582 : {
10583 935 : gsi = gsi_last_nondebug_bb (new_bb);
10584 935 : gsi_prev (&gsi);
10585 935 : e = split_block (new_bb, gsi_stmt (gsi));
10586 : }
10587 1741 : cond_bb = e->src;
10588 1741 : new_bb = e->dest;
10589 1741 : remove_edge (e);
10590 :
10591 1741 : then_bb = create_empty_bb (cond_bb);
10592 1741 : else_bb = create_empty_bb (then_bb);
10593 1741 : set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
10594 1741 : set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
10595 :
10596 1741 : stmt = gimple_build_cond_empty (cond);
10597 1741 : gsi = gsi_last_bb (cond_bb);
10598 1741 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
10599 :
10600 1741 : gsi = gsi_start_bb (then_bb);
10601 1741 : stmt = gimple_build_assign (tmp_var, *tp);
10602 1741 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
10603 1741 : adj_gsi = gsi;
10604 :
10605 1741 : gsi = gsi_start_bb (else_bb);
10606 1741 : if (is_gimple_omp_oacc (entry_stmt))
10607 1075 : stmt = gimple_build_assign (tmp_var,
10608 : BIT_IOR_EXPR,
10609 : *tp,
10610 : build_int_cst (integer_type_node,
10611 : GOACC_FLAG_HOST_FALLBACK));
10612 : else
10613 666 : stmt = gimple_build_assign (tmp_var,
10614 : build_int_cst (integer_type_node,
10615 : GOMP_DEVICE_HOST_FALLBACK));
10616 1741 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
10617 :
10618 1741 : make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
10619 1741 : make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
10620 1741 : add_bb_to_loop (then_bb, cond_bb->loop_father);
10621 1741 : add_bb_to_loop (else_bb, cond_bb->loop_father);
10622 1741 : make_edge (then_bb, new_bb, EDGE_FALLTHRU);
10623 1741 : make_edge (else_bb, new_bb, EDGE_FALLTHRU);
10624 :
10625 1741 : *tp = tmp_var;
10626 :
10627 1741 : gsi = gsi_last_nondebug_bb (new_bb);
10628 : }
10629 : else
10630 : {
10631 35412 : gsi = gsi_last_nondebug_bb (new_bb);
10632 :
10633 35412 : if (device != NULL_TREE)
10634 21702 : device = force_gimple_operand_gsi (&gsi, device, true, NULL_TREE,
10635 : true, GSI_SAME_STMT);
10636 35412 : if (need_device_adjustment)
10637 : {
10638 400 : tree tmp_var = create_tmp_var (TREE_TYPE (device));
10639 400 : stmt = gimple_build_assign (tmp_var, device);
10640 400 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
10641 400 : adj_gsi = gsi_for_stmt (stmt);
10642 400 : device = tmp_var;
10643 : }
10644 : }
10645 :
10646 37153 : if ((c = omp_find_clause (clauses, OMP_CLAUSE_SELF)) != NULL_TREE)
10647 : {
10648 324 : gcc_assert ((is_gimple_omp_oacc (entry_stmt) && offloaded)
10649 : || (gimple_omp_target_kind (entry_stmt)
10650 : == GF_OMP_TARGET_KIND_OACC_DATA_KERNELS));
10651 :
10652 87 : edge e;
10653 87 : if (offloaded)
10654 237 : e = split_block_after_labels (new_bb);
10655 : else
10656 : {
10657 87 : gsi = gsi_last_nondebug_bb (new_bb);
10658 87 : gsi_prev (&gsi);
10659 87 : e = split_block (new_bb, gsi_stmt (gsi));
10660 : }
10661 324 : basic_block cond_bb = e->src;
10662 324 : new_bb = e->dest;
10663 324 : remove_edge (e);
10664 :
10665 324 : basic_block then_bb = create_empty_bb (cond_bb);
10666 324 : basic_block else_bb = create_empty_bb (then_bb);
10667 324 : set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
10668 324 : set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
10669 :
10670 324 : tree self_cond = gimple_boolify (OMP_CLAUSE_SELF_EXPR (c));
10671 324 : stmt = gimple_build_cond_empty (self_cond);
10672 324 : gsi = gsi_last_bb (cond_bb);
10673 324 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
10674 :
10675 324 : tree tmp_var = create_tmp_var (TREE_TYPE (goacc_flags));
10676 324 : stmt = gimple_build_assign (tmp_var, BIT_IOR_EXPR, goacc_flags,
10677 : build_int_cst (integer_type_node,
10678 : GOACC_FLAG_LOCAL_DEVICE));
10679 324 : gsi = gsi_start_bb (then_bb);
10680 324 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
10681 :
10682 324 : gsi = gsi_start_bb (else_bb);
10683 324 : stmt = gimple_build_assign (tmp_var, goacc_flags);
10684 324 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
10685 :
10686 324 : make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
10687 324 : make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
10688 324 : add_bb_to_loop (then_bb, cond_bb->loop_father);
10689 324 : add_bb_to_loop (else_bb, cond_bb->loop_father);
10690 324 : make_edge (then_bb, new_bb, EDGE_FALLTHRU);
10691 324 : make_edge (else_bb, new_bb, EDGE_FALLTHRU);
10692 :
10693 324 : goacc_flags = tmp_var;
10694 324 : gsi = gsi_last_nondebug_bb (new_bb);
10695 : }
10696 :
10697 37153 : if (need_device_adjustment)
10698 : {
10699 874 : tree uns = fold_convert (unsigned_type_node, device);
10700 874 : uns = force_gimple_operand_gsi (&adj_gsi, uns, true, NULL_TREE,
10701 : false, GSI_CONTINUE_LINKING);
10702 874 : edge e = split_block (gsi_bb (adj_gsi), gsi_stmt (adj_gsi));
10703 874 : basic_block cond_bb = e->src;
10704 874 : basic_block else_bb = e->dest;
10705 874 : if (gsi_bb (adj_gsi) == new_bb)
10706 : {
10707 400 : new_bb = else_bb;
10708 400 : gsi = gsi_last_nondebug_bb (new_bb);
10709 : }
10710 :
10711 874 : basic_block then_bb = create_empty_bb (cond_bb);
10712 874 : set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
10713 :
10714 874 : cond = build2 (GT_EXPR, boolean_type_node, uns,
10715 : build_int_cst (unsigned_type_node,
10716 : GOMP_DEVICE_HOST_FALLBACK - 1));
10717 874 : stmt = gimple_build_cond_empty (cond);
10718 874 : adj_gsi = gsi_last_bb (cond_bb);
10719 874 : gsi_insert_after (&adj_gsi, stmt, GSI_CONTINUE_LINKING);
10720 :
10721 874 : adj_gsi = gsi_start_bb (then_bb);
10722 874 : tree add = build2 (PLUS_EXPR, integer_type_node, device,
10723 : build_int_cst (integer_type_node, -1));
10724 874 : stmt = gimple_build_assign (device, add);
10725 874 : gsi_insert_after (&adj_gsi, stmt, GSI_CONTINUE_LINKING);
10726 :
10727 874 : make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
10728 874 : e->flags = EDGE_FALSE_VALUE;
10729 874 : add_bb_to_loop (then_bb, cond_bb->loop_father);
10730 874 : make_edge (then_bb, else_bb, EDGE_FALLTHRU);
10731 : }
10732 :
10733 37153 : t = gimple_omp_target_data_arg (entry_stmt);
10734 37153 : if (t == NULL)
10735 : {
10736 4891 : t1 = size_zero_node;
10737 4891 : t2 = build_zero_cst (ptr_type_node);
10738 4891 : t3 = t2;
10739 4891 : t4 = t2;
10740 : }
10741 32262 : else if (TREE_VEC_LENGTH (t) == 3 || is_gimple_omp_oacc (entry_stmt))
10742 : {
10743 32120 : t1 = TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t, 1))));
10744 32120 : t1 = size_binop (PLUS_EXPR, t1, size_int (1));
10745 32120 : t2 = build_fold_addr_expr (TREE_VEC_ELT (t, 0));
10746 32120 : t3 = build_fold_addr_expr (TREE_VEC_ELT (t, 1));
10747 32120 : t4 = build_fold_addr_expr (TREE_VEC_ELT (t, 2));
10748 : }
10749 : else
10750 : {
10751 142 : t1 = force_gimple_operand_gsi (&gsi, TREE_VEC_ELT (t, 3), true, NULL_TREE,
10752 : true, GSI_SAME_STMT);
10753 142 : t2 = force_gimple_operand_gsi (&gsi, TREE_VEC_ELT (t, 0), true, NULL_TREE,
10754 : true, GSI_SAME_STMT);
10755 142 : t3 = force_gimple_operand_gsi (&gsi, TREE_VEC_ELT (t, 1), true, NULL_TREE,
10756 : true, GSI_SAME_STMT);
10757 142 : t4 = force_gimple_operand_gsi (&gsi, TREE_VEC_ELT (t, 2), true, NULL_TREE,
10758 : true, GSI_SAME_STMT);
10759 : }
10760 :
10761 37153 : gimple *g;
10762 37153 : bool tagging = false;
10763 : /* The maximum number used by any start_ix, without varargs. */
10764 37153 : auto_vec<tree, 11> args;
10765 37153 : if (is_gimple_omp_oacc (entry_stmt))
10766 : {
10767 14785 : tree goacc_flags_m = fold_build1 (GOACC_FLAGS_MARSHAL_OP,
10768 : TREE_TYPE (goacc_flags), goacc_flags);
10769 14785 : goacc_flags_m = force_gimple_operand_gsi (&gsi, goacc_flags_m, true,
10770 : NULL_TREE, true,
10771 : GSI_SAME_STMT);
10772 14785 : args.quick_push (goacc_flags_m);
10773 : }
10774 : else
10775 22368 : args.quick_push (device);
10776 37153 : if (offloaded)
10777 21185 : args.quick_push (build_fold_addr_expr (child_fn2 ? child_fn2 : child_fn));
10778 37153 : args.quick_push (t1);
10779 37153 : args.quick_push (t2);
10780 37153 : args.quick_push (t3);
10781 37153 : args.quick_push (t4);
10782 37153 : switch (start_ix)
10783 : {
10784 : case BUILT_IN_GOACC_DATA_START:
10785 : case BUILT_IN_GOACC_DECLARE:
10786 : case BUILT_IN_GOMP_TARGET_DATA:
10787 : break;
10788 20734 : case BUILT_IN_GOMP_TARGET:
10789 20734 : case BUILT_IN_GOMP_TARGET_UPDATE:
10790 20734 : case BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA:
10791 20734 : args.quick_push (build_int_cst (unsigned_type_node, flags_i));
10792 20734 : c = omp_find_clause (clauses, OMP_CLAUSE_DEPEND);
10793 20734 : if (c)
10794 329 : depend = OMP_CLAUSE_DECL (c);
10795 : else
10796 20405 : depend = build_int_cst (ptr_type_node, 0);
10797 20734 : args.quick_push (depend);
10798 20734 : if (start_ix == BUILT_IN_GOMP_TARGET)
10799 11790 : args.quick_push (get_target_arguments (&gsi, entry_stmt));
10800 : break;
10801 9395 : case BUILT_IN_GOACC_PARALLEL:
10802 9395 : if (lookup_attribute ("oacc serial", DECL_ATTRIBUTES (child_fn)) != NULL)
10803 : {
10804 : tree dims = NULL_TREE;
10805 : unsigned int ix;
10806 :
10807 : /* For serial constructs we set all dimensions to 1. */
10808 3024 : for (ix = GOMP_DIM_MAX; ix--;)
10809 2268 : dims = tree_cons (NULL_TREE, integer_one_node, dims);
10810 756 : oacc_replace_fn_attrib (child_fn, dims);
10811 : }
10812 : else
10813 8639 : oacc_set_fn_attrib (child_fn, clauses, &args);
10814 : tagging = true;
10815 : /* FALLTHRU */
10816 12136 : case BUILT_IN_GOACC_ENTER_DATA:
10817 12136 : case BUILT_IN_GOACC_EXIT_DATA:
10818 12136 : case BUILT_IN_GOACC_UPDATE:
10819 12136 : {
10820 12136 : tree t_async = NULL_TREE;
10821 :
10822 : /* If present, use the value specified by the respective
10823 : clause, making sure that is of the correct type. */
10824 12136 : c = omp_find_clause (clauses, OMP_CLAUSE_ASYNC);
10825 12136 : if (c)
10826 1681 : t_async = fold_convert_loc (OMP_CLAUSE_LOCATION (c),
10827 : integer_type_node,
10828 1681 : OMP_CLAUSE_ASYNC_EXPR (c));
10829 10455 : else if (!tagging)
10830 : /* Default values for t_async. */
10831 2431 : t_async = fold_convert_loc (gimple_location (entry_stmt),
10832 : integer_type_node,
10833 : build_int_cst (integer_type_node,
10834 : GOMP_ASYNC_SYNC));
10835 12136 : if (tagging && t_async)
10836 : {
10837 1371 : unsigned HOST_WIDE_INT i_async = GOMP_LAUNCH_OP_MAX;
10838 :
10839 1371 : if (TREE_CODE (t_async) == INTEGER_CST)
10840 : {
10841 : /* See if we can pack the async arg in to the tag's
10842 : operand. */
10843 1331 : i_async = TREE_INT_CST_LOW (t_async);
10844 1331 : if (i_async < GOMP_LAUNCH_OP_MAX)
10845 : t_async = NULL_TREE;
10846 : else
10847 825 : i_async = GOMP_LAUNCH_OP_MAX;
10848 : }
10849 1371 : args.safe_push (oacc_launch_pack (GOMP_LAUNCH_ASYNC, NULL_TREE,
10850 : i_async));
10851 : }
10852 12136 : if (t_async)
10853 3566 : args.safe_push (force_gimple_operand_gsi (&gsi, t_async, true,
10854 : NULL_TREE, true,
10855 : GSI_SAME_STMT));
10856 :
10857 : /* Save the argument index, and ... */
10858 12136 : unsigned t_wait_idx = args.length ();
10859 12136 : unsigned num_waits = 0;
10860 12136 : c = omp_find_clause (clauses, OMP_CLAUSE_WAIT);
10861 12136 : if (!tagging || c)
10862 : /* ... push a placeholder. */
10863 2944 : args.safe_push (integer_zero_node);
10864 :
10865 13986 : for (; c; c = OMP_CLAUSE_CHAIN (c))
10866 1850 : if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_WAIT)
10867 : {
10868 422 : tree arg = fold_convert_loc (OMP_CLAUSE_LOCATION (c),
10869 : integer_type_node,
10870 422 : OMP_CLAUSE_WAIT_EXPR (c));
10871 422 : arg = force_gimple_operand_gsi (&gsi, arg, true, NULL_TREE, true,
10872 : GSI_SAME_STMT);
10873 422 : args.safe_push (arg);
10874 422 : num_waits++;
10875 : }
10876 :
10877 12136 : if (!tagging || num_waits)
10878 : {
10879 203 : tree len;
10880 :
10881 : /* Now that we know the number, update the placeholder. */
10882 203 : if (tagging)
10883 203 : len = oacc_launch_pack (GOMP_LAUNCH_WAIT, NULL_TREE, num_waits);
10884 : else
10885 2741 : len = build_int_cst (integer_type_node, num_waits);
10886 2944 : len = fold_convert_loc (gimple_location (entry_stmt),
10887 : unsigned_type_node, len);
10888 2944 : args[t_wait_idx] = len;
10889 : }
10890 : }
10891 : break;
10892 : default:
10893 : gcc_unreachable ();
10894 : }
10895 23926 : if (tagging)
10896 : /* Push terminal marker - zero. */
10897 9395 : args.safe_push (oacc_launch_pack (0, NULL_TREE, 0));
10898 :
10899 37153 : if (child_fn2)
10900 : {
10901 : g = gimple_build_call_internal (IFN_GOMP_TARGET_REV, 1,
10902 : build_fold_addr_expr (child_fn));
10903 : gimple_set_location (g, gimple_location (entry_stmt));
10904 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
10905 : }
10906 :
10907 37153 : g = gimple_build_call_vec (builtin_decl_explicit (start_ix), args);
10908 37153 : gimple_set_location (g, gimple_location (entry_stmt));
10909 37153 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
10910 37153 : if (!offloaded)
10911 : {
10912 15968 : g = gsi_stmt (gsi);
10913 15968 : gcc_assert (g && gimple_code (g) == GIMPLE_OMP_TARGET);
10914 15968 : gsi_remove (&gsi, true);
10915 : }
10916 37153 : }
10917 :
10918 : /* Expand the parallel region tree rooted at REGION. Expansion
10919 : proceeds in depth-first order. Innermost regions are expanded
10920 : first. This way, parallel regions that require a new function to
10921 : be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
10922 : internal dependencies in their body. */
10923 :
10924 : static void
10925 82181 : expand_omp (struct omp_region *region)
10926 : {
10927 82181 : omp_any_child_fn_dumped = false;
10928 211214 : while (region)
10929 : {
10930 129033 : location_t saved_location;
10931 129033 : gimple *inner_stmt = NULL;
10932 :
10933 : /* First, determine whether this is a combined parallel+workshare
10934 : region. */
10935 129033 : if (region->type == GIMPLE_OMP_PARALLEL)
10936 16400 : determine_parallel_type (region);
10937 :
10938 129033 : if (region->type == GIMPLE_OMP_FOR
10939 129033 : && gimple_omp_for_combined_p (last_nondebug_stmt (region->entry)))
10940 14650 : inner_stmt = last_nondebug_stmt (region->inner->entry);
10941 :
10942 129033 : if (region->inner)
10943 57478 : expand_omp (region->inner);
10944 :
10945 129033 : saved_location = input_location;
10946 129033 : if (gimple_has_location (last_nondebug_stmt (region->entry)))
10947 128849 : input_location = gimple_location (last_nondebug_stmt (region->entry));
10948 :
10949 129033 : switch (region->type)
10950 : {
10951 20296 : case GIMPLE_OMP_PARALLEL:
10952 20296 : case GIMPLE_OMP_TASK:
10953 20296 : expand_omp_taskreg (region);
10954 20296 : break;
10955 :
10956 47801 : case GIMPLE_OMP_FOR:
10957 47801 : expand_omp_for (region, inner_stmt);
10958 47801 : break;
10959 :
10960 386 : case GIMPLE_OMP_SECTIONS:
10961 386 : expand_omp_sections (region);
10962 386 : break;
10963 :
10964 : case GIMPLE_OMP_SECTION:
10965 : /* Individual omp sections are handled together with their
10966 : parent GIMPLE_OMP_SECTIONS region. */
10967 : break;
10968 :
10969 0 : case GIMPLE_OMP_STRUCTURED_BLOCK:
10970 : /* We should have gotten rid of these in gimple lowering. */
10971 0 : gcc_unreachable ();
10972 :
10973 1288 : case GIMPLE_OMP_SINGLE:
10974 1288 : case GIMPLE_OMP_SCOPE:
10975 1288 : expand_omp_single (region);
10976 1288 : break;
10977 :
10978 1120 : case GIMPLE_OMP_ORDERED:
10979 1120 : {
10980 1120 : gomp_ordered *ord_stmt
10981 1120 : = as_a <gomp_ordered *> (last_nondebug_stmt (region->entry));
10982 1120 : if (gimple_omp_ordered_standalone_p (ord_stmt))
10983 : {
10984 : /* We'll expand these when expanding corresponding
10985 : worksharing region with ordered(n) clause. */
10986 709 : gcc_assert (region->outer
10987 : && region->outer->type == GIMPLE_OMP_FOR);
10988 709 : region->ord_stmt = ord_stmt;
10989 709 : break;
10990 : }
10991 : }
10992 : /* FALLTHRU */
10993 10872 : case GIMPLE_OMP_MASTER:
10994 10872 : case GIMPLE_OMP_MASKED:
10995 10872 : case GIMPLE_OMP_TASKGROUP:
10996 10872 : case GIMPLE_OMP_CRITICAL:
10997 10872 : case GIMPLE_OMP_TEAMS:
10998 10872 : expand_omp_synch (region);
10999 10872 : break;
11000 :
11001 9665 : case GIMPLE_OMP_ATOMIC_LOAD:
11002 9665 : expand_omp_atomic (region);
11003 9665 : break;
11004 :
11005 37153 : case GIMPLE_OMP_TARGET:
11006 37153 : expand_omp_target (region);
11007 37153 : break;
11008 :
11009 0 : default:
11010 0 : gcc_unreachable ();
11011 : }
11012 :
11013 129033 : input_location = saved_location;
11014 129033 : region = region->next;
11015 : }
11016 82181 : if (omp_any_child_fn_dumped)
11017 : {
11018 109 : if (dump_file)
11019 109 : dump_function_header (dump_file, current_function_decl, dump_flags);
11020 109 : omp_any_child_fn_dumped = false;
11021 : }
11022 82181 : }
11023 :
11024 : /* Helper for build_omp_regions. Scan the dominator tree starting at
11025 : block BB. PARENT is the region that contains BB. If SINGLE_TREE is
11026 : true, the function ends once a single tree is built (otherwise, whole
11027 : forest of OMP constructs may be built). */
11028 :
11029 : static void
11030 1407729 : build_omp_regions_1 (basic_block bb, struct omp_region *parent,
11031 : bool single_tree)
11032 : {
11033 1407729 : gimple_stmt_iterator gsi;
11034 1407729 : gimple *stmt;
11035 1407729 : basic_block son;
11036 :
11037 1407729 : gsi = gsi_last_nondebug_bb (bb);
11038 1407729 : if (!gsi_end_p (gsi) && is_gimple_omp (gsi_stmt (gsi)))
11039 : {
11040 290565 : struct omp_region *region;
11041 290565 : enum gimple_code code;
11042 :
11043 290565 : stmt = gsi_stmt (gsi);
11044 290565 : code = gimple_code (stmt);
11045 290565 : if (code == GIMPLE_OMP_RETURN)
11046 : {
11047 : /* STMT is the return point out of region PARENT. Mark it
11048 : as the exit point and make PARENT the immediately
11049 : enclosing region. */
11050 101816 : gcc_assert (parent);
11051 101816 : region = parent;
11052 101816 : region->exit = bb;
11053 101816 : parent = parent->outer;
11054 : }
11055 188749 : else if (code == GIMPLE_OMP_ATOMIC_STORE)
11056 : {
11057 : /* GIMPLE_OMP_ATOMIC_STORE is analogous to
11058 : GIMPLE_OMP_RETURN, but matches with
11059 : GIMPLE_OMP_ATOMIC_LOAD. */
11060 9665 : gcc_assert (parent);
11061 9665 : gcc_assert (parent->type == GIMPLE_OMP_ATOMIC_LOAD);
11062 9665 : region = parent;
11063 9665 : region->exit = bb;
11064 9665 : parent = parent->outer;
11065 : }
11066 179084 : else if (code == GIMPLE_OMP_CONTINUE)
11067 : {
11068 49665 : gcc_assert (parent);
11069 49665 : parent->cont = bb;
11070 : }
11071 129419 : else if (code == GIMPLE_OMP_SECTIONS_SWITCH)
11072 : {
11073 : /* GIMPLE_OMP_SECTIONS_SWITCH is part of
11074 : GIMPLE_OMP_SECTIONS, and we do nothing for it. */
11075 : }
11076 : else
11077 : {
11078 129033 : region = new_omp_region (bb, code, parent);
11079 : /* Otherwise... */
11080 129033 : if (code == GIMPLE_OMP_TARGET)
11081 : {
11082 37153 : switch (gimple_omp_target_kind (stmt))
11083 : {
11084 : case GF_OMP_TARGET_KIND_REGION:
11085 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
11086 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
11087 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
11088 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
11089 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
11090 : break;
11091 : case GF_OMP_TARGET_KIND_UPDATE:
11092 : case GF_OMP_TARGET_KIND_ENTER_DATA:
11093 : case GF_OMP_TARGET_KIND_EXIT_DATA:
11094 : case GF_OMP_TARGET_KIND_DATA:
11095 : case GF_OMP_TARGET_KIND_OACC_DATA:
11096 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
11097 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
11098 : case GF_OMP_TARGET_KIND_OACC_UPDATE:
11099 : case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
11100 : case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
11101 : case GF_OMP_TARGET_KIND_OACC_DECLARE:
11102 : /* ..., other than for those stand-alone directives...
11103 : To be precise, target data isn't stand-alone, but
11104 : gimplifier put the end API call into try finally block
11105 : for it, so omp expansion can treat it as such. */
11106 : region = NULL;
11107 : break;
11108 0 : default:
11109 0 : gcc_unreachable ();
11110 : }
11111 : }
11112 91880 : else if (code == GIMPLE_OMP_ORDERED
11113 91880 : && gimple_omp_ordered_standalone_p (stmt))
11114 : /* #pragma omp ordered depend is also just a stand-alone
11115 : directive. */
11116 : region = NULL;
11117 91171 : else if (code == GIMPLE_OMP_TASK
11118 91171 : && gimple_omp_task_taskwait_p (stmt))
11119 : /* #pragma omp taskwait depend(...) is a stand-alone directive. */
11120 : region = NULL;
11121 91087 : else if (code == GIMPLE_OMP_TASKGROUP)
11122 : /* #pragma omp taskgroup isn't a stand-alone directive, but
11123 : gimplifier put the end API call into try finally block
11124 : for it, so omp expansion can treat it as such. */
11125 : region = NULL;
11126 : /* ..., this directive becomes the parent for a new region. */
11127 : if (region)
11128 : parent = region;
11129 : }
11130 : }
11131 :
11132 1407729 : if (single_tree && !parent)
11133 0 : return;
11134 :
11135 1407729 : for (son = first_dom_son (CDI_DOMINATORS, bb);
11136 2738407 : son;
11137 1330678 : son = next_dom_son (CDI_DOMINATORS, son))
11138 1330678 : build_omp_regions_1 (son, parent, single_tree);
11139 : }
11140 :
11141 : /* Builds the tree of OMP regions rooted at ROOT, storing it to
11142 : root_omp_region. */
11143 :
11144 : static void
11145 0 : build_omp_regions_root (basic_block root)
11146 : {
11147 0 : gcc_assert (root_omp_region == NULL);
11148 0 : build_omp_regions_1 (root, NULL, true);
11149 0 : gcc_assert (root_omp_region != NULL);
11150 0 : }
11151 :
11152 : /* Expands omp construct (and its subconstructs) starting in HEAD. */
11153 :
11154 : void
11155 0 : omp_expand_local (basic_block head)
11156 : {
11157 0 : build_omp_regions_root (head);
11158 0 : if (dump_file && (dump_flags & TDF_DETAILS))
11159 : {
11160 0 : fprintf (dump_file, "\nOMP region tree\n\n");
11161 0 : dump_omp_region (dump_file, root_omp_region, 0);
11162 0 : fprintf (dump_file, "\n");
11163 : }
11164 :
11165 0 : remove_exit_barriers (root_omp_region);
11166 0 : expand_omp (root_omp_region);
11167 :
11168 0 : omp_free_regions ();
11169 0 : }
11170 :
11171 : /* Scan the CFG and build a tree of OMP regions. Return the root of
11172 : the OMP region tree. */
11173 :
11174 : static void
11175 77051 : build_omp_regions (void)
11176 : {
11177 77051 : gcc_assert (root_omp_region == NULL);
11178 77051 : calculate_dominance_info (CDI_DOMINATORS);
11179 77051 : build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, false);
11180 77051 : }
11181 :
11182 : /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
11183 :
11184 : static unsigned int
11185 77051 : execute_expand_omp (void)
11186 : {
11187 77051 : build_omp_regions ();
11188 :
11189 77051 : if (!root_omp_region)
11190 : return 0;
11191 :
11192 24703 : if (dump_file)
11193 : {
11194 284 : fprintf (dump_file, "\nOMP region tree\n\n");
11195 284 : dump_omp_region (dump_file, root_omp_region, 0);
11196 284 : fprintf (dump_file, "\n");
11197 : }
11198 :
11199 24703 : remove_exit_barriers (root_omp_region);
11200 :
11201 24703 : expand_omp (root_omp_region);
11202 :
11203 24703 : omp_free_regions ();
11204 :
11205 24703 : return (TODO_cleanup_cfg
11206 48857 : | (gimple_in_ssa_p (cfun) ? TODO_update_ssa_only_virtuals : 0));
11207 : }
11208 :
11209 : /* OMP expansion -- the default pass, run before creation of SSA form. */
11210 :
11211 : namespace {
11212 :
11213 : const pass_data pass_data_expand_omp =
11214 : {
11215 : GIMPLE_PASS, /* type */
11216 : "ompexp", /* name */
11217 : OPTGROUP_OMP, /* optinfo_flags */
11218 : TV_NONE, /* tv_id */
11219 : PROP_gimple_any, /* properties_required */
11220 : PROP_gimple_eomp, /* properties_provided */
11221 : 0, /* properties_destroyed */
11222 : 0, /* todo_flags_start */
11223 : 0, /* todo_flags_finish */
11224 : };
11225 :
11226 : class pass_expand_omp : public gimple_opt_pass
11227 : {
11228 : public:
11229 294196 : pass_expand_omp (gcc::context *ctxt)
11230 588392 : : gimple_opt_pass (pass_data_expand_omp, ctxt)
11231 : {}
11232 :
11233 : /* opt_pass methods: */
11234 3027071 : unsigned int execute (function *) final override
11235 : {
11236 3019799 : bool gate = ((flag_openacc != 0 || flag_openmp != 0
11237 2972197 : || flag_openmp_simd != 0)
11238 3076931 : && !seen_error ());
11239 :
11240 : /* This pass always runs, to provide PROP_gimple_eomp.
11241 : But often, there is nothing to do. */
11242 53034 : if (!gate)
11243 : return 0;
11244 :
11245 53034 : return execute_expand_omp ();
11246 : }
11247 :
11248 : }; // class pass_expand_omp
11249 :
11250 : } // anon namespace
11251 :
11252 : gimple_opt_pass *
11253 294196 : make_pass_expand_omp (gcc::context *ctxt)
11254 : {
11255 294196 : return new pass_expand_omp (ctxt);
11256 : }
11257 :
11258 : namespace {
11259 :
11260 : const pass_data pass_data_expand_omp_ssa =
11261 : {
11262 : GIMPLE_PASS, /* type */
11263 : "ompexpssa", /* name */
11264 : OPTGROUP_OMP, /* optinfo_flags */
11265 : TV_NONE, /* tv_id */
11266 : PROP_cfg | PROP_ssa, /* properties_required */
11267 : PROP_gimple_eomp, /* properties_provided */
11268 : 0, /* properties_destroyed */
11269 : 0, /* todo_flags_start */
11270 : TODO_cleanup_cfg | TODO_rebuild_alias, /* todo_flags_finish */
11271 : };
11272 :
11273 : class pass_expand_omp_ssa : public gimple_opt_pass
11274 : {
11275 : public:
11276 588392 : pass_expand_omp_ssa (gcc::context *ctxt)
11277 1176784 : : gimple_opt_pass (pass_data_expand_omp_ssa, ctxt)
11278 : {}
11279 :
11280 : /* opt_pass methods: */
11281 246135 : bool gate (function *fun) final override
11282 : {
11283 246135 : return !(fun->curr_properties & PROP_gimple_eomp);
11284 : }
11285 24017 : unsigned int execute (function *) final override
11286 : {
11287 24017 : return execute_expand_omp ();
11288 : }
11289 294196 : opt_pass * clone () final override
11290 : {
11291 294196 : return new pass_expand_omp_ssa (m_ctxt);
11292 : }
11293 :
11294 : }; // class pass_expand_omp_ssa
11295 :
11296 : } // anon namespace
11297 :
11298 : gimple_opt_pass *
11299 294196 : make_pass_expand_omp_ssa (gcc::context *ctxt)
11300 : {
11301 294196 : return new pass_expand_omp_ssa (ctxt);
11302 : }
11303 :
11304 : /* Called from tree-cfg.cc::make_edges to create cfg edges for all relevant
11305 : GIMPLE_* codes. */
11306 :
11307 : bool
11308 291622 : omp_make_gimple_edges (basic_block bb, struct omp_region **region,
11309 : int *region_idx)
11310 : {
11311 291622 : gimple *last = last_nondebug_stmt (bb);
11312 291622 : enum gimple_code code = gimple_code (last);
11313 291622 : struct omp_region *cur_region = *region;
11314 291622 : bool fallthru = false;
11315 :
11316 291622 : switch (code)
11317 : {
11318 75563 : case GIMPLE_OMP_PARALLEL:
11319 75563 : case GIMPLE_OMP_FOR:
11320 75563 : case GIMPLE_OMP_SINGLE:
11321 75563 : case GIMPLE_OMP_TEAMS:
11322 75563 : case GIMPLE_OMP_MASTER:
11323 75563 : case GIMPLE_OMP_MASKED:
11324 75563 : case GIMPLE_OMP_SCOPE:
11325 75563 : case GIMPLE_OMP_CRITICAL:
11326 75563 : case GIMPLE_OMP_SECTION:
11327 75563 : cur_region = new_omp_region (bb, code, cur_region);
11328 75563 : fallthru = true;
11329 75563 : break;
11330 :
11331 536 : case GIMPLE_OMP_TASKGROUP:
11332 536 : cur_region = new_omp_region (bb, code, cur_region);
11333 536 : fallthru = true;
11334 536 : cur_region = cur_region->outer;
11335 536 : break;
11336 :
11337 3896 : case GIMPLE_OMP_TASK:
11338 3896 : cur_region = new_omp_region (bb, code, cur_region);
11339 3896 : fallthru = true;
11340 3896 : if (gimple_omp_task_taskwait_p (last))
11341 84 : cur_region = cur_region->outer;
11342 : break;
11343 :
11344 1124 : case GIMPLE_OMP_ORDERED:
11345 1124 : cur_region = new_omp_region (bb, code, cur_region);
11346 1124 : fallthru = true;
11347 1124 : if (gimple_omp_ordered_standalone_p (last))
11348 713 : cur_region = cur_region->outer;
11349 : break;
11350 :
11351 37183 : case GIMPLE_OMP_TARGET:
11352 37183 : cur_region = new_omp_region (bb, code, cur_region);
11353 37183 : fallthru = true;
11354 37183 : switch (gimple_omp_target_kind (last))
11355 : {
11356 : case GF_OMP_TARGET_KIND_REGION:
11357 : case GF_OMP_TARGET_KIND_OACC_PARALLEL:
11358 : case GF_OMP_TARGET_KIND_OACC_KERNELS:
11359 : case GF_OMP_TARGET_KIND_OACC_SERIAL:
11360 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
11361 : case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
11362 : break;
11363 15970 : case GF_OMP_TARGET_KIND_UPDATE:
11364 15970 : case GF_OMP_TARGET_KIND_ENTER_DATA:
11365 15970 : case GF_OMP_TARGET_KIND_EXIT_DATA:
11366 15970 : case GF_OMP_TARGET_KIND_DATA:
11367 15970 : case GF_OMP_TARGET_KIND_OACC_DATA:
11368 15970 : case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
11369 15970 : case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
11370 15970 : case GF_OMP_TARGET_KIND_OACC_UPDATE:
11371 15970 : case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
11372 15970 : case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
11373 15970 : case GF_OMP_TARGET_KIND_OACC_DECLARE:
11374 15970 : cur_region = cur_region->outer;
11375 15970 : break;
11376 0 : default:
11377 0 : gcc_unreachable ();
11378 : }
11379 : break;
11380 :
11381 386 : case GIMPLE_OMP_SECTIONS:
11382 386 : cur_region = new_omp_region (bb, code, cur_region);
11383 386 : fallthru = true;
11384 386 : break;
11385 :
11386 : case GIMPLE_OMP_SECTIONS_SWITCH:
11387 : fallthru = false;
11388 : break;
11389 :
11390 19714 : case GIMPLE_OMP_ATOMIC_LOAD:
11391 19714 : case GIMPLE_OMP_ATOMIC_STORE:
11392 19714 : fallthru = true;
11393 19714 : break;
11394 :
11395 101385 : case GIMPLE_OMP_RETURN:
11396 : /* In the case of a GIMPLE_OMP_SECTION, the edge will go
11397 : somewhere other than the next block. This will be
11398 : created later. */
11399 101385 : cur_region->exit = bb;
11400 101385 : if (cur_region->type == GIMPLE_OMP_TASK)
11401 : /* Add an edge corresponding to not scheduling the task
11402 : immediately. */
11403 3812 : make_edge (cur_region->entry, bb, EDGE_ABNORMAL);
11404 101385 : fallthru = cur_region->type != GIMPLE_OMP_SECTION;
11405 101385 : cur_region = cur_region->outer;
11406 101385 : break;
11407 :
11408 51449 : case GIMPLE_OMP_CONTINUE:
11409 51449 : cur_region->cont = bb;
11410 51449 : switch (cur_region->type)
11411 : {
11412 47251 : case GIMPLE_OMP_FOR:
11413 : /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
11414 : succs edges as abnormal to prevent splitting
11415 : them. */
11416 47251 : single_succ_edge (cur_region->entry)->flags |= EDGE_ABNORMAL;
11417 : /* Make the loopback edge. */
11418 47251 : make_edge (bb, single_succ (cur_region->entry),
11419 : EDGE_ABNORMAL);
11420 :
11421 : /* Create an edge from GIMPLE_OMP_FOR to exit, which
11422 : corresponds to the case that the body of the loop
11423 : is not executed at all. */
11424 47251 : make_edge (cur_region->entry, bb->next_bb, EDGE_ABNORMAL);
11425 47251 : make_edge (bb, bb->next_bb, EDGE_FALLTHRU | EDGE_ABNORMAL);
11426 47251 : fallthru = false;
11427 47251 : break;
11428 :
11429 386 : case GIMPLE_OMP_SECTIONS:
11430 : /* Wire up the edges into and out of the nested sections. */
11431 386 : {
11432 386 : basic_block switch_bb = single_succ (cur_region->entry);
11433 :
11434 386 : struct omp_region *i;
11435 1249 : for (i = cur_region->inner; i ; i = i->next)
11436 : {
11437 863 : gcc_assert (i->type == GIMPLE_OMP_SECTION);
11438 863 : make_edge (switch_bb, i->entry, 0);
11439 863 : make_edge (i->exit, bb, EDGE_FALLTHRU);
11440 : }
11441 :
11442 : /* Make the loopback edge to the block with
11443 : GIMPLE_OMP_SECTIONS_SWITCH. */
11444 386 : make_edge (bb, switch_bb, 0);
11445 :
11446 : /* Make the edge from the switch to exit. */
11447 386 : make_edge (switch_bb, bb->next_bb, 0);
11448 386 : fallthru = false;
11449 : }
11450 386 : break;
11451 :
11452 : case GIMPLE_OMP_TASK:
11453 : fallthru = true;
11454 : break;
11455 :
11456 0 : default:
11457 0 : gcc_unreachable ();
11458 : }
11459 : break;
11460 :
11461 0 : default:
11462 0 : gcc_unreachable ();
11463 : }
11464 :
11465 291622 : if (*region != cur_region)
11466 : {
11467 202770 : *region = cur_region;
11468 202770 : if (cur_region)
11469 161318 : *region_idx = cur_region->entry->index;
11470 : else
11471 41452 : *region_idx = 0;
11472 : }
11473 :
11474 291622 : return fallthru;
11475 : }
|
