Line data Source code
1 : /* OMP constructs' SIMD clone supporting code.
2 :
3 : Copyright (C) 2005-2026 Free Software Foundation, Inc.
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "target.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "cfghooks.h"
29 : #include "alloc-pool.h"
30 : #include "tree-pass.h"
31 : #include "ssa.h"
32 : #include "cgraph.h"
33 : #include "pretty-print.h"
34 : #include "diagnostic-core.h"
35 : #include "fold-const.h"
36 : #include "stor-layout.h"
37 : #include "cfganal.h"
38 : #include "gimplify.h"
39 : #include "gimple-iterator.h"
40 : #include "gimplify-me.h"
41 : #include "gimple-walk.h"
42 : #include "langhooks.h"
43 : #include "tree-cfg.h"
44 : #include "tree-into-ssa.h"
45 : #include "tree-dfa.h"
46 : #include "cfgloop.h"
47 : #include "symbol-summary.h"
48 : #include "ipa-param-manipulation.h"
49 : #include "tree-eh.h"
50 : #include "varasm.h"
51 : #include "stringpool.h"
52 : #include "attribs.h"
53 : #include "omp-simd-clone.h"
54 : #include "omp-low.h"
55 : #include "omp-general.h"
56 :
57 : /* Print debug info for ok_for_auto_simd_clone to the dump file, logging
58 : failure reason EXCUSE for function DECL. Always returns false. */
59 : static bool
60 67 : auto_simd_fail (tree decl, const char *excuse)
61 : {
62 67 : if (dump_file && (dump_flags & TDF_DETAILS))
63 134 : fprintf (dump_file, "\nNot auto-cloning %s because %s\n",
64 67 : IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)),
65 : excuse);
66 67 : return false;
67 : }
68 :
69 : /* Helper function for ok_for_auto_simd_clone; return false if the statement
70 : violates restrictions for an "omp declare simd" function. Specifically,
71 : the function must not
72 : - throw or call setjmp/longjmp
73 : - write memory that could alias parallel calls
74 : - read volatile memory
75 : - include openmp directives or calls
76 : - call functions that might do those things */
77 :
78 : static bool
79 17 : auto_simd_check_stmt (gimple *stmt, tree outer)
80 : {
81 17 : tree decl;
82 :
83 17 : switch (gimple_code (stmt))
84 : {
85 4 : case GIMPLE_CALL:
86 :
87 : /* Calls to functions that are CONST or PURE are ok, even if they
88 : are internal functions without a decl. Reject other internal
89 : functions. */
90 4 : if (gimple_call_flags (stmt) & (ECF_CONST | ECF_PURE))
91 : break;
92 4 : if (gimple_call_internal_p (stmt))
93 0 : return auto_simd_fail (outer,
94 0 : "body contains internal function call");
95 :
96 4 : decl = gimple_call_fndecl (stmt);
97 :
98 : /* We can't know whether indirect calls are safe. */
99 4 : if (decl == NULL_TREE)
100 0 : return auto_simd_fail (outer, "body contains indirect call");
101 :
102 : /* Calls to functions that are already marked "omp declare simd" are
103 : OK. */
104 4 : if (lookup_attribute ("omp declare simd", DECL_ATTRIBUTES (decl)))
105 : break;
106 :
107 : /* Let recursive calls to the current function through. */
108 4 : if (decl == outer)
109 : break;
110 :
111 : /* Other function calls are not permitted. This covers all calls to
112 : the libgomp API and setjmp/longjmp, too, as well as things like
113 : __cxa_throw_ related to exception handling. */
114 4 : return auto_simd_fail (outer, "body contains unsafe function call");
115 :
116 : /* Reject EH-related constructs. Most of the EH gimple codes are
117 : already lowered by the time this pass runs during IPA.
118 : GIMPLE_EH_DISPATCH and GIMPLE_RESX remain and are lowered by
119 : pass_lower_eh_dispatch and pass_lower_resx, respectively; those
120 : passes run later. */
121 0 : case GIMPLE_EH_DISPATCH:
122 0 : case GIMPLE_RESX:
123 0 : return auto_simd_fail (outer, "body contains EH constructs");
124 :
125 : /* Asms are not permitted since we don't know what they do. */
126 0 : case GIMPLE_ASM:
127 0 : return auto_simd_fail (outer, "body contains inline asm");
128 :
129 : default:
130 : break;
131 : }
132 :
133 : /* Memory writes are not permitted.
134 : FIXME: this could be relaxed a little to permit writes to
135 : function-local variables that could not alias other instances
136 : of the function running in parallel. */
137 13 : if (gimple_store_p (stmt))
138 1 : return auto_simd_fail (outer, "body includes memory write");
139 :
140 : /* Volatile reads are not permitted. */
141 21 : if (gimple_has_volatile_ops (stmt))
142 1 : return auto_simd_fail (outer, "body includes volatile op");
143 :
144 : /* Otherwise OK. */
145 : return true;
146 : }
147 :
148 : /* Helper function for ok_for_auto_simd_clone: return true if type T is
149 : plausible for a cloneable function argument or return type. */
150 : static bool
151 42 : plausible_type_for_simd_clone (tree t)
152 : {
153 42 : if (VOID_TYPE_P (t))
154 : return true;
155 32 : else if (RECORD_OR_UNION_TYPE_P (t) || !is_a <scalar_mode> (TYPE_MODE (t)))
156 : /* Small record/union types may fit into a scalar mode, but are
157 : still not suitable. */
158 1 : return false;
159 31 : else if (TYPE_ATOMIC (t))
160 : /* Atomic types trigger warnings in simd_clone_clauses_extract. */
161 : return false;
162 : else
163 : return true;
164 : }
165 :
166 : /* Check if the function NODE appears suitable for auto-annotation
167 : with "declare simd". */
168 :
169 : static bool
170 71 : ok_for_auto_simd_clone (struct cgraph_node *node)
171 : {
172 71 : tree decl = node->decl;
173 71 : tree t;
174 71 : basic_block bb;
175 :
176 : /* Nothing to do if the function isn't a definition or doesn't
177 : have a body. */
178 71 : if (!node->definition || !node->has_gimple_body_p ())
179 46 : return auto_simd_fail (decl, "no definition or body");
180 :
181 : /* No point in trying to generate implicit clones if the function
182 : isn't used in the compilation unit. */
183 25 : if (!node->callers)
184 13 : return auto_simd_fail (decl, "function is not used");
185 :
186 : /* Nothing to do if the function already has the "omp declare simd"
187 : attribute, is marked noclone, or is not "omp declare target". */
188 12 : if (lookup_attribute ("omp declare simd", DECL_ATTRIBUTES (decl))
189 12 : || lookup_attribute ("noclone", DECL_ATTRIBUTES (decl))
190 24 : || !lookup_attribute ("omp declare target", DECL_ATTRIBUTES (decl)))
191 0 : return auto_simd_fail (decl, "incompatible attributes");
192 :
193 : /* Check whether the function is restricted host/nohost via the
194 : "omp declare target device_type" clause, and that doesn't match
195 : what we're compiling for. Internally, these translate into
196 : "omp declare target [no]host" attributes on the decl; "any"
197 : translates into both attributes, but the default (which is supposed
198 : to be equivalent to "any") is neither. */
199 12 : tree host = lookup_attribute ("omp declare target host",
200 12 : DECL_ATTRIBUTES (decl));
201 12 : tree nohost = lookup_attribute ("omp declare target nohost",
202 12 : DECL_ATTRIBUTES (decl));
203 : #ifdef ACCEL_COMPILER
204 : if (host && !nohost)
205 : return auto_simd_fail (decl, "device doesn't match for accel compiler");
206 : #else
207 12 : if (nohost && !host)
208 1 : return auto_simd_fail (decl, "device doesn't match for host compiler");
209 : #endif
210 :
211 : /* Backends will check for vectorizable arguments/return types in a
212 : target-specific way, but we can immediately filter out functions
213 : that have implausible argument/return types. */
214 11 : t = TREE_TYPE (TREE_TYPE (decl));
215 11 : if (!plausible_type_for_simd_clone (t))
216 0 : return auto_simd_fail (decl, "return type fails sniff test");
217 :
218 11 : if (TYPE_ARG_TYPES (TREE_TYPE (decl)))
219 : {
220 11 : for (tree temp = TYPE_ARG_TYPES (TREE_TYPE (decl));
221 41 : temp; temp = TREE_CHAIN (temp))
222 : {
223 31 : t = TREE_VALUE (temp);
224 31 : if (!plausible_type_for_simd_clone (t))
225 1 : return auto_simd_fail (decl, "argument type fails sniff test");
226 : }
227 : }
228 0 : else if (DECL_ARGUMENTS (decl))
229 : {
230 0 : for (tree temp = DECL_ARGUMENTS (decl); temp; temp = DECL_CHAIN (temp))
231 : {
232 0 : t = TREE_TYPE (temp);
233 0 : if (!plausible_type_for_simd_clone (t))
234 0 : return auto_simd_fail (decl, "argument type fails sniff test");
235 : }
236 : }
237 : else
238 0 : return auto_simd_fail (decl, "function has no arguments");
239 :
240 : /* Scan the function body to see if it is suitable for SIMD-ization. */
241 10 : node->get_body ();
242 :
243 17 : FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (decl))
244 : {
245 37 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);
246 11 : gsi_next (&gsi))
247 17 : if (!auto_simd_check_stmt (gsi_stmt (gsi), decl))
248 67 : return false;
249 : }
250 :
251 : /* All is good. */
252 4 : if (dump_file)
253 8 : fprintf (dump_file, "\nMarking %s for auto-cloning\n",
254 4 : IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)));
255 : return true;
256 : }
257 :
258 : /* Allocate a fresh `simd_clone' and return it. NARGS is the number
259 : of arguments to reserve space for. */
260 :
261 : static struct cgraph_simd_clone *
262 7590 : simd_clone_struct_alloc (int nargs)
263 : {
264 7590 : struct cgraph_simd_clone *clone_info;
265 7590 : size_t len = (sizeof (struct cgraph_simd_clone)
266 7590 : + nargs * sizeof (struct cgraph_simd_clone_arg));
267 7590 : clone_info = (struct cgraph_simd_clone *)
268 7590 : ggc_internal_cleared_alloc (len);
269 7590 : return clone_info;
270 : }
271 :
272 : /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
273 :
274 : static inline void
275 6119 : simd_clone_struct_copy (struct cgraph_simd_clone *to,
276 : struct cgraph_simd_clone *from)
277 : {
278 6119 : memcpy (to, from, (sizeof (struct cgraph_simd_clone)
279 6119 : + ((from->nargs - from->inbranch)
280 6119 : * sizeof (struct cgraph_simd_clone_arg))));
281 6119 : }
282 :
283 : /* Fill an empty vector ARGS with parameter types of function FNDECL. This
284 : uses TYPE_ARG_TYPES if available, otherwise falls back to types of
285 : DECL_ARGUMENTS types. */
286 :
287 : static void
288 4738 : simd_clone_vector_of_formal_parm_types (vec<tree> *args, tree fndecl)
289 : {
290 4738 : if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
291 : {
292 4694 : push_function_arg_types (args, TREE_TYPE (fndecl));
293 4694 : return;
294 : }
295 44 : push_function_arg_decls (args, fndecl);
296 44 : unsigned int i;
297 44 : tree arg;
298 93 : FOR_EACH_VEC_ELT (*args, i, arg)
299 5 : (*args)[i] = TREE_TYPE ((*args)[i]);
300 : }
301 :
302 : /* Given a simd function in NODE, extract the simd specific
303 : information from the OMP clauses passed in CLAUSES, and return
304 : the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
305 : is set to TRUE if the `inbranch' or `notinbranch' clause specified,
306 : otherwise set to FALSE. */
307 :
308 : static struct cgraph_simd_clone *
309 1471 : simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
310 : bool *inbranch_specified)
311 : {
312 1471 : auto_vec<tree> args;
313 1471 : simd_clone_vector_of_formal_parm_types (&args, node->decl);
314 1471 : tree t;
315 1471 : int n;
316 1471 : *inbranch_specified = false;
317 :
318 1471 : n = args.length ();
319 2930 : if (n > 0 && args.last () == void_type_node)
320 1463 : n--;
321 :
322 : /* Allocate one more than needed just in case this is an in-branch
323 : clone which will require a mask argument. */
324 1471 : struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
325 1471 : clone_info->nargs = n;
326 :
327 1471 : if (!clauses)
328 174 : goto out;
329 :
330 1297 : clauses = TREE_VALUE (clauses);
331 1297 : if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
332 101 : goto out;
333 :
334 3325 : for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
335 : {
336 2129 : switch (OMP_CLAUSE_CODE (t))
337 : {
338 110 : case OMP_CLAUSE_INBRANCH:
339 110 : clone_info->inbranch = 1;
340 110 : *inbranch_specified = true;
341 110 : break;
342 835 : case OMP_CLAUSE_NOTINBRANCH:
343 835 : clone_info->inbranch = 0;
344 835 : *inbranch_specified = true;
345 835 : break;
346 246 : case OMP_CLAUSE_SIMDLEN:
347 246 : clone_info->simdlen
348 246 : = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
349 246 : break;
350 486 : case OMP_CLAUSE_LINEAR:
351 486 : {
352 486 : tree decl = OMP_CLAUSE_DECL (t);
353 486 : tree step = OMP_CLAUSE_LINEAR_STEP (t);
354 486 : int argno = TREE_INT_CST_LOW (decl);
355 486 : if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
356 : {
357 31 : enum cgraph_simd_clone_arg_type arg_type;
358 31 : if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
359 11 : switch (OMP_CLAUSE_LINEAR_KIND (t))
360 : {
361 : case OMP_CLAUSE_LINEAR_REF:
362 : arg_type
363 : = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP;
364 : break;
365 : case OMP_CLAUSE_LINEAR_UVAL:
366 : arg_type
367 : = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP;
368 : break;
369 : case OMP_CLAUSE_LINEAR_VAL:
370 : case OMP_CLAUSE_LINEAR_DEFAULT:
371 : arg_type
372 : = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP;
373 : break;
374 0 : default:
375 0 : gcc_unreachable ();
376 : }
377 : else
378 : arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
379 31 : clone_info->args[argno].arg_type = arg_type;
380 31 : clone_info->args[argno].linear_step = tree_to_shwi (step);
381 31 : gcc_assert (clone_info->args[argno].linear_step >= 0
382 : && clone_info->args[argno].linear_step < n);
383 : }
384 : else
385 : {
386 455 : if (POINTER_TYPE_P (args[argno]))
387 176 : step = fold_convert (ssizetype, step);
388 455 : if (!tree_fits_shwi_p (step))
389 : {
390 0 : warning_at (OMP_CLAUSE_LOCATION (t), OPT_Wopenmp,
391 : "ignoring large linear step");
392 0 : return NULL;
393 : }
394 455 : else if (integer_zerop (step))
395 : {
396 0 : warning_at (OMP_CLAUSE_LOCATION (t), OPT_Wopenmp,
397 : "ignoring zero linear step");
398 0 : return NULL;
399 : }
400 : else
401 : {
402 455 : enum cgraph_simd_clone_arg_type arg_type;
403 455 : if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
404 113 : switch (OMP_CLAUSE_LINEAR_KIND (t))
405 : {
406 : case OMP_CLAUSE_LINEAR_REF:
407 : arg_type
408 : = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP;
409 : break;
410 : case OMP_CLAUSE_LINEAR_UVAL:
411 : arg_type
412 : = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP;
413 : break;
414 : case OMP_CLAUSE_LINEAR_VAL:
415 : case OMP_CLAUSE_LINEAR_DEFAULT:
416 : arg_type
417 : = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP;
418 : break;
419 0 : default:
420 0 : gcc_unreachable ();
421 : }
422 : else
423 : arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
424 455 : clone_info->args[argno].arg_type = arg_type;
425 455 : clone_info->args[argno].linear_step = tree_to_shwi (step);
426 : }
427 : }
428 : break;
429 : }
430 368 : case OMP_CLAUSE_UNIFORM:
431 368 : {
432 368 : tree decl = OMP_CLAUSE_DECL (t);
433 368 : int argno = tree_to_uhwi (decl);
434 368 : clone_info->args[argno].arg_type
435 368 : = SIMD_CLONE_ARG_TYPE_UNIFORM;
436 368 : break;
437 : }
438 84 : case OMP_CLAUSE_ALIGNED:
439 84 : {
440 : /* Ignore aligned (x) for declare simd, for the ABI we really
441 : need an alignment specified. */
442 84 : if (OMP_CLAUSE_ALIGNED_ALIGNMENT (t) == NULL_TREE)
443 : break;
444 80 : tree decl = OMP_CLAUSE_DECL (t);
445 80 : int argno = tree_to_uhwi (decl);
446 80 : clone_info->args[argno].alignment
447 80 : = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
448 80 : break;
449 : }
450 : default:
451 : break;
452 : }
453 : }
454 :
455 1196 : out:
456 1471 : if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node->decl))))
457 : {
458 0 : warning_at (DECL_SOURCE_LOCATION (node->decl), OPT_Wopenmp,
459 : "ignoring %<#pragma omp declare simd%> on function "
460 : "with %<_Atomic%> qualified return type");
461 0 : return NULL;
462 : }
463 :
464 3977 : for (unsigned int argno = 0; argno < clone_info->nargs; argno++)
465 2507 : if (TYPE_ATOMIC (args[argno])
466 2507 : && clone_info->args[argno].arg_type != SIMD_CLONE_ARG_TYPE_UNIFORM)
467 : {
468 1 : warning_at (DECL_SOURCE_LOCATION (node->decl), OPT_Wopenmp,
469 : "ignoring %<#pragma omp declare simd%> on function "
470 : "with %<_Atomic%> qualified non-%<uniform%> argument");
471 1 : args.release ();
472 1 : return NULL;
473 : }
474 :
475 : return clone_info;
476 1471 : }
477 :
478 : /* Given a SIMD clone in NODE, calculate the characteristic data
479 : type and return the coresponding type. The characteristic data
480 : type is computed as described in the Intel Vector ABI. */
481 :
482 : static tree
483 4195 : simd_clone_compute_base_data_type (struct cgraph_node *node,
484 : struct cgraph_simd_clone *clone_info)
485 : {
486 4195 : tree type = integer_type_node;
487 4195 : tree fndecl = node->decl;
488 :
489 : /* a) For non-void function, the characteristic data type is the
490 : return type. */
491 4195 : if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
492 3644 : type = TREE_TYPE (TREE_TYPE (fndecl));
493 :
494 : /* b) If the function has any non-uniform, non-linear parameters,
495 : then the characteristic data type is the type of the first
496 : such parameter. */
497 : else
498 : {
499 551 : auto_vec<tree> map;
500 551 : simd_clone_vector_of_formal_parm_types (&map, fndecl);
501 738 : for (unsigned int i = 0; i < clone_info->nargs; ++i)
502 362 : if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
503 : {
504 175 : type = map[i];
505 175 : break;
506 : }
507 551 : }
508 :
509 : /* c) If the characteristic data type determined by a) or b) above
510 : is struct, union, or class type which is pass-by-value (except
511 : for the type that maps to the built-in complex data type), the
512 : characteristic data type is int. */
513 4195 : if (RECORD_OR_UNION_TYPE_P (type)
514 2 : && !aggregate_value_p (type, NULL)
515 4197 : && TREE_CODE (type) != COMPLEX_TYPE)
516 2 : return integer_type_node;
517 :
518 : /* d) If none of the above three classes is applicable, the
519 : characteristic data type is int. */
520 :
521 : return type;
522 :
523 : /* e) For Intel Xeon Phi native and offload compilation, if the
524 : resulting characteristic data type is 8-bit or 16-bit integer
525 : data type, the characteristic data type is int. */
526 : /* Well, we don't handle Xeon Phi yet. */
527 : }
528 :
529 : static tree
530 7582 : simd_clone_mangle (struct cgraph_node *node,
531 : struct cgraph_simd_clone *clone_info)
532 : {
533 7582 : char vecsize_mangle = clone_info->vecsize_mangle;
534 7582 : char mask = clone_info->inbranch ? 'M' : 'N';
535 7582 : poly_uint64 simdlen = clone_info->simdlen;
536 7582 : unsigned int n;
537 7582 : pretty_printer pp;
538 :
539 7582 : gcc_assert (vecsize_mangle && maybe_ne (simdlen, 0U));
540 :
541 7582 : pp_string (&pp, "_ZGV");
542 7582 : pp_character (&pp, vecsize_mangle);
543 7582 : pp_character (&pp, mask);
544 :
545 7582 : unsigned HOST_WIDE_INT len;
546 7582 : if (simdlen.is_constant (&len))
547 7582 : pp_decimal_int (&pp, (int) (len));
548 : else
549 : pp_character (&pp, 'x');
550 :
551 20425 : for (n = 0; n < clone_info->nargs; ++n)
552 : {
553 12843 : struct cgraph_simd_clone_arg arg = clone_info->args[n];
554 :
555 12843 : switch (arg.arg_type)
556 : {
557 2063 : case SIMD_CLONE_ARG_TYPE_UNIFORM:
558 2063 : pp_character (&pp, 'u');
559 2063 : break;
560 1915 : case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
561 1915 : pp_character (&pp, 'l');
562 1915 : goto mangle_linear;
563 332 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
564 332 : pp_character (&pp, 'R');
565 332 : goto mangle_linear;
566 160 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
567 160 : pp_character (&pp, 'L');
568 160 : goto mangle_linear;
569 160 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
570 160 : pp_character (&pp, 'U');
571 160 : goto mangle_linear;
572 2567 : mangle_linear:
573 2567 : gcc_assert (arg.linear_step != 0);
574 2567 : if (arg.linear_step > 1)
575 1242 : pp_unsigned_wide_integer (&pp, arg.linear_step);
576 1325 : else if (arg.linear_step < 0)
577 : {
578 73 : pp_character (&pp, 'n');
579 73 : pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
580 : arg.linear_step));
581 : }
582 : break;
583 112 : case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
584 112 : pp_string (&pp, "ls");
585 112 : pp_unsigned_wide_integer (&pp, arg.linear_step);
586 112 : break;
587 28 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
588 28 : pp_string (&pp, "Rs");
589 28 : pp_unsigned_wide_integer (&pp, arg.linear_step);
590 28 : break;
591 12 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
592 12 : pp_string (&pp, "Ls");
593 12 : pp_unsigned_wide_integer (&pp, arg.linear_step);
594 12 : break;
595 12 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
596 12 : pp_string (&pp, "Us");
597 12 : pp_unsigned_wide_integer (&pp, arg.linear_step);
598 12 : break;
599 8049 : default:
600 8049 : pp_character (&pp, 'v');
601 : }
602 12843 : if (arg.alignment)
603 : {
604 520 : pp_character (&pp, 'a');
605 520 : pp_decimal_int (&pp, arg.alignment);
606 : }
607 : }
608 :
609 7582 : pp_underscore (&pp);
610 7582 : const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
611 7582 : if (*str == '*')
612 16 : ++str;
613 7582 : pp_string (&pp, str);
614 7582 : str = pp_formatted_text (&pp);
615 :
616 : /* If there already is a SIMD clone with the same mangled name, don't
617 : add another one. This can happen e.g. for
618 : #pragma omp declare simd
619 : #pragma omp declare simd simdlen(8)
620 : int foo (int, int);
621 : if the simdlen is assumed to be 8 for the first one, etc. */
622 31492 : for (struct cgraph_node *clone = node->simd_clones; clone;
623 23910 : clone = clone->simdclone->next_clone)
624 24304 : if (id_equal (DECL_ASSEMBLER_NAME (clone->decl), str))
625 : return NULL_TREE;
626 :
627 7188 : return get_identifier (str);
628 7582 : }
629 :
630 : /* Create a simd clone of OLD_NODE and return it. If FORCE_LOCAL is true,
631 : create it as a local symbol, otherwise copy the symbol linkage and
632 : visibility attributes from OLD_NODE. */
633 :
634 : static struct cgraph_node *
635 7188 : simd_clone_create (struct cgraph_node *old_node, bool force_local)
636 : {
637 7188 : struct cgraph_node *new_node;
638 7188 : if (old_node->definition)
639 : {
640 4472 : if (!old_node->has_gimple_body_p ())
641 : return NULL;
642 4472 : old_node->get_body ();
643 4472 : new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL,
644 : NULL, NULL,
645 : "simdclone");
646 : }
647 : else
648 : {
649 2716 : tree old_decl = old_node->decl;
650 2716 : tree new_decl = copy_node (old_node->decl);
651 2716 : DECL_NAME (new_decl) = clone_function_name_numbered (old_decl,
652 : "simdclone");
653 2716 : SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
654 2716 : SET_DECL_RTL (new_decl, NULL);
655 2716 : DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
656 2716 : DECL_STATIC_DESTRUCTOR (new_decl) = 0;
657 2716 : new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
658 2716 : if (old_node->in_other_partition)
659 16 : new_node->in_other_partition = 1;
660 : }
661 7188 : if (new_node == NULL)
662 : return new_node;
663 :
664 7188 : set_decl_built_in_function (new_node->decl, NOT_BUILT_IN, 0);
665 7188 : if (force_local)
666 : {
667 8 : TREE_PUBLIC (new_node->decl) = 0;
668 8 : DECL_COMDAT (new_node->decl) = 0;
669 8 : DECL_WEAK (new_node->decl) = 0;
670 8 : DECL_EXTERNAL (new_node->decl) = 0;
671 8 : DECL_VISIBILITY_SPECIFIED (new_node->decl) = 0;
672 8 : DECL_VISIBILITY (new_node->decl) = VISIBILITY_DEFAULT;
673 8 : DECL_DLLIMPORT_P (new_node->decl) = 0;
674 : }
675 : else
676 : {
677 7180 : TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
678 7180 : DECL_COMDAT (new_node->decl) = DECL_COMDAT (old_node->decl);
679 7180 : DECL_WEAK (new_node->decl) = DECL_WEAK (old_node->decl);
680 7180 : DECL_EXTERNAL (new_node->decl) = DECL_EXTERNAL (old_node->decl);
681 14360 : DECL_VISIBILITY_SPECIFIED (new_node->decl)
682 7180 : = DECL_VISIBILITY_SPECIFIED (old_node->decl);
683 7180 : DECL_VISIBILITY (new_node->decl) = DECL_VISIBILITY (old_node->decl);
684 7180 : DECL_DLLIMPORT_P (new_node->decl) = DECL_DLLIMPORT_P (old_node->decl);
685 7180 : if (DECL_ONE_ONLY (old_node->decl))
686 300 : make_decl_one_only (new_node->decl,
687 : DECL_ASSEMBLER_NAME (new_node->decl));
688 :
689 : /* The method cgraph_version_clone_with_body () will force the new
690 : symbol local. Undo this, and inherit external visibility from
691 : the old node. */
692 7180 : new_node->local = old_node->local;
693 7180 : new_node->externally_visible = old_node->externally_visible;
694 7180 : new_node->has_omp_variant_constructs
695 7180 : = old_node->has_omp_variant_constructs;
696 : }
697 :
698 : /* Mark clones with internal linkage as gc'able, so they will not be
699 : emitted unless the vectorizer can actually use them. */
700 7188 : if (!TREE_PUBLIC (new_node->decl))
701 112 : new_node->gc_candidate = true;
702 :
703 : return new_node;
704 : }
705 :
706 : /* Adjust the return type of the given function to its appropriate
707 : vector counterpart. */
708 :
709 : static void
710 7188 : simd_clone_adjust_return_type (struct cgraph_node *node)
711 : {
712 7188 : tree fndecl = node->decl;
713 7188 : tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
714 7188 : poly_uint64 veclen;
715 7188 : tree t;
716 :
717 : /* Adjust the function return type. */
718 7188 : if (orig_rettype == void_type_node)
719 752 : return;
720 6436 : t = TREE_TYPE (TREE_TYPE (fndecl));
721 6436 : if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t))
722 3115 : veclen = node->simdclone->vecsize_int;
723 : else
724 3321 : veclen = node->simdclone->vecsize_float;
725 6436 : if (known_eq (veclen, 0U))
726 0 : veclen = node->simdclone->simdlen;
727 : else
728 12872 : veclen = exact_div (veclen, GET_MODE_BITSIZE (SCALAR_TYPE_MODE (t)));
729 6436 : if (multiple_p (veclen, node->simdclone->simdlen))
730 6188 : veclen = node->simdclone->simdlen;
731 6436 : if (POINTER_TYPE_P (t))
732 18 : t = pointer_sized_int_node;
733 6436 : if (known_eq (veclen, node->simdclone->simdlen))
734 6188 : t = build_vector_type (t, node->simdclone->simdlen);
735 : else
736 : {
737 248 : t = build_vector_type (t, veclen);
738 248 : t = build_array_type_nelts (t, exact_div (node->simdclone->simdlen,
739 : veclen));
740 : }
741 6436 : TREE_TYPE (TREE_TYPE (fndecl)) = t;
742 : }
743 :
744 : /* Each vector argument has a corresponding array to be used locally
745 : as part of the eventual loop. Create such temporary array and
746 : return it.
747 :
748 : PREFIX is the prefix to be used for the temporary.
749 :
750 : TYPE is the inner element type.
751 :
752 : SIMDLEN is the number of elements. */
753 :
754 : static tree
755 6133 : create_tmp_simd_array (const char *prefix, tree type, poly_uint64 simdlen)
756 : {
757 6133 : tree atype = build_array_type_nelts (type, simdlen);
758 6133 : tree avar = create_tmp_var_raw (atype, prefix);
759 6133 : gimple_add_tmp_var (avar);
760 6133 : return avar;
761 : }
762 :
763 : /* Modify the function argument types to their corresponding vector
764 : counterparts if appropriate. Also, create one array for each simd
765 : argument to be used locally when using the function arguments as
766 : part of the loop.
767 :
768 : NODE is the function whose arguments are to be adjusted.
769 :
770 : If NODE does not represent function definition, returns NULL. Otherwise
771 : returns an adjustment class that will be filled describing how the argument
772 : declarations will be remapped. New arguments which are not to be remapped
773 : are marked with USER_FLAG. */
774 :
775 : static void
776 7188 : simd_clone_adjust_argument_types (struct cgraph_node *node)
777 : {
778 7188 : auto_vec<tree> args;
779 :
780 7188 : if (node->definition)
781 4472 : push_function_arg_decls (&args, node->decl);
782 : else
783 2716 : simd_clone_vector_of_formal_parm_types (&args, node->decl);
784 7188 : struct cgraph_simd_clone *sc = node->simdclone;
785 7188 : unsigned i, k;
786 7188 : poly_uint64 veclen;
787 7188 : auto_vec<tree> new_params;
788 :
789 19377 : for (i = 0; i < sc->nargs; ++i)
790 : {
791 12189 : tree parm = NULL_TREE;
792 12189 : tree parm_type = NULL_TREE;
793 12189 : if (i < args.length())
794 : {
795 12189 : parm = args[i];
796 12189 : parm_type = node->definition ? TREE_TYPE (parm) : parm;
797 : }
798 :
799 12189 : sc->args[i].orig_arg = node->definition ? parm : NULL_TREE;
800 12189 : sc->args[i].orig_type = parm_type;
801 :
802 12189 : switch (sc->args[i].arg_type)
803 : {
804 4138 : default:
805 4138 : new_params.safe_push (parm_type);
806 8448 : break;
807 172 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
808 172 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
809 172 : new_params.safe_push (parm_type);
810 172 : if (node->definition)
811 172 : sc->args[i].simd_array
812 172 : = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
813 172 : TREE_TYPE (parm_type),
814 : sc->simdlen);
815 : break;
816 7879 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
817 7879 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
818 7879 : case SIMD_CLONE_ARG_TYPE_VECTOR:
819 7879 : if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
820 3645 : veclen = sc->vecsize_int;
821 : else
822 4234 : veclen = sc->vecsize_float;
823 7879 : if (known_eq (veclen, 0U))
824 0 : veclen = sc->simdlen;
825 : else
826 7879 : veclen
827 7879 : = exact_div (veclen,
828 15758 : GET_MODE_BITSIZE (SCALAR_TYPE_MODE (parm_type)));
829 7879 : if (multiple_p (veclen, sc->simdlen))
830 6315 : veclen = sc->simdlen;
831 7879 : tree vtype;
832 7879 : if (POINTER_TYPE_P (parm_type))
833 842 : vtype = build_vector_type (pointer_sized_int_node, veclen);
834 : else
835 7037 : vtype = build_vector_type (parm_type, veclen);
836 7879 : sc->args[i].vector_type = vtype;
837 7879 : k = vector_unroll_factor (sc->simdlen, veclen);
838 17832 : for (unsigned j = 0; j < k; j++)
839 9953 : new_params.safe_push (vtype);
840 :
841 7879 : if (node->definition)
842 4471 : sc->args[i].simd_array
843 8930 : = create_tmp_simd_array (DECL_NAME (parm)
844 4459 : ? IDENTIFIER_POINTER (DECL_NAME (parm))
845 : : NULL, parm_type, sc->simdlen);
846 : }
847 : }
848 :
849 7188 : if (sc->inbranch)
850 : {
851 2243 : tree base_type = simd_clone_compute_base_data_type (sc->origin, sc);
852 2243 : tree mask_type;
853 2243 : if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
854 1617 : veclen = sc->vecsize_int;
855 : else
856 626 : veclen = sc->vecsize_float;
857 2243 : if (known_eq (veclen, 0U))
858 0 : veclen = sc->simdlen;
859 : else
860 4486 : veclen = exact_div (veclen,
861 4486 : GET_MODE_BITSIZE (SCALAR_TYPE_MODE (base_type)));
862 2243 : if (multiple_p (veclen, sc->simdlen))
863 2122 : veclen = sc->simdlen;
864 2243 : if (sc->mask_mode != VOIDmode)
865 550 : mask_type
866 550 : = lang_hooks.types.type_for_mode (sc->mask_mode, 1);
867 1693 : else if (POINTER_TYPE_P (base_type))
868 19 : mask_type = build_vector_type (pointer_sized_int_node, veclen);
869 : else
870 1674 : mask_type = build_vector_type (base_type, veclen);
871 :
872 2243 : k = vector_unroll_factor (sc->simdlen, veclen);
873 :
874 : /* We have previously allocated one extra entry for the mask. Use
875 : it and fill it. */
876 2243 : sc->nargs++;
877 2243 : if (sc->mask_mode != VOIDmode)
878 550 : base_type = boolean_type_node;
879 2243 : if (node->definition)
880 : {
881 1971 : sc->args[i].orig_arg
882 1971 : = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
883 1971 : if (sc->mask_mode == VOIDmode)
884 1489 : sc->args[i].simd_array
885 1489 : = create_tmp_simd_array ("mask", base_type, sc->simdlen);
886 482 : else if (k > 1)
887 1 : sc->args[i].simd_array
888 1 : = create_tmp_simd_array ("mask", mask_type, k);
889 : else
890 481 : sc->args[i].simd_array = NULL_TREE;
891 : }
892 2243 : sc->args[i].orig_type = base_type;
893 2243 : sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
894 2243 : sc->args[i].vector_type = mask_type;
895 : /* Record the number of mask copies when that is difficult to
896 : compute. */
897 2243 : if (sc->mask_mode != VOIDmode)
898 550 : sc->args[i].linear_step = k;
899 : }
900 :
901 7188 : if (!node->definition)
902 : {
903 2716 : tree new_arg_types = NULL_TREE, new_reversed;
904 2716 : bool last_parm_void = false;
905 5432 : if (args.length () > 0 && args.last () == void_type_node)
906 : last_parm_void = true;
907 :
908 2716 : gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
909 6879 : for (i = 0; i < new_params.length (); i++)
910 4163 : new_arg_types = tree_cons (NULL_TREE, new_params[i], new_arg_types);
911 2716 : new_reversed = nreverse (new_arg_types);
912 2716 : if (last_parm_void)
913 : {
914 2716 : if (new_reversed)
915 2716 : TREE_CHAIN (new_arg_types) = void_list_node;
916 : else
917 0 : new_reversed = void_list_node;
918 : }
919 2716 : TYPE_ARG_TYPES (TREE_TYPE (node->decl)) = new_reversed;
920 : }
921 7188 : }
922 :
923 : /* Initialize and copy the function arguments in NODE to their
924 : corresponding local simd arrays. Returns a fresh gimple_seq with
925 : the instruction sequence generated. */
926 :
927 : static gimple_seq
928 4472 : simd_clone_init_simd_arrays (struct cgraph_node *node,
929 : ipa_param_body_adjustments *adjustments)
930 : {
931 4472 : gimple_seq seq = NULL;
932 4472 : unsigned i = 0, j = 0, k;
933 :
934 4472 : for (tree arg = DECL_ARGUMENTS (node->decl);
935 14812 : arg;
936 10340 : arg = DECL_CHAIN (arg), i++, j++)
937 : {
938 10340 : ipa_adjusted_param adj = (*adjustments->m_adj_params)[j];
939 14238 : if (adj.op == IPA_PARAM_OP_COPY
940 10340 : || POINTER_TYPE_P (TREE_TYPE (arg)))
941 8278 : continue;
942 :
943 6442 : node->simdclone->args[i].vector_arg = arg;
944 :
945 6442 : tree array = node->simdclone->args[i].simd_array;
946 6442 : if (node->simdclone->mask_mode != VOIDmode
947 1554 : && adj.param_prefix_index == IPA_PARAM_PREFIX_MASK)
948 : {
949 482 : if (array == NULL_TREE)
950 481 : continue;
951 1 : unsigned int l
952 1 : = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array))));
953 5 : for (k = 0; k <= l; k++)
954 : {
955 4 : if (k)
956 : {
957 3 : arg = DECL_CHAIN (arg);
958 3 : j++;
959 : }
960 4 : tree t = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (array)),
961 4 : array, size_int (k), NULL, NULL);
962 4 : t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
963 4 : gimplify_and_add (t, &seq);
964 : }
965 1 : continue;
966 1 : }
967 5960 : if (!VECTOR_TYPE_P (TREE_TYPE (arg))
968 5960 : || known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)),
969 : node->simdclone->simdlen))
970 : {
971 4515 : tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
972 4515 : tree ptr = build_fold_addr_expr (array);
973 4515 : tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
974 : build_int_cst (ptype, 0));
975 4515 : t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
976 4515 : gimplify_and_add (t, &seq);
977 : }
978 : else
979 : {
980 1445 : poly_uint64 simdlen = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg));
981 1445 : unsigned int times = vector_unroll_factor (node->simdclone->simdlen,
982 : simdlen);
983 1445 : tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
984 4765 : for (k = 0; k < times; k++)
985 : {
986 3320 : tree ptr = build_fold_addr_expr (array);
987 3320 : int elemsize;
988 3320 : if (k)
989 : {
990 1875 : arg = DECL_CHAIN (arg);
991 1875 : j++;
992 : }
993 3320 : tree elemtype = TREE_TYPE (TREE_TYPE (arg));
994 3320 : elemsize = GET_MODE_SIZE (SCALAR_TYPE_MODE (elemtype));
995 3320 : tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
996 3320 : build_int_cst (ptype, k * elemsize * simdlen));
997 3320 : t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
998 3320 : gimplify_and_add (t, &seq);
999 : }
1000 : }
1001 : }
1002 4472 : return seq;
1003 : }
1004 :
1005 : /* Callback info for ipa_simd_modify_stmt_ops below. */
1006 :
1007 : struct modify_stmt_info {
1008 : ipa_param_body_adjustments *adjustments;
1009 : gimple *stmt;
1010 : gimple *after_stmt;
1011 : /* True if the parent statement was modified by
1012 : ipa_simd_modify_stmt_ops. */
1013 : bool modified;
1014 : };
1015 :
1016 : /* Callback for walk_gimple_op.
1017 :
1018 : Adjust operands from a given statement as specified in the
1019 : adjustments vector in the callback data. */
1020 :
1021 : static tree
1022 130909 : ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
1023 : {
1024 130909 : struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
1025 130909 : struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
1026 130909 : tree *orig_tp = tp;
1027 130909 : if (TREE_CODE (*tp) == ADDR_EXPR)
1028 9108 : tp = &TREE_OPERAND (*tp, 0);
1029 :
1030 130909 : if (TREE_CODE (*tp) == BIT_FIELD_REF
1031 130909 : || TREE_CODE (*tp) == IMAGPART_EXPR
1032 130909 : || TREE_CODE (*tp) == REALPART_EXPR)
1033 0 : tp = &TREE_OPERAND (*tp, 0);
1034 :
1035 130909 : tree repl = NULL_TREE;
1036 130909 : ipa_param_body_replacement *pbr = NULL;
1037 :
1038 130909 : if (TREE_CODE (*tp) == PARM_DECL)
1039 : {
1040 2084 : pbr = info->adjustments->get_expr_replacement (*tp, true);
1041 2084 : if (pbr)
1042 1304 : repl = pbr->repl;
1043 : }
1044 128825 : else if (TYPE_P (*tp))
1045 0 : *walk_subtrees = 0;
1046 :
1047 1304 : if (repl)
1048 1304 : repl = unshare_expr (repl);
1049 : else
1050 : {
1051 129605 : if (tp != orig_tp)
1052 : {
1053 8376 : *walk_subtrees = 0;
1054 8376 : bool modified = info->modified;
1055 8376 : info->modified = false;
1056 8376 : walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
1057 8376 : if (!info->modified)
1058 : {
1059 8192 : info->modified = modified;
1060 8192 : return NULL_TREE;
1061 : }
1062 184 : info->modified = modified;
1063 184 : repl = *tp;
1064 : }
1065 : else
1066 : return NULL_TREE;
1067 : }
1068 :
1069 1488 : if (tp != orig_tp)
1070 : {
1071 916 : if (gimple_code (info->stmt) == GIMPLE_PHI
1072 48 : && pbr
1073 40 : && TREE_CODE (*orig_tp) == ADDR_EXPR
1074 40 : && TREE_CODE (TREE_OPERAND (*orig_tp, 0)) == PARM_DECL
1075 956 : && pbr->dummy)
1076 : {
1077 0 : gcc_assert (TREE_CODE (pbr->dummy) == SSA_NAME);
1078 0 : *orig_tp = pbr->dummy;
1079 0 : info->modified = true;
1080 0 : return NULL_TREE;
1081 : }
1082 :
1083 916 : repl = build_fold_addr_expr (repl);
1084 916 : gimple *stmt;
1085 916 : if (is_gimple_debug (info->stmt))
1086 : {
1087 88 : tree vexpr = build_debug_expr_decl (TREE_TYPE (repl));
1088 88 : stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
1089 88 : repl = vexpr;
1090 : }
1091 : else
1092 : {
1093 828 : stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
1094 828 : repl = gimple_assign_lhs (stmt);
1095 : }
1096 916 : gimple_stmt_iterator gsi;
1097 916 : if (gimple_code (info->stmt) == GIMPLE_PHI)
1098 : {
1099 48 : if (info->after_stmt)
1100 8 : gsi = gsi_for_stmt (info->after_stmt);
1101 : else
1102 40 : gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
1103 : /* Cache SSA_NAME for next time. */
1104 48 : if (pbr
1105 40 : && TREE_CODE (*orig_tp) == ADDR_EXPR
1106 88 : && TREE_CODE (TREE_OPERAND (*orig_tp, 0)) == PARM_DECL)
1107 : {
1108 40 : gcc_assert (!pbr->dummy);
1109 40 : pbr->dummy = repl;
1110 : }
1111 : }
1112 : else
1113 868 : gsi = gsi_for_stmt (info->stmt);
1114 916 : if (info->after_stmt)
1115 8 : gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
1116 : else
1117 908 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
1118 916 : if (gimple_code (info->stmt) == GIMPLE_PHI)
1119 48 : info->after_stmt = stmt;
1120 916 : *orig_tp = repl;
1121 : }
1122 572 : else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
1123 : {
1124 0 : tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
1125 0 : *tp = vce;
1126 : }
1127 : else
1128 572 : *tp = repl;
1129 :
1130 1488 : info->modified = true;
1131 1488 : return NULL_TREE;
1132 : }
1133 :
1134 : /* Traverse the function body and perform all modifications as
1135 : described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
1136 : modified such that the replacement/reduction value will now be an
1137 : offset into the corresponding simd_array.
1138 :
1139 : This function will replace all function argument uses with their
1140 : corresponding simd array elements, and ajust the return values
1141 : accordingly. */
1142 :
1143 : static void
1144 4472 : ipa_simd_modify_function_body (struct cgraph_node *node,
1145 : ipa_param_body_adjustments *adjustments,
1146 : tree retval_array, tree iter)
1147 : {
1148 4472 : basic_block bb;
1149 4472 : unsigned int i, j;
1150 :
1151 :
1152 : /* Register replacements for every function argument use to an offset into
1153 : the corresponding simd_array. */
1154 14812 : for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
1155 : {
1156 16209 : if (!node->simdclone->args[i].vector_arg
1157 10340 : || (*adjustments->m_adj_params)[j].user_flag)
1158 5869 : continue;
1159 :
1160 4471 : tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
1161 4471 : tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
1162 4471 : tree r = build4 (ARRAY_REF, basetype, node->simdclone->args[i].simd_array,
1163 : iter, NULL_TREE, NULL_TREE);
1164 4471 : adjustments->register_replacement (&(*adjustments->m_adj_params)[j], r);
1165 :
1166 4471 : if (multiple_p (node->simdclone->simdlen, TYPE_VECTOR_SUBPARTS (vectype)))
1167 4471 : j += vector_unroll_factor (node->simdclone->simdlen,
1168 4471 : TYPE_VECTOR_SUBPARTS (vectype)) - 1;
1169 : }
1170 4472 : adjustments->sort_replacements ();
1171 :
1172 4472 : tree name;
1173 45983 : FOR_EACH_SSA_NAME (i, name, cfun)
1174 : {
1175 41511 : tree base_var;
1176 41511 : if (SSA_NAME_VAR (name)
1177 24001 : && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL
1178 7937 : && (base_var
1179 7937 : = adjustments->get_replacement_ssa_base (SSA_NAME_VAR (name))))
1180 : {
1181 4483 : if (SSA_NAME_IS_DEFAULT_DEF (name))
1182 : {
1183 4051 : tree old_decl = SSA_NAME_VAR (name);
1184 4051 : bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1185 4051 : gimple_stmt_iterator gsi = gsi_after_labels (bb);
1186 4051 : tree repl = adjustments->lookup_replacement (old_decl, 0);
1187 4051 : gcc_checking_assert (repl);
1188 4051 : repl = unshare_expr (repl);
1189 4051 : set_ssa_default_def (cfun, old_decl, NULL_TREE);
1190 4051 : SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
1191 4051 : SSA_NAME_IS_DEFAULT_DEF (name) = 0;
1192 4051 : gimple *stmt = gimple_build_assign (name, repl);
1193 4051 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
1194 : }
1195 : else
1196 864 : SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
1197 : }
1198 : }
1199 :
1200 4472 : struct modify_stmt_info info;
1201 4472 : info.adjustments = adjustments;
1202 :
1203 19670 : FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
1204 : {
1205 15198 : gimple_stmt_iterator gsi;
1206 :
1207 17902 : for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1208 : {
1209 2704 : gphi *phi = as_a <gphi *> (gsi_stmt (gsi));
1210 2704 : int i, n = gimple_phi_num_args (phi);
1211 2704 : info.stmt = phi;
1212 2704 : info.after_stmt = NULL;
1213 2704 : struct walk_stmt_info wi;
1214 2704 : memset (&wi, 0, sizeof (wi));
1215 2704 : info.modified = false;
1216 2704 : wi.info = &info;
1217 9004 : for (i = 0; i < n; ++i)
1218 : {
1219 6300 : int walk_subtrees = 1;
1220 6300 : tree arg = gimple_phi_arg_def (phi, i);
1221 6300 : tree op = arg;
1222 6300 : ipa_simd_modify_stmt_ops (&op, &walk_subtrees, &wi);
1223 6300 : if (op != arg)
1224 : {
1225 40 : SET_PHI_ARG_DEF (phi, i, op);
1226 40 : gcc_assert (TREE_CODE (op) == SSA_NAME);
1227 40 : if (gimple_phi_arg_edge (phi, i)->flags & EDGE_ABNORMAL)
1228 0 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op) = 1;
1229 : }
1230 : }
1231 : }
1232 :
1233 15198 : gsi = gsi_start_bb (bb);
1234 62531 : while (!gsi_end_p (gsi))
1235 : {
1236 47333 : gimple *stmt = gsi_stmt (gsi);
1237 47333 : info.stmt = stmt;
1238 47333 : info.after_stmt = NULL;
1239 47333 : struct walk_stmt_info wi;
1240 :
1241 47333 : memset (&wi, 0, sizeof (wi));
1242 47333 : info.modified = false;
1243 47333 : wi.info = &info;
1244 47333 : walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
1245 :
1246 47333 : if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
1247 : {
1248 4392 : tree retval = gimple_return_retval (return_stmt);
1249 4392 : edge e = find_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun));
1250 4392 : e->flags |= EDGE_FALLTHRU;
1251 4392 : if (!retval)
1252 : {
1253 720 : gsi_remove (&gsi, true);
1254 744 : continue;
1255 : }
1256 :
1257 : /* Replace `return foo' with `retval_array[iter] = foo'. */
1258 3672 : tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
1259 : retval_array, iter, NULL, NULL);
1260 3672 : stmt = gimple_build_assign (ref, retval);
1261 3672 : gsi_replace (&gsi, stmt, true);
1262 3672 : info.modified = true;
1263 : }
1264 :
1265 46613 : if (info.modified)
1266 : {
1267 4792 : update_stmt (stmt);
1268 : /* If the above changed the var of a debug bind into something
1269 : different, remove the debug stmt. We could also for all the
1270 : replaced parameters add VAR_DECLs for debug info purposes,
1271 : add debug stmts for those to be the simd array accesses and
1272 : replace debug stmt var operand with that var. Debugging of
1273 : vectorized loops doesn't work too well, so don't bother for
1274 : now. */
1275 4848 : if ((gimple_debug_bind_p (stmt)
1276 80 : && !DECL_P (gimple_debug_bind_get_var (stmt)))
1277 136 : || (gimple_debug_source_bind_p (stmt)
1278 0 : && !DECL_P (gimple_debug_source_bind_get_var (stmt))))
1279 : {
1280 24 : gsi_remove (&gsi, true);
1281 24 : continue;
1282 : }
1283 4768 : if (maybe_clean_eh_stmt (stmt))
1284 0 : gimple_purge_dead_eh_edges (gimple_bb (stmt));
1285 : }
1286 46589 : gsi_next (&gsi);
1287 : }
1288 : }
1289 4472 : }
1290 :
1291 : /* Helper function of simd_clone_adjust, return linear step addend
1292 : of Ith argument. */
1293 :
1294 : static tree
1295 2071 : simd_clone_linear_addend (struct cgraph_node *node, unsigned int i,
1296 : tree addtype, basic_block entry_bb)
1297 : {
1298 2071 : tree ptype = NULL_TREE;
1299 2071 : switch (node->simdclone->args[i].arg_type)
1300 : {
1301 1919 : case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
1302 1919 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
1303 1919 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
1304 1919 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
1305 1919 : return build_int_cst (addtype, node->simdclone->args[i].linear_step);
1306 140 : case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
1307 140 : case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
1308 140 : ptype = TREE_TYPE (node->simdclone->args[i].orig_arg);
1309 140 : break;
1310 12 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
1311 12 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
1312 12 : ptype = TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg));
1313 12 : break;
1314 0 : default:
1315 0 : gcc_unreachable ();
1316 : }
1317 :
1318 152 : unsigned int idx = node->simdclone->args[i].linear_step;
1319 152 : tree arg = node->simdclone->args[idx].orig_arg;
1320 152 : gcc_assert (is_gimple_reg_type (TREE_TYPE (arg)));
1321 152 : gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1322 152 : gimple *g;
1323 152 : tree ret;
1324 152 : if (is_gimple_reg (arg))
1325 152 : ret = get_or_create_ssa_default_def (cfun, arg);
1326 : else
1327 : {
1328 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (arg)), arg);
1329 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1330 0 : ret = gimple_assign_lhs (g);
1331 : }
1332 152 : if (TREE_CODE (TREE_TYPE (arg)) == REFERENCE_TYPE)
1333 : {
1334 28 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg))),
1335 : build_simple_mem_ref (ret));
1336 28 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1337 28 : ret = gimple_assign_lhs (g);
1338 : }
1339 152 : if (!useless_type_conversion_p (addtype, TREE_TYPE (ret)))
1340 : {
1341 60 : g = gimple_build_assign (make_ssa_name (addtype), NOP_EXPR, ret);
1342 60 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1343 60 : ret = gimple_assign_lhs (g);
1344 : }
1345 152 : if (POINTER_TYPE_P (ptype))
1346 : {
1347 60 : tree size = TYPE_SIZE_UNIT (TREE_TYPE (ptype));
1348 60 : if (size && TREE_CODE (size) == INTEGER_CST)
1349 : {
1350 60 : g = gimple_build_assign (make_ssa_name (addtype), MULT_EXPR,
1351 : ret, fold_convert (addtype, size));
1352 60 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1353 60 : ret = gimple_assign_lhs (g);
1354 : }
1355 : }
1356 : return ret;
1357 : }
1358 :
1359 : /* Adjust the argument types in NODE to their appropriate vector
1360 : counterparts. */
1361 :
1362 : static void
1363 4472 : simd_clone_adjust (struct cgraph_node *node)
1364 : {
1365 4472 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
1366 :
1367 4472 : tree orig_rettype = TREE_TYPE (TREE_TYPE (node->decl));
1368 4472 : TREE_TYPE (node->decl) = build_distinct_type_copy (TREE_TYPE (node->decl));
1369 4472 : simd_clone_adjust_return_type (node);
1370 4472 : simd_clone_adjust_argument_types (node);
1371 4472 : targetm.simd_clone.adjust (node);
1372 4472 : tree retval = NULL_TREE;
1373 4472 : if (orig_rettype != void_type_node)
1374 : {
1375 3728 : poly_uint64 veclen;
1376 3728 : if (INTEGRAL_TYPE_P (orig_rettype) || POINTER_TYPE_P (orig_rettype))
1377 2819 : veclen = node->simdclone->vecsize_int;
1378 : else
1379 909 : veclen = node->simdclone->vecsize_float;
1380 3728 : if (known_eq (veclen, 0U))
1381 0 : veclen = node->simdclone->simdlen;
1382 : else
1383 7456 : veclen = exact_div (veclen,
1384 7456 : GET_MODE_BITSIZE (SCALAR_TYPE_MODE (orig_rettype)));
1385 3728 : if (multiple_p (veclen, node->simdclone->simdlen))
1386 3521 : veclen = node->simdclone->simdlen;
1387 :
1388 3728 : retval = DECL_RESULT (node->decl);
1389 : /* Adjust the DECL_RESULT. */
1390 3728 : TREE_TYPE (retval) = TREE_TYPE (TREE_TYPE (node->decl));
1391 3728 : relayout_decl (retval);
1392 :
1393 7456 : tree atype = build_array_type_nelts (orig_rettype,
1394 3728 : node->simdclone->simdlen);
1395 3728 : if (maybe_ne (veclen, node->simdclone->simdlen))
1396 207 : retval = build1 (VIEW_CONVERT_EXPR, atype, retval);
1397 : else
1398 : {
1399 : /* Set up a SIMD array to use as the return value. */
1400 3521 : retval = create_tmp_var_raw (atype, "retval");
1401 3521 : gimple_add_tmp_var (retval);
1402 : }
1403 : }
1404 :
1405 4472 : struct cgraph_simd_clone *sc = node->simdclone;
1406 4472 : vec<ipa_adjusted_param, va_gc> *new_params = NULL;
1407 4472 : vec_safe_reserve (new_params, sc->nargs);
1408 4472 : unsigned i, j, k;
1409 14812 : for (i = 0; i < sc->nargs; ++i)
1410 : {
1411 10340 : ipa_adjusted_param adj;
1412 10340 : memset (&adj, 0, sizeof (adj));
1413 10340 : poly_uint64 veclen;
1414 10340 : tree elem_type;
1415 :
1416 10340 : adj.base_index = i;
1417 10340 : adj.prev_clone_index = i;
1418 10340 : switch (sc->args[i].arg_type)
1419 : {
1420 3886 : default:
1421 : /* No adjustment necessary for scalar arguments. */
1422 3886 : adj.op = IPA_PARAM_OP_COPY;
1423 3886 : break;
1424 12 : case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
1425 12 : adj.op = IPA_PARAM_OP_COPY;
1426 12 : break;
1427 6442 : case SIMD_CLONE_ARG_TYPE_MASK:
1428 6442 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
1429 6442 : case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
1430 6442 : case SIMD_CLONE_ARG_TYPE_VECTOR:
1431 6442 : if (sc->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK
1432 1971 : && sc->mask_mode != VOIDmode)
1433 482 : elem_type = simd_clone_compute_base_data_type (sc->origin, sc);
1434 : else
1435 5960 : elem_type = TREE_TYPE (sc->args[i].vector_type);
1436 6442 : if (INTEGRAL_TYPE_P (elem_type) || POINTER_TYPE_P (elem_type))
1437 4738 : veclen = sc->vecsize_int;
1438 : else
1439 1704 : veclen = sc->vecsize_float;
1440 6442 : if (known_eq (veclen, 0U))
1441 0 : veclen = sc->simdlen;
1442 : else
1443 6442 : veclen
1444 6442 : = exact_div (veclen,
1445 12884 : GET_MODE_BITSIZE (SCALAR_TYPE_MODE (elem_type)));
1446 6442 : if (multiple_p (veclen, sc->simdlen))
1447 4996 : veclen = sc->simdlen;
1448 6442 : if (sc->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK)
1449 : {
1450 1971 : adj.user_flag = 1;
1451 1971 : adj.param_prefix_index = IPA_PARAM_PREFIX_MASK;
1452 : }
1453 : else
1454 4471 : adj.param_prefix_index = IPA_PARAM_PREFIX_SIMD;
1455 6442 : adj.op = IPA_PARAM_OP_NEW;
1456 6442 : adj.type = sc->args[i].vector_type;
1457 6442 : k = vector_unroll_factor (sc->simdlen, veclen);
1458 8320 : for (j = 1; j < k; j++)
1459 : {
1460 1878 : vec_safe_push (new_params, adj);
1461 1878 : if (j == 1)
1462 : {
1463 1446 : memset (&adj, 0, sizeof (adj));
1464 1446 : adj.op = IPA_PARAM_OP_NEW;
1465 1446 : adj.user_flag = 1;
1466 1446 : if (sc->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK)
1467 95 : adj.param_prefix_index = IPA_PARAM_PREFIX_MASK;
1468 : else
1469 1351 : adj.param_prefix_index = IPA_PARAM_PREFIX_SIMD;
1470 1446 : adj.base_index = i;
1471 1446 : adj.prev_clone_index = i;
1472 1446 : adj.type = sc->args[i].vector_type;
1473 : }
1474 : }
1475 : }
1476 10340 : vec_safe_push (new_params, adj);
1477 : }
1478 4472 : ipa_param_body_adjustments *adjustments
1479 4472 : = new ipa_param_body_adjustments (new_params, node->decl);
1480 4472 : adjustments->modify_formal_parameters ();
1481 :
1482 4472 : push_gimplify_context ();
1483 :
1484 4472 : gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
1485 :
1486 : /* Adjust all uses of vector arguments accordingly. Adjust all
1487 : return values accordingly. */
1488 4472 : tree iter = create_tmp_var (unsigned_type_node, "iter");
1489 4472 : tree iter1 = make_ssa_name (iter);
1490 4472 : tree iter2 = NULL_TREE;
1491 4472 : ipa_simd_modify_function_body (node, adjustments, retval, iter1);
1492 4472 : delete adjustments;
1493 :
1494 : /* Initialize the iteration variable. */
1495 4472 : basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1496 4472 : basic_block body_bb = split_block_after_labels (entry_bb)->dest;
1497 4472 : gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1498 : /* Insert the SIMD array and iv initialization at function
1499 : entry. */
1500 4472 : gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
1501 :
1502 4472 : pop_gimplify_context (NULL);
1503 :
1504 4472 : gimple *g;
1505 4472 : basic_block incr_bb = NULL;
1506 4472 : class loop *loop = NULL;
1507 :
1508 : /* Create a new BB right before the original exit BB, to hold the
1509 : iteration increment and the condition/branch. */
1510 4472 : if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1511 : {
1512 4352 : basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
1513 4352 : incr_bb = create_empty_bb (orig_exit);
1514 4352 : incr_bb->count = profile_count::zero ();
1515 4352 : add_bb_to_loop (incr_bb, body_bb->loop_father);
1516 13096 : while (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1517 : {
1518 4392 : edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1519 4392 : redirect_edge_succ (e, incr_bb);
1520 4392 : incr_bb->count += e->count ();
1521 : }
1522 : }
1523 120 : else if (node->simdclone->inbranch)
1524 : {
1525 60 : incr_bb = create_empty_bb (entry_bb);
1526 60 : incr_bb->count = profile_count::zero ();
1527 60 : add_bb_to_loop (incr_bb, body_bb->loop_father);
1528 : }
1529 :
1530 60 : if (incr_bb)
1531 : {
1532 4412 : make_single_succ_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1533 4412 : gsi = gsi_last_bb (incr_bb);
1534 4412 : iter2 = make_ssa_name (iter);
1535 4412 : g = gimple_build_assign (iter2, PLUS_EXPR, iter1,
1536 : build_int_cst (unsigned_type_node, 1));
1537 4412 : gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1538 :
1539 : /* Mostly annotate the loop for the vectorizer (the rest is done
1540 : below). */
1541 4412 : loop = alloc_loop ();
1542 4412 : cfun->has_force_vectorize_loops = true;
1543 : /* We can assert that safelen is the 'minimum' simdlen. */
1544 4412 : loop->safelen = constant_lower_bound (node->simdclone->simdlen);
1545 4412 : loop->force_vectorize = true;
1546 4412 : loop->header = body_bb;
1547 : }
1548 :
1549 : /* Branch around the body if the mask applies. */
1550 4472 : if (node->simdclone->inbranch)
1551 : {
1552 1971 : gsi = gsi_last_bb (loop->header);
1553 1971 : tree mask_array
1554 1971 : = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
1555 1971 : tree mask;
1556 1971 : if (node->simdclone->mask_mode != VOIDmode)
1557 : {
1558 482 : tree shift_cnt;
1559 482 : if (mask_array == NULL_TREE)
1560 : {
1561 481 : tree arg = node->simdclone->args[node->simdclone->nargs
1562 : - 1].vector_arg;
1563 481 : mask = get_or_create_ssa_default_def (cfun, arg);
1564 481 : shift_cnt = iter1;
1565 : }
1566 : else
1567 : {
1568 1 : tree maskt = TREE_TYPE (mask_array);
1569 1 : int c = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt)));
1570 : /* For now, c must be constant here. */
1571 1 : c = exact_div (node->simdclone->simdlen, c + 1).to_constant ();
1572 1 : int s = exact_log2 (c);
1573 1 : gcc_assert (s > 0);
1574 1 : c--;
1575 1 : tree idx = make_ssa_name (TREE_TYPE (iter1));
1576 1 : g = gimple_build_assign (idx, RSHIFT_EXPR, iter1,
1577 1 : build_int_cst (NULL_TREE, s));
1578 1 : gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1579 1 : mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1580 1 : tree aref = build4 (ARRAY_REF,
1581 1 : TREE_TYPE (TREE_TYPE (mask_array)),
1582 : mask_array, idx, NULL, NULL);
1583 1 : g = gimple_build_assign (mask, aref);
1584 1 : gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1585 1 : shift_cnt = make_ssa_name (TREE_TYPE (iter1));
1586 1 : g = gimple_build_assign (shift_cnt, BIT_AND_EXPR, iter1,
1587 1 : build_int_cst (TREE_TYPE (iter1), c));
1588 1 : gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1589 : }
1590 482 : tree shift_cnt_conv = shift_cnt;
1591 482 : if (!useless_type_conversion_p (TREE_TYPE (mask),
1592 482 : TREE_TYPE (shift_cnt)))
1593 : {
1594 9 : shift_cnt_conv = make_ssa_name (TREE_TYPE (mask));
1595 9 : g = gimple_build_assign (shift_cnt_conv, NOP_EXPR, shift_cnt);
1596 9 : gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1597 : }
1598 482 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1599 : RSHIFT_EXPR, mask, shift_cnt_conv);
1600 482 : gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1601 482 : mask = gimple_assign_lhs (g);
1602 482 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1603 : BIT_AND_EXPR, mask,
1604 482 : build_one_cst (TREE_TYPE (mask)));
1605 482 : gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1606 482 : mask = gimple_assign_lhs (g);
1607 : }
1608 : else
1609 : {
1610 1489 : mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1611 1489 : tree aref = build4 (ARRAY_REF,
1612 1489 : TREE_TYPE (TREE_TYPE (mask_array)),
1613 : mask_array, iter1, NULL, NULL);
1614 1489 : g = gimple_build_assign (mask, aref);
1615 1489 : gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1616 1489 : int bitsize = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (aref)));
1617 1489 : if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
1618 : {
1619 364 : aref = build1 (VIEW_CONVERT_EXPR,
1620 : build_nonstandard_integer_type (bitsize, 0),
1621 : mask);
1622 364 : mask = make_ssa_name (TREE_TYPE (aref));
1623 364 : g = gimple_build_assign (mask, aref);
1624 364 : gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1625 : }
1626 : }
1627 :
1628 1971 : g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
1629 : NULL, NULL);
1630 1971 : gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1631 1971 : edge e = make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
1632 1971 : e->probability = profile_probability::unlikely ().guessed ();
1633 1971 : incr_bb->count += e->count ();
1634 1971 : edge fallthru = FALLTHRU_EDGE (loop->header);
1635 1971 : fallthru->flags = EDGE_FALSE_VALUE;
1636 1971 : fallthru->probability = profile_probability::likely ().guessed ();
1637 : }
1638 :
1639 4472 : basic_block latch_bb = NULL;
1640 4472 : basic_block new_exit_bb = NULL;
1641 :
1642 : /* Generate the condition. */
1643 4472 : if (incr_bb)
1644 : {
1645 4412 : gsi = gsi_last_bb (incr_bb);
1646 4412 : g = gimple_build_cond (LT_EXPR, iter2,
1647 : build_int_cst (unsigned_type_node,
1648 4412 : node->simdclone->simdlen),
1649 : NULL, NULL);
1650 4412 : gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1651 4412 : edge e = split_block (incr_bb, gsi_stmt (gsi));
1652 4412 : latch_bb = e->dest;
1653 4412 : new_exit_bb = split_block_after_labels (latch_bb)->dest;
1654 4412 : loop->latch = latch_bb;
1655 :
1656 4412 : redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
1657 :
1658 4412 : edge new_e = make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
1659 :
1660 : /* FIXME: Do we need to distribute probabilities for the conditional? */
1661 4412 : new_e->probability = profile_probability::guessed_never ();
1662 : /* The successor of incr_bb is already pointing to latch_bb; just
1663 : change the flags.
1664 : make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1665 4412 : FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
1666 : }
1667 :
1668 4472 : gphi *phi = create_phi_node (iter1, body_bb);
1669 4472 : edge preheader_edge = find_edge (entry_bb, body_bb);
1670 4472 : edge latch_edge = NULL;
1671 4472 : add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
1672 : UNKNOWN_LOCATION);
1673 4472 : if (incr_bb)
1674 : {
1675 4412 : latch_edge = single_succ_edge (latch_bb);
1676 4412 : add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1677 :
1678 : /* Generate the new return. */
1679 4412 : gsi = gsi_last_bb (new_exit_bb);
1680 4412 : if (retval
1681 3716 : && TREE_CODE (retval) == VIEW_CONVERT_EXPR
1682 4619 : && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
1683 207 : retval = TREE_OPERAND (retval, 0);
1684 4205 : else if (retval)
1685 : {
1686 3509 : retval = build1 (VIEW_CONVERT_EXPR,
1687 3509 : TREE_TYPE (TREE_TYPE (node->decl)),
1688 : retval);
1689 3509 : retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
1690 : false, GSI_CONTINUE_LINKING);
1691 : }
1692 4412 : g = gimple_build_return (retval);
1693 4412 : gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1694 : }
1695 :
1696 : /* Handle aligned clauses by replacing default defs of the aligned
1697 : uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1698 : lhs. Handle linear by adding PHIs. */
1699 14812 : for (unsigned i = 0; i < node->simdclone->nargs; i++)
1700 10340 : if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1701 10340 : && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg)
1702 1523 : || !is_gimple_reg_type
1703 1523 : (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1704 : {
1705 236 : tree orig_arg = node->simdclone->args[i].orig_arg;
1706 236 : if (is_gimple_reg_type (TREE_TYPE (orig_arg)))
1707 208 : iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1708 : else
1709 : {
1710 28 : iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg));
1711 28 : gimple_add_tmp_var (iter1);
1712 : }
1713 236 : gsi = gsi_after_labels (entry_bb);
1714 236 : g = gimple_build_assign (iter1, orig_arg);
1715 236 : gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1716 236 : gsi = gsi_after_labels (body_bb);
1717 236 : g = gimple_build_assign (orig_arg, iter1);
1718 236 : gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1719 : }
1720 10104 : else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1721 1495 : && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg)
1722 276 : && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1723 : == REFERENCE_TYPE
1724 10364 : && TREE_ADDRESSABLE
1725 : (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1726 : {
1727 52 : tree orig_arg = node->simdclone->args[i].orig_arg;
1728 52 : tree def = ssa_default_def (cfun, orig_arg);
1729 52 : if (def && !has_zero_uses (def))
1730 : {
1731 52 : iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg)));
1732 52 : gimple_add_tmp_var (iter1);
1733 52 : gsi = gsi_after_labels (entry_bb);
1734 52 : g = gimple_build_assign (iter1, build_simple_mem_ref (def));
1735 52 : gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1736 52 : gsi = gsi_after_labels (body_bb);
1737 52 : g = gimple_build_assign (build_simple_mem_ref (def), iter1);
1738 52 : gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1739 : }
1740 : }
1741 10052 : else if (node->simdclone->args[i].alignment
1742 364 : && node->simdclone->args[i].arg_type
1743 : == SIMD_CLONE_ARG_TYPE_UNIFORM
1744 192 : && (node->simdclone->args[i].alignment
1745 192 : & (node->simdclone->args[i].alignment - 1)) == 0
1746 10244 : && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1747 : == POINTER_TYPE)
1748 : {
1749 192 : unsigned int alignment = node->simdclone->args[i].alignment;
1750 192 : tree orig_arg = node->simdclone->args[i].orig_arg;
1751 192 : tree def = ssa_default_def (cfun, orig_arg);
1752 192 : if (def && !has_zero_uses (def))
1753 : {
1754 184 : tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
1755 184 : gimple_seq seq = NULL;
1756 184 : bool need_cvt = false;
1757 184 : gcall *call
1758 184 : = gimple_build_call (fn, 2, def, size_int (alignment));
1759 184 : g = call;
1760 184 : if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
1761 : ptr_type_node))
1762 0 : need_cvt = true;
1763 0 : tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg);
1764 184 : gimple_call_set_lhs (g, t);
1765 184 : gimple_seq_add_stmt_without_update (&seq, g);
1766 184 : if (need_cvt)
1767 : {
1768 0 : t = make_ssa_name (orig_arg);
1769 0 : g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g));
1770 0 : gimple_seq_add_stmt_without_update (&seq, g);
1771 : }
1772 184 : gsi_insert_seq_on_edge_immediate
1773 184 : (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
1774 :
1775 184 : entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1776 184 : node->create_edge (cgraph_node::get_create (fn),
1777 : call, entry_bb->count);
1778 :
1779 184 : imm_use_iterator iter;
1780 184 : use_operand_p use_p;
1781 184 : gimple *use_stmt;
1782 184 : tree repl = gimple_get_lhs (g);
1783 736 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1784 368 : if (is_gimple_debug (use_stmt) || use_stmt == call)
1785 184 : continue;
1786 : else
1787 552 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1788 368 : SET_USE (use_p, repl);
1789 : }
1790 : }
1791 9860 : else if ((node->simdclone->args[i].arg_type
1792 : == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
1793 8337 : || (node->simdclone->args[i].arg_type
1794 : == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
1795 8005 : || (node->simdclone->args[i].arg_type
1796 : == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
1797 7893 : || (node->simdclone->args[i].arg_type
1798 : == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP))
1799 : {
1800 1995 : tree orig_arg = node->simdclone->args[i].orig_arg;
1801 1995 : gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1802 : || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
1803 1995 : tree def = NULL_TREE;
1804 1995 : if (TREE_ADDRESSABLE (orig_arg))
1805 : {
1806 100 : def = make_ssa_name (TREE_TYPE (orig_arg));
1807 100 : iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1808 100 : if (incr_bb)
1809 88 : iter2 = make_ssa_name (TREE_TYPE (orig_arg));
1810 100 : gsi = gsi_after_labels (entry_bb);
1811 100 : g = gimple_build_assign (def, orig_arg);
1812 100 : gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1813 : }
1814 : else
1815 : {
1816 1895 : def = ssa_default_def (cfun, orig_arg);
1817 1895 : if (!def || has_zero_uses (def))
1818 : def = NULL_TREE;
1819 : else
1820 : {
1821 1847 : iter1 = make_ssa_name (orig_arg);
1822 1847 : if (incr_bb)
1823 1823 : iter2 = make_ssa_name (orig_arg);
1824 : }
1825 : }
1826 1947 : if (def)
1827 : {
1828 1947 : phi = create_phi_node (iter1, body_bb);
1829 1947 : add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
1830 1947 : if (incr_bb)
1831 : {
1832 1911 : add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1833 3822 : enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1834 1911 : ? PLUS_EXPR : POINTER_PLUS_EXPR;
1835 3822 : tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1836 3822 : ? TREE_TYPE (orig_arg) : sizetype;
1837 1911 : tree addcst = simd_clone_linear_addend (node, i, addtype,
1838 : entry_bb);
1839 1911 : gsi = gsi_last_bb (incr_bb);
1840 1911 : g = gimple_build_assign (iter2, code, iter1, addcst);
1841 1911 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1842 : }
1843 :
1844 1947 : imm_use_iterator iter;
1845 1947 : use_operand_p use_p;
1846 1947 : gimple *use_stmt;
1847 1947 : if (TREE_ADDRESSABLE (orig_arg))
1848 : {
1849 100 : gsi = gsi_after_labels (body_bb);
1850 100 : g = gimple_build_assign (orig_arg, iter1);
1851 100 : gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1852 : }
1853 : else
1854 7648 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1855 3954 : if (use_stmt == phi)
1856 1847 : continue;
1857 : else
1858 6321 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1859 3954 : SET_USE (use_p, iter1);
1860 : }
1861 : }
1862 7865 : else if (node->simdclone->args[i].arg_type
1863 : == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1864 7865 : || (node->simdclone->args[i].arg_type
1865 : == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP))
1866 : {
1867 172 : tree orig_arg = node->simdclone->args[i].orig_arg;
1868 172 : tree def = ssa_default_def (cfun, orig_arg);
1869 172 : gcc_assert (!TREE_ADDRESSABLE (orig_arg)
1870 : && TREE_CODE (TREE_TYPE (orig_arg)) == REFERENCE_TYPE);
1871 172 : if (def && !has_zero_uses (def))
1872 : {
1873 172 : tree rtype = TREE_TYPE (TREE_TYPE (orig_arg));
1874 172 : iter1 = make_ssa_name (orig_arg);
1875 172 : if (incr_bb)
1876 160 : iter2 = make_ssa_name (orig_arg);
1877 172 : tree iter3 = make_ssa_name (rtype);
1878 172 : tree iter4 = make_ssa_name (rtype);
1879 172 : tree iter5 = incr_bb ? make_ssa_name (rtype) : NULL_TREE;
1880 172 : gsi = gsi_after_labels (entry_bb);
1881 172 : gimple *load
1882 172 : = gimple_build_assign (iter3, build_simple_mem_ref (def));
1883 172 : gsi_insert_before (&gsi, load, GSI_NEW_STMT);
1884 :
1885 172 : tree array = node->simdclone->args[i].simd_array;
1886 172 : TREE_ADDRESSABLE (array) = 1;
1887 172 : tree ptr = build_fold_addr_expr (array);
1888 172 : phi = create_phi_node (iter1, body_bb);
1889 172 : add_phi_arg (phi, ptr, preheader_edge, UNKNOWN_LOCATION);
1890 172 : if (incr_bb)
1891 : {
1892 160 : add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1893 160 : g = gimple_build_assign (iter2, POINTER_PLUS_EXPR, iter1,
1894 160 : TYPE_SIZE_UNIT (TREE_TYPE (iter3)));
1895 160 : gsi = gsi_last_bb (incr_bb);
1896 160 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1897 : }
1898 :
1899 172 : phi = create_phi_node (iter4, body_bb);
1900 172 : add_phi_arg (phi, iter3, preheader_edge, UNKNOWN_LOCATION);
1901 172 : if (incr_bb)
1902 : {
1903 160 : add_phi_arg (phi, iter5, latch_edge, UNKNOWN_LOCATION);
1904 320 : enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1905 160 : ? PLUS_EXPR : POINTER_PLUS_EXPR;
1906 320 : tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1907 320 : ? TREE_TYPE (iter3) : sizetype;
1908 160 : tree addcst = simd_clone_linear_addend (node, i, addtype,
1909 : entry_bb);
1910 160 : g = gimple_build_assign (iter5, code, iter4, addcst);
1911 160 : gsi = gsi_last_bb (incr_bb);
1912 160 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1913 : }
1914 :
1915 172 : g = gimple_build_assign (build_simple_mem_ref (iter1), iter4);
1916 172 : gsi = gsi_after_labels (body_bb);
1917 172 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1918 :
1919 172 : imm_use_iterator iter;
1920 172 : use_operand_p use_p;
1921 172 : gimple *use_stmt;
1922 848 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1923 504 : if (use_stmt == load)
1924 172 : continue;
1925 : else
1926 996 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1927 332 : SET_USE (use_p, iter1);
1928 :
1929 172 : if (!TYPE_READONLY (rtype) && incr_bb)
1930 : {
1931 112 : tree v = make_ssa_name (rtype);
1932 112 : tree aref = build4 (ARRAY_REF, rtype, array,
1933 : size_zero_node, NULL_TREE,
1934 : NULL_TREE);
1935 112 : gsi = gsi_after_labels (new_exit_bb);
1936 112 : g = gimple_build_assign (v, aref);
1937 112 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1938 112 : g = gimple_build_assign (build_simple_mem_ref (def), v);
1939 112 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1940 : }
1941 : }
1942 : }
1943 :
1944 4472 : calculate_dominance_info (CDI_DOMINATORS);
1945 4472 : if (loop)
1946 4412 : add_loop (loop, loop->header->loop_father);
1947 4472 : update_ssa (TODO_update_ssa);
1948 :
1949 4472 : pop_cfun ();
1950 4472 : }
1951 :
1952 : /* If the function in NODE is tagged as an elemental SIMD function,
1953 : create the appropriate SIMD clones. */
1954 :
1955 : void
1956 4621879 : expand_simd_clones (struct cgraph_node *node)
1957 : {
1958 4621879 : tree attr;
1959 4621879 : bool explicit_p = true;
1960 :
1961 4621879 : if (node->inlined_to
1962 4621879 : || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
1963 1495302 : return;
1964 :
1965 3126577 : attr = lookup_attribute ("omp declare simd",
1966 3126577 : DECL_ATTRIBUTES (node->decl));
1967 :
1968 : /* See if we can add an "omp declare simd" directive implicitly
1969 : before giving up. */
1970 : /* FIXME: OpenACC "#pragma acc routine" translates into
1971 : "omp declare target", but appears also to have some other effects
1972 : that conflict with generating SIMD clones, causing ICEs. So don't
1973 : do this if we've got OpenACC instead of OpenMP. */
1974 3126577 : if (attr == NULL_TREE
1975 : #ifdef ACCEL_COMPILER
1976 : && (flag_openmp_target_simd_clone == OMP_TARGET_SIMD_CLONE_ANY
1977 : || flag_openmp_target_simd_clone == OMP_TARGET_SIMD_CLONE_NOHOST)
1978 : #else
1979 3125315 : && (flag_openmp_target_simd_clone == OMP_TARGET_SIMD_CLONE_ANY
1980 3125244 : || flag_openmp_target_simd_clone == OMP_TARGET_SIMD_CLONE_HOST)
1981 : #endif
1982 71 : && !oacc_get_fn_attrib (node->decl)
1983 3126648 : && ok_for_auto_simd_clone (node))
1984 : {
1985 4 : attr = tree_cons (get_identifier ("omp declare simd"), NULL,
1986 4 : DECL_ATTRIBUTES (node->decl));
1987 4 : DECL_ATTRIBUTES (node->decl) = attr;
1988 4 : explicit_p = false;
1989 : }
1990 :
1991 3126577 : if (attr == NULL_TREE)
1992 : return;
1993 :
1994 : /* Ignore
1995 : #pragma omp declare simd
1996 : extern int foo ();
1997 : in C, there we don't know the argument types at all. */
1998 1266 : if (!node->definition
1999 1266 : && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
2000 : return;
2001 :
2002 : /* Call this before creating clone_info, as it might ggc_collect. */
2003 1266 : if (node->definition && node->has_gimple_body_p ())
2004 661 : node->get_body ();
2005 :
2006 1471 : do
2007 : {
2008 : /* Start with parsing the "omp declare simd" attribute(s). */
2009 1471 : bool inbranch_clause_specified;
2010 1471 : struct cgraph_simd_clone *clone_info
2011 1471 : = simd_clone_clauses_extract (node, TREE_VALUE (attr),
2012 : &inbranch_clause_specified);
2013 1471 : if (clone_info == NULL)
2014 8 : continue;
2015 :
2016 1470 : poly_uint64 orig_simdlen = clone_info->simdlen;
2017 1470 : tree base_type = simd_clone_compute_base_data_type (node, clone_info);
2018 :
2019 : /* The target can return 0 (no simd clones should be created),
2020 : 1 (just one ISA of simd clones should be created) or higher
2021 : count of ISA variants. In that case, clone_info is initialized
2022 : for the first ISA variant. */
2023 1470 : int count
2024 1470 : = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
2025 : base_type, 0,
2026 : explicit_p);
2027 1470 : if (count == 0)
2028 7 : continue;
2029 :
2030 : /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
2031 : also create one inbranch and one !inbranch clone of it. */
2032 12735 : for (int i = 0; i < count * 2; i++)
2033 : {
2034 11272 : struct cgraph_simd_clone *clone = clone_info;
2035 11272 : if (inbranch_clause_specified && (i & 1) != 0)
2036 3690 : continue;
2037 :
2038 7582 : if (i != 0)
2039 : {
2040 12238 : clone = simd_clone_struct_alloc (clone_info->nargs
2041 6119 : + ((i & 1) != 0));
2042 6119 : simd_clone_struct_copy (clone, clone_info);
2043 : /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
2044 : and simd_clone_adjust_argument_types did to the first
2045 : clone's info. */
2046 6119 : clone->nargs -= clone_info->inbranch;
2047 6119 : clone->simdlen = orig_simdlen;
2048 : /* And call the target hook again to get the right ISA. */
2049 6119 : targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
2050 : base_type,
2051 : i / 2,
2052 : explicit_p);
2053 6119 : if ((i & 1) != 0)
2054 1946 : clone->inbranch = 1;
2055 : }
2056 :
2057 : /* simd_clone_mangle might fail if such a clone has been created
2058 : already. */
2059 7582 : tree id = simd_clone_mangle (node, clone);
2060 7582 : if (id == NULL_TREE)
2061 : {
2062 394 : if (i == 0)
2063 74 : clone->nargs += clone->inbranch;
2064 394 : continue;
2065 : }
2066 :
2067 : /* Only when we are sure we want to create the clone actually
2068 : clone the function (or definitions) or create another
2069 : extern FUNCTION_DECL (for prototypes without definitions). */
2070 7188 : struct cgraph_node *n = simd_clone_create (node, !explicit_p);
2071 7188 : if (n == NULL)
2072 : {
2073 0 : if (i == 0)
2074 0 : clone->nargs += clone->inbranch;
2075 0 : continue;
2076 : }
2077 :
2078 7188 : n->simdclone = clone;
2079 7188 : clone->origin = node;
2080 7188 : clone->next_clone = NULL;
2081 7188 : if (node->simd_clones == NULL)
2082 : {
2083 1258 : clone->prev_clone = n;
2084 1258 : node->simd_clones = n;
2085 : }
2086 : else
2087 : {
2088 5930 : clone->prev_clone = node->simd_clones->simdclone->prev_clone;
2089 5930 : clone->prev_clone->simdclone->next_clone = n;
2090 5930 : node->simd_clones->simdclone->prev_clone = n;
2091 : }
2092 7188 : symtab->change_decl_assembler_name (n->decl, id);
2093 : /* And finally adjust the return type, parameters and for
2094 : definitions also function body. */
2095 7188 : if (node->definition)
2096 4472 : simd_clone_adjust (n);
2097 : else
2098 : {
2099 2716 : TREE_TYPE (n->decl)
2100 2716 : = build_distinct_type_copy (TREE_TYPE (n->decl));
2101 2716 : simd_clone_adjust_return_type (n);
2102 2716 : simd_clone_adjust_argument_types (n);
2103 2716 : targetm.simd_clone.adjust (n);
2104 : }
2105 7188 : if (dump_file)
2106 8 : fprintf (dump_file, "\nGenerated %s clone %s\n",
2107 8 : (TREE_PUBLIC (n->decl) ? "global" : "local"),
2108 8 : IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (n->decl)));
2109 : }
2110 : }
2111 1471 : while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
2112 : }
2113 :
2114 : /* Entry point for IPA simd clone creation pass. */
2115 :
2116 : static unsigned int
2117 230428 : ipa_omp_simd_clone (void)
2118 : {
2119 230428 : struct cgraph_node *node;
2120 4852305 : FOR_EACH_FUNCTION (node)
2121 4621877 : expand_simd_clones (node);
2122 230428 : return 0;
2123 : }
2124 :
2125 : namespace {
2126 :
2127 : const pass_data pass_data_omp_simd_clone =
2128 : {
2129 : SIMPLE_IPA_PASS, /* type */
2130 : "simdclone", /* name */
2131 : OPTGROUP_OMP, /* optinfo_flags */
2132 : TV_NONE, /* tv_id */
2133 : ( PROP_ssa | PROP_cfg ), /* properties_required */
2134 : 0, /* properties_provided */
2135 : 0, /* properties_destroyed */
2136 : 0, /* todo_flags_start */
2137 : 0, /* todo_flags_finish */
2138 : };
2139 :
2140 : class pass_omp_simd_clone : public simple_ipa_opt_pass
2141 : {
2142 : public:
2143 285722 : pass_omp_simd_clone(gcc::context *ctxt)
2144 571444 : : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
2145 : {}
2146 :
2147 : /* opt_pass methods: */
2148 : bool gate (function *) final override;
2149 230428 : unsigned int execute (function *) final override
2150 : {
2151 230428 : return ipa_omp_simd_clone ();
2152 : }
2153 : };
2154 :
2155 : bool
2156 230433 : pass_omp_simd_clone::gate (function *)
2157 : {
2158 230433 : return targetm.simd_clone.compute_vecsize_and_simdlen != NULL;
2159 : }
2160 :
2161 : } // anon namespace
2162 :
2163 : simple_ipa_opt_pass *
2164 285722 : make_pass_omp_simd_clone (gcc::context *ctxt)
2165 : {
2166 285722 : return new pass_omp_simd_clone (ctxt);
2167 : }
|
