Line data Source code
1 : /* Lower vector operations to scalar operations.
2 : Copyright (C) 2004-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it
7 : under the terms of the GNU General Public License as published by the
8 : Free Software Foundation; either version 3, or (at your option) any
9 : later version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT
12 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "rtl.h"
25 : #include "tree.h"
26 : #include "gimple.h"
27 : #include "tree-pass.h"
28 : #include "ssa.h"
29 : #include "expmed.h"
30 : #include "optabs-tree.h"
31 : #include "diagnostic.h"
32 : #include "fold-const.h"
33 : #include "stor-layout.h"
34 : #include "langhooks.h"
35 : #include "tree-eh.h"
36 : #include "gimple-iterator.h"
37 : #include "gimplify-me.h"
38 : #include "gimplify.h"
39 : #include "tree-cfg.h"
40 : #include "tree-vector-builder.h"
41 : #include "vec-perm-indices.h"
42 : #include "insn-config.h"
43 : #include "gimple-fold.h"
44 : #include "gimple-match.h"
45 : #include "recog.h" /* FIXME: for insn_data */
46 : #include "optabs-libfuncs.h"
47 : #include "cfgloop.h"
48 : #include "tree-vectorizer.h"
49 :
50 :
51 : /* Build a ternary operation and gimplify it. Emit code before GSI.
52 : Return the gimple_val holding the result. */
53 :
54 : static tree
55 12579 : gimplify_build3 (gimple_stmt_iterator *gsi, enum tree_code code,
56 : tree type, tree a, tree b, tree c)
57 : {
58 12579 : location_t loc = gimple_location (gsi_stmt (*gsi));
59 12579 : return gimple_build (gsi, true, GSI_SAME_STMT, loc, code, type, a, b, c);
60 : }
61 :
62 : /* Build a binary operation and gimplify it. Emit code before GSI.
63 : Return the gimple_val holding the result. */
64 :
65 : static tree
66 22111 : gimplify_build2 (gimple_stmt_iterator *gsi, enum tree_code code,
67 : tree type, tree a, tree b)
68 : {
69 22111 : location_t loc = gimple_location (gsi_stmt (*gsi));
70 22111 : return gimple_build (gsi, true, GSI_SAME_STMT, loc, code, type, a, b);
71 : }
72 :
73 : /* Build a unary operation and gimplify it. Emit code before GSI.
74 : Return the gimple_val holding the result. */
75 :
76 : static tree
77 20159 : gimplify_build1 (gimple_stmt_iterator *gsi, enum tree_code code, tree type,
78 : tree a)
79 : {
80 20159 : location_t loc = gimple_location (gsi_stmt (*gsi));
81 20159 : return gimple_build (gsi, true, GSI_SAME_STMT, loc, code, type, a);
82 : }
83 :
84 :
85 : /* Return the number of elements in a vector type TYPE that we have
86 : already decided needs to be expanded piecewise. We don't support
87 : this kind of expansion for variable-length vectors, since we should
88 : always check for target support before introducing uses of those. */
89 : static unsigned int
90 141306 : nunits_for_known_piecewise_op (const_tree type)
91 : {
92 140309 : return TYPE_VECTOR_SUBPARTS (type).to_constant ();
93 : }
94 :
95 : /* Return true if TYPE1 has more elements than TYPE2, where either
96 : type may be a vector or a scalar. */
97 :
98 : static inline bool
99 300 : subparts_gt (tree type1, tree type2)
100 : {
101 300 : poly_uint64 n1 = VECTOR_TYPE_P (type1) ? TYPE_VECTOR_SUBPARTS (type1) : 1;
102 300 : poly_uint64 n2 = VECTOR_TYPE_P (type2) ? TYPE_VECTOR_SUBPARTS (type2) : 1;
103 300 : return known_gt (n1, n2);
104 : }
105 :
106 : static GTY(()) tree vector_inner_type;
107 : static GTY(()) tree vector_last_type;
108 : static GTY(()) int vector_last_nunits;
109 :
110 : /* Return a suitable vector types made of SUBPARTS units each of mode
111 : "word_mode" (the global variable). */
112 : static tree
113 52 : build_word_mode_vector_type (int nunits)
114 : {
115 52 : if (!vector_inner_type)
116 40 : vector_inner_type = lang_hooks.types.type_for_mode (word_mode, 1);
117 12 : else if (vector_last_nunits == nunits)
118 : {
119 10 : gcc_assert (TREE_CODE (vector_last_type) == VECTOR_TYPE);
120 : return vector_last_type;
121 : }
122 :
123 42 : vector_last_nunits = nunits;
124 42 : vector_last_type = build_vector_type (vector_inner_type, nunits);
125 42 : return vector_last_type;
126 : }
127 :
128 : typedef tree (*elem_op_func) (gimple_stmt_iterator *,
129 : tree, tree, tree, tree, tree, enum tree_code,
130 : tree);
131 :
132 : /* Extract the vector element of type TYPE at BITPOS with BITSIZE from T
133 : and return it. */
134 :
135 : tree
136 77734 : tree_vec_extract (gimple_stmt_iterator *gsi, tree type,
137 : tree t, tree bitsize, tree bitpos)
138 : {
139 : /* We're using the resimplify API and maybe_push_res_to_seq to
140 : simplify the BIT_FIELD_REF but restrict the simplification to
141 : a single stmt while at the same time following SSA edges for
142 : simplification with already emitted CTORs. */
143 77734 : gimple_match_op opr;
144 77734 : opr.set_op (BIT_FIELD_REF, type, t, bitsize, bitpos);
145 77734 : opr.resimplify (NULL, follow_all_ssa_edges);
146 77734 : gimple_seq stmts = NULL;
147 77734 : tree res = maybe_push_res_to_seq (&opr, &stmts);
148 77734 : if (!res)
149 : {
150 : /* This can happen if SSA_NAME_OCCURS_IN_ABNORMAL_PHI are
151 : used. Build BIT_FIELD_REF manually otherwise. */
152 64 : t = build3 (BIT_FIELD_REF, type, t, bitsize, bitpos);
153 64 : res = make_ssa_name (type);
154 64 : gimple *g = gimple_build_assign (res, t);
155 64 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
156 64 : return res;
157 : }
158 77670 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
159 77670 : return res;
160 : }
161 :
162 : static tree
163 638 : do_unop (gimple_stmt_iterator *gsi, tree inner_type, tree a,
164 : tree b ATTRIBUTE_UNUSED, tree bitpos, tree bitsize,
165 : enum tree_code code, tree type ATTRIBUTE_UNUSED)
166 : {
167 638 : tree rhs_type = inner_type;
168 :
169 : /* For ABSU_EXPR, use the signed type for the rhs if the rhs was signed. */
170 638 : if (code == ABSU_EXPR
171 9 : && ANY_INTEGRAL_TYPE_P (TREE_TYPE (a))
172 647 : && !TYPE_UNSIGNED (TREE_TYPE (a)))
173 9 : rhs_type = signed_type_for (rhs_type);
174 :
175 638 : a = tree_vec_extract (gsi, rhs_type, a, bitsize, bitpos);
176 638 : return gimplify_build1 (gsi, code, inner_type, a);
177 : }
178 :
179 : static tree
180 13511 : do_binop (gimple_stmt_iterator *gsi, tree inner_type, tree a, tree b,
181 : tree bitpos, tree bitsize, enum tree_code code,
182 : tree type ATTRIBUTE_UNUSED)
183 : {
184 13511 : if (VECTOR_TYPE_P (TREE_TYPE (a)))
185 13511 : a = tree_vec_extract (gsi, inner_type, a, bitsize, bitpos);
186 13511 : if (VECTOR_TYPE_P (TREE_TYPE (b)))
187 13191 : b = tree_vec_extract (gsi, inner_type, b, bitsize, bitpos);
188 13511 : return gimplify_build2 (gsi, code, inner_type, a, b);
189 : }
190 :
191 : /* Construct expression (A[BITPOS] code B[BITPOS]) ? -1 : 0
192 :
193 : INNER_TYPE is the type of A and B elements
194 :
195 : returned expression is of signed integer type with the
196 : size equal to the size of INNER_TYPE. */
197 : static tree
198 6500 : do_compare (gimple_stmt_iterator *gsi, tree inner_type, tree a, tree b,
199 : tree bitpos, tree bitsize, enum tree_code code, tree type)
200 : {
201 6500 : tree stype = TREE_TYPE (type);
202 6500 : tree cst_false = build_zero_cst (stype);
203 6500 : tree cst_true = build_all_ones_cst (stype);
204 6500 : tree cmp;
205 :
206 6500 : a = tree_vec_extract (gsi, inner_type, a, bitsize, bitpos);
207 6500 : b = tree_vec_extract (gsi, inner_type, b, bitsize, bitpos);
208 :
209 6500 : cmp = build2 (code, boolean_type_node, a, b);
210 6500 : return gimplify_build3 (gsi, COND_EXPR, stype, cmp, cst_true, cst_false);
211 : }
212 :
213 : /* Expand vector addition to scalars. This does bit twiddling
214 : in order to increase parallelism:
215 :
216 : a + b = (((int) a & 0x7f7f7f7f) + ((int) b & 0x7f7f7f7f)) ^
217 : (a ^ b) & 0x80808080
218 :
219 : a - b = (((int) a | 0x80808080) - ((int) b & 0x7f7f7f7f)) ^
220 : (a ^ ~b) & 0x80808080
221 :
222 : -b = (0x80808080 - ((int) b & 0x7f7f7f7f)) ^ (~b & 0x80808080)
223 :
224 : This optimization should be done only if 4 vector items or more
225 : fit into a word. */
226 : static tree
227 6 : do_plus_minus (gimple_stmt_iterator *gsi, tree word_type, tree a, tree b,
228 : tree bitpos ATTRIBUTE_UNUSED, tree bitsize ATTRIBUTE_UNUSED,
229 : enum tree_code code, tree type ATTRIBUTE_UNUSED)
230 : {
231 6 : unsigned int width = vector_element_bits (TREE_TYPE (a));
232 6 : tree inner_type = TREE_TYPE (TREE_TYPE (a));
233 6 : unsigned HOST_WIDE_INT max;
234 6 : tree low_bits, high_bits, a_low, b_low, result_low, signs;
235 :
236 6 : max = GET_MODE_MASK (TYPE_MODE (inner_type));
237 6 : low_bits = build_replicated_int_cst (word_type, width, max >> 1);
238 6 : high_bits = build_replicated_int_cst (word_type, width, max & ~(max >> 1));
239 :
240 6 : a = tree_vec_extract (gsi, word_type, a, bitsize, bitpos);
241 6 : b = tree_vec_extract (gsi, word_type, b, bitsize, bitpos);
242 :
243 6 : signs = gimplify_build2 (gsi, BIT_XOR_EXPR, word_type, a, b);
244 6 : b_low = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, b, low_bits);
245 6 : if (code == PLUS_EXPR)
246 2 : a_low = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, a, low_bits);
247 : else
248 : {
249 4 : a_low = gimplify_build2 (gsi, BIT_IOR_EXPR, word_type, a, high_bits);
250 4 : signs = gimplify_build1 (gsi, BIT_NOT_EXPR, word_type, signs);
251 : }
252 :
253 6 : signs = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, signs, high_bits);
254 6 : result_low = gimplify_build2 (gsi, code, word_type, a_low, b_low);
255 6 : return gimplify_build2 (gsi, BIT_XOR_EXPR, word_type, result_low, signs);
256 : }
257 :
258 : static tree
259 0 : do_negate (gimple_stmt_iterator *gsi, tree word_type, tree b,
260 : tree unused ATTRIBUTE_UNUSED, tree bitpos ATTRIBUTE_UNUSED,
261 : tree bitsize ATTRIBUTE_UNUSED,
262 : enum tree_code code ATTRIBUTE_UNUSED,
263 : tree type ATTRIBUTE_UNUSED)
264 : {
265 0 : unsigned int width = vector_element_bits (TREE_TYPE (b));
266 0 : tree inner_type = TREE_TYPE (TREE_TYPE (b));
267 0 : HOST_WIDE_INT max;
268 0 : tree low_bits, high_bits, b_low, result_low, signs;
269 :
270 0 : max = GET_MODE_MASK (TYPE_MODE (inner_type));
271 0 : low_bits = build_replicated_int_cst (word_type, width, max >> 1);
272 0 : high_bits = build_replicated_int_cst (word_type, width, max & ~(max >> 1));
273 :
274 0 : b = tree_vec_extract (gsi, word_type, b, bitsize, bitpos);
275 :
276 0 : b_low = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, b, low_bits);
277 0 : signs = gimplify_build1 (gsi, BIT_NOT_EXPR, word_type, b);
278 0 : signs = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, signs, high_bits);
279 0 : result_low = gimplify_build2 (gsi, MINUS_EXPR, word_type, high_bits, b_low);
280 0 : return gimplify_build2 (gsi, BIT_XOR_EXPR, word_type, result_low, signs);
281 : }
282 :
283 : /* Expand a vector operation to scalars, by using many operations
284 : whose type is the vector type's inner type. */
285 : static tree
286 4099 : expand_vector_piecewise (gimple_stmt_iterator *gsi, elem_op_func f,
287 : tree type, tree inner_type,
288 : tree a, tree b, enum tree_code code,
289 : bool parallel_p, tree ret_type = NULL_TREE)
290 : {
291 4099 : vec<constructor_elt, va_gc> *v;
292 4099 : tree part_width = TYPE_SIZE (inner_type);
293 4099 : tree index = bitsize_int (0);
294 4099 : int nunits = nunits_for_known_piecewise_op (type);
295 4099 : int delta = (VECTOR_TYPE_P (inner_type)
296 5096 : ? nunits_for_known_piecewise_op (inner_type) : 1);
297 4099 : int i;
298 4099 : location_t loc = gimple_location (gsi_stmt (*gsi));
299 :
300 4099 : if (nunits == 1
301 4099 : || warning_suppressed_p (gsi_stmt (*gsi),
302 : OPT_Wvector_operation_performance))
303 : /* Do not diagnose decomposing single element vectors or when
304 : decomposing vectorizer produced operations. */
305 : ;
306 3752 : else if (ret_type || !parallel_p)
307 3700 : warning_at (loc, OPT_Wvector_operation_performance,
308 : "vector operation will be expanded piecewise");
309 : else
310 52 : warning_at (loc, OPT_Wvector_operation_performance,
311 : "vector operation will be expanded in parallel");
312 :
313 4099 : if (!ret_type)
314 3743 : ret_type = type;
315 4099 : vec_alloc (v, (nunits + delta - 1) / delta);
316 4099 : bool constant_p = true;
317 47805 : for (i = 0; i < nunits;
318 39607 : i += delta, index = int_const_binop (PLUS_EXPR, index, part_width))
319 : {
320 39607 : tree result = f (gsi, inner_type, a, b, index, part_width, code,
321 : ret_type);
322 39607 : if (!CONSTANT_CLASS_P (result))
323 39174 : constant_p = false;
324 39607 : constructor_elt ce = {NULL_TREE, result};
325 39607 : v->quick_push (ce);
326 : }
327 :
328 4099 : if (constant_p)
329 2 : return build_vector_from_ctor (ret_type, v);
330 : else
331 4097 : return build_constructor (ret_type, v);
332 : }
333 :
334 : /* Expand a vector operation to scalars with the freedom to use
335 : a scalar integer type, or to use a different size for the items
336 : in the vector type. */
337 : static tree
338 104 : expand_vector_parallel (gimple_stmt_iterator *gsi, elem_op_func f, tree type,
339 : tree a, tree b, enum tree_code code)
340 : {
341 104 : tree result, compute_type;
342 104 : int n_words = tree_to_uhwi (TYPE_SIZE_UNIT (type)) / UNITS_PER_WORD;
343 104 : location_t loc = gimple_location (gsi_stmt (*gsi));
344 :
345 : /* We have three strategies. If the type is already correct, just do
346 : the operation an element at a time. Else, if the vector is wider than
347 : one word, do it a word at a time; finally, if the vector is smaller
348 : than one word, do it as a scalar. */
349 104 : if (TYPE_MODE (TREE_TYPE (type)) == word_mode)
350 2 : return expand_vector_piecewise (gsi, f,
351 2 : type, TREE_TYPE (type),
352 2 : a, b, code, true);
353 102 : else if (n_words > 1)
354 : {
355 52 : tree word_type = build_word_mode_vector_type (n_words);
356 52 : result = expand_vector_piecewise (gsi, f,
357 52 : word_type, TREE_TYPE (word_type),
358 : a, b, code, true);
359 52 : result = force_gimple_operand_gsi (gsi, result, true, NULL, true,
360 : GSI_SAME_STMT);
361 : }
362 : else
363 : {
364 : /* Use a single scalar operation with a mode no wider than word_mode. */
365 50 : if (!warning_suppressed_p (gsi_stmt (*gsi),
366 : OPT_Wvector_operation_performance))
367 50 : warning_at (loc, OPT_Wvector_operation_performance,
368 : "vector operation will be expanded with a "
369 : "single scalar operation");
370 50 : scalar_int_mode mode
371 50 : = int_mode_for_size (tree_to_uhwi (TYPE_SIZE (type)), 0).require ();
372 50 : compute_type = lang_hooks.types.type_for_mode (mode, 1);
373 50 : result = f (gsi, compute_type, a, b, bitsize_zero_node,
374 50 : TYPE_SIZE (compute_type), code, type);
375 : }
376 :
377 : return result;
378 : }
379 :
380 : /* Expand a vector operation to scalars; for integer types we can use
381 : special bit twiddling tricks to do the sums a word at a time, using
382 : function F_PARALLEL instead of F. These tricks are done only if
383 : they can process at least four items, that is, only if the vector
384 : holds at least four items and if a word can hold four items. */
385 : static tree
386 210 : expand_vector_addition (gimple_stmt_iterator *gsi,
387 : elem_op_func f, elem_op_func f_parallel,
388 : tree type, tree a, tree b, enum tree_code code)
389 : {
390 210 : int parts_per_word = BITS_PER_WORD / vector_element_bits (type);
391 :
392 420 : if (INTEGRAL_TYPE_P (TREE_TYPE (type))
393 210 : && parts_per_word >= 4
394 220 : && nunits_for_known_piecewise_op (type) >= 4)
395 3 : return expand_vector_parallel (gsi, f_parallel,
396 3 : type, a, b, code);
397 : else
398 207 : return expand_vector_piecewise (gsi, f,
399 207 : type, TREE_TYPE (type),
400 207 : a, b, code, false);
401 : }
402 :
403 : /* Expand vector comparison expression OP0 CODE OP1 if the compare optab
404 : is not implemented. */
405 :
406 : static tree
407 27566 : expand_vector_comparison (gimple_stmt_iterator *gsi, tree type, tree op0,
408 : tree op1, enum tree_code code)
409 : {
410 27566 : if (expand_vec_cmp_expr_p (TREE_TYPE (op0), type, code))
411 : return NULL_TREE;
412 :
413 809 : tree t;
414 809 : if (VECTOR_BOOLEAN_TYPE_P (type)
415 809 : && SCALAR_INT_MODE_P (TYPE_MODE (type))
416 935 : && known_lt (GET_MODE_BITSIZE (TYPE_MODE (type)),
417 : TYPE_VECTOR_SUBPARTS (type)
418 : * GET_MODE_BITSIZE (SCALAR_TYPE_MODE
419 : (TREE_TYPE (type)))))
420 : {
421 0 : tree inner_type = TREE_TYPE (TREE_TYPE (op0));
422 0 : tree part_width = vector_element_bits_tree (TREE_TYPE (op0));
423 0 : tree index = bitsize_int (0);
424 0 : int nunits = nunits_for_known_piecewise_op (TREE_TYPE (op0));
425 0 : int prec = GET_MODE_PRECISION (SCALAR_TYPE_MODE (type));
426 0 : tree ret_type = build_nonstandard_integer_type (prec, 1);
427 0 : tree ret_inner_type = boolean_type_node;
428 0 : int i;
429 0 : location_t loc = gimple_location (gsi_stmt (*gsi));
430 0 : t = build_zero_cst (ret_type);
431 :
432 0 : if (TYPE_PRECISION (ret_inner_type) != 1)
433 0 : ret_inner_type = build_nonstandard_integer_type (1, 1);
434 0 : if (!warning_suppressed_p (gsi_stmt (*gsi),
435 : OPT_Wvector_operation_performance))
436 0 : warning_at (loc, OPT_Wvector_operation_performance,
437 : "vector operation will be expanded piecewise");
438 0 : for (i = 0; i < nunits;
439 0 : i++, index = int_const_binop (PLUS_EXPR, index, part_width))
440 : {
441 0 : tree a = tree_vec_extract (gsi, inner_type, op0, part_width,
442 : index);
443 0 : tree b = tree_vec_extract (gsi, inner_type, op1, part_width,
444 : index);
445 0 : tree result = gimplify_build2 (gsi, code, ret_inner_type, a, b);
446 0 : t = gimplify_build3 (gsi, BIT_INSERT_EXPR, ret_type, t, result,
447 0 : bitsize_int (i));
448 : }
449 0 : t = gimplify_build1 (gsi, VIEW_CONVERT_EXPR, type, t);
450 : }
451 : else
452 809 : t = expand_vector_piecewise (gsi, do_compare, type,
453 809 : TREE_TYPE (TREE_TYPE (op0)), op0, op1,
454 : code, false);
455 :
456 : return t;
457 : }
458 :
459 : /* Helper function of expand_vector_divmod. Gimplify a RSHIFT_EXPR in type
460 : of OP0 with shift counts in SHIFTCNTS array and return the temporary holding
461 : the result if successful, otherwise return NULL_TREE. */
462 : static tree
463 716 : add_shift (gimple_stmt_iterator *gsi, tree type, tree op0, int *shiftcnts,
464 : enum tree_code code)
465 : {
466 716 : optab op;
467 716 : unsigned int i, nunits = nunits_for_known_piecewise_op (type);
468 716 : bool scalar_shift = true;
469 :
470 4204 : for (i = 1; i < nunits; i++)
471 : {
472 3488 : if (shiftcnts[i] != shiftcnts[0])
473 96 : scalar_shift = false;
474 : }
475 :
476 716 : if (scalar_shift && shiftcnts[0] == 0)
477 : return op0;
478 :
479 578 : if (scalar_shift)
480 : {
481 578 : op = optab_for_tree_code (code, type, optab_scalar);
482 578 : if (op != unknown_optab
483 578 : && can_implement_p (op, TYPE_MODE (type)))
484 578 : return gimplify_build2 (gsi, code, type, op0,
485 578 : build_int_cst (integer_type_node, shiftcnts[0]));
486 : }
487 :
488 32 : op = optab_for_tree_code (code, type, optab_vector);
489 32 : if (op != unknown_optab
490 32 : && can_implement_p (op, TYPE_MODE (type)))
491 : {
492 0 : tree_vector_builder vec (type, nunits, 1);
493 0 : for (i = 0; i < nunits; i++)
494 0 : vec.quick_push (build_int_cst (TREE_TYPE (type), shiftcnts[i]));
495 0 : return gimplify_build2 (gsi, code, type, op0, vec.build ());
496 0 : }
497 :
498 : return NULL_TREE;
499 : }
500 :
501 : /* Verify that the target has optabs of VECTYPE to perform all the steps
502 : needed by the multiplication-by-immediate synthesis algorithm described by
503 : ALG and VAR. Return true iff the target supports all the steps. */
504 : static bool
505 1 : target_supports_mult_synth_alg (struct algorithm *alg, mult_variant var,
506 : tree vectype)
507 : {
508 1 : if (alg->op[0] != alg_zero && alg->op[0] != alg_m)
509 : return false;
510 :
511 1 : bool plus_supported_p = target_supports_op_p (vectype, PLUS_EXPR,
512 : optab_vector);
513 1 : bool minus_supported_p = target_supports_op_p (vectype, MINUS_EXPR,
514 : optab_vector);
515 1 : bool negate_supported_p = target_supports_op_p (vectype, NEGATE_EXPR,
516 : optab_vector);
517 :
518 1 : if ((var == negate_variant && !negate_supported_p)
519 1 : || (var == add_variant && !plus_supported_p))
520 : return false;
521 :
522 1 : bool lshift_supported_p
523 1 : = (target_supports_op_p (vectype, LSHIFT_EXPR, optab_scalar)
524 1 : || target_supports_op_p (vectype, LSHIFT_EXPR, optab_vector));
525 :
526 5 : for (int i = 1; i < alg->ops; i++)
527 : {
528 4 : switch (alg->op[i])
529 : {
530 2 : case alg_shift:
531 2 : if (!lshift_supported_p)
532 : return false;
533 : break;
534 0 : case alg_add_t_m2:
535 0 : case alg_add_t2_m:
536 0 : case alg_add_factor:
537 0 : if (!plus_supported_p || !lshift_supported_p)
538 : return false;
539 : break;
540 2 : case alg_sub_t_m2:
541 2 : case alg_sub_t2_m:
542 2 : case alg_sub_factor:
543 2 : if (!minus_supported_p || !lshift_supported_p)
544 : return false;
545 : break;
546 : case alg_unknown:
547 : case alg_m:
548 : case alg_zero:
549 : case alg_impossible:
550 : return false;
551 0 : default:
552 0 : gcc_unreachable ();
553 : }
554 : }
555 :
556 : return true;
557 : }
558 : /* For integer multiplications with exact power of 2, use left shifts.
559 : In constant is a uniform vector, then check if it can be synthesized with
560 : shifts and adds/subs. Use scalar multiplication cost to compare with the
561 : synthesized vector code as done in expmed.cc. */
562 : static tree
563 6 : expand_vector_mult (gimple_stmt_iterator *gsi, tree type, tree op0,
564 : tree op1)
565 : {
566 6 : if (!CONSTANT_CLASS_P (op1))
567 : return NULL_TREE;
568 6 : unsigned int nunits = nunits_for_known_piecewise_op (type);
569 6 : int *shifts = XALLOCAVEC (int, nunits);
570 :
571 6 : bool perfect_pow2_p = true;
572 6 : bool uniform_const_p = true;
573 6 : unsigned HOST_WIDE_INT first_cst, const_elt;
574 13 : for (unsigned int i = 0; i < nunits; i++)
575 : {
576 7 : tree cst = VECTOR_CST_ELT (op1, i);
577 7 : if (TREE_CODE (cst) != INTEGER_CST || tree_int_cst_sgn (cst) <= 0)
578 : return NULL_TREE;
579 :
580 7 : perfect_pow2_p &= integer_pow2p (cst);
581 :
582 7 : if (uniform_const_p && tree_fits_uhwi_p (cst))
583 : {
584 2 : const_elt = TREE_INT_CST_LOW (cst);
585 2 : if (i == 0)
586 : first_cst = const_elt;
587 1 : else if (first_cst != const_elt)
588 0 : uniform_const_p = false;
589 : }
590 : else
591 : uniform_const_p = false;
592 :
593 7 : shifts[i] = tree_log2 (cst);
594 : }
595 6 : if (perfect_pow2_p)
596 0 : return add_shift (gsi, type, op0, shifts, LSHIFT_EXPR);
597 6 : else if (uniform_const_p)
598 : {
599 1 : tree scalar_type = type;
600 1 : gcc_assert (TREE_CODE (type) == VECTOR_TYPE);
601 1 : scalar_type = TREE_TYPE (type);
602 :
603 : // handle signed ints
604 1 : bool cast_to_unsigned_p = !TYPE_OVERFLOW_WRAPS (scalar_type);
605 0 : tree multtype = cast_to_unsigned_p
606 0 : ? unsigned_type_for (scalar_type) : scalar_type;
607 0 : tree vectype = cast_to_unsigned_p ? unsigned_type_for (type) : type;
608 :
609 : // estimate the scalar MULT cost by generating dummy rtx insns
610 1 : machine_mode mode = TYPE_MODE (multtype);
611 1 : rtx rop0 = gen_rtx_REG (mode, 0);
612 1 : rtx rop1 = gen_rtx_REG (mode, 1);
613 1 : rtx mult_rtx = gen_rtx_MULT (mode, rop0, rop1);
614 1 : int max_cost = nunits * set_src_cost (mult_rtx, mode, true);
615 :
616 1 : struct algorithm alg;
617 1 : mult_variant variant;
618 :
619 1 : bool possible = choose_mult_variant (TYPE_MODE (vectype), const_elt, &alg,
620 : &variant, max_cost);
621 1 : if (!possible || !target_supports_mult_synth_alg (&alg, variant, vectype))
622 : return NULL_TREE;
623 1 : if (cast_to_unsigned_p)
624 : {
625 0 : tree tmp_op = gimplify_build1 (gsi, VIEW_CONVERT_EXPR, vectype, op0);
626 0 : op0 = tmp_op;
627 : }
628 1 : tree accumulator, tmp_var;
629 1 : if (alg.op[0] == alg_zero)
630 0 : accumulator = build_zero_cst (vectype);
631 : else
632 : accumulator = op0;
633 :
634 5 : for (int i = 1; i < alg.ops; i++)
635 : {
636 12 : for (unsigned int j = 0; j < nunits; j++)
637 8 : shifts[j] = alg.log[i];
638 :
639 4 : switch (alg.op[i])
640 : {
641 2 : case alg_shift:
642 2 : accumulator = add_shift (gsi, vectype, accumulator, shifts,
643 : LSHIFT_EXPR);
644 2 : break;
645 0 : case alg_add_t_m2:
646 0 : tmp_var = add_shift (gsi, vectype, op0, shifts, LSHIFT_EXPR);
647 0 : accumulator = gimplify_build2 (gsi, PLUS_EXPR, vectype, tmp_var,
648 : accumulator);
649 0 : break;
650 1 : case alg_sub_t_m2:
651 1 : tmp_var = add_shift (gsi, vectype, op0, shifts, LSHIFT_EXPR);
652 1 : accumulator = gimplify_build2 (gsi, MINUS_EXPR, vectype,
653 : accumulator, tmp_var);
654 1 : break;
655 0 : case alg_add_t2_m:
656 0 : tmp_var = add_shift (gsi, vectype, accumulator, shifts,
657 : LSHIFT_EXPR);
658 0 : accumulator = gimplify_build2 (gsi, PLUS_EXPR, vectype, tmp_var,
659 : op0);
660 0 : break;
661 0 : case alg_sub_t2_m:
662 0 : tmp_var = add_shift (gsi, vectype, accumulator, shifts,
663 : LSHIFT_EXPR);
664 0 : accumulator = gimplify_build2 (gsi, MINUS_EXPR, vectype,
665 : tmp_var, op0);
666 0 : break;
667 0 : case alg_add_factor:
668 0 : tmp_var = add_shift (gsi, vectype, accumulator, shifts,
669 : LSHIFT_EXPR);
670 0 : accumulator = gimplify_build2 (gsi, PLUS_EXPR, vectype,
671 : accumulator, tmp_var);
672 0 : break;
673 1 : case alg_sub_factor:
674 1 : tmp_var = add_shift (gsi, vectype, accumulator, shifts,
675 : LSHIFT_EXPR);
676 1 : accumulator = gimplify_build2 (gsi, MINUS_EXPR, vectype,
677 : tmp_var, accumulator);
678 1 : break;
679 0 : default:
680 0 : gcc_unreachable ();
681 : }
682 : }
683 1 : if (variant == negate_variant)
684 0 : accumulator = gimplify_build1 (gsi, NEGATE_EXPR, vectype, accumulator);
685 1 : else if (variant == add_variant)
686 0 : accumulator = gimplify_build2 (gsi, PLUS_EXPR, vectype, accumulator,
687 : op0);
688 :
689 1 : if (cast_to_unsigned_p)
690 0 : accumulator = gimplify_build1 (gsi, VIEW_CONVERT_EXPR, vectype,
691 : accumulator);
692 : return accumulator;
693 : }
694 : return NULL_TREE;
695 : }
696 : /* Try to expand integer vector division by constant using
697 : widening multiply, shifts and additions. */
698 : static tree
699 634 : expand_vector_divmod (gimple_stmt_iterator *gsi, tree type, tree op0,
700 : tree op1, enum tree_code code)
701 : {
702 634 : bool use_pow2 = true;
703 634 : bool has_vector_shift = true;
704 634 : bool has_vector_negate = true;
705 634 : bool use_abs_op1 = false;
706 : /* For the power-of-two path: 1 if the quotient is negated (all divisors
707 : are negative), 0 if not, -1 if undetermined. */
708 634 : int negate_p = -1;
709 634 : int mode = -1, this_mode;
710 634 : int pre_shift = -1, post_shift;
711 634 : unsigned int nunits = nunits_for_known_piecewise_op (type);
712 634 : int *shifts = XALLOCAVEC (int, nunits * 4);
713 634 : int *pre_shifts = shifts + nunits;
714 634 : int *post_shifts = pre_shifts + nunits;
715 634 : int *shift_temps = post_shifts + nunits;
716 634 : unsigned HOST_WIDE_INT *mulc = XALLOCAVEC (unsigned HOST_WIDE_INT, nunits);
717 634 : int prec = TYPE_PRECISION (TREE_TYPE (type));
718 634 : unsigned int i;
719 634 : signop sign_p = TYPE_SIGN (TREE_TYPE (type));
720 634 : unsigned HOST_WIDE_INT mask = GET_MODE_MASK (TYPE_MODE (TREE_TYPE (type)));
721 634 : tree cur_op, mulcst, tem;
722 634 : optab op;
723 :
724 634 : if (prec > HOST_BITS_PER_WIDE_INT)
725 : return NULL_TREE;
726 :
727 633 : op = optab_for_tree_code (RSHIFT_EXPR, type, optab_vector);
728 633 : if (op == unknown_optab
729 633 : || !can_implement_p (op, TYPE_MODE (type)))
730 : has_vector_shift = false;
731 :
732 : /* expand_unop falls back to 0 - x for a missing vector negation. */
733 633 : op = optab_for_tree_code (NEGATE_EXPR, type, optab_default);
734 633 : if (op == unknown_optab
735 633 : || !can_implement_p (op, TYPE_MODE (type)))
736 : {
737 0 : op = optab_for_tree_code (MINUS_EXPR, type, optab_default);
738 0 : if (op == unknown_optab
739 0 : || !can_implement_p (op, TYPE_MODE (type)))
740 : has_vector_negate = false;
741 : }
742 :
743 : /* Analysis phase. Determine if all op1 elements are either power
744 : of two and it is possible to expand it using shifts (or for remainder
745 : using masking). Additionally compute the multiplicative constants
746 : and pre and post shifts if the division is to be expanded using
747 : widening or high part multiplication plus shifts. */
748 4187 : for (i = 0; i < nunits; i++)
749 : {
750 3564 : tree cst = VECTOR_CST_ELT (op1, i);
751 3564 : unsigned HOST_WIDE_INT ml;
752 :
753 3564 : if (TREE_CODE (cst) != INTEGER_CST || integer_zerop (cst))
754 10 : return NULL_TREE;
755 3556 : pre_shifts[i] = 0;
756 3556 : post_shifts[i] = 0;
757 3556 : mulc[i] = 0;
758 3556 : if (use_pow2)
759 : {
760 : /* Divide by a negative power of two as by its absolute value and
761 : negate the quotient, as expand_divmod does. The quotient needs
762 : all elements to have the same sign; the remainder does not
763 : depend on the sign, as x % d == x % -d. INT_MIN is excluded,
764 : as its magnitude does not fit. */
765 1713 : int this_neg = 0;
766 1713 : wide_int w = wi::to_wide (cst);
767 1713 : if (sign_p == SIGNED && wi::neg_p (w))
768 : {
769 584 : this_neg = code == TRUNC_DIV_EXPR;
770 584 : w = wi::neg (w);
771 : }
772 1713 : int log2 = wi::exact_log2 (w);
773 1713 : if (log2 < 0
774 1427 : || (sign_p == SIGNED && log2 >= prec - 1)
775 1388 : || (this_neg && !has_vector_negate)
776 1388 : || (negate_p >= 0 && negate_p != this_neg))
777 : use_pow2 = false;
778 : else
779 : {
780 1380 : negate_p = this_neg;
781 1380 : shifts[i] = log2;
782 1380 : if (shifts[i] != shifts[0]
783 352 : && code == TRUNC_DIV_EXPR
784 352 : && !has_vector_shift)
785 1713 : use_pow2 = false;
786 : }
787 1713 : }
788 3556 : if (mode == -2)
789 1272 : continue;
790 2433 : if (sign_p == UNSIGNED)
791 : {
792 822 : unsigned HOST_WIDE_INT mh;
793 822 : unsigned HOST_WIDE_INT d = TREE_INT_CST_LOW (cst) & mask;
794 :
795 822 : if (d >= (HOST_WIDE_INT_1U << (prec - 1)))
796 : /* FIXME: Can transform this into op0 >= op1 ? 1 : 0. */
797 : return NULL_TREE;
798 :
799 820 : if (d <= 1)
800 : {
801 32 : mode = -2;
802 32 : continue;
803 : }
804 :
805 : /* Find a suitable multiplier and right shift count instead of
806 : directly dividing by D. */
807 788 : mh = choose_multiplier (d, prec, prec, &ml, &post_shift);
808 :
809 : /* If the suggested multiplier is more than PREC bits, we can
810 : do better for even divisors, using an initial right shift. */
811 788 : if ((mh != 0 && (d & 1) == 0)
812 646 : || (!has_vector_shift && pre_shift != -1))
813 : {
814 142 : if (has_vector_shift)
815 0 : pre_shift = ctz_or_zero (d);
816 174 : else if (pre_shift == -1)
817 : {
818 : unsigned int j;
819 518 : for (j = 0; j < nunits; j++)
820 : {
821 440 : tree cst2 = VECTOR_CST_ELT (op1, j);
822 440 : unsigned HOST_WIDE_INT d2;
823 440 : int this_pre_shift;
824 :
825 440 : if (!tree_fits_uhwi_p (cst2))
826 : return NULL_TREE;
827 440 : d2 = tree_to_uhwi (cst2) & mask;
828 440 : if (d2 == 0)
829 : return NULL_TREE;
830 440 : this_pre_shift = floor_log2 (d2 & -d2);
831 440 : if (pre_shift == -1 || this_pre_shift < pre_shift)
832 78 : pre_shift = this_pre_shift;
833 : }
834 78 : if (i != 0 && pre_shift != 0)
835 : {
836 : /* Restart. */
837 0 : i = -1U;
838 0 : mode = -1;
839 0 : continue;
840 : }
841 : }
842 174 : if (pre_shift != 0)
843 : {
844 174 : if ((d >> pre_shift) <= 1)
845 : {
846 46 : mode = -2;
847 46 : continue;
848 : }
849 128 : mh = choose_multiplier (d >> pre_shift, prec,
850 : prec - pre_shift,
851 : &ml, &post_shift);
852 128 : gcc_assert (!mh);
853 128 : pre_shifts[i] = pre_shift;
854 : }
855 : }
856 742 : if (!mh)
857 : this_mode = 0;
858 : else
859 192 : this_mode = 1;
860 : }
861 : else
862 : {
863 1611 : HOST_WIDE_INT d = TREE_INT_CST_LOW (cst);
864 1611 : unsigned HOST_WIDE_INT abs_d;
865 :
866 1611 : if (d == -1)
867 : return NULL_TREE;
868 :
869 : /* Since d might be INT_MIN, we have to cast to
870 : unsigned HOST_WIDE_INT before negating to avoid
871 : undefined signed overflow. */
872 1611 : abs_d = (d >= 0
873 : ? (unsigned HOST_WIDE_INT) d
874 : : - (unsigned HOST_WIDE_INT) d);
875 :
876 : /* n rem d = n rem -d */
877 1611 : if (code == TRUNC_MOD_EXPR && d < 0)
878 : {
879 365 : d = abs_d;
880 365 : use_abs_op1 = true;
881 : }
882 1611 : if (abs_d == HOST_WIDE_INT_1U << (prec - 1))
883 : {
884 : /* This case is not handled correctly below. */
885 39 : mode = -2;
886 39 : continue;
887 : }
888 1572 : if (abs_d <= 1)
889 : {
890 32 : mode = -2;
891 32 : continue;
892 : }
893 :
894 1540 : choose_multiplier (abs_d, prec, prec - 1, &ml,
895 : &post_shift);
896 1540 : if (ml >= HOST_WIDE_INT_1U << (prec - 1))
897 : {
898 916 : this_mode = 4 + (d < 0);
899 916 : ml |= HOST_WIDE_INT_M1U << (prec - 1);
900 : }
901 : else
902 624 : this_mode = 2 + (d < 0);
903 : }
904 2282 : mulc[i] = ml;
905 2282 : post_shifts[i] = post_shift;
906 2282 : if ((i && !has_vector_shift && post_shifts[0] != post_shift)
907 2106 : || post_shift >= prec
908 2106 : || pre_shifts[i] >= prec)
909 176 : this_mode = -2;
910 :
911 2282 : if (i == 0)
912 : mode = this_mode;
913 1802 : else if (mode != this_mode)
914 184 : mode = -2;
915 : }
916 :
917 623 : if (use_pow2)
918 : {
919 226 : tree addend = NULL_TREE;
920 226 : if (sign_p == SIGNED)
921 : {
922 : tree uns_type;
923 :
924 : /* Both division and remainder sequences need
925 : op0 < 0 ? mask : 0 computed. It can be either computed as
926 : (type) (((uns_type) (op0 >> (prec - 1))) >> (prec - shifts[i]))
927 : if none of the shifts is 0, or as the conditional. */
928 920 : for (i = 0; i < nunits; i++)
929 780 : if (shifts[i] == 0)
930 : break;
931 164 : uns_type
932 164 : = build_vector_type (build_nonstandard_integer_type (prec, 1),
933 164 : nunits);
934 164 : if (i == nunits && TYPE_MODE (uns_type) == TYPE_MODE (type))
935 : {
936 880 : for (i = 0; i < nunits; i++)
937 740 : shift_temps[i] = prec - 1;
938 140 : cur_op = add_shift (gsi, type, op0, shift_temps, RSHIFT_EXPR);
939 140 : if (cur_op != NULL_TREE)
940 : {
941 140 : cur_op = gimplify_build1 (gsi, VIEW_CONVERT_EXPR,
942 : uns_type, cur_op);
943 1020 : for (i = 0; i < nunits; i++)
944 740 : shift_temps[i] = prec - shifts[i];
945 140 : cur_op = add_shift (gsi, uns_type, cur_op, shift_temps,
946 : RSHIFT_EXPR);
947 140 : if (cur_op != NULL_TREE)
948 108 : addend = gimplify_build1 (gsi, VIEW_CONVERT_EXPR,
949 : type, cur_op);
950 : }
951 : }
952 164 : tree mask_type = truth_type_for (type);
953 164 : if (addend == NULL_TREE
954 56 : && expand_vec_cmp_expr_p (type, mask_type, LT_EXPR)
955 212 : && expand_vec_cond_expr_p (type, mask_type))
956 : {
957 48 : tree zero, cst, mask_type, mask;
958 48 : gimple *stmt, *cond;
959 :
960 48 : mask_type = truth_type_for (type);
961 48 : zero = build_zero_cst (type);
962 48 : mask = make_ssa_name (mask_type);
963 48 : cond = gimple_build_assign (mask, LT_EXPR, op0, zero);
964 48 : gsi_insert_before (gsi, cond, GSI_SAME_STMT);
965 48 : tree_vector_builder vec (type, nunits, 1);
966 352 : for (i = 0; i < nunits; i++)
967 256 : vec.quick_push (build_int_cst (TREE_TYPE (type),
968 : (HOST_WIDE_INT_1U
969 256 : << shifts[i]) - 1));
970 48 : cst = vec.build ();
971 48 : addend = make_ssa_name (type);
972 48 : stmt
973 48 : = gimple_build_assign (addend, VEC_COND_EXPR, mask, cst, zero);
974 48 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
975 48 : }
976 : }
977 226 : if (code == TRUNC_DIV_EXPR)
978 : {
979 80 : if (sign_p == UNSIGNED)
980 : {
981 : /* q = op0 >> shift; */
982 30 : cur_op = add_shift (gsi, type, op0, shifts, RSHIFT_EXPR);
983 30 : if (cur_op != NULL_TREE)
984 : return cur_op;
985 : }
986 50 : else if (addend != NULL_TREE)
987 : {
988 : /* t1 = op0 + addend;
989 : q = t1 >> shift;
990 : and, for a negative power-of-two divisor, q = -q. */
991 50 : op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
992 50 : if (op != unknown_optab
993 50 : && can_implement_p (op, TYPE_MODE (type)))
994 : {
995 50 : cur_op = gimplify_build2 (gsi, PLUS_EXPR, type, op0, addend);
996 50 : cur_op = add_shift (gsi, type, cur_op, shifts, RSHIFT_EXPR);
997 50 : if (cur_op != NULL_TREE)
998 : {
999 50 : if (negate_p == 1)
1000 26 : cur_op = gimplify_build1 (gsi, NEGATE_EXPR, type,
1001 : cur_op);
1002 : return cur_op;
1003 : }
1004 : }
1005 : }
1006 : }
1007 : else
1008 : {
1009 146 : tree mask;
1010 146 : tree_vector_builder vec (type, nunits, 1);
1011 1068 : for (i = 0; i < nunits; i++)
1012 776 : vec.quick_push (build_int_cst (TREE_TYPE (type),
1013 : (HOST_WIDE_INT_1U
1014 776 : << shifts[i]) - 1));
1015 146 : mask = vec.build ();
1016 146 : op = optab_for_tree_code (BIT_AND_EXPR, type, optab_default);
1017 146 : if (op != unknown_optab
1018 146 : && can_implement_p (op, TYPE_MODE (type)))
1019 : {
1020 146 : if (sign_p == UNSIGNED)
1021 : /* r = op0 & mask; */
1022 32 : return gimplify_build2 (gsi, BIT_AND_EXPR, type, op0, mask);
1023 114 : else if (addend != NULL_TREE)
1024 : {
1025 : /* t1 = op0 + addend;
1026 : t2 = t1 & mask;
1027 : r = t2 - addend; */
1028 106 : op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
1029 106 : if (op != unknown_optab
1030 106 : && can_implement_p (op, TYPE_MODE (type)))
1031 : {
1032 106 : cur_op = gimplify_build2 (gsi, PLUS_EXPR, type, op0,
1033 : addend);
1034 106 : cur_op = gimplify_build2 (gsi, BIT_AND_EXPR, type,
1035 : cur_op, mask);
1036 106 : op = optab_for_tree_code (MINUS_EXPR, type,
1037 : optab_default);
1038 106 : if (op != unknown_optab
1039 106 : && can_implement_p (op, TYPE_MODE (type)))
1040 106 : return gimplify_build2 (gsi, MINUS_EXPR, type,
1041 106 : cur_op, addend);
1042 : }
1043 : }
1044 : }
1045 146 : }
1046 : }
1047 :
1048 405 : if (mode == -2 || BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN)
1049 : return NULL_TREE;
1050 :
1051 182 : if (!can_mult_highpart_p (TYPE_MODE (type), TYPE_UNSIGNED (type)))
1052 : return NULL_TREE;
1053 :
1054 176 : cur_op = op0;
1055 :
1056 176 : switch (mode)
1057 : {
1058 69 : case 0:
1059 69 : gcc_assert (sign_p == UNSIGNED);
1060 : /* t1 = oprnd0 >> pre_shift;
1061 : t2 = t1 h* ml;
1062 : q = t2 >> post_shift; */
1063 69 : cur_op = add_shift (gsi, type, cur_op, pre_shifts, RSHIFT_EXPR);
1064 69 : if (cur_op == NULL_TREE)
1065 : return NULL_TREE;
1066 : break;
1067 32 : case 1:
1068 32 : gcc_assert (sign_p == UNSIGNED);
1069 224 : for (i = 0; i < nunits; i++)
1070 : {
1071 192 : shift_temps[i] = 1;
1072 192 : post_shifts[i]--;
1073 : }
1074 : break;
1075 75 : case 2:
1076 75 : case 3:
1077 75 : case 4:
1078 75 : case 5:
1079 75 : gcc_assert (sign_p == SIGNED);
1080 619 : for (i = 0; i < nunits; i++)
1081 544 : shift_temps[i] = prec - 1;
1082 : break;
1083 : default:
1084 : return NULL_TREE;
1085 : }
1086 :
1087 176 : tree_vector_builder vec (type, nunits, 1);
1088 1492 : for (i = 0; i < nunits; i++)
1089 1140 : vec.quick_push (build_int_cst (TREE_TYPE (type), mulc[i]));
1090 176 : mulcst = vec.build ();
1091 :
1092 176 : cur_op = gimplify_build2 (gsi, MULT_HIGHPART_EXPR, type, cur_op, mulcst);
1093 :
1094 176 : switch (mode)
1095 : {
1096 69 : case 0:
1097 : /* t1 = oprnd0 >> pre_shift;
1098 : t2 = t1 h* ml;
1099 : q = t2 >> post_shift; */
1100 69 : cur_op = add_shift (gsi, type, cur_op, post_shifts, RSHIFT_EXPR);
1101 69 : break;
1102 32 : case 1:
1103 : /* t1 = oprnd0 h* ml;
1104 : t2 = oprnd0 - t1;
1105 : t3 = t2 >> 1;
1106 : t4 = t1 + t3;
1107 : q = t4 >> (post_shift - 1); */
1108 32 : op = optab_for_tree_code (MINUS_EXPR, type, optab_default);
1109 32 : if (op == unknown_optab
1110 32 : || !can_implement_p (op, TYPE_MODE (type)))
1111 : return NULL_TREE;
1112 32 : tem = gimplify_build2 (gsi, MINUS_EXPR, type, op0, cur_op);
1113 32 : tem = add_shift (gsi, type, tem, shift_temps, RSHIFT_EXPR);
1114 32 : op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
1115 32 : if (op == unknown_optab
1116 32 : || !can_implement_p (op, TYPE_MODE (type)))
1117 : return NULL_TREE;
1118 32 : tem = gimplify_build2 (gsi, PLUS_EXPR, type, cur_op, tem);
1119 32 : cur_op = add_shift (gsi, type, tem, post_shifts, RSHIFT_EXPR);
1120 32 : if (cur_op == NULL_TREE)
1121 : return NULL_TREE;
1122 : break;
1123 75 : case 2:
1124 75 : case 3:
1125 75 : case 4:
1126 75 : case 5:
1127 : /* t1 = oprnd0 h* ml;
1128 : t2 = t1; [ iff (mode & 2) != 0 ]
1129 : t2 = t1 + oprnd0; [ iff (mode & 2) == 0 ]
1130 : t3 = t2 >> post_shift;
1131 : t4 = oprnd0 >> (prec - 1);
1132 : q = t3 - t4; [ iff (mode & 1) == 0 ]
1133 : q = t4 - t3; [ iff (mode & 1) != 0 ] */
1134 75 : if ((mode & 2) == 0)
1135 : {
1136 16 : op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
1137 16 : if (op == unknown_optab
1138 16 : || !can_implement_p (op, TYPE_MODE (type)))
1139 : return NULL_TREE;
1140 16 : cur_op = gimplify_build2 (gsi, PLUS_EXPR, type, cur_op, op0);
1141 : }
1142 75 : cur_op = add_shift (gsi, type, cur_op, post_shifts, RSHIFT_EXPR);
1143 75 : if (cur_op == NULL_TREE)
1144 : return NULL_TREE;
1145 75 : tem = add_shift (gsi, type, op0, shift_temps, RSHIFT_EXPR);
1146 75 : if (tem == NULL_TREE)
1147 : return NULL_TREE;
1148 75 : op = optab_for_tree_code (MINUS_EXPR, type, optab_default);
1149 75 : if (op == unknown_optab
1150 75 : || !can_implement_p (op, TYPE_MODE (type)))
1151 : return NULL_TREE;
1152 75 : if ((mode & 1) == 0)
1153 75 : cur_op = gimplify_build2 (gsi, MINUS_EXPR, type, cur_op, tem);
1154 : else
1155 0 : cur_op = gimplify_build2 (gsi, MINUS_EXPR, type, tem, cur_op);
1156 : break;
1157 : default:
1158 : gcc_unreachable ();
1159 : }
1160 :
1161 176 : if (code == TRUNC_DIV_EXPR)
1162 : return cur_op;
1163 :
1164 : /* We divided. Now finish by:
1165 : t1 = q * oprnd1;
1166 : r = oprnd0 - t1; */
1167 87 : op = optab_for_tree_code (MULT_EXPR, type, optab_default);
1168 87 : if (op == unknown_optab
1169 87 : || !can_implement_p (op, TYPE_MODE (type)))
1170 : return NULL_TREE;
1171 87 : if (use_abs_op1)
1172 : {
1173 7 : tree_vector_builder elts;
1174 7 : if (!elts.new_unary_operation (type, op1, false))
1175 : return NULL_TREE;
1176 7 : unsigned int count = elts.encoded_nelts ();
1177 14 : for (unsigned int i = 0; i < count; ++i)
1178 : {
1179 7 : tree elem1 = VECTOR_CST_ELT (op1, i);
1180 :
1181 7 : tree elt = const_unop (ABS_EXPR, TREE_TYPE (elem1), elem1);
1182 7 : if (elt == NULL_TREE)
1183 0 : return NULL_TREE;
1184 7 : elts.quick_push (elt);
1185 : }
1186 7 : op1 = elts.build ();
1187 7 : }
1188 87 : tem = gimplify_build2 (gsi, MULT_EXPR, type, cur_op, op1);
1189 87 : op = optab_for_tree_code (MINUS_EXPR, type, optab_default);
1190 87 : if (op == unknown_optab
1191 87 : || !can_implement_p (op, TYPE_MODE (type)))
1192 : return NULL_TREE;
1193 87 : return gimplify_build2 (gsi, MINUS_EXPR, type, op0, tem);
1194 : }
1195 :
1196 : /* Expand a vector condition to scalars, by using many conditions
1197 : on the vector's elements. */
1198 :
1199 : static bool
1200 23907 : expand_vector_condition (gimple_stmt_iterator *gsi)
1201 : {
1202 23907 : gassign *stmt = as_a <gassign *> (gsi_stmt (*gsi));
1203 23907 : tree type = TREE_TYPE (gimple_assign_lhs (stmt));
1204 23907 : tree a = gimple_assign_rhs1 (stmt);
1205 23907 : tree a1 = a;
1206 23907 : tree a2 = NULL_TREE;
1207 23907 : bool a_is_scalar_bitmask = false;
1208 23907 : tree b = gimple_assign_rhs2 (stmt);
1209 23907 : tree c = gimple_assign_rhs3 (stmt);
1210 23907 : vec<constructor_elt, va_gc> *v;
1211 23907 : tree constr;
1212 23907 : tree inner_type = TREE_TYPE (type);
1213 23907 : tree width = vector_element_bits_tree (type);
1214 23907 : tree cond_type = TREE_TYPE (TREE_TYPE (a));
1215 23907 : tree index = bitsize_int (0);
1216 23907 : tree comp_width = width;
1217 23907 : tree comp_index = index;
1218 23907 : location_t loc = gimple_location (gsi_stmt (*gsi));
1219 :
1220 23907 : gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (a)));
1221 :
1222 23907 : if (expand_vec_cond_expr_p (type, TREE_TYPE (a)))
1223 : return true;
1224 :
1225 : /* Handle vector boolean types with bitmasks. We can transform
1226 : vbfld_1 = tmp_6 ? vbfld_4 : vbfld_5;
1227 : into
1228 : tmp_7 = tmp_6 & vbfld_4;
1229 : tmp_8 = ~tmp_6;
1230 : tmp_9 = tmp_8 & vbfld_5;
1231 : vbfld_1 = tmp_7 | tmp_9; */
1232 539 : if (VECTOR_BOOLEAN_TYPE_P (type)
1233 0 : && SCALAR_INT_MODE_P (TYPE_MODE (type))
1234 539 : && useless_type_conversion_p (type, TREE_TYPE (a)))
1235 : {
1236 0 : a1 = gimplify_build2 (gsi, BIT_AND_EXPR, type, a, b);
1237 0 : a2 = gimplify_build1 (gsi, BIT_NOT_EXPR, type, a);
1238 0 : a2 = gimplify_build2 (gsi, BIT_AND_EXPR, type, a2, c);
1239 0 : a = gimplify_build2 (gsi, BIT_IOR_EXPR, type, a1, a2);
1240 0 : gimple_assign_set_rhs_from_tree (gsi, a);
1241 0 : update_stmt (gsi_stmt (*gsi));
1242 0 : return true;
1243 : }
1244 :
1245 : /* TODO: try and find a smaller vector type. */
1246 :
1247 539 : if (!warning_suppressed_p (stmt, OPT_Wvector_operation_performance))
1248 539 : warning_at (loc, OPT_Wvector_operation_performance,
1249 : "vector condition will be expanded piecewise");
1250 :
1251 1042 : if (SCALAR_INT_MODE_P (TYPE_MODE (TREE_TYPE (a)))
1252 611 : && known_lt (GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (a))),
1253 : TYPE_VECTOR_SUBPARTS (TREE_TYPE (a))
1254 : * GET_MODE_BITSIZE (SCALAR_TYPE_MODE
1255 : (TREE_TYPE (TREE_TYPE (a))))))
1256 : {
1257 0 : a_is_scalar_bitmask = true;
1258 0 : int prec = GET_MODE_PRECISION (SCALAR_TYPE_MODE (TREE_TYPE (a)));
1259 0 : tree atype = build_nonstandard_integer_type (prec, 1);
1260 0 : a = gimplify_build1 (gsi, VIEW_CONVERT_EXPR, atype, a);
1261 : }
1262 : else
1263 539 : comp_width = vector_element_bits_tree (TREE_TYPE (a));
1264 :
1265 539 : int nunits = nunits_for_known_piecewise_op (type);
1266 539 : vec_alloc (v, nunits);
1267 539 : bool constant_p = true;
1268 7143 : for (int i = 0; i < nunits; i++)
1269 : {
1270 6065 : tree aa, result;
1271 6065 : tree bb = tree_vec_extract (gsi, inner_type, b, width, index);
1272 6065 : tree cc = tree_vec_extract (gsi, inner_type, c, width, index);
1273 6065 : if (a_is_scalar_bitmask)
1274 : {
1275 0 : wide_int w = wi::set_bit_in_zero (i, TYPE_PRECISION (TREE_TYPE (a)));
1276 0 : result = gimplify_build2 (gsi, BIT_AND_EXPR, TREE_TYPE (a),
1277 0 : a, wide_int_to_tree (TREE_TYPE (a), w));
1278 0 : aa = gimplify_build2 (gsi, NE_EXPR, boolean_type_node, result,
1279 0 : build_zero_cst (TREE_TYPE (a)));
1280 0 : }
1281 : else
1282 : {
1283 6065 : result = tree_vec_extract (gsi, cond_type, a, comp_width, comp_index);
1284 6065 : aa = gimplify_build2 (gsi, NE_EXPR, boolean_type_node, result,
1285 : build_zero_cst (cond_type));
1286 : }
1287 6065 : result = gimplify_build3 (gsi, COND_EXPR, inner_type, aa, bb, cc);
1288 6065 : if (!CONSTANT_CLASS_P (result))
1289 5981 : constant_p = false;
1290 6065 : constructor_elt ce = {NULL_TREE, result};
1291 6065 : v->quick_push (ce);
1292 6065 : index = int_const_binop (PLUS_EXPR, index, width);
1293 6065 : if (width == comp_width)
1294 : comp_index = index;
1295 : else
1296 0 : comp_index = int_const_binop (PLUS_EXPR, comp_index, comp_width);
1297 : }
1298 :
1299 539 : if (constant_p)
1300 0 : constr = build_vector_from_ctor (type, v);
1301 : else
1302 539 : constr = build_constructor (type, v);
1303 539 : gimple_assign_set_rhs_from_tree (gsi, constr);
1304 539 : update_stmt (gsi_stmt (*gsi));
1305 :
1306 539 : return false;
1307 : }
1308 :
1309 : static tree
1310 30938 : expand_vector_operation (gimple_stmt_iterator *gsi, tree type, tree compute_type,
1311 : gassign *assign, enum tree_code code)
1312 : {
1313 30938 : machine_mode compute_mode = TYPE_MODE (compute_type);
1314 :
1315 : /* If the compute mode is not a vector mode (hence we are not decomposing
1316 : a BLKmode vector to smaller, hardware-supported vectors), we may want
1317 : to expand the operations in parallel. */
1318 30938 : if (!VECTOR_MODE_P (compute_mode))
1319 29957 : switch (code)
1320 : {
1321 351 : case PLUS_EXPR:
1322 351 : case MINUS_EXPR:
1323 351 : if (ANY_INTEGRAL_TYPE_P (type) && !TYPE_OVERFLOW_TRAPS (type))
1324 404 : return expand_vector_addition (gsi, do_binop, do_plus_minus, type,
1325 : gimple_assign_rhs1 (assign),
1326 202 : gimple_assign_rhs2 (assign), code);
1327 : break;
1328 :
1329 9 : case NEGATE_EXPR:
1330 9 : if (ANY_INTEGRAL_TYPE_P (type) && !TYPE_OVERFLOW_TRAPS (type))
1331 8 : return expand_vector_addition (gsi, do_unop, do_negate, type,
1332 : gimple_assign_rhs1 (assign),
1333 8 : NULL_TREE, code);
1334 : break;
1335 :
1336 92 : case BIT_AND_EXPR:
1337 92 : case BIT_IOR_EXPR:
1338 92 : case BIT_XOR_EXPR:
1339 184 : return expand_vector_parallel (gsi, do_binop, type,
1340 : gimple_assign_rhs1 (assign),
1341 92 : gimple_assign_rhs2 (assign), code);
1342 :
1343 9 : case BIT_NOT_EXPR:
1344 9 : return expand_vector_parallel (gsi, do_unop, type,
1345 : gimple_assign_rhs1 (assign),
1346 9 : NULL_TREE, code);
1347 27566 : case EQ_EXPR:
1348 27566 : case NE_EXPR:
1349 27566 : case GT_EXPR:
1350 27566 : case LT_EXPR:
1351 27566 : case GE_EXPR:
1352 27566 : case LE_EXPR:
1353 27566 : case UNEQ_EXPR:
1354 27566 : case UNGT_EXPR:
1355 27566 : case UNLT_EXPR:
1356 27566 : case UNGE_EXPR:
1357 27566 : case UNLE_EXPR:
1358 27566 : case LTGT_EXPR:
1359 27566 : case ORDERED_EXPR:
1360 27566 : case UNORDERED_EXPR:
1361 27566 : {
1362 27566 : tree rhs1 = gimple_assign_rhs1 (assign);
1363 27566 : tree rhs2 = gimple_assign_rhs2 (assign);
1364 :
1365 27566 : return expand_vector_comparison (gsi, type, rhs1, rhs2, code);
1366 : }
1367 :
1368 991 : case TRUNC_DIV_EXPR:
1369 991 : case TRUNC_MOD_EXPR:
1370 991 : {
1371 991 : tree rhs1 = gimple_assign_rhs1 (assign);
1372 991 : tree rhs2 = gimple_assign_rhs2 (assign);
1373 991 : tree ret;
1374 :
1375 991 : if (!optimize
1376 872 : || !VECTOR_INTEGER_TYPE_P (type)
1377 872 : || TREE_CODE (rhs2) != VECTOR_CST
1378 1631 : || !VECTOR_MODE_P (TYPE_MODE (type)))
1379 : break;
1380 :
1381 634 : ret = expand_vector_divmod (gsi, type, rhs1, rhs2, code);
1382 634 : if (ret != NULL_TREE)
1383 : return ret;
1384 : break;
1385 : }
1386 109 : case MULT_EXPR:
1387 109 : {
1388 109 : tree rhs1 = gimple_assign_rhs1 (assign);
1389 109 : tree rhs2 = gimple_assign_rhs2 (assign);
1390 109 : tree ret;
1391 :
1392 109 : if (!optimize
1393 87 : || !VECTOR_INTEGER_TYPE_P (type)
1394 66 : || TREE_CODE (rhs2) != VECTOR_CST
1395 117 : || !VECTOR_MODE_P (TYPE_MODE (type)))
1396 : break;
1397 :
1398 6 : ret = expand_vector_mult (gsi, type, rhs1, rhs2);
1399 6 : if (ret != NULL_TREE)
1400 : return ret;
1401 : break;
1402 : }
1403 : default:
1404 : break;
1405 : }
1406 :
1407 2666 : if (TREE_CODE_CLASS (code) == tcc_unary)
1408 140 : return expand_vector_piecewise (gsi, do_unop, type, compute_type,
1409 : gimple_assign_rhs1 (assign),
1410 140 : NULL_TREE, code, false);
1411 : else
1412 5052 : return expand_vector_piecewise (gsi, do_binop, type, compute_type,
1413 : gimple_assign_rhs1 (assign),
1414 2526 : gimple_assign_rhs2 (assign), code, false);
1415 : }
1416 :
1417 : /* Try to optimize
1418 : a_5 = { b_7, b_7 + 3, b_7 + 6, b_7 + 9 };
1419 : style stmts into:
1420 : _9 = { b_7, b_7, b_7, b_7 };
1421 : a_5 = _9 + { 0, 3, 6, 9 };
1422 : because vector splat operation is usually more efficient
1423 : than piecewise initialization of the vector. */
1424 :
1425 : static void
1426 148070 : optimize_vector_constructor (gimple_stmt_iterator *gsi)
1427 : {
1428 148070 : gassign *stmt = as_a <gassign *> (gsi_stmt (*gsi));
1429 148070 : tree lhs = gimple_assign_lhs (stmt);
1430 148070 : tree rhs = gimple_assign_rhs1 (stmt);
1431 148070 : tree type = TREE_TYPE (rhs);
1432 148070 : unsigned int i, j;
1433 148070 : unsigned HOST_WIDE_INT nelts;
1434 148070 : bool all_same = true;
1435 148070 : constructor_elt *elt;
1436 148070 : gimple *g;
1437 148070 : tree base = NULL_TREE;
1438 148070 : optab op;
1439 :
1440 149841 : if (!TYPE_VECTOR_SUBPARTS (type).is_constant (&nelts)
1441 148070 : || nelts <= 2
1442 46744 : || CONSTRUCTOR_NELTS (rhs) != nelts)
1443 148058 : return;
1444 21604 : op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
1445 21604 : if (op == unknown_optab
1446 21604 : || !can_implement_p (op, TYPE_MODE (type)))
1447 : return;
1448 104527 : FOR_EACH_VEC_SAFE_ELT (CONSTRUCTOR_ELTS (rhs), i, elt)
1449 92746 : if (TREE_CODE (elt->value) != SSA_NAME
1450 92746 : || TREE_CODE (TREE_TYPE (elt->value)) == VECTOR_TYPE)
1451 : return;
1452 : else
1453 : {
1454 90613 : tree this_base = elt->value;
1455 90613 : if (this_base != CONSTRUCTOR_ELT (rhs, 0)->value)
1456 7194 : all_same = false;
1457 91694 : for (j = 0; j < nelts + 1; j++)
1458 : {
1459 91694 : g = SSA_NAME_DEF_STMT (this_base);
1460 91694 : if (is_gimple_assign (g)
1461 58295 : && gimple_assign_rhs_code (g) == PLUS_EXPR
1462 2590 : && TREE_CODE (gimple_assign_rhs2 (g)) == INTEGER_CST
1463 1081 : && TREE_CODE (gimple_assign_rhs1 (g)) == SSA_NAME
1464 92775 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (gimple_assign_rhs1 (g)))
1465 1081 : this_base = gimple_assign_rhs1 (g);
1466 : else
1467 : break;
1468 : }
1469 90613 : if (i == 0)
1470 : base = this_base;
1471 70093 : else if (this_base != base)
1472 : return;
1473 : }
1474 11781 : if (all_same)
1475 : return;
1476 12 : tree_vector_builder cst (type, nelts, 1);
1477 72 : for (i = 0; i < nelts; i++)
1478 : {
1479 48 : tree this_base = CONSTRUCTOR_ELT (rhs, i)->value;
1480 48 : tree elt = build_zero_cst (TREE_TYPE (base));
1481 93 : while (this_base != base)
1482 : {
1483 45 : g = SSA_NAME_DEF_STMT (this_base);
1484 45 : elt = fold_binary (PLUS_EXPR, TREE_TYPE (base),
1485 : elt, gimple_assign_rhs2 (g));
1486 45 : if (elt == NULL_TREE
1487 45 : || TREE_CODE (elt) != INTEGER_CST
1488 90 : || TREE_OVERFLOW (elt))
1489 0 : return;
1490 45 : this_base = gimple_assign_rhs1 (g);
1491 : }
1492 48 : cst.quick_push (elt);
1493 : }
1494 60 : for (i = 0; i < nelts; i++)
1495 48 : CONSTRUCTOR_ELT (rhs, i)->value = base;
1496 12 : g = gimple_build_assign (make_ssa_name (type), rhs);
1497 12 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
1498 12 : g = gimple_build_assign (lhs, PLUS_EXPR, gimple_assign_lhs (g),
1499 : cst.build ());
1500 12 : gsi_replace (gsi, g, false);
1501 12 : }
1502 :
1503 : /* Return a type for the widest vector mode with the same element type as
1504 : type ORIGINAL_VECTOR_TYPE, with at most the same number of elements as type
1505 : ORIGINAL_VECTOR_TYPE and that is supported by the target for an operation
1506 : with optab OP, or return NULL_TREE if none is found. */
1507 :
1508 : static tree
1509 3877 : type_for_widest_vector_mode (tree original_vector_type, optab op)
1510 : {
1511 3877 : gcc_assert (VECTOR_TYPE_P (original_vector_type));
1512 3877 : tree type = TREE_TYPE (original_vector_type);
1513 3877 : machine_mode inner_mode = TYPE_MODE (type);
1514 3877 : machine_mode best_mode = VOIDmode, mode;
1515 3877 : poly_int64 best_nunits = 0;
1516 :
1517 3877 : if (SCALAR_FLOAT_MODE_P (inner_mode))
1518 : mode = MIN_MODE_VECTOR_FLOAT;
1519 3514 : else if (SCALAR_FRACT_MODE_P (inner_mode))
1520 : mode = MIN_MODE_VECTOR_FRACT;
1521 3514 : else if (SCALAR_UFRACT_MODE_P (inner_mode))
1522 : mode = MIN_MODE_VECTOR_UFRACT;
1523 3514 : else if (SCALAR_ACCUM_MODE_P (inner_mode))
1524 : mode = MIN_MODE_VECTOR_ACCUM;
1525 3514 : else if (SCALAR_UACCUM_MODE_P (inner_mode))
1526 : mode = MIN_MODE_VECTOR_UACCUM;
1527 3514 : else if (inner_mode == BImode)
1528 : mode = MIN_MODE_VECTOR_BOOL;
1529 : else
1530 3877 : mode = MIN_MODE_VECTOR_INT;
1531 :
1532 121886 : FOR_EACH_MODE_FROM (mode, mode)
1533 118009 : if (GET_MODE_INNER (mode) == inner_mode
1534 45876 : && maybe_gt (GET_MODE_NUNITS (mode), best_nunits)
1535 22938 : && can_implement_p (op, mode)
1536 143959 : && known_le (GET_MODE_NUNITS (mode),
1537 : TYPE_VECTOR_SUBPARTS (original_vector_type)))
1538 4482 : best_mode = mode, best_nunits = GET_MODE_NUNITS (mode);
1539 :
1540 3877 : if (best_mode == VOIDmode)
1541 : return NULL_TREE;
1542 : else
1543 1205 : return build_vector_type_for_mode (type, best_mode);
1544 : }
1545 :
1546 :
1547 : /* Build a reference to the element of the vector VECT. Function
1548 : returns either the element itself, either BIT_FIELD_REF, or an
1549 : ARRAY_REF expression.
1550 :
1551 : GSI is required to insert temporary variables while building a
1552 : reference to the element of the vector VECT.
1553 :
1554 : PTMPVEC is a pointer to the temporary variable for caching
1555 : purposes. In case when PTMPVEC is NULL new temporary variable
1556 : will be created. */
1557 : static tree
1558 134271 : vector_element (gimple_stmt_iterator *gsi, tree vect, tree idx, tree *ptmpvec)
1559 : {
1560 134271 : tree vect_type, vect_elt_type;
1561 134271 : gimple *asgn;
1562 134271 : tree tmpvec;
1563 134271 : tree arraytype;
1564 134271 : bool need_asgn = true;
1565 134271 : unsigned int elements;
1566 :
1567 134271 : vect_type = TREE_TYPE (vect);
1568 134271 : vect_elt_type = TREE_TYPE (vect_type);
1569 134271 : elements = nunits_for_known_piecewise_op (vect_type);
1570 :
1571 134271 : if (TREE_CODE (idx) == INTEGER_CST)
1572 : {
1573 124346 : unsigned HOST_WIDE_INT index;
1574 :
1575 : /* Given that we're about to compute a binary modulus,
1576 : we don't care about the high bits of the value. */
1577 124346 : index = TREE_INT_CST_LOW (idx);
1578 124346 : if (!tree_fits_uhwi_p (idx) || index >= elements)
1579 : {
1580 0 : index &= elements - 1;
1581 0 : idx = build_int_cst (TREE_TYPE (idx), index);
1582 : }
1583 :
1584 : /* When lowering a vector statement sequence do some easy
1585 : simplification by looking through intermediate vector results. */
1586 124346 : if (TREE_CODE (vect) == SSA_NAME)
1587 : {
1588 65539 : gimple *def_stmt = SSA_NAME_DEF_STMT (vect);
1589 65539 : if (is_gimple_assign (def_stmt)
1590 65539 : && (gimple_assign_rhs_code (def_stmt) == VECTOR_CST
1591 64089 : || gimple_assign_rhs_code (def_stmt) == CONSTRUCTOR))
1592 4061 : vect = gimple_assign_rhs1 (def_stmt);
1593 : }
1594 :
1595 124346 : if (TREE_CODE (vect) == VECTOR_CST)
1596 60094 : return VECTOR_CST_ELT (vect, index);
1597 64252 : else if (TREE_CODE (vect) == CONSTRUCTOR
1598 64252 : && (CONSTRUCTOR_NELTS (vect) == 0
1599 2774 : || TREE_CODE (TREE_TYPE (CONSTRUCTOR_ELT (vect, 0)->value))
1600 : != VECTOR_TYPE))
1601 : {
1602 485 : if (index < CONSTRUCTOR_NELTS (vect))
1603 485 : return CONSTRUCTOR_ELT (vect, index)->value;
1604 0 : return build_zero_cst (vect_elt_type);
1605 : }
1606 : else
1607 : {
1608 63767 : tree size = vector_element_bits_tree (vect_type);
1609 63767 : tree pos = fold_build2 (MULT_EXPR, bitsizetype, bitsize_int (index),
1610 : size);
1611 63767 : return fold_build3 (BIT_FIELD_REF, vect_elt_type, vect, size, pos);
1612 : }
1613 : }
1614 :
1615 9925 : if (!ptmpvec)
1616 0 : tmpvec = create_tmp_var (vect_type, "vectmp");
1617 9925 : else if (!*ptmpvec)
1618 1296 : tmpvec = *ptmpvec = create_tmp_var (vect_type, "vectmp");
1619 : else
1620 : {
1621 : tmpvec = *ptmpvec;
1622 : need_asgn = false;
1623 : }
1624 :
1625 1296 : if (need_asgn)
1626 : {
1627 1296 : TREE_ADDRESSABLE (tmpvec) = 1;
1628 1296 : asgn = gimple_build_assign (tmpvec, vect);
1629 1296 : gsi_insert_before (gsi, asgn, GSI_SAME_STMT);
1630 : }
1631 :
1632 9925 : arraytype = build_array_type_nelts (vect_elt_type, elements);
1633 9925 : return build4 (ARRAY_REF, vect_elt_type,
1634 : build1 (VIEW_CONVERT_EXPR, arraytype, tmpvec),
1635 9925 : idx, NULL_TREE, NULL_TREE);
1636 : }
1637 :
1638 : /* Check if VEC_PERM_EXPR within the given setting is supported
1639 : by hardware, or lower it piecewise.
1640 :
1641 : When VEC_PERM_EXPR has the same first and second operands:
1642 : VEC_PERM_EXPR <v0, v0, mask> the lowered version would be
1643 : {v0[mask[0]], v0[mask[1]], ...}
1644 : MASK and V0 must have the same number of elements.
1645 :
1646 : Otherwise VEC_PERM_EXPR <v0, v1, mask> is lowered to
1647 : {mask[0] < len(v0) ? v0[mask[0]] : v1[mask[0]], ...}
1648 : V0 and V1 must have the same type. MASK, V0, V1 must have the
1649 : same number of arguments. */
1650 :
1651 : static void
1652 91859 : lower_vec_perm (gimple_stmt_iterator *gsi)
1653 : {
1654 91859 : gassign *stmt = as_a <gassign *> (gsi_stmt (*gsi));
1655 91859 : tree mask = gimple_assign_rhs3 (stmt);
1656 91859 : tree vec0 = gimple_assign_rhs1 (stmt);
1657 91859 : tree vec1 = gimple_assign_rhs2 (stmt);
1658 91859 : tree res_vect_type = TREE_TYPE (gimple_assign_lhs (stmt));
1659 91859 : tree vect_type = TREE_TYPE (vec0);
1660 91859 : tree mask_type = TREE_TYPE (mask);
1661 91859 : tree vect_elt_type = TREE_TYPE (vect_type);
1662 : /* Perform the index computations in an unsigned type, the mask
1663 : elements are interpreted modulo the number of elements and with
1664 : a signed type masking with ELEMENTS - 1 can be a no-op when
1665 : ELEMENTS - 1 is not representable, leaving negative indices. */
1666 91859 : tree mask_elt_type = unsigned_type_for (TREE_TYPE (mask_type));
1667 91859 : unsigned HOST_WIDE_INT elements, in_elements;
1668 91859 : vec<constructor_elt, va_gc> *v;
1669 91859 : tree constr, t, si, i_val;
1670 91859 : tree vec0tmp = NULL_TREE, vec1tmp = NULL_TREE, masktmp = NULL_TREE;
1671 91859 : bool two_operand_p = !operand_equal_p (vec0, vec1, 0);
1672 91859 : location_t loc = gimple_location (gsi_stmt (*gsi));
1673 91859 : unsigned i;
1674 :
1675 91859 : if (!TYPE_VECTOR_SUBPARTS (res_vect_type).is_constant (&elements)
1676 91859 : || !TYPE_VECTOR_SUBPARTS (vect_type).is_constant (&in_elements))
1677 82291 : return;
1678 :
1679 91859 : if (TREE_CODE (mask) == SSA_NAME)
1680 : {
1681 901 : gimple *def_stmt = SSA_NAME_DEF_STMT (mask);
1682 901 : if (is_gimple_assign (def_stmt)
1683 901 : && gimple_assign_rhs_code (def_stmt) == VECTOR_CST)
1684 13 : mask = gimple_assign_rhs1 (def_stmt);
1685 : }
1686 :
1687 :
1688 91859 : vec_perm_indices indices;
1689 91859 : if (TREE_CODE (mask) == VECTOR_CST
1690 91859 : && tree_to_vec_perm_indices (&indices, vec0, vec1, mask))
1691 : {
1692 90970 : machine_mode vmode = TYPE_MODE (vect_type);
1693 90970 : tree lhs_type = TREE_TYPE (gimple_assign_lhs (stmt));
1694 90970 : machine_mode lhs_mode = TYPE_MODE (lhs_type);
1695 90970 : if (can_vec_perm_const_p (lhs_mode, vmode, indices, true))
1696 : {
1697 82272 : gimple_assign_set_rhs3 (stmt, mask);
1698 82272 : update_stmt (stmt);
1699 82272 : return;
1700 : }
1701 : }
1702 889 : else if (can_vec_perm_var_p (TYPE_MODE (vect_type)))
1703 : return;
1704 :
1705 9568 : if (!warning_suppressed_p (stmt, OPT_Wvector_operation_performance))
1706 9568 : warning_at (loc, OPT_Wvector_operation_performance,
1707 : "vector shuffling operation will be expanded piecewise");
1708 :
1709 9568 : vec_alloc (v, elements);
1710 9568 : bool constant_p = true;
1711 85032 : for (i = 0; i < elements; i++)
1712 : {
1713 65896 : si = size_int (i);
1714 65896 : i_val = vector_element (gsi, mask, si, &masktmp);
1715 65896 : i_val = fold_convert (mask_elt_type, i_val);
1716 :
1717 65896 : if (TREE_CODE (i_val) == INTEGER_CST)
1718 : {
1719 58450 : unsigned HOST_WIDE_INT index;
1720 :
1721 58450 : index = TREE_INT_CST_LOW (i_val);
1722 58450 : if (!tree_fits_uhwi_p (i_val) || index >= in_elements)
1723 18943 : i_val = build_int_cst (mask_elt_type, index & (in_elements - 1));
1724 :
1725 58450 : if (two_operand_p && (index & in_elements) != 0)
1726 18811 : t = vector_element (gsi, vec1, i_val, &vec1tmp);
1727 : else
1728 39639 : t = vector_element (gsi, vec0, i_val, &vec0tmp);
1729 :
1730 58450 : t = force_gimple_operand_gsi (gsi, t, true, NULL_TREE,
1731 : true, GSI_SAME_STMT);
1732 : }
1733 : else
1734 : {
1735 7446 : tree cond = NULL_TREE, v0_val;
1736 :
1737 7446 : if (two_operand_p)
1738 : {
1739 2479 : cond = fold_build2 (BIT_AND_EXPR, mask_elt_type, i_val,
1740 : build_int_cst (mask_elt_type, elements));
1741 2479 : cond = force_gimple_operand_gsi (gsi, cond, true, NULL_TREE,
1742 : true, GSI_SAME_STMT);
1743 : }
1744 :
1745 7446 : i_val = fold_build2 (BIT_AND_EXPR, mask_elt_type, i_val,
1746 : build_int_cst (mask_elt_type, elements - 1));
1747 7446 : i_val = force_gimple_operand_gsi (gsi, i_val, true, NULL_TREE,
1748 : true, GSI_SAME_STMT);
1749 :
1750 7446 : v0_val = vector_element (gsi, vec0, i_val, &vec0tmp);
1751 7446 : v0_val = force_gimple_operand_gsi (gsi, v0_val, true, NULL_TREE,
1752 : true, GSI_SAME_STMT);
1753 :
1754 7446 : if (two_operand_p)
1755 : {
1756 2479 : tree v1_val;
1757 :
1758 2479 : v1_val = vector_element (gsi, vec1, i_val, &vec1tmp);
1759 2479 : v1_val = force_gimple_operand_gsi (gsi, v1_val, true, NULL_TREE,
1760 : true, GSI_SAME_STMT);
1761 :
1762 2479 : cond = fold_build2 (EQ_EXPR, boolean_type_node,
1763 : cond, build_zero_cst (mask_elt_type));
1764 2479 : cond = fold_build3 (COND_EXPR, vect_elt_type,
1765 : cond, v0_val, v1_val);
1766 2479 : t = force_gimple_operand_gsi (gsi, cond, true, NULL_TREE,
1767 : true, GSI_SAME_STMT);
1768 : }
1769 : else
1770 : t = v0_val;
1771 : }
1772 :
1773 65896 : if (!CONSTANT_CLASS_P (t))
1774 63629 : constant_p = false;
1775 65896 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, t);
1776 : }
1777 :
1778 9568 : if (constant_p)
1779 8 : constr = build_vector_from_ctor (res_vect_type, v);
1780 : else
1781 9560 : constr = build_constructor (res_vect_type, v);
1782 9568 : gimple_assign_set_rhs_from_tree (gsi, constr);
1783 9568 : update_stmt (gsi_stmt (*gsi));
1784 91859 : }
1785 :
1786 : /* Return the type that should be used to implement OP on type TYPE.
1787 : This is TYPE itself if the target can do the operation directly,
1788 : otherwise it is a scalar type or a smaller vector type. */
1789 :
1790 : static tree
1791 230401 : get_compute_type (optab op, tree type)
1792 : {
1793 230401 : if (op)
1794 : {
1795 405670 : if (VECTOR_MODE_P (TYPE_MODE (type))
1796 404042 : && can_implement_p (op, TYPE_MODE (type)))
1797 : return type;
1798 :
1799 : /* For very wide vectors, try using a smaller vector mode. */
1800 3800 : tree vector_compute_type = type_for_widest_vector_mode (type, op);
1801 3800 : if (vector_compute_type != NULL_TREE
1802 3800 : && maybe_ne (TYPE_VECTOR_SUBPARTS (vector_compute_type), 1U)
1803 4873 : && can_implement_p (op, TYPE_MODE (vector_compute_type)))
1804 1073 : return vector_compute_type;
1805 : }
1806 :
1807 : /* There is no operation in hardware, so fall back to scalars. */
1808 30293 : return TREE_TYPE (type);
1809 : }
1810 :
1811 : static tree
1812 14 : do_cond (gimple_stmt_iterator *gsi, tree inner_type, tree a, tree b,
1813 : tree bitpos, tree bitsize, enum tree_code code,
1814 : tree type ATTRIBUTE_UNUSED)
1815 : {
1816 14 : if (VECTOR_TYPE_P (TREE_TYPE (a)))
1817 14 : a = tree_vec_extract (gsi, inner_type, a, bitsize, bitpos);
1818 14 : if (VECTOR_TYPE_P (TREE_TYPE (b)))
1819 14 : b = tree_vec_extract (gsi, inner_type, b, bitsize, bitpos);
1820 14 : tree cond = gimple_assign_rhs1 (gsi_stmt (*gsi));
1821 14 : return gimplify_build3 (gsi, code, inner_type, unshare_expr (cond), a, b);
1822 : }
1823 :
1824 : /* Expand a vector COND_EXPR to scalars, piecewise. */
1825 : static void
1826 7 : expand_vector_scalar_condition (gimple_stmt_iterator *gsi)
1827 : {
1828 7 : gassign *stmt = as_a <gassign *> (gsi_stmt (*gsi));
1829 7 : tree lhs = gimple_assign_lhs (stmt);
1830 7 : tree type = TREE_TYPE (lhs);
1831 7 : tree compute_type = get_compute_type (mov_optab, type);
1832 7 : machine_mode compute_mode = TYPE_MODE (compute_type);
1833 7 : gcc_assert (compute_mode != BLKmode);
1834 7 : tree rhs2 = gimple_assign_rhs2 (stmt);
1835 7 : tree rhs3 = gimple_assign_rhs3 (stmt);
1836 7 : tree new_rhs;
1837 :
1838 : /* If the compute mode is not a vector mode (hence we are not decomposing
1839 : a BLKmode vector to smaller, hardware-supported vectors), we may want
1840 : to expand the operations in parallel. */
1841 7 : if (!VECTOR_MODE_P (compute_mode))
1842 0 : new_rhs = expand_vector_parallel (gsi, do_cond, type, rhs2, rhs3,
1843 : COND_EXPR);
1844 : else
1845 7 : new_rhs = expand_vector_piecewise (gsi, do_cond, type, compute_type,
1846 : rhs2, rhs3, COND_EXPR, false);
1847 7 : if (!useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (new_rhs)))
1848 0 : new_rhs = gimplify_build1 (gsi, VIEW_CONVERT_EXPR, TREE_TYPE (lhs),
1849 : new_rhs);
1850 :
1851 : /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
1852 : way to do it is change expand_vector_operation and its callees to
1853 : return a tree_code, RHS1 and RHS2 instead of a tree. */
1854 7 : gimple_assign_set_rhs_from_tree (gsi, new_rhs);
1855 7 : update_stmt (gsi_stmt (*gsi));
1856 7 : }
1857 :
1858 : /* Callback for expand_vector_piecewise to do VEC_CONVERT ifn call
1859 : lowering. If INNER_TYPE is not a vector type, this is a scalar
1860 : fallback. */
1861 :
1862 : static tree
1863 18972 : do_vec_conversion (gimple_stmt_iterator *gsi, tree inner_type, tree a,
1864 : tree decl, tree bitpos, tree bitsize,
1865 : enum tree_code code, tree type)
1866 : {
1867 18972 : a = tree_vec_extract (gsi, inner_type, a, bitsize, bitpos);
1868 18972 : if (!VECTOR_TYPE_P (inner_type))
1869 18972 : return gimplify_build1 (gsi, code, TREE_TYPE (type), a);
1870 0 : if (code == CALL_EXPR)
1871 : {
1872 0 : gimple *g = gimple_build_call (decl, 1, a);
1873 0 : tree lhs = make_ssa_name (TREE_TYPE (TREE_TYPE (decl)));
1874 0 : gimple_call_set_lhs (g, lhs);
1875 0 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
1876 0 : return lhs;
1877 : }
1878 : else
1879 : {
1880 0 : tree outer_type = build_vector_type (TREE_TYPE (type),
1881 0 : TYPE_VECTOR_SUBPARTS (inner_type));
1882 0 : return gimplify_build1 (gsi, code, outer_type, a);
1883 : }
1884 : }
1885 :
1886 : /* Similarly, but for narrowing conversion. */
1887 :
1888 : static tree
1889 53 : do_vec_narrow_conversion (gimple_stmt_iterator *gsi, tree inner_type, tree a,
1890 : tree, tree bitpos, tree, enum tree_code code,
1891 : tree type)
1892 : {
1893 53 : tree itype = build_vector_type (TREE_TYPE (inner_type),
1894 53 : exact_div (TYPE_VECTOR_SUBPARTS (inner_type),
1895 : 2));
1896 53 : tree b = tree_vec_extract (gsi, itype, a, TYPE_SIZE (itype), bitpos);
1897 53 : tree c = tree_vec_extract (gsi, itype, a, TYPE_SIZE (itype),
1898 : int_const_binop (PLUS_EXPR, bitpos,
1899 53 : TYPE_SIZE (itype)));
1900 53 : tree outer_type = build_vector_type (TREE_TYPE (type),
1901 53 : TYPE_VECTOR_SUBPARTS (inner_type));
1902 53 : return gimplify_build2 (gsi, code, outer_type, b, c);
1903 : }
1904 :
1905 : /* Expand VEC_CONVERT ifn call. */
1906 :
1907 : static void
1908 1193 : expand_vector_conversion (gimple_stmt_iterator *gsi)
1909 : {
1910 1193 : gimple *stmt = gsi_stmt (*gsi);
1911 1193 : gimple *g;
1912 1193 : tree lhs = gimple_call_lhs (stmt);
1913 1193 : if (lhs == NULL_TREE)
1914 : {
1915 1 : g = gimple_build_nop ();
1916 1 : gsi_replace (gsi, g, false);
1917 847 : return;
1918 : }
1919 1192 : tree arg = gimple_call_arg (stmt, 0);
1920 1192 : tree ret_type = TREE_TYPE (lhs);
1921 1192 : tree arg_type = TREE_TYPE (arg);
1922 1192 : tree new_rhs, new_lhs, compute_type = TREE_TYPE (arg_type);
1923 1192 : enum tree_code code = NOP_EXPR;
1924 1192 : enum tree_code code1 = ERROR_MARK;
1925 1192 : enum { NARROW, NONE, WIDEN } modifier = NONE;
1926 1192 : optab optab1 = unknown_optab;
1927 :
1928 1192 : gcc_checking_assert (VECTOR_TYPE_P (ret_type) && VECTOR_TYPE_P (arg_type));
1929 2384 : if (INTEGRAL_TYPE_P (TREE_TYPE (ret_type))
1930 2012 : && SCALAR_FLOAT_TYPE_P (TREE_TYPE (arg_type)))
1931 : code = FIX_TRUNC_EXPR;
1932 1868 : else if (INTEGRAL_TYPE_P (TREE_TYPE (arg_type))
1933 1772 : && SCALAR_FLOAT_TYPE_P (TREE_TYPE (ret_type)))
1934 : code = FLOAT_EXPR;
1935 1192 : unsigned int ret_elt_bits = vector_element_bits (ret_type);
1936 1192 : unsigned int arg_elt_bits = vector_element_bits (arg_type);
1937 1192 : if (ret_elt_bits < arg_elt_bits)
1938 : modifier = NARROW;
1939 732 : else if (ret_elt_bits > arg_elt_bits)
1940 468 : modifier = WIDEN;
1941 :
1942 1192 : auto_vec<std::pair<tree, tree_code>, 2> converts;
1943 1192 : if (supportable_indirect_convert_operation (code, ret_type, arg_type,
1944 : converts))
1945 : {
1946 684 : new_rhs = arg;
1947 684 : gcc_assert (!converts.is_empty ());
1948 1504 : for (unsigned int i = 0; i < converts.length () - 1; i++)
1949 : {
1950 68 : new_lhs = make_ssa_name (converts[i].first);
1951 68 : g = gimple_build_assign (new_lhs, converts[i].second, new_rhs);
1952 68 : new_rhs = new_lhs;
1953 68 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
1954 : }
1955 2052 : g = gimple_build_assign (lhs,
1956 684 : converts[converts.length() - 1].second,
1957 : new_rhs);
1958 684 : gsi_replace (gsi, g, false);
1959 684 : return;
1960 : }
1961 :
1962 508 : if (modifier == NONE && (code == FIX_TRUNC_EXPR || code == FLOAT_EXPR))
1963 : {
1964 : /* Can't use get_compute_type here, as supportable_convert_operation
1965 : doesn't necessarily use an optab and needs two arguments. */
1966 77 : tree vec_compute_type
1967 77 : = type_for_widest_vector_mode (arg_type, mov_optab);
1968 77 : if (vec_compute_type
1969 77 : && VECTOR_MODE_P (TYPE_MODE (vec_compute_type)))
1970 : {
1971 77 : unsigned HOST_WIDE_INT nelts
1972 77 : = constant_lower_bound (TYPE_VECTOR_SUBPARTS (vec_compute_type));
1973 156 : while (nelts > 1)
1974 : {
1975 79 : tree ret1_type = build_vector_type (TREE_TYPE (ret_type), nelts);
1976 79 : tree arg1_type = build_vector_type (TREE_TYPE (arg_type), nelts);
1977 79 : if (supportable_convert_operation (code, ret1_type, arg1_type))
1978 : {
1979 0 : new_rhs = expand_vector_piecewise (gsi, do_vec_conversion,
1980 : ret_type, arg1_type, arg,
1981 : NULL_TREE, code, false);
1982 0 : g = gimple_build_assign (lhs, new_rhs);
1983 0 : gsi_replace (gsi, g, false);
1984 0 : return;
1985 : }
1986 79 : nelts = nelts / 2;
1987 : }
1988 : }
1989 : }
1990 431 : else if (modifier == NARROW)
1991 : {
1992 270 : switch (code)
1993 : {
1994 210 : CASE_CONVERT:
1995 210 : code1 = VEC_PACK_TRUNC_EXPR;
1996 210 : optab1 = optab_for_tree_code (code1, arg_type, optab_default);
1997 210 : break;
1998 50 : case FIX_TRUNC_EXPR:
1999 50 : code1 = VEC_PACK_FIX_TRUNC_EXPR;
2000 : /* The signedness is determined from output operand. */
2001 50 : optab1 = optab_for_tree_code (code1, ret_type, optab_default);
2002 50 : break;
2003 10 : case FLOAT_EXPR:
2004 10 : code1 = VEC_PACK_FLOAT_EXPR;
2005 10 : optab1 = optab_for_tree_code (code1, arg_type, optab_default);
2006 10 : break;
2007 : default:
2008 : gcc_unreachable ();
2009 : }
2010 :
2011 270 : if (optab1)
2012 270 : compute_type = get_compute_type (optab1, arg_type);
2013 270 : enum insn_code icode1;
2014 270 : if (VECTOR_TYPE_P (compute_type)
2015 259 : && ((icode1 = optab_handler (optab1, TYPE_MODE (compute_type)))
2016 : != CODE_FOR_nothing)
2017 529 : && VECTOR_MODE_P (insn_data[icode1].operand[0].mode))
2018 : {
2019 259 : tree cretd_type
2020 259 : = build_vector_type (TREE_TYPE (ret_type),
2021 259 : TYPE_VECTOR_SUBPARTS (compute_type) * 2);
2022 259 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (cretd_type))
2023 : {
2024 127 : if (compute_type == arg_type)
2025 : {
2026 81 : new_rhs = gimplify_build2 (gsi, code1, cretd_type,
2027 : arg, build_zero_cst (arg_type));
2028 162 : new_rhs = tree_vec_extract (gsi, ret_type, new_rhs,
2029 81 : TYPE_SIZE (ret_type),
2030 : bitsize_int (0));
2031 81 : g = gimple_build_assign (lhs, new_rhs);
2032 81 : gsi_replace (gsi, g, false);
2033 81 : return;
2034 : }
2035 46 : tree dcompute_type
2036 46 : = build_vector_type (TREE_TYPE (compute_type),
2037 46 : TYPE_VECTOR_SUBPARTS (compute_type) * 2);
2038 46 : if (TYPE_MAIN_VARIANT (dcompute_type)
2039 46 : == TYPE_MAIN_VARIANT (arg_type))
2040 37 : new_rhs = do_vec_narrow_conversion (gsi, dcompute_type, arg,
2041 : NULL_TREE, bitsize_int (0),
2042 : NULL_TREE, code1,
2043 : ret_type);
2044 : else
2045 9 : new_rhs = expand_vector_piecewise (gsi,
2046 : do_vec_narrow_conversion,
2047 : arg_type, dcompute_type,
2048 : arg, NULL_TREE, code1,
2049 : false, ret_type);
2050 46 : g = gimple_build_assign (lhs, new_rhs);
2051 46 : gsi_replace (gsi, g, false);
2052 46 : return;
2053 : }
2054 : }
2055 : }
2056 161 : else if (modifier == WIDEN)
2057 : {
2058 158 : enum tree_code code2 = ERROR_MARK;
2059 158 : optab optab2 = unknown_optab;
2060 158 : switch (code)
2061 : {
2062 : CASE_CONVERT:
2063 : code1 = VEC_UNPACK_LO_EXPR;
2064 : code2 = VEC_UNPACK_HI_EXPR;
2065 : break;
2066 0 : case FIX_TRUNC_EXPR:
2067 0 : code1 = VEC_UNPACK_FIX_TRUNC_LO_EXPR;
2068 0 : code2 = VEC_UNPACK_FIX_TRUNC_HI_EXPR;
2069 0 : break;
2070 : case FLOAT_EXPR:
2071 : code1 = VEC_UNPACK_FLOAT_LO_EXPR;
2072 : code2 = VEC_UNPACK_FLOAT_HI_EXPR;
2073 : break;
2074 : default:
2075 : gcc_unreachable ();
2076 : }
2077 68 : if (BYTES_BIG_ENDIAN)
2078 : std::swap (code1, code2);
2079 :
2080 68 : if (code == FIX_TRUNC_EXPR)
2081 : {
2082 : /* The signedness is determined from output operand. */
2083 0 : optab1 = optab_for_tree_code (code1, ret_type, optab_default);
2084 0 : optab2 = optab_for_tree_code (code2, ret_type, optab_default);
2085 : }
2086 : else
2087 : {
2088 158 : optab1 = optab_for_tree_code (code1, arg_type, optab_default);
2089 158 : optab2 = optab_for_tree_code (code2, arg_type, optab_default);
2090 : }
2091 :
2092 158 : if (optab1 && optab2)
2093 158 : compute_type = get_compute_type (optab1, arg_type);
2094 :
2095 158 : enum insn_code icode1, icode2;
2096 158 : if (VECTOR_TYPE_P (compute_type)
2097 54 : && ((icode1 = optab_handler (optab1, TYPE_MODE (compute_type)))
2098 : != CODE_FOR_nothing)
2099 54 : && ((icode2 = optab_handler (optab2, TYPE_MODE (compute_type)))
2100 : != CODE_FOR_nothing)
2101 54 : && VECTOR_MODE_P (insn_data[icode1].operand[0].mode)
2102 158 : && (insn_data[icode1].operand[0].mode
2103 54 : == insn_data[icode2].operand[0].mode))
2104 : {
2105 54 : poly_uint64 nunits
2106 54 : = exact_div (TYPE_VECTOR_SUBPARTS (compute_type), 2);
2107 54 : tree cretd_type = build_vector_type (TREE_TYPE (ret_type), nunits);
2108 54 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (cretd_type))
2109 : {
2110 34 : vec<constructor_elt, va_gc> *v;
2111 34 : tree part_width = TYPE_SIZE (compute_type);
2112 34 : tree index = bitsize_int (0);
2113 34 : int nunits = nunits_for_known_piecewise_op (arg_type);
2114 34 : int delta = tree_to_uhwi (part_width) / arg_elt_bits;
2115 34 : int i;
2116 34 : location_t loc = gimple_location (gsi_stmt (*gsi));
2117 :
2118 34 : if (compute_type != arg_type)
2119 : {
2120 0 : if (!warning_suppressed_p (gsi_stmt (*gsi),
2121 : OPT_Wvector_operation_performance))
2122 0 : warning_at (loc, OPT_Wvector_operation_performance,
2123 : "vector operation will be expanded piecewise");
2124 : }
2125 : else
2126 : {
2127 : nunits = 1;
2128 : delta = 1;
2129 : }
2130 :
2131 34 : vec_alloc (v, (nunits + delta - 1) / delta * 2);
2132 34 : bool constant_p = true;
2133 102 : for (i = 0; i < nunits;
2134 34 : i += delta, index = int_const_binop (PLUS_EXPR, index,
2135 : part_width))
2136 : {
2137 34 : tree a = arg;
2138 34 : if (compute_type != arg_type)
2139 0 : a = tree_vec_extract (gsi, compute_type, a, part_width,
2140 : index);
2141 34 : tree result = gimplify_build1 (gsi, code1, cretd_type, a);
2142 34 : constructor_elt ce = { NULL_TREE, result };
2143 34 : if (!CONSTANT_CLASS_P (ce.value))
2144 34 : constant_p = false;
2145 34 : v->quick_push (ce);
2146 34 : ce.value = gimplify_build1 (gsi, code2, cretd_type, a);
2147 34 : if (!CONSTANT_CLASS_P (ce.value))
2148 34 : constant_p = false;
2149 34 : v->quick_push (ce);
2150 : }
2151 :
2152 34 : if (constant_p)
2153 0 : new_rhs = build_vector_from_ctor (ret_type, v);
2154 : else
2155 34 : new_rhs = build_constructor (ret_type, v);
2156 34 : g = gimple_build_assign (lhs, new_rhs);
2157 34 : gsi_replace (gsi, g, false);
2158 34 : return;
2159 : }
2160 : }
2161 : }
2162 :
2163 347 : new_rhs = expand_vector_piecewise (gsi, do_vec_conversion, arg_type,
2164 347 : TREE_TYPE (arg_type), arg,
2165 : NULL_TREE, code, false, ret_type);
2166 347 : g = gimple_build_assign (lhs, new_rhs);
2167 347 : gsi_replace (gsi, g, false);
2168 1192 : }
2169 :
2170 : /* Process one statement. If we identify a vector operation, expand it. */
2171 :
2172 : static void
2173 103170311 : expand_vector_operations_1 (gimple_stmt_iterator *gsi)
2174 : {
2175 103170311 : tree lhs, rhs1, rhs2 = NULL, type, compute_type = NULL_TREE;
2176 103170311 : enum tree_code code;
2177 103170311 : optab op = unknown_optab;
2178 103170311 : enum gimple_rhs_class rhs_class;
2179 103170311 : tree new_rhs;
2180 :
2181 : /* Only consider code == GIMPLE_ASSIGN. */
2182 103170311 : gassign *stmt = dyn_cast <gassign *> (gsi_stmt (*gsi));
2183 103170311 : if (!stmt)
2184 : {
2185 68817398 : if (gimple_call_internal_p (gsi_stmt (*gsi), IFN_VEC_CONVERT))
2186 1193 : expand_vector_conversion (gsi);
2187 : return;
2188 : }
2189 :
2190 34352913 : code = gimple_assign_rhs_code (stmt);
2191 34352913 : rhs_class = get_gimple_rhs_class (code);
2192 34352913 : lhs = gimple_assign_lhs (stmt);
2193 :
2194 34352913 : if (code == VEC_PERM_EXPR)
2195 : {
2196 91859 : lower_vec_perm (gsi);
2197 91859 : return;
2198 : }
2199 :
2200 34261054 : if (code == VEC_COND_EXPR)
2201 : {
2202 23907 : expand_vector_condition (gsi);
2203 23907 : return;
2204 : }
2205 :
2206 34237147 : if (code == COND_EXPR
2207 13884 : && TREE_CODE (TREE_TYPE (gimple_assign_lhs (stmt))) == VECTOR_TYPE
2208 34237154 : && TYPE_MODE (TREE_TYPE (gimple_assign_lhs (stmt))) == BLKmode)
2209 : {
2210 7 : expand_vector_scalar_condition (gsi);
2211 7 : return;
2212 : }
2213 :
2214 34237140 : if (code == CONSTRUCTOR
2215 1901276 : && TREE_CODE (lhs) == SSA_NAME
2216 150825 : && VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (lhs)))
2217 150037 : && !gimple_clobber_p (stmt)
2218 34387177 : && optimize)
2219 : {
2220 148070 : optimize_vector_constructor (gsi);
2221 148070 : return;
2222 : }
2223 :
2224 34089070 : if (rhs_class != GIMPLE_UNARY_RHS && rhs_class != GIMPLE_BINARY_RHS)
2225 : return;
2226 :
2227 12519940 : rhs1 = gimple_assign_rhs1 (stmt);
2228 12519940 : if (rhs_class == GIMPLE_BINARY_RHS)
2229 8494245 : rhs2 = gimple_assign_rhs2 (stmt);
2230 :
2231 12519940 : type = TREE_TYPE (lhs);
2232 12519940 : if (!VECTOR_TYPE_P (type)
2233 12519940 : || !VECTOR_TYPE_P (TREE_TYPE (rhs1)))
2234 : return;
2235 :
2236 : /* A scalar operation pretending to be a vector one. */
2237 271758 : if (VECTOR_BOOLEAN_TYPE_P (type)
2238 38348 : && !VECTOR_MODE_P (TYPE_MODE (type))
2239 12004 : && TYPE_MODE (type) != BLKmode
2240 11670 : && (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt)) != tcc_comparison
2241 6613 : || (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (rhs1))
2242 0 : && !VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (rhs1)))
2243 0 : && TYPE_MODE (TREE_TYPE (rhs1)) != BLKmode)))
2244 : return;
2245 :
2246 : /* If the vector operation is operating on all same vector elements
2247 : implement it with a scalar operation and a splat if the target
2248 : supports the scalar operation. */
2249 266701 : tree srhs1, srhs2 = NULL_TREE;
2250 266701 : if ((srhs1 = ssa_uniform_vector_p (rhs1)) != NULL_TREE
2251 16941 : && (rhs2 == NULL_TREE
2252 16702 : || (! VECTOR_TYPE_P (TREE_TYPE (rhs2))
2253 : && (srhs2 = rhs2))
2254 16682 : || (srhs2 = ssa_uniform_vector_p (rhs2)) != NULL_TREE)
2255 : /* As we query direct optabs restrict to non-convert operations. */
2256 268315 : && TYPE_MODE (TREE_TYPE (type)) == TYPE_MODE (TREE_TYPE (srhs1)))
2257 : {
2258 1426 : op = optab_for_tree_code (code, TREE_TYPE (type), optab_scalar);
2259 1426 : if (op >= FIRST_NORM_OPTAB && op <= LAST_NORM_OPTAB
2260 1426 : && can_implement_p (op, TYPE_MODE (TREE_TYPE (type))))
2261 : {
2262 981 : tree stype = TREE_TYPE (TREE_TYPE (lhs));
2263 981 : tree slhs = (rhs2 != NULL_TREE)
2264 981 : ? gimplify_build2 (gsi, code, stype, srhs1, srhs2)
2265 101 : : gimplify_build1 (gsi, code, stype, srhs1);
2266 981 : gimple_assign_set_rhs_from_tree (gsi,
2267 : build_vector_from_val (type, slhs));
2268 981 : update_stmt (stmt);
2269 981 : return;
2270 : }
2271 : }
2272 :
2273 : /* Plain moves do not need lowering. */
2274 265720 : if (code == SSA_NAME
2275 265720 : || code == VIEW_CONVERT_EXPR
2276 : || code == PAREN_EXPR)
2277 : return;
2278 :
2279 : if (CONVERT_EXPR_CODE_P (code)
2280 : || code == FLOAT_EXPR
2281 : || code == FIX_TRUNC_EXPR)
2282 : return;
2283 :
2284 : /* The signedness is determined from input argument. */
2285 : if (code == VEC_UNPACK_FLOAT_HI_EXPR
2286 : || code == VEC_UNPACK_FLOAT_LO_EXPR
2287 : || code == VEC_PACK_FLOAT_EXPR)
2288 : {
2289 : /* We do not know how to scalarize those. */
2290 : return;
2291 : }
2292 :
2293 : /* For widening/narrowing vector operations, the relevant type is of the
2294 : arguments, not the widened result. VEC_UNPACK_FLOAT_*_EXPR is
2295 : calculated in the same way above. */
2296 : if (code == WIDEN_SUM_EXPR
2297 : || code == VEC_WIDEN_MULT_HI_EXPR
2298 : || code == VEC_WIDEN_MULT_LO_EXPR
2299 : || code == VEC_WIDEN_MULT_EVEN_EXPR
2300 : || code == VEC_WIDEN_MULT_ODD_EXPR
2301 : || code == VEC_UNPACK_HI_EXPR
2302 : || code == VEC_UNPACK_LO_EXPR
2303 : || code == VEC_UNPACK_FIX_TRUNC_HI_EXPR
2304 : || code == VEC_UNPACK_FIX_TRUNC_LO_EXPR
2305 : || code == VEC_PACK_TRUNC_EXPR
2306 : || code == VEC_PACK_SAT_EXPR
2307 : || code == VEC_PACK_FIX_TRUNC_EXPR
2308 : || code == VEC_WIDEN_LSHIFT_HI_EXPR
2309 : || code == VEC_WIDEN_LSHIFT_LO_EXPR)
2310 : {
2311 : /* We do not know how to scalarize those. */
2312 : return;
2313 : }
2314 :
2315 : /* Choose between vector shift/rotate by vector and vector shift/rotate by
2316 : scalar */
2317 : if (code == LSHIFT_EXPR
2318 : || code == RSHIFT_EXPR
2319 : || code == LROTATE_EXPR
2320 : || code == RROTATE_EXPR)
2321 : {
2322 11728 : optab opv;
2323 :
2324 : /* Check whether we have vector <op> {x,x,x,x} where x
2325 : could be a scalar variable or a constant. Transform
2326 : vector <op> {x,x,x,x} ==> vector <op> scalar. */
2327 11728 : if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2)))
2328 : {
2329 2117 : tree first;
2330 :
2331 2117 : if ((first = ssa_uniform_vector_p (rhs2)) != NULL_TREE)
2332 : {
2333 103 : gimple_assign_set_rhs2 (stmt, first);
2334 103 : update_stmt (stmt);
2335 103 : rhs2 = first;
2336 : }
2337 : }
2338 :
2339 11728 : opv = optab_for_tree_code (code, type, optab_vector);
2340 11728 : if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2)))
2341 : op = opv;
2342 : else
2343 : {
2344 9714 : op = optab_for_tree_code (code, type, optab_scalar);
2345 :
2346 9714 : compute_type = get_compute_type (op, type);
2347 9714 : if (compute_type == type)
2348 : return;
2349 : /* The rtl expander will expand vector/scalar as vector/vector
2350 : if necessary. Pick one with wider vector type. */
2351 180 : tree compute_vtype = get_compute_type (opv, type);
2352 180 : if (subparts_gt (compute_vtype, compute_type))
2353 : {
2354 0 : compute_type = compute_vtype;
2355 0 : op = opv;
2356 : }
2357 : }
2358 :
2359 2194 : if (code == LROTATE_EXPR || code == RROTATE_EXPR)
2360 : {
2361 41 : if (compute_type == NULL_TREE)
2362 6 : compute_type = get_compute_type (op, type);
2363 41 : if (compute_type == type)
2364 : return;
2365 : /* Before splitting vector rotates into scalar rotates,
2366 : see if we can't use vector shifts and BIT_IOR_EXPR
2367 : instead. For vector by vector rotates we'd also
2368 : need to check BIT_AND_EXPR and NEGATE_EXPR, punt there
2369 : for now, fold doesn't seem to create such rotates anyway. */
2370 35 : if (compute_type == TREE_TYPE (type)
2371 35 : && !VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2)))
2372 : {
2373 30 : optab oplv = vashl_optab, opl = ashl_optab;
2374 30 : optab oprv = vlshr_optab, opr = lshr_optab, opo = ior_optab;
2375 30 : tree compute_lvtype = get_compute_type (oplv, type);
2376 30 : tree compute_rvtype = get_compute_type (oprv, type);
2377 30 : tree compute_otype = get_compute_type (opo, type);
2378 30 : tree compute_ltype = get_compute_type (opl, type);
2379 30 : tree compute_rtype = get_compute_type (opr, type);
2380 : /* The rtl expander will expand vector/scalar as vector/vector
2381 : if necessary. Pick one with wider vector type. */
2382 30 : if (subparts_gt (compute_lvtype, compute_ltype))
2383 : {
2384 0 : compute_ltype = compute_lvtype;
2385 0 : opl = oplv;
2386 : }
2387 30 : if (subparts_gt (compute_rvtype, compute_rtype))
2388 : {
2389 0 : compute_rtype = compute_rvtype;
2390 0 : opr = oprv;
2391 : }
2392 : /* Pick the narrowest type from LSHIFT_EXPR, RSHIFT_EXPR and
2393 : BIT_IOR_EXPR. */
2394 30 : compute_type = compute_ltype;
2395 30 : if (subparts_gt (compute_type, compute_rtype))
2396 0 : compute_type = compute_rtype;
2397 30 : if (subparts_gt (compute_type, compute_otype))
2398 0 : compute_type = compute_otype;
2399 : /* Verify all 3 operations can be performed in that type. */
2400 30 : if (compute_type != TREE_TYPE (type))
2401 : {
2402 30 : if (!can_implement_p (opl, TYPE_MODE (compute_type))
2403 30 : || !can_implement_p (opr, TYPE_MODE (compute_type))
2404 60 : || !can_implement_p (opo, TYPE_MODE (compute_type)))
2405 0 : compute_type = TREE_TYPE (type);
2406 : }
2407 : }
2408 : }
2409 : }
2410 : else
2411 217908 : op = optab_for_tree_code (code, type, optab_default);
2412 :
2413 : /* Optabs will try converting a negation into a subtraction, so
2414 : look for it as well. TODO: negation of floating-point vectors
2415 : might be turned into an exclusive OR toggling the sign bit. */
2416 220096 : if (op == unknown_optab
2417 220096 : && code == NEGATE_EXPR
2418 220096 : && INTEGRAL_TYPE_P (TREE_TYPE (type)))
2419 0 : op = optab_for_tree_code (MINUS_EXPR, type, optab_default);
2420 :
2421 220096 : if (compute_type == NULL_TREE)
2422 219916 : compute_type = get_compute_type (op, type);
2423 220096 : if (compute_type == type)
2424 : return;
2425 :
2426 30938 : new_rhs = expand_vector_operation (gsi, type, compute_type, stmt, code);
2427 :
2428 : /* Leave expression untouched for later expansion. */
2429 30938 : if (new_rhs == NULL_TREE)
2430 : return;
2431 :
2432 4181 : if (!useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (new_rhs)))
2433 102 : new_rhs = gimplify_build1 (gsi, VIEW_CONVERT_EXPR, TREE_TYPE (lhs),
2434 : new_rhs);
2435 :
2436 : /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
2437 : way to do it is change expand_vector_operation and its callees to
2438 : return a tree_code, RHS1 and RHS2 instead of a tree. */
2439 4181 : gimple_assign_set_rhs_from_tree (gsi, new_rhs);
2440 4181 : update_stmt (gsi_stmt (*gsi));
2441 : }
2442 :
2443 : /* Use this to lower vector operations introduced by the vectorizer,
2444 : if it may need the bit-twiddling tricks implemented in this file. */
2445 :
2446 : static unsigned int
2447 1518056 : expand_vector_operations (void)
2448 : {
2449 1518056 : gimple_stmt_iterator gsi;
2450 1518056 : basic_block bb;
2451 1518056 : bool cfg_changed = false;
2452 :
2453 16167214 : FOR_EACH_BB_FN (bb, cfun)
2454 : {
2455 132468627 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2456 : {
2457 103170311 : expand_vector_operations_1 (&gsi);
2458 : /* ??? If we do not cleanup EH then we will ICE in
2459 : verification. But in reality we have created wrong-code
2460 : as we did not properly transition EH info and edges to
2461 : the piecewise computations. */
2462 103170311 : if (maybe_clean_eh_stmt (gsi_stmt (gsi))
2463 103170311 : && gimple_purge_dead_eh_edges (bb))
2464 : cfg_changed = true;
2465 : /* If a .LOOP_DIST_ALIAS call prevailed loops got elided
2466 : before vectorization got a chance to get at them. Simply
2467 : fold as if loop distribution wasn't performed. */
2468 103170311 : if (gimple_call_internal_p (gsi_stmt (gsi), IFN_LOOP_DIST_ALIAS))
2469 : {
2470 0 : fold_loop_internal_call (gsi_stmt (gsi), boolean_false_node);
2471 0 : cfg_changed = true;
2472 : }
2473 : }
2474 : }
2475 :
2476 1518056 : return cfg_changed ? TODO_cleanup_cfg : 0;
2477 : }
2478 :
2479 : namespace {
2480 :
2481 : const pass_data pass_data_lower_vector =
2482 : {
2483 : GIMPLE_PASS, /* type */
2484 : "veclower", /* name */
2485 : OPTGROUP_VEC, /* optinfo_flags */
2486 : TV_NONE, /* tv_id */
2487 : PROP_cfg, /* properties_required */
2488 : PROP_gimple_lvec, /* properties_provided */
2489 : 0, /* properties_destroyed */
2490 : 0, /* todo_flags_start */
2491 : TODO_update_ssa, /* todo_flags_finish */
2492 : };
2493 :
2494 : class pass_lower_vector : public gimple_opt_pass
2495 : {
2496 : public:
2497 297180 : pass_lower_vector (gcc::context *ctxt)
2498 594360 : : gimple_opt_pass (pass_data_lower_vector, ctxt)
2499 : {}
2500 :
2501 : /* opt_pass methods: */
2502 1517917 : bool gate (function *fun) final override
2503 : {
2504 1517917 : return !(fun->curr_properties & PROP_gimple_lvec);
2505 : }
2506 :
2507 452614 : unsigned int execute (function *) final override
2508 : {
2509 452614 : return expand_vector_operations ();
2510 : }
2511 :
2512 : }; // class pass_lower_vector
2513 :
2514 : } // anon namespace
2515 :
2516 : gimple_opt_pass *
2517 297180 : make_pass_lower_vector (gcc::context *ctxt)
2518 : {
2519 297180 : return new pass_lower_vector (ctxt);
2520 : }
2521 :
2522 : namespace {
2523 :
2524 : const pass_data pass_data_lower_vector_ssa =
2525 : {
2526 : GIMPLE_PASS, /* type */
2527 : "veclower2", /* name */
2528 : OPTGROUP_VEC, /* optinfo_flags */
2529 : TV_NONE, /* tv_id */
2530 : PROP_cfg, /* properties_required */
2531 : PROP_gimple_lvec, /* properties_provided */
2532 : 0, /* properties_destroyed */
2533 : 0, /* todo_flags_start */
2534 : ( TODO_update_ssa
2535 : | TODO_cleanup_cfg ), /* todo_flags_finish */
2536 : };
2537 :
2538 : class pass_lower_vector_ssa : public gimple_opt_pass
2539 : {
2540 : public:
2541 594360 : pass_lower_vector_ssa (gcc::context *ctxt)
2542 1188720 : : gimple_opt_pass (pass_data_lower_vector_ssa, ctxt)
2543 : {}
2544 :
2545 : /* opt_pass methods: */
2546 297180 : opt_pass * clone () final override
2547 : {
2548 297180 : return new pass_lower_vector_ssa (m_ctxt);
2549 : }
2550 1065442 : unsigned int execute (function *) final override
2551 : {
2552 1065442 : return expand_vector_operations ();
2553 : }
2554 :
2555 : }; // class pass_lower_vector_ssa
2556 :
2557 : } // anon namespace
2558 :
2559 : gimple_opt_pass *
2560 297180 : make_pass_lower_vector_ssa (gcc::context *ctxt)
2561 : {
2562 297180 : return new pass_lower_vector_ssa (ctxt);
2563 : }
2564 :
2565 : #include "gt-tree-vect-generic.h"
|