Line data Source code
1 : /* Lower vector operations to scalar operations.
2 : Copyright (C) 2004-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it
7 : under the terms of the GNU General Public License as published by the
8 : Free Software Foundation; either version 3, or (at your option) any
9 : later version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT
12 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "rtl.h"
25 : #include "tree.h"
26 : #include "gimple.h"
27 : #include "tree-pass.h"
28 : #include "ssa.h"
29 : #include "expmed.h"
30 : #include "optabs-tree.h"
31 : #include "diagnostic.h"
32 : #include "fold-const.h"
33 : #include "stor-layout.h"
34 : #include "langhooks.h"
35 : #include "tree-eh.h"
36 : #include "gimple-iterator.h"
37 : #include "gimplify-me.h"
38 : #include "gimplify.h"
39 : #include "tree-cfg.h"
40 : #include "tree-vector-builder.h"
41 : #include "vec-perm-indices.h"
42 : #include "insn-config.h"
43 : #include "gimple-fold.h"
44 : #include "gimple-match.h"
45 : #include "recog.h" /* FIXME: for insn_data */
46 : #include "optabs-libfuncs.h"
47 : #include "cfgloop.h"
48 : #include "tree-vectorizer.h"
49 :
50 :
51 : /* Build a ternary operation and gimplify it. Emit code before GSI.
52 : Return the gimple_val holding the result. */
53 :
54 : static tree
55 11231 : gimplify_build3 (gimple_stmt_iterator *gsi, enum tree_code code,
56 : tree type, tree a, tree b, tree c)
57 : {
58 11231 : location_t loc = gimple_location (gsi_stmt (*gsi));
59 11231 : return gimple_build (gsi, true, GSI_SAME_STMT, loc, code, type, a, b, c);
60 : }
61 :
62 : /* Build a binary operation and gimplify it. Emit code before GSI.
63 : Return the gimple_val holding the result. */
64 :
65 : static tree
66 17882 : gimplify_build2 (gimple_stmt_iterator *gsi, enum tree_code code,
67 : tree type, tree a, tree b)
68 : {
69 17882 : location_t loc = gimple_location (gsi_stmt (*gsi));
70 17882 : return gimple_build (gsi, true, GSI_SAME_STMT, loc, code, type, a, b);
71 : }
72 :
73 : /* Build a unary operation and gimplify it. Emit code before GSI.
74 : Return the gimple_val holding the result. */
75 :
76 : static tree
77 18813 : gimplify_build1 (gimple_stmt_iterator *gsi, enum tree_code code, tree type,
78 : tree a)
79 : {
80 18813 : location_t loc = gimple_location (gsi_stmt (*gsi));
81 18813 : return gimple_build (gsi, true, GSI_SAME_STMT, loc, code, type, a);
82 : }
83 :
84 :
85 : /* Return the number of elements in a vector type TYPE that we have
86 : already decided needs to be expanded piecewise. We don't support
87 : this kind of expansion for variable-length vectors, since we should
88 : always check for target support before introducing uses of those. */
89 : static unsigned int
90 111273 : nunits_for_known_piecewise_op (const_tree type)
91 : {
92 110365 : return TYPE_VECTOR_SUBPARTS (type).to_constant ();
93 : }
94 :
95 : /* Return true if TYPE1 has more elements than TYPE2, where either
96 : type may be a vector or a scalar. */
97 :
98 : static inline bool
99 292 : subparts_gt (tree type1, tree type2)
100 : {
101 292 : poly_uint64 n1 = VECTOR_TYPE_P (type1) ? TYPE_VECTOR_SUBPARTS (type1) : 1;
102 292 : poly_uint64 n2 = VECTOR_TYPE_P (type2) ? TYPE_VECTOR_SUBPARTS (type2) : 1;
103 292 : return known_gt (n1, n2);
104 : }
105 :
106 : static GTY(()) tree vector_inner_type;
107 : static GTY(()) tree vector_last_type;
108 : static GTY(()) int vector_last_nunits;
109 :
110 : /* Return a suitable vector types made of SUBPARTS units each of mode
111 : "word_mode" (the global variable). */
112 : static tree
113 52 : build_word_mode_vector_type (int nunits)
114 : {
115 52 : if (!vector_inner_type)
116 40 : vector_inner_type = lang_hooks.types.type_for_mode (word_mode, 1);
117 12 : else if (vector_last_nunits == nunits)
118 : {
119 10 : gcc_assert (TREE_CODE (vector_last_type) == VECTOR_TYPE);
120 : return vector_last_type;
121 : }
122 :
123 42 : vector_last_nunits = nunits;
124 42 : vector_last_type = build_vector_type (vector_inner_type, nunits);
125 42 : return vector_last_type;
126 : }
127 :
128 : typedef tree (*elem_op_func) (gimple_stmt_iterator *,
129 : tree, tree, tree, tree, tree, enum tree_code,
130 : tree);
131 :
132 : /* Extract the vector element of type TYPE at BITPOS with BITSIZE from T
133 : and return it. */
134 :
135 : tree
136 66974 : tree_vec_extract (gimple_stmt_iterator *gsi, tree type,
137 : tree t, tree bitsize, tree bitpos)
138 : {
139 : /* We're using the resimplify API and maybe_push_res_to_seq to
140 : simplify the BIT_FIELD_REF but restrict the simplification to
141 : a single stmt while at the same time following SSA edges for
142 : simplification with already emitted CTORs. */
143 66974 : gimple_match_op opr;
144 66974 : opr.set_op (BIT_FIELD_REF, type, t, bitsize, bitpos);
145 66974 : opr.resimplify (NULL, follow_all_ssa_edges);
146 66974 : gimple_seq stmts = NULL;
147 66974 : tree res = maybe_push_res_to_seq (&opr, &stmts);
148 66974 : if (!res)
149 : {
150 : /* This can happen if SSA_NAME_OCCURS_IN_ABNORMAL_PHI are
151 : used. Build BIT_FIELD_REF manually otherwise. */
152 64 : t = build3 (BIT_FIELD_REF, type, t, bitsize, bitpos);
153 64 : res = make_ssa_name (type);
154 64 : gimple *g = gimple_build_assign (res, t);
155 64 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
156 64 : return res;
157 : }
158 66910 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
159 66910 : return res;
160 : }
161 :
162 : static tree
163 388 : do_unop (gimple_stmt_iterator *gsi, tree inner_type, tree a,
164 : tree b ATTRIBUTE_UNUSED, tree bitpos, tree bitsize,
165 : enum tree_code code, tree type ATTRIBUTE_UNUSED)
166 : {
167 388 : tree rhs_type = inner_type;
168 :
169 : /* For ABSU_EXPR, use the signed type for the rhs if the rhs was signed. */
170 388 : if (code == ABSU_EXPR
171 9 : && ANY_INTEGRAL_TYPE_P (TREE_TYPE (a))
172 397 : && !TYPE_UNSIGNED (TREE_TYPE (a)))
173 9 : rhs_type = signed_type_for (rhs_type);
174 :
175 388 : a = tree_vec_extract (gsi, rhs_type, a, bitsize, bitpos);
176 388 : return gimplify_build1 (gsi, code, inner_type, a);
177 : }
178 :
179 : static tree
180 10329 : do_binop (gimple_stmt_iterator *gsi, tree inner_type, tree a, tree b,
181 : tree bitpos, tree bitsize, enum tree_code code,
182 : tree type ATTRIBUTE_UNUSED)
183 : {
184 10329 : if (VECTOR_TYPE_P (TREE_TYPE (a)))
185 10329 : a = tree_vec_extract (gsi, inner_type, a, bitsize, bitpos);
186 10329 : if (VECTOR_TYPE_P (TREE_TYPE (b)))
187 10019 : b = tree_vec_extract (gsi, inner_type, b, bitsize, bitpos);
188 10329 : return gimplify_build2 (gsi, code, inner_type, a, b);
189 : }
190 :
191 : /* Construct expression (A[BITPOS] code B[BITPOS]) ? -1 : 0
192 :
193 : INNER_TYPE is the type of A and B elements
194 :
195 : returned expression is of signed integer type with the
196 : size equal to the size of INNER_TYPE. */
197 : static tree
198 5630 : do_compare (gimple_stmt_iterator *gsi, tree inner_type, tree a, tree b,
199 : tree bitpos, tree bitsize, enum tree_code code, tree type)
200 : {
201 5630 : tree stype = TREE_TYPE (type);
202 5630 : tree cst_false = build_zero_cst (stype);
203 5630 : tree cst_true = build_all_ones_cst (stype);
204 5630 : tree cmp;
205 :
206 5630 : a = tree_vec_extract (gsi, inner_type, a, bitsize, bitpos);
207 5630 : b = tree_vec_extract (gsi, inner_type, b, bitsize, bitpos);
208 :
209 5630 : cmp = build2 (code, boolean_type_node, a, b);
210 5630 : return gimplify_build3 (gsi, COND_EXPR, stype, cmp, cst_true, cst_false);
211 : }
212 :
213 : /* Expand vector addition to scalars. This does bit twiddling
214 : in order to increase parallelism:
215 :
216 : a + b = (((int) a & 0x7f7f7f7f) + ((int) b & 0x7f7f7f7f)) ^
217 : (a ^ b) & 0x80808080
218 :
219 : a - b = (((int) a | 0x80808080) - ((int) b & 0x7f7f7f7f)) ^
220 : (a ^ ~b) & 0x80808080
221 :
222 : -b = (0x80808080 - ((int) b & 0x7f7f7f7f)) ^ (~b & 0x80808080)
223 :
224 : This optimization should be done only if 4 vector items or more
225 : fit into a word. */
226 : static tree
227 6 : do_plus_minus (gimple_stmt_iterator *gsi, tree word_type, tree a, tree b,
228 : tree bitpos ATTRIBUTE_UNUSED, tree bitsize ATTRIBUTE_UNUSED,
229 : enum tree_code code, tree type ATTRIBUTE_UNUSED)
230 : {
231 6 : unsigned int width = vector_element_bits (TREE_TYPE (a));
232 6 : tree inner_type = TREE_TYPE (TREE_TYPE (a));
233 6 : unsigned HOST_WIDE_INT max;
234 6 : tree low_bits, high_bits, a_low, b_low, result_low, signs;
235 :
236 6 : max = GET_MODE_MASK (TYPE_MODE (inner_type));
237 6 : low_bits = build_replicated_int_cst (word_type, width, max >> 1);
238 6 : high_bits = build_replicated_int_cst (word_type, width, max & ~(max >> 1));
239 :
240 6 : a = tree_vec_extract (gsi, word_type, a, bitsize, bitpos);
241 6 : b = tree_vec_extract (gsi, word_type, b, bitsize, bitpos);
242 :
243 6 : signs = gimplify_build2 (gsi, BIT_XOR_EXPR, word_type, a, b);
244 6 : b_low = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, b, low_bits);
245 6 : if (code == PLUS_EXPR)
246 2 : a_low = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, a, low_bits);
247 : else
248 : {
249 4 : a_low = gimplify_build2 (gsi, BIT_IOR_EXPR, word_type, a, high_bits);
250 4 : signs = gimplify_build1 (gsi, BIT_NOT_EXPR, word_type, signs);
251 : }
252 :
253 6 : signs = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, signs, high_bits);
254 6 : result_low = gimplify_build2 (gsi, code, word_type, a_low, b_low);
255 6 : return gimplify_build2 (gsi, BIT_XOR_EXPR, word_type, result_low, signs);
256 : }
257 :
258 : static tree
259 0 : do_negate (gimple_stmt_iterator *gsi, tree word_type, tree b,
260 : tree unused ATTRIBUTE_UNUSED, tree bitpos ATTRIBUTE_UNUSED,
261 : tree bitsize ATTRIBUTE_UNUSED,
262 : enum tree_code code ATTRIBUTE_UNUSED,
263 : tree type ATTRIBUTE_UNUSED)
264 : {
265 0 : unsigned int width = vector_element_bits (TREE_TYPE (b));
266 0 : tree inner_type = TREE_TYPE (TREE_TYPE (b));
267 0 : HOST_WIDE_INT max;
268 0 : tree low_bits, high_bits, b_low, result_low, signs;
269 :
270 0 : max = GET_MODE_MASK (TYPE_MODE (inner_type));
271 0 : low_bits = build_replicated_int_cst (word_type, width, max >> 1);
272 0 : high_bits = build_replicated_int_cst (word_type, width, max & ~(max >> 1));
273 :
274 0 : b = tree_vec_extract (gsi, word_type, b, bitsize, bitpos);
275 :
276 0 : b_low = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, b, low_bits);
277 0 : signs = gimplify_build1 (gsi, BIT_NOT_EXPR, word_type, b);
278 0 : signs = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, signs, high_bits);
279 0 : result_low = gimplify_build2 (gsi, MINUS_EXPR, word_type, high_bits, b_low);
280 0 : return gimplify_build2 (gsi, BIT_XOR_EXPR, word_type, result_low, signs);
281 : }
282 :
283 : /* Expand a vector operation to scalars, by using many operations
284 : whose type is the vector type's inner type. */
285 : static tree
286 2852 : expand_vector_piecewise (gimple_stmt_iterator *gsi, elem_op_func f,
287 : tree type, tree inner_type,
288 : tree a, tree b, enum tree_code code,
289 : bool parallel_p, tree ret_type = NULL_TREE)
290 : {
291 2852 : vec<constructor_elt, va_gc> *v;
292 2852 : tree part_width = TYPE_SIZE (inner_type);
293 2852 : tree index = bitsize_int (0);
294 2852 : int nunits = nunits_for_known_piecewise_op (type);
295 2852 : int delta = (VECTOR_TYPE_P (inner_type)
296 3760 : ? nunits_for_known_piecewise_op (inner_type) : 1);
297 2852 : int i;
298 2852 : location_t loc = gimple_location (gsi_stmt (*gsi));
299 :
300 2852 : if (nunits == 1
301 2852 : || warning_suppressed_p (gsi_stmt (*gsi),
302 : OPT_Wvector_operation_performance))
303 : /* Do not diagnose decomposing single element vectors or when
304 : decomposing vectorizer produced operations. */
305 : ;
306 2521 : else if (ret_type || !parallel_p)
307 2469 : warning_at (loc, OPT_Wvector_operation_performance,
308 : "vector operation will be expanded piecewise");
309 : else
310 52 : warning_at (loc, OPT_Wvector_operation_performance,
311 : "vector operation will be expanded in parallel");
312 :
313 2852 : if (!ret_type)
314 2752 : ret_type = type;
315 2852 : vec_alloc (v, (nunits + delta - 1) / delta);
316 2852 : bool constant_p = true;
317 37263 : for (i = 0; i < nunits;
318 34411 : i += delta, index = int_const_binop (PLUS_EXPR, index, part_width))
319 : {
320 34411 : tree result = f (gsi, inner_type, a, b, index, part_width, code,
321 : ret_type);
322 34411 : if (!CONSTANT_CLASS_P (result))
323 33979 : constant_p = false;
324 34411 : constructor_elt ce = {NULL_TREE, result};
325 34411 : v->quick_push (ce);
326 : }
327 :
328 2852 : if (constant_p)
329 2 : return build_vector_from_ctor (ret_type, v);
330 : else
331 2850 : return build_constructor (ret_type, v);
332 : }
333 :
334 : /* Expand a vector operation to scalars with the freedom to use
335 : a scalar integer type, or to use a different size for the items
336 : in the vector type. */
337 : static tree
338 94 : expand_vector_parallel (gimple_stmt_iterator *gsi, elem_op_func f, tree type,
339 : tree a, tree b, enum tree_code code)
340 : {
341 94 : tree result, compute_type;
342 94 : int n_words = tree_to_uhwi (TYPE_SIZE_UNIT (type)) / UNITS_PER_WORD;
343 94 : location_t loc = gimple_location (gsi_stmt (*gsi));
344 :
345 : /* We have three strategies. If the type is already correct, just do
346 : the operation an element at a time. Else, if the vector is wider than
347 : one word, do it a word at a time; finally, if the vector is smaller
348 : than one word, do it as a scalar. */
349 94 : if (TYPE_MODE (TREE_TYPE (type)) == word_mode)
350 2 : return expand_vector_piecewise (gsi, f,
351 2 : type, TREE_TYPE (type),
352 2 : a, b, code, true);
353 92 : else if (n_words > 1)
354 : {
355 52 : tree word_type = build_word_mode_vector_type (n_words);
356 52 : result = expand_vector_piecewise (gsi, f,
357 52 : word_type, TREE_TYPE (word_type),
358 : a, b, code, true);
359 52 : result = force_gimple_operand_gsi (gsi, result, true, NULL, true,
360 : GSI_SAME_STMT);
361 : }
362 : else
363 : {
364 : /* Use a single scalar operation with a mode no wider than word_mode. */
365 40 : if (!warning_suppressed_p (gsi_stmt (*gsi),
366 : OPT_Wvector_operation_performance))
367 40 : warning_at (loc, OPT_Wvector_operation_performance,
368 : "vector operation will be expanded with a "
369 : "single scalar operation");
370 40 : scalar_int_mode mode
371 40 : = int_mode_for_size (tree_to_uhwi (TYPE_SIZE (type)), 0).require ();
372 40 : compute_type = lang_hooks.types.type_for_mode (mode, 1);
373 40 : result = f (gsi, compute_type, a, b, bitsize_zero_node,
374 40 : TYPE_SIZE (compute_type), code, type);
375 : }
376 :
377 : return result;
378 : }
379 :
380 : /* Expand a vector operation to scalars; for integer types we can use
381 : special bit twiddling tricks to do the sums a word at a time, using
382 : function F_PARALLEL instead of F. These tricks are done only if
383 : they can process at least four items, that is, only if the vector
384 : holds at least four items and if a word can hold four items. */
385 : static tree
386 202 : expand_vector_addition (gimple_stmt_iterator *gsi,
387 : elem_op_func f, elem_op_func f_parallel,
388 : tree type, tree a, tree b, enum tree_code code)
389 : {
390 202 : int parts_per_word = BITS_PER_WORD / vector_element_bits (type);
391 :
392 404 : if (INTEGRAL_TYPE_P (TREE_TYPE (type))
393 202 : && parts_per_word >= 4
394 212 : && nunits_for_known_piecewise_op (type) >= 4)
395 3 : return expand_vector_parallel (gsi, f_parallel,
396 3 : type, a, b, code);
397 : else
398 199 : return expand_vector_piecewise (gsi, f,
399 199 : type, TREE_TYPE (type),
400 199 : a, b, code, false);
401 : }
402 :
403 : /* Expand vector comparison expression OP0 CODE OP1 if the compare optab
404 : is not implemented. */
405 :
406 : static tree
407 23426 : expand_vector_comparison (gimple_stmt_iterator *gsi, tree type, tree op0,
408 : tree op1, enum tree_code code)
409 : {
410 23426 : if (expand_vec_cmp_expr_p (TREE_TYPE (op0), type, code))
411 : return NULL_TREE;
412 :
413 585 : tree t;
414 585 : if (VECTOR_BOOLEAN_TYPE_P (type)
415 585 : && SCALAR_INT_MODE_P (TYPE_MODE (type))
416 711 : && known_lt (GET_MODE_BITSIZE (TYPE_MODE (type)),
417 : TYPE_VECTOR_SUBPARTS (type)
418 : * GET_MODE_BITSIZE (SCALAR_TYPE_MODE
419 : (TREE_TYPE (type)))))
420 : {
421 0 : tree inner_type = TREE_TYPE (TREE_TYPE (op0));
422 0 : tree part_width = vector_element_bits_tree (TREE_TYPE (op0));
423 0 : tree index = bitsize_int (0);
424 0 : int nunits = nunits_for_known_piecewise_op (TREE_TYPE (op0));
425 0 : int prec = GET_MODE_PRECISION (SCALAR_TYPE_MODE (type));
426 0 : tree ret_type = build_nonstandard_integer_type (prec, 1);
427 0 : tree ret_inner_type = boolean_type_node;
428 0 : int i;
429 0 : location_t loc = gimple_location (gsi_stmt (*gsi));
430 0 : t = build_zero_cst (ret_type);
431 :
432 0 : if (TYPE_PRECISION (ret_inner_type) != 1)
433 0 : ret_inner_type = build_nonstandard_integer_type (1, 1);
434 0 : if (!warning_suppressed_p (gsi_stmt (*gsi),
435 : OPT_Wvector_operation_performance))
436 0 : warning_at (loc, OPT_Wvector_operation_performance,
437 : "vector operation will be expanded piecewise");
438 0 : for (i = 0; i < nunits;
439 0 : i++, index = int_const_binop (PLUS_EXPR, index, part_width))
440 : {
441 0 : tree a = tree_vec_extract (gsi, inner_type, op0, part_width,
442 : index);
443 0 : tree b = tree_vec_extract (gsi, inner_type, op1, part_width,
444 : index);
445 0 : tree result = gimplify_build2 (gsi, code, ret_inner_type, a, b);
446 0 : t = gimplify_build3 (gsi, BIT_INSERT_EXPR, ret_type, t, result,
447 0 : bitsize_int (i));
448 : }
449 0 : t = gimplify_build1 (gsi, VIEW_CONVERT_EXPR, type, t);
450 : }
451 : else
452 585 : t = expand_vector_piecewise (gsi, do_compare, type,
453 585 : TREE_TYPE (TREE_TYPE (op0)), op0, op1,
454 : code, false);
455 :
456 : return t;
457 : }
458 :
459 : /* Helper function of expand_vector_divmod. Gimplify a RSHIFT_EXPR in type
460 : of OP0 with shift counts in SHIFTCNTS array and return the temporary holding
461 : the result if successful, otherwise return NULL_TREE. */
462 : static tree
463 494 : add_shift (gimple_stmt_iterator *gsi, tree type, tree op0, int *shiftcnts,
464 : enum tree_code code)
465 : {
466 494 : optab op;
467 494 : unsigned int i, nunits = nunits_for_known_piecewise_op (type);
468 494 : bool scalar_shift = true;
469 :
470 3128 : for (i = 1; i < nunits; i++)
471 : {
472 2634 : if (shiftcnts[i] != shiftcnts[0])
473 0 : scalar_shift = false;
474 : }
475 :
476 494 : if (scalar_shift && shiftcnts[0] == 0)
477 : return op0;
478 :
479 394 : if (scalar_shift)
480 : {
481 394 : op = optab_for_tree_code (code, type, optab_scalar);
482 394 : if (op != unknown_optab
483 394 : && can_implement_p (op, TYPE_MODE (type)))
484 394 : return gimplify_build2 (gsi, code, type, op0,
485 394 : build_int_cst (NULL_TREE, shiftcnts[0]));
486 : }
487 :
488 0 : op = optab_for_tree_code (code, type, optab_vector);
489 0 : if (op != unknown_optab
490 0 : && can_implement_p (op, TYPE_MODE (type)))
491 : {
492 0 : tree_vector_builder vec (type, nunits, 1);
493 0 : for (i = 0; i < nunits; i++)
494 0 : vec.quick_push (build_int_cst (TREE_TYPE (type), shiftcnts[i]));
495 0 : return gimplify_build2 (gsi, code, type, op0, vec.build ());
496 0 : }
497 :
498 : return NULL_TREE;
499 : }
500 :
501 : /* Verify that the target has optabs of VECTYPE to perform all the steps
502 : needed by the multiplication-by-immediate synthesis algorithm described by
503 : ALG and VAR. Return true iff the target supports all the steps. */
504 : static bool
505 1 : target_supports_mult_synth_alg (struct algorithm *alg, mult_variant var,
506 : tree vectype)
507 : {
508 1 : if (alg->op[0] != alg_zero && alg->op[0] != alg_m)
509 : return false;
510 :
511 1 : bool plus_supported_p = target_supports_op_p (vectype, PLUS_EXPR,
512 : optab_vector);
513 1 : bool minus_supported_p = target_supports_op_p (vectype, MINUS_EXPR,
514 : optab_vector);
515 1 : bool negate_supported_p = target_supports_op_p (vectype, NEGATE_EXPR,
516 : optab_vector);
517 :
518 1 : if ((var == negate_variant && !negate_supported_p)
519 1 : || (var == add_variant && !plus_supported_p))
520 : return false;
521 :
522 1 : bool lshift_supported_p
523 1 : = (target_supports_op_p (vectype, LSHIFT_EXPR, optab_scalar)
524 1 : || target_supports_op_p (vectype, LSHIFT_EXPR, optab_vector));
525 :
526 5 : for (int i = 1; i < alg->ops; i++)
527 : {
528 4 : switch (alg->op[i])
529 : {
530 2 : case alg_shift:
531 2 : if (!lshift_supported_p)
532 : return false;
533 : break;
534 0 : case alg_add_t_m2:
535 0 : case alg_add_t2_m:
536 0 : case alg_add_factor:
537 0 : if (!plus_supported_p || !lshift_supported_p)
538 : return false;
539 : break;
540 2 : case alg_sub_t_m2:
541 2 : case alg_sub_t2_m:
542 2 : case alg_sub_factor:
543 2 : if (!minus_supported_p || !lshift_supported_p)
544 : return false;
545 : break;
546 : case alg_unknown:
547 : case alg_m:
548 : case alg_zero:
549 : case alg_impossible:
550 : return false;
551 0 : default:
552 0 : gcc_unreachable ();
553 : }
554 : }
555 :
556 : return true;
557 : }
558 : /* For integer multiplications with exact power of 2, use left shifts.
559 : In constant is a uniform vector, then check if it can be synthesized with
560 : shifts and adds/subs. Use scalar multiplication cost to compare with the
561 : synthesized vector code as done in expmed.cc. */
562 : static tree
563 1 : expand_vector_mult (gimple_stmt_iterator *gsi, tree type, tree op0,
564 : tree op1)
565 : {
566 1 : if (!CONSTANT_CLASS_P (op1))
567 : return NULL_TREE;
568 1 : unsigned int nunits = nunits_for_known_piecewise_op (type);
569 1 : int *shifts = XALLOCAVEC (int, nunits);
570 :
571 1 : bool perfect_pow2_p = true;
572 1 : bool uniform_const_p = true;
573 1 : unsigned HOST_WIDE_INT first_cst, const_elt;
574 3 : for (unsigned int i = 0; i < nunits; i++)
575 : {
576 2 : tree cst = VECTOR_CST_ELT (op1, i);
577 2 : if (TREE_CODE (cst) != INTEGER_CST || tree_int_cst_sgn (cst) <= 0)
578 0 : return NULL_TREE;
579 :
580 2 : perfect_pow2_p &= integer_pow2p (cst);
581 :
582 2 : if (uniform_const_p && tree_fits_uhwi_p (cst))
583 : {
584 2 : const_elt = TREE_INT_CST_LOW (cst);
585 2 : if (i == 0)
586 : first_cst = const_elt;
587 1 : else if (first_cst != const_elt)
588 0 : uniform_const_p = false;
589 : }
590 : else
591 : uniform_const_p = false;
592 :
593 2 : shifts[i] = tree_log2 (cst);
594 : }
595 1 : if (perfect_pow2_p)
596 0 : return add_shift (gsi, type, op0, shifts, LSHIFT_EXPR);
597 1 : else if (uniform_const_p)
598 : {
599 1 : tree scalar_type = type;
600 1 : gcc_assert (TREE_CODE (type) == VECTOR_TYPE);
601 1 : scalar_type = TREE_TYPE (type);
602 :
603 : // handle signed ints
604 1 : bool cast_to_unsigned_p = !TYPE_OVERFLOW_WRAPS (scalar_type);
605 0 : tree multtype = cast_to_unsigned_p
606 0 : ? unsigned_type_for (scalar_type) : scalar_type;
607 0 : tree vectype = cast_to_unsigned_p ? unsigned_type_for (type) : type;
608 :
609 : // estimate the scalar MULT cost by generating dummy rtx insns
610 1 : machine_mode mode = TYPE_MODE (multtype);
611 1 : rtx rop0 = gen_rtx_REG (mode, 0);
612 1 : rtx rop1 = gen_rtx_REG (mode, 1);
613 1 : rtx mult_rtx = gen_rtx_MULT (mode, rop0, rop1);
614 1 : int max_cost = nunits * set_src_cost (mult_rtx, mode, true);
615 :
616 1 : struct algorithm alg;
617 1 : mult_variant variant;
618 :
619 1 : bool possible = choose_mult_variant (TYPE_MODE (vectype), const_elt, &alg,
620 : &variant, max_cost);
621 1 : if (!possible || !target_supports_mult_synth_alg (&alg, variant, vectype))
622 0 : return NULL_TREE;
623 1 : if (cast_to_unsigned_p)
624 : {
625 0 : tree tmp_op = gimplify_build1 (gsi, VIEW_CONVERT_EXPR, vectype, op0);
626 0 : op0 = tmp_op;
627 : }
628 1 : tree accumulator, tmp_var;
629 1 : if (alg.op[0] == alg_zero)
630 0 : accumulator = build_zero_cst (vectype);
631 : else
632 : accumulator = op0;
633 :
634 5 : for (int i = 1; i < alg.ops; i++)
635 : {
636 12 : for (unsigned int j = 0; j < nunits; j++)
637 8 : shifts[j] = alg.log[i];
638 :
639 4 : switch (alg.op[i])
640 : {
641 2 : case alg_shift:
642 2 : accumulator = add_shift (gsi, vectype, accumulator, shifts,
643 : LSHIFT_EXPR);
644 2 : break;
645 0 : case alg_add_t_m2:
646 0 : tmp_var = add_shift (gsi, vectype, op0, shifts, LSHIFT_EXPR);
647 0 : accumulator = gimplify_build2 (gsi, PLUS_EXPR, vectype, tmp_var,
648 : accumulator);
649 0 : break;
650 1 : case alg_sub_t_m2:
651 1 : tmp_var = add_shift (gsi, vectype, op0, shifts, LSHIFT_EXPR);
652 1 : accumulator = gimplify_build2 (gsi, MINUS_EXPR, vectype,
653 : accumulator, tmp_var);
654 1 : break;
655 0 : case alg_add_t2_m:
656 0 : tmp_var = add_shift (gsi, vectype, accumulator, shifts,
657 : LSHIFT_EXPR);
658 0 : accumulator = gimplify_build2 (gsi, PLUS_EXPR, vectype, tmp_var,
659 : op0);
660 0 : break;
661 0 : case alg_sub_t2_m:
662 0 : tmp_var = add_shift (gsi, vectype, accumulator, shifts,
663 : LSHIFT_EXPR);
664 0 : accumulator = gimplify_build2 (gsi, MINUS_EXPR, vectype,
665 : tmp_var, op0);
666 0 : break;
667 0 : case alg_add_factor:
668 0 : tmp_var = add_shift (gsi, vectype, accumulator, shifts,
669 : LSHIFT_EXPR);
670 0 : accumulator = gimplify_build2 (gsi, PLUS_EXPR, vectype,
671 : accumulator, tmp_var);
672 0 : break;
673 1 : case alg_sub_factor:
674 1 : tmp_var = add_shift (gsi, vectype, accumulator, shifts,
675 : LSHIFT_EXPR);
676 1 : accumulator = gimplify_build2 (gsi, MINUS_EXPR, vectype,
677 : tmp_var, accumulator);
678 1 : break;
679 0 : default:
680 0 : gcc_unreachable ();
681 : }
682 : }
683 1 : if (variant == negate_variant)
684 0 : accumulator = gimplify_build1 (gsi, NEGATE_EXPR, vectype, accumulator);
685 1 : else if (variant == add_variant)
686 0 : accumulator = gimplify_build2 (gsi, PLUS_EXPR, vectype, accumulator,
687 : op0);
688 :
689 1 : if (cast_to_unsigned_p)
690 0 : accumulator = gimplify_build1 (gsi, VIEW_CONVERT_EXPR, vectype,
691 : accumulator);
692 1 : return accumulator;
693 : }
694 : return NULL_TREE;
695 : }
696 : /* Try to expand integer vector division by constant using
697 : widening multiply, shifts and additions. */
698 : static tree
699 448 : expand_vector_divmod (gimple_stmt_iterator *gsi, tree type, tree op0,
700 : tree op1, enum tree_code code)
701 : {
702 448 : bool use_pow2 = true;
703 448 : bool has_vector_shift = true;
704 448 : bool use_abs_op1 = false;
705 448 : int mode = -1, this_mode;
706 448 : int pre_shift = -1, post_shift;
707 448 : unsigned int nunits = nunits_for_known_piecewise_op (type);
708 448 : int *shifts = XALLOCAVEC (int, nunits * 4);
709 448 : int *pre_shifts = shifts + nunits;
710 448 : int *post_shifts = pre_shifts + nunits;
711 448 : int *shift_temps = post_shifts + nunits;
712 448 : unsigned HOST_WIDE_INT *mulc = XALLOCAVEC (unsigned HOST_WIDE_INT, nunits);
713 448 : int prec = TYPE_PRECISION (TREE_TYPE (type));
714 448 : unsigned int i;
715 448 : signop sign_p = TYPE_SIGN (TREE_TYPE (type));
716 448 : unsigned HOST_WIDE_INT mask = GET_MODE_MASK (TYPE_MODE (TREE_TYPE (type)));
717 448 : tree cur_op, mulcst, tem;
718 448 : optab op;
719 :
720 448 : if (prec > HOST_BITS_PER_WIDE_INT)
721 : return NULL_TREE;
722 :
723 447 : op = optab_for_tree_code (RSHIFT_EXPR, type, optab_vector);
724 447 : if (op == unknown_optab
725 447 : || !can_implement_p (op, TYPE_MODE (type)))
726 : has_vector_shift = false;
727 :
728 : /* Analysis phase. Determine if all op1 elements are either power
729 : of two and it is possible to expand it using shifts (or for remainder
730 : using masking). Additionally compute the multiplicative constants
731 : and pre and post shifts if the division is to be expanded using
732 : widening or high part multiplication plus shifts. */
733 3189 : for (i = 0; i < nunits; i++)
734 : {
735 2752 : tree cst = VECTOR_CST_ELT (op1, i);
736 2752 : unsigned HOST_WIDE_INT ml;
737 :
738 2752 : if (TREE_CODE (cst) != INTEGER_CST || integer_zerop (cst))
739 10 : return NULL_TREE;
740 2744 : pre_shifts[i] = 0;
741 2744 : post_shifts[i] = 0;
742 2744 : mulc[i] = 0;
743 2744 : if (use_pow2
744 2744 : && (!integer_pow2p (cst) || tree_int_cst_sgn (cst) != 1))
745 : use_pow2 = false;
746 2457 : if (use_pow2)
747 : {
748 824 : shifts[i] = tree_log2 (cst);
749 824 : if (shifts[i] != shifts[0]
750 192 : && code == TRUNC_DIV_EXPR
751 192 : && !has_vector_shift)
752 1952 : use_pow2 = false;
753 : }
754 2744 : if (mode == -2)
755 1000 : continue;
756 1861 : if (sign_p == UNSIGNED)
757 : {
758 814 : unsigned HOST_WIDE_INT mh;
759 814 : unsigned HOST_WIDE_INT d = TREE_INT_CST_LOW (cst) & mask;
760 :
761 814 : if (d >= (HOST_WIDE_INT_1U << (prec - 1)))
762 : /* FIXME: Can transform this into op0 >= op1 ? 1 : 0. */
763 : return NULL_TREE;
764 :
765 812 : if (d <= 1)
766 : {
767 32 : mode = -2;
768 32 : continue;
769 : }
770 :
771 : /* Find a suitable multiplier and right shift count instead of
772 : directly dividing by D. */
773 780 : mh = choose_multiplier (d, prec, prec, &ml, &post_shift);
774 :
775 : /* If the suggested multiplier is more than PREC bits, we can
776 : do better for even divisors, using an initial right shift. */
777 780 : if ((mh != 0 && (d & 1) == 0)
778 638 : || (!has_vector_shift && pre_shift != -1))
779 : {
780 142 : if (has_vector_shift)
781 0 : pre_shift = ctz_or_zero (d);
782 174 : else if (pre_shift == -1)
783 : {
784 : unsigned int j;
785 518 : for (j = 0; j < nunits; j++)
786 : {
787 440 : tree cst2 = VECTOR_CST_ELT (op1, j);
788 440 : unsigned HOST_WIDE_INT d2;
789 440 : int this_pre_shift;
790 :
791 440 : if (!tree_fits_uhwi_p (cst2))
792 : return NULL_TREE;
793 440 : d2 = tree_to_uhwi (cst2) & mask;
794 440 : if (d2 == 0)
795 : return NULL_TREE;
796 440 : this_pre_shift = floor_log2 (d2 & -d2);
797 440 : if (pre_shift == -1 || this_pre_shift < pre_shift)
798 78 : pre_shift = this_pre_shift;
799 : }
800 78 : if (i != 0 && pre_shift != 0)
801 : {
802 : /* Restart. */
803 0 : i = -1U;
804 0 : mode = -1;
805 0 : continue;
806 : }
807 : }
808 174 : if (pre_shift != 0)
809 : {
810 174 : if ((d >> pre_shift) <= 1)
811 : {
812 46 : mode = -2;
813 46 : continue;
814 : }
815 128 : mh = choose_multiplier (d >> pre_shift, prec,
816 : prec - pre_shift,
817 : &ml, &post_shift);
818 128 : gcc_assert (!mh);
819 128 : pre_shifts[i] = pre_shift;
820 : }
821 : }
822 734 : if (!mh)
823 : this_mode = 0;
824 : else
825 192 : this_mode = 1;
826 : }
827 : else
828 : {
829 1047 : HOST_WIDE_INT d = TREE_INT_CST_LOW (cst);
830 1047 : unsigned HOST_WIDE_INT abs_d;
831 :
832 1047 : if (d == -1)
833 : return NULL_TREE;
834 :
835 : /* Since d might be INT_MIN, we have to cast to
836 : unsigned HOST_WIDE_INT before negating to avoid
837 : undefined signed overflow. */
838 1047 : abs_d = (d >= 0
839 : ? (unsigned HOST_WIDE_INT) d
840 : : - (unsigned HOST_WIDE_INT) d);
841 :
842 : /* n rem d = n rem -d */
843 1047 : if (code == TRUNC_MOD_EXPR && d < 0)
844 : {
845 119 : d = abs_d;
846 119 : use_abs_op1 = true;
847 : }
848 1047 : if (abs_d == HOST_WIDE_INT_1U << (prec - 1))
849 : {
850 : /* This case is not handled correctly below. */
851 7 : mode = -2;
852 7 : continue;
853 : }
854 1040 : if (abs_d <= 1)
855 : {
856 32 : mode = -2;
857 32 : continue;
858 : }
859 :
860 1008 : choose_multiplier (abs_d, prec, prec - 1, &ml,
861 : &post_shift);
862 1008 : if (ml >= HOST_WIDE_INT_1U << (prec - 1))
863 : {
864 432 : this_mode = 4 + (d < 0);
865 432 : ml |= HOST_WIDE_INT_M1U << (prec - 1);
866 : }
867 : else
868 576 : this_mode = 2 + (d < 0);
869 : }
870 1742 : mulc[i] = ml;
871 1742 : post_shifts[i] = post_shift;
872 1742 : if ((i && !has_vector_shift && post_shifts[0] != post_shift)
873 1646 : || post_shift >= prec
874 1646 : || pre_shifts[i] >= prec)
875 96 : this_mode = -2;
876 :
877 1742 : if (i == 0)
878 : mode = this_mode;
879 1416 : else if (mode != this_mode)
880 96 : mode = -2;
881 : }
882 :
883 437 : if (use_pow2)
884 : {
885 126 : tree addend = NULL_TREE;
886 126 : if (sign_p == SIGNED)
887 : {
888 : tree uns_type;
889 :
890 : /* Both division and remainder sequences need
891 : op0 < 0 ? mask : 0 computed. It can be either computed as
892 : (type) (((uns_type) (op0 >> (prec - 1))) >> (prec - shifts[i]))
893 : if none of the shifts is 0, or as the conditional. */
894 384 : for (i = 0; i < nunits; i++)
895 336 : if (shifts[i] == 0)
896 : break;
897 64 : uns_type
898 64 : = build_vector_type (build_nonstandard_integer_type (prec, 1),
899 64 : nunits);
900 64 : if (i == nunits && TYPE_MODE (uns_type) == TYPE_MODE (type))
901 : {
902 368 : for (i = 0; i < nunits; i++)
903 320 : shift_temps[i] = prec - 1;
904 48 : cur_op = add_shift (gsi, type, op0, shift_temps, RSHIFT_EXPR);
905 48 : if (cur_op != NULL_TREE)
906 : {
907 48 : cur_op = gimplify_build1 (gsi, VIEW_CONVERT_EXPR,
908 : uns_type, cur_op);
909 416 : for (i = 0; i < nunits; i++)
910 320 : shift_temps[i] = prec - shifts[i];
911 48 : cur_op = add_shift (gsi, uns_type, cur_op, shift_temps,
912 : RSHIFT_EXPR);
913 48 : if (cur_op != NULL_TREE)
914 48 : addend = gimplify_build1 (gsi, VIEW_CONVERT_EXPR,
915 : type, cur_op);
916 : }
917 : }
918 64 : tree mask_type = truth_type_for (type);
919 64 : if (addend == NULL_TREE
920 16 : && expand_vec_cmp_expr_p (type, mask_type, LT_EXPR)
921 80 : && expand_vec_cond_expr_p (type, mask_type))
922 : {
923 16 : tree zero, cst, mask_type, mask;
924 16 : gimple *stmt, *cond;
925 :
926 16 : mask_type = truth_type_for (type);
927 16 : zero = build_zero_cst (type);
928 16 : mask = make_ssa_name (mask_type);
929 16 : cond = gimple_build_assign (mask, LT_EXPR, op0, zero);
930 16 : gsi_insert_before (gsi, cond, GSI_SAME_STMT);
931 16 : tree_vector_builder vec (type, nunits, 1);
932 128 : for (i = 0; i < nunits; i++)
933 96 : vec.quick_push (build_int_cst (TREE_TYPE (type),
934 : (HOST_WIDE_INT_1U
935 96 : << shifts[i]) - 1));
936 16 : cst = vec.build ();
937 16 : addend = make_ssa_name (type);
938 16 : stmt
939 16 : = gimple_build_assign (addend, VEC_COND_EXPR, mask, cst, zero);
940 16 : gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
941 16 : }
942 : }
943 126 : if (code == TRUNC_DIV_EXPR)
944 : {
945 54 : if (sign_p == UNSIGNED)
946 : {
947 : /* q = op0 >> shift; */
948 30 : cur_op = add_shift (gsi, type, op0, shifts, RSHIFT_EXPR);
949 30 : if (cur_op != NULL_TREE)
950 : return cur_op;
951 : }
952 24 : else if (addend != NULL_TREE)
953 : {
954 : /* t1 = op0 + addend;
955 : q = t1 >> shift; */
956 24 : op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
957 24 : if (op != unknown_optab
958 24 : && can_implement_p (op, TYPE_MODE (type)))
959 : {
960 24 : cur_op = gimplify_build2 (gsi, PLUS_EXPR, type, op0, addend);
961 24 : cur_op = add_shift (gsi, type, cur_op, shifts, RSHIFT_EXPR);
962 24 : if (cur_op != NULL_TREE)
963 : return cur_op;
964 : }
965 : }
966 : }
967 : else
968 : {
969 72 : tree mask;
970 72 : tree_vector_builder vec (type, nunits, 1);
971 592 : for (i = 0; i < nunits; i++)
972 448 : vec.quick_push (build_int_cst (TREE_TYPE (type),
973 : (HOST_WIDE_INT_1U
974 448 : << shifts[i]) - 1));
975 72 : mask = vec.build ();
976 72 : op = optab_for_tree_code (BIT_AND_EXPR, type, optab_default);
977 72 : if (op != unknown_optab
978 72 : && can_implement_p (op, TYPE_MODE (type)))
979 : {
980 72 : if (sign_p == UNSIGNED)
981 : /* r = op0 & mask; */
982 32 : return gimplify_build2 (gsi, BIT_AND_EXPR, type, op0, mask);
983 40 : else if (addend != NULL_TREE)
984 : {
985 : /* t1 = op0 + addend;
986 : t2 = t1 & mask;
987 : r = t2 - addend; */
988 40 : op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
989 40 : if (op != unknown_optab
990 40 : && can_implement_p (op, TYPE_MODE (type)))
991 : {
992 40 : cur_op = gimplify_build2 (gsi, PLUS_EXPR, type, op0,
993 : addend);
994 40 : cur_op = gimplify_build2 (gsi, BIT_AND_EXPR, type,
995 : cur_op, mask);
996 40 : op = optab_for_tree_code (MINUS_EXPR, type,
997 : optab_default);
998 40 : if (op != unknown_optab
999 40 : && can_implement_p (op, TYPE_MODE (type)))
1000 40 : return gimplify_build2 (gsi, MINUS_EXPR, type,
1001 40 : cur_op, addend);
1002 : }
1003 : }
1004 : }
1005 72 : }
1006 : }
1007 :
1008 311 : if (mode == -2 || BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN)
1009 : return NULL_TREE;
1010 :
1011 176 : if (!can_mult_highpart_p (TYPE_MODE (type), TYPE_UNSIGNED (type)))
1012 : return NULL_TREE;
1013 :
1014 170 : cur_op = op0;
1015 :
1016 170 : switch (mode)
1017 : {
1018 67 : case 0:
1019 67 : gcc_assert (sign_p == UNSIGNED);
1020 : /* t1 = oprnd0 >> pre_shift;
1021 : t2 = t1 h* ml;
1022 : q = t2 >> post_shift; */
1023 67 : cur_op = add_shift (gsi, type, cur_op, pre_shifts, RSHIFT_EXPR);
1024 67 : if (cur_op == NULL_TREE)
1025 : return NULL_TREE;
1026 : break;
1027 32 : case 1:
1028 32 : gcc_assert (sign_p == UNSIGNED);
1029 224 : for (i = 0; i < nunits; i++)
1030 : {
1031 192 : shift_temps[i] = 1;
1032 192 : post_shifts[i]--;
1033 : }
1034 : break;
1035 71 : case 2:
1036 71 : case 3:
1037 71 : case 4:
1038 71 : case 5:
1039 71 : gcc_assert (sign_p == SIGNED);
1040 567 : for (i = 0; i < nunits; i++)
1041 496 : shift_temps[i] = prec - 1;
1042 : break;
1043 : default:
1044 : return NULL_TREE;
1045 : }
1046 :
1047 170 : tree_vector_builder vec (type, nunits, 1);
1048 1424 : for (i = 0; i < nunits; i++)
1049 1084 : vec.quick_push (build_int_cst (TREE_TYPE (type), mulc[i]));
1050 170 : mulcst = vec.build ();
1051 :
1052 170 : cur_op = gimplify_build2 (gsi, MULT_HIGHPART_EXPR, type, cur_op, mulcst);
1053 :
1054 170 : switch (mode)
1055 : {
1056 67 : case 0:
1057 : /* t1 = oprnd0 >> pre_shift;
1058 : t2 = t1 h* ml;
1059 : q = t2 >> post_shift; */
1060 67 : cur_op = add_shift (gsi, type, cur_op, post_shifts, RSHIFT_EXPR);
1061 67 : break;
1062 32 : case 1:
1063 : /* t1 = oprnd0 h* ml;
1064 : t2 = oprnd0 - t1;
1065 : t3 = t2 >> 1;
1066 : t4 = t1 + t3;
1067 : q = t4 >> (post_shift - 1); */
1068 32 : op = optab_for_tree_code (MINUS_EXPR, type, optab_default);
1069 32 : if (op == unknown_optab
1070 32 : || !can_implement_p (op, TYPE_MODE (type)))
1071 0 : return NULL_TREE;
1072 32 : tem = gimplify_build2 (gsi, MINUS_EXPR, type, op0, cur_op);
1073 32 : tem = add_shift (gsi, type, tem, shift_temps, RSHIFT_EXPR);
1074 32 : op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
1075 32 : if (op == unknown_optab
1076 32 : || !can_implement_p (op, TYPE_MODE (type)))
1077 0 : return NULL_TREE;
1078 32 : tem = gimplify_build2 (gsi, PLUS_EXPR, type, cur_op, tem);
1079 32 : cur_op = add_shift (gsi, type, tem, post_shifts, RSHIFT_EXPR);
1080 32 : if (cur_op == NULL_TREE)
1081 : return NULL_TREE;
1082 : break;
1083 71 : case 2:
1084 71 : case 3:
1085 71 : case 4:
1086 71 : case 5:
1087 : /* t1 = oprnd0 h* ml;
1088 : t2 = t1; [ iff (mode & 2) != 0 ]
1089 : t2 = t1 + oprnd0; [ iff (mode & 2) == 0 ]
1090 : t3 = t2 >> post_shift;
1091 : t4 = oprnd0 >> (prec - 1);
1092 : q = t3 - t4; [ iff (mode & 1) == 0 ]
1093 : q = t4 - t3; [ iff (mode & 1) != 0 ] */
1094 71 : if ((mode & 2) == 0)
1095 : {
1096 16 : op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
1097 16 : if (op == unknown_optab
1098 16 : || !can_implement_p (op, TYPE_MODE (type)))
1099 0 : return NULL_TREE;
1100 16 : cur_op = gimplify_build2 (gsi, PLUS_EXPR, type, cur_op, op0);
1101 : }
1102 71 : cur_op = add_shift (gsi, type, cur_op, post_shifts, RSHIFT_EXPR);
1103 71 : if (cur_op == NULL_TREE)
1104 : return NULL_TREE;
1105 71 : tem = add_shift (gsi, type, op0, shift_temps, RSHIFT_EXPR);
1106 71 : if (tem == NULL_TREE)
1107 : return NULL_TREE;
1108 71 : op = optab_for_tree_code (MINUS_EXPR, type, optab_default);
1109 71 : if (op == unknown_optab
1110 71 : || !can_implement_p (op, TYPE_MODE (type)))
1111 0 : return NULL_TREE;
1112 71 : if ((mode & 1) == 0)
1113 71 : cur_op = gimplify_build2 (gsi, MINUS_EXPR, type, cur_op, tem);
1114 : else
1115 0 : cur_op = gimplify_build2 (gsi, MINUS_EXPR, type, tem, cur_op);
1116 : break;
1117 : default:
1118 : gcc_unreachable ();
1119 : }
1120 :
1121 170 : if (code == TRUNC_DIV_EXPR)
1122 : return cur_op;
1123 :
1124 : /* We divided. Now finish by:
1125 : t1 = q * oprnd1;
1126 : r = oprnd0 - t1; */
1127 87 : op = optab_for_tree_code (MULT_EXPR, type, optab_default);
1128 87 : if (op == unknown_optab
1129 87 : || !can_implement_p (op, TYPE_MODE (type)))
1130 0 : return NULL_TREE;
1131 87 : if (use_abs_op1)
1132 : {
1133 7 : tree_vector_builder elts;
1134 7 : if (!elts.new_unary_operation (type, op1, false))
1135 : return NULL_TREE;
1136 7 : unsigned int count = elts.encoded_nelts ();
1137 14 : for (unsigned int i = 0; i < count; ++i)
1138 : {
1139 7 : tree elem1 = VECTOR_CST_ELT (op1, i);
1140 :
1141 7 : tree elt = const_unop (ABS_EXPR, TREE_TYPE (elem1), elem1);
1142 7 : if (elt == NULL_TREE)
1143 0 : return NULL_TREE;
1144 7 : elts.quick_push (elt);
1145 : }
1146 7 : op1 = elts.build ();
1147 7 : }
1148 87 : tem = gimplify_build2 (gsi, MULT_EXPR, type, cur_op, op1);
1149 87 : op = optab_for_tree_code (MINUS_EXPR, type, optab_default);
1150 87 : if (op == unknown_optab
1151 87 : || !can_implement_p (op, TYPE_MODE (type)))
1152 0 : return NULL_TREE;
1153 87 : return gimplify_build2 (gsi, MINUS_EXPR, type, op0, tem);
1154 : }
1155 :
1156 : /* Expand a vector condition to scalars, by using many conditions
1157 : on the vector's elements. */
1158 :
1159 : static bool
1160 20410 : expand_vector_condition (gimple_stmt_iterator *gsi)
1161 : {
1162 20410 : gassign *stmt = as_a <gassign *> (gsi_stmt (*gsi));
1163 20410 : tree type = TREE_TYPE (gimple_assign_lhs (stmt));
1164 20410 : tree a = gimple_assign_rhs1 (stmt);
1165 20410 : tree a1 = a;
1166 20410 : tree a2 = NULL_TREE;
1167 20410 : bool a_is_scalar_bitmask = false;
1168 20410 : tree b = gimple_assign_rhs2 (stmt);
1169 20410 : tree c = gimple_assign_rhs3 (stmt);
1170 20410 : vec<constructor_elt, va_gc> *v;
1171 20410 : tree constr;
1172 20410 : tree inner_type = TREE_TYPE (type);
1173 20410 : tree width = vector_element_bits_tree (type);
1174 20410 : tree cond_type = TREE_TYPE (TREE_TYPE (a));
1175 20410 : tree index = bitsize_int (0);
1176 20410 : tree comp_width = width;
1177 20410 : tree comp_index = index;
1178 20410 : location_t loc = gimple_location (gsi_stmt (*gsi));
1179 :
1180 20410 : gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (a)));
1181 :
1182 20410 : if (expand_vec_cond_expr_p (type, TREE_TYPE (a)))
1183 : return true;
1184 :
1185 : /* Handle vector boolean types with bitmasks. We can transform
1186 : vbfld_1 = tmp_6 ? vbfld_4 : vbfld_5;
1187 : into
1188 : tmp_7 = tmp_6 & vbfld_4;
1189 : tmp_8 = ~tmp_6;
1190 : tmp_9 = tmp_8 & vbfld_5;
1191 : vbfld_1 = tmp_7 | tmp_9; */
1192 511 : if (VECTOR_BOOLEAN_TYPE_P (type)
1193 0 : && SCALAR_INT_MODE_P (TYPE_MODE (type))
1194 511 : && useless_type_conversion_p (type, TREE_TYPE (a)))
1195 : {
1196 0 : a1 = gimplify_build2 (gsi, BIT_AND_EXPR, type, a, b);
1197 0 : a2 = gimplify_build1 (gsi, BIT_NOT_EXPR, type, a);
1198 0 : a2 = gimplify_build2 (gsi, BIT_AND_EXPR, type, a2, c);
1199 0 : a = gimplify_build2 (gsi, BIT_IOR_EXPR, type, a1, a2);
1200 0 : gimple_assign_set_rhs_from_tree (gsi, a);
1201 0 : update_stmt (gsi_stmt (*gsi));
1202 0 : return true;
1203 : }
1204 :
1205 : /* TODO: try and find a smaller vector type. */
1206 :
1207 511 : if (!warning_suppressed_p (stmt, OPT_Wvector_operation_performance))
1208 511 : warning_at (loc, OPT_Wvector_operation_performance,
1209 : "vector condition will be expanded piecewise");
1210 :
1211 986 : if (SCALAR_INT_MODE_P (TYPE_MODE (TREE_TYPE (a)))
1212 583 : && known_lt (GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (a))),
1213 : TYPE_VECTOR_SUBPARTS (TREE_TYPE (a))
1214 : * GET_MODE_BITSIZE (SCALAR_TYPE_MODE
1215 : (TREE_TYPE (TREE_TYPE (a))))))
1216 : {
1217 0 : a_is_scalar_bitmask = true;
1218 0 : int prec = GET_MODE_PRECISION (SCALAR_TYPE_MODE (TREE_TYPE (a)));
1219 0 : tree atype = build_nonstandard_integer_type (prec, 1);
1220 0 : a = gimplify_build1 (gsi, VIEW_CONVERT_EXPR, atype, a);
1221 : }
1222 : else
1223 511 : comp_width = vector_element_bits_tree (TREE_TYPE (a));
1224 :
1225 511 : int nunits = nunits_for_known_piecewise_op (type);
1226 511 : vec_alloc (v, nunits);
1227 511 : bool constant_p = true;
1228 6098 : for (int i = 0; i < nunits; i++)
1229 : {
1230 5587 : tree aa, result;
1231 5587 : tree bb = tree_vec_extract (gsi, inner_type, b, width, index);
1232 5587 : tree cc = tree_vec_extract (gsi, inner_type, c, width, index);
1233 5587 : if (a_is_scalar_bitmask)
1234 : {
1235 0 : wide_int w = wi::set_bit_in_zero (i, TYPE_PRECISION (TREE_TYPE (a)));
1236 0 : result = gimplify_build2 (gsi, BIT_AND_EXPR, TREE_TYPE (a),
1237 0 : a, wide_int_to_tree (TREE_TYPE (a), w));
1238 0 : aa = gimplify_build2 (gsi, NE_EXPR, boolean_type_node, result,
1239 0 : build_zero_cst (TREE_TYPE (a)));
1240 0 : }
1241 : else
1242 : {
1243 5587 : result = tree_vec_extract (gsi, cond_type, a, comp_width, comp_index);
1244 5587 : aa = gimplify_build2 (gsi, NE_EXPR, boolean_type_node, result,
1245 : build_zero_cst (cond_type));
1246 : }
1247 5587 : result = gimplify_build3 (gsi, COND_EXPR, inner_type, aa, bb, cc);
1248 5587 : if (!CONSTANT_CLASS_P (result))
1249 5503 : constant_p = false;
1250 5587 : constructor_elt ce = {NULL_TREE, result};
1251 5587 : v->quick_push (ce);
1252 5587 : index = int_const_binop (PLUS_EXPR, index, width);
1253 5587 : if (width == comp_width)
1254 : comp_index = index;
1255 : else
1256 0 : comp_index = int_const_binop (PLUS_EXPR, comp_index, comp_width);
1257 : }
1258 :
1259 511 : if (constant_p)
1260 0 : constr = build_vector_from_ctor (type, v);
1261 : else
1262 511 : constr = build_constructor (type, v);
1263 511 : gimple_assign_set_rhs_from_tree (gsi, constr);
1264 511 : update_stmt (gsi_stmt (*gsi));
1265 :
1266 511 : return false;
1267 : }
1268 :
1269 : static tree
1270 25923 : expand_vector_operation (gimple_stmt_iterator *gsi, tree type, tree compute_type,
1271 : gassign *assign, enum tree_code code)
1272 : {
1273 25923 : machine_mode compute_mode = TYPE_MODE (compute_type);
1274 :
1275 : /* If the compute mode is not a vector mode (hence we are not decomposing
1276 : a BLKmode vector to smaller, hardware-supported vectors), we may want
1277 : to expand the operations in parallel. */
1278 25923 : if (!VECTOR_MODE_P (compute_mode))
1279 25031 : switch (code)
1280 : {
1281 349 : case PLUS_EXPR:
1282 349 : case MINUS_EXPR:
1283 349 : if (ANY_INTEGRAL_TYPE_P (type) && !TYPE_OVERFLOW_TRAPS (type))
1284 400 : return expand_vector_addition (gsi, do_binop, do_plus_minus, type,
1285 : gimple_assign_rhs1 (assign),
1286 200 : gimple_assign_rhs2 (assign), code);
1287 : break;
1288 :
1289 3 : case NEGATE_EXPR:
1290 3 : if (ANY_INTEGRAL_TYPE_P (type) && !TYPE_OVERFLOW_TRAPS (type))
1291 2 : return expand_vector_addition (gsi, do_unop, do_negate, type,
1292 : gimple_assign_rhs1 (assign),
1293 2 : NULL_TREE, code);
1294 : break;
1295 :
1296 82 : case BIT_AND_EXPR:
1297 82 : case BIT_IOR_EXPR:
1298 82 : case BIT_XOR_EXPR:
1299 164 : return expand_vector_parallel (gsi, do_binop, type,
1300 : gimple_assign_rhs1 (assign),
1301 82 : gimple_assign_rhs2 (assign), code);
1302 :
1303 9 : case BIT_NOT_EXPR:
1304 9 : return expand_vector_parallel (gsi, do_unop, type,
1305 : gimple_assign_rhs1 (assign),
1306 9 : NULL_TREE, code);
1307 23426 : case EQ_EXPR:
1308 23426 : case NE_EXPR:
1309 23426 : case GT_EXPR:
1310 23426 : case LT_EXPR:
1311 23426 : case GE_EXPR:
1312 23426 : case LE_EXPR:
1313 23426 : case UNEQ_EXPR:
1314 23426 : case UNGT_EXPR:
1315 23426 : case UNLT_EXPR:
1316 23426 : case UNGE_EXPR:
1317 23426 : case UNLE_EXPR:
1318 23426 : case LTGT_EXPR:
1319 23426 : case ORDERED_EXPR:
1320 23426 : case UNORDERED_EXPR:
1321 23426 : {
1322 23426 : tree rhs1 = gimple_assign_rhs1 (assign);
1323 23426 : tree rhs2 = gimple_assign_rhs2 (assign);
1324 :
1325 23426 : return expand_vector_comparison (gsi, type, rhs1, rhs2, code);
1326 : }
1327 :
1328 765 : case TRUNC_DIV_EXPR:
1329 765 : case TRUNC_MOD_EXPR:
1330 765 : {
1331 765 : tree rhs1 = gimple_assign_rhs1 (assign);
1332 765 : tree rhs2 = gimple_assign_rhs2 (assign);
1333 765 : tree ret;
1334 :
1335 765 : if (!optimize
1336 668 : || !VECTOR_INTEGER_TYPE_P (type)
1337 668 : || TREE_CODE (rhs2) != VECTOR_CST
1338 1219 : || !VECTOR_MODE_P (TYPE_MODE (type)))
1339 : break;
1340 :
1341 448 : ret = expand_vector_divmod (gsi, type, rhs1, rhs2, code);
1342 448 : if (ret != NULL_TREE)
1343 : return ret;
1344 : break;
1345 : }
1346 103 : case MULT_EXPR:
1347 103 : {
1348 103 : tree rhs1 = gimple_assign_rhs1 (assign);
1349 103 : tree rhs2 = gimple_assign_rhs2 (assign);
1350 103 : tree ret;
1351 :
1352 103 : if (!optimize
1353 82 : || !VECTOR_INTEGER_TYPE_P (type)
1354 61 : || TREE_CODE (rhs2) != VECTOR_CST
1355 106 : || !VECTOR_MODE_P (TYPE_MODE (type)))
1356 : break;
1357 :
1358 1 : ret = expand_vector_mult (gsi, type, rhs1, rhs2);
1359 1 : if (ret != NULL_TREE)
1360 : return ret;
1361 : break;
1362 : }
1363 : default:
1364 : break;
1365 : }
1366 :
1367 1907 : if (TREE_CODE_CLASS (code) == tcc_unary)
1368 98 : return expand_vector_piecewise (gsi, do_unop, type, compute_type,
1369 : gimple_assign_rhs1 (assign),
1370 98 : NULL_TREE, code, false);
1371 : else
1372 3618 : return expand_vector_piecewise (gsi, do_binop, type, compute_type,
1373 : gimple_assign_rhs1 (assign),
1374 1809 : gimple_assign_rhs2 (assign), code, false);
1375 : }
1376 :
1377 : /* Try to optimize
1378 : a_5 = { b_7, b_7 + 3, b_7 + 6, b_7 + 9 };
1379 : style stmts into:
1380 : _9 = { b_7, b_7, b_7, b_7 };
1381 : a_5 = _9 + { 0, 3, 6, 9 };
1382 : because vector splat operation is usually more efficient
1383 : than piecewise initialization of the vector. */
1384 :
1385 : static void
1386 142569 : optimize_vector_constructor (gimple_stmt_iterator *gsi)
1387 : {
1388 142569 : gassign *stmt = as_a <gassign *> (gsi_stmt (*gsi));
1389 142569 : tree lhs = gimple_assign_lhs (stmt);
1390 142569 : tree rhs = gimple_assign_rhs1 (stmt);
1391 142569 : tree type = TREE_TYPE (rhs);
1392 142569 : unsigned int i, j;
1393 142569 : unsigned HOST_WIDE_INT nelts;
1394 142569 : bool all_same = true;
1395 142569 : constructor_elt *elt;
1396 142569 : gimple *g;
1397 142569 : tree base = NULL_TREE;
1398 142569 : optab op;
1399 :
1400 142569 : if (!TYPE_VECTOR_SUBPARTS (type).is_constant (&nelts)
1401 142569 : || nelts <= 2
1402 44245 : || CONSTRUCTOR_NELTS (rhs) != nelts)
1403 142565 : return;
1404 20688 : op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
1405 20688 : if (op == unknown_optab
1406 20688 : || !can_implement_p (op, TYPE_MODE (type)))
1407 513 : return;
1408 101751 : FOR_EACH_VEC_SAFE_ELT (CONSTRUCTOR_ELTS (rhs), i, elt)
1409 90201 : if (TREE_CODE (elt->value) != SSA_NAME
1410 90201 : || TREE_CODE (TREE_TYPE (elt->value)) == VECTOR_TYPE)
1411 : return;
1412 : else
1413 : {
1414 88302 : tree this_base = elt->value;
1415 88302 : if (this_base != CONSTRUCTOR_ELT (rhs, 0)->value)
1416 6738 : all_same = false;
1417 89363 : for (j = 0; j < nelts + 1; j++)
1418 : {
1419 89363 : g = SSA_NAME_DEF_STMT (this_base);
1420 89363 : if (is_gimple_assign (g)
1421 56857 : && gimple_assign_rhs_code (g) == PLUS_EXPR
1422 2592 : && TREE_CODE (gimple_assign_rhs2 (g)) == INTEGER_CST
1423 1061 : && TREE_CODE (gimple_assign_rhs1 (g)) == SSA_NAME
1424 90424 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (gimple_assign_rhs1 (g)))
1425 1061 : this_base = gimple_assign_rhs1 (g);
1426 : else
1427 : break;
1428 : }
1429 88302 : if (i == 0)
1430 : base = this_base;
1431 68559 : else if (this_base != base)
1432 : return;
1433 : }
1434 11550 : if (all_same)
1435 : return;
1436 4 : tree_vector_builder cst (type, nelts, 1);
1437 20 : for (i = 0; i < nelts; i++)
1438 : {
1439 16 : tree this_base = CONSTRUCTOR_ELT (rhs, i)->value;
1440 16 : tree elt = build_zero_cst (TREE_TYPE (base));
1441 29 : while (this_base != base)
1442 : {
1443 13 : g = SSA_NAME_DEF_STMT (this_base);
1444 13 : elt = fold_binary (PLUS_EXPR, TREE_TYPE (base),
1445 : elt, gimple_assign_rhs2 (g));
1446 13 : if (elt == NULL_TREE
1447 13 : || TREE_CODE (elt) != INTEGER_CST
1448 26 : || TREE_OVERFLOW (elt))
1449 0 : return;
1450 13 : this_base = gimple_assign_rhs1 (g);
1451 : }
1452 16 : cst.quick_push (elt);
1453 : }
1454 20 : for (i = 0; i < nelts; i++)
1455 16 : CONSTRUCTOR_ELT (rhs, i)->value = base;
1456 4 : g = gimple_build_assign (make_ssa_name (type), rhs);
1457 4 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
1458 4 : g = gimple_build_assign (lhs, PLUS_EXPR, gimple_assign_lhs (g),
1459 : cst.build ());
1460 4 : gsi_replace (gsi, g, false);
1461 4 : }
1462 :
1463 : /* Return a type for the widest vector mode with the same element type as
1464 : type ORIGINAL_VECTOR_TYPE, with at most the same number of elements as type
1465 : ORIGINAL_VECTOR_TYPE and that is supported by the target for an operation
1466 : with optab OP, or return NULL_TREE if none is found. */
1467 :
1468 : static tree
1469 2862 : type_for_widest_vector_mode (tree original_vector_type, optab op)
1470 : {
1471 2862 : gcc_assert (VECTOR_TYPE_P (original_vector_type));
1472 2862 : tree type = TREE_TYPE (original_vector_type);
1473 2862 : machine_mode inner_mode = TYPE_MODE (type);
1474 2862 : machine_mode best_mode = VOIDmode, mode;
1475 2862 : poly_int64 best_nunits = 0;
1476 :
1477 2862 : if (SCALAR_FLOAT_MODE_P (inner_mode))
1478 : mode = MIN_MODE_VECTOR_FLOAT;
1479 2506 : else if (SCALAR_FRACT_MODE_P (inner_mode))
1480 : mode = MIN_MODE_VECTOR_FRACT;
1481 2506 : else if (SCALAR_UFRACT_MODE_P (inner_mode))
1482 : mode = MIN_MODE_VECTOR_UFRACT;
1483 2506 : else if (SCALAR_ACCUM_MODE_P (inner_mode))
1484 : mode = MIN_MODE_VECTOR_ACCUM;
1485 2506 : else if (SCALAR_UACCUM_MODE_P (inner_mode))
1486 : mode = MIN_MODE_VECTOR_UACCUM;
1487 2506 : else if (inner_mode == BImode)
1488 : mode = MIN_MODE_VECTOR_BOOL;
1489 : else
1490 2862 : mode = MIN_MODE_VECTOR_INT;
1491 :
1492 89448 : FOR_EACH_MODE_FROM (mode, mode)
1493 86586 : if (GET_MODE_INNER (mode) == inner_mode
1494 33380 : && maybe_gt (GET_MODE_NUNITS (mode), best_nunits)
1495 16690 : && can_implement_p (op, mode)
1496 105885 : && known_le (GET_MODE_NUNITS (mode),
1497 : TYPE_VECTOR_SUBPARTS (original_vector_type)))
1498 4166 : best_mode = mode, best_nunits = GET_MODE_NUNITS (mode);
1499 :
1500 2862 : if (best_mode == VOIDmode)
1501 : return NULL_TREE;
1502 : else
1503 1090 : return build_vector_type_for_mode (type, best_mode);
1504 : }
1505 :
1506 :
1507 : /* Build a reference to the element of the vector VECT. Function
1508 : returns either the element itself, either BIT_FIELD_REF, or an
1509 : ARRAY_REF expression.
1510 :
1511 : GSI is required to insert temporary variables while building a
1512 : refernece to the element of the vector VECT.
1513 :
1514 : PTMPVEC is a pointer to the temporary variable for caching
1515 : purposes. In case when PTMPVEC is NULL new temporary variable
1516 : will be created. */
1517 : static tree
1518 106015 : vector_element (gimple_stmt_iterator *gsi, tree vect, tree idx, tree *ptmpvec)
1519 : {
1520 106015 : tree vect_type, vect_elt_type;
1521 106015 : gimple *asgn;
1522 106015 : tree tmpvec;
1523 106015 : tree arraytype;
1524 106015 : bool need_asgn = true;
1525 106015 : unsigned int elements;
1526 :
1527 106015 : vect_type = TREE_TYPE (vect);
1528 106015 : vect_elt_type = TREE_TYPE (vect_type);
1529 106015 : elements = nunits_for_known_piecewise_op (vect_type);
1530 :
1531 106015 : if (TREE_CODE (idx) == INTEGER_CST)
1532 : {
1533 98394 : unsigned HOST_WIDE_INT index;
1534 :
1535 : /* Given that we're about to compute a binary modulus,
1536 : we don't care about the high bits of the value. */
1537 98394 : index = TREE_INT_CST_LOW (idx);
1538 98394 : if (!tree_fits_uhwi_p (idx) || index >= elements)
1539 : {
1540 0 : index &= elements - 1;
1541 0 : idx = build_int_cst (TREE_TYPE (idx), index);
1542 : }
1543 :
1544 : /* When lowering a vector statement sequence do some easy
1545 : simplification by looking through intermediate vector results. */
1546 98394 : if (TREE_CODE (vect) == SSA_NAME)
1547 : {
1548 51462 : gimple *def_stmt = SSA_NAME_DEF_STMT (vect);
1549 51462 : if (is_gimple_assign (def_stmt)
1550 51462 : && (gimple_assign_rhs_code (def_stmt) == VECTOR_CST
1551 50047 : || gimple_assign_rhs_code (def_stmt) == CONSTRUCTOR))
1552 2487 : vect = gimple_assign_rhs1 (def_stmt);
1553 : }
1554 :
1555 98394 : if (TREE_CODE (vect) == VECTOR_CST)
1556 48219 : return VECTOR_CST_ELT (vect, index);
1557 50175 : else if (TREE_CODE (vect) == CONSTRUCTOR
1558 50175 : && (CONSTRUCTOR_NELTS (vect) == 0
1559 1200 : || TREE_CODE (TREE_TYPE (CONSTRUCTOR_ELT (vect, 0)->value))
1560 : != VECTOR_TYPE))
1561 : {
1562 169 : if (index < CONSTRUCTOR_NELTS (vect))
1563 169 : return CONSTRUCTOR_ELT (vect, index)->value;
1564 0 : return build_zero_cst (vect_elt_type);
1565 : }
1566 : else
1567 : {
1568 50006 : tree size = vector_element_bits_tree (vect_type);
1569 50006 : tree pos = fold_build2 (MULT_EXPR, bitsizetype, bitsize_int (index),
1570 : size);
1571 50006 : return fold_build3 (BIT_FIELD_REF, vect_elt_type, vect, size, pos);
1572 : }
1573 : }
1574 :
1575 7621 : if (!ptmpvec)
1576 0 : tmpvec = create_tmp_var (vect_type, "vectmp");
1577 7621 : else if (!*ptmpvec)
1578 1287 : tmpvec = *ptmpvec = create_tmp_var (vect_type, "vectmp");
1579 : else
1580 : {
1581 : tmpvec = *ptmpvec;
1582 : need_asgn = false;
1583 : }
1584 :
1585 1287 : if (need_asgn)
1586 : {
1587 1287 : TREE_ADDRESSABLE (tmpvec) = 1;
1588 1287 : asgn = gimple_build_assign (tmpvec, vect);
1589 1287 : gsi_insert_before (gsi, asgn, GSI_SAME_STMT);
1590 : }
1591 :
1592 7621 : arraytype = build_array_type_nelts (vect_elt_type, elements);
1593 7621 : return build4 (ARRAY_REF, vect_elt_type,
1594 : build1 (VIEW_CONVERT_EXPR, arraytype, tmpvec),
1595 7621 : idx, NULL_TREE, NULL_TREE);
1596 : }
1597 :
1598 : /* Check if VEC_PERM_EXPR within the given setting is supported
1599 : by hardware, or lower it piecewise.
1600 :
1601 : When VEC_PERM_EXPR has the same first and second operands:
1602 : VEC_PERM_EXPR <v0, v0, mask> the lowered version would be
1603 : {v0[mask[0]], v0[mask[1]], ...}
1604 : MASK and V0 must have the same number of elements.
1605 :
1606 : Otherwise VEC_PERM_EXPR <v0, v1, mask> is lowered to
1607 : {mask[0] < len(v0) ? v0[mask[0]] : v1[mask[0]], ...}
1608 : V0 and V1 must have the same type. MASK, V0, V1 must have the
1609 : same number of arguments. */
1610 :
1611 : static void
1612 90253 : lower_vec_perm (gimple_stmt_iterator *gsi)
1613 : {
1614 90253 : gassign *stmt = as_a <gassign *> (gsi_stmt (*gsi));
1615 90253 : tree mask = gimple_assign_rhs3 (stmt);
1616 90253 : tree vec0 = gimple_assign_rhs1 (stmt);
1617 90253 : tree vec1 = gimple_assign_rhs2 (stmt);
1618 90253 : tree res_vect_type = TREE_TYPE (gimple_assign_lhs (stmt));
1619 90253 : tree vect_type = TREE_TYPE (vec0);
1620 90253 : tree mask_type = TREE_TYPE (mask);
1621 90253 : tree vect_elt_type = TREE_TYPE (vect_type);
1622 90253 : tree mask_elt_type = TREE_TYPE (mask_type);
1623 90253 : unsigned HOST_WIDE_INT elements, in_elements;
1624 90253 : vec<constructor_elt, va_gc> *v;
1625 90253 : tree constr, t, si, i_val;
1626 90253 : tree vec0tmp = NULL_TREE, vec1tmp = NULL_TREE, masktmp = NULL_TREE;
1627 90253 : bool two_operand_p = !operand_equal_p (vec0, vec1, 0);
1628 90253 : location_t loc = gimple_location (gsi_stmt (*gsi));
1629 90253 : unsigned i;
1630 :
1631 90253 : if (!TYPE_VECTOR_SUBPARTS (res_vect_type).is_constant (&elements)
1632 90253 : || !TYPE_VECTOR_SUBPARTS (vect_type).is_constant (&in_elements))
1633 81301 : return;
1634 :
1635 90253 : if (TREE_CODE (mask) == SSA_NAME)
1636 : {
1637 883 : gimple *def_stmt = SSA_NAME_DEF_STMT (mask);
1638 883 : if (is_gimple_assign (def_stmt)
1639 883 : && gimple_assign_rhs_code (def_stmt) == VECTOR_CST)
1640 12 : mask = gimple_assign_rhs1 (def_stmt);
1641 : }
1642 :
1643 90253 : vec_perm_builder sel_int;
1644 :
1645 90253 : if (TREE_CODE (mask) == VECTOR_CST
1646 90253 : && tree_to_vec_perm_builder (&sel_int, mask))
1647 : {
1648 89381 : vec_perm_indices indices (sel_int, 2, in_elements);
1649 89381 : machine_mode vmode = TYPE_MODE (vect_type);
1650 89381 : tree lhs_type = TREE_TYPE (gimple_assign_lhs (stmt));
1651 89381 : machine_mode lhs_mode = TYPE_MODE (lhs_type);
1652 89381 : if (can_vec_perm_const_p (lhs_mode, vmode, indices))
1653 : {
1654 81258 : gimple_assign_set_rhs3 (stmt, mask);
1655 81258 : update_stmt (stmt);
1656 81258 : return;
1657 : }
1658 : /* Also detect vec_shr pattern - VEC_PERM_EXPR with zero
1659 : vector as VEC1 and a right element shift MASK. */
1660 8123 : if (can_implement_p (vec_shr_optab, TYPE_MODE (vect_type))
1661 7356 : && TREE_CODE (vec1) == VECTOR_CST
1662 31 : && initializer_zerop (vec1)
1663 31 : && maybe_ne (indices[0], 0)
1664 15479 : && known_lt (poly_uint64 (indices[0]), elements))
1665 : {
1666 28 : bool ok_p = indices.series_p (0, 1, indices[0], 1);
1667 28 : if (!ok_p)
1668 : {
1669 1 : for (i = 1; i < elements; ++i)
1670 : {
1671 1 : poly_uint64 actual = indices[i];
1672 1 : poly_uint64 expected = i + indices[0];
1673 : /* Indices into the second vector are all equivalent. */
1674 1 : if (maybe_lt (actual, elements)
1675 1 : ? maybe_ne (actual, expected)
1676 1 : : maybe_lt (expected, elements))
1677 : break;
1678 : }
1679 1 : ok_p = i == elements;
1680 : }
1681 1 : if (ok_p)
1682 : {
1683 27 : gimple_assign_set_rhs3 (stmt, mask);
1684 27 : update_stmt (stmt);
1685 27 : return;
1686 : }
1687 : }
1688 : /* And similarly vec_shl pattern. */
1689 8096 : if (can_implement_p (vec_shl_optab, TYPE_MODE (vect_type))
1690 7329 : && TREE_CODE (vec0) == VECTOR_CST
1691 8104 : && initializer_zerop (vec0))
1692 : {
1693 : unsigned int first = 0;
1694 49 : for (i = 0; i < elements; ++i)
1695 42 : if (known_eq (poly_uint64 (indices[i]), elements))
1696 : {
1697 5 : if (i == 0 || first)
1698 : break;
1699 : first = i;
1700 : }
1701 74 : else if (first
1702 74 : ? maybe_ne (poly_uint64 (indices[i]),
1703 14 : elements + i - first)
1704 37 : : maybe_ge (poly_uint64 (indices[i]), elements))
1705 : break;
1706 8 : if (first && i == elements)
1707 : {
1708 5 : gimple_assign_set_rhs3 (stmt, mask);
1709 5 : update_stmt (stmt);
1710 5 : return;
1711 : }
1712 : }
1713 89381 : }
1714 872 : else if (can_vec_perm_var_p (TYPE_MODE (vect_type)))
1715 : return;
1716 :
1717 8952 : if (!warning_suppressed_p (stmt, OPT_Wvector_operation_performance))
1718 8952 : warning_at (loc, OPT_Wvector_operation_performance,
1719 : "vector shuffling operation will be expanded piecewise");
1720 :
1721 8952 : vec_alloc (v, elements);
1722 8952 : bool constant_p = true;
1723 60720 : for (i = 0; i < elements; i++)
1724 : {
1725 51768 : si = size_int (i);
1726 51768 : i_val = vector_element (gsi, mask, si, &masktmp);
1727 :
1728 51768 : if (TREE_CODE (i_val) == INTEGER_CST)
1729 : {
1730 46626 : unsigned HOST_WIDE_INT index;
1731 :
1732 46626 : index = TREE_INT_CST_LOW (i_val);
1733 46626 : if (!tree_fits_uhwi_p (i_val) || index >= in_elements)
1734 14677 : i_val = build_int_cst (mask_elt_type, index & (in_elements - 1));
1735 :
1736 46626 : if (two_operand_p && (index & in_elements) != 0)
1737 14663 : t = vector_element (gsi, vec1, i_val, &vec1tmp);
1738 : else
1739 31963 : t = vector_element (gsi, vec0, i_val, &vec0tmp);
1740 :
1741 46626 : t = force_gimple_operand_gsi (gsi, t, true, NULL_TREE,
1742 : true, GSI_SAME_STMT);
1743 : }
1744 : else
1745 : {
1746 5142 : tree cond = NULL_TREE, v0_val;
1747 :
1748 5142 : if (two_operand_p)
1749 : {
1750 2479 : cond = fold_build2 (BIT_AND_EXPR, mask_elt_type, i_val,
1751 : build_int_cst (mask_elt_type, elements));
1752 2479 : cond = force_gimple_operand_gsi (gsi, cond, true, NULL_TREE,
1753 : true, GSI_SAME_STMT);
1754 : }
1755 :
1756 5142 : i_val = fold_build2 (BIT_AND_EXPR, mask_elt_type, i_val,
1757 : build_int_cst (mask_elt_type, elements - 1));
1758 5142 : i_val = force_gimple_operand_gsi (gsi, i_val, true, NULL_TREE,
1759 : true, GSI_SAME_STMT);
1760 :
1761 5142 : v0_val = vector_element (gsi, vec0, i_val, &vec0tmp);
1762 5142 : v0_val = force_gimple_operand_gsi (gsi, v0_val, true, NULL_TREE,
1763 : true, GSI_SAME_STMT);
1764 :
1765 5142 : if (two_operand_p)
1766 : {
1767 2479 : tree v1_val;
1768 :
1769 2479 : v1_val = vector_element (gsi, vec1, i_val, &vec1tmp);
1770 2479 : v1_val = force_gimple_operand_gsi (gsi, v1_val, true, NULL_TREE,
1771 : true, GSI_SAME_STMT);
1772 :
1773 2479 : cond = fold_build2 (EQ_EXPR, boolean_type_node,
1774 : cond, build_zero_cst (mask_elt_type));
1775 2479 : cond = fold_build3 (COND_EXPR, vect_elt_type,
1776 : cond, v0_val, v1_val);
1777 2479 : t = force_gimple_operand_gsi (gsi, cond, true, NULL_TREE,
1778 : true, GSI_SAME_STMT);
1779 : }
1780 : else
1781 : t = v0_val;
1782 : }
1783 :
1784 51768 : if (!CONSTANT_CLASS_P (t))
1785 49554 : constant_p = false;
1786 51768 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, t);
1787 : }
1788 :
1789 8952 : if (constant_p)
1790 7 : constr = build_vector_from_ctor (res_vect_type, v);
1791 : else
1792 8945 : constr = build_constructor (res_vect_type, v);
1793 8952 : gimple_assign_set_rhs_from_tree (gsi, constr);
1794 8952 : update_stmt (gsi_stmt (*gsi));
1795 90253 : }
1796 :
1797 : /* Return the type that should be used to implement OP on type TYPE.
1798 : This is TYPE itself if the target can do the operation directly,
1799 : otherwise it is a scalar type or a smaller vector type. */
1800 :
1801 : static tree
1802 216107 : get_compute_type (optab op, tree type)
1803 : {
1804 216107 : if (op)
1805 : {
1806 385362 : if (VECTOR_MODE_P (TYPE_MODE (type))
1807 383849 : && can_implement_p (op, TYPE_MODE (type)))
1808 : return type;
1809 :
1810 : /* For very wide vectors, try using a smaller vector mode. */
1811 2791 : tree vector_compute_type = type_for_widest_vector_mode (type, op);
1812 2791 : if (vector_compute_type != NULL_TREE
1813 2791 : && maybe_ne (TYPE_VECTOR_SUBPARTS (vector_compute_type), 1U)
1814 3755 : && can_implement_p (op, TYPE_MODE (vector_compute_type)))
1815 964 : return vector_compute_type;
1816 : }
1817 :
1818 : /* There is no operation in hardware, so fall back to scalars. */
1819 25253 : return TREE_TYPE (type);
1820 : }
1821 :
1822 : static tree
1823 14 : do_cond (gimple_stmt_iterator *gsi, tree inner_type, tree a, tree b,
1824 : tree bitpos, tree bitsize, enum tree_code code,
1825 : tree type ATTRIBUTE_UNUSED)
1826 : {
1827 14 : if (VECTOR_TYPE_P (TREE_TYPE (a)))
1828 14 : a = tree_vec_extract (gsi, inner_type, a, bitsize, bitpos);
1829 14 : if (VECTOR_TYPE_P (TREE_TYPE (b)))
1830 14 : b = tree_vec_extract (gsi, inner_type, b, bitsize, bitpos);
1831 14 : tree cond = gimple_assign_rhs1 (gsi_stmt (*gsi));
1832 14 : return gimplify_build3 (gsi, code, inner_type, unshare_expr (cond), a, b);
1833 : }
1834 :
1835 : /* Expand a vector COND_EXPR to scalars, piecewise. */
1836 : static void
1837 7 : expand_vector_scalar_condition (gimple_stmt_iterator *gsi)
1838 : {
1839 7 : gassign *stmt = as_a <gassign *> (gsi_stmt (*gsi));
1840 7 : tree lhs = gimple_assign_lhs (stmt);
1841 7 : tree type = TREE_TYPE (lhs);
1842 7 : tree compute_type = get_compute_type (mov_optab, type);
1843 7 : machine_mode compute_mode = TYPE_MODE (compute_type);
1844 7 : gcc_assert (compute_mode != BLKmode);
1845 7 : tree rhs2 = gimple_assign_rhs2 (stmt);
1846 7 : tree rhs3 = gimple_assign_rhs3 (stmt);
1847 7 : tree new_rhs;
1848 :
1849 : /* If the compute mode is not a vector mode (hence we are not decomposing
1850 : a BLKmode vector to smaller, hardware-supported vectors), we may want
1851 : to expand the operations in parallel. */
1852 7 : if (!VECTOR_MODE_P (compute_mode))
1853 0 : new_rhs = expand_vector_parallel (gsi, do_cond, type, rhs2, rhs3,
1854 : COND_EXPR);
1855 : else
1856 7 : new_rhs = expand_vector_piecewise (gsi, do_cond, type, compute_type,
1857 : rhs2, rhs3, COND_EXPR, false);
1858 7 : if (!useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (new_rhs)))
1859 0 : new_rhs = gimplify_build1 (gsi, VIEW_CONVERT_EXPR, TREE_TYPE (lhs),
1860 : new_rhs);
1861 :
1862 : /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
1863 : way to do it is change expand_vector_operation and its callees to
1864 : return a tree_code, RHS1 and RHS2 instead of a tree. */
1865 7 : gimple_assign_set_rhs_from_tree (gsi, new_rhs);
1866 7 : update_stmt (gsi_stmt (*gsi));
1867 7 : }
1868 :
1869 : /* Callback for expand_vector_piecewise to do VEC_CONVERT ifn call
1870 : lowering. If INNER_TYPE is not a vector type, this is a scalar
1871 : fallback. */
1872 :
1873 : static tree
1874 18068 : do_vec_conversion (gimple_stmt_iterator *gsi, tree inner_type, tree a,
1875 : tree decl, tree bitpos, tree bitsize,
1876 : enum tree_code code, tree type)
1877 : {
1878 18068 : a = tree_vec_extract (gsi, inner_type, a, bitsize, bitpos);
1879 18068 : if (!VECTOR_TYPE_P (inner_type))
1880 18068 : return gimplify_build1 (gsi, code, TREE_TYPE (type), a);
1881 0 : if (code == CALL_EXPR)
1882 : {
1883 0 : gimple *g = gimple_build_call (decl, 1, a);
1884 0 : tree lhs = make_ssa_name (TREE_TYPE (TREE_TYPE (decl)));
1885 0 : gimple_call_set_lhs (g, lhs);
1886 0 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
1887 0 : return lhs;
1888 : }
1889 : else
1890 : {
1891 0 : tree outer_type = build_vector_type (TREE_TYPE (type),
1892 0 : TYPE_VECTOR_SUBPARTS (inner_type));
1893 0 : return gimplify_build1 (gsi, code, outer_type, a);
1894 : }
1895 : }
1896 :
1897 : /* Similarly, but for narrowing conversion. */
1898 :
1899 : static tree
1900 43 : do_vec_narrow_conversion (gimple_stmt_iterator *gsi, tree inner_type, tree a,
1901 : tree, tree bitpos, tree, enum tree_code code,
1902 : tree type)
1903 : {
1904 43 : tree itype = build_vector_type (TREE_TYPE (inner_type),
1905 43 : exact_div (TYPE_VECTOR_SUBPARTS (inner_type),
1906 : 2));
1907 43 : tree b = tree_vec_extract (gsi, itype, a, TYPE_SIZE (itype), bitpos);
1908 43 : tree c = tree_vec_extract (gsi, itype, a, TYPE_SIZE (itype),
1909 : int_const_binop (PLUS_EXPR, bitpos,
1910 43 : TYPE_SIZE (itype)));
1911 43 : tree outer_type = build_vector_type (TREE_TYPE (type),
1912 43 : TYPE_VECTOR_SUBPARTS (inner_type));
1913 43 : return gimplify_build2 (gsi, code, outer_type, b, c);
1914 : }
1915 :
1916 : /* Expand VEC_CONVERT ifn call. */
1917 :
1918 : static void
1919 478 : expand_vector_conversion (gimple_stmt_iterator *gsi)
1920 : {
1921 478 : gimple *stmt = gsi_stmt (*gsi);
1922 478 : gimple *g;
1923 478 : tree lhs = gimple_call_lhs (stmt);
1924 478 : if (lhs == NULL_TREE)
1925 : {
1926 1 : g = gimple_build_nop ();
1927 1 : gsi_replace (gsi, g, false);
1928 388 : return;
1929 : }
1930 477 : tree arg = gimple_call_arg (stmt, 0);
1931 477 : tree ret_type = TREE_TYPE (lhs);
1932 477 : tree arg_type = TREE_TYPE (arg);
1933 477 : tree new_rhs, new_lhs, compute_type = TREE_TYPE (arg_type);
1934 477 : enum tree_code code = NOP_EXPR;
1935 477 : enum tree_code code1 = ERROR_MARK;
1936 477 : enum { NARROW, NONE, WIDEN } modifier = NONE;
1937 477 : optab optab1 = unknown_optab;
1938 :
1939 477 : gcc_checking_assert (VECTOR_TYPE_P (ret_type) && VECTOR_TYPE_P (arg_type));
1940 954 : if (INTEGRAL_TYPE_P (TREE_TYPE (ret_type))
1941 778 : && SCALAR_FLOAT_TYPE_P (TREE_TYPE (arg_type)))
1942 : code = FIX_TRUNC_EXPR;
1943 648 : else if (INTEGRAL_TYPE_P (TREE_TYPE (arg_type))
1944 596 : && SCALAR_FLOAT_TYPE_P (TREE_TYPE (ret_type)))
1945 : code = FLOAT_EXPR;
1946 477 : unsigned int ret_elt_bits = vector_element_bits (ret_type);
1947 477 : unsigned int arg_elt_bits = vector_element_bits (arg_type);
1948 477 : if (ret_elt_bits < arg_elt_bits)
1949 : modifier = NARROW;
1950 272 : else if (ret_elt_bits > arg_elt_bits)
1951 130 : modifier = WIDEN;
1952 :
1953 477 : auto_vec<std::pair<tree, tree_code>, 2> converts;
1954 477 : if (supportable_indirect_convert_operation (code, ret_type, arg_type,
1955 : converts))
1956 : {
1957 293 : new_rhs = arg;
1958 293 : gcc_assert (!converts.is_empty ());
1959 682 : for (unsigned int i = 0; i < converts.length () - 1; i++)
1960 : {
1961 48 : new_lhs = make_ssa_name (converts[i].first);
1962 48 : g = gimple_build_assign (new_lhs, converts[i].second, new_rhs);
1963 48 : new_rhs = new_lhs;
1964 48 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
1965 : }
1966 879 : g = gimple_build_assign (lhs,
1967 293 : converts[converts.length() - 1].second,
1968 : new_rhs);
1969 293 : gsi_replace (gsi, g, false);
1970 293 : return;
1971 : }
1972 :
1973 184 : if (modifier == NONE && (code == FIX_TRUNC_EXPR || code == FLOAT_EXPR))
1974 : {
1975 : /* Can't use get_compute_type here, as supportable_convert_operation
1976 : doesn't necessarily use an optab and needs two arguments. */
1977 71 : tree vec_compute_type
1978 71 : = type_for_widest_vector_mode (arg_type, mov_optab);
1979 71 : if (vec_compute_type
1980 71 : && VECTOR_MODE_P (TYPE_MODE (vec_compute_type)))
1981 : {
1982 71 : unsigned HOST_WIDE_INT nelts
1983 71 : = constant_lower_bound (TYPE_VECTOR_SUBPARTS (vec_compute_type));
1984 144 : while (nelts > 1)
1985 : {
1986 73 : tree ret1_type = build_vector_type (TREE_TYPE (ret_type), nelts);
1987 73 : tree arg1_type = build_vector_type (TREE_TYPE (arg_type), nelts);
1988 73 : if (supportable_convert_operation (code, ret1_type, arg1_type,
1989 : &code1))
1990 : {
1991 0 : new_rhs = expand_vector_piecewise (gsi, do_vec_conversion,
1992 : ret_type, arg1_type, arg,
1993 : NULL_TREE, code1, false);
1994 0 : g = gimple_build_assign (lhs, new_rhs);
1995 0 : gsi_replace (gsi, g, false);
1996 0 : return;
1997 : }
1998 73 : nelts = nelts / 2;
1999 : }
2000 : }
2001 : }
2002 113 : else if (modifier == NARROW)
2003 : {
2004 72 : switch (code)
2005 : {
2006 42 : CASE_CONVERT:
2007 42 : code1 = VEC_PACK_TRUNC_EXPR;
2008 42 : optab1 = optab_for_tree_code (code1, arg_type, optab_default);
2009 42 : break;
2010 26 : case FIX_TRUNC_EXPR:
2011 26 : code1 = VEC_PACK_FIX_TRUNC_EXPR;
2012 : /* The signedness is determined from output operand. */
2013 26 : optab1 = optab_for_tree_code (code1, ret_type, optab_default);
2014 26 : break;
2015 4 : case FLOAT_EXPR:
2016 4 : code1 = VEC_PACK_FLOAT_EXPR;
2017 4 : optab1 = optab_for_tree_code (code1, arg_type, optab_default);
2018 4 : break;
2019 0 : default:
2020 0 : gcc_unreachable ();
2021 : }
2022 :
2023 72 : if (optab1)
2024 72 : compute_type = get_compute_type (optab1, arg_type);
2025 72 : enum insn_code icode1;
2026 72 : if (VECTOR_TYPE_P (compute_type)
2027 67 : && ((icode1 = optab_handler (optab1, TYPE_MODE (compute_type)))
2028 : != CODE_FOR_nothing)
2029 139 : && VECTOR_MODE_P (insn_data[icode1].operand[0].mode))
2030 : {
2031 67 : tree cretd_type
2032 67 : = build_vector_type (TREE_TYPE (ret_type),
2033 67 : TYPE_VECTOR_SUBPARTS (compute_type) * 2);
2034 67 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (cretd_type))
2035 : {
2036 59 : if (compute_type == arg_type)
2037 : {
2038 23 : new_rhs = gimplify_build2 (gsi, code1, cretd_type,
2039 : arg, build_zero_cst (arg_type));
2040 46 : new_rhs = tree_vec_extract (gsi, ret_type, new_rhs,
2041 23 : TYPE_SIZE (ret_type),
2042 : bitsize_int (0));
2043 23 : g = gimple_build_assign (lhs, new_rhs);
2044 23 : gsi_replace (gsi, g, false);
2045 23 : return;
2046 : }
2047 36 : tree dcompute_type
2048 36 : = build_vector_type (TREE_TYPE (compute_type),
2049 36 : TYPE_VECTOR_SUBPARTS (compute_type) * 2);
2050 36 : if (TYPE_MAIN_VARIANT (dcompute_type)
2051 36 : == TYPE_MAIN_VARIANT (arg_type))
2052 27 : new_rhs = do_vec_narrow_conversion (gsi, dcompute_type, arg,
2053 : NULL_TREE, bitsize_int (0),
2054 : NULL_TREE, code1,
2055 : ret_type);
2056 : else
2057 9 : new_rhs = expand_vector_piecewise (gsi,
2058 : do_vec_narrow_conversion,
2059 : arg_type, dcompute_type,
2060 : arg, NULL_TREE, code1,
2061 : false, ret_type);
2062 36 : g = gimple_build_assign (lhs, new_rhs);
2063 36 : gsi_replace (gsi, g, false);
2064 36 : return;
2065 : }
2066 : }
2067 : }
2068 41 : else if (modifier == WIDEN)
2069 : {
2070 38 : enum tree_code code2 = ERROR_MARK;
2071 38 : optab optab2 = unknown_optab;
2072 38 : switch (code)
2073 : {
2074 8 : CASE_CONVERT:
2075 8 : code1 = VEC_UNPACK_LO_EXPR;
2076 8 : code2 = VEC_UNPACK_HI_EXPR;
2077 8 : break;
2078 0 : case FIX_TRUNC_EXPR:
2079 0 : code1 = VEC_UNPACK_FIX_TRUNC_LO_EXPR;
2080 0 : code2 = VEC_UNPACK_FIX_TRUNC_HI_EXPR;
2081 0 : break;
2082 30 : case FLOAT_EXPR:
2083 30 : code1 = VEC_UNPACK_FLOAT_LO_EXPR;
2084 30 : code2 = VEC_UNPACK_FLOAT_HI_EXPR;
2085 30 : break;
2086 0 : default:
2087 0 : gcc_unreachable ();
2088 : }
2089 38 : if (BYTES_BIG_ENDIAN)
2090 : std::swap (code1, code2);
2091 :
2092 38 : if (code == FIX_TRUNC_EXPR)
2093 : {
2094 : /* The signedness is determined from output operand. */
2095 0 : optab1 = optab_for_tree_code (code1, ret_type, optab_default);
2096 0 : optab2 = optab_for_tree_code (code2, ret_type, optab_default);
2097 : }
2098 : else
2099 : {
2100 38 : optab1 = optab_for_tree_code (code1, arg_type, optab_default);
2101 38 : optab2 = optab_for_tree_code (code2, arg_type, optab_default);
2102 : }
2103 :
2104 38 : if (optab1 && optab2)
2105 38 : compute_type = get_compute_type (optab1, arg_type);
2106 :
2107 38 : enum insn_code icode1, icode2;
2108 38 : if (VECTOR_TYPE_P (compute_type)
2109 34 : && ((icode1 = optab_handler (optab1, TYPE_MODE (compute_type)))
2110 : != CODE_FOR_nothing)
2111 34 : && ((icode2 = optab_handler (optab2, TYPE_MODE (compute_type)))
2112 : != CODE_FOR_nothing)
2113 34 : && VECTOR_MODE_P (insn_data[icode1].operand[0].mode)
2114 38 : && (insn_data[icode1].operand[0].mode
2115 34 : == insn_data[icode2].operand[0].mode))
2116 : {
2117 34 : poly_uint64 nunits
2118 34 : = exact_div (TYPE_VECTOR_SUBPARTS (compute_type), 2);
2119 34 : tree cretd_type = build_vector_type (TREE_TYPE (ret_type), nunits);
2120 34 : if (insn_data[icode1].operand[0].mode == TYPE_MODE (cretd_type))
2121 : {
2122 34 : vec<constructor_elt, va_gc> *v;
2123 34 : tree part_width = TYPE_SIZE (compute_type);
2124 34 : tree index = bitsize_int (0);
2125 34 : int nunits = nunits_for_known_piecewise_op (arg_type);
2126 34 : int delta = tree_to_uhwi (part_width) / arg_elt_bits;
2127 34 : int i;
2128 34 : location_t loc = gimple_location (gsi_stmt (*gsi));
2129 :
2130 34 : if (compute_type != arg_type)
2131 : {
2132 0 : if (!warning_suppressed_p (gsi_stmt (*gsi),
2133 : OPT_Wvector_operation_performance))
2134 0 : warning_at (loc, OPT_Wvector_operation_performance,
2135 : "vector operation will be expanded piecewise");
2136 : }
2137 : else
2138 : {
2139 : nunits = 1;
2140 : delta = 1;
2141 : }
2142 :
2143 34 : vec_alloc (v, (nunits + delta - 1) / delta * 2);
2144 34 : bool constant_p = true;
2145 68 : for (i = 0; i < nunits;
2146 34 : i += delta, index = int_const_binop (PLUS_EXPR, index,
2147 : part_width))
2148 : {
2149 34 : tree a = arg;
2150 34 : if (compute_type != arg_type)
2151 0 : a = tree_vec_extract (gsi, compute_type, a, part_width,
2152 : index);
2153 34 : tree result = gimplify_build1 (gsi, code1, cretd_type, a);
2154 34 : constructor_elt ce = { NULL_TREE, result };
2155 34 : if (!CONSTANT_CLASS_P (ce.value))
2156 34 : constant_p = false;
2157 34 : v->quick_push (ce);
2158 34 : ce.value = gimplify_build1 (gsi, code2, cretd_type, a);
2159 34 : if (!CONSTANT_CLASS_P (ce.value))
2160 34 : constant_p = false;
2161 34 : v->quick_push (ce);
2162 : }
2163 :
2164 34 : if (constant_p)
2165 0 : new_rhs = build_vector_from_ctor (ret_type, v);
2166 : else
2167 34 : new_rhs = build_constructor (ret_type, v);
2168 34 : g = gimple_build_assign (lhs, new_rhs);
2169 34 : gsi_replace (gsi, g, false);
2170 34 : return;
2171 : }
2172 : }
2173 : }
2174 :
2175 91 : new_rhs = expand_vector_piecewise (gsi, do_vec_conversion, arg_type,
2176 91 : TREE_TYPE (arg_type), arg,
2177 : NULL_TREE, code, false, ret_type);
2178 91 : g = gimple_build_assign (lhs, new_rhs);
2179 91 : gsi_replace (gsi, g, false);
2180 477 : }
2181 :
2182 : /* Process one statement. If we identify a vector operation, expand it. */
2183 :
2184 : static void
2185 96885881 : expand_vector_operations_1 (gimple_stmt_iterator *gsi)
2186 : {
2187 96885881 : tree lhs, rhs1, rhs2 = NULL, type, compute_type = NULL_TREE;
2188 96885881 : enum tree_code code;
2189 96885881 : optab op = unknown_optab;
2190 96885881 : enum gimple_rhs_class rhs_class;
2191 96885881 : tree new_rhs;
2192 :
2193 : /* Only consider code == GIMPLE_ASSIGN. */
2194 96885881 : gassign *stmt = dyn_cast <gassign *> (gsi_stmt (*gsi));
2195 96885881 : if (!stmt)
2196 : {
2197 63840678 : if (gimple_call_internal_p (gsi_stmt (*gsi), IFN_VEC_CONVERT))
2198 478 : expand_vector_conversion (gsi);
2199 63840678 : return;
2200 : }
2201 :
2202 33045203 : code = gimple_assign_rhs_code (stmt);
2203 33045203 : rhs_class = get_gimple_rhs_class (code);
2204 33045203 : lhs = gimple_assign_lhs (stmt);
2205 :
2206 33045203 : if (code == VEC_PERM_EXPR)
2207 : {
2208 90253 : lower_vec_perm (gsi);
2209 90253 : return;
2210 : }
2211 :
2212 32954950 : if (code == VEC_COND_EXPR)
2213 : {
2214 20410 : expand_vector_condition (gsi);
2215 20410 : return;
2216 : }
2217 :
2218 32934540 : if (code == COND_EXPR
2219 13142 : && TREE_CODE (TREE_TYPE (gimple_assign_lhs (stmt))) == VECTOR_TYPE
2220 32934547 : && TYPE_MODE (TREE_TYPE (gimple_assign_lhs (stmt))) == BLKmode)
2221 : {
2222 7 : expand_vector_scalar_condition (gsi);
2223 7 : return;
2224 : }
2225 :
2226 32934533 : if (code == CONSTRUCTOR
2227 1749539 : && TREE_CODE (lhs) == SSA_NAME
2228 145101 : && VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (lhs)))
2229 144531 : && !gimple_clobber_p (stmt)
2230 33079064 : && optimize)
2231 : {
2232 142569 : optimize_vector_constructor (gsi);
2233 142569 : return;
2234 : }
2235 :
2236 32791964 : if (rhs_class != GIMPLE_UNARY_RHS && rhs_class != GIMPLE_BINARY_RHS)
2237 : return;
2238 :
2239 12031444 : rhs1 = gimple_assign_rhs1 (stmt);
2240 12031444 : if (rhs_class == GIMPLE_BINARY_RHS)
2241 8166339 : rhs2 = gimple_assign_rhs2 (stmt);
2242 :
2243 12031444 : type = TREE_TYPE (lhs);
2244 12031444 : if (!VECTOR_TYPE_P (type)
2245 12031444 : || !VECTOR_TYPE_P (TREE_TYPE (rhs1)))
2246 : return;
2247 :
2248 : /* A scalar operation pretending to be a vector one. */
2249 255054 : if (VECTOR_BOOLEAN_TYPE_P (type)
2250 34216 : && !VECTOR_MODE_P (TYPE_MODE (type))
2251 11756 : && TYPE_MODE (type) != BLKmode
2252 11452 : && (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt)) != tcc_comparison
2253 6462 : || (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (rhs1))
2254 0 : && !VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (rhs1)))
2255 0 : && TYPE_MODE (TREE_TYPE (rhs1)) != BLKmode)))
2256 4990 : return;
2257 :
2258 : /* If the vector operation is operating on all same vector elements
2259 : implement it with a scalar operation and a splat if the target
2260 : supports the scalar operation. */
2261 250064 : tree srhs1, srhs2 = NULL_TREE;
2262 250064 : if ((srhs1 = ssa_uniform_vector_p (rhs1)) != NULL_TREE
2263 15925 : && (rhs2 == NULL_TREE
2264 15646 : || (! VECTOR_TYPE_P (TREE_TYPE (rhs2))
2265 : && (srhs2 = rhs2))
2266 15588 : || (srhs2 = ssa_uniform_vector_p (rhs2)) != NULL_TREE)
2267 : /* As we query direct optabs restrict to non-convert operations. */
2268 251413 : && TYPE_MODE (TREE_TYPE (type)) == TYPE_MODE (TREE_TYPE (srhs1)))
2269 : {
2270 1202 : op = optab_for_tree_code (code, TREE_TYPE (type), optab_scalar);
2271 1202 : if (op >= FIRST_NORM_OPTAB && op <= LAST_NORM_OPTAB
2272 1202 : && can_implement_p (op, TYPE_MODE (TREE_TYPE (type))))
2273 : {
2274 894 : tree stype = TREE_TYPE (TREE_TYPE (lhs));
2275 894 : tree slhs = (rhs2 != NULL_TREE)
2276 894 : ? gimplify_build2 (gsi, code, stype, srhs1, srhs2)
2277 97 : : gimplify_build1 (gsi, code, stype, srhs1);
2278 894 : gimple_assign_set_rhs_from_tree (gsi,
2279 : build_vector_from_val (type, slhs));
2280 894 : update_stmt (stmt);
2281 894 : return;
2282 : }
2283 : }
2284 :
2285 : /* Plain moves do not need lowering. */
2286 249170 : if (code == SSA_NAME
2287 249170 : || code == VIEW_CONVERT_EXPR
2288 : || code == PAREN_EXPR)
2289 : return;
2290 :
2291 : if (CONVERT_EXPR_CODE_P (code)
2292 : || code == FLOAT_EXPR
2293 : || code == FIX_TRUNC_EXPR)
2294 : return;
2295 :
2296 : /* The signedness is determined from input argument. */
2297 : if (code == VEC_UNPACK_FLOAT_HI_EXPR
2298 : || code == VEC_UNPACK_FLOAT_LO_EXPR
2299 : || code == VEC_PACK_FLOAT_EXPR)
2300 : {
2301 : /* We do not know how to scalarize those. */
2302 : return;
2303 : }
2304 :
2305 : /* For widening/narrowing vector operations, the relevant type is of the
2306 : arguments, not the widened result. VEC_UNPACK_FLOAT_*_EXPR is
2307 : calculated in the same way above. */
2308 : if (code == WIDEN_SUM_EXPR
2309 : || code == VEC_WIDEN_MULT_HI_EXPR
2310 : || code == VEC_WIDEN_MULT_LO_EXPR
2311 : || code == VEC_WIDEN_MULT_EVEN_EXPR
2312 : || code == VEC_WIDEN_MULT_ODD_EXPR
2313 : || code == VEC_UNPACK_HI_EXPR
2314 : || code == VEC_UNPACK_LO_EXPR
2315 : || code == VEC_UNPACK_FIX_TRUNC_HI_EXPR
2316 : || code == VEC_UNPACK_FIX_TRUNC_LO_EXPR
2317 : || code == VEC_PACK_TRUNC_EXPR
2318 : || code == VEC_PACK_SAT_EXPR
2319 : || code == VEC_PACK_FIX_TRUNC_EXPR
2320 : || code == VEC_WIDEN_LSHIFT_HI_EXPR
2321 : || code == VEC_WIDEN_LSHIFT_LO_EXPR)
2322 : {
2323 : /* We do not know how to scalarize those. */
2324 : return;
2325 : }
2326 :
2327 : /* Choose between vector shift/rotate by vector and vector shift/rotate by
2328 : scalar */
2329 : if (code == LSHIFT_EXPR
2330 : || code == RSHIFT_EXPR
2331 : || code == LROTATE_EXPR
2332 : || code == RROTATE_EXPR)
2333 : {
2334 9168 : optab opv;
2335 :
2336 : /* Check whether we have vector <op> {x,x,x,x} where x
2337 : could be a scalar variable or a constant. Transform
2338 : vector <op> {x,x,x,x} ==> vector <op> scalar. */
2339 9168 : if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2)))
2340 : {
2341 1555 : tree first;
2342 :
2343 1555 : if ((first = ssa_uniform_vector_p (rhs2)) != NULL_TREE)
2344 : {
2345 7 : gimple_assign_set_rhs2 (stmt, first);
2346 7 : update_stmt (stmt);
2347 7 : rhs2 = first;
2348 : }
2349 : }
2350 :
2351 9168 : opv = optab_for_tree_code (code, type, optab_vector);
2352 9168 : if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2)))
2353 : op = opv;
2354 : else
2355 : {
2356 7620 : op = optab_for_tree_code (code, type, optab_scalar);
2357 :
2358 7620 : compute_type = get_compute_type (op, type);
2359 7620 : if (compute_type == type)
2360 : return;
2361 : /* The rtl expander will expand vector/scalar as vector/vector
2362 : if necessary. Pick one with wider vector type. */
2363 172 : tree compute_vtype = get_compute_type (opv, type);
2364 172 : if (subparts_gt (compute_vtype, compute_type))
2365 : {
2366 0 : compute_type = compute_vtype;
2367 0 : op = opv;
2368 : }
2369 : }
2370 :
2371 1720 : if (code == LROTATE_EXPR || code == RROTATE_EXPR)
2372 : {
2373 39 : if (compute_type == NULL_TREE)
2374 6 : compute_type = get_compute_type (op, type);
2375 39 : if (compute_type == type)
2376 : return;
2377 : /* Before splitting vector rotates into scalar rotates,
2378 : see if we can't use vector shifts and BIT_IOR_EXPR
2379 : instead. For vector by vector rotates we'd also
2380 : need to check BIT_AND_EXPR and NEGATE_EXPR, punt there
2381 : for now, fold doesn't seem to create such rotates anyway. */
2382 33 : if (compute_type == TREE_TYPE (type)
2383 33 : && !VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2)))
2384 : {
2385 30 : optab oplv = vashl_optab, opl = ashl_optab;
2386 30 : optab oprv = vlshr_optab, opr = lshr_optab, opo = ior_optab;
2387 30 : tree compute_lvtype = get_compute_type (oplv, type);
2388 30 : tree compute_rvtype = get_compute_type (oprv, type);
2389 30 : tree compute_otype = get_compute_type (opo, type);
2390 30 : tree compute_ltype = get_compute_type (opl, type);
2391 30 : tree compute_rtype = get_compute_type (opr, type);
2392 : /* The rtl expander will expand vector/scalar as vector/vector
2393 : if necessary. Pick one with wider vector type. */
2394 30 : if (subparts_gt (compute_lvtype, compute_ltype))
2395 : {
2396 0 : compute_ltype = compute_lvtype;
2397 0 : opl = oplv;
2398 : }
2399 30 : if (subparts_gt (compute_rvtype, compute_rtype))
2400 : {
2401 0 : compute_rtype = compute_rvtype;
2402 0 : opr = oprv;
2403 : }
2404 : /* Pick the narrowest type from LSHIFT_EXPR, RSHIFT_EXPR and
2405 : BIT_IOR_EXPR. */
2406 30 : compute_type = compute_ltype;
2407 30 : if (subparts_gt (compute_type, compute_rtype))
2408 0 : compute_type = compute_rtype;
2409 30 : if (subparts_gt (compute_type, compute_otype))
2410 0 : compute_type = compute_otype;
2411 : /* Verify all 3 operations can be performed in that type. */
2412 30 : if (compute_type != TREE_TYPE (type))
2413 : {
2414 30 : if (!can_implement_p (opl, TYPE_MODE (compute_type))
2415 30 : || !can_implement_p (opr, TYPE_MODE (compute_type))
2416 60 : || !can_implement_p (opo, TYPE_MODE (compute_type)))
2417 0 : compute_type = TREE_TYPE (type);
2418 : }
2419 : }
2420 : }
2421 : }
2422 : else
2423 206500 : op = optab_for_tree_code (code, type, optab_default);
2424 :
2425 : /* Optabs will try converting a negation into a subtraction, so
2426 : look for it as well. TODO: negation of floating-point vectors
2427 : might be turned into an exclusive OR toggling the sign bit. */
2428 208214 : if (op == unknown_optab
2429 208214 : && code == NEGATE_EXPR
2430 208214 : && INTEGRAL_TYPE_P (TREE_TYPE (type)))
2431 0 : op = optab_for_tree_code (MINUS_EXPR, type, optab_default);
2432 :
2433 208214 : if (compute_type == NULL_TREE)
2434 208042 : compute_type = get_compute_type (op, type);
2435 208214 : if (compute_type == type)
2436 : return;
2437 :
2438 25923 : new_rhs = expand_vector_operation (gsi, type, compute_type, stmt, code);
2439 :
2440 : /* Leave expression untouched for later expansion. */
2441 25923 : if (new_rhs == NULL_TREE)
2442 : return;
2443 :
2444 3082 : if (!useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (new_rhs)))
2445 92 : new_rhs = gimplify_build1 (gsi, VIEW_CONVERT_EXPR, TREE_TYPE (lhs),
2446 : new_rhs);
2447 :
2448 : /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
2449 : way to do it is change expand_vector_operation and its callees to
2450 : return a tree_code, RHS1 and RHS2 instead of a tree. */
2451 3082 : gimple_assign_set_rhs_from_tree (gsi, new_rhs);
2452 3082 : update_stmt (gsi_stmt (*gsi));
2453 : }
2454 :
2455 : /* Use this to lower vector operations introduced by the vectorizer,
2456 : if it may need the bit-twiddling tricks implemented in this file. */
2457 :
2458 : static unsigned int
2459 1472253 : expand_vector_operations (void)
2460 : {
2461 1472253 : gimple_stmt_iterator gsi;
2462 1472253 : basic_block bb;
2463 1472253 : bool cfg_changed = false;
2464 :
2465 15725563 : FOR_EACH_BB_FN (bb, cfun)
2466 : {
2467 125392501 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2468 : {
2469 96885881 : expand_vector_operations_1 (&gsi);
2470 : /* ??? If we do not cleanup EH then we will ICE in
2471 : verification. But in reality we have created wrong-code
2472 : as we did not properly transition EH info and edges to
2473 : the piecewise computations. */
2474 96885881 : if (maybe_clean_eh_stmt (gsi_stmt (gsi))
2475 96885881 : && gimple_purge_dead_eh_edges (bb))
2476 : cfg_changed = true;
2477 : /* If a .LOOP_DIST_ALIAS call prevailed loops got elided
2478 : before vectorization got a chance to get at them. Simply
2479 : fold as if loop distribution wasn't performed. */
2480 96885881 : if (gimple_call_internal_p (gsi_stmt (gsi), IFN_LOOP_DIST_ALIAS))
2481 : {
2482 0 : fold_loop_internal_call (gsi_stmt (gsi), boolean_false_node);
2483 0 : cfg_changed = true;
2484 : }
2485 : }
2486 : }
2487 :
2488 1472253 : return cfg_changed ? TODO_cleanup_cfg : 0;
2489 : }
2490 :
2491 : namespace {
2492 :
2493 : const pass_data pass_data_lower_vector =
2494 : {
2495 : GIMPLE_PASS, /* type */
2496 : "veclower", /* name */
2497 : OPTGROUP_VEC, /* optinfo_flags */
2498 : TV_NONE, /* tv_id */
2499 : PROP_cfg, /* properties_required */
2500 : PROP_gimple_lvec, /* properties_provided */
2501 : 0, /* properties_destroyed */
2502 : 0, /* todo_flags_start */
2503 : TODO_update_ssa, /* todo_flags_finish */
2504 : };
2505 :
2506 : class pass_lower_vector : public gimple_opt_pass
2507 : {
2508 : public:
2509 285722 : pass_lower_vector (gcc::context *ctxt)
2510 571444 : : gimple_opt_pass (pass_data_lower_vector, ctxt)
2511 : {}
2512 :
2513 : /* opt_pass methods: */
2514 1472150 : bool gate (function *fun) final override
2515 : {
2516 1472150 : return !(fun->curr_properties & PROP_gimple_lvec);
2517 : }
2518 :
2519 428124 : unsigned int execute (function *) final override
2520 : {
2521 428124 : return expand_vector_operations ();
2522 : }
2523 :
2524 : }; // class pass_lower_vector
2525 :
2526 : } // anon namespace
2527 :
2528 : gimple_opt_pass *
2529 285722 : make_pass_lower_vector (gcc::context *ctxt)
2530 : {
2531 285722 : return new pass_lower_vector (ctxt);
2532 : }
2533 :
2534 : namespace {
2535 :
2536 : const pass_data pass_data_lower_vector_ssa =
2537 : {
2538 : GIMPLE_PASS, /* type */
2539 : "veclower2", /* name */
2540 : OPTGROUP_VEC, /* optinfo_flags */
2541 : TV_NONE, /* tv_id */
2542 : PROP_cfg, /* properties_required */
2543 : PROP_gimple_lvec, /* properties_provided */
2544 : 0, /* properties_destroyed */
2545 : 0, /* todo_flags_start */
2546 : ( TODO_update_ssa
2547 : | TODO_cleanup_cfg ), /* todo_flags_finish */
2548 : };
2549 :
2550 : class pass_lower_vector_ssa : public gimple_opt_pass
2551 : {
2552 : public:
2553 571444 : pass_lower_vector_ssa (gcc::context *ctxt)
2554 1142888 : : gimple_opt_pass (pass_data_lower_vector_ssa, ctxt)
2555 : {}
2556 :
2557 : /* opt_pass methods: */
2558 285722 : opt_pass * clone () final override
2559 : {
2560 285722 : return new pass_lower_vector_ssa (m_ctxt);
2561 : }
2562 1044129 : unsigned int execute (function *) final override
2563 : {
2564 1044129 : return expand_vector_operations ();
2565 : }
2566 :
2567 : }; // class pass_lower_vector_ssa
2568 :
2569 : } // anon namespace
2570 :
2571 : gimple_opt_pass *
2572 285722 : make_pass_lower_vector_ssa (gcc::context *ctxt)
2573 : {
2574 285722 : return new pass_lower_vector_ssa (ctxt);
2575 : }
2576 :
2577 : #include "gt-tree-vect-generic.h"
|
