Line data Source code
1 : /* Analysis Utilities for Loop Vectorization.
2 : Copyright (C) 2006-2026 Free Software Foundation, Inc.
3 : Contributed by Dorit Nuzman <dorit@il.ibm.com>
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "gimple-iterator.h"
29 : #include "gimple-fold.h"
30 : #include "ssa.h"
31 : #include "expmed.h"
32 : #include "optabs-tree.h"
33 : #include "insn-config.h"
34 : #include "recog.h" /* FIXME: for insn_data */
35 : #include "fold-const.h"
36 : #include "stor-layout.h"
37 : #include "tree-eh.h"
38 : #include "gimplify.h"
39 : #include "gimplify-me.h"
40 : #include "cfgloop.h"
41 : #include "tree-vectorizer.h"
42 : #include "dumpfile.h"
43 : #include "builtins.h"
44 : #include "internal-fn.h"
45 : #include "case-cfn-macros.h"
46 : #include "fold-const-call.h"
47 : #include "attribs.h"
48 : #include "cgraph.h"
49 : #include "omp-simd-clone.h"
50 : #include "predict.h"
51 : #include "tree-vector-builder.h"
52 : #include "tree-ssa-loop-ivopts.h"
53 : #include "vec-perm-indices.h"
54 : #include "gimple-range.h"
55 : #include "alias.h"
56 :
57 :
58 : /* TODO: Note the vectorizer still builds COND_EXPRs with GENERIC compares
59 : in the first operand. Disentangling this is future work, the
60 : IL is properly transferred to VEC_COND_EXPRs with separate compares. */
61 :
62 :
63 : /* Return true if we have a useful VR_RANGE range for VAR, storing it
64 : in *MIN_VALUE and *MAX_VALUE if so. Note the range in the dump files. */
65 :
66 : bool
67 12984807 : vect_get_range_info (tree var, wide_int *min_value, wide_int *max_value)
68 : {
69 12984807 : int_range_max vr;
70 12984807 : tree vr_min, vr_max;
71 25969614 : get_range_query (cfun)->range_of_expr (vr, var);
72 12984807 : if (vr.undefined_p ())
73 53 : vr.set_varying (TREE_TYPE (var));
74 12984807 : value_range_kind vr_type = get_legacy_range (vr, vr_min, vr_max);
75 12984807 : *min_value = wi::to_wide (vr_min);
76 12984807 : *max_value = wi::to_wide (vr_max);
77 12984807 : wide_int nonzero = get_nonzero_bits (var);
78 12984807 : signop sgn = TYPE_SIGN (TREE_TYPE (var));
79 12984807 : if (intersect_range_with_nonzero_bits (vr_type, min_value, max_value,
80 : nonzero, sgn) == VR_RANGE)
81 : {
82 6271496 : if (dump_enabled_p ())
83 : {
84 92867 : dump_generic_expr_loc (MSG_NOTE, vect_location, TDF_SLIM, var);
85 92867 : dump_printf (MSG_NOTE, " has range [");
86 92867 : dump_hex (MSG_NOTE, *min_value);
87 92867 : dump_printf (MSG_NOTE, ", ");
88 92867 : dump_hex (MSG_NOTE, *max_value);
89 92867 : dump_printf (MSG_NOTE, "]\n");
90 : }
91 : return true;
92 : }
93 : else
94 : {
95 6713311 : if (dump_enabled_p ())
96 : {
97 70696 : dump_generic_expr_loc (MSG_NOTE, vect_location, TDF_SLIM, var);
98 70696 : dump_printf (MSG_NOTE, " has no range info\n");
99 : }
100 : return false;
101 : }
102 12984807 : }
103 :
104 : /* Report that we've found an instance of pattern PATTERN in
105 : statement STMT. */
106 :
107 : static void
108 1385313 : vect_pattern_detected (const char *name, gimple *stmt)
109 : {
110 1385313 : if (dump_enabled_p ())
111 27319 : dump_printf_loc (MSG_NOTE, vect_location, "%s: detected: %G", name, stmt);
112 1385313 : }
113 :
114 : /* Associate pattern statement PATTERN_STMT with ORIG_STMT_INFO and
115 : return the pattern statement's stmt_vec_info. Set its vector type to
116 : VECTYPE if it doesn't have one already. */
117 :
118 : static stmt_vec_info
119 2597006 : vect_init_pattern_stmt (vec_info *vinfo, gimple *pattern_stmt,
120 : stmt_vec_info orig_stmt_info, tree vectype)
121 : {
122 2597006 : stmt_vec_info pattern_stmt_info = vinfo->lookup_stmt (pattern_stmt);
123 2597006 : if (pattern_stmt_info == NULL)
124 1386739 : pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
125 2597006 : gimple_set_bb (pattern_stmt, gimple_bb (orig_stmt_info->stmt));
126 :
127 2597006 : pattern_stmt_info->pattern_stmt_p = true;
128 2597006 : STMT_VINFO_RELATED_STMT (pattern_stmt_info) = orig_stmt_info;
129 2597006 : STMT_VINFO_DEF_TYPE (pattern_stmt_info)
130 2597006 : = STMT_VINFO_DEF_TYPE (orig_stmt_info);
131 2597006 : if (!STMT_VINFO_VECTYPE (pattern_stmt_info))
132 : {
133 2323797 : gcc_assert (!vectype
134 : || is_a <gcond *> (pattern_stmt)
135 : || (VECTOR_BOOLEAN_TYPE_P (vectype)
136 : == vect_use_mask_type_p (orig_stmt_info)));
137 1397325 : STMT_VINFO_VECTYPE (pattern_stmt_info) = vectype;
138 1397325 : pattern_stmt_info->mask_precision = orig_stmt_info->mask_precision;
139 1397325 : if (dump_enabled_p ())
140 : {
141 27046 : if (pattern_stmt_info->mask_precision == ~0U)
142 63 : dump_printf_loc (MSG_NOTE, vect_location,
143 : "using normal nonmask vectors for %G",
144 : pattern_stmt_info->stmt);
145 : else
146 26983 : dump_printf_loc (MSG_NOTE, vect_location,
147 : "using boolean precision %d for %G",
148 : pattern_stmt_info->mask_precision,
149 : pattern_stmt_info->stmt);
150 : }
151 : }
152 2597006 : return pattern_stmt_info;
153 : }
154 :
155 : /* Set the pattern statement of ORIG_STMT_INFO to PATTERN_STMT.
156 : Also set the vector type of PATTERN_STMT to VECTYPE, if it doesn't
157 : have one already. */
158 :
159 : static void
160 1111180 : vect_set_pattern_stmt (vec_info *vinfo, gimple *pattern_stmt,
161 : stmt_vec_info orig_stmt_info, tree vectype)
162 : {
163 1111180 : STMT_VINFO_IN_PATTERN_P (orig_stmt_info) = true;
164 1111180 : STMT_VINFO_RELATED_STMT (orig_stmt_info)
165 0 : = vect_init_pattern_stmt (vinfo, pattern_stmt, orig_stmt_info, vectype);
166 0 : }
167 :
168 : /* Add NEW_STMT to STMT_INFO's pattern definition statements. If VECTYPE
169 : is nonnull, record that NEW_STMT's vector type is VECTYPE, which might
170 : be different from the vector type of the final pattern statement.
171 : If VECTYPE is a mask type, SCALAR_TYPE_FOR_MASK is the scalar type
172 : from which it was derived. */
173 :
174 : static inline void
175 1446478 : append_pattern_def_seq (vec_info *vinfo,
176 : stmt_vec_info stmt_info, gimple *new_stmt,
177 : tree vectype = NULL_TREE,
178 : tree scalar_type_for_mask = NULL_TREE)
179 : {
180 2205495 : gcc_assert (!scalar_type_for_mask
181 : == (!vectype || !VECTOR_BOOLEAN_TYPE_P (vectype)));
182 1446478 : if (vectype)
183 : {
184 1199899 : stmt_vec_info new_stmt_info = vinfo->add_stmt (new_stmt);
185 1199899 : STMT_VINFO_VECTYPE (new_stmt_info) = vectype;
186 1199899 : if (scalar_type_for_mask)
187 : {
188 687461 : new_stmt_info->mask_precision
189 687461 : = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (scalar_type_for_mask));
190 687461 : if (dump_enabled_p ())
191 : {
192 6543 : if (new_stmt_info->mask_precision == ~0U)
193 : dump_printf_loc (MSG_NOTE, vect_location,
194 : "using normal nonmask vectors for %G",
195 : new_stmt_info->stmt);
196 : else
197 6543 : dump_printf_loc (MSG_NOTE, vect_location,
198 : "using boolean precision %d for %G",
199 : new_stmt_info->mask_precision,
200 : new_stmt_info->stmt);
201 : }
202 : }
203 : }
204 1446478 : gimple_seq_add_stmt_without_update (&STMT_VINFO_PATTERN_DEF_SEQ (stmt_info),
205 : new_stmt);
206 1446478 : }
207 :
208 :
209 : /* Add NEW_STMT to VINFO's invariant pattern definition statements. These
210 : statements are not vectorized but are materialized as scalar in the loop
211 : preheader. */
212 :
213 : static inline void
214 1350 : append_inv_pattern_def_seq (vec_info *vinfo, gimple *new_stmt)
215 : {
216 1350 : gimple_seq_add_stmt_without_update (&vinfo->inv_pattern_def_seq, new_stmt);
217 : }
218 :
219 : /* The caller wants to perform new operations on vect_external variable
220 : VAR, so that the result of the operations would also be vect_external.
221 : Return the edge on which the operations can be performed, if one exists.
222 : Return null if the operations should instead be treated as part of
223 : the pattern that needs them. */
224 :
225 : static edge
226 9671 : vect_get_external_def_edge (vec_info *vinfo, tree var)
227 : {
228 9671 : edge e = NULL;
229 9671 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
230 : {
231 978 : e = loop_preheader_edge (loop_vinfo->loop);
232 978 : if (!SSA_NAME_IS_DEFAULT_DEF (var))
233 : {
234 774 : basic_block bb = gimple_bb (SSA_NAME_DEF_STMT (var));
235 774 : if (bb == NULL
236 774 : || !dominated_by_p (CDI_DOMINATORS, e->dest, bb))
237 : e = NULL;
238 : }
239 : }
240 9671 : return e;
241 : }
242 :
243 : /* Return true if the target supports a vector version of CODE,
244 : where CODE is known to map to a direct optab with the given SUBTYPE.
245 : ITYPE specifies the type of (some of) the scalar inputs and OTYPE
246 : specifies the type of the scalar result.
247 :
248 : If CODE allows the inputs and outputs to have different type
249 : (such as for WIDEN_SUM_EXPR), it is the input mode rather
250 : than the output mode that determines the appropriate target pattern.
251 : Operand 0 of the target pattern then specifies the mode that the output
252 : must have.
253 :
254 : When returning true, set *VECOTYPE_OUT to the vector version of OTYPE.
255 : Also set *VECITYPE_OUT to the vector version of ITYPE if VECITYPE_OUT
256 : is nonnull. */
257 :
258 : static bool
259 938 : vect_supportable_direct_optab_p (vec_info *vinfo, tree otype, tree_code code,
260 : tree itype, tree *vecotype_out,
261 : tree *vecitype_out = NULL,
262 : enum optab_subtype subtype = optab_default)
263 : {
264 938 : tree vecitype = get_vectype_for_scalar_type (vinfo, itype);
265 938 : if (!vecitype)
266 : return false;
267 :
268 936 : tree vecotype = get_vectype_for_scalar_type (vinfo, otype);
269 936 : if (!vecotype)
270 : return false;
271 :
272 936 : optab optab = optab_for_tree_code (code, vecitype, subtype);
273 936 : if (!optab)
274 : return false;
275 :
276 936 : insn_code icode = optab_handler (optab, TYPE_MODE (vecitype));
277 936 : if (icode == CODE_FOR_nothing
278 936 : || insn_data[icode].operand[0].mode != TYPE_MODE (vecotype))
279 : return false;
280 :
281 491 : *vecotype_out = vecotype;
282 491 : if (vecitype_out)
283 460 : *vecitype_out = vecitype;
284 : return true;
285 : }
286 :
287 : /* Return true if the target supports a vector version of CODE,
288 : where CODE is known to map to a conversion optab with the given SUBTYPE.
289 : ITYPE specifies the type of (some of) the scalar inputs and OTYPE
290 : specifies the type of the scalar result.
291 :
292 : When returning true, set *VECOTYPE_OUT to the vector version of OTYPE.
293 : Also set *VECITYPE_OUT to the vector version of ITYPE if VECITYPE_OUT
294 : is nonnull. */
295 :
296 : static bool
297 5734 : vect_supportable_conv_optab_p (vec_info *vinfo, tree otype, tree_code code,
298 : tree itype, tree *vecotype_out,
299 : tree *vecitype_out = NULL,
300 : enum optab_subtype subtype = optab_default)
301 : {
302 5734 : tree vecitype = get_vectype_for_scalar_type (vinfo, itype);
303 5734 : tree vecotype = get_vectype_for_scalar_type (vinfo, otype);
304 5734 : if (!vecitype || !vecotype)
305 : return false;
306 :
307 5200 : if (!directly_supported_p (code, vecotype, vecitype, subtype))
308 : return false;
309 :
310 732 : *vecotype_out = vecotype;
311 732 : if (vecitype_out)
312 732 : *vecitype_out = vecitype;
313 : return true;
314 : }
315 :
316 : /* Round bit precision PRECISION up to a full element. */
317 :
318 : static unsigned int
319 3637217 : vect_element_precision (unsigned int precision)
320 : {
321 275446 : precision = 1 << ceil_log2 (precision);
322 5307286 : return MAX (precision, BITS_PER_UNIT);
323 : }
324 :
325 : /* If OP is defined by a statement that's being considered for vectorization,
326 : return information about that statement, otherwise return NULL. */
327 :
328 : static stmt_vec_info
329 382661 : vect_get_internal_def (vec_info *vinfo, tree op)
330 : {
331 382661 : stmt_vec_info def_stmt_info = vinfo->lookup_def (op);
332 382661 : if (def_stmt_info
333 368804 : && STMT_VINFO_DEF_TYPE (def_stmt_info) == vect_internal_def)
334 350636 : return vect_stmt_to_vectorize (def_stmt_info);
335 : return NULL;
336 : }
337 :
338 : /* Holds information about an input operand after some sign changes
339 : and type promotions have been peeled away. */
340 : class vect_unpromoted_value {
341 : public:
342 : vect_unpromoted_value ();
343 :
344 : void set_op (tree, vect_def_type, stmt_vec_info = NULL);
345 :
346 : /* The value obtained after peeling away zero or more casts. */
347 : tree op;
348 :
349 : /* The type of OP. */
350 : tree type;
351 :
352 : /* The definition type of OP. */
353 : vect_def_type dt;
354 :
355 : /* If OP is the result of peeling at least one cast, and if the cast
356 : of OP itself is a vectorizable statement, CASTER identifies that
357 : statement, otherwise it is null. */
358 : stmt_vec_info caster;
359 : };
360 :
361 315727880 : inline vect_unpromoted_value::vect_unpromoted_value ()
362 315727880 : : op (NULL_TREE),
363 315727880 : type (NULL_TREE),
364 315727880 : dt (vect_uninitialized_def),
365 3326222 : caster (NULL)
366 : {
367 : }
368 :
369 : /* Set the operand to OP_IN, its definition type to DT_IN, and the
370 : statement that casts it to CASTER_IN. */
371 :
372 : inline void
373 12013428 : vect_unpromoted_value::set_op (tree op_in, vect_def_type dt_in,
374 : stmt_vec_info caster_in)
375 : {
376 12013428 : op = op_in;
377 12013428 : type = TREE_TYPE (op);
378 12013428 : dt = dt_in;
379 12013428 : caster = caster_in;
380 12013428 : }
381 :
382 : /* If OP is a vectorizable SSA name, strip a sequence of integer conversions
383 : to reach some vectorizable inner operand OP', continuing as long as it
384 : is possible to convert OP' back to OP using a possible sign change
385 : followed by a possible promotion P. Return this OP', or null if OP is
386 : not a vectorizable SSA name. If there is a promotion P, describe its
387 : input in UNPROM, otherwise describe OP' in UNPROM. If SINGLE_USE_P
388 : is nonnull, set *SINGLE_USE_P to false if any of the SSA names involved
389 : have more than one user.
390 :
391 : A successful return means that it is possible to go from OP' to OP
392 : via UNPROM. The cast from OP' to UNPROM is at most a sign change,
393 : whereas the cast from UNPROM to OP might be a promotion, a sign
394 : change, or a nop.
395 :
396 : E.g. say we have:
397 :
398 : signed short *ptr = ...;
399 : signed short C = *ptr;
400 : unsigned short B = (unsigned short) C; // sign change
401 : signed int A = (signed int) B; // unsigned promotion
402 : ...possible other uses of A...
403 : unsigned int OP = (unsigned int) A; // sign change
404 :
405 : In this case it's possible to go directly from C to OP using:
406 :
407 : OP = (unsigned int) (unsigned short) C;
408 : +------------+ +--------------+
409 : promotion sign change
410 :
411 : so OP' would be C. The input to the promotion is B, so UNPROM
412 : would describe B. */
413 :
414 : static tree
415 8844197 : vect_look_through_possible_promotion (vec_info *vinfo, tree op,
416 : vect_unpromoted_value *unprom,
417 : bool *single_use_p = NULL)
418 : {
419 8844197 : tree op_type = TREE_TYPE (op);
420 8844197 : if (!INTEGRAL_TYPE_P (op_type))
421 : return NULL_TREE;
422 :
423 8792948 : tree res = NULL_TREE;
424 8792948 : unsigned int orig_precision = TYPE_PRECISION (op_type);
425 8792948 : unsigned int min_precision = orig_precision;
426 8792948 : stmt_vec_info caster = NULL;
427 10548634 : while (TREE_CODE (op) == SSA_NAME && INTEGRAL_TYPE_P (op_type))
428 : {
429 : /* See whether OP is simple enough to vectorize. */
430 10323179 : stmt_vec_info def_stmt_info;
431 10323179 : gimple *def_stmt;
432 10323179 : vect_def_type dt;
433 10323179 : if (!vect_is_simple_use (op, vinfo, &dt, &def_stmt_info, &def_stmt))
434 : break;
435 :
436 : /* If OP is the input of a demotion, skip over it to see whether
437 : OP is itself the result of a promotion. If so, the combined
438 : effect of the promotion and the demotion might fit the required
439 : pattern, otherwise neither operation fits.
440 :
441 : This copes with cases such as the result of an arithmetic
442 : operation being truncated before being stored, and where that
443 : arithmetic operation has been recognized as an over-widened one. */
444 10306031 : if (TYPE_PRECISION (op_type) <= min_precision)
445 : {
446 : /* Use OP as the UNPROM described above if we haven't yet
447 : found a promotion, or if using the new input preserves the
448 : sign of the previous promotion. */
449 10171674 : if (!res
450 1498036 : || TYPE_PRECISION (unprom->type) == orig_precision
451 39765 : || TYPE_SIGN (unprom->type) == TYPE_SIGN (op_type)
452 10207948 : || (TYPE_UNSIGNED (op_type)
453 25662 : && TYPE_PRECISION (op_type) < TYPE_PRECISION (unprom->type)))
454 : {
455 10135859 : unprom->set_op (op, dt, caster);
456 10135859 : min_precision = TYPE_PRECISION (op_type);
457 : }
458 : /* Stop if we've already seen a promotion and if this
459 : conversion does more than change the sign. */
460 35815 : else if (TYPE_PRECISION (op_type)
461 35815 : != TYPE_PRECISION (unprom->type))
462 : break;
463 :
464 : /* The sequence now extends to OP. */
465 : res = op;
466 : }
467 :
468 : /* See whether OP is defined by a cast. Record it as CASTER if
469 : the cast is potentially vectorizable. */
470 10306001 : if (!def_stmt)
471 : break;
472 10085382 : caster = def_stmt_info;
473 :
474 : /* Ignore pattern statements, since we don't link uses for them. */
475 10085382 : if (caster
476 10085382 : && single_use_p
477 2023856 : && !STMT_VINFO_RELATED_STMT (caster)
478 11953901 : && !has_single_use (res))
479 1116732 : *single_use_p = false;
480 :
481 14979232 : gassign *assign = dyn_cast <gassign *> (def_stmt);
482 6411739 : if (!assign || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def_stmt)))
483 : break;
484 :
485 : /* Continue with the input to the cast. */
486 1755686 : op = gimple_assign_rhs1 (def_stmt);
487 1755686 : op_type = TREE_TYPE (op);
488 : }
489 : return res;
490 : }
491 :
492 : /* OP is an integer operand to an operation that returns TYPE, and we
493 : want to treat the operation as a widening one. So far we can treat
494 : it as widening from *COMMON_TYPE.
495 :
496 : Return true if OP is suitable for such a widening operation,
497 : either widening from *COMMON_TYPE or from some supertype of it.
498 : Update *COMMON_TYPE to the supertype in the latter case.
499 :
500 : SHIFT_P is true if OP is a shift amount. */
501 :
502 : static bool
503 353947 : vect_joust_widened_integer (tree type, bool shift_p, tree op,
504 : tree *common_type)
505 : {
506 : /* Calculate the minimum precision required by OP, without changing
507 : the sign of either operand. */
508 353947 : unsigned int precision;
509 353947 : if (shift_p)
510 : {
511 13535 : if (!wi::leu_p (wi::to_widest (op), TYPE_PRECISION (type) / 2))
512 : return false;
513 10914 : precision = TREE_INT_CST_LOW (op);
514 : }
515 : else
516 : {
517 340412 : precision = wi::min_precision (wi::to_widest (op),
518 340412 : TYPE_SIGN (*common_type));
519 340412 : if (precision * 2 > TYPE_PRECISION (type))
520 : return false;
521 : }
522 :
523 : /* If OP requires a wider type, switch to that type. The checks
524 : above ensure that this is still narrower than the result. */
525 335010 : precision = vect_element_precision (precision);
526 335010 : if (TYPE_PRECISION (*common_type) < precision)
527 6802 : *common_type = build_nonstandard_integer_type
528 6802 : (precision, TYPE_UNSIGNED (*common_type));
529 : return true;
530 : }
531 :
532 : /* Return true if the common supertype of NEW_TYPE and *COMMON_TYPE
533 : is narrower than type, storing the supertype in *COMMON_TYPE if so. */
534 :
535 : static bool
536 51806 : vect_joust_widened_type (tree type, tree new_type, tree *common_type)
537 : {
538 51806 : if (types_compatible_p (*common_type, new_type))
539 : return true;
540 :
541 : /* See if *COMMON_TYPE can hold all values of NEW_TYPE. */
542 4785 : if ((TYPE_PRECISION (new_type) < TYPE_PRECISION (*common_type))
543 4785 : && (TYPE_UNSIGNED (new_type) || !TYPE_UNSIGNED (*common_type)))
544 : return true;
545 :
546 : /* See if NEW_TYPE can hold all values of *COMMON_TYPE. */
547 3824 : if (TYPE_PRECISION (*common_type) < TYPE_PRECISION (new_type)
548 3824 : && (TYPE_UNSIGNED (*common_type) || !TYPE_UNSIGNED (new_type)))
549 : {
550 924 : *common_type = new_type;
551 924 : return true;
552 : }
553 :
554 : /* We have mismatched signs, with the signed type being
555 : no wider than the unsigned type. In this case we need
556 : a wider signed type. */
557 2900 : unsigned int precision = MAX (TYPE_PRECISION (*common_type),
558 : TYPE_PRECISION (new_type));
559 2900 : precision *= 2;
560 :
561 2900 : if (precision * 2 > TYPE_PRECISION (type))
562 : return false;
563 :
564 40 : *common_type = build_nonstandard_integer_type (precision, false);
565 40 : return true;
566 : }
567 :
568 : /* If the range of UNPROM->OP at STMT fits in a narrower element type than
569 : UNPROM->TYPE, change UNPROM->TYPE to that type and return true. */
570 :
571 : static bool
572 276708 : vect_narrow_unprom_to_range (vect_unpromoted_value *unprom, gimple *stmt)
573 : {
574 276708 : int_range_max r;
575 553416 : get_range_query (cfun)->range_of_expr (r, unprom->op, stmt);
576 276708 : if (r.undefined_p ())
577 : return false;
578 :
579 275446 : signop sgn = TYPE_SIGN (unprom->type);
580 275446 : unsigned int precision
581 826338 : = vect_element_precision (MAX (wi::min_precision (r.lower_bound (), sgn),
582 : wi::min_precision (r.upper_bound (), sgn)));
583 275446 : if (precision >= TYPE_PRECISION (unprom->type))
584 : return false;
585 :
586 35336 : unprom->type = build_nonstandard_integer_type (precision, sgn == UNSIGNED);
587 35336 : return true;
588 276708 : }
589 :
590 : /* Check whether STMT_INFO can be viewed as a tree of integer operations
591 : in which each node either performs CODE or WIDENED_CODE, and where
592 : each leaf operand is narrower than the result of STMT_INFO. MAX_NOPS
593 : specifies the maximum number of leaf operands. SHIFT_P says whether
594 : CODE and WIDENED_CODE are some sort of shift.
595 :
596 : If STMT_INFO is such a tree, return the number of leaf operands
597 : and describe them in UNPROM[0] onwards. Also set *COMMON_TYPE
598 : to a type that (a) is narrower than the result of STMT_INFO and
599 : (b) can hold all leaf operand values.
600 :
601 : If SUBTYPE then allow that the signs of the operands
602 : may differ in signs but not in precision. SUBTYPE is updated to reflect
603 : this.
604 :
605 : Return 0 if STMT_INFO isn't such a tree, or if no such COMMON_TYPE
606 : exists. */
607 :
608 : static unsigned int
609 132793883 : vect_widened_op_tree (vec_info *vinfo, stmt_vec_info stmt_info, tree_code code,
610 : code_helper widened_code, bool shift_p,
611 : unsigned int max_nops,
612 : vect_unpromoted_value *unprom, tree *common_type,
613 : enum optab_subtype *subtype = NULL)
614 : {
615 : /* Check for an integer operation with the right code. */
616 132793883 : gimple* stmt = stmt_info->stmt;
617 132793883 : if (!(is_gimple_assign (stmt) || is_gimple_call (stmt)))
618 : return 0;
619 :
620 109088118 : code_helper rhs_code;
621 109088118 : if (is_gimple_assign (stmt))
622 93472598 : rhs_code = gimple_assign_rhs_code (stmt);
623 15615520 : else if (is_gimple_call (stmt))
624 15615520 : rhs_code = gimple_call_combined_fn (stmt);
625 : else
626 : return 0;
627 :
628 109088118 : if (rhs_code != code
629 109088118 : && rhs_code != widened_code)
630 : return 0;
631 :
632 6919527 : tree lhs = gimple_get_lhs (stmt);
633 6919527 : tree type = TREE_TYPE (lhs);
634 6919527 : if (!INTEGRAL_TYPE_P (type))
635 : return 0;
636 :
637 : /* Assume that both operands will be leaf operands. */
638 6029447 : max_nops -= 2;
639 :
640 : /* Check the operands. */
641 6029447 : unsigned int next_op = 0;
642 6914530 : for (unsigned int i = 0; i < 2; ++i)
643 : {
644 6530169 : vect_unpromoted_value *this_unprom = &unprom[next_op];
645 6530169 : unsigned int nops = 1;
646 6530169 : tree op = gimple_arg (stmt, i);
647 6530169 : if (i == 1 && TREE_CODE (op) == INTEGER_CST)
648 : {
649 : /* We already have a common type from earlier operands.
650 : Update it to account for OP. */
651 353947 : this_unprom->set_op (op, vect_constant_def);
652 353947 : if (!vect_joust_widened_integer (type, shift_p, op, common_type))
653 : return 0;
654 : }
655 : else
656 : {
657 : /* Only allow shifts by constants. */
658 6176222 : if (shift_p && i == 1)
659 : return 0;
660 :
661 6170111 : if (rhs_code != code)
662 : {
663 : /* If rhs_code is widened_code, don't look through further
664 : possible promotions, there is a promotion already embedded
665 : in the WIDEN_*_EXPR. */
666 1723 : if (TREE_CODE (op) != SSA_NAME
667 1723 : || !INTEGRAL_TYPE_P (TREE_TYPE (op)))
668 0 : return 0;
669 :
670 1723 : stmt_vec_info def_stmt_info;
671 1723 : gimple *def_stmt;
672 1723 : vect_def_type dt;
673 1723 : if (!vect_is_simple_use (op, vinfo, &dt, &def_stmt_info,
674 : &def_stmt))
675 : return 0;
676 1723 : this_unprom->set_op (op, dt, NULL);
677 : }
678 6168388 : else if (!vect_look_through_possible_promotion (vinfo, op,
679 : this_unprom))
680 : return 0;
681 :
682 6055173 : if (TYPE_PRECISION (this_unprom->type) == TYPE_PRECISION (type)
683 6331881 : && !(this_unprom->dt == vect_external_def
684 276708 : && vect_narrow_unprom_to_range (this_unprom, stmt)))
685 : {
686 : /* The operand isn't widened. If STMT_INFO has the code
687 : for an unwidened operation, recursively check whether
688 : this operand is a node of the tree. */
689 5498535 : if (rhs_code != code
690 5498535 : || max_nops == 0
691 5499060 : || this_unprom->dt != vect_internal_def)
692 : return 0;
693 :
694 : /* Give back the leaf slot allocated above now that we're
695 : not treating this as a leaf operand. */
696 525 : max_nops += 1;
697 :
698 : /* Recursively process the definition of the operand. */
699 525 : stmt_vec_info def_stmt_info
700 525 : = vect_get_internal_def (vinfo, this_unprom->op);
701 :
702 525 : nops = vect_widened_op_tree (vinfo, def_stmt_info, code,
703 : widened_code, shift_p, max_nops,
704 : this_unprom, common_type,
705 : subtype);
706 525 : if (nops == 0)
707 : return 0;
708 :
709 367 : max_nops -= nops;
710 : }
711 : else
712 : {
713 : /* Make sure that the operand is narrower than the result. */
714 556638 : if (TYPE_PRECISION (this_unprom->type) * 2
715 556638 : > TYPE_PRECISION (type))
716 : return 0;
717 :
718 : /* Update COMMON_TYPE for the new operand. */
719 552275 : if (i == 0)
720 500469 : *common_type = this_unprom->type;
721 51806 : else if (!vect_joust_widened_type (type, this_unprom->type,
722 : common_type))
723 : {
724 2860 : if (subtype)
725 : {
726 : /* See if we can sign extend the smaller type. */
727 291 : if (TYPE_PRECISION (this_unprom->type)
728 291 : > TYPE_PRECISION (*common_type))
729 27 : *common_type = this_unprom->type;
730 291 : *subtype = optab_vector_mixed_sign;
731 : }
732 : else
733 : return 0;
734 : }
735 : }
736 : }
737 885083 : next_op += nops;
738 : }
739 : return next_op;
740 : }
741 :
742 : /* Helper to return a new temporary for pattern of TYPE for STMT. If STMT
743 : is NULL, the caller must set SSA_NAME_DEF_STMT for the returned SSA var. */
744 :
745 : static tree
746 2210269 : vect_recog_temp_ssa_var (tree type, gimple *stmt = NULL)
747 : {
748 0 : return make_temp_ssa_name (type, stmt, "patt");
749 : }
750 :
751 : /* STMT2_INFO describes a type conversion that could be split into STMT1
752 : followed by a version of STMT2_INFO that takes NEW_RHS as its first
753 : input. Try to do this using pattern statements, returning true on
754 : success. */
755 :
756 : static bool
757 29011 : vect_split_statement (vec_info *vinfo, stmt_vec_info stmt2_info, tree new_rhs,
758 : gimple *stmt1, tree vectype)
759 : {
760 29011 : if (is_pattern_stmt_p (stmt2_info))
761 : {
762 : /* STMT2_INFO is part of a pattern. Get the statement to which
763 : the pattern is attached. */
764 463 : stmt_vec_info orig_stmt2_info = STMT_VINFO_RELATED_STMT (stmt2_info);
765 463 : vect_init_pattern_stmt (vinfo, stmt1, orig_stmt2_info, vectype);
766 :
767 463 : if (dump_enabled_p ())
768 19 : dump_printf_loc (MSG_NOTE, vect_location,
769 : "Splitting pattern statement: %G", stmt2_info->stmt);
770 :
771 : /* Since STMT2_INFO is a pattern statement, we can change it
772 : in-situ without worrying about changing the code for the
773 : containing block. */
774 463 : gimple_assign_set_rhs1 (stmt2_info->stmt, new_rhs);
775 :
776 463 : if (dump_enabled_p ())
777 : {
778 19 : dump_printf_loc (MSG_NOTE, vect_location, "into: %G", stmt1);
779 19 : dump_printf_loc (MSG_NOTE, vect_location, "and: %G",
780 : stmt2_info->stmt);
781 : }
782 :
783 463 : gimple_seq *def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt2_info);
784 463 : if (STMT_VINFO_RELATED_STMT (orig_stmt2_info) == stmt2_info)
785 : /* STMT2_INFO is the actual pattern statement. Add STMT1
786 : to the end of the definition sequence. */
787 454 : gimple_seq_add_stmt_without_update (def_seq, stmt1);
788 : else
789 : {
790 : /* STMT2_INFO belongs to the definition sequence. Insert STMT1
791 : before it. */
792 9 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt2_info->stmt, def_seq);
793 9 : gsi_insert_before_without_update (&gsi, stmt1, GSI_SAME_STMT);
794 : }
795 : return true;
796 : }
797 : else
798 : {
799 : /* STMT2_INFO doesn't yet have a pattern. Try to create a
800 : two-statement pattern now. */
801 28548 : gcc_assert (!STMT_VINFO_RELATED_STMT (stmt2_info));
802 28548 : tree lhs_type = TREE_TYPE (gimple_get_lhs (stmt2_info->stmt));
803 28548 : tree lhs_vectype = get_vectype_for_scalar_type (vinfo, lhs_type);
804 28548 : if (!lhs_vectype)
805 : return false;
806 :
807 28548 : if (dump_enabled_p ())
808 2061 : dump_printf_loc (MSG_NOTE, vect_location,
809 : "Splitting statement: %G", stmt2_info->stmt);
810 :
811 : /* Add STMT1 as a singleton pattern definition sequence. */
812 28548 : gimple_seq *def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (stmt2_info);
813 28548 : vect_init_pattern_stmt (vinfo, stmt1, stmt2_info, vectype);
814 28548 : gimple_seq_add_stmt_without_update (def_seq, stmt1);
815 :
816 : /* Build the second of the two pattern statements. */
817 28548 : tree new_lhs = vect_recog_temp_ssa_var (lhs_type, NULL);
818 28548 : gassign *new_stmt2 = gimple_build_assign (new_lhs, NOP_EXPR, new_rhs);
819 28548 : vect_set_pattern_stmt (vinfo, new_stmt2, stmt2_info, lhs_vectype);
820 :
821 28548 : if (dump_enabled_p ())
822 : {
823 2061 : dump_printf_loc (MSG_NOTE, vect_location,
824 : "into pattern statements: %G", stmt1);
825 2061 : dump_printf_loc (MSG_NOTE, vect_location, "and: %G",
826 : (gimple *) new_stmt2);
827 : }
828 :
829 : return true;
830 : }
831 : }
832 :
833 : /* Look for the following pattern
834 : X = x[i]
835 : Y = y[i]
836 : DIFF = X - Y
837 : DAD = ABS_EXPR<DIFF>
838 :
839 : ABS_STMT should point to a statement of code ABS_EXPR or ABSU_EXPR.
840 : HALF_TYPE and UNPROM will be set should the statement be found to
841 : be a widened operation.
842 : DIFF_STMT will be set to the MINUS_EXPR
843 : statement that precedes the ABS_STMT if it is a MINUS_EXPR..
844 : */
845 : static bool
846 22976596 : vect_recog_absolute_difference (vec_info *vinfo, gassign *abs_stmt,
847 : tree *half_type,
848 : vect_unpromoted_value unprom[2],
849 : gassign **diff_stmt)
850 : {
851 22976596 : if (!abs_stmt)
852 : return false;
853 :
854 : /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
855 : inside the loop (in case we are analyzing an outer-loop). */
856 22976596 : enum tree_code code = gimple_assign_rhs_code (abs_stmt);
857 22976596 : if (code != ABS_EXPR && code != ABSU_EXPR)
858 : return false;
859 :
860 34454 : tree abs_oprnd = gimple_assign_rhs1 (abs_stmt);
861 34454 : tree abs_type = TREE_TYPE (abs_oprnd);
862 34454 : if (!abs_oprnd)
863 : return false;
864 27039 : if (!ANY_INTEGRAL_TYPE_P (abs_type)
865 7666 : || TYPE_OVERFLOW_WRAPS (abs_type)
866 41960 : || TYPE_UNSIGNED (abs_type))
867 : return false;
868 :
869 : /* Peel off conversions from the ABS input. This can involve sign
870 : changes (e.g. from an unsigned subtraction to a signed ABS input)
871 : or signed promotion, but it can't include unsigned promotion.
872 : (Note that ABS of an unsigned promotion should have been folded
873 : away before now anyway.) */
874 7506 : vect_unpromoted_value unprom_diff;
875 7506 : abs_oprnd = vect_look_through_possible_promotion (vinfo, abs_oprnd,
876 : &unprom_diff);
877 7506 : if (!abs_oprnd)
878 : return false;
879 7255 : if (TYPE_PRECISION (unprom_diff.type) != TYPE_PRECISION (abs_type)
880 7255 : && TYPE_UNSIGNED (unprom_diff.type))
881 : return false;
882 :
883 : /* We then detect if the operand of abs_expr is defined by a minus_expr. */
884 7255 : stmt_vec_info diff_stmt_vinfo = vect_get_internal_def (vinfo, abs_oprnd);
885 7255 : if (!diff_stmt_vinfo)
886 : return false;
887 :
888 7043 : gassign *diff = dyn_cast <gassign *> (STMT_VINFO_STMT (diff_stmt_vinfo));
889 7043 : if (diff_stmt && diff
890 5801 : && gimple_assign_rhs_code (diff) == MINUS_EXPR
891 8896 : && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (abs_oprnd)))
892 251 : *diff_stmt = diff;
893 :
894 : /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
895 : inside the loop (in case we are analyzing an outer-loop). */
896 7043 : if (vect_widened_op_tree (vinfo, diff_stmt_vinfo,
897 7043 : MINUS_EXPR, IFN_VEC_WIDEN_MINUS,
898 : false, 2, unprom, half_type))
899 2408 : return true;
900 :
901 : return false;
902 : }
903 :
904 : /* Convert UNPROM to TYPE and return the result, adding new statements
905 : to STMT_INFO's pattern definition statements if no better way is
906 : available. VECTYPE is the vector form of TYPE.
907 :
908 : If SUBTYPE then convert the type based on the subtype. */
909 :
910 : static tree
911 516427 : vect_convert_input (vec_info *vinfo, stmt_vec_info stmt_info, tree type,
912 : vect_unpromoted_value *unprom, tree vectype,
913 : enum optab_subtype subtype = optab_default)
914 : {
915 : /* Update the type if the signs differ. */
916 516427 : if (subtype == optab_vector_mixed_sign)
917 : {
918 328 : gcc_assert (!TYPE_UNSIGNED (type));
919 328 : if (TYPE_UNSIGNED (TREE_TYPE (unprom->op)))
920 : {
921 164 : type = unsigned_type_for (type);
922 164 : vectype = unsigned_type_for (vectype);
923 : }
924 : }
925 :
926 : /* Check for a no-op conversion. */
927 516427 : if (types_compatible_p (type, TREE_TYPE (unprom->op)))
928 182301 : return unprom->op;
929 :
930 : /* Allow the caller to create constant vect_unpromoted_values. */
931 334126 : if (TREE_CODE (unprom->op) == INTEGER_CST)
932 207003 : return wide_int_to_tree (type, wi::to_widest (unprom->op));
933 :
934 127123 : tree input = unprom->op;
935 127123 : if (unprom->caster)
936 : {
937 67161 : tree lhs = gimple_get_lhs (unprom->caster->stmt);
938 67161 : tree lhs_type = TREE_TYPE (lhs);
939 :
940 : /* If the result of the existing cast is the right width, use it
941 : instead of the source of the cast. */
942 67161 : if (TYPE_PRECISION (lhs_type) == TYPE_PRECISION (type))
943 : input = lhs;
944 : /* If the precision we want is between the source and result
945 : precisions of the existing cast, try splitting the cast into
946 : two and tapping into a mid-way point. */
947 64876 : else if (TYPE_PRECISION (lhs_type) > TYPE_PRECISION (type)
948 64876 : && TYPE_PRECISION (type) > TYPE_PRECISION (unprom->type))
949 : {
950 : /* In order to preserve the semantics of the original cast,
951 : give the mid-way point the same signedness as the input value.
952 :
953 : It would be possible to use a signed type here instead if
954 : TYPE is signed and UNPROM->TYPE is unsigned, but that would
955 : make the sign of the midtype sensitive to the order in
956 : which we process the statements, since the signedness of
957 : TYPE is the signedness required by just one of possibly
958 : many users. Also, unsigned promotions are usually as cheap
959 : as or cheaper than signed ones, so it's better to keep an
960 : unsigned promotion. */
961 29011 : tree midtype = build_nonstandard_integer_type
962 29011 : (TYPE_PRECISION (type), TYPE_UNSIGNED (unprom->type));
963 29011 : tree vec_midtype = get_vectype_for_scalar_type (vinfo, midtype);
964 29011 : if (vec_midtype)
965 : {
966 29011 : input = vect_recog_temp_ssa_var (midtype, NULL);
967 29011 : gassign *new_stmt = gimple_build_assign (input, NOP_EXPR,
968 : unprom->op);
969 29011 : if (!vect_split_statement (vinfo, unprom->caster, input, new_stmt,
970 : vec_midtype))
971 0 : append_pattern_def_seq (vinfo, stmt_info,
972 : new_stmt, vec_midtype);
973 : }
974 : }
975 :
976 : /* See if we can reuse an existing result. */
977 67161 : if (types_compatible_p (type, TREE_TYPE (input)))
978 : return input;
979 : }
980 :
981 : /* We need a new conversion statement. */
982 106242 : tree new_op = vect_recog_temp_ssa_var (type, NULL);
983 106242 : gassign *new_stmt = gimple_build_assign (new_op, NOP_EXPR, input);
984 :
985 : /* If OP is an external value, see if we can insert the new statement
986 : on an incoming edge. */
987 106242 : if (input == unprom->op && unprom->dt == vect_external_def)
988 9656 : if (edge e = vect_get_external_def_edge (vinfo, input))
989 : {
990 963 : basic_block new_bb = gsi_insert_on_edge_immediate (e, new_stmt);
991 963 : gcc_assert (!new_bb);
992 : return new_op;
993 : }
994 :
995 : /* As a (common) last resort, add the statement to the pattern itself. */
996 105279 : append_pattern_def_seq (vinfo, stmt_info, new_stmt, vectype);
997 105279 : return new_op;
998 : }
999 :
1000 : /* Invoke vect_convert_input for N elements of UNPROM and store the
1001 : result in the corresponding elements of RESULT.
1002 :
1003 : If SUBTYPE then convert the type based on the subtype. */
1004 :
1005 : static void
1006 262294 : vect_convert_inputs (vec_info *vinfo, stmt_vec_info stmt_info, unsigned int n,
1007 : tree *result, tree type, vect_unpromoted_value *unprom,
1008 : tree vectype, enum optab_subtype subtype = optab_default)
1009 : {
1010 778801 : for (unsigned int i = 0; i < n; ++i)
1011 : {
1012 : unsigned int j;
1013 770132 : for (j = 0; j < i; ++j)
1014 254213 : if (unprom[j].op == unprom[i].op)
1015 : break;
1016 :
1017 516507 : if (j < i)
1018 588 : result[i] = result[j];
1019 : else
1020 515919 : result[i] = vect_convert_input (vinfo, stmt_info,
1021 515919 : type, &unprom[i], vectype, subtype);
1022 : }
1023 262294 : }
1024 :
1025 : /* The caller has created a (possibly empty) sequence of pattern definition
1026 : statements followed by a single statement PATTERN_STMT. Cast the result
1027 : of this final statement to TYPE. If a new statement is needed, add
1028 : PATTERN_STMT to the end of STMT_INFO's pattern definition statements
1029 : and return the new statement, otherwise return PATTERN_STMT as-is.
1030 : VECITYPE is the vector form of PATTERN_STMT's result type. */
1031 :
1032 : static gimple *
1033 289086 : vect_convert_output (vec_info *vinfo, stmt_vec_info stmt_info, tree type,
1034 : gimple *pattern_stmt, tree vecitype)
1035 : {
1036 289086 : tree lhs = gimple_get_lhs (pattern_stmt);
1037 289086 : if (!types_compatible_p (type, TREE_TYPE (lhs)))
1038 : {
1039 258010 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vecitype);
1040 258010 : tree cast_var = vect_recog_temp_ssa_var (type, NULL);
1041 258010 : pattern_stmt = gimple_build_assign (cast_var, NOP_EXPR, lhs);
1042 : }
1043 289086 : return pattern_stmt;
1044 : }
1045 :
1046 : /* Return true if STMT_VINFO describes a reduction for which reassociation
1047 : is allowed. If STMT_INFO is part of a group, assume that it's part of
1048 : a reduction chain and optimistically assume that all statements
1049 : except the last allow reassociation.
1050 : Also require it to have code CODE and to be a reduction
1051 : in the outermost loop. When returning true, store the operands in
1052 : *OP0_OUT and *OP1_OUT. */
1053 :
1054 : static bool
1055 99053199 : vect_reassociating_reduction_p (vec_info *vinfo,
1056 : stmt_vec_info stmt_info, tree_code code,
1057 : tree *op0_out, tree *op1_out)
1058 : {
1059 99053199 : loop_vec_info loop_info = dyn_cast <loop_vec_info> (vinfo);
1060 13843368 : if (!loop_info)
1061 : return false;
1062 :
1063 13843368 : gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
1064 15105339 : if (!assign || gimple_assign_rhs_code (assign) != code)
1065 : return false;
1066 :
1067 : /* We don't allow changing the order of the computation in the inner-loop
1068 : when doing outer-loop vectorization. */
1069 2694575 : class loop *loop = LOOP_VINFO_LOOP (loop_info);
1070 2694575 : if (loop && nested_in_vect_loop_p (loop, stmt_info))
1071 : return false;
1072 :
1073 2641313 : if (!vect_is_reduction (stmt_info))
1074 : return false;
1075 :
1076 173180 : if (needs_fold_left_reduction_p (TREE_TYPE (gimple_assign_lhs (assign)),
1077 173180 : code))
1078 : return false;
1079 :
1080 160256 : *op0_out = gimple_assign_rhs1 (assign);
1081 160256 : *op1_out = gimple_assign_rhs2 (assign);
1082 160256 : if (commutative_tree_code (code) && STMT_VINFO_REDUC_IDX (stmt_info) == 0)
1083 64014 : std::swap (*op0_out, *op1_out);
1084 : return true;
1085 : }
1086 :
1087 : /* Return true iff the target has a vector optab implementing the operation
1088 : CODE on type VECTYPE with SUBTYPE. */
1089 :
1090 : static bool
1091 895799 : target_has_vecop_for_code (tree_code code, tree vectype,
1092 : enum optab_subtype subtype = optab_vector)
1093 : {
1094 895799 : optab voptab = optab_for_tree_code (code, vectype, subtype);
1095 895799 : return voptab && can_implement_p (voptab, TYPE_MODE (vectype));
1096 : }
1097 :
1098 : /* match.pd function to match
1099 : (cond (cmp@3 a b) (convert@1 c) (convert@2 d))
1100 : with conditions:
1101 : 1) @1, @2, c, d, a, b are all integral type.
1102 : 2) There's single_use for both @1 and @2.
1103 : 3) a, c have same precision.
1104 : 4) c and @1 have different precision.
1105 : 5) c, d are the same type or they can differ in sign when convert is
1106 : truncation.
1107 :
1108 : record a and c and d and @3. */
1109 :
1110 : extern bool gimple_cond_expr_convert_p (tree, tree*, tree (*)(tree));
1111 :
1112 : /* Function vect_recog_cond_expr_convert
1113 :
1114 : Try to find the following pattern:
1115 :
1116 : TYPE_AB A,B;
1117 : TYPE_CD C,D;
1118 : TYPE_E E;
1119 : TYPE_E op_true = (TYPE_E) A;
1120 : TYPE_E op_false = (TYPE_E) B;
1121 :
1122 : E = C cmp D ? op_true : op_false;
1123 :
1124 : where
1125 : TYPE_PRECISION (TYPE_E) != TYPE_PRECISION (TYPE_CD);
1126 : TYPE_PRECISION (TYPE_AB) == TYPE_PRECISION (TYPE_CD);
1127 : single_use of op_true and op_false.
1128 : TYPE_AB could differ in sign when (TYPE_E) A is a truncation.
1129 :
1130 : Input:
1131 :
1132 : * STMT_VINFO: The stmt from which the pattern search begins.
1133 : here it starts with E = c cmp D ? op_true : op_false;
1134 :
1135 : Output:
1136 :
1137 : TYPE1 E' = C cmp D ? A : B;
1138 : TYPE3 E = (TYPE3) E';
1139 :
1140 : There may extra nop_convert for A or B to handle different signness.
1141 :
1142 : * TYPE_OUT: The vector type of the output of this pattern.
1143 :
1144 : * Return value: A new stmt that will be used to replace the sequence of
1145 : stmts that constitute the pattern. In this case it will be:
1146 : E = (TYPE3)E';
1147 : E' = C cmp D ? A : B; is recorded in pattern definition statements; */
1148 :
1149 : static gimple *
1150 33099295 : vect_recog_cond_expr_convert_pattern (vec_info *vinfo,
1151 : stmt_vec_info stmt_vinfo, tree *type_out)
1152 : {
1153 33099295 : gassign *last_stmt = dyn_cast <gassign *> (stmt_vinfo->stmt);
1154 23078018 : tree lhs, match[4], temp, type, new_lhs, op2, op1;
1155 23078018 : gimple *cond_stmt;
1156 23078018 : gimple *pattern_stmt;
1157 56177313 : enum tree_code code = NOP_EXPR;
1158 :
1159 23078018 : if (!last_stmt)
1160 : return NULL;
1161 :
1162 23078018 : lhs = gimple_assign_lhs (last_stmt);
1163 :
1164 : /* Find E = C cmp D ? (TYPE3) A ? (TYPE3) B;
1165 : TYPE_PRECISION (A) == TYPE_PRECISION (C). */
1166 23078018 : if (!gimple_cond_expr_convert_p (lhs, &match[0], NULL))
1167 : return NULL;
1168 :
1169 33 : if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (lhs)))
1170 20 : code = INTEGRAL_TYPE_P (TREE_TYPE (match[1])) ? FLOAT_EXPR : CONVERT_EXPR;
1171 13 : else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (match[1])))
1172 0 : code = FIX_TRUNC_EXPR;
1173 :
1174 33 : op1 = match[1];
1175 33 : op2 = match[2];
1176 33 : type = TREE_TYPE (op1);
1177 : /* When op1/op2 is REAL_CST, the conversion must be CONVERT_EXPR from
1178 : SCALAR_FLOAT_TYPE_P which is restricted in gimple_cond_expr_convert_p.
1179 : Otherwise, the conversion could be FLOAT_EXPR, FIX_TRUNC_EXPR
1180 : or CONVERT_EXPR. */
1181 33 : if (TREE_CODE (op1) == REAL_CST)
1182 : {
1183 20 : op1 = const_unop (CONVERT_EXPR, TREE_TYPE (op2), op1);
1184 20 : type = TREE_TYPE (op2);
1185 20 : if (op1 == NULL_TREE)
1186 : return NULL;
1187 : }
1188 13 : else if (TREE_CODE (op2) == REAL_CST)
1189 : {
1190 0 : op2 = const_unop (FLOAT_EXPR, TREE_TYPE (op1), op2);
1191 0 : if (op2 == NULL_TREE)
1192 : return NULL;
1193 : }
1194 13 : else if (code == NOP_EXPR)
1195 : {
1196 13 : if (TYPE_SIGN (type) != TYPE_SIGN (TREE_TYPE (match[2])))
1197 : {
1198 9 : op2 = vect_recog_temp_ssa_var (type, NULL);
1199 9 : gimple* nop_stmt = gimple_build_assign (op2, NOP_EXPR, match[2]);
1200 9 : append_pattern_def_seq (vinfo, stmt_vinfo, nop_stmt);
1201 : }
1202 : }
1203 :
1204 33 : vect_pattern_detected ("vect_recog_cond_expr_convert_pattern", last_stmt);
1205 :
1206 33 : temp = vect_recog_temp_ssa_var (type, NULL);
1207 33 : cond_stmt = gimple_build_assign (temp, build3 (COND_EXPR, type, match[3],
1208 : op1, op2));
1209 33 : append_pattern_def_seq (vinfo, stmt_vinfo, cond_stmt);
1210 33 : new_lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
1211 33 : pattern_stmt = gimple_build_assign (new_lhs, code, temp);
1212 33 : *type_out = NULL_TREE;
1213 :
1214 33 : if (dump_enabled_p ())
1215 24 : dump_printf_loc (MSG_NOTE, vect_location,
1216 : "created pattern stmt: %G", pattern_stmt);
1217 : return pattern_stmt;
1218 : }
1219 :
1220 : /* Function vect_recog_dot_prod_pattern
1221 :
1222 : Try to find the following pattern:
1223 :
1224 : type1a x_t
1225 : type1b y_t;
1226 : TYPE1 prod;
1227 : TYPE2 sum = init;
1228 : loop:
1229 : sum_0 = phi <init, sum_1>
1230 : S1 x_t = ...
1231 : S2 y_t = ...
1232 : S3 x_T = (TYPE1) x_t;
1233 : S4 y_T = (TYPE1) y_t;
1234 : S5 prod = x_T * y_T;
1235 : [S6 prod = (TYPE2) prod; #optional]
1236 : S7 sum_1 = prod + sum_0;
1237 :
1238 : where 'TYPE1' is exactly double the size of type 'type1a' and 'type1b',
1239 : the sign of 'TYPE1' must be one of 'type1a' or 'type1b' but the sign of
1240 : 'type1a' and 'type1b' can differ.
1241 :
1242 : Input:
1243 :
1244 : * STMT_VINFO: The stmt from which the pattern search begins. In the
1245 : example, when this function is called with S7, the pattern {S3,S4,S5,S6,S7}
1246 : will be detected.
1247 :
1248 : Output:
1249 :
1250 : * TYPE_OUT: The type of the output of this pattern.
1251 :
1252 : * Return value: A new stmt that will be used to replace the sequence of
1253 : stmts that constitute the pattern. In this case it will be:
1254 : WIDEN_DOT_PRODUCT <x_t, y_t, sum_0>
1255 :
1256 : Note: The dot-prod idiom is a widening reduction pattern that is
1257 : vectorized without preserving all the intermediate results. It
1258 : produces only N/2 (widened) results (by summing up pairs of
1259 : intermediate results) rather than all N results. Therefore, we
1260 : cannot allow this pattern when we want to get all the results and in
1261 : the correct order (as is the case when this computation is in an
1262 : inner-loop nested in an outer-loop that us being vectorized). */
1263 :
1264 : static gimple *
1265 33018371 : vect_recog_dot_prod_pattern (vec_info *vinfo,
1266 : stmt_vec_info stmt_vinfo, tree *type_out)
1267 : {
1268 33018371 : tree oprnd0, oprnd1;
1269 33018371 : gimple *last_stmt = stmt_vinfo->stmt;
1270 33018371 : tree type, half_type;
1271 33018371 : gimple *pattern_stmt;
1272 33018371 : tree var;
1273 :
1274 : /* Look for the following pattern
1275 : DX = (TYPE1) X;
1276 : DY = (TYPE1) Y;
1277 : DPROD = DX * DY;
1278 : DDPROD = (TYPE2) DPROD;
1279 : sum_1 = DDPROD + sum_0;
1280 : In which
1281 : - DX is double the size of X
1282 : - DY is double the size of Y
1283 : - DX, DY, DPROD all have the same type but the sign
1284 : between X, Y and DPROD can differ.
1285 : - sum is the same size of DPROD or bigger
1286 : - sum has been recognized as a reduction variable.
1287 :
1288 : This is equivalent to:
1289 : DPROD = X w* Y; #widen mult
1290 : sum_1 = DPROD w+ sum_0; #widen summation
1291 : or
1292 : DPROD = X w* Y; #widen mult
1293 : sum_1 = DPROD + sum_0; #summation
1294 : */
1295 :
1296 : /* Starting from LAST_STMT, follow the defs of its uses in search
1297 : of the above pattern. */
1298 :
1299 33018371 : if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, PLUS_EXPR,
1300 : &oprnd0, &oprnd1))
1301 : return NULL;
1302 :
1303 54060 : type = TREE_TYPE (gimple_get_lhs (last_stmt));
1304 :
1305 54060 : vect_unpromoted_value unprom_mult;
1306 54060 : oprnd0 = vect_look_through_possible_promotion (vinfo, oprnd0, &unprom_mult);
1307 :
1308 : /* So far so good. Since last_stmt was detected as a (summation) reduction,
1309 : we know that oprnd1 is the reduction variable (defined by a loop-header
1310 : phi), and oprnd0 is an ssa-name defined by a stmt in the loop body.
1311 : Left to check that oprnd0 is defined by a (widen_)mult_expr */
1312 54060 : if (!oprnd0)
1313 : return NULL;
1314 :
1315 36762 : stmt_vec_info mult_vinfo = vect_get_internal_def (vinfo, oprnd0);
1316 36762 : if (!mult_vinfo)
1317 : return NULL;
1318 :
1319 : /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
1320 : inside the loop (in case we are analyzing an outer-loop). */
1321 107736 : vect_unpromoted_value unprom0[2];
1322 35912 : enum optab_subtype subtype = optab_vector;
1323 35912 : if (!vect_widened_op_tree (vinfo, mult_vinfo, MULT_EXPR, WIDEN_MULT_EXPR,
1324 : false, 2, unprom0, &half_type, &subtype))
1325 : return NULL;
1326 :
1327 : /* If there are two widening operations, make sure they agree on the sign
1328 : of the extension. The result of an optab_vector_mixed_sign operation
1329 : is signed; otherwise, the result has the same sign as the operands. */
1330 1386 : if (TYPE_PRECISION (unprom_mult.type) != TYPE_PRECISION (type)
1331 2127 : && (subtype == optab_vector_mixed_sign
1332 741 : ? TYPE_UNSIGNED (unprom_mult.type)
1333 532 : : TYPE_SIGN (unprom_mult.type) != TYPE_SIGN (half_type)))
1334 : return NULL;
1335 :
1336 1305 : vect_pattern_detected ("vect_recog_dot_prod_pattern", last_stmt);
1337 :
1338 : /* If the inputs have mixed signs, canonicalize on using the signed
1339 : input type for analysis. This also helps when emulating mixed-sign
1340 : operations using signed operations. */
1341 1305 : if (subtype == optab_vector_mixed_sign)
1342 246 : half_type = signed_type_for (half_type);
1343 :
1344 1305 : tree half_vectype;
1345 1305 : if (!vect_supportable_conv_optab_p (vinfo, type, DOT_PROD_EXPR, half_type,
1346 : type_out, &half_vectype, subtype))
1347 : {
1348 : /* We can emulate a mixed-sign dot-product using a sequence of
1349 : signed dot-products; see vect_emulate_mixed_dot_prod for details. */
1350 588 : if (subtype != optab_vector_mixed_sign
1351 588 : || !vect_supportable_conv_optab_p (vinfo, signed_type_for (type),
1352 : DOT_PROD_EXPR, half_type,
1353 : type_out, &half_vectype,
1354 : optab_vector))
1355 : return NULL;
1356 :
1357 15 : *type_out = signed_or_unsigned_type_for (TYPE_UNSIGNED (type),
1358 : *type_out);
1359 : }
1360 :
1361 : /* Get the inputs in the appropriate types. */
1362 732 : tree mult_oprnd[2];
1363 732 : vect_convert_inputs (vinfo, stmt_vinfo, 2, mult_oprnd, half_type,
1364 : unprom0, half_vectype, subtype);
1365 :
1366 732 : var = vect_recog_temp_ssa_var (type, NULL);
1367 732 : pattern_stmt = gimple_build_assign (var, DOT_PROD_EXPR,
1368 : mult_oprnd[0], mult_oprnd[1], oprnd1);
1369 :
1370 732 : return pattern_stmt;
1371 : }
1372 :
1373 :
1374 : /* Function vect_recog_sad_pattern
1375 :
1376 : Try to find the following Sum of Absolute Difference (SAD) pattern:
1377 :
1378 : type x_t, y_t;
1379 : signed TYPE1 diff, abs_diff;
1380 : TYPE2 sum = init;
1381 : loop:
1382 : sum_0 = phi <init, sum_1>
1383 : S1 x_t = ...
1384 : S2 y_t = ...
1385 : S3 x_T = (TYPE1) x_t;
1386 : S4 y_T = (TYPE1) y_t;
1387 : S5 diff = x_T - y_T;
1388 : S6 abs_diff = ABS_EXPR <diff>;
1389 : [S7 abs_diff = (TYPE2) abs_diff; #optional]
1390 : S8 sum_1 = abs_diff + sum_0;
1391 :
1392 : where 'TYPE1' is at least double the size of type 'type', and 'TYPE2' is the
1393 : same size of 'TYPE1' or bigger. This is a special case of a reduction
1394 : computation.
1395 :
1396 : Input:
1397 :
1398 : * STMT_VINFO: The stmt from which the pattern search begins. In the
1399 : example, when this function is called with S8, the pattern
1400 : {S3,S4,S5,S6,S7,S8} will be detected.
1401 :
1402 : Output:
1403 :
1404 : * TYPE_OUT: The type of the output of this pattern.
1405 :
1406 : * Return value: A new stmt that will be used to replace the sequence of
1407 : stmts that constitute the pattern. In this case it will be:
1408 : SAD_EXPR <x_t, y_t, sum_0>
1409 : */
1410 :
1411 : static gimple *
1412 33017644 : vect_recog_sad_pattern (vec_info *vinfo,
1413 : stmt_vec_info stmt_vinfo, tree *type_out)
1414 : {
1415 33017644 : gimple *last_stmt = stmt_vinfo->stmt;
1416 33017644 : tree half_type;
1417 :
1418 : /* Look for the following pattern
1419 : DX = (TYPE1) X;
1420 : DY = (TYPE1) Y;
1421 : DDIFF = DX - DY;
1422 : DAD = ABS_EXPR <DDIFF>;
1423 : DDPROD = (TYPE2) DPROD;
1424 : sum_1 = DAD + sum_0;
1425 : In which
1426 : - DX is at least double the size of X
1427 : - DY is at least double the size of Y
1428 : - DX, DY, DDIFF, DAD all have the same type
1429 : - sum is the same size of DAD or bigger
1430 : - sum has been recognized as a reduction variable.
1431 :
1432 : This is equivalent to:
1433 : DDIFF = X w- Y; #widen sub
1434 : DAD = ABS_EXPR <DDIFF>;
1435 : sum_1 = DAD w+ sum_0; #widen summation
1436 : or
1437 : DDIFF = X w- Y; #widen sub
1438 : DAD = ABS_EXPR <DDIFF>;
1439 : sum_1 = DAD + sum_0; #summation
1440 : */
1441 :
1442 : /* Starting from LAST_STMT, follow the defs of its uses in search
1443 : of the above pattern. */
1444 :
1445 33017644 : tree plus_oprnd0, plus_oprnd1;
1446 33017644 : if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, PLUS_EXPR,
1447 : &plus_oprnd0, &plus_oprnd1))
1448 : return NULL;
1449 :
1450 53328 : tree sum_type = TREE_TYPE (gimple_get_lhs (last_stmt));
1451 :
1452 : /* Any non-truncating sequence of conversions is OK here, since
1453 : with a successful match, the result of the ABS(U) is known to fit
1454 : within the nonnegative range of the result type. (It cannot be the
1455 : negative of the minimum signed value due to the range of the widening
1456 : MINUS_EXPR.) */
1457 53328 : vect_unpromoted_value unprom_abs;
1458 53328 : plus_oprnd0 = vect_look_through_possible_promotion (vinfo, plus_oprnd0,
1459 : &unprom_abs);
1460 :
1461 : /* So far so good. Since last_stmt was detected as a (summation) reduction,
1462 : we know that plus_oprnd1 is the reduction variable (defined by a loop-header
1463 : phi), and plus_oprnd0 is an ssa-name defined by a stmt in the loop body.
1464 : Then check that plus_oprnd0 is defined by an abs_expr. */
1465 :
1466 53328 : if (!plus_oprnd0)
1467 : return NULL;
1468 :
1469 36030 : stmt_vec_info abs_stmt_vinfo = vect_get_internal_def (vinfo, plus_oprnd0);
1470 36030 : if (!abs_stmt_vinfo)
1471 : return NULL;
1472 :
1473 : /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
1474 : inside the loop (in case we are analyzing an outer-loop). */
1475 35180 : gassign *abs_stmt = dyn_cast <gassign *> (abs_stmt_vinfo->stmt);
1476 105540 : vect_unpromoted_value unprom[2];
1477 :
1478 35180 : if (!abs_stmt)
1479 : {
1480 367 : gcall *abd_stmt = dyn_cast <gcall *> (abs_stmt_vinfo->stmt);
1481 302 : if (!abd_stmt
1482 302 : || !gimple_call_internal_p (abd_stmt)
1483 0 : || gimple_call_num_args (abd_stmt) != 2)
1484 : return NULL;
1485 :
1486 0 : tree abd_oprnd0 = gimple_call_arg (abd_stmt, 0);
1487 0 : tree abd_oprnd1 = gimple_call_arg (abd_stmt, 1);
1488 :
1489 0 : if (gimple_call_internal_fn (abd_stmt) == IFN_ABD
1490 0 : || gimple_call_internal_fn (abd_stmt) == IFN_VEC_WIDEN_ABD)
1491 : {
1492 0 : unprom[0].op = abd_oprnd0;
1493 0 : unprom[0].type = TREE_TYPE (abd_oprnd0);
1494 0 : unprom[1].op = abd_oprnd1;
1495 0 : unprom[1].type = TREE_TYPE (abd_oprnd1);
1496 : }
1497 : else
1498 : return NULL;
1499 :
1500 0 : half_type = unprom[0].type;
1501 : }
1502 34813 : else if (!vect_recog_absolute_difference (vinfo, abs_stmt, &half_type,
1503 : unprom, NULL))
1504 : return NULL;
1505 :
1506 854 : vect_pattern_detected ("vect_recog_sad_pattern", last_stmt);
1507 :
1508 854 : tree half_vectype;
1509 854 : if (!vect_supportable_direct_optab_p (vinfo, sum_type, SAD_EXPR, half_type,
1510 : type_out, &half_vectype))
1511 : return NULL;
1512 :
1513 : /* Get the inputs to the SAD_EXPR in the appropriate types. */
1514 460 : tree sad_oprnd[2];
1515 460 : vect_convert_inputs (vinfo, stmt_vinfo, 2, sad_oprnd, half_type,
1516 : unprom, half_vectype);
1517 :
1518 460 : tree var = vect_recog_temp_ssa_var (sum_type, NULL);
1519 460 : gimple *pattern_stmt = gimple_build_assign (var, SAD_EXPR, sad_oprnd[0],
1520 : sad_oprnd[1], plus_oprnd1);
1521 :
1522 460 : return pattern_stmt;
1523 : }
1524 :
1525 : /* Function vect_recog_abd_pattern
1526 :
1527 : Try to find the following ABsolute Difference (ABD) or
1528 : widening ABD (WIDEN_ABD) pattern:
1529 :
1530 : TYPE1 x;
1531 : TYPE2 y;
1532 : TYPE3 x_cast = (TYPE3) x; // widening or no-op
1533 : TYPE3 y_cast = (TYPE3) y; // widening or no-op
1534 : TYPE3 diff = x_cast - y_cast;
1535 : TYPE4 diff_cast = (TYPE4) diff; // widening or no-op
1536 : TYPE5 abs = ABS(U)_EXPR <diff_cast>;
1537 :
1538 : WIDEN_ABD exists to optimize the case where TYPE4 is at least
1539 : twice as wide as TYPE3.
1540 :
1541 : Input:
1542 :
1543 : * STMT_VINFO: The stmt from which the pattern search begins
1544 :
1545 : Output:
1546 :
1547 : * TYPE_OUT: The type of the output of this pattern
1548 :
1549 : * Return value: A new stmt that will be used to replace the sequence of
1550 : stmts that constitute the pattern, principally:
1551 : out = IFN_ABD (x, y)
1552 : out = IFN_WIDEN_ABD (x, y)
1553 : */
1554 :
1555 : static gimple *
1556 32962854 : vect_recog_abd_pattern (vec_info *vinfo,
1557 : stmt_vec_info stmt_vinfo, tree *type_out)
1558 : {
1559 32962854 : gassign *last_stmt = dyn_cast <gassign *> (STMT_VINFO_STMT (stmt_vinfo));
1560 22941783 : if (!last_stmt)
1561 : return NULL;
1562 :
1563 22941783 : tree out_type = TREE_TYPE (gimple_assign_lhs (last_stmt));
1564 :
1565 68825349 : vect_unpromoted_value unprom[2];
1566 22941783 : gassign *diff_stmt = NULL;
1567 22941783 : tree abd_in_type;
1568 22941783 : if (!vect_recog_absolute_difference (vinfo, last_stmt, &abd_in_type,
1569 : unprom, &diff_stmt))
1570 : {
1571 : /* We cannot try further without having a non-widening MINUS. */
1572 22940229 : if (!diff_stmt)
1573 : return NULL;
1574 :
1575 251 : unprom[0].op = gimple_assign_rhs1 (diff_stmt);
1576 251 : unprom[1].op = gimple_assign_rhs2 (diff_stmt);
1577 251 : abd_in_type = signed_type_for (out_type);
1578 : }
1579 :
1580 1805 : tree abd_out_type = abd_in_type;
1581 :
1582 1805 : tree vectype_in = get_vectype_for_scalar_type (vinfo, abd_in_type);
1583 1805 : if (!vectype_in)
1584 : return NULL;
1585 :
1586 1787 : internal_fn ifn = IFN_ABD;
1587 1787 : tree vectype_out = vectype_in;
1588 :
1589 1787 : if (TYPE_PRECISION (out_type) >= TYPE_PRECISION (abd_in_type) * 2
1590 1787 : && stmt_vinfo->min_output_precision >= TYPE_PRECISION (abd_in_type) * 2)
1591 : {
1592 1445 : tree mid_type
1593 1445 : = build_nonstandard_integer_type (TYPE_PRECISION (abd_in_type) * 2,
1594 1445 : TYPE_UNSIGNED (abd_in_type));
1595 1445 : tree mid_vectype = get_vectype_for_scalar_type (vinfo, mid_type);
1596 :
1597 1445 : code_helper dummy_code;
1598 1445 : int dummy_int;
1599 1445 : auto_vec<tree> dummy_vec;
1600 1445 : if (mid_vectype
1601 1445 : && supportable_widening_operation (IFN_VEC_WIDEN_ABD,
1602 : mid_vectype, vectype_in, false,
1603 : &dummy_code, &dummy_code,
1604 : &dummy_int, &dummy_vec))
1605 : {
1606 0 : ifn = IFN_VEC_WIDEN_ABD;
1607 0 : abd_out_type = mid_type;
1608 0 : vectype_out = mid_vectype;
1609 : }
1610 1445 : }
1611 :
1612 1445 : if (ifn == IFN_ABD
1613 1787 : && !direct_internal_fn_supported_p (ifn, vectype_in,
1614 : OPTIMIZE_FOR_SPEED))
1615 : return NULL;
1616 :
1617 0 : vect_pattern_detected ("vect_recog_abd_pattern", last_stmt);
1618 :
1619 0 : tree abd_oprnds[2];
1620 0 : vect_convert_inputs (vinfo, stmt_vinfo, 2, abd_oprnds,
1621 : abd_in_type, unprom, vectype_in);
1622 :
1623 0 : *type_out = get_vectype_for_scalar_type (vinfo, out_type);
1624 :
1625 0 : tree abd_result = vect_recog_temp_ssa_var (abd_out_type, NULL);
1626 0 : gcall *abd_stmt = gimple_build_call_internal (ifn, 2,
1627 : abd_oprnds[0], abd_oprnds[1]);
1628 0 : gimple_call_set_lhs (abd_stmt, abd_result);
1629 0 : gimple_set_location (abd_stmt, gimple_location (last_stmt));
1630 :
1631 0 : gimple *stmt = abd_stmt;
1632 0 : if (TYPE_PRECISION (abd_in_type) == TYPE_PRECISION (abd_out_type)
1633 0 : && TYPE_PRECISION (abd_out_type) < TYPE_PRECISION (out_type)
1634 0 : && !TYPE_UNSIGNED (abd_out_type))
1635 : {
1636 0 : tree unsign = unsigned_type_for (abd_out_type);
1637 0 : stmt = vect_convert_output (vinfo, stmt_vinfo, unsign, stmt, vectype_out);
1638 0 : vectype_out = get_vectype_for_scalar_type (vinfo, unsign);
1639 : }
1640 :
1641 0 : return vect_convert_output (vinfo, stmt_vinfo, out_type, stmt, vectype_out);
1642 : }
1643 :
1644 : /* Recognize an operation that performs ORIG_CODE on widened inputs,
1645 : so that it can be treated as though it had the form:
1646 :
1647 : A_TYPE a;
1648 : B_TYPE b;
1649 : HALF_TYPE a_cast = (HALF_TYPE) a; // possible no-op
1650 : HALF_TYPE b_cast = (HALF_TYPE) b; // possible no-op
1651 : | RES_TYPE a_extend = (RES_TYPE) a_cast; // promotion from HALF_TYPE
1652 : | RES_TYPE b_extend = (RES_TYPE) b_cast; // promotion from HALF_TYPE
1653 : | RES_TYPE res = a_extend ORIG_CODE b_extend;
1654 :
1655 : Try to replace the pattern with:
1656 :
1657 : A_TYPE a;
1658 : B_TYPE b;
1659 : HALF_TYPE a_cast = (HALF_TYPE) a; // possible no-op
1660 : HALF_TYPE b_cast = (HALF_TYPE) b; // possible no-op
1661 : | EXT_TYPE ext = a_cast WIDE_CODE b_cast;
1662 : | RES_TYPE res = (EXT_TYPE) ext; // possible no-op
1663 :
1664 : where EXT_TYPE is wider than HALF_TYPE but has the same signedness.
1665 :
1666 : SHIFT_P is true if ORIG_CODE and WIDE_CODE are shifts. NAME is the
1667 : name of the pattern being matched, for dump purposes. */
1668 :
1669 : static gimple *
1670 132707033 : vect_recog_widen_op_pattern (vec_info *vinfo,
1671 : stmt_vec_info last_stmt_info, tree *type_out,
1672 : tree_code orig_code, code_helper wide_code,
1673 : bool shift_p, const char *name)
1674 : {
1675 132707033 : gimple *last_stmt = last_stmt_info->stmt;
1676 :
1677 398121099 : vect_unpromoted_value unprom[2];
1678 132707033 : tree half_type;
1679 132707033 : if (!vect_widened_op_tree (vinfo, last_stmt_info, orig_code, orig_code,
1680 : shift_p, 2, unprom, &half_type))
1681 :
1682 : return NULL;
1683 :
1684 : /* Pattern detected. */
1685 368492 : vect_pattern_detected (name, last_stmt);
1686 :
1687 368492 : tree type = TREE_TYPE (gimple_get_lhs (last_stmt));
1688 368492 : tree itype = type;
1689 368492 : if (TYPE_PRECISION (type) != TYPE_PRECISION (half_type) * 2
1690 368492 : || TYPE_UNSIGNED (type) != TYPE_UNSIGNED (half_type))
1691 255318 : itype = build_nonstandard_integer_type (TYPE_PRECISION (half_type) * 2,
1692 255318 : TYPE_UNSIGNED (half_type));
1693 :
1694 : /* Check target support */
1695 368492 : tree vectype = get_vectype_for_scalar_type (vinfo, half_type);
1696 368492 : tree vecitype = get_vectype_for_scalar_type (vinfo, itype);
1697 368492 : tree ctype = itype;
1698 368492 : tree vecctype = vecitype;
1699 368492 : if (orig_code == MINUS_EXPR
1700 8928 : && TYPE_UNSIGNED (itype)
1701 375388 : && TYPE_PRECISION (type) > TYPE_PRECISION (itype))
1702 : {
1703 : /* Subtraction is special, even if half_type is unsigned and no matter
1704 : whether type is signed or unsigned, if type is wider than itype,
1705 : we need to sign-extend from the widening operation result to the
1706 : result type.
1707 : Consider half_type unsigned char, operand 1 0xfe, operand 2 0xff,
1708 : itype unsigned short and type either int or unsigned int.
1709 : Widened (unsigned short) 0xfe - (unsigned short) 0xff is
1710 : (unsigned short) 0xffff, but for type int we want the result -1
1711 : and for type unsigned int 0xffffffff rather than 0xffff. */
1712 3042 : ctype = build_nonstandard_integer_type (TYPE_PRECISION (itype), 0);
1713 3042 : vecctype = get_vectype_for_scalar_type (vinfo, ctype);
1714 : }
1715 :
1716 368492 : code_helper dummy_code;
1717 368492 : int dummy_int;
1718 368492 : auto_vec<tree> dummy_vec;
1719 : if (!vectype
1720 368492 : || !vecitype
1721 286300 : || !vecctype
1722 654792 : || !supportable_widening_operation (wide_code, vecitype, vectype, true,
1723 : &dummy_code, &dummy_code,
1724 : &dummy_int, &dummy_vec))
1725 : return NULL;
1726 :
1727 116395 : *type_out = get_vectype_for_scalar_type (vinfo, type);
1728 116395 : if (!*type_out)
1729 : return NULL;
1730 :
1731 116395 : tree oprnd[2];
1732 116395 : vect_convert_inputs (vinfo, last_stmt_info,
1733 : 2, oprnd, half_type, unprom, vectype);
1734 :
1735 116395 : tree var = vect_recog_temp_ssa_var (itype, NULL);
1736 116395 : gimple *pattern_stmt = vect_gimple_build (var, wide_code, oprnd[0], oprnd[1]);
1737 :
1738 116395 : if (vecctype != vecitype)
1739 0 : pattern_stmt = vect_convert_output (vinfo, last_stmt_info, ctype,
1740 : pattern_stmt, vecitype);
1741 :
1742 116395 : return vect_convert_output (vinfo, last_stmt_info,
1743 116395 : type, pattern_stmt, vecctype);
1744 368492 : }
1745 :
1746 : /* Try to detect multiplication on widened inputs, converting MULT_EXPR
1747 : to WIDEN_MULT_EXPR. See vect_recog_widen_op_pattern for details. */
1748 :
1749 : static gimple *
1750 33045989 : vect_recog_widen_mult_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
1751 : tree *type_out)
1752 : {
1753 33045989 : return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
1754 33045989 : MULT_EXPR, WIDEN_MULT_EXPR, false,
1755 33045989 : "vect_recog_widen_mult_pattern");
1756 : }
1757 :
1758 : /* Try to detect addition on widened inputs, converting PLUS_EXPR
1759 : to IFN_VEC_WIDEN_PLUS. See vect_recog_widen_op_pattern for details. */
1760 :
1761 : static gimple *
1762 33321659 : vect_recog_widen_plus_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
1763 : tree *type_out)
1764 : {
1765 33321659 : return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
1766 33321659 : PLUS_EXPR, IFN_VEC_WIDEN_PLUS,
1767 33321659 : false, "vect_recog_widen_plus_pattern");
1768 : }
1769 :
1770 : /* Try to detect subtraction on widened inputs, converting MINUS_EXPR
1771 : to IFN_VEC_WIDEN_MINUS. See vect_recog_widen_op_pattern for details. */
1772 : static gimple *
1773 33321659 : vect_recog_widen_minus_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
1774 : tree *type_out)
1775 : {
1776 33321659 : return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
1777 33321659 : MINUS_EXPR, IFN_VEC_WIDEN_MINUS,
1778 33321659 : false, "vect_recog_widen_minus_pattern");
1779 : }
1780 :
1781 : /* Try to detect abd on widened inputs, converting IFN_ABD
1782 : to IFN_VEC_WIDEN_ABD. */
1783 : static gimple *
1784 33321659 : vect_recog_widen_abd_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
1785 : tree *type_out)
1786 : {
1787 33321659 : gassign *last_stmt = dyn_cast <gassign *> (STMT_VINFO_STMT (stmt_vinfo));
1788 31331547 : if (!last_stmt || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (last_stmt)))
1789 : return NULL;
1790 :
1791 3350582 : tree last_rhs = gimple_assign_rhs1 (last_stmt);
1792 :
1793 3350582 : tree in_type = TREE_TYPE (last_rhs);
1794 3350582 : tree out_type = TREE_TYPE (gimple_assign_lhs (last_stmt));
1795 3350582 : if (!INTEGRAL_TYPE_P (in_type)
1796 2929004 : || !INTEGRAL_TYPE_P (out_type)
1797 2802244 : || TYPE_PRECISION (in_type) * 2 != TYPE_PRECISION (out_type)
1798 4023767 : || !TYPE_UNSIGNED (in_type))
1799 : return NULL;
1800 :
1801 243083 : vect_unpromoted_value unprom;
1802 243083 : tree op = vect_look_through_possible_promotion (vinfo, last_rhs, &unprom);
1803 243083 : if (!op || TYPE_PRECISION (TREE_TYPE (op)) != TYPE_PRECISION (in_type))
1804 : return NULL;
1805 :
1806 240530 : stmt_vec_info abd_pattern_vinfo = vect_get_internal_def (vinfo, op);
1807 240530 : if (!abd_pattern_vinfo)
1808 : return NULL;
1809 :
1810 215149 : gcall *abd_stmt = dyn_cast <gcall *> (STMT_VINFO_STMT (abd_pattern_vinfo));
1811 9286 : if (!abd_stmt
1812 9286 : || !gimple_call_internal_p (abd_stmt)
1813 314 : || gimple_call_internal_fn (abd_stmt) != IFN_ABD)
1814 : return NULL;
1815 :
1816 0 : tree vectype_in = get_vectype_for_scalar_type (vinfo, in_type);
1817 0 : tree vectype_out = get_vectype_for_scalar_type (vinfo, out_type);
1818 :
1819 0 : code_helper dummy_code;
1820 0 : int dummy_int;
1821 0 : auto_vec<tree> dummy_vec;
1822 0 : if (!supportable_widening_operation (IFN_VEC_WIDEN_ABD, vectype_out,
1823 : vectype_in, false,
1824 : &dummy_code, &dummy_code,
1825 : &dummy_int, &dummy_vec))
1826 : return NULL;
1827 :
1828 0 : vect_pattern_detected ("vect_recog_widen_abd_pattern", last_stmt);
1829 :
1830 0 : *type_out = vectype_out;
1831 :
1832 0 : tree abd_oprnd0 = gimple_call_arg (abd_stmt, 0);
1833 0 : tree abd_oprnd1 = gimple_call_arg (abd_stmt, 1);
1834 0 : tree widen_abd_result = vect_recog_temp_ssa_var (out_type, NULL);
1835 0 : gcall *widen_abd_stmt = gimple_build_call_internal (IFN_VEC_WIDEN_ABD, 2,
1836 : abd_oprnd0, abd_oprnd1);
1837 0 : gimple_call_set_lhs (widen_abd_stmt, widen_abd_result);
1838 0 : gimple_set_location (widen_abd_stmt, gimple_location (last_stmt));
1839 0 : return widen_abd_stmt;
1840 0 : }
1841 :
1842 : /* Function vect_recog_ctz_ffs_pattern
1843 :
1844 : Try to find the following pattern:
1845 :
1846 : TYPE1 A;
1847 : TYPE1 B;
1848 :
1849 : B = __builtin_ctz{,l,ll} (A);
1850 :
1851 : or
1852 :
1853 : B = __builtin_ffs{,l,ll} (A);
1854 :
1855 : Input:
1856 :
1857 : * STMT_VINFO: The stmt from which the pattern search begins.
1858 : here it starts with B = __builtin_* (A);
1859 :
1860 : Output:
1861 :
1862 : * TYPE_OUT: The vector type of the output of this pattern.
1863 :
1864 : * Return value: A new stmt that will be used to replace the sequence of
1865 : stmts that constitute the pattern, using clz or popcount builtins. */
1866 :
1867 : static gimple *
1868 33017533 : vect_recog_ctz_ffs_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
1869 : tree *type_out)
1870 : {
1871 33017533 : gimple *call_stmt = stmt_vinfo->stmt;
1872 33017533 : gimple *pattern_stmt;
1873 33017533 : tree rhs_oprnd, rhs_type, lhs_oprnd, lhs_type, vec_type, vec_rhs_type;
1874 33017533 : tree new_var;
1875 33017533 : internal_fn ifn = IFN_LAST, ifnnew = IFN_LAST;
1876 33017533 : bool defined_at_zero = true, defined_at_zero_new = false;
1877 33017533 : int val = 0, val_new = 0, val_cmp = 0;
1878 33017533 : int prec;
1879 33017533 : int sub = 0, add = 0;
1880 33017533 : location_t loc;
1881 :
1882 33017533 : if (!is_gimple_call (call_stmt))
1883 : return NULL;
1884 :
1885 3907020 : if (gimple_call_num_args (call_stmt) != 1
1886 3907020 : && gimple_call_num_args (call_stmt) != 2)
1887 : return NULL;
1888 :
1889 2230047 : rhs_oprnd = gimple_call_arg (call_stmt, 0);
1890 2230047 : rhs_type = TREE_TYPE (rhs_oprnd);
1891 2230047 : lhs_oprnd = gimple_call_lhs (call_stmt);
1892 2230047 : if (!lhs_oprnd)
1893 : return NULL;
1894 1068962 : lhs_type = TREE_TYPE (lhs_oprnd);
1895 1068962 : if (!INTEGRAL_TYPE_P (lhs_type)
1896 356290 : || !INTEGRAL_TYPE_P (rhs_type)
1897 54274 : || !type_has_mode_precision_p (rhs_type)
1898 1120251 : || TREE_CODE (rhs_oprnd) != SSA_NAME)
1899 : return NULL;
1900 :
1901 38017 : switch (gimple_call_combined_fn (call_stmt))
1902 : {
1903 2037 : CASE_CFN_CTZ:
1904 2037 : ifn = IFN_CTZ;
1905 2037 : if (!gimple_call_internal_p (call_stmt)
1906 2037 : || gimple_call_num_args (call_stmt) != 2)
1907 : defined_at_zero = false;
1908 : else
1909 121 : val = tree_to_shwi (gimple_call_arg (call_stmt, 1));
1910 : break;
1911 : CASE_CFN_FFS:
1912 : ifn = IFN_FFS;
1913 : break;
1914 : default:
1915 : return NULL;
1916 : }
1917 :
1918 2272 : prec = TYPE_PRECISION (rhs_type);
1919 2272 : loc = gimple_location (call_stmt);
1920 :
1921 2272 : vec_type = get_vectype_for_scalar_type (vinfo, lhs_type);
1922 2272 : if (!vec_type)
1923 : return NULL;
1924 :
1925 2266 : vec_rhs_type = get_vectype_for_scalar_type (vinfo, rhs_type);
1926 2266 : if (!vec_rhs_type)
1927 : return NULL;
1928 :
1929 : /* Do it only if the backend doesn't have ctz<vector_mode>2 or
1930 : ffs<vector_mode>2 pattern but does have clz<vector_mode>2 or
1931 : popcount<vector_mode>2. */
1932 2000 : if (!vec_type
1933 2000 : || direct_internal_fn_supported_p (ifn, vec_rhs_type,
1934 : OPTIMIZE_FOR_SPEED))
1935 : return NULL;
1936 :
1937 2000 : if (ifn == IFN_FFS
1938 2000 : && direct_internal_fn_supported_p (IFN_CTZ, vec_rhs_type,
1939 : OPTIMIZE_FOR_SPEED))
1940 : {
1941 0 : ifnnew = IFN_CTZ;
1942 0 : defined_at_zero_new
1943 0 : = CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_rhs_type),
1944 : val_new) == 2;
1945 : }
1946 2000 : else if (direct_internal_fn_supported_p (IFN_CLZ, vec_rhs_type,
1947 : OPTIMIZE_FOR_SPEED))
1948 : {
1949 166 : ifnnew = IFN_CLZ;
1950 166 : defined_at_zero_new
1951 166 : = CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_rhs_type),
1952 : val_new) == 2;
1953 : }
1954 166 : if ((ifnnew == IFN_LAST
1955 166 : || (defined_at_zero && !defined_at_zero_new))
1956 1834 : && direct_internal_fn_supported_p (IFN_POPCOUNT, vec_rhs_type,
1957 : OPTIMIZE_FOR_SPEED))
1958 : {
1959 : ifnnew = IFN_POPCOUNT;
1960 : defined_at_zero_new = true;
1961 : val_new = prec;
1962 : }
1963 1874 : if (ifnnew == IFN_LAST)
1964 : return NULL;
1965 :
1966 292 : vect_pattern_detected ("vec_recog_ctz_ffs_pattern", call_stmt);
1967 :
1968 292 : val_cmp = val_new;
1969 292 : if ((ifnnew == IFN_CLZ
1970 292 : && defined_at_zero
1971 106 : && defined_at_zero_new
1972 106 : && val == prec
1973 54 : && val_new == prec)
1974 238 : || (ifnnew == IFN_POPCOUNT && ifn == IFN_CTZ))
1975 : {
1976 137 : if (vect_is_reduction (stmt_vinfo))
1977 : return NULL;
1978 :
1979 : /* .CTZ (X) = PREC - .CLZ ((X - 1) & ~X)
1980 : .CTZ (X) = .POPCOUNT ((X - 1) & ~X). */
1981 137 : if (ifnnew == IFN_CLZ)
1982 54 : sub = prec;
1983 137 : val_cmp = prec;
1984 :
1985 137 : if (!TYPE_UNSIGNED (rhs_type))
1986 : {
1987 12 : rhs_type = unsigned_type_for (rhs_type);
1988 12 : vec_rhs_type = get_vectype_for_scalar_type (vinfo, rhs_type);
1989 12 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
1990 12 : pattern_stmt = gimple_build_assign (new_var, NOP_EXPR, rhs_oprnd);
1991 12 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
1992 : vec_rhs_type);
1993 12 : rhs_oprnd = new_var;
1994 : }
1995 :
1996 137 : tree m1 = vect_recog_temp_ssa_var (rhs_type, NULL);
1997 137 : pattern_stmt = gimple_build_assign (m1, PLUS_EXPR, rhs_oprnd,
1998 : build_int_cst (rhs_type, -1));
1999 137 : gimple_set_location (pattern_stmt, loc);
2000 137 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2001 :
2002 137 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
2003 137 : pattern_stmt = gimple_build_assign (new_var, BIT_NOT_EXPR, rhs_oprnd);
2004 137 : gimple_set_location (pattern_stmt, loc);
2005 137 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2006 137 : rhs_oprnd = new_var;
2007 :
2008 137 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
2009 137 : pattern_stmt = gimple_build_assign (new_var, BIT_AND_EXPR,
2010 : m1, rhs_oprnd);
2011 137 : gimple_set_location (pattern_stmt, loc);
2012 137 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2013 137 : rhs_oprnd = new_var;
2014 137 : }
2015 155 : else if (ifnnew == IFN_CLZ)
2016 : {
2017 112 : if (vect_is_reduction (stmt_vinfo))
2018 : return NULL;
2019 :
2020 : /* .CTZ (X) = (PREC - 1) - .CLZ (X & -X)
2021 : .FFS (X) = PREC - .CLZ (X & -X). */
2022 106 : sub = prec - (ifn == IFN_CTZ);
2023 106 : val_cmp = sub - val_new;
2024 :
2025 106 : tree neg = vect_recog_temp_ssa_var (rhs_type, NULL);
2026 106 : pattern_stmt = gimple_build_assign (neg, NEGATE_EXPR, rhs_oprnd);
2027 106 : gimple_set_location (pattern_stmt, loc);
2028 106 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2029 :
2030 106 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
2031 106 : pattern_stmt = gimple_build_assign (new_var, BIT_AND_EXPR,
2032 : rhs_oprnd, neg);
2033 106 : gimple_set_location (pattern_stmt, loc);
2034 106 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2035 106 : rhs_oprnd = new_var;
2036 : }
2037 43 : else if (ifnnew == IFN_POPCOUNT)
2038 : {
2039 43 : if (vect_is_reduction (stmt_vinfo))
2040 : return NULL;
2041 :
2042 : /* .CTZ (X) = PREC - .POPCOUNT (X | -X)
2043 : .FFS (X) = (PREC + 1) - .POPCOUNT (X | -X). */
2044 43 : sub = prec + (ifn == IFN_FFS);
2045 43 : val_cmp = sub;
2046 :
2047 43 : tree neg = vect_recog_temp_ssa_var (rhs_type, NULL);
2048 43 : pattern_stmt = gimple_build_assign (neg, NEGATE_EXPR, rhs_oprnd);
2049 43 : gimple_set_location (pattern_stmt, loc);
2050 43 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2051 :
2052 43 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
2053 43 : pattern_stmt = gimple_build_assign (new_var, BIT_IOR_EXPR,
2054 : rhs_oprnd, neg);
2055 43 : gimple_set_location (pattern_stmt, loc);
2056 43 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2057 43 : rhs_oprnd = new_var;
2058 : }
2059 0 : else if (ifnnew == IFN_CTZ)
2060 : {
2061 : /* .FFS (X) = .CTZ (X) + 1. */
2062 0 : add = 1;
2063 0 : val_cmp++;
2064 :
2065 0 : if (vect_is_reduction (stmt_vinfo)
2066 0 : && defined_at_zero
2067 0 : && (!defined_at_zero_new || val != val_cmp))
2068 : return NULL;
2069 : }
2070 :
2071 : /* Create B = .IFNNEW (A). */
2072 286 : new_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2073 286 : if ((ifnnew == IFN_CLZ || ifnnew == IFN_CTZ) && defined_at_zero_new)
2074 160 : pattern_stmt
2075 160 : = gimple_build_call_internal (ifnnew, 2, rhs_oprnd,
2076 : build_int_cst (integer_type_node,
2077 160 : val_new));
2078 : else
2079 126 : pattern_stmt = gimple_build_call_internal (ifnnew, 1, rhs_oprnd);
2080 286 : gimple_call_set_lhs (pattern_stmt, new_var);
2081 286 : gimple_set_location (pattern_stmt, loc);
2082 286 : *type_out = vec_type;
2083 :
2084 286 : if (sub)
2085 : {
2086 203 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2087 203 : tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2088 203 : pattern_stmt = gimple_build_assign (ret_var, MINUS_EXPR,
2089 203 : build_int_cst (lhs_type, sub),
2090 : new_var);
2091 203 : gimple_set_location (pattern_stmt, loc);
2092 203 : new_var = ret_var;
2093 : }
2094 83 : else if (add)
2095 : {
2096 0 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2097 0 : tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2098 0 : pattern_stmt = gimple_build_assign (ret_var, PLUS_EXPR, new_var,
2099 0 : build_int_cst (lhs_type, add));
2100 0 : gimple_set_location (pattern_stmt, loc);
2101 0 : new_var = ret_var;
2102 : }
2103 :
2104 286 : if (defined_at_zero
2105 210 : && (!defined_at_zero_new || val != val_cmp))
2106 : {
2107 43 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2108 43 : tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2109 43 : rhs_oprnd = gimple_call_arg (call_stmt, 0);
2110 43 : rhs_type = TREE_TYPE (rhs_oprnd);
2111 43 : tree cmp = vect_recog_temp_ssa_var (boolean_type_node, NULL);
2112 43 : pattern_stmt = gimple_build_assign (cmp, NE_EXPR, rhs_oprnd,
2113 : build_zero_cst (rhs_type));
2114 43 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
2115 : truth_type_for (vec_type), rhs_type);
2116 43 : pattern_stmt = gimple_build_assign (ret_var, COND_EXPR, cmp,
2117 : new_var,
2118 43 : build_int_cst (lhs_type, val));
2119 : }
2120 :
2121 286 : if (dump_enabled_p ())
2122 36 : dump_printf_loc (MSG_NOTE, vect_location,
2123 : "created pattern stmt: %G", pattern_stmt);
2124 :
2125 : return pattern_stmt;
2126 : }
2127 :
2128 : /* Function vect_recog_popcount_clz_ctz_ffs_pattern
2129 :
2130 : Try to find the following pattern:
2131 :
2132 : UTYPE1 A;
2133 : TYPE1 B;
2134 : UTYPE2 temp_in;
2135 : TYPE3 temp_out;
2136 : temp_in = (UTYPE2)A;
2137 :
2138 : temp_out = __builtin_popcount{,l,ll} (temp_in);
2139 : B = (TYPE1) temp_out;
2140 :
2141 : TYPE2 may or may not be equal to TYPE3.
2142 : i.e. TYPE2 is equal to TYPE3 for __builtin_popcount
2143 : i.e. TYPE2 is not equal to TYPE3 for __builtin_popcountll
2144 :
2145 : Input:
2146 :
2147 : * STMT_VINFO: The stmt from which the pattern search begins.
2148 : here it starts with B = (TYPE1) temp_out;
2149 :
2150 : Output:
2151 :
2152 : * TYPE_OUT: The vector type of the output of this pattern.
2153 :
2154 : * Return value: A new stmt that will be used to replace the sequence of
2155 : stmts that constitute the pattern. In this case it will be:
2156 : B = .POPCOUNT (A);
2157 :
2158 : Similarly for clz, ctz and ffs.
2159 : */
2160 :
2161 : static gimple *
2162 33017167 : vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
2163 : stmt_vec_info stmt_vinfo,
2164 : tree *type_out)
2165 : {
2166 33017167 : gassign *last_stmt = dyn_cast <gassign *> (stmt_vinfo->stmt);
2167 22991385 : gimple *call_stmt, *pattern_stmt;
2168 22991385 : tree rhs_oprnd, rhs_origin, lhs_oprnd, lhs_type, vec_type, new_var;
2169 56008552 : internal_fn ifn = IFN_LAST;
2170 56008552 : int addend = 0;
2171 :
2172 : /* Find B = (TYPE1) temp_out. */
2173 22991385 : if (!last_stmt)
2174 : return NULL;
2175 22991385 : tree_code code = gimple_assign_rhs_code (last_stmt);
2176 22991385 : if (!CONVERT_EXPR_CODE_P (code))
2177 : return NULL;
2178 :
2179 3194204 : lhs_oprnd = gimple_assign_lhs (last_stmt);
2180 3194204 : lhs_type = TREE_TYPE (lhs_oprnd);
2181 3194204 : if (!INTEGRAL_TYPE_P (lhs_type))
2182 : return NULL;
2183 :
2184 2905146 : rhs_oprnd = gimple_assign_rhs1 (last_stmt);
2185 2905146 : if (TREE_CODE (rhs_oprnd) != SSA_NAME
2186 2905146 : || !has_single_use (rhs_oprnd))
2187 : return NULL;
2188 1486881 : call_stmt = SSA_NAME_DEF_STMT (rhs_oprnd);
2189 :
2190 : /* Find temp_out = __builtin_popcount{,l,ll} (temp_in); */
2191 1486881 : if (!is_gimple_call (call_stmt))
2192 : return NULL;
2193 60369 : switch (gimple_call_combined_fn (call_stmt))
2194 : {
2195 : int val;
2196 : CASE_CFN_POPCOUNT:
2197 : ifn = IFN_POPCOUNT;
2198 : break;
2199 2427 : CASE_CFN_CLZ:
2200 2427 : ifn = IFN_CLZ;
2201 : /* Punt if call result is unsigned and defined value at zero
2202 : is negative, as the negative value doesn't extend correctly. */
2203 2427 : if (TYPE_UNSIGNED (TREE_TYPE (rhs_oprnd))
2204 0 : && gimple_call_internal_p (call_stmt)
2205 2427 : && CLZ_DEFINED_VALUE_AT_ZERO
2206 : (SCALAR_INT_TYPE_MODE (TREE_TYPE (rhs_oprnd)), val) == 2
2207 2427 : && val < 0)
2208 : return NULL;
2209 : break;
2210 753 : CASE_CFN_CTZ:
2211 753 : ifn = IFN_CTZ;
2212 : /* Punt if call result is unsigned and defined value at zero
2213 : is negative, as the negative value doesn't extend correctly. */
2214 753 : if (TYPE_UNSIGNED (TREE_TYPE (rhs_oprnd))
2215 0 : && gimple_call_internal_p (call_stmt)
2216 753 : && CTZ_DEFINED_VALUE_AT_ZERO
2217 : (SCALAR_INT_TYPE_MODE (TREE_TYPE (rhs_oprnd)), val) == 2
2218 753 : && val < 0)
2219 : return NULL;
2220 : break;
2221 57 : CASE_CFN_FFS:
2222 57 : ifn = IFN_FFS;
2223 57 : break;
2224 : default:
2225 : return NULL;
2226 : }
2227 :
2228 3591 : if (gimple_call_num_args (call_stmt) != 1
2229 3591 : && gimple_call_num_args (call_stmt) != 2)
2230 : return NULL;
2231 :
2232 3591 : rhs_oprnd = gimple_call_arg (call_stmt, 0);
2233 3591 : vect_unpromoted_value unprom_diff;
2234 3591 : rhs_origin
2235 3591 : = vect_look_through_possible_promotion (vinfo, rhs_oprnd, &unprom_diff);
2236 :
2237 3591 : if (!rhs_origin)
2238 : return NULL;
2239 :
2240 : /* Input and output of .POPCOUNT should be same-precision integer. */
2241 3581 : if (TYPE_PRECISION (unprom_diff.type) != TYPE_PRECISION (lhs_type))
2242 : return NULL;
2243 :
2244 1831 : vec_type = get_vectype_for_scalar_type (vinfo, lhs_type);
2245 1831 : if (!vec_type || !VECTOR_MODE_P (TYPE_MODE (vec_type)))
2246 : return NULL;
2247 :
2248 : /* Also A should be unsigned or same precision as temp_in, otherwise
2249 : different builtins/internal functions have different behaviors. */
2250 1699 : if (TYPE_PRECISION (unprom_diff.type)
2251 1699 : != TYPE_PRECISION (TREE_TYPE (rhs_oprnd)))
2252 319 : switch (ifn)
2253 : {
2254 147 : case IFN_POPCOUNT:
2255 : /* For popcount require zero extension, which doesn't add any
2256 : further bits to the count. */
2257 147 : if (!TYPE_UNSIGNED (unprom_diff.type))
2258 : return NULL;
2259 : break;
2260 109 : case IFN_CLZ:
2261 : /* clzll (x) == clz (x) + 32 for unsigned x != 0, so ok
2262 : if it is undefined at zero or if it matches also for the
2263 : defined value there. */
2264 109 : if (!TYPE_UNSIGNED (unprom_diff.type))
2265 : return NULL;
2266 109 : if (!type_has_mode_precision_p (lhs_type)
2267 109 : || !type_has_mode_precision_p (TREE_TYPE (rhs_oprnd)))
2268 : return NULL;
2269 109 : addend = (TYPE_PRECISION (TREE_TYPE (rhs_oprnd))
2270 109 : - TYPE_PRECISION (lhs_type));
2271 109 : if (gimple_call_internal_p (call_stmt)
2272 109 : && gimple_call_num_args (call_stmt) == 2)
2273 : {
2274 0 : int val1, val2;
2275 0 : val1 = tree_to_shwi (gimple_call_arg (call_stmt, 1));
2276 0 : int d2
2277 0 : = CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val2);
2278 0 : if (d2 != 2 || val1 != val2 + addend)
2279 : return NULL;
2280 : }
2281 : break;
2282 43 : case IFN_CTZ:
2283 : /* ctzll (x) == ctz (x) for unsigned or signed x != 0, so ok
2284 : if it is undefined at zero or if it matches also for the
2285 : defined value there. */
2286 43 : if (gimple_call_internal_p (call_stmt)
2287 43 : && gimple_call_num_args (call_stmt) == 2)
2288 : {
2289 0 : int val1, val2;
2290 0 : val1 = tree_to_shwi (gimple_call_arg (call_stmt, 1));
2291 0 : int d2
2292 0 : = CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val2);
2293 0 : if (d2 != 2 || val1 != val2)
2294 : return NULL;
2295 : }
2296 : break;
2297 : case IFN_FFS:
2298 : /* ffsll (x) == ffs (x) for unsigned or signed x. */
2299 : break;
2300 : default:
2301 : gcc_unreachable ();
2302 : }
2303 :
2304 : /* Do it only if the backend has popcount<vector_mode>2 etc. pattern. */
2305 1699 : bool supported
2306 1699 : = direct_internal_fn_supported_p (ifn, vec_type, OPTIMIZE_FOR_SPEED);
2307 : /* We cannot rely on the reduction check in vect_recog_ctz_ffs_pattern
2308 : because we do not set up the temporary pattern stmt finally for
2309 : the recursive analysis. */
2310 1699 : if (!supported && vect_is_reduction (stmt_vinfo))
2311 : return NULL;
2312 1683 : if (!supported)
2313 1506 : switch (ifn)
2314 : {
2315 : case IFN_POPCOUNT:
2316 : case IFN_CLZ:
2317 : return NULL;
2318 57 : case IFN_FFS:
2319 : /* vect_recog_ctz_ffs_pattern can implement ffs using ctz. */
2320 57 : if (direct_internal_fn_supported_p (IFN_CTZ, vec_type,
2321 : OPTIMIZE_FOR_SPEED))
2322 : break;
2323 : /* FALLTHRU */
2324 540 : case IFN_CTZ:
2325 : /* vect_recog_ctz_ffs_pattern can implement ffs or ctz using
2326 : clz or popcount. */
2327 540 : if (direct_internal_fn_supported_p (IFN_CLZ, vec_type,
2328 : OPTIMIZE_FOR_SPEED))
2329 : break;
2330 480 : if (direct_internal_fn_supported_p (IFN_POPCOUNT, vec_type,
2331 : OPTIMIZE_FOR_SPEED))
2332 : break;
2333 : return NULL;
2334 : default:
2335 : gcc_unreachable ();
2336 : }
2337 :
2338 318 : vect_pattern_detected ("vec_recog_popcount_clz_ctz_ffs_pattern",
2339 : call_stmt);
2340 :
2341 318 : tree orig_lhs_type = lhs_type;
2342 318 : gimple *cast_stmt = NULL;
2343 318 : if (!type_has_mode_precision_p (lhs_type))
2344 : {
2345 0 : lhs_type = TREE_TYPE (vec_type);
2346 0 : cast_stmt
2347 0 : = gimple_build_assign (vect_recog_temp_ssa_var (lhs_type, NULL),
2348 : NOP_EXPR, unprom_diff.op);
2349 0 : unprom_diff.op = gimple_assign_lhs (cast_stmt);
2350 : }
2351 :
2352 : /* Create B = .POPCOUNT (A). */
2353 318 : new_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2354 318 : tree arg2 = NULL_TREE;
2355 318 : int val;
2356 318 : if (ifn == IFN_CLZ
2357 368 : && CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type),
2358 : val) == 2)
2359 48 : arg2 = build_int_cst (integer_type_node, val);
2360 270 : else if (ifn == IFN_CTZ
2361 363 : && CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type),
2362 : val) == 2)
2363 93 : arg2 = build_int_cst (integer_type_node, val);
2364 141 : if (arg2)
2365 141 : pattern_stmt = gimple_build_call_internal (ifn, 2, unprom_diff.op, arg2);
2366 : else
2367 177 : pattern_stmt = gimple_build_call_internal (ifn, 1, unprom_diff.op);
2368 318 : gimple_call_set_lhs (pattern_stmt, new_var);
2369 318 : gimple_set_location (pattern_stmt, gimple_location (last_stmt));
2370 318 : *type_out = vec_type;
2371 :
2372 318 : if (dump_enabled_p ())
2373 24 : dump_printf_loc (MSG_NOTE, vect_location,
2374 : "created pattern stmt: %G", pattern_stmt);
2375 :
2376 318 : if (addend)
2377 : {
2378 12 : gcc_assert (supported);
2379 12 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2380 12 : tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2381 12 : pattern_stmt = gimple_build_assign (ret_var, PLUS_EXPR, new_var,
2382 12 : build_int_cst (lhs_type, addend));
2383 : }
2384 306 : else if (!supported)
2385 : {
2386 141 : stmt_vec_info new_stmt_info = vinfo->add_stmt (pattern_stmt);
2387 141 : STMT_VINFO_VECTYPE (new_stmt_info) = vec_type;
2388 141 : pattern_stmt
2389 141 : = vect_recog_ctz_ffs_pattern (vinfo, new_stmt_info, type_out);
2390 141 : if (pattern_stmt == NULL)
2391 : return NULL;
2392 141 : if (cast_stmt)
2393 0 : append_pattern_def_seq (vinfo, stmt_vinfo, cast_stmt, vec_type);
2394 141 : if (gimple_seq seq = STMT_VINFO_PATTERN_DEF_SEQ (new_stmt_info))
2395 : {
2396 141 : gimple_seq *pseq = &STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo);
2397 141 : gimple_seq_add_seq_without_update (pseq, seq);
2398 : }
2399 : }
2400 165 : else if (cast_stmt)
2401 0 : append_pattern_def_seq (vinfo, stmt_vinfo, cast_stmt, vec_type);
2402 :
2403 153 : if (cast_stmt)
2404 : {
2405 0 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2406 0 : tree ret_var = vect_recog_temp_ssa_var (orig_lhs_type, NULL);
2407 0 : pattern_stmt = gimple_build_assign (ret_var, NOP_EXPR,
2408 : gimple_get_lhs (pattern_stmt));
2409 : }
2410 : return pattern_stmt;
2411 : }
2412 :
2413 : /* Function vect_recog_pow_pattern
2414 :
2415 : Try to find the following pattern:
2416 :
2417 : x = POW (y, N);
2418 :
2419 : with POW being one of pow, powf, powi, powif and N being
2420 : either 2 or 0.5.
2421 :
2422 : Input:
2423 :
2424 : * STMT_VINFO: The stmt from which the pattern search begins.
2425 :
2426 : Output:
2427 :
2428 : * TYPE_OUT: The type of the output of this pattern.
2429 :
2430 : * Return value: A new stmt that will be used to replace the sequence of
2431 : stmts that constitute the pattern. In this case it will be:
2432 : x = x * x
2433 : or
2434 : x = sqrt (x)
2435 : */
2436 :
2437 : static gimple *
2438 33017184 : vect_recog_pow_pattern (vec_info *vinfo,
2439 : stmt_vec_info stmt_vinfo, tree *type_out)
2440 : {
2441 33017184 : gimple *last_stmt = stmt_vinfo->stmt;
2442 33017184 : tree base, exp;
2443 33017184 : gimple *stmt;
2444 33017184 : tree var;
2445 :
2446 33017184 : if (!is_gimple_call (last_stmt) || gimple_call_lhs (last_stmt) == NULL)
2447 : return NULL;
2448 :
2449 1568476 : switch (gimple_call_combined_fn (last_stmt))
2450 : {
2451 530 : CASE_CFN_POW:
2452 530 : CASE_CFN_POWI:
2453 530 : break;
2454 :
2455 : default:
2456 : return NULL;
2457 : }
2458 :
2459 530 : base = gimple_call_arg (last_stmt, 0);
2460 530 : exp = gimple_call_arg (last_stmt, 1);
2461 530 : if (TREE_CODE (exp) != REAL_CST
2462 475 : && TREE_CODE (exp) != INTEGER_CST)
2463 : {
2464 475 : if (flag_unsafe_math_optimizations
2465 37 : && TREE_CODE (base) == REAL_CST
2466 478 : && gimple_call_builtin_p (last_stmt, BUILT_IN_NORMAL))
2467 : {
2468 3 : combined_fn log_cfn;
2469 3 : built_in_function exp_bfn;
2470 3 : switch (DECL_FUNCTION_CODE (gimple_call_fndecl (last_stmt)))
2471 : {
2472 : case BUILT_IN_POW:
2473 : log_cfn = CFN_BUILT_IN_LOG;
2474 : exp_bfn = BUILT_IN_EXP;
2475 : break;
2476 0 : case BUILT_IN_POWF:
2477 0 : log_cfn = CFN_BUILT_IN_LOGF;
2478 0 : exp_bfn = BUILT_IN_EXPF;
2479 0 : break;
2480 0 : case BUILT_IN_POWL:
2481 0 : log_cfn = CFN_BUILT_IN_LOGL;
2482 0 : exp_bfn = BUILT_IN_EXPL;
2483 0 : break;
2484 : default:
2485 : return NULL;
2486 : }
2487 3 : tree logc = fold_const_call (log_cfn, TREE_TYPE (base), base);
2488 3 : tree exp_decl = builtin_decl_implicit (exp_bfn);
2489 : /* Optimize pow (C, x) as exp (log (C) * x). Normally match.pd
2490 : does that, but if C is a power of 2, we want to use
2491 : exp2 (log2 (C) * x) in the non-vectorized version, but for
2492 : vectorization we don't have vectorized exp2. */
2493 3 : if (logc
2494 3 : && TREE_CODE (logc) == REAL_CST
2495 3 : && exp_decl
2496 6 : && lookup_attribute ("omp declare simd",
2497 3 : DECL_ATTRIBUTES (exp_decl)))
2498 : {
2499 3 : cgraph_node *node = cgraph_node::get_create (exp_decl);
2500 3 : if (node->simd_clones == NULL)
2501 : {
2502 2 : if (targetm.simd_clone.compute_vecsize_and_simdlen == NULL
2503 2 : || node->definition)
2504 : return NULL;
2505 2 : expand_simd_clones (node);
2506 2 : if (node->simd_clones == NULL)
2507 : return NULL;
2508 : }
2509 3 : *type_out = get_vectype_for_scalar_type (vinfo, TREE_TYPE (base));
2510 3 : if (!*type_out)
2511 : return NULL;
2512 3 : tree def = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
2513 3 : gimple *g = gimple_build_assign (def, MULT_EXPR, exp, logc);
2514 3 : append_pattern_def_seq (vinfo, stmt_vinfo, g);
2515 3 : tree res = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
2516 3 : g = gimple_build_call (exp_decl, 1, def);
2517 3 : gimple_call_set_lhs (g, res);
2518 3 : return g;
2519 : }
2520 : }
2521 :
2522 : return NULL;
2523 : }
2524 :
2525 : /* We now have a pow or powi builtin function call with a constant
2526 : exponent. */
2527 :
2528 : /* Catch squaring. */
2529 55 : if ((tree_fits_shwi_p (exp)
2530 0 : && tree_to_shwi (exp) == 2)
2531 55 : || (TREE_CODE (exp) == REAL_CST
2532 55 : && real_equal (&TREE_REAL_CST (exp), &dconst2)))
2533 : {
2534 11 : if (!vect_supportable_direct_optab_p (vinfo, TREE_TYPE (base), MULT_EXPR,
2535 11 : TREE_TYPE (base), type_out))
2536 : return NULL;
2537 :
2538 9 : var = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
2539 9 : stmt = gimple_build_assign (var, MULT_EXPR, base, base);
2540 9 : return stmt;
2541 : }
2542 :
2543 : /* Catch square root. */
2544 44 : if (TREE_CODE (exp) == REAL_CST
2545 44 : && real_equal (&TREE_REAL_CST (exp), &dconsthalf))
2546 : {
2547 10 : *type_out = get_vectype_for_scalar_type (vinfo, TREE_TYPE (base));
2548 10 : if (*type_out
2549 10 : && direct_internal_fn_supported_p (IFN_SQRT, *type_out,
2550 : OPTIMIZE_FOR_SPEED))
2551 : {
2552 8 : gcall *stmt = gimple_build_call_internal (IFN_SQRT, 1, base);
2553 8 : var = vect_recog_temp_ssa_var (TREE_TYPE (base), stmt);
2554 8 : gimple_call_set_lhs (stmt, var);
2555 8 : gimple_call_set_nothrow (stmt, true);
2556 8 : return stmt;
2557 : }
2558 : }
2559 :
2560 : return NULL;
2561 : }
2562 :
2563 :
2564 : /* Function vect_recog_widen_sum_pattern
2565 :
2566 : Try to find the following pattern:
2567 :
2568 : type x_t;
2569 : TYPE x_T, sum = init;
2570 : loop:
2571 : sum_0 = phi <init, sum_1>
2572 : S1 x_t = *p;
2573 : S2 x_T = (TYPE) x_t;
2574 : S3 sum_1 = x_T + sum_0;
2575 :
2576 : where type 'TYPE' is at least double the size of type 'type', i.e - we're
2577 : summing elements of type 'type' into an accumulator of type 'TYPE'. This is
2578 : a special case of a reduction computation.
2579 :
2580 : Input:
2581 :
2582 : * STMT_VINFO: The stmt from which the pattern search begins. In the example,
2583 : when this function is called with S3, the pattern {S2,S3} will be detected.
2584 :
2585 : Output:
2586 :
2587 : * TYPE_OUT: The type of the output of this pattern.
2588 :
2589 : * Return value: A new stmt that will be used to replace the sequence of
2590 : stmts that constitute the pattern. In this case it will be:
2591 : WIDEN_SUM <x_t, sum_0>
2592 :
2593 : Note: The widening-sum idiom is a widening reduction pattern that is
2594 : vectorized without preserving all the intermediate results. It
2595 : produces only N/2 (widened) results (by summing up pairs of
2596 : intermediate results) rather than all N results. Therefore, we
2597 : cannot allow this pattern when we want to get all the results and in
2598 : the correct order (as is the case when this computation is in an
2599 : inner-loop nested in an outer-loop that us being vectorized). */
2600 :
2601 : static gimple *
2602 33017184 : vect_recog_widen_sum_pattern (vec_info *vinfo,
2603 : stmt_vec_info stmt_vinfo, tree *type_out)
2604 : {
2605 33017184 : gimple *last_stmt = stmt_vinfo->stmt;
2606 33017184 : tree oprnd0, oprnd1;
2607 33017184 : tree type;
2608 33017184 : gimple *pattern_stmt;
2609 33017184 : tree var;
2610 :
2611 : /* Look for the following pattern
2612 : DX = (TYPE) X;
2613 : sum_1 = DX + sum_0;
2614 : In which DX is at least double the size of X, and sum_1 has been
2615 : recognized as a reduction variable.
2616 : */
2617 :
2618 : /* Starting from LAST_STMT, follow the defs of its uses in search
2619 : of the above pattern. */
2620 :
2621 33017184 : if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, PLUS_EXPR,
2622 : &oprnd0, &oprnd1)
2623 52868 : || TREE_CODE (oprnd0) != SSA_NAME
2624 33069783 : || !vinfo->lookup_def (oprnd0))
2625 : return NULL;
2626 :
2627 52522 : type = TREE_TYPE (gimple_get_lhs (last_stmt));
2628 :
2629 : /* So far so good. Since last_stmt was detected as a (summation) reduction,
2630 : we know that oprnd1 is the reduction variable (defined by a loop-header
2631 : phi), and oprnd0 is an ssa-name defined by a stmt in the loop body.
2632 : Left to check that oprnd0 is defined by a cast from type 'type' to type
2633 : 'TYPE'. */
2634 :
2635 52522 : vect_unpromoted_value unprom0;
2636 52522 : if (!vect_look_through_possible_promotion (vinfo, oprnd0, &unprom0)
2637 52522 : || TYPE_PRECISION (unprom0.type) * 2 > TYPE_PRECISION (type))
2638 : return NULL;
2639 :
2640 2337 : vect_pattern_detected ("vect_recog_widen_sum_pattern", last_stmt);
2641 :
2642 : /* If the exact input type is not supported, widen it to an intermediate
2643 : type before adding it to the accumulator. Start with the narrowest
2644 : type in order to retain the largest vectorization factor. */
2645 2337 : tree input_type = unprom0.type;
2646 2337 : tree input_vectype = NULL_TREE;
2647 :
2648 4332 : while (!vect_supportable_conv_optab_p (vinfo, type, WIDEN_SUM_EXPR,
2649 : input_type, type_out, &input_vectype))
2650 : {
2651 4332 : unsigned int precision
2652 4332 : = vect_element_precision (TYPE_PRECISION (input_type) + 1);
2653 :
2654 4332 : if (precision > TYPE_PRECISION (type) / 2)
2655 : return NULL;
2656 :
2657 1995 : input_type = build_nonstandard_integer_type
2658 1995 : (precision, TYPE_UNSIGNED (unprom0.type));
2659 : }
2660 :
2661 0 : tree input = vect_convert_input (vinfo, stmt_vinfo, input_type, &unprom0,
2662 : input_vectype);
2663 :
2664 0 : var = vect_recog_temp_ssa_var (type, NULL);
2665 0 : pattern_stmt = gimple_build_assign (var, WIDEN_SUM_EXPR, input, oprnd1);
2666 :
2667 0 : return pattern_stmt;
2668 : }
2669 :
2670 : /* Function vect_recog_bitfield_ref_pattern
2671 :
2672 : Try to find the following pattern:
2673 :
2674 : bf_value = BIT_FIELD_REF (container, bitsize, bitpos);
2675 : result = (type_out) bf_value;
2676 :
2677 : or
2678 :
2679 : if (BIT_FIELD_REF (container, bitsize, bitpos) `cmp` <constant>)
2680 :
2681 : where type_out is a non-bitfield type, that is to say, it's precision matches
2682 : 2^(TYPE_SIZE(type_out) - (TYPE_UNSIGNED (type_out) ? 1 : 2)).
2683 :
2684 : Input:
2685 :
2686 : * STMT_VINFO: The stmt from which the pattern search begins.
2687 : here it starts with:
2688 : result = (type_out) bf_value;
2689 :
2690 : or
2691 :
2692 : if (BIT_FIELD_REF (container, bitsize, bitpos) `cmp` <constant>)
2693 :
2694 : Output:
2695 :
2696 : * TYPE_OUT: The vector type of the output of this pattern.
2697 :
2698 : * Return value: A new stmt that will be used to replace the sequence of
2699 : stmts that constitute the pattern. If the precision of type_out is bigger
2700 : than the precision type of _1 we perform the widening before the shifting,
2701 : since the new precision will be large enough to shift the value and moving
2702 : widening operations up the statement chain enables the generation of
2703 : widening loads. If we are widening and the operation after the pattern is
2704 : an addition then we mask first and shift later, to enable the generation of
2705 : shifting adds. In the case of narrowing we will always mask first, shift
2706 : last and then perform a narrowing operation. This will enable the
2707 : generation of narrowing shifts.
2708 :
2709 : Widening with mask first, shift later:
2710 : container = (type_out) container;
2711 : masked = container & (((1 << bitsize) - 1) << bitpos);
2712 : result = masked >> bitpos;
2713 :
2714 : Widening with shift first, mask last:
2715 : container = (type_out) container;
2716 : shifted = container >> bitpos;
2717 : result = shifted & ((1 << bitsize) - 1);
2718 :
2719 : Narrowing:
2720 : masked = container & (((1 << bitsize) - 1) << bitpos);
2721 : result = masked >> bitpos;
2722 : result = (type_out) result;
2723 :
2724 : If the bitfield is signed and it's wider than type_out, we need to
2725 : keep the result sign-extended:
2726 : container = (type) container;
2727 : masked = container << (prec - bitsize - bitpos);
2728 : result = (type_out) (masked >> (prec - bitsize));
2729 :
2730 : Here type is the signed variant of the wider of type_out and the type
2731 : of container.
2732 :
2733 : The shifting is always optional depending on whether bitpos != 0.
2734 :
2735 : When the original bitfield was inside a gcond then an new gcond is also
2736 : generated with the newly `result` as the operand to the comparison.
2737 :
2738 : */
2739 :
2740 : static gimple *
2741 32959348 : vect_recog_bitfield_ref_pattern (vec_info *vinfo, stmt_vec_info stmt_info,
2742 : tree *type_out)
2743 : {
2744 32959348 : gimple *bf_stmt = NULL;
2745 32959348 : tree lhs = NULL_TREE;
2746 32959348 : tree ret_type = NULL_TREE;
2747 32959348 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
2748 32959348 : if (gcond *cond_stmt = dyn_cast <gcond *> (stmt))
2749 : {
2750 5505292 : tree op = gimple_cond_lhs (cond_stmt);
2751 5505292 : if (TREE_CODE (op) != SSA_NAME)
2752 : return NULL;
2753 5504980 : bf_stmt = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (op));
2754 5504980 : if (TREE_CODE (gimple_cond_rhs (cond_stmt)) != INTEGER_CST)
2755 : return NULL;
2756 : }
2757 27454056 : else if (is_gimple_assign (stmt)
2758 22937658 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
2759 30567296 : && TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME)
2760 : {
2761 3070700 : gimple *second_stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmt));
2762 3070700 : bf_stmt = dyn_cast <gassign *> (second_stmt);
2763 3070700 : lhs = gimple_assign_lhs (stmt);
2764 3070700 : ret_type = TREE_TYPE (lhs);
2765 : }
2766 :
2767 6633959 : if (!bf_stmt
2768 6633959 : || gimple_assign_rhs_code (bf_stmt) != BIT_FIELD_REF)
2769 : return NULL;
2770 :
2771 17332 : tree bf_ref = gimple_assign_rhs1 (bf_stmt);
2772 17332 : tree container = TREE_OPERAND (bf_ref, 0);
2773 17332 : ret_type = ret_type ? ret_type : TREE_TYPE (container);
2774 :
2775 17332 : if (!bit_field_offset (bf_ref).is_constant ()
2776 17332 : || !bit_field_size (bf_ref).is_constant ()
2777 17332 : || !tree_fits_uhwi_p (TYPE_SIZE (TREE_TYPE (container))))
2778 : return NULL;
2779 :
2780 34286 : if (!INTEGRAL_TYPE_P (TREE_TYPE (bf_ref))
2781 17330 : || !INTEGRAL_TYPE_P (TREE_TYPE (container))
2782 19505 : || TYPE_MODE (TREE_TYPE (container)) == E_BLKmode)
2783 : return NULL;
2784 :
2785 2173 : gimple *use_stmt, *pattern_stmt;
2786 2173 : use_operand_p use_p;
2787 2173 : bool shift_first = true;
2788 2173 : tree container_type = TREE_TYPE (container);
2789 2173 : tree vectype = get_vectype_for_scalar_type (vinfo, container_type);
2790 :
2791 : /* Calculate shift_n before the adjustments for widening loads, otherwise
2792 : the container may change and we have to consider offset change for
2793 : widening loads on big endianness. The shift_n calculated here can be
2794 : independent of widening. */
2795 2173 : unsigned HOST_WIDE_INT shift_n = bit_field_offset (bf_ref).to_constant ();
2796 2173 : unsigned HOST_WIDE_INT mask_width = bit_field_size (bf_ref).to_constant ();
2797 2173 : unsigned HOST_WIDE_INT prec = tree_to_uhwi (TYPE_SIZE (container_type));
2798 2173 : if (BYTES_BIG_ENDIAN)
2799 : shift_n = prec - shift_n - mask_width;
2800 :
2801 2173 : bool ref_sext = (!TYPE_UNSIGNED (TREE_TYPE (bf_ref)) &&
2802 1400 : TYPE_PRECISION (ret_type) > mask_width);
2803 2173 : bool load_widen = (TYPE_PRECISION (TREE_TYPE (container)) <
2804 2173 : TYPE_PRECISION (ret_type));
2805 :
2806 : /* We move the conversion earlier if the loaded type is smaller than the
2807 : return type to enable the use of widening loads. And if we need a
2808 : sign extension, we need to convert the loaded value early to a signed
2809 : type as well. */
2810 2173 : if (ref_sext || load_widen)
2811 : {
2812 947 : tree type = load_widen ? ret_type : container_type;
2813 947 : if (ref_sext)
2814 908 : type = gimple_signed_type (type);
2815 947 : pattern_stmt = gimple_build_assign (vect_recog_temp_ssa_var (type),
2816 : NOP_EXPR, container);
2817 947 : container = gimple_get_lhs (pattern_stmt);
2818 947 : container_type = TREE_TYPE (container);
2819 947 : prec = tree_to_uhwi (TYPE_SIZE (container_type));
2820 947 : vectype = get_vectype_for_scalar_type (vinfo, container_type);
2821 947 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2822 : }
2823 1226 : else if (!useless_type_conversion_p (TREE_TYPE (container), ret_type))
2824 : /* If we are doing the conversion last then also delay the shift as we may
2825 : be able to combine the shift and conversion in certain cases. */
2826 : shift_first = false;
2827 :
2828 : /* If the only use of the result of this BIT_FIELD_REF + CONVERT is a
2829 : PLUS_EXPR then do the shift last as some targets can combine the shift and
2830 : add into a single instruction. */
2831 1416 : if (lhs && !is_pattern_stmt_p (stmt_info)
2832 3589 : && single_imm_use (lhs, &use_p, &use_stmt))
2833 : {
2834 1049 : if (gimple_code (use_stmt) == GIMPLE_ASSIGN
2835 1049 : && gimple_assign_rhs_code (use_stmt) == PLUS_EXPR)
2836 : shift_first = false;
2837 : }
2838 :
2839 : /* If we don't have to shift we only generate the mask, so just fix the
2840 : code-path to shift_first. */
2841 2173 : if (shift_n == 0)
2842 763 : shift_first = true;
2843 :
2844 2173 : tree result;
2845 2173 : if (shift_first && !ref_sext)
2846 : {
2847 513 : tree shifted = container;
2848 513 : if (shift_n)
2849 : {
2850 65 : pattern_stmt
2851 65 : = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
2852 : RSHIFT_EXPR, container,
2853 65 : build_int_cst (sizetype, shift_n));
2854 65 : shifted = gimple_assign_lhs (pattern_stmt);
2855 65 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2856 : }
2857 :
2858 513 : tree mask = wide_int_to_tree (container_type,
2859 513 : wi::mask (mask_width, false, prec));
2860 :
2861 513 : pattern_stmt
2862 513 : = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
2863 : BIT_AND_EXPR, shifted, mask);
2864 513 : result = gimple_assign_lhs (pattern_stmt);
2865 : }
2866 : else
2867 : {
2868 1660 : tree temp = vect_recog_temp_ssa_var (container_type);
2869 1660 : if (!ref_sext)
2870 : {
2871 752 : tree mask = wide_int_to_tree (container_type,
2872 752 : wi::shifted_mask (shift_n,
2873 : mask_width,
2874 : false, prec));
2875 752 : pattern_stmt = gimple_build_assign (temp, BIT_AND_EXPR,
2876 : container, mask);
2877 : }
2878 : else
2879 : {
2880 908 : HOST_WIDE_INT shl = prec - shift_n - mask_width;
2881 908 : shift_n += shl;
2882 908 : pattern_stmt = gimple_build_assign (temp, LSHIFT_EXPR,
2883 : container,
2884 : build_int_cst (sizetype,
2885 908 : shl));
2886 : }
2887 :
2888 1660 : tree masked = gimple_assign_lhs (pattern_stmt);
2889 1660 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2890 1660 : pattern_stmt
2891 1660 : = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
2892 : RSHIFT_EXPR, masked,
2893 1660 : build_int_cst (sizetype, shift_n));
2894 1660 : result = gimple_assign_lhs (pattern_stmt);
2895 : }
2896 :
2897 2173 : if (!useless_type_conversion_p (TREE_TYPE (result), ret_type))
2898 : {
2899 1444 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2900 1444 : pattern_stmt
2901 1444 : = gimple_build_assign (vect_recog_temp_ssa_var (ret_type),
2902 : NOP_EXPR, result);
2903 : }
2904 :
2905 2173 : if (!lhs)
2906 : {
2907 757 : if (!vectype)
2908 : return NULL;
2909 :
2910 619 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2911 619 : vectype = truth_type_for (vectype);
2912 :
2913 : /* FIXME: This part extracts the boolean value out of the bitfield in the
2914 : same way as vect_recog_gcond_pattern does. However because
2915 : patterns cannot match the same root twice, when we handle and
2916 : lower the bitfield in the gcond, vect_recog_gcond_pattern can't
2917 : apply anymore. We should really fix it so that we don't need to
2918 : duplicate transformations like these. */
2919 619 : tree new_lhs = vect_recog_temp_ssa_var (boolean_type_node, NULL);
2920 619 : gcond *cond_stmt = dyn_cast <gcond *> (stmt_info->stmt);
2921 619 : tree cond_cst = gimple_cond_rhs (cond_stmt);
2922 619 : gimple *new_stmt
2923 619 : = gimple_build_assign (new_lhs, gimple_cond_code (cond_stmt),
2924 : gimple_get_lhs (pattern_stmt),
2925 : fold_convert (container_type, cond_cst));
2926 619 : append_pattern_def_seq (vinfo, stmt_info, new_stmt, vectype, container_type);
2927 619 : pattern_stmt
2928 619 : = gimple_build_cond (NE_EXPR, new_lhs,
2929 619 : build_zero_cst (TREE_TYPE (new_lhs)),
2930 : NULL_TREE, NULL_TREE);
2931 : }
2932 :
2933 2035 : *type_out = STMT_VINFO_VECTYPE (stmt_info);
2934 2035 : vect_pattern_detected ("bitfield_ref pattern", stmt_info->stmt);
2935 :
2936 2035 : return pattern_stmt;
2937 : }
2938 :
2939 : /* Function vect_recog_bit_insert_pattern
2940 :
2941 : Try to find the following pattern:
2942 :
2943 : written = BIT_INSERT_EXPR (container, value, bitpos);
2944 :
2945 : Input:
2946 :
2947 : * STMT_VINFO: The stmt we want to replace.
2948 :
2949 : Output:
2950 :
2951 : * TYPE_OUT: The vector type of the output of this pattern.
2952 :
2953 : * Return value: A new stmt that will be used to replace the sequence of
2954 : stmts that constitute the pattern. In this case it will be:
2955 : value = (container_type) value; // Make sure
2956 : shifted = value << bitpos; // Shift value into place
2957 : masked = shifted & (mask << bitpos); // Mask off the non-relevant bits in
2958 : // the 'to-write value'.
2959 : cleared = container & ~(mask << bitpos); // Clearing the bits we want to
2960 : // write to from the value we want
2961 : // to write to.
2962 : written = cleared | masked; // Write bits.
2963 :
2964 :
2965 : where mask = ((1 << TYPE_PRECISION (value)) - 1), a mask to keep the number of
2966 : bits corresponding to the real size of the bitfield value we are writing to.
2967 : The shifting is always optional depending on whether bitpos != 0.
2968 :
2969 : */
2970 :
2971 : static gimple *
2972 32962398 : vect_recog_bit_insert_pattern (vec_info *vinfo, stmt_vec_info stmt_info,
2973 : tree *type_out)
2974 : {
2975 32962398 : gassign *bf_stmt = dyn_cast <gassign *> (stmt_info->stmt);
2976 30590827 : if (!bf_stmt || gimple_assign_rhs_code (bf_stmt) != BIT_INSERT_EXPR)
2977 : return NULL;
2978 :
2979 690 : tree container = gimple_assign_rhs1 (bf_stmt);
2980 690 : tree value = gimple_assign_rhs2 (bf_stmt);
2981 690 : tree shift = gimple_assign_rhs3 (bf_stmt);
2982 :
2983 690 : tree bf_type = TREE_TYPE (value);
2984 690 : tree container_type = TREE_TYPE (container);
2985 :
2986 690 : if (!INTEGRAL_TYPE_P (container_type)
2987 690 : || !tree_fits_uhwi_p (TYPE_SIZE (container_type)))
2988 : return NULL;
2989 :
2990 508 : gimple *pattern_stmt;
2991 :
2992 508 : vect_unpromoted_value unprom;
2993 508 : unprom.set_op (value, vect_internal_def);
2994 508 : value = vect_convert_input (vinfo, stmt_info, container_type, &unprom,
2995 : get_vectype_for_scalar_type (vinfo,
2996 : container_type));
2997 :
2998 508 : unsigned HOST_WIDE_INT mask_width = TYPE_PRECISION (bf_type);
2999 508 : unsigned HOST_WIDE_INT prec = tree_to_uhwi (TYPE_SIZE (container_type));
3000 508 : unsigned HOST_WIDE_INT shift_n = tree_to_uhwi (shift);
3001 508 : if (BYTES_BIG_ENDIAN)
3002 : {
3003 : shift_n = prec - shift_n - mask_width;
3004 : shift = build_int_cst (TREE_TYPE (shift), shift_n);
3005 : }
3006 :
3007 508 : if (!useless_type_conversion_p (TREE_TYPE (value), container_type))
3008 : {
3009 0 : pattern_stmt =
3010 0 : gimple_build_assign (vect_recog_temp_ssa_var (container_type),
3011 : NOP_EXPR, value);
3012 0 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt);
3013 0 : value = gimple_get_lhs (pattern_stmt);
3014 : }
3015 :
3016 : /* Shift VALUE into place. */
3017 508 : tree shifted = value;
3018 508 : if (shift_n)
3019 : {
3020 249 : gimple_seq stmts = NULL;
3021 249 : shifted
3022 249 : = gimple_build (&stmts, LSHIFT_EXPR, container_type, value, shift);
3023 249 : if (!gimple_seq_empty_p (stmts))
3024 112 : append_pattern_def_seq (vinfo, stmt_info,
3025 : gimple_seq_first_stmt (stmts));
3026 : }
3027 :
3028 508 : tree mask_t
3029 508 : = wide_int_to_tree (container_type,
3030 508 : wi::shifted_mask (shift_n, mask_width, false, prec));
3031 :
3032 : /* Clear bits we don't want to write back from SHIFTED. */
3033 508 : gimple_seq stmts = NULL;
3034 508 : tree masked = gimple_build (&stmts, BIT_AND_EXPR, container_type, shifted,
3035 : mask_t);
3036 508 : if (!gimple_seq_empty_p (stmts))
3037 : {
3038 118 : pattern_stmt = gimple_seq_first_stmt (stmts);
3039 118 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt);
3040 : }
3041 :
3042 : /* Mask off the bits in the container that we are to write to. */
3043 508 : mask_t = wide_int_to_tree (container_type,
3044 508 : wi::shifted_mask (shift_n, mask_width, true, prec));
3045 508 : tree cleared = vect_recog_temp_ssa_var (container_type);
3046 508 : pattern_stmt = gimple_build_assign (cleared, BIT_AND_EXPR, container, mask_t);
3047 508 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt);
3048 :
3049 : /* Write MASKED into CLEARED. */
3050 508 : pattern_stmt
3051 508 : = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
3052 : BIT_IOR_EXPR, cleared, masked);
3053 :
3054 508 : *type_out = STMT_VINFO_VECTYPE (stmt_info);
3055 508 : vect_pattern_detected ("bit_insert pattern", stmt_info->stmt);
3056 :
3057 508 : return pattern_stmt;
3058 : }
3059 :
3060 :
3061 : /* Recognize cases in which an operation is performed in one type WTYPE
3062 : but could be done more efficiently in a narrower type NTYPE. For example,
3063 : if we have:
3064 :
3065 : ATYPE a; // narrower than NTYPE
3066 : BTYPE b; // narrower than NTYPE
3067 : WTYPE aw = (WTYPE) a;
3068 : WTYPE bw = (WTYPE) b;
3069 : WTYPE res = aw + bw; // only uses of aw and bw
3070 :
3071 : then it would be more efficient to do:
3072 :
3073 : NTYPE an = (NTYPE) a;
3074 : NTYPE bn = (NTYPE) b;
3075 : NTYPE resn = an + bn;
3076 : WTYPE res = (WTYPE) resn;
3077 :
3078 : Other situations include things like:
3079 :
3080 : ATYPE a; // NTYPE or narrower
3081 : WTYPE aw = (WTYPE) a;
3082 : WTYPE res = aw + b;
3083 :
3084 : when only "(NTYPE) res" is significant. In that case it's more efficient
3085 : to truncate "b" and do the operation on NTYPE instead:
3086 :
3087 : NTYPE an = (NTYPE) a;
3088 : NTYPE bn = (NTYPE) b; // truncation
3089 : NTYPE resn = an + bn;
3090 : WTYPE res = (WTYPE) resn;
3091 :
3092 : All users of "res" should then use "resn" instead, making the final
3093 : statement dead (not marked as relevant). The final statement is still
3094 : needed to maintain the type correctness of the IR.
3095 :
3096 : vect_determine_precisions has already determined the minimum
3097 : precision of the operation and the minimum precision required
3098 : by users of the result. */
3099 :
3100 : static gimple *
3101 32962854 : vect_recog_over_widening_pattern (vec_info *vinfo,
3102 : stmt_vec_info last_stmt_info, tree *type_out)
3103 : {
3104 32962854 : gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
3105 22941783 : if (!last_stmt)
3106 : return NULL;
3107 :
3108 : /* See whether we have found that this operation can be done on a
3109 : narrower type without changing its semantics. */
3110 22941783 : unsigned int new_precision = last_stmt_info->operation_precision;
3111 22941783 : if (!new_precision)
3112 : return NULL;
3113 :
3114 1680115 : tree lhs = gimple_assign_lhs (last_stmt);
3115 1680115 : tree type = TREE_TYPE (lhs);
3116 1680115 : tree_code code = gimple_assign_rhs_code (last_stmt);
3117 :
3118 : /* Punt for reductions where we don't handle the type conversions. */
3119 1680115 : if (vect_is_reduction (last_stmt_info))
3120 : return NULL;
3121 :
3122 : /* Keep the first operand of a COND_EXPR as-is: only the other two
3123 : operands are interesting. */
3124 1671783 : unsigned int first_op = (code == COND_EXPR ? 2 : 1);
3125 :
3126 : /* Check the operands. */
3127 1671783 : unsigned int nops = gimple_num_ops (last_stmt) - first_op;
3128 1671783 : auto_vec <vect_unpromoted_value, 3> unprom (nops);
3129 1671783 : unprom.quick_grow_cleared (nops);
3130 1671783 : unsigned int min_precision = 0;
3131 1671783 : bool single_use_p = false;
3132 6666718 : for (unsigned int i = 0; i < nops; ++i)
3133 : {
3134 3324866 : tree op = gimple_op (last_stmt, first_op + i);
3135 3324866 : if (TREE_CODE (op) == INTEGER_CST)
3136 1521391 : unprom[i].set_op (op, vect_constant_def);
3137 1803475 : else if (TREE_CODE (op) == SSA_NAME)
3138 : {
3139 1803475 : bool op_single_use_p = true;
3140 1803475 : if (!vect_look_through_possible_promotion (vinfo, op, &unprom[i],
3141 : &op_single_use_p))
3142 1714 : return NULL;
3143 : /* If:
3144 :
3145 : (1) N bits of the result are needed;
3146 : (2) all inputs are widened from M<N bits; and
3147 : (3) one operand OP is a single-use SSA name
3148 :
3149 : we can shift the M->N widening from OP to the output
3150 : without changing the number or type of extensions involved.
3151 : This then reduces the number of copies of STMT_INFO.
3152 :
3153 : If instead of (3) more than one operand is a single-use SSA name,
3154 : shifting the extension to the output is even more of a win.
3155 :
3156 : If instead:
3157 :
3158 : (1) N bits of the result are needed;
3159 : (2) one operand OP2 is widened from M2<N bits;
3160 : (3) another operand OP1 is widened from M1<M2 bits; and
3161 : (4) both OP1 and OP2 are single-use
3162 :
3163 : the choice is between:
3164 :
3165 : (a) truncating OP2 to M1, doing the operation on M1,
3166 : and then widening the result to N
3167 :
3168 : (b) widening OP1 to M2, doing the operation on M2, and then
3169 : widening the result to N
3170 :
3171 : Both shift the M2->N widening of the inputs to the output.
3172 : (a) additionally shifts the M1->M2 widening to the output;
3173 : it requires fewer copies of STMT_INFO but requires an extra
3174 : M2->M1 truncation.
3175 :
3176 : Which is better will depend on the complexity and cost of
3177 : STMT_INFO, which is hard to predict at this stage. However,
3178 : a clear tie-breaker in favor of (b) is the fact that the
3179 : truncation in (a) increases the length of the operation chain.
3180 :
3181 : If instead of (4) only one of OP1 or OP2 is single-use,
3182 : (b) is still a win over doing the operation in N bits:
3183 : it still shifts the M2->N widening on the single-use operand
3184 : to the output and reduces the number of STMT_INFO copies.
3185 :
3186 : If neither operand is single-use then operating on fewer than
3187 : N bits might lead to more extensions overall. Whether it does
3188 : or not depends on global information about the vectorization
3189 : region, and whether that's a good trade-off would again
3190 : depend on the complexity and cost of the statements involved,
3191 : as well as things like register pressure that are not normally
3192 : modelled at this stage. We therefore ignore these cases
3193 : and just optimize the clear single-use wins above.
3194 :
3195 : Thus we take the maximum precision of the unpromoted operands
3196 : and record whether any operand is single-use. */
3197 1801761 : if (unprom[i].dt == vect_internal_def)
3198 : {
3199 1038521 : min_precision = MAX (min_precision,
3200 : TYPE_PRECISION (unprom[i].type));
3201 1038521 : single_use_p |= op_single_use_p;
3202 : }
3203 : }
3204 : else
3205 : return NULL;
3206 : }
3207 :
3208 : /* Although the operation could be done in operation_precision, we have
3209 : to balance that against introducing extra truncations or extensions.
3210 : Calculate the minimum precision that can be handled efficiently.
3211 :
3212 : The loop above determined that the operation could be handled
3213 : efficiently in MIN_PRECISION if SINGLE_USE_P; this would shift an
3214 : extension from the inputs to the output without introducing more
3215 : instructions, and would reduce the number of instructions required
3216 : for STMT_INFO itself.
3217 :
3218 : vect_determine_precisions has also determined that the result only
3219 : needs min_output_precision bits. Truncating by a factor of N times
3220 : requires a tree of N - 1 instructions, so if TYPE is N times wider
3221 : than min_output_precision, doing the operation in TYPE and truncating
3222 : the result requires N + (N - 1) = 2N - 1 instructions per output vector.
3223 : In contrast:
3224 :
3225 : - truncating the input to a unary operation and doing the operation
3226 : in the new type requires at most N - 1 + 1 = N instructions per
3227 : output vector
3228 :
3229 : - doing the same for a binary operation requires at most
3230 : (N - 1) * 2 + 1 = 2N - 1 instructions per output vector
3231 :
3232 : Both unary and binary operations require fewer instructions than
3233 : this if the operands were extended from a suitable truncated form.
3234 : Thus there is usually nothing to lose by doing operations in
3235 : min_output_precision bits, but there can be something to gain. */
3236 1670069 : if (!single_use_p)
3237 1313855 : min_precision = last_stmt_info->min_output_precision;
3238 : else
3239 356214 : min_precision = MIN (min_precision, last_stmt_info->min_output_precision);
3240 :
3241 : /* Apply the minimum efficient precision we just calculated. */
3242 1670069 : if (new_precision < min_precision)
3243 : new_precision = min_precision;
3244 1670069 : new_precision = vect_element_precision (new_precision);
3245 1670069 : if (new_precision >= TYPE_PRECISION (type))
3246 : return NULL;
3247 :
3248 159264 : vect_pattern_detected ("vect_recog_over_widening_pattern", last_stmt);
3249 :
3250 159264 : *type_out = get_vectype_for_scalar_type (vinfo, type);
3251 159264 : if (!*type_out)
3252 : return NULL;
3253 :
3254 : /* We've found a viable pattern. Get the new type of the operation. */
3255 140391 : bool unsigned_p = (last_stmt_info->operation_sign == UNSIGNED);
3256 140391 : tree new_type = build_nonstandard_integer_type (new_precision, unsigned_p);
3257 :
3258 : /* If we're truncating an operation, we need to make sure that we
3259 : don't introduce new undefined overflow. The codes tested here are
3260 : a subset of those accepted by vect_truncatable_operation_p. */
3261 140391 : tree op_type = new_type;
3262 140391 : if (TYPE_OVERFLOW_UNDEFINED (new_type)
3263 181301 : && (code == PLUS_EXPR || code == MINUS_EXPR || code == MULT_EXPR))
3264 27991 : op_type = build_nonstandard_integer_type (new_precision, true);
3265 :
3266 140391 : tree new_vectype = get_vectype_for_scalar_type (vinfo, new_type);
3267 140391 : tree op_vectype = get_vectype_for_scalar_type (vinfo, op_type);
3268 140391 : if (!new_vectype || !op_vectype)
3269 : return NULL;
3270 :
3271 : /* Verify we can handle the new operation. For shifts and rotates
3272 : apply heuristic of whether we are likely facing vector-vector or
3273 : vector-scalar operation. Since we are eventually expecting that
3274 : a later pattern might eventually want to rewrite an unsupported
3275 : into a supported case error on that side in case the original
3276 : operation was not supported either or this is a binary operation
3277 : and the 2nd operand is constant. */
3278 140391 : if (code == RSHIFT_EXPR || code == LSHIFT_EXPR || code == RROTATE_EXPR)
3279 : {
3280 29637 : if (!target_has_vecop_for_code (code, op_vectype, optab_vector)
3281 28560 : && ((unprom[1].dt != vect_external_def
3282 28148 : && unprom[1].dt != vect_constant_def)
3283 19610 : || !target_has_vecop_for_code (code, op_vectype, optab_scalar))
3284 38610 : && !(!target_has_vecop_for_code (code, *type_out, optab_vector)
3285 7875 : && ((unprom[1].dt != vect_external_def
3286 7875 : || unprom[1].dt != vect_constant_def)
3287 : || !target_has_vecop_for_code (code, *type_out,
3288 : optab_scalar))))
3289 : return NULL;
3290 : }
3291 110754 : else if (!target_has_vecop_for_code (code, op_vectype, optab_vector)
3292 110754 : && (target_has_vecop_for_code (code, *type_out, optab_vector)
3293 29 : && !(nops == 2 && unprom[1].dt == vect_constant_def)))
3294 : return NULL;
3295 :
3296 139283 : if (dump_enabled_p ())
3297 4643 : dump_printf_loc (MSG_NOTE, vect_location, "demoting %T to %T\n",
3298 : type, new_type);
3299 :
3300 : /* Calculate the rhs operands for an operation on OP_TYPE. */
3301 139283 : tree ops[3] = {};
3302 139508 : for (unsigned int i = 1; i < first_op; ++i)
3303 225 : ops[i - 1] = gimple_op (last_stmt, i);
3304 139283 : vect_convert_inputs (vinfo, last_stmt_info, nops, &ops[first_op - 1],
3305 139283 : op_type, &unprom[0], op_vectype);
3306 :
3307 : /* Use the operation to produce a result of type OP_TYPE. */
3308 139283 : tree new_var = vect_recog_temp_ssa_var (op_type, NULL);
3309 139283 : gimple *pattern_stmt = gimple_build_assign (new_var, code,
3310 : ops[0], ops[1], ops[2]);
3311 139283 : gimple_set_location (pattern_stmt, gimple_location (last_stmt));
3312 :
3313 139283 : if (dump_enabled_p ())
3314 4643 : dump_printf_loc (MSG_NOTE, vect_location,
3315 : "created pattern stmt: %G", pattern_stmt);
3316 :
3317 : /* Convert back to the original signedness, if OP_TYPE is different
3318 : from NEW_TYPE. */
3319 139283 : if (op_type != new_type)
3320 27984 : pattern_stmt = vect_convert_output (vinfo, last_stmt_info, new_type,
3321 : pattern_stmt, op_vectype);
3322 :
3323 : /* Promote the result to the original type. */
3324 139283 : pattern_stmt = vect_convert_output (vinfo, last_stmt_info, type,
3325 : pattern_stmt, new_vectype);
3326 :
3327 139283 : return pattern_stmt;
3328 1671783 : }
3329 :
3330 : /* Recognize the following patterns:
3331 :
3332 : ATYPE a; // narrower than TYPE
3333 : BTYPE b; // narrower than TYPE
3334 :
3335 : 1) Multiply high with scaling
3336 : TYPE res = ((TYPE) a * (TYPE) b) >> c;
3337 : Here, c is bitsize (TYPE) / 2 - 1.
3338 :
3339 : 2) ... or also with rounding
3340 : TYPE res = (((TYPE) a * (TYPE) b) >> d + 1) >> 1;
3341 : Here, d is bitsize (TYPE) / 2 - 2.
3342 :
3343 : 3) Normal multiply high
3344 : TYPE res = ((TYPE) a * (TYPE) b) >> e;
3345 : Here, e is bitsize (TYPE) / 2.
3346 :
3347 : where only the bottom half of res is used. */
3348 :
3349 : static gimple *
3350 33099304 : vect_recog_mulhs_pattern (vec_info *vinfo,
3351 : stmt_vec_info last_stmt_info, tree *type_out)
3352 : {
3353 : /* Check for a right shift. */
3354 33099304 : gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
3355 23078027 : if (!last_stmt
3356 23078027 : || gimple_assign_rhs_code (last_stmt) != RSHIFT_EXPR)
3357 : return NULL;
3358 :
3359 : /* Check that the shift result is wider than the users of the
3360 : result need (i.e. that narrowing would be a natural choice). */
3361 370977 : tree lhs_type = TREE_TYPE (gimple_assign_lhs (last_stmt));
3362 370977 : unsigned int target_precision
3363 370977 : = vect_element_precision (last_stmt_info->min_output_precision);
3364 370977 : if (!INTEGRAL_TYPE_P (lhs_type)
3365 370977 : || target_precision >= TYPE_PRECISION (lhs_type))
3366 : return NULL;
3367 :
3368 : /* Look through any change in sign on the outer shift input. */
3369 64085 : vect_unpromoted_value unprom_rshift_input;
3370 64085 : tree rshift_input = vect_look_through_possible_promotion
3371 64085 : (vinfo, gimple_assign_rhs1 (last_stmt), &unprom_rshift_input);
3372 64085 : if (!rshift_input
3373 64085 : || TYPE_PRECISION (TREE_TYPE (rshift_input))
3374 63459 : != TYPE_PRECISION (lhs_type))
3375 : return NULL;
3376 :
3377 : /* Get the definition of the shift input. */
3378 59157 : stmt_vec_info rshift_input_stmt_info
3379 59157 : = vect_get_internal_def (vinfo, rshift_input);
3380 59157 : if (!rshift_input_stmt_info)
3381 : return NULL;
3382 54431 : gassign *rshift_input_stmt
3383 54431 : = dyn_cast <gassign *> (rshift_input_stmt_info->stmt);
3384 51005 : if (!rshift_input_stmt)
3385 : return NULL;
3386 :
3387 51005 : stmt_vec_info mulh_stmt_info;
3388 51005 : tree scale_term;
3389 51005 : bool rounding_p = false;
3390 :
3391 : /* Check for the presence of the rounding term. */
3392 58895 : if (gimple_assign_rhs_code (rshift_input_stmt) == PLUS_EXPR)
3393 : {
3394 : /* Check that the outer shift was by 1. */
3395 17912 : if (!integer_onep (gimple_assign_rhs2 (last_stmt)))
3396 8889 : return NULL;
3397 :
3398 : /* Check that the second operand of the PLUS_EXPR is 1. */
3399 436 : if (!integer_onep (gimple_assign_rhs2 (rshift_input_stmt)))
3400 : return NULL;
3401 :
3402 : /* Look through any change in sign on the addition input. */
3403 69 : vect_unpromoted_value unprom_plus_input;
3404 69 : tree plus_input = vect_look_through_possible_promotion
3405 69 : (vinfo, gimple_assign_rhs1 (rshift_input_stmt), &unprom_plus_input);
3406 69 : if (!plus_input
3407 69 : || TYPE_PRECISION (TREE_TYPE (plus_input))
3408 69 : != TYPE_PRECISION (TREE_TYPE (rshift_input)))
3409 : return NULL;
3410 :
3411 : /* Get the definition of the multiply-high-scale part. */
3412 69 : stmt_vec_info plus_input_stmt_info
3413 69 : = vect_get_internal_def (vinfo, plus_input);
3414 69 : if (!plus_input_stmt_info)
3415 : return NULL;
3416 69 : gassign *plus_input_stmt
3417 8958 : = dyn_cast <gassign *> (plus_input_stmt_info->stmt);
3418 69 : if (!plus_input_stmt
3419 69 : || gimple_assign_rhs_code (plus_input_stmt) != RSHIFT_EXPR)
3420 : return NULL;
3421 :
3422 : /* Look through any change in sign on the scaling input. */
3423 67 : vect_unpromoted_value unprom_scale_input;
3424 67 : tree scale_input = vect_look_through_possible_promotion
3425 67 : (vinfo, gimple_assign_rhs1 (plus_input_stmt), &unprom_scale_input);
3426 67 : if (!scale_input
3427 67 : || TYPE_PRECISION (TREE_TYPE (scale_input))
3428 67 : != TYPE_PRECISION (TREE_TYPE (plus_input)))
3429 : return NULL;
3430 :
3431 : /* Get the definition of the multiply-high part. */
3432 67 : mulh_stmt_info = vect_get_internal_def (vinfo, scale_input);
3433 67 : if (!mulh_stmt_info)
3434 : return NULL;
3435 :
3436 : /* Get the scaling term. */
3437 67 : scale_term = gimple_assign_rhs2 (plus_input_stmt);
3438 67 : rounding_p = true;
3439 : }
3440 : else
3441 : {
3442 42049 : mulh_stmt_info = rshift_input_stmt_info;
3443 42049 : scale_term = gimple_assign_rhs2 (last_stmt);
3444 : }
3445 :
3446 : /* Check that the scaling factor is constant. */
3447 42116 : if (TREE_CODE (scale_term) != INTEGER_CST)
3448 : return NULL;
3449 :
3450 : /* Check whether the scaling input term can be seen as two widened
3451 : inputs multiplied together. */
3452 123330 : vect_unpromoted_value unprom_mult[2];
3453 41110 : tree new_type;
3454 41110 : unsigned int nops
3455 41110 : = vect_widened_op_tree (vinfo, mulh_stmt_info, MULT_EXPR, WIDEN_MULT_EXPR,
3456 : false, 2, unprom_mult, &new_type);
3457 41110 : if (nops != 2)
3458 : return NULL;
3459 :
3460 : /* Adjust output precision. */
3461 10749 : if (TYPE_PRECISION (new_type) < target_precision)
3462 0 : new_type = build_nonstandard_integer_type
3463 0 : (target_precision, TYPE_UNSIGNED (new_type));
3464 :
3465 10749 : unsigned mult_precision = TYPE_PRECISION (new_type);
3466 10749 : internal_fn ifn;
3467 : /* Check that the scaling factor is expected. Instead of
3468 : target_precision, we should use the one that we actually
3469 : use for internal function. */
3470 10749 : if (rounding_p)
3471 : {
3472 : /* Check pattern 2). */
3473 134 : if (wi::to_widest (scale_term) + mult_precision + 2
3474 201 : != TYPE_PRECISION (lhs_type))
3475 : return NULL;
3476 :
3477 : ifn = IFN_MULHRS;
3478 : }
3479 : else
3480 : {
3481 : /* Check for pattern 1). */
3482 21364 : if (wi::to_widest (scale_term) + mult_precision + 1
3483 32046 : == TYPE_PRECISION (lhs_type))
3484 : ifn = IFN_MULHS;
3485 : /* Check for pattern 3). */
3486 10648 : else if (wi::to_widest (scale_term) + mult_precision
3487 21296 : == TYPE_PRECISION (lhs_type))
3488 : ifn = IFN_MULH;
3489 : else
3490 : return NULL;
3491 : }
3492 :
3493 10660 : vect_pattern_detected ("vect_recog_mulhs_pattern", last_stmt);
3494 :
3495 : /* Check for target support. */
3496 10660 : tree new_vectype = get_vectype_for_scalar_type (vinfo, new_type);
3497 10660 : if (!new_vectype
3498 17982 : || !direct_internal_fn_supported_p
3499 7322 : (ifn, new_vectype, OPTIMIZE_FOR_SPEED))
3500 : return NULL;
3501 :
3502 : /* The result is cast back to LHS_TYPE, a cast that the over-widening
3503 : machinery then removes. LHS_TYPE need not have a vector type, as for the
3504 : 128-bit product of a 64-bit high-part multiply, so leave *TYPE_OUT null
3505 : rather than giving up. */
3506 4525 : *type_out = get_vectype_for_scalar_type (vinfo, lhs_type);
3507 :
3508 : /* Generate the IFN_MULHRS call. */
3509 4525 : tree new_var = vect_recog_temp_ssa_var (new_type, NULL);
3510 4525 : tree new_ops[2];
3511 4525 : vect_convert_inputs (vinfo, last_stmt_info, 2, new_ops, new_type,
3512 : unprom_mult, new_vectype);
3513 4525 : gcall *mulhrs_stmt
3514 4525 : = gimple_build_call_internal (ifn, 2, new_ops[0], new_ops[1]);
3515 4525 : gimple_call_set_lhs (mulhrs_stmt, new_var);
3516 4525 : gimple_set_location (mulhrs_stmt, gimple_location (last_stmt));
3517 :
3518 4525 : if (dump_enabled_p ())
3519 3 : dump_printf_loc (MSG_NOTE, vect_location,
3520 : "created pattern stmt: %G", (gimple *) mulhrs_stmt);
3521 :
3522 4525 : return vect_convert_output (vinfo, last_stmt_info, lhs_type,
3523 4525 : mulhrs_stmt, new_vectype);
3524 : }
3525 :
3526 : /* Recognize the patterns:
3527 :
3528 : ATYPE a; // narrower than TYPE
3529 : BTYPE b; // narrower than TYPE
3530 : (1) TYPE avg = ((TYPE) a + (TYPE) b) >> 1;
3531 : or (2) TYPE avg = ((TYPE) a + (TYPE) b + 1) >> 1;
3532 :
3533 : where only the bottom half of avg is used. Try to transform them into:
3534 :
3535 : (1) NTYPE avg' = .AVG_FLOOR ((NTYPE) a, (NTYPE) b);
3536 : or (2) NTYPE avg' = .AVG_CEIL ((NTYPE) a, (NTYPE) b);
3537 :
3538 : followed by:
3539 :
3540 : TYPE avg = (TYPE) avg';
3541 :
3542 : where NTYPE is no wider than half of TYPE. Since only the bottom half
3543 : of avg is used, all or part of the cast of avg' should become redundant.
3544 :
3545 : If there is no target support available, generate code to distribute rshift
3546 : over plus and add a carry. */
3547 :
3548 : static gimple *
3549 33095067 : vect_recog_average_pattern (vec_info *vinfo,
3550 : stmt_vec_info last_stmt_info, tree *type_out)
3551 : {
3552 : /* Check for a shift right by one bit. */
3553 33095067 : gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
3554 23073996 : if (!last_stmt
3555 23073996 : || gimple_assign_rhs_code (last_stmt) != RSHIFT_EXPR
3556 370490 : || !integer_onep (gimple_assign_rhs2 (last_stmt)))
3557 : return NULL;
3558 :
3559 : /* Check that the shift result is wider than the users of the
3560 : result need (i.e. that narrowing would be a natural choice). */
3561 58014 : tree lhs = gimple_assign_lhs (last_stmt);
3562 58014 : tree type = TREE_TYPE (lhs);
3563 58014 : unsigned int target_precision
3564 58014 : = vect_element_precision (last_stmt_info->min_output_precision);
3565 58014 : if (!INTEGRAL_TYPE_P (type) || target_precision >= TYPE_PRECISION (type))
3566 : return NULL;
3567 :
3568 : /* Look through any change in sign on the shift input. */
3569 2272 : tree rshift_rhs = gimple_assign_rhs1 (last_stmt);
3570 2272 : vect_unpromoted_value unprom_plus;
3571 2272 : rshift_rhs = vect_look_through_possible_promotion (vinfo, rshift_rhs,
3572 : &unprom_plus);
3573 2272 : if (!rshift_rhs
3574 2272 : || TYPE_PRECISION (TREE_TYPE (rshift_rhs)) != TYPE_PRECISION (type))
3575 : return NULL;
3576 :
3577 : /* Get the definition of the shift input. */
3578 2266 : stmt_vec_info plus_stmt_info = vect_get_internal_def (vinfo, rshift_rhs);
3579 2266 : if (!plus_stmt_info)
3580 : return NULL;
3581 :
3582 : /* Check whether the shift input can be seen as a tree of additions on
3583 : 2 or 3 widened inputs.
3584 :
3585 : Note that the pattern should be a win even if the result of one or
3586 : more additions is reused elsewhere: if the pattern matches, we'd be
3587 : replacing 2N RSHIFT_EXPRs and N VEC_PACK_*s with N IFN_AVG_*s. */
3588 2260 : internal_fn ifn = IFN_AVG_FLOOR;
3589 9040 : vect_unpromoted_value unprom[3];
3590 2260 : tree new_type;
3591 2260 : unsigned int nops = vect_widened_op_tree (vinfo, plus_stmt_info, PLUS_EXPR,
3592 2260 : IFN_VEC_WIDEN_PLUS, false, 3,
3593 : unprom, &new_type);
3594 2260 : if (nops == 0)
3595 : return NULL;
3596 959 : if (nops == 3)
3597 : {
3598 : /* Check that one operand is 1. */
3599 : unsigned int i;
3600 1155 : for (i = 0; i < 3; ++i)
3601 1101 : if (integer_onep (unprom[i].op))
3602 : break;
3603 367 : if (i == 3)
3604 : return NULL;
3605 : /* Throw away the 1 operand and keep the other two. */
3606 313 : if (i < 2)
3607 0 : unprom[i] = unprom[2];
3608 : ifn = IFN_AVG_CEIL;
3609 : }
3610 :
3611 905 : vect_pattern_detected ("vect_recog_average_pattern", last_stmt);
3612 :
3613 : /* We know that:
3614 :
3615 : (a) the operation can be viewed as:
3616 :
3617 : TYPE widened0 = (TYPE) UNPROM[0];
3618 : TYPE widened1 = (TYPE) UNPROM[1];
3619 : TYPE tmp1 = widened0 + widened1 {+ 1};
3620 : TYPE tmp2 = tmp1 >> 1; // LAST_STMT_INFO
3621 :
3622 : (b) the first two statements are equivalent to:
3623 :
3624 : TYPE widened0 = (TYPE) (NEW_TYPE) UNPROM[0];
3625 : TYPE widened1 = (TYPE) (NEW_TYPE) UNPROM[1];
3626 :
3627 : (c) vect_recog_over_widening_pattern has already tried to narrow TYPE
3628 : where sensible;
3629 :
3630 : (d) all the operations can be performed correctly at twice the width of
3631 : NEW_TYPE, due to the nature of the average operation; and
3632 :
3633 : (e) users of the result of the right shift need only TARGET_PRECISION
3634 : bits, where TARGET_PRECISION is no more than half of TYPE's
3635 : precision.
3636 :
3637 : Under these circumstances, the only situation in which NEW_TYPE
3638 : could be narrower than TARGET_PRECISION is if widened0, widened1
3639 : and an addition result are all used more than once. Thus we can
3640 : treat any widening of UNPROM[0] and UNPROM[1] to TARGET_PRECISION
3641 : as "free", whereas widening the result of the average instruction
3642 : from NEW_TYPE to TARGET_PRECISION would be a new operation. It's
3643 : therefore better not to go narrower than TARGET_PRECISION. */
3644 905 : if (TYPE_PRECISION (new_type) < target_precision)
3645 0 : new_type = build_nonstandard_integer_type (target_precision,
3646 0 : TYPE_UNSIGNED (new_type));
3647 :
3648 : /* Check for target support. */
3649 905 : tree new_vectype = get_vectype_for_scalar_type (vinfo, new_type);
3650 905 : if (!new_vectype)
3651 : return NULL;
3652 :
3653 905 : bool fallback_p = !direct_internal_fn_supported_p (ifn, new_vectype,
3654 905 : OPTIMIZE_FOR_SPEED);
3655 905 : if (fallback_p
3656 905 : && (!target_supports_op_p (new_vectype, RSHIFT_EXPR, optab_scalar)
3657 697 : || !target_supports_op_p (new_vectype, PLUS_EXPR, optab_default)
3658 697 : || !target_supports_op_p (new_vectype, BIT_AND_EXPR, optab_default)
3659 697 : || (ifn == IFN_AVG_CEIL
3660 105 : && !target_supports_op_p (new_vectype, BIT_IOR_EXPR,
3661 : optab_default))))
3662 : return NULL;
3663 :
3664 : /* The IR requires a valid vector type for the cast result, even though
3665 : it's likely to be discarded. */
3666 903 : *type_out = get_vectype_for_scalar_type (vinfo, type);
3667 903 : if (!*type_out)
3668 : return NULL;
3669 :
3670 899 : tree new_var = vect_recog_temp_ssa_var (new_type, NULL);
3671 899 : tree new_ops[2];
3672 899 : vect_convert_inputs (vinfo, last_stmt_info, 2, new_ops, new_type,
3673 : unprom, new_vectype);
3674 :
3675 899 : if (fallback_p)
3676 : {
3677 : /* As a fallback, generate code for following sequence:
3678 :
3679 : shifted_op0 = new_ops[0] >> 1;
3680 : shifted_op1 = new_ops[1] >> 1;
3681 : sum_of_shifted = shifted_op0 + shifted_op1;
3682 : unmasked_carry = new_ops[0] and/or new_ops[1];
3683 : carry = unmasked_carry & 1;
3684 : new_var = sum_of_shifted + carry;
3685 :
3686 : For signed types, arithmetic shifts round down and the carry is one
3687 : when both operands are odd (or either operand for IFN_AVG_CEIL). */
3688 :
3689 693 : tree one_cst = build_one_cst (new_type);
3690 693 : gassign *g;
3691 :
3692 693 : tree shifted_op0 = vect_recog_temp_ssa_var (new_type, NULL);
3693 693 : g = gimple_build_assign (shifted_op0, RSHIFT_EXPR, new_ops[0], one_cst);
3694 693 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3695 :
3696 693 : tree shifted_op1 = vect_recog_temp_ssa_var (new_type, NULL);
3697 693 : g = gimple_build_assign (shifted_op1, RSHIFT_EXPR, new_ops[1], one_cst);
3698 693 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3699 :
3700 693 : tree sum_of_shifted = vect_recog_temp_ssa_var (new_type, NULL);
3701 693 : g = gimple_build_assign (sum_of_shifted, PLUS_EXPR,
3702 : shifted_op0, shifted_op1);
3703 693 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3704 :
3705 693 : tree unmasked_carry = vect_recog_temp_ssa_var (new_type, NULL);
3706 693 : tree_code c = (ifn == IFN_AVG_CEIL) ? BIT_IOR_EXPR : BIT_AND_EXPR;
3707 693 : g = gimple_build_assign (unmasked_carry, c, new_ops[0], new_ops[1]);
3708 693 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3709 :
3710 693 : tree carry = vect_recog_temp_ssa_var (new_type, NULL);
3711 693 : g = gimple_build_assign (carry, BIT_AND_EXPR, unmasked_carry, one_cst);
3712 693 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3713 :
3714 693 : g = gimple_build_assign (new_var, PLUS_EXPR, sum_of_shifted, carry);
3715 693 : return vect_convert_output (vinfo, last_stmt_info, type, g, new_vectype);
3716 : }
3717 :
3718 : /* Generate the IFN_AVG* call. */
3719 206 : gcall *average_stmt = gimple_build_call_internal (ifn, 2, new_ops[0],
3720 : new_ops[1]);
3721 206 : gimple_call_set_lhs (average_stmt, new_var);
3722 206 : gimple_set_location (average_stmt, gimple_location (last_stmt));
3723 :
3724 206 : if (dump_enabled_p ())
3725 79 : dump_printf_loc (MSG_NOTE, vect_location,
3726 : "created pattern stmt: %G", (gimple *) average_stmt);
3727 :
3728 206 : return vect_convert_output (vinfo, last_stmt_info,
3729 206 : type, average_stmt, new_vectype);
3730 : }
3731 :
3732 : /* Recognize cases in which the input to a cast is wider than its
3733 : output, and the input is fed by a widening operation. Fold this
3734 : by removing the unnecessary intermediate widening. E.g.:
3735 :
3736 : unsigned char a;
3737 : unsigned int b = (unsigned int) a;
3738 : unsigned short c = (unsigned short) b;
3739 :
3740 : -->
3741 :
3742 : unsigned short c = (unsigned short) a;
3743 :
3744 : Although this is rare in input IR, it is an expected side-effect
3745 : of the over-widening pattern above.
3746 :
3747 : This is beneficial also for integer-to-float conversions, if the
3748 : widened integer has more bits than the float, and if the unwidened
3749 : input doesn't. */
3750 :
3751 : static gimple *
3752 33099304 : vect_recog_cast_forwprop_pattern (vec_info *vinfo,
3753 : stmt_vec_info last_stmt_info, tree *type_out)
3754 : {
3755 : /* Check for a cast, including an integer-to-float conversion. */
3756 33099304 : gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
3757 23073502 : if (!last_stmt)
3758 : return NULL;
3759 23073502 : tree_code code = gimple_assign_rhs_code (last_stmt);
3760 23073502 : if (!CONVERT_EXPR_CODE_P (code) && code != FLOAT_EXPR)
3761 : return NULL;
3762 :
3763 : /* Make sure that the rhs is a scalar with a natural bitsize. */
3764 3373969 : tree lhs = gimple_assign_lhs (last_stmt);
3765 3373969 : if (!lhs)
3766 : return NULL;
3767 3373969 : tree lhs_type = TREE_TYPE (lhs);
3768 3373969 : scalar_mode lhs_mode;
3769 3352956 : if (VECT_SCALAR_BOOLEAN_TYPE_P (lhs_type)
3770 6725135 : || !is_a <scalar_mode> (TYPE_MODE (lhs_type), &lhs_mode))
3771 : return NULL;
3772 :
3773 : /* Check for a narrowing operation (from a vector point of view). */
3774 3347431 : tree rhs = gimple_assign_rhs1 (last_stmt);
3775 3347431 : tree rhs_type = TREE_TYPE (rhs);
3776 3347431 : if (!INTEGRAL_TYPE_P (rhs_type)
3777 2928060 : || VECT_SCALAR_BOOLEAN_TYPE_P (rhs_type)
3778 9050839 : || TYPE_PRECISION (rhs_type) <= GET_MODE_BITSIZE (lhs_mode))
3779 : return NULL;
3780 :
3781 : /* Try to find an unpromoted input. */
3782 391678 : vect_unpromoted_value unprom;
3783 391678 : if (!vect_look_through_possible_promotion (vinfo, rhs, &unprom)
3784 391678 : || TYPE_PRECISION (unprom.type) >= TYPE_PRECISION (rhs_type))
3785 : return NULL;
3786 :
3787 : /* If the bits above RHS_TYPE matter, make sure that they're the
3788 : same when extending from UNPROM as they are when extending from RHS. */
3789 53435 : if (!INTEGRAL_TYPE_P (lhs_type)
3790 53435 : && TYPE_SIGN (rhs_type) != TYPE_SIGN (unprom.type))
3791 : return NULL;
3792 :
3793 : /* We can get the same result by casting UNPROM directly, to avoid
3794 : the unnecessary widening and narrowing. */
3795 53315 : vect_pattern_detected ("vect_recog_cast_forwprop_pattern", last_stmt);
3796 :
3797 53315 : *type_out = get_vectype_for_scalar_type (vinfo, lhs_type);
3798 53315 : if (!*type_out)
3799 : return NULL;
3800 :
3801 53315 : tree new_var = vect_recog_temp_ssa_var (lhs_type, NULL);
3802 53315 : gimple *pattern_stmt = gimple_build_assign (new_var, code, unprom.op);
3803 53315 : gimple_set_location (pattern_stmt, gimple_location (last_stmt));
3804 :
3805 53315 : return pattern_stmt;
3806 : }
3807 :
3808 : /* Try to detect a shift left of a widened input, converting LSHIFT_EXPR
3809 : to WIDEN_LSHIFT_EXPR. See vect_recog_widen_op_pattern for details. */
3810 :
3811 : static gimple *
3812 33017726 : vect_recog_widen_shift_pattern (vec_info *vinfo,
3813 : stmt_vec_info last_stmt_info, tree *type_out)
3814 : {
3815 33017726 : return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
3816 33017726 : LSHIFT_EXPR, WIDEN_LSHIFT_EXPR, true,
3817 33017726 : "vect_recog_widen_shift_pattern");
3818 : }
3819 :
3820 : /* Detect a rotate pattern wouldn't be otherwise vectorized:
3821 :
3822 : type a_t, b_t, c_t;
3823 :
3824 : S0 a_t = b_t r<< c_t;
3825 :
3826 : Input/Output:
3827 :
3828 : * STMT_VINFO: The stmt from which the pattern search begins,
3829 : i.e. the shift/rotate stmt. The original stmt (S0) is replaced
3830 : with a sequence:
3831 :
3832 : S1 d_t = -c_t;
3833 : S2 e_t = d_t & (B - 1);
3834 : S3 f_t = b_t << c_t;
3835 : S4 g_t = b_t >> e_t;
3836 : S0 a_t = f_t | g_t;
3837 :
3838 : where B is element bitsize of type.
3839 :
3840 : Output:
3841 :
3842 : * TYPE_OUT: The type of the output of this pattern.
3843 :
3844 : * Return value: A new stmt that will be used to replace the rotate
3845 : S0 stmt. */
3846 :
3847 : static gimple *
3848 33017726 : vect_recog_rotate_pattern (vec_info *vinfo,
3849 : stmt_vec_info stmt_vinfo, tree *type_out)
3850 : {
3851 33017726 : gimple *last_stmt = stmt_vinfo->stmt;
3852 33017726 : tree oprnd0, oprnd1, lhs, var, var1, var2, vectype, type, stype, def, def2;
3853 33017726 : gimple *pattern_stmt, *def_stmt;
3854 33017726 : enum tree_code rhs_code;
3855 33017726 : enum vect_def_type dt;
3856 33017726 : optab optab1, optab2;
3857 33017726 : edge ext_def = NULL;
3858 33017726 : bool bswap16_p = false;
3859 :
3860 33017726 : if (is_gimple_assign (last_stmt))
3861 : {
3862 22991874 : rhs_code = gimple_assign_rhs_code (last_stmt);
3863 22991874 : switch (rhs_code)
3864 : {
3865 8658 : case LROTATE_EXPR:
3866 8658 : case RROTATE_EXPR:
3867 8658 : break;
3868 : default:
3869 : return NULL;
3870 : }
3871 :
3872 8658 : lhs = gimple_assign_lhs (last_stmt);
3873 8658 : oprnd0 = gimple_assign_rhs1 (last_stmt);
3874 8658 : type = TREE_TYPE (oprnd0);
3875 8658 : oprnd1 = gimple_assign_rhs2 (last_stmt);
3876 : }
3877 10025852 : else if (gimple_call_builtin_p (last_stmt, BUILT_IN_BSWAP16))
3878 : {
3879 : /* __builtin_bswap16 (x) is another form of x r>> 8.
3880 : The vectorizer has bswap support, but only if the argument isn't
3881 : promoted. */
3882 206 : lhs = gimple_call_lhs (last_stmt);
3883 206 : oprnd0 = gimple_call_arg (last_stmt, 0);
3884 206 : type = TREE_TYPE (oprnd0);
3885 206 : if (!lhs
3886 206 : || TYPE_PRECISION (TREE_TYPE (lhs)) != 16
3887 206 : || TYPE_PRECISION (type) <= 16
3888 0 : || TREE_CODE (oprnd0) != SSA_NAME
3889 206 : || BITS_PER_UNIT != 8)
3890 206 : return NULL;
3891 :
3892 0 : stmt_vec_info def_stmt_info;
3893 0 : if (!vect_is_simple_use (oprnd0, vinfo, &dt, &def_stmt_info, &def_stmt))
3894 : return NULL;
3895 :
3896 0 : if (dt != vect_internal_def)
3897 : return NULL;
3898 :
3899 0 : if (gimple_assign_cast_p (def_stmt))
3900 : {
3901 0 : def = gimple_assign_rhs1 (def_stmt);
3902 0 : if (INTEGRAL_TYPE_P (TREE_TYPE (def))
3903 0 : && TYPE_PRECISION (TREE_TYPE (def)) == 16)
3904 : oprnd0 = def;
3905 : }
3906 :
3907 0 : type = TREE_TYPE (lhs);
3908 0 : vectype = get_vectype_for_scalar_type (vinfo, type);
3909 0 : if (vectype == NULL_TREE)
3910 : return NULL;
3911 :
3912 0 : if (tree char_vectype = get_same_sized_vectype (char_type_node, vectype))
3913 : {
3914 : /* The encoding uses one stepped pattern for each byte in the
3915 : 16-bit word. */
3916 0 : vec_perm_builder elts (TYPE_VECTOR_SUBPARTS (char_vectype), 2, 3);
3917 0 : for (unsigned i = 0; i < 3; ++i)
3918 0 : for (unsigned j = 0; j < 2; ++j)
3919 0 : elts.quick_push ((i + 1) * 2 - j - 1);
3920 :
3921 0 : vec_perm_indices indices (elts, 1,
3922 0 : TYPE_VECTOR_SUBPARTS (char_vectype));
3923 0 : machine_mode vmode = TYPE_MODE (char_vectype);
3924 0 : if (can_vec_perm_const_p (vmode, vmode, indices))
3925 : {
3926 : /* vectorizable_bswap can handle the __builtin_bswap16 if we
3927 : undo the argument promotion. */
3928 0 : if (!useless_type_conversion_p (type, TREE_TYPE (oprnd0)))
3929 : {
3930 0 : def = vect_recog_temp_ssa_var (type, NULL);
3931 0 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd0);
3932 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
3933 0 : oprnd0 = def;
3934 : }
3935 :
3936 : /* Pattern detected. */
3937 0 : vect_pattern_detected ("vect_recog_rotate_pattern", last_stmt);
3938 :
3939 0 : *type_out = vectype;
3940 :
3941 : /* Pattern supported. Create a stmt to be used to replace the
3942 : pattern, with the unpromoted argument. */
3943 0 : var = vect_recog_temp_ssa_var (type, NULL);
3944 0 : pattern_stmt = gimple_build_call (gimple_call_fndecl (last_stmt),
3945 : 1, oprnd0);
3946 0 : gimple_call_set_lhs (pattern_stmt, var);
3947 0 : gimple_call_set_fntype (as_a <gcall *> (pattern_stmt),
3948 : gimple_call_fntype (last_stmt));
3949 0 : return pattern_stmt;
3950 : }
3951 0 : }
3952 :
3953 0 : oprnd1 = build_int_cst (integer_type_node, 8);
3954 0 : rhs_code = LROTATE_EXPR;
3955 0 : bswap16_p = true;
3956 : }
3957 : else
3958 : return NULL;
3959 :
3960 8658 : if (TREE_CODE (oprnd0) != SSA_NAME
3961 8538 : || !INTEGRAL_TYPE_P (type)
3962 16875 : || TYPE_PRECISION (TREE_TYPE (lhs)) != TYPE_PRECISION (type))
3963 : return NULL;
3964 :
3965 8217 : stmt_vec_info def_stmt_info;
3966 8217 : if (!vect_is_simple_use (oprnd1, vinfo, &dt, &def_stmt_info, &def_stmt))
3967 : return NULL;
3968 :
3969 8217 : if (dt != vect_internal_def
3970 7150 : && dt != vect_constant_def
3971 25 : && dt != vect_external_def)
3972 : return NULL;
3973 :
3974 8211 : vectype = get_vectype_for_scalar_type (vinfo, type);
3975 8211 : if (vectype == NULL_TREE)
3976 : return NULL;
3977 :
3978 : /* If vector/vector or vector/scalar rotate is supported by the target,
3979 : don't do anything here. */
3980 7844 : optab1 = optab_for_tree_code (rhs_code, vectype, optab_vector);
3981 7844 : if (optab1
3982 7844 : && can_implement_p (optab1, TYPE_MODE (vectype)))
3983 : {
3984 564 : use_rotate:
3985 564 : if (bswap16_p)
3986 : {
3987 0 : if (!useless_type_conversion_p (type, TREE_TYPE (oprnd0)))
3988 : {
3989 0 : def = vect_recog_temp_ssa_var (type, NULL);
3990 0 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd0);
3991 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
3992 0 : oprnd0 = def;
3993 : }
3994 :
3995 : /* Pattern detected. */
3996 0 : vect_pattern_detected ("vect_recog_rotate_pattern", last_stmt);
3997 :
3998 0 : *type_out = vectype;
3999 :
4000 : /* Pattern supported. Create a stmt to be used to replace the
4001 : pattern. */
4002 0 : var = vect_recog_temp_ssa_var (type, NULL);
4003 0 : pattern_stmt = gimple_build_assign (var, LROTATE_EXPR, oprnd0,
4004 : oprnd1);
4005 0 : return pattern_stmt;
4006 : }
4007 : return NULL;
4008 : }
4009 :
4010 7820 : if (is_a <bb_vec_info> (vinfo) || dt != vect_internal_def)
4011 : {
4012 7736 : optab2 = optab_for_tree_code (rhs_code, vectype, optab_scalar);
4013 7736 : if (optab2
4014 7736 : && can_implement_p (optab2, TYPE_MODE (vectype)))
4015 540 : goto use_rotate;
4016 : }
4017 :
4018 : /* We may not use a reduction operand twice. */
4019 7280 : if (vect_is_reduction (stmt_vinfo))
4020 : return NULL;
4021 :
4022 7259 : tree utype = unsigned_type_for (type);
4023 7259 : tree uvectype = get_vectype_for_scalar_type (vinfo, utype);
4024 7259 : if (!uvectype)
4025 : return NULL;
4026 :
4027 : /* If vector/vector or vector/scalar shifts aren't supported by the target,
4028 : don't do anything here either. */
4029 7259 : optab1 = optab_for_tree_code (LSHIFT_EXPR, uvectype, optab_vector);
4030 7259 : optab2 = optab_for_tree_code (RSHIFT_EXPR, uvectype, optab_vector);
4031 7259 : if (!optab1
4032 7259 : || !can_implement_p (optab1, TYPE_MODE (uvectype))
4033 746 : || !optab2
4034 8005 : || !can_implement_p (optab2, TYPE_MODE (uvectype)))
4035 : {
4036 6513 : if (! is_a <bb_vec_info> (vinfo) && dt == vect_internal_def)
4037 : return NULL;
4038 6450 : optab1 = optab_for_tree_code (LSHIFT_EXPR, uvectype, optab_scalar);
4039 6450 : optab2 = optab_for_tree_code (RSHIFT_EXPR, uvectype, optab_scalar);
4040 6450 : if (!optab1
4041 6450 : || !can_implement_p (optab1, TYPE_MODE (uvectype))
4042 4800 : || !optab2
4043 11250 : || !can_implement_p (optab2, TYPE_MODE (uvectype)))
4044 : return NULL;
4045 : }
4046 :
4047 5546 : *type_out = vectype;
4048 :
4049 5546 : if (!useless_type_conversion_p (utype, TREE_TYPE (oprnd0)))
4050 : {
4051 58 : def = vect_recog_temp_ssa_var (utype, NULL);
4052 58 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd0);
4053 58 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
4054 58 : oprnd0 = def;
4055 : }
4056 :
4057 5546 : if (dt == vect_external_def && TREE_CODE (oprnd1) == SSA_NAME)
4058 15 : ext_def = vect_get_external_def_edge (vinfo, oprnd1);
4059 :
4060 5546 : def = NULL_TREE;
4061 5546 : scalar_int_mode mode = SCALAR_INT_TYPE_MODE (utype);
4062 5546 : if (dt != vect_internal_def || TYPE_MODE (TREE_TYPE (oprnd1)) == mode)
4063 : def = oprnd1;
4064 124 : else if (def_stmt && gimple_assign_cast_p (def_stmt))
4065 : {
4066 96 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
4067 96 : if (TYPE_MODE (TREE_TYPE (rhs1)) == mode
4068 96 : && TYPE_PRECISION (TREE_TYPE (rhs1))
4069 0 : == TYPE_PRECISION (type))
4070 : def = rhs1;
4071 : }
4072 :
4073 5422 : if (def == NULL_TREE)
4074 : {
4075 124 : def = vect_recog_temp_ssa_var (utype, NULL);
4076 124 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd1);
4077 124 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
4078 : }
4079 5546 : stype = TREE_TYPE (def);
4080 :
4081 5546 : if (TREE_CODE (def) == INTEGER_CST)
4082 : {
4083 4968 : if (!tree_fits_uhwi_p (def)
4084 4968 : || tree_to_uhwi (def) >= GET_MODE_PRECISION (mode)
4085 9936 : || integer_zerop (def))
4086 : return NULL;
4087 4968 : def2 = build_int_cst (stype,
4088 4968 : GET_MODE_PRECISION (mode) - tree_to_uhwi (def));
4089 : }
4090 : else
4091 : {
4092 578 : tree vecstype = get_vectype_for_scalar_type (vinfo, stype);
4093 :
4094 578 : if (vecstype == NULL_TREE)
4095 : return NULL;
4096 578 : def2 = vect_recog_temp_ssa_var (stype, NULL);
4097 578 : def_stmt = gimple_build_assign (def2, NEGATE_EXPR, def);
4098 578 : if (ext_def)
4099 : {
4100 15 : basic_block new_bb
4101 15 : = gsi_insert_on_edge_immediate (ext_def, def_stmt);
4102 15 : gcc_assert (!new_bb);
4103 : }
4104 : else
4105 563 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecstype);
4106 :
4107 578 : def2 = vect_recog_temp_ssa_var (stype, NULL);
4108 578 : tree mask = build_int_cst (stype, GET_MODE_PRECISION (mode) - 1);
4109 578 : def_stmt = gimple_build_assign (def2, BIT_AND_EXPR,
4110 : gimple_assign_lhs (def_stmt), mask);
4111 578 : if (ext_def)
4112 : {
4113 15 : basic_block new_bb
4114 15 : = gsi_insert_on_edge_immediate (ext_def, def_stmt);
4115 15 : gcc_assert (!new_bb);
4116 : }
4117 : else
4118 563 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecstype);
4119 : }
4120 :
4121 5546 : var1 = vect_recog_temp_ssa_var (utype, NULL);
4122 10781 : def_stmt = gimple_build_assign (var1, rhs_code == LROTATE_EXPR
4123 : ? LSHIFT_EXPR : RSHIFT_EXPR,
4124 : oprnd0, def);
4125 5546 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
4126 :
4127 5546 : var2 = vect_recog_temp_ssa_var (utype, NULL);
4128 10781 : def_stmt = gimple_build_assign (var2, rhs_code == LROTATE_EXPR
4129 : ? RSHIFT_EXPR : LSHIFT_EXPR,
4130 : oprnd0, def2);
4131 5546 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
4132 :
4133 : /* Pattern detected. */
4134 5546 : vect_pattern_detected ("vect_recog_rotate_pattern", last_stmt);
4135 :
4136 : /* Pattern supported. Create a stmt to be used to replace the pattern. */
4137 5546 : var = vect_recog_temp_ssa_var (utype, NULL);
4138 5546 : pattern_stmt = gimple_build_assign (var, BIT_IOR_EXPR, var1, var2);
4139 :
4140 5546 : if (!useless_type_conversion_p (type, utype))
4141 : {
4142 58 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, uvectype);
4143 58 : tree result = vect_recog_temp_ssa_var (type, NULL);
4144 58 : pattern_stmt = gimple_build_assign (result, NOP_EXPR, var);
4145 : }
4146 : return pattern_stmt;
4147 : }
4148 :
4149 : /* Detect a vector by vector shift pattern that wouldn't be otherwise
4150 : vectorized:
4151 :
4152 : type a_t;
4153 : TYPE b_T, res_T;
4154 :
4155 : S1 a_t = ;
4156 : S2 b_T = ;
4157 : S3 res_T = b_T op a_t;
4158 :
4159 : where type 'TYPE' is a type with different size than 'type',
4160 : and op is <<, >> or rotate.
4161 :
4162 : Also detect cases:
4163 :
4164 : type a_t;
4165 : TYPE b_T, c_T, res_T;
4166 :
4167 : S0 c_T = ;
4168 : S1 a_t = (type) c_T;
4169 : S2 b_T = ;
4170 : S3 res_T = b_T op a_t;
4171 :
4172 : Input/Output:
4173 :
4174 : * STMT_VINFO: The stmt from which the pattern search begins,
4175 : i.e. the shift/rotate stmt. The original stmt (S3) is replaced
4176 : with a shift/rotate which has same type on both operands, in the
4177 : second case just b_T op c_T, in the first case with added cast
4178 : from a_t to c_T in STMT_VINFO_PATTERN_DEF_SEQ.
4179 :
4180 : Output:
4181 :
4182 : * TYPE_OUT: The type of the output of this pattern.
4183 :
4184 : * Return value: A new stmt that will be used to replace the shift/rotate
4185 : S3 stmt. */
4186 :
4187 : static gimple *
4188 33024638 : vect_recog_vector_vector_shift_pattern (vec_info *vinfo,
4189 : stmt_vec_info stmt_vinfo,
4190 : tree *type_out)
4191 : {
4192 33024638 : gimple *last_stmt = stmt_vinfo->stmt;
4193 33024638 : tree oprnd0, oprnd1, lhs, var;
4194 33024638 : gimple *pattern_stmt;
4195 33024638 : enum tree_code rhs_code;
4196 :
4197 33024638 : if (!is_gimple_assign (last_stmt))
4198 : return NULL;
4199 :
4200 22998786 : rhs_code = gimple_assign_rhs_code (last_stmt);
4201 22998786 : switch (rhs_code)
4202 : {
4203 517190 : case LSHIFT_EXPR:
4204 517190 : case RSHIFT_EXPR:
4205 517190 : case LROTATE_EXPR:
4206 517190 : case RROTATE_EXPR:
4207 517190 : break;
4208 : default:
4209 : return NULL;
4210 : }
4211 :
4212 517190 : lhs = gimple_assign_lhs (last_stmt);
4213 517190 : oprnd0 = gimple_assign_rhs1 (last_stmt);
4214 517190 : oprnd1 = gimple_assign_rhs2 (last_stmt);
4215 517190 : if (TREE_CODE (oprnd1) != SSA_NAME
4216 111938 : || TYPE_MODE (TREE_TYPE (oprnd0)) == TYPE_MODE (TREE_TYPE (oprnd1))
4217 49781 : || !INTEGRAL_TYPE_P (TREE_TYPE (oprnd0))
4218 49328 : || !type_has_mode_precision_p (TREE_TYPE (oprnd1))
4219 566518 : || TYPE_PRECISION (TREE_TYPE (lhs))
4220 49328 : != TYPE_PRECISION (TREE_TYPE (oprnd0)))
4221 : return NULL;
4222 :
4223 49328 : stmt_vec_info def_vinfo = vinfo->lookup_def (oprnd1);
4224 49328 : if (!def_vinfo || STMT_VINFO_DEF_TYPE (def_vinfo) == vect_external_def)
4225 : return NULL;
4226 :
4227 46484 : def_vinfo = vect_stmt_to_vectorize (def_vinfo);
4228 1134 : gcc_assert (def_vinfo);
4229 :
4230 46484 : *type_out = get_vectype_for_scalar_type (vinfo, TREE_TYPE (oprnd0));
4231 46484 : if (*type_out == NULL_TREE)
4232 : return NULL;
4233 :
4234 33525 : tree def = NULL_TREE;
4235 33525 : gassign *def_stmt = dyn_cast <gassign *> (def_vinfo->stmt);
4236 19671 : if (def_stmt && gimple_assign_cast_p (def_stmt))
4237 : {
4238 5497 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
4239 5497 : if (TYPE_MODE (TREE_TYPE (rhs1)) == TYPE_MODE (TREE_TYPE (oprnd0))
4240 5497 : && TYPE_PRECISION (TREE_TYPE (rhs1))
4241 1284 : == TYPE_PRECISION (TREE_TYPE (oprnd0)))
4242 : {
4243 1284 : if (TYPE_PRECISION (TREE_TYPE (oprnd1))
4244 1284 : >= TYPE_PRECISION (TREE_TYPE (rhs1)))
4245 : def = rhs1;
4246 : else
4247 : {
4248 1182 : tree mask
4249 1182 : = build_low_bits_mask (TREE_TYPE (rhs1),
4250 1182 : TYPE_PRECISION (TREE_TYPE (oprnd1)));
4251 1182 : def = vect_recog_temp_ssa_var (TREE_TYPE (rhs1), NULL);
4252 1182 : def_stmt = gimple_build_assign (def, BIT_AND_EXPR, rhs1, mask);
4253 1182 : tree vecstype = get_vectype_for_scalar_type (vinfo,
4254 1182 : TREE_TYPE (rhs1));
4255 1182 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecstype);
4256 : }
4257 : }
4258 : }
4259 :
4260 1284 : if (def == NULL_TREE)
4261 : {
4262 32241 : def = vect_recog_temp_ssa_var (TREE_TYPE (oprnd0), NULL);
4263 32241 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd1);
4264 32241 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
4265 : }
4266 :
4267 : /* Pattern detected. */
4268 33525 : vect_pattern_detected ("vect_recog_vector_vector_shift_pattern", last_stmt);
4269 :
4270 : /* Pattern supported. Create a stmt to be used to replace the pattern. */
4271 33525 : var = vect_recog_temp_ssa_var (TREE_TYPE (oprnd0), NULL);
4272 33525 : pattern_stmt = gimple_build_assign (var, rhs_code, oprnd0, def);
4273 :
4274 33525 : return pattern_stmt;
4275 : }
4276 :
4277 : /* Verify that the target has optabs of VECTYPE to perform all the steps
4278 : needed by the multiplication-by-immediate synthesis algorithm described by
4279 : ALG and VAR. If SYNTH_SHIFT_P is true ensure that vector addition is
4280 : present. Return true iff the target supports all the steps. */
4281 :
4282 : static bool
4283 311886 : target_supports_mult_synth_alg (struct algorithm *alg, mult_variant var,
4284 : tree vectype, bool synth_shift_p)
4285 : {
4286 311886 : if (alg->op[0] != alg_zero && alg->op[0] != alg_m)
4287 : return false;
4288 :
4289 311886 : bool supports_vminus = target_has_vecop_for_code (MINUS_EXPR, vectype);
4290 311886 : bool supports_vplus = target_has_vecop_for_code (PLUS_EXPR, vectype);
4291 :
4292 311886 : if (var == negate_variant
4293 311886 : && !target_has_vecop_for_code (NEGATE_EXPR, vectype))
4294 : return false;
4295 :
4296 : /* If we must synthesize shifts with additions make sure that vector
4297 : addition is available. */
4298 311229 : if ((var == add_variant || synth_shift_p) && !supports_vplus)
4299 : return false;
4300 :
4301 150004 : for (int i = 1; i < alg->ops; i++)
4302 : {
4303 112605 : switch (alg->op[i])
4304 : {
4305 : case alg_shift:
4306 : break;
4307 27591 : case alg_add_t_m2:
4308 27591 : case alg_add_t2_m:
4309 27591 : case alg_add_factor:
4310 27591 : if (!supports_vplus)
4311 : return false;
4312 : break;
4313 17635 : case alg_sub_t_m2:
4314 17635 : case alg_sub_t2_m:
4315 17635 : case alg_sub_factor:
4316 17635 : if (!supports_vminus)
4317 : return false;
4318 : break;
4319 : case alg_unknown:
4320 : case alg_m:
4321 : case alg_zero:
4322 : case alg_impossible:
4323 : return false;
4324 0 : default:
4325 0 : gcc_unreachable ();
4326 : }
4327 : }
4328 :
4329 : return true;
4330 : }
4331 :
4332 : /* Synthesize a left shift of OP by AMNT bits using a series of additions and
4333 : putting the final result in DEST. Append all statements but the last into
4334 : VINFO. Return the last statement. */
4335 :
4336 : static gimple *
4337 0 : synth_lshift_by_additions (vec_info *vinfo,
4338 : tree dest, tree op, HOST_WIDE_INT amnt,
4339 : stmt_vec_info stmt_info, tree vectype)
4340 : {
4341 0 : HOST_WIDE_INT i;
4342 0 : tree itype = TREE_TYPE (op);
4343 0 : tree prev_res = op;
4344 0 : gcc_assert (amnt >= 0);
4345 0 : for (i = 0; i < amnt; i++)
4346 : {
4347 0 : tree tmp_var = (i < amnt - 1) ? vect_recog_temp_ssa_var (itype, NULL)
4348 : : dest;
4349 0 : gimple *stmt
4350 0 : = gimple_build_assign (tmp_var, PLUS_EXPR, prev_res, prev_res);
4351 0 : prev_res = tmp_var;
4352 0 : if (i < amnt - 1)
4353 0 : append_pattern_def_seq (vinfo, stmt_info, stmt, vectype);
4354 : else
4355 0 : return stmt;
4356 : }
4357 0 : gcc_unreachable ();
4358 : return NULL;
4359 : }
4360 :
4361 : /* Helper for vect_synth_mult_by_constant. Apply a binary operation
4362 : CODE to operands OP1 and OP2, creating a new temporary SSA var in
4363 : the process if necessary. Append the resulting assignment statements
4364 : to the sequence in STMT_VINFO. Return the SSA variable that holds the
4365 : result of the binary operation. If SYNTH_SHIFT_P is true synthesize
4366 : left shifts using additions. */
4367 :
4368 : static tree
4369 44908 : apply_binop_and_append_stmt (vec_info *vinfo,
4370 : tree_code code, tree op1, tree op2,
4371 : stmt_vec_info stmt_vinfo, tree vectype,
4372 : bool synth_shift_p)
4373 : {
4374 44908 : if (integer_zerop (op2)
4375 44908 : && (code == LSHIFT_EXPR
4376 39029 : || code == PLUS_EXPR))
4377 : {
4378 39029 : gcc_assert (TREE_CODE (op1) == SSA_NAME);
4379 : return op1;
4380 : }
4381 :
4382 5879 : gimple *stmt;
4383 5879 : tree itype = TREE_TYPE (op1);
4384 5879 : tree tmp_var = vect_recog_temp_ssa_var (itype, NULL);
4385 :
4386 5879 : if (code == LSHIFT_EXPR
4387 5879 : && synth_shift_p)
4388 : {
4389 0 : stmt = synth_lshift_by_additions (vinfo, tmp_var, op1,
4390 0 : TREE_INT_CST_LOW (op2), stmt_vinfo,
4391 : vectype);
4392 0 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4393 0 : return tmp_var;
4394 : }
4395 :
4396 5879 : stmt = gimple_build_assign (tmp_var, code, op1, op2);
4397 5879 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4398 5879 : return tmp_var;
4399 : }
4400 :
4401 : /* Synthesize a multiplication of OP by an INTEGER_CST VAL using shifts
4402 : and simple arithmetic operations to be vectorized. Record the statements
4403 : produced in STMT_VINFO and return the last statement in the sequence or
4404 : NULL if it's not possible to synthesize such a multiplication.
4405 : This function mirrors the behavior of expand_mult_const in expmed.cc but
4406 : works on tree-ssa form. */
4407 :
4408 : static gimple *
4409 315373 : vect_synth_mult_by_constant (vec_info *vinfo, tree op, tree val,
4410 : stmt_vec_info stmt_vinfo)
4411 : {
4412 315373 : tree itype = TREE_TYPE (op);
4413 315373 : machine_mode mode = TYPE_MODE (itype);
4414 315373 : struct algorithm alg;
4415 315373 : mult_variant variant;
4416 315373 : if (!tree_fits_shwi_p (val))
4417 : return NULL;
4418 :
4419 : /* Multiplication synthesis by shifts, adds and subs can introduce
4420 : signed overflow where the original operation didn't. Perform the
4421 : operations on an unsigned type and cast back to avoid this.
4422 : In the future we may want to relax this for synthesis algorithms
4423 : that we can prove do not cause unexpected overflow. */
4424 311901 : bool cast_to_unsigned_p = !TYPE_OVERFLOW_WRAPS (itype);
4425 :
4426 60087 : tree multtype = cast_to_unsigned_p ? unsigned_type_for (itype) : itype;
4427 311901 : tree vectype = get_vectype_for_scalar_type (vinfo, multtype);
4428 311901 : if (!vectype)
4429 : return NULL;
4430 :
4431 : /* Targets that don't support vector shifts but support vector additions
4432 : can synthesize shifts that way. */
4433 311901 : bool synth_shift_p = !vect_supportable_shift (vinfo, LSHIFT_EXPR, multtype);
4434 :
4435 311901 : HOST_WIDE_INT hwval = tree_to_shwi (val);
4436 : /* Use MAX_COST here as we don't want to limit the sequence on rtx costs.
4437 : The vectorizer's benefit analysis will decide whether it's beneficial
4438 : to do this. */
4439 623606 : bool possible = choose_mult_variant (VECTOR_MODE_P (TYPE_MODE (vectype))
4440 311705 : ? TYPE_MODE (vectype) : mode,
4441 : hwval, &alg, &variant, MAX_COST);
4442 311901 : if (!possible)
4443 : return NULL;
4444 :
4445 311901 : if (vect_is_reduction (stmt_vinfo))
4446 : {
4447 26 : int op_uses = alg.op[0] != alg_zero;
4448 45 : for (int i = 1; i < alg.ops; i++)
4449 32 : switch (alg.op[i])
4450 : {
4451 4 : case alg_add_t_m2:
4452 4 : case alg_sub_t_m2:
4453 4 : if (synth_shift_p && alg.log[i])
4454 : return NULL;
4455 : else
4456 4 : op_uses++;
4457 4 : break;
4458 0 : case alg_add_t2_m:
4459 0 : case alg_sub_t2_m:
4460 0 : op_uses++;
4461 : /* Fallthru. */
4462 28 : case alg_shift:
4463 28 : if (synth_shift_p && alg.log[i])
4464 : return NULL;
4465 : break;
4466 : case alg_add_factor:
4467 : case alg_sub_factor:
4468 : return NULL;
4469 : default:
4470 : break;
4471 : }
4472 13 : if (variant == add_variant)
4473 0 : op_uses++;
4474 : /* When we'll synthesize more than a single use of the reduction
4475 : operand the reduction constraints are violated. Avoid this
4476 : situation. */
4477 13 : if (op_uses > 1)
4478 : return NULL;
4479 : }
4480 :
4481 311886 : if (!target_supports_mult_synth_alg (&alg, variant, vectype, synth_shift_p))
4482 : return NULL;
4483 :
4484 37399 : tree accumulator;
4485 :
4486 : /* Clear out the sequence of statements so we can populate it below. */
4487 37399 : gimple *stmt = NULL;
4488 :
4489 37399 : if (cast_to_unsigned_p)
4490 : {
4491 12464 : tree tmp_op = vect_recog_temp_ssa_var (multtype, NULL);
4492 12464 : stmt = gimple_build_assign (tmp_op, CONVERT_EXPR, op);
4493 12464 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4494 12464 : op = tmp_op;
4495 : }
4496 :
4497 37399 : if (alg.op[0] == alg_zero)
4498 198 : accumulator = build_int_cst (multtype, 0);
4499 : else
4500 : accumulator = op;
4501 :
4502 37399 : bool needs_fixup = (variant == negate_variant)
4503 37399 : || (variant == add_variant);
4504 :
4505 149401 : for (int i = 1; i < alg.ops; i++)
4506 : {
4507 112002 : tree shft_log = build_int_cst (multtype, alg.log[i]);
4508 112002 : tree accum_tmp = vect_recog_temp_ssa_var (multtype, NULL);
4509 112002 : tree tmp_var = NULL_TREE;
4510 :
4511 112002 : switch (alg.op[i])
4512 : {
4513 67094 : case alg_shift:
4514 67094 : if (synth_shift_p)
4515 0 : stmt
4516 0 : = synth_lshift_by_additions (vinfo, accum_tmp, accumulator,
4517 0 : alg.log[i], stmt_vinfo, vectype);
4518 : else
4519 67094 : stmt = gimple_build_assign (accum_tmp, LSHIFT_EXPR, accumulator,
4520 : shft_log);
4521 : break;
4522 22627 : case alg_add_t_m2:
4523 22627 : tmp_var
4524 22627 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, op, shft_log,
4525 : stmt_vinfo, vectype, synth_shift_p);
4526 22627 : stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, accumulator,
4527 : tmp_var);
4528 22627 : break;
4529 16605 : case alg_sub_t_m2:
4530 16605 : tmp_var = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, op,
4531 : shft_log, stmt_vinfo,
4532 : vectype, synth_shift_p);
4533 : /* In some algorithms the first step involves zeroing the
4534 : accumulator. If subtracting from such an accumulator
4535 : just emit the negation directly. */
4536 16605 : if (integer_zerop (accumulator))
4537 198 : stmt = gimple_build_assign (accum_tmp, NEGATE_EXPR, tmp_var);
4538 : else
4539 16407 : stmt = gimple_build_assign (accum_tmp, MINUS_EXPR, accumulator,
4540 : tmp_var);
4541 : break;
4542 0 : case alg_add_t2_m:
4543 0 : tmp_var
4544 0 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
4545 : shft_log, stmt_vinfo, vectype,
4546 : synth_shift_p);
4547 0 : stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, tmp_var, op);
4548 0 : break;
4549 0 : case alg_sub_t2_m:
4550 0 : tmp_var
4551 0 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
4552 : shft_log, stmt_vinfo, vectype,
4553 : synth_shift_p);
4554 0 : stmt = gimple_build_assign (accum_tmp, MINUS_EXPR, tmp_var, op);
4555 0 : break;
4556 4840 : case alg_add_factor:
4557 4840 : tmp_var
4558 4840 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
4559 : shft_log, stmt_vinfo, vectype,
4560 : synth_shift_p);
4561 4840 : stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, accumulator,
4562 : tmp_var);
4563 4840 : break;
4564 836 : case alg_sub_factor:
4565 836 : tmp_var
4566 836 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
4567 : shft_log, stmt_vinfo, vectype,
4568 : synth_shift_p);
4569 836 : stmt = gimple_build_assign (accum_tmp, MINUS_EXPR, tmp_var,
4570 : accumulator);
4571 836 : break;
4572 0 : default:
4573 0 : gcc_unreachable ();
4574 : }
4575 : /* We don't want to append the last stmt in the sequence to stmt_vinfo
4576 : but rather return it directly. */
4577 :
4578 112002 : if ((i < alg.ops - 1) || needs_fixup || cast_to_unsigned_p)
4579 87385 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4580 112002 : accumulator = accum_tmp;
4581 : }
4582 37399 : if (variant == negate_variant)
4583 : {
4584 481 : tree accum_tmp = vect_recog_temp_ssa_var (multtype, NULL);
4585 481 : stmt = gimple_build_assign (accum_tmp, NEGATE_EXPR, accumulator);
4586 481 : accumulator = accum_tmp;
4587 481 : if (cast_to_unsigned_p)
4588 169 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4589 : }
4590 36918 : else if (variant == add_variant)
4591 : {
4592 97 : tree accum_tmp = vect_recog_temp_ssa_var (multtype, NULL);
4593 97 : stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, accumulator, op);
4594 97 : accumulator = accum_tmp;
4595 97 : if (cast_to_unsigned_p)
4596 91 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4597 : }
4598 : /* Move back to a signed if needed. */
4599 37081 : if (cast_to_unsigned_p)
4600 : {
4601 12464 : tree accum_tmp = vect_recog_temp_ssa_var (itype, NULL);
4602 12464 : stmt = gimple_build_assign (accum_tmp, CONVERT_EXPR, accumulator);
4603 : }
4604 :
4605 : return stmt;
4606 : }
4607 :
4608 : /* Detect multiplication by constant and convert it into a sequence of
4609 : shifts and additions, subtractions, negations. We reuse the
4610 : choose_mult_variant algorithms from expmed.cc
4611 :
4612 : Input/Output:
4613 :
4614 : STMT_VINFO: The stmt from which the pattern search begins,
4615 : i.e. the mult stmt.
4616 :
4617 : Output:
4618 :
4619 : * TYPE_OUT: The type of the output of this pattern.
4620 :
4621 : * Return value: A new stmt that will be used to replace
4622 : the multiplication. */
4623 :
4624 : static gimple *
4625 33220615 : vect_recog_mult_pattern (vec_info *vinfo,
4626 : stmt_vec_info stmt_vinfo, tree *type_out)
4627 : {
4628 33220615 : gimple *last_stmt = stmt_vinfo->stmt;
4629 33220615 : tree oprnd0, oprnd1, vectype, itype;
4630 33220615 : gimple *pattern_stmt;
4631 :
4632 33220615 : if (!is_gimple_assign (last_stmt))
4633 : return NULL;
4634 :
4635 23194763 : if (gimple_assign_rhs_code (last_stmt) != MULT_EXPR)
4636 : return NULL;
4637 :
4638 1564271 : oprnd0 = gimple_assign_rhs1 (last_stmt);
4639 1564271 : oprnd1 = gimple_assign_rhs2 (last_stmt);
4640 1564271 : itype = TREE_TYPE (oprnd0);
4641 :
4642 1564271 : if (TREE_CODE (oprnd0) != SSA_NAME
4643 1564204 : || TREE_CODE (oprnd1) != INTEGER_CST
4644 938833 : || !INTEGRAL_TYPE_P (itype)
4645 2503104 : || !type_has_mode_precision_p (itype))
4646 : return NULL;
4647 :
4648 938778 : vectype = get_vectype_for_scalar_type (vinfo, itype);
4649 938778 : if (vectype == NULL_TREE)
4650 : return NULL;
4651 :
4652 : /* If the target can handle vectorized multiplication natively,
4653 : don't attempt to optimize this. */
4654 763629 : optab mul_optab = optab_for_tree_code (MULT_EXPR, vectype, optab_default);
4655 763629 : if (mul_optab != unknown_optab
4656 763629 : && can_implement_p (mul_optab, TYPE_MODE (vectype)))
4657 : return NULL;
4658 :
4659 315373 : pattern_stmt = vect_synth_mult_by_constant (vinfo,
4660 : oprnd0, oprnd1, stmt_vinfo);
4661 315373 : if (!pattern_stmt)
4662 : return NULL;
4663 :
4664 : /* Pattern detected. */
4665 37399 : vect_pattern_detected ("vect_recog_mult_pattern", last_stmt);
4666 :
4667 37399 : *type_out = vectype;
4668 :
4669 37399 : return pattern_stmt;
4670 : }
4671 :
4672 : extern bool gimple_unsigned_integer_sat_add (tree, tree*, tree (*)(tree));
4673 : extern bool gimple_unsigned_integer_sat_sub (tree, tree*, tree (*)(tree));
4674 : extern bool gimple_unsigned_integer_sat_trunc (tree, tree*, tree (*)(tree));
4675 :
4676 : extern bool gimple_unsigned_integer_narrow_clip (tree, tree*, tree (*)(tree));
4677 :
4678 : extern bool gimple_signed_integer_sat_add (tree, tree*, tree (*)(tree));
4679 : extern bool gimple_signed_integer_sat_sub (tree, tree*, tree (*)(tree));
4680 : extern bool gimple_signed_integer_sat_trunc (tree, tree*, tree (*)(tree));
4681 :
4682 : static gimple *
4683 1269 : vect_recog_build_binary_gimple_stmt (vec_info *vinfo, stmt_vec_info stmt_info,
4684 : internal_fn fn, tree *type_out,
4685 : tree lhs, tree op_0, tree op_1)
4686 : {
4687 1269 : tree itype = TREE_TYPE (op_0);
4688 1269 : tree otype = TREE_TYPE (lhs);
4689 1269 : tree v_itype = get_vectype_for_scalar_type (vinfo, itype);
4690 1269 : tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
4691 :
4692 1269 : if (v_itype != NULL_TREE && v_otype != NULL_TREE
4693 1269 : && direct_internal_fn_supported_p (fn, v_itype, OPTIMIZE_FOR_BOTH))
4694 : {
4695 1011 : gcall *call = gimple_build_call_internal (fn, 2, op_0, op_1);
4696 1011 : tree in_ssa = vect_recog_temp_ssa_var (itype, NULL);
4697 :
4698 1011 : gimple_call_set_lhs (call, in_ssa);
4699 1011 : gimple_call_set_nothrow (call, /* nothrow_p */ false);
4700 1011 : gimple_set_location (call, gimple_location (STMT_VINFO_STMT (stmt_info)));
4701 :
4702 1011 : *type_out = v_otype;
4703 :
4704 1011 : if (types_compatible_p (itype, otype))
4705 : return call;
4706 : else
4707 : {
4708 0 : append_pattern_def_seq (vinfo, stmt_info, call, v_itype);
4709 0 : tree out_ssa = vect_recog_temp_ssa_var (otype, NULL);
4710 :
4711 0 : return gimple_build_assign (out_ssa, NOP_EXPR, in_ssa);
4712 : }
4713 : }
4714 :
4715 : return NULL;
4716 : }
4717 :
4718 : /*
4719 : * Try to detect saturation add pattern (SAT_ADD), aka below gimple:
4720 : * _7 = _4 + _6;
4721 : * _8 = _4 > _7;
4722 : * _9 = (long unsigned int) _8;
4723 : * _10 = -_9;
4724 : * _12 = _7 | _10;
4725 : *
4726 : * And then simplified to
4727 : * _12 = .SAT_ADD (_4, _6);
4728 : */
4729 :
4730 : static gimple *
4731 33294386 : vect_recog_sat_add_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
4732 : tree *type_out)
4733 : {
4734 33294386 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
4735 :
4736 33294386 : if (!is_gimple_assign (last_stmt))
4737 : return NULL;
4738 :
4739 23268534 : tree ops[2];
4740 23268534 : tree lhs = gimple_assign_lhs (last_stmt);
4741 :
4742 23268534 : if (gimple_unsigned_integer_sat_add (lhs, ops, NULL)
4743 23268534 : || gimple_signed_integer_sat_add (lhs, ops, NULL))
4744 : {
4745 : /* .SAT_ADD saturates in the type of its first operand, which must be
4746 : the type of the result. */
4747 831 : gcc_checking_assert (types_compatible_p (TREE_TYPE (lhs),
4748 : TREE_TYPE (ops[0])));
4749 831 : if (TREE_CODE (ops[1]) == INTEGER_CST)
4750 13 : ops[1] = fold_convert (TREE_TYPE (ops[0]), ops[1]);
4751 :
4752 831 : gimple *stmt = vect_recog_build_binary_gimple_stmt (vinfo, stmt_vinfo,
4753 : IFN_SAT_ADD, type_out,
4754 : lhs, ops[0], ops[1]);
4755 831 : if (stmt)
4756 : {
4757 796 : vect_pattern_detected ("vect_recog_sat_add_pattern", last_stmt);
4758 796 : return stmt;
4759 : }
4760 : }
4761 :
4762 : return NULL;
4763 : }
4764 :
4765 : /*
4766 : * Try to transform the truncation for .SAT_SUB pattern, mostly occurs in
4767 : * the benchmark zip. Aka:
4768 : *
4769 : * unsigned int _1;
4770 : * unsigned int _2;
4771 : * unsigned short int _4;
4772 : * _9 = (unsigned short int).SAT_SUB (_1, _2);
4773 : *
4774 : * if _1 is known to be in the range of unsigned short int. For example
4775 : * there is a def _1 = (unsigned short int)_4. Then we can transform the
4776 : * truncation to:
4777 : *
4778 : * _3 = (unsigned short int) MIN (65535, _2); // aka _3 = .SAT_TRUNC (_2);
4779 : * _9 = .SAT_SUB (_4, _3);
4780 : *
4781 : * Then, we can better vectorized code and avoid the unnecessary narrowing
4782 : * stmt during vectorization with below stmt(s).
4783 : *
4784 : * _3 = .SAT_TRUNC(_2); // SI => HI
4785 : * _9 = .SAT_SUB (_4, _3);
4786 : */
4787 : static void
4788 438 : vect_recog_sat_sub_pattern_transform (vec_info *vinfo,
4789 : stmt_vec_info stmt_vinfo,
4790 : tree lhs, tree *ops)
4791 : {
4792 438 : tree otype = TREE_TYPE (lhs);
4793 438 : tree itype = TREE_TYPE (ops[0]);
4794 438 : unsigned itype_prec = TYPE_PRECISION (itype);
4795 438 : unsigned otype_prec = TYPE_PRECISION (otype);
4796 :
4797 438 : if (types_compatible_p (otype, itype) || otype_prec >= itype_prec)
4798 438 : return;
4799 :
4800 0 : tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
4801 0 : tree v_itype = get_vectype_for_scalar_type (vinfo, itype);
4802 0 : tree_pair v_pair = tree_pair (v_otype, v_itype);
4803 :
4804 0 : if (v_otype == NULL_TREE || v_itype == NULL_TREE
4805 0 : || !direct_internal_fn_supported_p (IFN_SAT_TRUNC, v_pair,
4806 : OPTIMIZE_FOR_BOTH))
4807 : return;
4808 :
4809 : /* 1. Find the _4 and update ops[0] as above example. */
4810 0 : vect_unpromoted_value unprom;
4811 0 : tree tmp = vect_look_through_possible_promotion (vinfo, ops[0], &unprom);
4812 :
4813 0 : if (tmp == NULL_TREE || TYPE_PRECISION (unprom.type) != otype_prec)
4814 : return;
4815 :
4816 0 : ops[0] = tmp;
4817 :
4818 : /* 2. Generate _3 = .SAT_TRUNC (_2) and update ops[1] as above example. */
4819 0 : tree trunc_lhs_ssa = vect_recog_temp_ssa_var (otype, NULL);
4820 0 : gcall *call = gimple_build_call_internal (IFN_SAT_TRUNC, 1, ops[1]);
4821 :
4822 0 : gimple_call_set_lhs (call, trunc_lhs_ssa);
4823 0 : gimple_call_set_nothrow (call, /* nothrow_p */ false);
4824 0 : append_pattern_def_seq (vinfo, stmt_vinfo, call, v_otype);
4825 :
4826 0 : ops[1] = trunc_lhs_ssa;
4827 : }
4828 :
4829 : /*
4830 : * Try to detect saturation sub pattern (SAT_ADD), aka below gimple:
4831 : * Unsigned:
4832 : * _7 = _1 >= _2;
4833 : * _8 = _1 - _2;
4834 : * _10 = (long unsigned int) _7;
4835 : * _9 = _8 * _10;
4836 : *
4837 : * And then simplified to
4838 : * _9 = .SAT_SUB (_1, _2);
4839 : *
4840 : * Signed:
4841 : * x.0_4 = (unsigned char) x_16;
4842 : * y.1_5 = (unsigned char) y_18;
4843 : * _6 = x.0_4 - y.1_5;
4844 : * minus_19 = (int8_t) _6;
4845 : * _7 = x_16 ^ y_18;
4846 : * _8 = x_16 ^ minus_19;
4847 : * _44 = _7 < 0;
4848 : * _23 = x_16 < 0;
4849 : * _24 = (signed char) _23;
4850 : * _58 = (unsigned char) _24;
4851 : * _59 = -_58;
4852 : * _25 = (signed char) _59;
4853 : * _26 = _25 ^ 127;
4854 : * _42 = _8 < 0;
4855 : * _41 = _42 & _44;
4856 : * iftmp.2_11 = _41 ? _26 : minus_19;
4857 : *
4858 : * And then simplified to
4859 : * iftmp.2_11 = .SAT_SUB (x_16, y_18);
4860 : */
4861 :
4862 : static gimple *
4863 33293590 : vect_recog_sat_sub_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
4864 : tree *type_out)
4865 : {
4866 33293590 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
4867 :
4868 33293590 : if (!is_gimple_assign (last_stmt))
4869 : return NULL;
4870 :
4871 23267738 : tree ops[2];
4872 23267738 : tree lhs = gimple_assign_lhs (last_stmt);
4873 :
4874 23267738 : if (gimple_unsigned_integer_sat_sub (lhs, ops, NULL)
4875 23267738 : || gimple_signed_integer_sat_sub (lhs, ops, NULL))
4876 : {
4877 438 : vect_recog_sat_sub_pattern_transform (vinfo, stmt_vinfo, lhs, ops);
4878 438 : gimple *stmt = vect_recog_build_binary_gimple_stmt (vinfo, stmt_vinfo,
4879 : IFN_SAT_SUB, type_out,
4880 : lhs, ops[0], ops[1]);
4881 438 : if (stmt)
4882 : {
4883 215 : vect_pattern_detected ("vect_recog_sat_sub_pattern", last_stmt);
4884 215 : return stmt;
4885 : }
4886 : }
4887 :
4888 : return NULL;
4889 : }
4890 :
4891 : /* Return the vector type for ITYPE if the target supports the MAX_EXPR
4892 : and SAT_TRUNC sequence. */
4893 :
4894 : static tree
4895 73 : vect_sat_trunc_clip_input_vectype (vec_info *vinfo, tree v_otype,
4896 : tree itype)
4897 : {
4898 73 : tree v_itype;
4899 73 : if (!vect_supportable_direct_optab_p
4900 73 : (vinfo, itype, MAX_EXPR, itype, &v_itype)
4901 95 : || !direct_internal_fn_supported_p
4902 95 : (IFN_SAT_TRUNC, tree_pair (v_otype, v_itype),
4903 : OPTIMIZE_FOR_BOTH))
4904 : return NULL_TREE;
4905 :
4906 0 : return v_itype;
4907 : }
4908 :
4909 : /*
4910 : * Try to detect saturation truncation pattern (SAT_TRUNC), aka below gimple:
4911 : * overflow_5 = x_4(D) > 4294967295;
4912 : * _1 = (unsigned int) x_4(D);
4913 : * _2 = (unsigned int) overflow_5;
4914 : * _3 = -_2;
4915 : * _6 = _1 | _3;
4916 : *
4917 : * And then simplified to
4918 : * _6 = .SAT_TRUNC (x_4(D));
4919 : */
4920 :
4921 : static gimple *
4922 33293375 : vect_recog_sat_trunc_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
4923 : tree *type_out)
4924 : {
4925 33293375 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
4926 :
4927 33293375 : if (!is_gimple_assign (last_stmt))
4928 : return NULL;
4929 :
4930 23267523 : tree ops[1];
4931 23267523 : tree lhs = gimple_assign_lhs (last_stmt);
4932 23267523 : tree otype = TREE_TYPE (lhs);
4933 :
4934 23267523 : if ((gimple_unsigned_integer_narrow_clip (lhs, ops, NULL))
4935 23267523 : && type_has_mode_precision_p (otype))
4936 : {
4937 73 : tree input = ops[0];
4938 73 : tree itype = TREE_TYPE (input);
4939 73 : tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
4940 73 : internal_fn fn = IFN_SAT_TRUNC;
4941 73 : tree v_itype = NULL_TREE;
4942 :
4943 73 : if (v_otype != NULL_TREE)
4944 : {
4945 : /* Avoid widening an input only to narrow it again. A signed
4946 : promotion does not affect MAX (X, 0), so use the unpromoted
4947 : value when the target can narrow it directly to the output
4948 : type. */
4949 73 : vect_unpromoted_value unprom;
4950 73 : bool single_use_p = true;
4951 73 : if (vect_look_through_possible_promotion (vinfo, input, &unprom,
4952 : &single_use_p)
4953 73 : && single_use_p
4954 34 : && INTEGRAL_TYPE_P (unprom.type)
4955 34 : && TYPE_SIGN (unprom.type) == TYPE_SIGN (itype)
4956 34 : && type_has_mode_precision_p (unprom.type)
4957 34 : && TYPE_PRECISION (otype) < TYPE_PRECISION (unprom.type)
4958 107 : && TYPE_PRECISION (unprom.type) < TYPE_PRECISION (itype))
4959 : {
4960 0 : tree v_unprom_type
4961 0 : = vect_sat_trunc_clip_input_vectype (vinfo, v_otype,
4962 : unprom.type);
4963 0 : if (v_unprom_type != NULL_TREE)
4964 : {
4965 0 : input = unprom.op;
4966 0 : itype = unprom.type;
4967 0 : v_itype = v_unprom_type;
4968 : }
4969 : }
4970 :
4971 0 : if (v_itype == NULL_TREE)
4972 73 : v_itype
4973 73 : = vect_sat_trunc_clip_input_vectype (vinfo, v_otype, itype);
4974 :
4975 73 : if (v_itype != NULL_TREE)
4976 : {
4977 0 : tree temp = vect_recog_temp_ssa_var (itype, NULL);
4978 0 : gimple *max_stmt
4979 0 : = gimple_build_assign (temp, MAX_EXPR,
4980 : build_zero_cst (itype), input);
4981 0 : append_pattern_def_seq (vinfo, stmt_vinfo, max_stmt, v_itype);
4982 :
4983 0 : gcall *call = gimple_build_call_internal (fn, 1, temp);
4984 0 : tree out_ssa = vect_recog_temp_ssa_var (otype, NULL);
4985 :
4986 0 : gimple_call_set_lhs (call, out_ssa);
4987 0 : gimple_call_set_nothrow (call, false);
4988 0 : gimple_set_location (call, gimple_location (last_stmt));
4989 :
4990 0 : *type_out = v_otype;
4991 :
4992 0 : return call;
4993 : }
4994 : }
4995 : }
4996 :
4997 23267523 : if ((gimple_unsigned_integer_sat_trunc (lhs, ops, NULL)
4998 23267138 : || gimple_signed_integer_sat_trunc (lhs, ops, NULL))
4999 23267695 : && type_has_mode_precision_p (otype))
5000 : {
5001 545 : tree itype = TREE_TYPE (ops[0]);
5002 545 : tree v_itype = get_vectype_for_scalar_type (vinfo, itype);
5003 545 : tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
5004 545 : internal_fn fn = IFN_SAT_TRUNC;
5005 :
5006 537 : if (v_itype != NULL_TREE && v_otype != NULL_TREE
5007 1082 : && direct_internal_fn_supported_p (fn, tree_pair (v_otype, v_itype),
5008 : OPTIMIZE_FOR_BOTH))
5009 : {
5010 0 : gcall *call = gimple_build_call_internal (fn, 1, ops[0]);
5011 0 : tree out_ssa = vect_recog_temp_ssa_var (otype, NULL);
5012 :
5013 0 : gimple_call_set_lhs (call, out_ssa);
5014 0 : gimple_call_set_nothrow (call, /* nothrow_p */ false);
5015 0 : gimple_set_location (call, gimple_location (last_stmt));
5016 :
5017 0 : *type_out = v_otype;
5018 :
5019 0 : return call;
5020 : }
5021 : }
5022 :
5023 : return NULL;
5024 : }
5025 :
5026 :
5027 : /* Function add_code_for_floorceilround_divmod
5028 : A helper function to add compensation code for implementing FLOOR_MOD_EXPR,
5029 : FLOOR_DIV_EXPR, CEIL_MOD_EXPR, CEIL_DIV_EXPR, ROUND_MOD_EXPR and
5030 : ROUND_DIV_EXPR
5031 : The quotient and remainder are needed for implemented these operators.
5032 : FLOOR cases
5033 : r = x %[fl] y; r = x/[fl] y;
5034 : is
5035 : r = x % y; if (r && (x ^ y) < 0) r += y;
5036 : r = x % y; d = x/y; if (r && (x ^ y) < 0) d--; Respectively
5037 : Produce following sequence
5038 : v0 = x^y
5039 : v1 = -r
5040 : v2 = r | -r
5041 : v3 = v0 & v2
5042 : v4 = v3 < 0
5043 : if (floor_mod)
5044 : v5 = v4 ? y : 0
5045 : v6 = r + v5
5046 : if (floor_div)
5047 : v5 = v4 ? 1 : 0
5048 : v6 = d - 1
5049 : Similar sequences of vector instructions are produces for following cases
5050 : CEIL cases
5051 : r = x %[cl] y; r = x/[cl] y;
5052 : is
5053 : r = x % y; if (r && (x ^ y) >= 0) r -= y;
5054 : r = x % y; if (r) r -= y; (unsigned)
5055 : r = x % y; d = x/y; if (r && (x ^ y) >= 0) d++;
5056 : r = x % y; d = x/y; if (r) d++; (unsigned)
5057 : ROUND cases
5058 : r = x %[rd] y; r = x/[rd] y;
5059 : is
5060 : r = x % y; if (r > ((y-1)/2)) if ((x ^ y) >= 0) r -= y; else r += y;
5061 : r = x % y; if (r > ((y-1)/2)) r -= y; (unsigned)
5062 : r = x % y; d = x/y; if (r > ((y-1)/2)) if ((x ^ y) >= 0) d++; else d--;
5063 : r = x % y; d = x/y; if (r > ((y-1)/2)) d++; (unsigned)
5064 : Inputs:
5065 : VECTYPE: Vector type of the operands
5066 : STMT_VINFO: Statement where pattern begins
5067 : RHS_CODE: Should either be FLOOR_MOD_EXPR or FLOOR_DIV_EXPR
5068 : Q: The quotient of division
5069 : R: Remainder of division
5070 : OPRDN0/OPRND1: Actual operands involved
5071 : ITYPE: tree type of oprnd0
5072 : Output:
5073 : NULL if vectorization not possible
5074 : Gimple statement based on rhs_code
5075 : */
5076 : static gimple *
5077 414 : add_code_for_floorceilround_divmod (tree vectype, vec_info *vinfo,
5078 : stmt_vec_info stmt_vinfo,
5079 : enum tree_code rhs_code, tree q, tree r,
5080 : tree oprnd0, tree oprnd1, tree itype)
5081 : {
5082 414 : gimple *def_stmt;
5083 414 : tree mask_vectype = truth_type_for (vectype);
5084 414 : if (!mask_vectype)
5085 : return NULL;
5086 414 : tree bool_cond;
5087 414 : bool unsigned_p = TYPE_UNSIGNED (itype);
5088 :
5089 414 : switch (rhs_code)
5090 : {
5091 378 : case FLOOR_MOD_EXPR:
5092 378 : case FLOOR_DIV_EXPR:
5093 378 : case CEIL_MOD_EXPR:
5094 378 : case CEIL_DIV_EXPR:
5095 378 : {
5096 378 : if (!target_has_vecop_for_code (NEGATE_EXPR, vectype)
5097 354 : || !target_has_vecop_for_code (BIT_XOR_EXPR, vectype)
5098 354 : || !target_has_vecop_for_code (BIT_IOR_EXPR, vectype)
5099 354 : || !target_has_vecop_for_code (PLUS_EXPR, vectype)
5100 354 : || !target_has_vecop_for_code (MINUS_EXPR, vectype)
5101 354 : || !expand_vec_cmp_expr_p (vectype, mask_vectype, LT_EXPR)
5102 602 : || !expand_vec_cond_expr_p (vectype, mask_vectype))
5103 : return NULL;
5104 224 : if (unsigned_p)
5105 : {
5106 18 : gcc_assert (rhs_code == CEIL_MOD_EXPR || rhs_code == CEIL_DIV_EXPR);
5107 :
5108 18 : if (!expand_vec_cmp_expr_p (vectype, mask_vectype, GT_EXPR))
5109 : return NULL;
5110 18 : bool is_mod = rhs_code == CEIL_MOD_EXPR;
5111 : // r > 0
5112 18 : bool_cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5113 18 : def_stmt = gimple_build_assign (bool_cond, GT_EXPR, r,
5114 : build_int_cst (itype, 0));
5115 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5116 : itype);
5117 :
5118 : // (r > 0) ? y : 0 (mod)
5119 : // (r > 0) ? 1 : 0 (ceil)
5120 18 : tree extr_cond = vect_recog_temp_ssa_var (itype, NULL);
5121 18 : def_stmt
5122 27 : = gimple_build_assign (extr_cond, COND_EXPR, bool_cond,
5123 9 : is_mod ? oprnd1 : build_int_cst (itype, 1),
5124 : build_int_cst (itype, 0));
5125 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5126 :
5127 : // r -= (r > 0) ? y : 0 (mod)
5128 : // d += (x^y < 0 && r) ? -1 : 0 (ceil)
5129 18 : tree result = vect_recog_temp_ssa_var (itype, NULL);
5130 27 : return gimple_build_assign (result, is_mod ? MINUS_EXPR : PLUS_EXPR,
5131 18 : is_mod ? r : q, extr_cond);
5132 : }
5133 : else
5134 : {
5135 206 : bool ceil_p
5136 206 : = (rhs_code == CEIL_MOD_EXPR || rhs_code == CEIL_DIV_EXPR);
5137 206 : if (ceil_p && !target_has_vecop_for_code (BIT_NOT_EXPR, vectype))
5138 : return NULL;
5139 : // x ^ y
5140 206 : tree xort = vect_recog_temp_ssa_var (itype, NULL);
5141 206 : def_stmt = gimple_build_assign (xort, BIT_XOR_EXPR, oprnd0, oprnd1);
5142 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5143 :
5144 206 : tree cond_reg = xort;
5145 : // ~(x ^ y) (ceil)
5146 206 : if (ceil_p)
5147 : {
5148 18 : cond_reg = vect_recog_temp_ssa_var (itype, NULL);
5149 18 : def_stmt = gimple_build_assign (cond_reg, BIT_NOT_EXPR, xort);
5150 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5151 : }
5152 :
5153 : // -r
5154 206 : tree negate_r = vect_recog_temp_ssa_var (itype, NULL);
5155 206 : def_stmt = gimple_build_assign (negate_r, NEGATE_EXPR, r);
5156 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5157 :
5158 : // r | -r , sign bit is set if r!=0
5159 206 : tree r_or_negr = vect_recog_temp_ssa_var (itype, NULL);
5160 206 : def_stmt
5161 206 : = gimple_build_assign (r_or_negr, BIT_IOR_EXPR, r, negate_r);
5162 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5163 :
5164 : // (x ^ y) & (r | -r)
5165 : // ~(x ^ y) & (r | -r) (ceil)
5166 206 : tree r_or_negr_and_xor = vect_recog_temp_ssa_var (itype, NULL);
5167 206 : def_stmt = gimple_build_assign (r_or_negr_and_xor, BIT_AND_EXPR,
5168 : r_or_negr, cond_reg);
5169 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5170 :
5171 : // (x ^ y) & (r | -r) < 0 which is equivalent to (x^y < 0 && r!=0)
5172 206 : bool_cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5173 206 : def_stmt
5174 206 : = gimple_build_assign (bool_cond, LT_EXPR, r_or_negr_and_xor,
5175 : build_int_cst (itype, 0));
5176 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5177 : itype);
5178 :
5179 : // (x^y < 0 && r) ? y : 0 (mod)
5180 : // (x^y < 0 && r) ? -1 : 0 (div)
5181 206 : bool is_mod
5182 206 : = (rhs_code == FLOOR_MOD_EXPR || rhs_code == CEIL_MOD_EXPR);
5183 206 : tree extr_cond = vect_recog_temp_ssa_var (itype, NULL);
5184 246 : def_stmt = gimple_build_assign (extr_cond, COND_EXPR, bool_cond,
5185 : is_mod ? oprnd1
5186 40 : : build_int_cst (itype, -1),
5187 : build_int_cst (itype, 0));
5188 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5189 :
5190 : // r += (x ^ y < 0 && r) ? y : 0 (floor mod)
5191 : // d += (x^y < 0 && r) ? -1 : 0 (floor div)
5192 : // r -= (x ^ y < 0 && r) ? y : 0 (ceil mod)
5193 : // d -= (x^y < 0 && r) ? -1 : 0 (ceil div)
5194 206 : tree result = vect_recog_temp_ssa_var (itype, NULL);
5195 412 : return gimple_build_assign (result,
5196 206 : (rhs_code == FLOOR_MOD_EXPR
5197 206 : || rhs_code == FLOOR_DIV_EXPR)
5198 : ? PLUS_EXPR
5199 : : MINUS_EXPR,
5200 206 : is_mod ? r : q, extr_cond);
5201 : }
5202 : }
5203 36 : case ROUND_MOD_EXPR:
5204 36 : case ROUND_DIV_EXPR:
5205 36 : {
5206 36 : if (!target_has_vecop_for_code (BIT_AND_EXPR, vectype)
5207 36 : || !target_has_vecop_for_code (PLUS_EXPR, vectype)
5208 36 : || !expand_vec_cmp_expr_p (vectype, mask_vectype, LT_EXPR)
5209 36 : || !expand_vec_cmp_expr_p (vectype, mask_vectype, GT_EXPR)
5210 72 : || !expand_vec_cond_expr_p (vectype, mask_vectype))
5211 : return NULL;
5212 :
5213 36 : bool is_mod = rhs_code == ROUND_MOD_EXPR;
5214 36 : HOST_WIDE_INT d = TREE_INT_CST_LOW (oprnd1);
5215 36 : unsigned HOST_WIDE_INT abs_d
5216 : = (d >= 0 ? (unsigned HOST_WIDE_INT) d : -(unsigned HOST_WIDE_INT) d);
5217 36 : unsigned HOST_WIDE_INT mid_d = (abs_d - 1) >> 1;
5218 36 : if (!unsigned_p)
5219 : {
5220 : // check availability of abs expression for vector
5221 18 : if (!target_has_vecop_for_code (ABS_EXPR, vectype))
5222 : return NULL;
5223 : // abs (r)
5224 18 : tree abs_r = vect_recog_temp_ssa_var (itype, NULL);
5225 18 : def_stmt = gimple_build_assign (abs_r, ABS_EXPR, r);
5226 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5227 :
5228 : // abs (r) > (abs (y-1) >> 1)
5229 18 : tree round_p = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5230 18 : def_stmt = gimple_build_assign (round_p, GT_EXPR, abs_r,
5231 18 : build_int_cst (itype, mid_d));
5232 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5233 : itype);
5234 :
5235 : // x ^ y
5236 18 : tree cond_reg = vect_recog_temp_ssa_var (itype, NULL);
5237 18 : def_stmt
5238 18 : = gimple_build_assign (cond_reg, BIT_XOR_EXPR, oprnd0, oprnd1);
5239 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5240 :
5241 : // x ^ y < 0
5242 18 : bool_cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5243 18 : def_stmt = gimple_build_assign (bool_cond, LT_EXPR, cond_reg,
5244 : build_int_cst (itype, 0));
5245 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5246 : itype);
5247 :
5248 : // x ^ y < 0 ? y : -y (mod)
5249 : // x ^ y < 0 ? -1 : 1 (div)
5250 18 : tree val1 = vect_recog_temp_ssa_var (itype, NULL);
5251 18 : def_stmt
5252 36 : = gimple_build_assign (val1, COND_EXPR, bool_cond,
5253 27 : build_int_cst (itype, is_mod ? d : -1),
5254 18 : build_int_cst (itype, is_mod ? -d : 1));
5255 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5256 18 : int precision = TYPE_PRECISION (itype);
5257 18 : wide_int wmask = wi::mask (precision, false, precision);
5258 :
5259 : // abs (r) > (abs (y-1) >> 1) ? 0xffffffff : 0
5260 18 : tree val2 = vect_recog_temp_ssa_var (itype, NULL);
5261 36 : def_stmt = gimple_build_assign (val2, COND_EXPR, round_p,
5262 18 : wide_int_to_tree (itype, wmask),
5263 : build_int_cst (itype, 0));
5264 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5265 :
5266 18 : tree fval = vect_recog_temp_ssa_var (itype, NULL);
5267 18 : def_stmt = gimple_build_assign (fval, BIT_AND_EXPR, val1, val2);
5268 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5269 :
5270 18 : tree result = vect_recog_temp_ssa_var (itype, NULL);
5271 27 : return gimple_build_assign (result, PLUS_EXPR, is_mod ? r : q,
5272 : fval);
5273 18 : }
5274 : else
5275 : {
5276 : // r > (y-1 >> 1)
5277 18 : tree round_p = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5278 18 : def_stmt = gimple_build_assign (round_p, GT_EXPR, r,
5279 18 : build_int_cst (itype, mid_d));
5280 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5281 : itype);
5282 :
5283 : // (r > (y-1)>>1) ? -d : 1
5284 18 : tree val2 = vect_recog_temp_ssa_var (itype, NULL);
5285 18 : def_stmt
5286 36 : = gimple_build_assign (val2, COND_EXPR, round_p,
5287 18 : build_int_cst (itype, is_mod ? -d : 1),
5288 : build_int_cst (itype, 0));
5289 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5290 :
5291 18 : tree result = vect_recog_temp_ssa_var (itype, NULL);
5292 27 : return gimple_build_assign (result, PLUS_EXPR, is_mod ? r : q,
5293 18 : val2);
5294 : }
5295 : }
5296 : default:
5297 : return NULL;
5298 : }
5299 : }
5300 :
5301 : /* Detect a signed division by a constant that wouldn't be
5302 : otherwise vectorized:
5303 :
5304 : type a_t, b_t;
5305 :
5306 : S1 a_t = b_t / N;
5307 :
5308 : where type 'type' is an integral type and N is a constant.
5309 :
5310 : Similarly handle modulo by a constant:
5311 :
5312 : S4 a_t = b_t % N;
5313 :
5314 : Input/Output:
5315 :
5316 : * STMT_VINFO: The stmt from which the pattern search begins,
5317 : i.e. the division stmt. S1 is replaced by if N is a power
5318 : of two constant and type is signed:
5319 : S3 y_t = b_t < 0 ? N - 1 : 0;
5320 : S2 x_t = b_t + y_t;
5321 : S1' a_t = x_t >> log2 (N);
5322 :
5323 : S4 is replaced if N is a power of two constant and
5324 : type is signed by (where *_T temporaries have unsigned type):
5325 : S9 y_T = b_t < 0 ? -1U : 0U;
5326 : S8 z_T = y_T >> (sizeof (type_t) * CHAR_BIT - log2 (N));
5327 : S7 z_t = (type) z_T;
5328 : S6 w_t = b_t + z_t;
5329 : S5 x_t = w_t & (N - 1);
5330 : S4' a_t = x_t - z_t;
5331 :
5332 : Output:
5333 :
5334 : * TYPE_OUT: The type of the output of this pattern.
5335 :
5336 : * Return value: A new stmt that will be used to replace the division
5337 : S1 or modulo S4 stmt. */
5338 :
5339 : static gimple *
5340 33024536 : vect_recog_divmod_pattern (vec_info *vinfo,
5341 : stmt_vec_info stmt_vinfo, tree *type_out)
5342 : {
5343 33024536 : gimple *last_stmt = stmt_vinfo->stmt;
5344 33024536 : tree oprnd0, oprnd1, vectype, itype, cond;
5345 33024536 : gimple *pattern_stmt = NULL;
5346 33024536 : gimple *def_stmt = NULL;
5347 33024536 : enum tree_code rhs_code;
5348 33024536 : optab optab;
5349 33024536 : tree q, cst;
5350 33024536 : int prec;
5351 :
5352 33024536 : if (!is_gimple_assign (last_stmt)
5353 : /* The pattern will disrupt the reduction chain with multiple uses. */
5354 33024536 : || vect_is_reduction (stmt_vinfo))
5355 : return NULL;
5356 :
5357 22885208 : rhs_code = gimple_assign_rhs_code (last_stmt);
5358 22885208 : switch (rhs_code)
5359 : {
5360 292779 : case TRUNC_DIV_EXPR:
5361 292779 : case EXACT_DIV_EXPR:
5362 292779 : case TRUNC_MOD_EXPR:
5363 292779 : case FLOOR_MOD_EXPR:
5364 292779 : case FLOOR_DIV_EXPR:
5365 292779 : case CEIL_MOD_EXPR:
5366 292779 : case CEIL_DIV_EXPR:
5367 292779 : case ROUND_MOD_EXPR:
5368 292779 : case ROUND_DIV_EXPR:
5369 292779 : break;
5370 : default:
5371 : return NULL;
5372 : }
5373 :
5374 292779 : oprnd0 = gimple_assign_rhs1 (last_stmt);
5375 292779 : oprnd1 = gimple_assign_rhs2 (last_stmt);
5376 292779 : itype = TREE_TYPE (oprnd0);
5377 292779 : if (TREE_CODE (oprnd0) != SSA_NAME
5378 274065 : || TREE_CODE (oprnd1) != INTEGER_CST
5379 162227 : || TREE_CODE (itype) != INTEGER_TYPE
5380 455006 : || !type_has_mode_precision_p (itype))
5381 : return NULL;
5382 :
5383 162213 : scalar_int_mode itype_mode = SCALAR_INT_TYPE_MODE (itype);
5384 162213 : vectype = get_vectype_for_scalar_type (vinfo, itype);
5385 162213 : if (vectype == NULL_TREE)
5386 : return NULL;
5387 :
5388 130693 : if (optimize_bb_for_size_p (gimple_bb (last_stmt)))
5389 : {
5390 : /* If the target can handle vectorized division or modulo natively,
5391 : don't attempt to optimize this, since native division is likely
5392 : to give smaller code. */
5393 1986 : optab = optab_for_tree_code (rhs_code, vectype, optab_default);
5394 1986 : if (optab != unknown_optab
5395 1986 : && can_implement_p (optab, TYPE_MODE (vectype)))
5396 : return NULL;
5397 : }
5398 :
5399 130693 : prec = TYPE_PRECISION (itype);
5400 :
5401 261386 : bool is_flclrd_moddiv_p
5402 130693 : = rhs_code == FLOOR_MOD_EXPR || rhs_code == FLOOR_DIV_EXPR
5403 : || rhs_code == CEIL_MOD_EXPR || rhs_code == CEIL_DIV_EXPR
5404 130084 : || rhs_code == ROUND_MOD_EXPR || rhs_code == ROUND_DIV_EXPR;
5405 130693 : if (integer_pow2p (oprnd1))
5406 : {
5407 77543 : if ((TYPE_UNSIGNED (itype)
5408 36 : && (rhs_code == FLOOR_MOD_EXPR || rhs_code == FLOOR_DIV_EXPR))
5409 77579 : || tree_int_cst_sgn (oprnd1) != 1)
5410 : return NULL;
5411 :
5412 : /* Pattern detected. */
5413 77543 : vect_pattern_detected ("vect_recog_divmod_pattern", last_stmt);
5414 :
5415 77543 : *type_out = vectype;
5416 :
5417 : /* Check if the target supports this internal function. */
5418 77543 : internal_fn ifn = IFN_DIV_POW2;
5419 77543 : if (!TYPE_UNSIGNED (itype)
5420 77543 : && direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
5421 : {
5422 0 : tree shift = build_int_cst (itype, tree_log2 (oprnd1));
5423 :
5424 0 : tree var_div = vect_recog_temp_ssa_var (itype, NULL);
5425 0 : gimple *div_stmt = gimple_build_call_internal (ifn, 2, oprnd0, shift);
5426 0 : gimple_call_set_lhs (div_stmt, var_div);
5427 0 : if (rhs_code == TRUNC_MOD_EXPR || is_flclrd_moddiv_p)
5428 : {
5429 0 : append_pattern_def_seq (vinfo, stmt_vinfo, div_stmt);
5430 0 : tree t1 = vect_recog_temp_ssa_var (itype, NULL);
5431 0 : def_stmt
5432 0 : = gimple_build_assign (t1, LSHIFT_EXPR, var_div, shift);
5433 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5434 0 : tree r = vect_recog_temp_ssa_var (itype, NULL);
5435 0 : pattern_stmt = gimple_build_assign (r, MINUS_EXPR, oprnd0, t1);
5436 0 : if (is_flclrd_moddiv_p)
5437 : {
5438 0 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5439 0 : pattern_stmt
5440 0 : = add_code_for_floorceilround_divmod (vectype, vinfo,
5441 : stmt_vinfo, rhs_code,
5442 : var_div, r, oprnd0,
5443 : oprnd1, itype);
5444 0 : if (pattern_stmt == NULL)
5445 : return NULL;
5446 : }
5447 : }
5448 : else
5449 : pattern_stmt = div_stmt;
5450 0 : gimple_set_location (pattern_stmt, gimple_location (last_stmt));
5451 :
5452 0 : return pattern_stmt;
5453 : }
5454 :
5455 77543 : cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5456 77543 : def_stmt = gimple_build_assign (cond, LT_EXPR, oprnd0,
5457 : build_int_cst (itype, 0));
5458 77543 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt,
5459 : truth_type_for (vectype), itype);
5460 77543 : tree div_result = NULL_TREE;
5461 77543 : if (rhs_code == TRUNC_DIV_EXPR
5462 77543 : || rhs_code == EXACT_DIV_EXPR
5463 : || rhs_code == FLOOR_DIV_EXPR
5464 2630 : || rhs_code == CEIL_DIV_EXPR
5465 2468 : || rhs_code == ROUND_DIV_EXPR)
5466 : {
5467 75087 : tree var = vect_recog_temp_ssa_var (itype, NULL);
5468 75087 : tree shift;
5469 75087 : def_stmt
5470 75087 : = gimple_build_assign (var, COND_EXPR, cond,
5471 : fold_build2 (MINUS_EXPR, itype, oprnd1,
5472 : build_int_cst (itype, 1)),
5473 : build_int_cst (itype, 0));
5474 75087 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5475 75087 : var = vect_recog_temp_ssa_var (itype, NULL);
5476 75087 : def_stmt
5477 75087 : = gimple_build_assign (var, PLUS_EXPR, oprnd0,
5478 : gimple_assign_lhs (def_stmt));
5479 75087 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5480 :
5481 75087 : shift = build_int_cst (itype, tree_log2 (oprnd1));
5482 75087 : div_result = vect_recog_temp_ssa_var (itype, NULL);
5483 75087 : pattern_stmt
5484 75087 : = gimple_build_assign (div_result, RSHIFT_EXPR, var, shift);
5485 : }
5486 77543 : if (rhs_code == TRUNC_MOD_EXPR || is_flclrd_moddiv_p)
5487 : {
5488 2630 : if (rhs_code == FLOOR_DIV_EXPR
5489 : || rhs_code == CEIL_DIV_EXPR
5490 2630 : || rhs_code == ROUND_DIV_EXPR)
5491 174 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5492 :
5493 2630 : tree signmask;
5494 2630 : if (compare_tree_int (oprnd1, 2) == 0)
5495 : {
5496 1342 : signmask = vect_recog_temp_ssa_var (itype, NULL);
5497 1342 : def_stmt = gimple_build_assign (signmask, COND_EXPR, cond,
5498 : build_int_cst (itype, 1),
5499 : build_int_cst (itype, 0));
5500 1342 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5501 : }
5502 : else
5503 : {
5504 1288 : tree utype
5505 1288 : = build_nonstandard_integer_type (prec, 1);
5506 1288 : tree vecutype = get_vectype_for_scalar_type (vinfo, utype);
5507 1288 : tree shift
5508 1288 : = build_int_cst (utype, GET_MODE_BITSIZE (itype_mode)
5509 1288 : - tree_log2 (oprnd1));
5510 1288 : tree var = vect_recog_temp_ssa_var (utype, NULL);
5511 :
5512 1288 : def_stmt = gimple_build_assign (var, COND_EXPR, cond,
5513 : build_int_cst (utype, -1),
5514 : build_int_cst (utype, 0));
5515 1288 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecutype);
5516 1288 : var = vect_recog_temp_ssa_var (utype, NULL);
5517 1288 : def_stmt = gimple_build_assign (var, RSHIFT_EXPR,
5518 : gimple_assign_lhs (def_stmt),
5519 : shift);
5520 1288 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecutype);
5521 1288 : signmask = vect_recog_temp_ssa_var (itype, NULL);
5522 1288 : def_stmt
5523 1288 : = gimple_build_assign (signmask, NOP_EXPR, var);
5524 1288 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5525 : }
5526 2630 : def_stmt
5527 2630 : = gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL),
5528 : PLUS_EXPR, oprnd0, signmask);
5529 2630 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5530 2630 : def_stmt
5531 2630 : = gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL),
5532 : BIT_AND_EXPR, gimple_assign_lhs (def_stmt),
5533 : fold_build2 (MINUS_EXPR, itype, oprnd1,
5534 : build_int_cst (itype, 1)));
5535 2630 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5536 :
5537 2630 : tree r = vect_recog_temp_ssa_var (itype, NULL);
5538 2630 : pattern_stmt
5539 2630 : = gimple_build_assign (r, MINUS_EXPR, gimple_assign_lhs (def_stmt),
5540 : signmask);
5541 2630 : if (is_flclrd_moddiv_p)
5542 : {
5543 268 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5544 268 : pattern_stmt
5545 268 : = add_code_for_floorceilround_divmod (vectype, vinfo,
5546 : stmt_vinfo, rhs_code,
5547 : div_result, r, oprnd0,
5548 : oprnd1, itype);
5549 268 : if (pattern_stmt == NULL)
5550 : return NULL;
5551 : }
5552 : }
5553 :
5554 77389 : return pattern_stmt;
5555 : }
5556 :
5557 53150 : if ((cst = uniform_integer_cst_p (oprnd1))
5558 53150 : && TYPE_UNSIGNED (itype)
5559 : && rhs_code == TRUNC_DIV_EXPR
5560 30167 : && vectype
5561 71383 : && targetm.vectorize.preferred_div_as_shifts_over_mult (vectype))
5562 : {
5563 : /* We can use the relationship:
5564 :
5565 : x // N == ((x+N+2) // (N+1) + x) // (N+1) for 0 <= x < N(N+3)
5566 :
5567 : to optimize cases where N+1 is a power of 2, and where // (N+1)
5568 : is therefore a shift right. When operating in modes that are
5569 : multiples of a byte in size, there are two cases:
5570 :
5571 : (1) N(N+3) is not representable, in which case the question
5572 : becomes whether the replacement expression overflows.
5573 : It is enough to test that x+N+2 does not overflow,
5574 : i.e. that x < MAX-(N+1).
5575 :
5576 : (2) N(N+3) is representable, in which case it is the (only)
5577 : bound that we need to check.
5578 :
5579 : ??? For now we just handle the case where // (N+1) is a shift
5580 : right by half the precision, since some architectures can
5581 : optimize the associated addition and shift combinations
5582 : into single instructions. */
5583 :
5584 7516 : auto wcst = wi::to_wide (cst);
5585 7516 : int pow = wi::exact_log2 (wcst + 1);
5586 7516 : if (pow == prec / 2)
5587 : {
5588 : /* Check that no overflow will occur. If we don't have range
5589 : information we can't perform the optimization. */
5590 :
5591 466 : int_range_max r;
5592 932 : if (get_range_query (cfun)->range_of_expr (r, oprnd0, last_stmt)
5593 466 : && !r.undefined_p ())
5594 : {
5595 466 : wide_int max = r.upper_bound ();
5596 466 : wide_int one = wi::shwi (1, prec);
5597 466 : wide_int adder = wi::add (one, wi::lshift (one, pow));
5598 466 : wi::overflow_type ovf;
5599 466 : wi::add (max, adder, UNSIGNED, &ovf);
5600 466 : if (ovf == wi::OVF_NONE)
5601 : {
5602 299 : *type_out = vectype;
5603 299 : tree tadder = wide_int_to_tree (itype, adder);
5604 299 : tree rshift = wide_int_to_tree (itype, pow);
5605 :
5606 299 : tree new_lhs1 = vect_recog_temp_ssa_var (itype, NULL);
5607 299 : gassign *patt1
5608 299 : = gimple_build_assign (new_lhs1, PLUS_EXPR, oprnd0, tadder);
5609 299 : append_pattern_def_seq (vinfo, stmt_vinfo, patt1, vectype);
5610 :
5611 299 : tree new_lhs2 = vect_recog_temp_ssa_var (itype, NULL);
5612 299 : patt1 = gimple_build_assign (new_lhs2, RSHIFT_EXPR, new_lhs1,
5613 : rshift);
5614 299 : append_pattern_def_seq (vinfo, stmt_vinfo, patt1, vectype);
5615 :
5616 299 : tree new_lhs3 = vect_recog_temp_ssa_var (itype, NULL);
5617 299 : patt1 = gimple_build_assign (new_lhs3, PLUS_EXPR, new_lhs2,
5618 : oprnd0);
5619 299 : append_pattern_def_seq (vinfo, stmt_vinfo, patt1, vectype);
5620 :
5621 299 : tree new_lhs4 = vect_recog_temp_ssa_var (itype, NULL);
5622 299 : pattern_stmt = gimple_build_assign (new_lhs4, RSHIFT_EXPR,
5623 : new_lhs3, rshift);
5624 :
5625 299 : return pattern_stmt;
5626 : }
5627 466 : }
5628 466 : }
5629 : }
5630 :
5631 52851 : if (prec > HOST_BITS_PER_WIDE_INT
5632 52851 : || integer_zerop (oprnd1))
5633 : return NULL;
5634 :
5635 52554 : if (!can_mult_highpart_p (TYPE_MODE (vectype), TYPE_UNSIGNED (itype)))
5636 : return NULL;
5637 :
5638 22206 : if (TYPE_UNSIGNED (itype))
5639 : {
5640 17329 : unsigned HOST_WIDE_INT mh, ml;
5641 17329 : int pre_shift, post_shift;
5642 17329 : unsigned HOST_WIDE_INT d = (TREE_INT_CST_LOW (oprnd1)
5643 17329 : & GET_MODE_MASK (itype_mode));
5644 17329 : tree t1, t2, t3, t4;
5645 :
5646 17329 : if (d >= (HOST_WIDE_INT_1U << (prec - 1)))
5647 : /* FIXME: Can transform this into oprnd0 >= oprnd1 ? 1 : 0. */
5648 45 : return NULL;
5649 :
5650 : /* Find a suitable multiplier and right shift count instead of
5651 : directly dividing by D. */
5652 17284 : mh = choose_multiplier (d, prec, prec, &ml, &post_shift);
5653 :
5654 : /* If the suggested multiplier is more than PREC bits, we can do better
5655 : for even divisors, using an initial right shift. */
5656 17284 : if (mh != 0 && (d & 1) == 0)
5657 : {
5658 1551 : pre_shift = ctz_or_zero (d);
5659 1551 : mh = choose_multiplier (d >> pre_shift, prec, prec - pre_shift,
5660 : &ml, &post_shift);
5661 1551 : gcc_assert (!mh);
5662 : }
5663 : else
5664 : pre_shift = 0;
5665 :
5666 1219 : if (mh != 0)
5667 : {
5668 1219 : if (post_shift - 1 >= prec)
5669 : return NULL;
5670 :
5671 : /* t1 = oprnd0 h* ml;
5672 : t2 = oprnd0 - t1;
5673 : t3 = t2 >> 1;
5674 : t4 = t1 + t3;
5675 : q = t4 >> (post_shift - 1); */
5676 1219 : t1 = vect_recog_temp_ssa_var (itype, NULL);
5677 1219 : def_stmt = gimple_build_assign (t1, MULT_HIGHPART_EXPR, oprnd0,
5678 1219 : build_int_cst (itype, ml));
5679 1219 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5680 :
5681 1219 : t2 = vect_recog_temp_ssa_var (itype, NULL);
5682 1219 : def_stmt
5683 1219 : = gimple_build_assign (t2, MINUS_EXPR, oprnd0, t1);
5684 1219 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5685 :
5686 1219 : t3 = vect_recog_temp_ssa_var (itype, NULL);
5687 1219 : def_stmt
5688 1219 : = gimple_build_assign (t3, RSHIFT_EXPR, t2, integer_one_node);
5689 1219 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5690 :
5691 1219 : t4 = vect_recog_temp_ssa_var (itype, NULL);
5692 1219 : def_stmt
5693 1219 : = gimple_build_assign (t4, PLUS_EXPR, t1, t3);
5694 :
5695 1219 : if (post_shift != 1)
5696 : {
5697 1219 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5698 :
5699 1219 : q = vect_recog_temp_ssa_var (itype, NULL);
5700 1219 : pattern_stmt
5701 1219 : = gimple_build_assign (q, RSHIFT_EXPR, t4,
5702 1219 : build_int_cst (itype, post_shift - 1));
5703 : }
5704 : else
5705 : {
5706 : q = t4;
5707 : pattern_stmt = def_stmt;
5708 : }
5709 : }
5710 : else
5711 : {
5712 16065 : if (pre_shift >= prec || post_shift >= prec)
5713 : return NULL;
5714 :
5715 : /* t1 = oprnd0 >> pre_shift;
5716 : t2 = t1 h* ml;
5717 : q = t2 >> post_shift; */
5718 16065 : if (pre_shift)
5719 : {
5720 1551 : t1 = vect_recog_temp_ssa_var (itype, NULL);
5721 1551 : def_stmt
5722 1551 : = gimple_build_assign (t1, RSHIFT_EXPR, oprnd0,
5723 1551 : build_int_cst (integer_type_node, pre_shift));
5724 1551 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5725 : }
5726 : else
5727 : t1 = oprnd0;
5728 :
5729 16065 : t2 = vect_recog_temp_ssa_var (itype, NULL);
5730 16065 : def_stmt = gimple_build_assign (t2, MULT_HIGHPART_EXPR, t1,
5731 16065 : build_int_cst (itype, ml));
5732 :
5733 16065 : if (post_shift)
5734 : {
5735 16019 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5736 :
5737 16019 : q = vect_recog_temp_ssa_var (itype, NULL);
5738 16019 : def_stmt
5739 16019 : = gimple_build_assign (q, RSHIFT_EXPR, t2,
5740 16019 : build_int_cst (itype, post_shift));
5741 : }
5742 : else
5743 : q = t2;
5744 :
5745 : pattern_stmt = def_stmt;
5746 : }
5747 : }
5748 : else
5749 : {
5750 4877 : unsigned HOST_WIDE_INT ml;
5751 4877 : int post_shift;
5752 4877 : HOST_WIDE_INT d = TREE_INT_CST_LOW (oprnd1);
5753 4877 : unsigned HOST_WIDE_INT abs_d;
5754 4877 : bool add = false;
5755 4877 : tree t1, t2, t3, t4;
5756 :
5757 : /* Give up for -1. */
5758 4877 : if (d == -1)
5759 0 : return NULL;
5760 :
5761 : /* Since d might be INT_MIN, we have to cast to
5762 : unsigned HOST_WIDE_INT before negating to avoid
5763 : undefined signed overflow. */
5764 4877 : abs_d = (d >= 0
5765 4877 : ? (unsigned HOST_WIDE_INT) d
5766 : : - (unsigned HOST_WIDE_INT) d);
5767 :
5768 : /* n rem d = n rem -d */
5769 4877 : if (rhs_code == TRUNC_MOD_EXPR && d < 0)
5770 : {
5771 0 : d = abs_d;
5772 0 : oprnd1 = build_int_cst (itype, abs_d);
5773 : }
5774 4877 : if (HOST_BITS_PER_WIDE_INT >= prec
5775 4877 : && abs_d == HOST_WIDE_INT_1U << (prec - 1))
5776 : /* This case is not handled correctly below. */
5777 : return NULL;
5778 :
5779 4877 : choose_multiplier (abs_d, prec, prec - 1, &ml, &post_shift);
5780 4877 : if (ml >= HOST_WIDE_INT_1U << (prec - 1))
5781 : {
5782 1621 : add = true;
5783 1621 : ml |= HOST_WIDE_INT_M1U << (prec - 1);
5784 : }
5785 4877 : if (post_shift >= prec)
5786 : return NULL;
5787 :
5788 : /* t1 = oprnd0 h* ml; */
5789 4877 : t1 = vect_recog_temp_ssa_var (itype, NULL);
5790 4877 : def_stmt = gimple_build_assign (t1, MULT_HIGHPART_EXPR, oprnd0,
5791 4877 : build_int_cst (itype, ml));
5792 :
5793 4877 : if (add)
5794 : {
5795 : /* t2 = t1 + oprnd0; */
5796 1621 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5797 1621 : t2 = vect_recog_temp_ssa_var (itype, NULL);
5798 1621 : def_stmt = gimple_build_assign (t2, PLUS_EXPR, t1, oprnd0);
5799 : }
5800 : else
5801 : t2 = t1;
5802 :
5803 4877 : if (post_shift)
5804 : {
5805 : /* t3 = t2 >> post_shift; */
5806 4045 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5807 4045 : t3 = vect_recog_temp_ssa_var (itype, NULL);
5808 4045 : def_stmt = gimple_build_assign (t3, RSHIFT_EXPR, t2,
5809 4045 : build_int_cst (itype, post_shift));
5810 : }
5811 : else
5812 : t3 = t2;
5813 :
5814 4877 : int msb = 1;
5815 4877 : int_range_max r;
5816 9754 : get_range_query (cfun)->range_of_expr (r, oprnd0);
5817 4877 : if (!r.varying_p () && !r.undefined_p ())
5818 : {
5819 3024 : if (!wi::neg_p (r.lower_bound (), TYPE_SIGN (itype)))
5820 : msb = 0;
5821 734 : else if (wi::neg_p (r.upper_bound (), TYPE_SIGN (itype)))
5822 : msb = -1;
5823 : }
5824 :
5825 2290 : if (msb == 0 && d >= 0)
5826 : {
5827 : /* q = t3; */
5828 : q = t3;
5829 : pattern_stmt = def_stmt;
5830 : }
5831 : else
5832 : {
5833 : /* t4 = oprnd0 >> (prec - 1);
5834 : or if we know from VRP that oprnd0 >= 0
5835 : t4 = 0;
5836 : or if we know from VRP that oprnd0 < 0
5837 : t4 = -1; */
5838 2655 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5839 2655 : t4 = vect_recog_temp_ssa_var (itype, NULL);
5840 2655 : if (msb != 1)
5841 76 : def_stmt = gimple_build_assign (t4, INTEGER_CST,
5842 76 : build_int_cst (itype, msb));
5843 : else
5844 2579 : def_stmt = gimple_build_assign (t4, RSHIFT_EXPR, oprnd0,
5845 2579 : build_int_cst (itype, prec - 1));
5846 2655 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5847 :
5848 : /* q = t3 - t4; or q = t4 - t3; */
5849 2655 : q = vect_recog_temp_ssa_var (itype, NULL);
5850 5126 : pattern_stmt = gimple_build_assign (q, MINUS_EXPR, d < 0 ? t4 : t3,
5851 : d < 0 ? t3 : t4);
5852 : }
5853 4877 : }
5854 :
5855 22161 : if (rhs_code == TRUNC_MOD_EXPR || is_flclrd_moddiv_p)
5856 : {
5857 9770 : tree r, t1;
5858 :
5859 : /* We divided. Now finish by:
5860 : t1 = q * oprnd1;
5861 : r = oprnd0 - t1; */
5862 9770 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5863 :
5864 9770 : t1 = vect_recog_temp_ssa_var (itype, NULL);
5865 9770 : def_stmt = gimple_build_assign (t1, MULT_EXPR, q, oprnd1);
5866 9770 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5867 :
5868 9770 : r = vect_recog_temp_ssa_var (itype, NULL);
5869 9770 : pattern_stmt = gimple_build_assign (r, MINUS_EXPR, oprnd0, t1);
5870 :
5871 9770 : if (is_flclrd_moddiv_p)
5872 : {
5873 146 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5874 146 : pattern_stmt
5875 146 : = add_code_for_floorceilround_divmod (vectype, vinfo, stmt_vinfo,
5876 : rhs_code, q, r, oprnd0, oprnd1,
5877 : itype);
5878 146 : if (pattern_stmt == NULL)
5879 : return NULL;
5880 : }
5881 : }
5882 :
5883 : /* Pattern detected. */
5884 22161 : vect_pattern_detected ("vect_recog_divmod_pattern", last_stmt);
5885 :
5886 22161 : *type_out = vectype;
5887 22161 : return pattern_stmt;
5888 : }
5889 :
5890 : /* Detects pattern with a modulo operation (S1) where both arguments
5891 : are variables of integral type.
5892 : The statement is replaced by division, multiplication, and subtraction.
5893 : The last statement (S4) is returned.
5894 :
5895 : Example:
5896 : S1 c_t = a_t % b_t;
5897 :
5898 : is replaced by
5899 : S2 x_t = a_t / b_t;
5900 : S3 y_t = x_t * b_t;
5901 : S4 z_t = a_t - y_t; */
5902 :
5903 : static gimple *
5904 33220615 : vect_recog_mod_var_pattern (vec_info *vinfo,
5905 : stmt_vec_info stmt_vinfo, tree *type_out)
5906 : {
5907 33220615 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
5908 33220615 : tree oprnd0, oprnd1, vectype, itype;
5909 33220615 : gimple *pattern_stmt, *def_stmt;
5910 33220615 : enum tree_code rhs_code;
5911 :
5912 33220615 : if (!is_gimple_assign (last_stmt) || vect_is_reduction (stmt_vinfo))
5913 : return NULL;
5914 :
5915 23081287 : rhs_code = gimple_assign_rhs_code (last_stmt);
5916 23081287 : if (rhs_code != TRUNC_MOD_EXPR)
5917 : return NULL;
5918 :
5919 75680 : oprnd0 = gimple_assign_rhs1 (last_stmt);
5920 75680 : oprnd1 = gimple_assign_rhs2 (last_stmt);
5921 75680 : itype = TREE_TYPE (oprnd0);
5922 75680 : if (TREE_CODE (oprnd0) != SSA_NAME
5923 66781 : || TREE_CODE (oprnd1) != SSA_NAME
5924 53590 : || TREE_CODE (itype) != INTEGER_TYPE)
5925 : return NULL;
5926 :
5927 53463 : vectype = get_vectype_for_scalar_type (vinfo, itype);
5928 :
5929 53463 : if (!vectype
5930 44728 : || target_has_vecop_for_code (TRUNC_MOD_EXPR, vectype)
5931 44728 : || !target_has_vecop_for_code (TRUNC_DIV_EXPR, vectype)
5932 0 : || !target_has_vecop_for_code (MULT_EXPR, vectype)
5933 53463 : || !target_has_vecop_for_code (MINUS_EXPR, vectype))
5934 : return NULL;
5935 :
5936 0 : tree q, tmp, r;
5937 0 : q = vect_recog_temp_ssa_var (itype, NULL);
5938 0 : def_stmt = gimple_build_assign (q, TRUNC_DIV_EXPR, oprnd0, oprnd1);
5939 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vectype);
5940 :
5941 0 : tmp = vect_recog_temp_ssa_var (itype, NULL);
5942 0 : def_stmt = gimple_build_assign (tmp, MULT_EXPR, q, oprnd1);
5943 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vectype);
5944 :
5945 0 : r = vect_recog_temp_ssa_var (itype, NULL);
5946 0 : pattern_stmt = gimple_build_assign (r, MINUS_EXPR, oprnd0, tmp);
5947 :
5948 : /* Pattern detected. */
5949 0 : *type_out = vectype;
5950 0 : vect_pattern_detected ("vect_recog_mod_var_pattern", last_stmt);
5951 :
5952 0 : return pattern_stmt;
5953 : }
5954 :
5955 :
5956 : /* Return the proper type for converting bool VAR into
5957 : an integer value or NULL_TREE if no such type exists.
5958 : The type is chosen so that the converted value has the
5959 : same number of elements as VAR's vector type. */
5960 :
5961 : static tree
5962 4736317 : integer_type_for_mask (tree var, vec_info *vinfo, vect_def_type *dt = nullptr)
5963 : {
5964 4736317 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (var)))
5965 : return NULL_TREE;
5966 :
5967 2152549 : stmt_vec_info def_stmt_info = vinfo->lookup_def (var);
5968 2152549 : if (dt)
5969 : {
5970 381806 : if (!def_stmt_info)
5971 3490 : *dt = vect_external_def;
5972 : else
5973 378316 : *dt = STMT_VINFO_DEF_TYPE (def_stmt_info);
5974 : }
5975 381806 : if (!def_stmt_info
5976 2055384 : || STMT_VINFO_DEF_TYPE (def_stmt_info) == vect_external_def
5977 3826127 : || !vect_use_mask_type_p (def_stmt_info))
5978 : return NULL_TREE;
5979 :
5980 1316280 : return build_nonstandard_integer_type (def_stmt_info->mask_precision, 1);
5981 : }
5982 :
5983 : /* Function vect_recog_gcond_pattern
5984 :
5985 : Try to find pattern like following:
5986 :
5987 : if (a op b)
5988 :
5989 : where operator 'op' is not != and convert it to an adjusted boolean pattern
5990 :
5991 : mask = a op b
5992 : if (mask != 0)
5993 :
5994 : and set the mask type on MASK.
5995 :
5996 : Input:
5997 :
5998 : * STMT_VINFO: The stmt at the end from which the pattern
5999 : search begins, i.e. cast of a bool to
6000 : an integer type.
6001 :
6002 : Output:
6003 :
6004 : * TYPE_OUT: The type of the output of this pattern.
6005 :
6006 : * Return value: A new stmt that will be used to replace the pattern. */
6007 :
6008 : static gimple *
6009 33293375 : vect_recog_gcond_pattern (vec_info *vinfo,
6010 : stmt_vec_info stmt_vinfo, tree *type_out)
6011 : {
6012 : /* Currently we only support this for loop vectorization and when multiple
6013 : exits. */
6014 33293375 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6015 4639458 : if (!loop_vinfo || !LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
6016 : return NULL;
6017 :
6018 1728317 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
6019 1728317 : gcond* cond = NULL;
6020 1728317 : if (!(cond = dyn_cast <gcond *> (last_stmt)))
6021 : return NULL;
6022 :
6023 406192 : auto lhs = gimple_cond_lhs (cond);
6024 406192 : auto rhs = gimple_cond_rhs (cond);
6025 406192 : auto code = gimple_cond_code (cond);
6026 :
6027 406192 : tree scalar_type = TREE_TYPE (lhs);
6028 406192 : if (VECTOR_TYPE_P (scalar_type))
6029 : return NULL;
6030 :
6031 : /* If the input is a boolean then try to figure out the precision that the
6032 : vector type should use. We cannot use the scalar precision as this would
6033 : later mismatch. This is similar to what recog_bool does. */
6034 406192 : if (VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type))
6035 : {
6036 10460 : if (tree stype = integer_type_for_mask (lhs, vinfo))
6037 406192 : scalar_type = stype;
6038 : }
6039 :
6040 406192 : tree vectype = get_mask_type_for_scalar_type (vinfo, scalar_type);
6041 406192 : if (vectype == NULL_TREE)
6042 : return NULL;
6043 :
6044 385300 : tree new_lhs = vect_recog_temp_ssa_var (boolean_type_node, NULL);
6045 385300 : gimple *new_stmt = gimple_build_assign (new_lhs, code, lhs, rhs);
6046 385300 : append_pattern_def_seq (vinfo, stmt_vinfo, new_stmt, vectype, scalar_type);
6047 :
6048 385300 : gimple *pattern_stmt
6049 385300 : = gimple_build_cond (NE_EXPR, new_lhs,
6050 385300 : build_int_cst (TREE_TYPE (new_lhs), 0),
6051 : NULL_TREE, NULL_TREE);
6052 385300 : *type_out = vectype;
6053 385300 : vect_pattern_detected ("vect_recog_gcond_pattern", last_stmt);
6054 385300 : return pattern_stmt;
6055 : }
6056 :
6057 :
6058 : /* A helper for vect_recog_mask_conversion_pattern. Build
6059 : conversion of MASK to a type suitable for masking VECTYPE.
6060 : Built statement gets required vectype and is appended to
6061 : a pattern sequence of STMT_VINFO.
6062 : If VECTYPE is a mask type, SCALAR_TYPE_FOR_MASK is the scalar type
6063 : from which it was derived.
6064 :
6065 : Return converted mask. */
6066 :
6067 : static tree
6068 147429 : build_mask_conversion (vec_info *vinfo,
6069 : tree mask, tree vectype, stmt_vec_info stmt_vinfo,
6070 : tree scalar_type_for_mask = NULL_TREE)
6071 : {
6072 147429 : gimple *stmt;
6073 147429 : tree masktype, tmp;
6074 :
6075 237733 : gcc_assert (!scalar_type_for_mask == !VECTOR_BOOLEAN_TYPE_P (vectype));
6076 :
6077 147429 : masktype = truth_type_for (vectype);
6078 147429 : tmp = vect_recog_temp_ssa_var (TREE_TYPE (masktype), NULL);
6079 147429 : stmt = gimple_build_assign (tmp, CONVERT_EXPR, mask);
6080 237733 : append_pattern_def_seq (vinfo, stmt_vinfo,
6081 : stmt, masktype,
6082 : scalar_type_for_mask
6083 90304 : ? scalar_type_for_mask : TREE_TYPE (vectype));
6084 :
6085 147429 : return tmp;
6086 : }
6087 :
6088 :
6089 : /* Return MASK if MASK is suitable for masking an operation on vectors
6090 : of type VECTYPE, otherwise convert it into such a form and return
6091 : the result. Associate any conversion statements with STMT_INFO's
6092 : pattern. If VECTYPE is a mask type, SCALAR_TYPE_FOR_MASK is the scalar
6093 : type from which it was derived. */
6094 :
6095 : static tree
6096 80722 : vect_convert_mask_for_vectype (tree mask, tree vectype,
6097 : stmt_vec_info stmt_info, vec_info *vinfo,
6098 : tree scalar_type_for_mask = NULL_TREE)
6099 : {
6100 80722 : tree mask_type = integer_type_for_mask (mask, vinfo);
6101 80722 : if (mask_type)
6102 : {
6103 80722 : tree mask_vectype = get_mask_type_for_scalar_type (vinfo, mask_type);
6104 80722 : if (mask_vectype
6105 161444 : && maybe_ne (TYPE_VECTOR_SUBPARTS (vectype),
6106 98271 : TYPE_VECTOR_SUBPARTS (mask_vectype)))
6107 63173 : mask = build_mask_conversion (vinfo, mask, vectype, stmt_info,
6108 : scalar_type_for_mask);
6109 : }
6110 80722 : return mask;
6111 : }
6112 :
6113 :
6114 : /* Function vect_recog_bool_pattern
6115 :
6116 : Try to find pattern like following:
6117 :
6118 : bool a_b, b_b, c_b, d_b, e_b;
6119 : TYPE f_T;
6120 : loop:
6121 : S1 a_b = x1 CMP1 y1;
6122 : S2 b_b = x2 CMP2 y2;
6123 : S3 c_b = a_b & b_b;
6124 : S4 d_b = x3 CMP3 y3;
6125 : S5 e_b = c_b | d_b;
6126 : S6 f_T = (TYPE) e_b;
6127 :
6128 : where type 'TYPE' is an integral type. Or a similar pattern
6129 : ending in
6130 :
6131 : S6 f_Y = e_b ? r_Y : s_Y;
6132 :
6133 : as results from if-conversion of a complex condition.
6134 :
6135 : Input:
6136 :
6137 : * STMT_VINFO: The stmt at the end from which the pattern
6138 : search begins, i.e. cast of a bool to
6139 : an integer type.
6140 :
6141 : Output:
6142 :
6143 : * TYPE_OUT: The type of the output of this pattern.
6144 :
6145 : * Return value: A new stmt that will be used to replace the pattern.
6146 :
6147 : Assuming size of TYPE is the same as size of all comparisons
6148 : (otherwise some casts would be added where needed), the above
6149 : sequence we create related pattern stmts:
6150 : S1' a_T = x1 CMP1 y1 ? 1 : 0;
6151 : S3' c_T = x2 CMP2 y2 ? a_T : 0;
6152 : S4' d_T = x3 CMP3 y3 ? 1 : 0;
6153 : S5' e_T = c_T | d_T;
6154 : S6' f_T = e_T;
6155 :
6156 : Instead of the above S3' we could emit:
6157 : S2' b_T = x2 CMP2 y2 ? 1 : 0;
6158 : S3' c_T = a_T | b_T;
6159 : but the above is more efficient. */
6160 :
6161 : static gimple *
6162 33293375 : vect_recog_bool_pattern (vec_info *vinfo,
6163 : stmt_vec_info stmt_vinfo, tree *type_out)
6164 : {
6165 33293375 : gimple *last_stmt = stmt_vinfo->stmt;
6166 33293375 : enum tree_code rhs_code;
6167 33293375 : tree var, lhs, rhs, vectype;
6168 33293375 : gimple *pattern_stmt;
6169 :
6170 33293375 : if (!is_gimple_assign (last_stmt))
6171 : return NULL;
6172 :
6173 23652823 : var = gimple_assign_rhs1 (last_stmt);
6174 23652823 : lhs = gimple_assign_lhs (last_stmt);
6175 23652823 : rhs_code = gimple_assign_rhs_code (last_stmt);
6176 :
6177 23652823 : if (rhs_code == VIEW_CONVERT_EXPR)
6178 200133 : var = TREE_OPERAND (var, 0);
6179 :
6180 23652823 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (var)))
6181 : return NULL;
6182 :
6183 766698 : hash_set<gimple *> bool_stmts;
6184 :
6185 766698 : if (CONVERT_EXPR_CODE_P (rhs_code)
6186 : || rhs_code == VIEW_CONVERT_EXPR
6187 : || rhs_code == FLOAT_EXPR)
6188 : {
6189 169562 : if (! (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
6190 2423 : || SCALAR_FLOAT_TYPE_P (TREE_TYPE (lhs)))
6191 167673 : || VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)))
6192 : return NULL;
6193 78186 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6194 :
6195 78186 : tree type = integer_type_for_mask (var, vinfo);
6196 78186 : tree cst0, cst1, tmp;
6197 :
6198 78186 : if (!type)
6199 : return NULL;
6200 :
6201 : /* We may directly use cond with narrowed type to avoid multiple cond
6202 : exprs with following result packing and perform single cond with
6203 : packed mask instead. In case of widening we better make cond first
6204 : and then extract results. */
6205 37874 : if (TYPE_MODE (type) == TYPE_MODE (TREE_TYPE (lhs)))
6206 24711 : type = TREE_TYPE (lhs);
6207 :
6208 37874 : cst0 = build_int_cst (type, 0);
6209 37874 : cst1 = build_int_cst (type, 1);
6210 37874 : tmp = vect_recog_temp_ssa_var (type, NULL);
6211 37874 : pattern_stmt = gimple_build_assign (tmp, COND_EXPR, var, cst1, cst0);
6212 :
6213 37874 : if (!useless_type_conversion_p (type, TREE_TYPE (lhs)))
6214 : {
6215 13163 : tree new_vectype = get_vectype_for_scalar_type (vinfo, type);
6216 13163 : append_pattern_def_seq (vinfo, stmt_vinfo,
6217 : pattern_stmt, new_vectype);
6218 :
6219 13163 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6220 13163 : pattern_stmt
6221 26018 : = gimple_build_assign (lhs, (rhs_code == FLOAT_EXPR
6222 : ? FLOAT_EXPR : CONVERT_EXPR), tmp);
6223 : }
6224 :
6225 37874 : *type_out = vectype;
6226 37874 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6227 :
6228 37874 : return pattern_stmt;
6229 : }
6230 : else if (rhs_code == COND_EXPR
6231 213306 : && TREE_CODE (var) == SSA_NAME)
6232 : {
6233 213306 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6234 213306 : if (vectype == NULL_TREE)
6235 : return NULL;
6236 :
6237 : /* Build a scalar type for the boolean result that when
6238 : vectorized matches the vector type of the result in
6239 : size and number of elements. */
6240 197912 : unsigned prec
6241 197912 : = vector_element_size (tree_to_poly_uint64 (TYPE_SIZE (vectype)),
6242 : TYPE_VECTOR_SUBPARTS (vectype));
6243 :
6244 197912 : tree type
6245 395824 : = build_nonstandard_integer_type (prec,
6246 197912 : TYPE_UNSIGNED (TREE_TYPE (var)));
6247 197912 : if (get_vectype_for_scalar_type (vinfo, type) == NULL_TREE)
6248 : return NULL;
6249 :
6250 197912 : enum vect_def_type dt;
6251 197912 : if (integer_type_for_mask (var, vinfo))
6252 : return NULL;
6253 37280 : else if (TREE_CODE (TREE_TYPE (var)) == BOOLEAN_TYPE
6254 37280 : && vect_is_simple_use (var, vinfo, &dt)
6255 37280 : && (dt == vect_external_def
6256 37273 : || dt == vect_constant_def))
6257 : {
6258 : /* If the condition is already a boolean then manually convert it to a
6259 : mask of the given integer type but don't set a vectype. */
6260 1350 : tree lhs_ivar = vect_recog_temp_ssa_var (type, NULL);
6261 1350 : pattern_stmt = gimple_build_assign (lhs_ivar, COND_EXPR, var,
6262 : build_all_ones_cst (type),
6263 : build_zero_cst (type));
6264 1350 : append_inv_pattern_def_seq (vinfo, pattern_stmt);
6265 1350 : var = lhs_ivar;
6266 : }
6267 :
6268 37280 : tree lhs_var = vect_recog_temp_ssa_var (boolean_type_node, NULL);
6269 37280 : pattern_stmt = gimple_build_assign (lhs_var, NE_EXPR, var,
6270 37280 : build_zero_cst (TREE_TYPE (var)));
6271 :
6272 37280 : tree new_vectype = get_mask_type_for_scalar_type (vinfo, TREE_TYPE (var));
6273 37280 : if (!new_vectype)
6274 : return NULL;
6275 :
6276 37280 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, new_vectype,
6277 37280 : TREE_TYPE (var));
6278 :
6279 37280 : lhs_var = vect_convert_mask_for_vectype (lhs_var, vectype, stmt_vinfo,
6280 : vinfo);
6281 :
6282 37280 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6283 37280 : pattern_stmt
6284 37280 : = gimple_build_assign (lhs, COND_EXPR, lhs_var,
6285 : gimple_assign_rhs2 (last_stmt),
6286 : gimple_assign_rhs3 (last_stmt));
6287 37280 : *type_out = vectype;
6288 37280 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6289 :
6290 37280 : return pattern_stmt;
6291 : }
6292 468236 : else if (rhs_code == BIT_NOT_EXPR
6293 468236 : && !vect_use_mask_type_p (stmt_vinfo))
6294 : {
6295 : /* When we have a bool data inversion rewrite that to an XOR to
6296 : cope with the fact that we'll use a wider vector element type. */
6297 10354 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6298 10354 : pattern_stmt
6299 10354 : = gimple_build_assign (lhs, BIT_XOR_EXPR, var,
6300 10354 : build_all_ones_cst (TREE_TYPE (var)));
6301 10354 : *type_out = NULL_TREE;
6302 10354 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6303 :
6304 10354 : return pattern_stmt;
6305 : }
6306 457882 : else if ((rhs_code == BIT_XOR_EXPR
6307 : || rhs_code == BIT_AND_EXPR
6308 457882 : || rhs_code == BIT_IOR_EXPR)
6309 362849 : && TREE_CODE (var) == SSA_NAME)
6310 : {
6311 362849 : tree rhs2 = gimple_assign_rhs2 (last_stmt);
6312 362849 : if (TREE_CODE (rhs2) != SSA_NAME)
6313 : return NULL;
6314 362849 : tree lhs_type = integer_type_for_mask (lhs, vinfo);
6315 362849 : if (!lhs_type)
6316 : return NULL;
6317 190903 : vectype = get_mask_type_for_scalar_type (vinfo, lhs_type);
6318 190903 : if (!vectype)
6319 : return NULL;
6320 190903 : vect_def_type dt1, dt2;
6321 190903 : tree rhs1_type = integer_type_for_mask (var, vinfo, &dt1);
6322 190903 : tree rhs2_type = integer_type_for_mask (rhs2, vinfo, &dt2);
6323 190903 : if ((rhs1_type || dt1 == vect_external_def)
6324 174950 : && (rhs2_type || dt2 == vect_external_def))
6325 : return NULL;
6326 : /* When one input is a mask and the other is not create a pattern
6327 : stmt sequence that creates a mask for the non-mask input and
6328 : convert it to one suitable for the output mask used. */
6329 38969 : if (rhs1_type && !rhs2_type)
6330 : {
6331 23016 : tree rhs1_vectype = get_mask_type_for_scalar_type (vinfo, rhs1_type);
6332 23016 : if (!rhs1_vectype)
6333 : return NULL;
6334 23016 : tree rhs2_vectype = get_vectype_for_scalar_type (vinfo,
6335 23016 : TREE_TYPE (rhs2));
6336 23016 : if (!rhs2_vectype)
6337 : return NULL;
6338 23016 : tree new_vectype = truth_type_for (rhs2_vectype);
6339 23016 : tree tem = vect_recog_temp_ssa_var (TREE_TYPE (new_vectype), NULL);
6340 23016 : pattern_stmt = gimple_build_assign (tem, NE_EXPR, rhs2,
6341 : build_zero_cst
6342 23016 : (TREE_TYPE (rhs2)));
6343 23016 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
6344 23016 : new_vectype, TREE_TYPE (new_vectype));
6345 23016 : rhs2 = vect_convert_mask_for_vectype (tem, rhs1_vectype,
6346 : stmt_vinfo, vinfo, rhs1_type);
6347 : }
6348 15953 : else if (!rhs1_type && rhs2_type)
6349 : {
6350 15953 : tree rhs2_vectype = get_mask_type_for_scalar_type (vinfo, rhs2_type);
6351 15953 : if (!rhs2_vectype)
6352 : return NULL;
6353 15953 : tree rhs1_vectype = get_vectype_for_scalar_type (vinfo,
6354 15953 : TREE_TYPE (var));
6355 15953 : if (!rhs1_vectype)
6356 : return NULL;
6357 15953 : tree new_vectype = truth_type_for (rhs1_vectype);
6358 15953 : tree tem = vect_recog_temp_ssa_var (TREE_TYPE (new_vectype), NULL);
6359 15953 : pattern_stmt = gimple_build_assign (tem, NE_EXPR, var,
6360 : build_zero_cst
6361 15953 : (TREE_TYPE (var)));
6362 15953 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
6363 15953 : new_vectype, TREE_TYPE (new_vectype));
6364 15953 : var = vect_convert_mask_for_vectype (tem, rhs2_vectype,
6365 : stmt_vinfo, vinfo, rhs2_type);
6366 : }
6367 38969 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6368 38969 : pattern_stmt = gimple_build_assign (lhs, rhs_code, var, rhs2);
6369 38969 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6370 38969 : *type_out = vectype;
6371 38969 : return pattern_stmt;
6372 : }
6373 95033 : else if (rhs_code == SSA_NAME
6374 25951 : && STMT_VINFO_DATA_REF (stmt_vinfo))
6375 : {
6376 7870 : stmt_vec_info pattern_stmt_info;
6377 7870 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6378 7870 : if (!vectype || !VECTOR_MODE_P (TYPE_MODE (vectype)))
6379 : return NULL;
6380 :
6381 7870 : tree type = integer_type_for_mask (var, vinfo);
6382 7870 : if (!type)
6383 : return NULL;
6384 :
6385 4473 : var = vect_convert_mask_for_vectype (var, vectype, stmt_vinfo, vinfo);
6386 :
6387 4473 : tree cst0 = build_int_cst (TREE_TYPE (vectype), 0);
6388 4473 : tree cst1 = build_int_cst (TREE_TYPE (vectype), 1);
6389 4473 : rhs = vect_recog_temp_ssa_var (TREE_TYPE (vectype), NULL);
6390 4473 : pattern_stmt = gimple_build_assign (rhs, COND_EXPR, var, cst1, cst0);
6391 4473 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vectype);
6392 :
6393 4473 : lhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (vectype), lhs);
6394 4473 : pattern_stmt = gimple_build_assign (lhs, SSA_NAME, rhs);
6395 4473 : pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
6396 4473 : vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
6397 4473 : *type_out = vectype;
6398 4473 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6399 :
6400 4473 : return pattern_stmt;
6401 : }
6402 : else
6403 : return NULL;
6404 766698 : }
6405 :
6406 :
6407 : /* Function vect_recog_mask_conversion_pattern
6408 :
6409 : Try to find statements which require boolean type
6410 : conversion. Additional conversion statements are
6411 : added to handle such cases. For example:
6412 :
6413 : bool m_1, m_2, m_3;
6414 : int i_4, i_5;
6415 : double d_6, d_7;
6416 : char c_1, c_2, c_3;
6417 :
6418 : S1 m_1 = i_4 > i_5;
6419 : S2 m_2 = d_6 < d_7;
6420 : S3 m_3 = m_1 & m_2;
6421 : S4 c_1 = m_3 ? c_2 : c_3;
6422 :
6423 : Will be transformed into:
6424 :
6425 : S1 m_1 = i_4 > i_5;
6426 : S2 m_2 = d_6 < d_7;
6427 : S3'' m_2' = (_Bool[bitsize=32])m_2
6428 : S3' m_3' = m_1 & m_2';
6429 : S4'' m_3'' = (_Bool[bitsize=8])m_3'
6430 : S4' c_1' = m_3'' ? c_2 : c_3; */
6431 :
6432 : static gimple *
6433 33321613 : vect_recog_mask_conversion_pattern (vec_info *vinfo,
6434 : stmt_vec_info stmt_vinfo, tree *type_out)
6435 : {
6436 33321613 : gimple *last_stmt = stmt_vinfo->stmt;
6437 33321613 : enum tree_code rhs_code;
6438 33321613 : tree lhs = NULL_TREE, rhs1, rhs2, tmp, rhs1_type, rhs2_type;
6439 33321613 : tree vectype1, vectype2;
6440 33321613 : stmt_vec_info pattern_stmt_info;
6441 :
6442 : /* Check for MASK_LOAD and MASK_STORE as well as COND_OP calls requiring mask
6443 : conversion. */
6444 33321613 : if (is_gimple_call (last_stmt)
6445 33321613 : && gimple_call_internal_p (last_stmt))
6446 : {
6447 133846 : gcall *pattern_stmt;
6448 :
6449 133846 : internal_fn ifn = gimple_call_internal_fn (last_stmt);
6450 133846 : int mask_argno = internal_fn_mask_index (ifn);
6451 133846 : if (mask_argno < 0)
6452 : return NULL;
6453 :
6454 14612 : bool store_p = internal_store_fn_p (ifn);
6455 14612 : bool load_p = internal_store_fn_p (ifn);
6456 14612 : if (store_p)
6457 : {
6458 2488 : int rhs_index = internal_fn_stored_value_index (ifn);
6459 2488 : tree rhs = gimple_call_arg (last_stmt, rhs_index);
6460 2488 : vectype1 = get_vectype_for_scalar_type (vinfo, TREE_TYPE (rhs));
6461 : }
6462 : else
6463 : {
6464 12124 : lhs = gimple_call_lhs (last_stmt);
6465 12124 : if (!lhs)
6466 : return NULL;
6467 12124 : vectype1 = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6468 : }
6469 :
6470 14612 : if (!vectype1)
6471 : return NULL;
6472 :
6473 14292 : tree mask_arg = gimple_call_arg (last_stmt, mask_argno);
6474 14292 : tree mask_arg_type = integer_type_for_mask (mask_arg, vinfo);
6475 14292 : if (mask_arg_type)
6476 : {
6477 12665 : vectype2 = get_mask_type_for_scalar_type (vinfo, mask_arg_type);
6478 :
6479 12665 : if (!vectype2
6480 12665 : || known_eq (TYPE_VECTOR_SUBPARTS (vectype1),
6481 : TYPE_VECTOR_SUBPARTS (vectype2)))
6482 : return NULL;
6483 : }
6484 1627 : else if (store_p || load_p)
6485 : return NULL;
6486 :
6487 6113 : tmp = build_mask_conversion (vinfo, mask_arg, vectype1, stmt_vinfo);
6488 :
6489 6113 : auto_vec<tree, 8> args;
6490 6113 : unsigned int nargs = gimple_call_num_args (last_stmt);
6491 6113 : args.safe_grow (nargs, true);
6492 36678 : for (unsigned int i = 0; i < nargs; ++i)
6493 24452 : args[i] = ((int) i == mask_argno
6494 24452 : ? tmp
6495 18339 : : gimple_call_arg (last_stmt, i));
6496 6113 : pattern_stmt = gimple_build_call_internal_vec (ifn, args);
6497 :
6498 6113 : if (!store_p)
6499 : {
6500 5841 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6501 5841 : gimple_call_set_lhs (pattern_stmt, lhs);
6502 : }
6503 :
6504 5841 : if (load_p || store_p)
6505 272 : gimple_call_set_nothrow (pattern_stmt, true);
6506 :
6507 6113 : pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
6508 6113 : if (STMT_VINFO_DATA_REF (stmt_vinfo))
6509 1804 : vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
6510 :
6511 6113 : *type_out = vectype1;
6512 6113 : vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
6513 :
6514 6113 : return pattern_stmt;
6515 6113 : }
6516 :
6517 33187767 : if (!is_gimple_assign (last_stmt))
6518 : return NULL;
6519 :
6520 23681061 : gimple *pattern_stmt;
6521 23681061 : lhs = gimple_assign_lhs (last_stmt);
6522 23681061 : rhs1 = gimple_assign_rhs1 (last_stmt);
6523 23681061 : rhs_code = gimple_assign_rhs_code (last_stmt);
6524 :
6525 : /* Check for cond expression requiring mask conversion. */
6526 23681061 : if (rhs_code == COND_EXPR)
6527 : {
6528 193792 : vectype1 = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6529 :
6530 193792 : gcc_assert (! COMPARISON_CLASS_P (rhs1));
6531 193792 : if (TREE_CODE (rhs1) == SSA_NAME)
6532 : {
6533 193792 : rhs1_type = integer_type_for_mask (rhs1, vinfo);
6534 193792 : if (!rhs1_type)
6535 : return NULL;
6536 : }
6537 : else
6538 : return NULL;
6539 :
6540 180558 : vectype2 = get_mask_type_for_scalar_type (vinfo, rhs1_type);
6541 :
6542 180558 : if (!vectype1 || !vectype2)
6543 : return NULL;
6544 :
6545 : /* Continue if a conversion is needed. Also continue if we have
6546 : a comparison whose vector type would normally be different from
6547 : VECTYPE2 when considered in isolation. In that case we'll
6548 : replace the comparison with an SSA name (so that we can record
6549 : its vector type) and behave as though the comparison was an SSA
6550 : name from the outset. */
6551 178398 : if (known_eq (TYPE_VECTOR_SUBPARTS (vectype1),
6552 : TYPE_VECTOR_SUBPARTS (vectype2)))
6553 : return NULL;
6554 :
6555 45915 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1),
6556 91830 : TYPE_VECTOR_SUBPARTS (vectype2)))
6557 45915 : tmp = build_mask_conversion (vinfo, rhs1, vectype1, stmt_vinfo);
6558 : else
6559 : tmp = rhs1;
6560 :
6561 45915 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6562 45915 : pattern_stmt = gimple_build_assign (lhs, COND_EXPR, tmp,
6563 : gimple_assign_rhs2 (last_stmt),
6564 : gimple_assign_rhs3 (last_stmt));
6565 :
6566 45915 : *type_out = vectype1;
6567 45915 : vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
6568 :
6569 45915 : return pattern_stmt;
6570 : }
6571 :
6572 : /* Now check for binary boolean operations requiring conversion for
6573 : one of operands. */
6574 23487269 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)))
6575 : return NULL;
6576 :
6577 1891218 : if (rhs_code != BIT_IOR_EXPR
6578 : && rhs_code != BIT_XOR_EXPR
6579 1891218 : && rhs_code != BIT_AND_EXPR
6580 1567338 : && TREE_CODE_CLASS (rhs_code) != tcc_comparison)
6581 : return NULL;
6582 :
6583 1704214 : rhs2 = gimple_assign_rhs2 (last_stmt);
6584 :
6585 1704214 : rhs1_type = integer_type_for_mask (rhs1, vinfo);
6586 1704214 : rhs2_type = integer_type_for_mask (rhs2, vinfo);
6587 :
6588 1704214 : if (!rhs1_type || !rhs2_type
6589 1704214 : || TYPE_PRECISION (rhs1_type) == TYPE_PRECISION (rhs2_type))
6590 : return NULL;
6591 :
6592 32228 : if (TYPE_PRECISION (rhs1_type) < TYPE_PRECISION (rhs2_type))
6593 : {
6594 24746 : vectype1 = get_mask_type_for_scalar_type (vinfo, rhs1_type);
6595 24746 : if (!vectype1)
6596 : return NULL;
6597 24746 : rhs2 = build_mask_conversion (vinfo, rhs2, vectype1, stmt_vinfo,
6598 : rhs1_type);
6599 : }
6600 : else
6601 : {
6602 7482 : vectype1 = get_mask_type_for_scalar_type (vinfo, rhs2_type);
6603 7482 : if (!vectype1)
6604 : return NULL;
6605 7482 : rhs1 = build_mask_conversion (vinfo, rhs1, vectype1, stmt_vinfo,
6606 : rhs2_type);
6607 : }
6608 :
6609 32228 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6610 32228 : pattern_stmt = gimple_build_assign (lhs, rhs_code, rhs1, rhs2);
6611 :
6612 32228 : *type_out = vectype1;
6613 32228 : vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
6614 :
6615 32228 : return pattern_stmt;
6616 : }
6617 :
6618 : /* STMT_INFO is a load or store. If the load or store is conditional, return
6619 : the boolean condition under which it occurs, otherwise return null. */
6620 :
6621 : static tree
6622 106102 : vect_get_load_store_mask (stmt_vec_info stmt_info)
6623 : {
6624 106102 : if (gassign *def_assign = dyn_cast <gassign *> (stmt_info->stmt))
6625 : {
6626 105079 : gcc_assert (gimple_assign_single_p (def_assign));
6627 : return NULL_TREE;
6628 : }
6629 :
6630 1023 : if (gcall *def_call = dyn_cast <gcall *> (stmt_info->stmt))
6631 : {
6632 1023 : internal_fn ifn = gimple_call_internal_fn (def_call);
6633 1023 : int mask_index = internal_fn_mask_index (ifn);
6634 1023 : return gimple_call_arg (def_call, mask_index);
6635 : }
6636 :
6637 0 : gcc_unreachable ();
6638 : }
6639 :
6640 : /* Return the equivalent of:
6641 :
6642 : fold_convert (TYPE, VALUE)
6643 :
6644 : with the expectation that the operation will be vectorized.
6645 : If new statements are needed, add them as pattern statements
6646 : to STMT_INFO. */
6647 :
6648 : static tree
6649 0 : vect_add_conversion_to_pattern (vec_info *vinfo,
6650 : tree type, tree value, stmt_vec_info stmt_info)
6651 : {
6652 0 : if (useless_type_conversion_p (type, TREE_TYPE (value)))
6653 : return value;
6654 :
6655 0 : tree new_value = vect_recog_temp_ssa_var (type, NULL);
6656 0 : gassign *conversion = gimple_build_assign (new_value, CONVERT_EXPR, value);
6657 0 : append_pattern_def_seq (vinfo, stmt_info, conversion,
6658 : get_vectype_for_scalar_type (vinfo, type));
6659 0 : return new_value;
6660 : }
6661 :
6662 : /* Try to convert STMT_INFO into a call to a gather load or scatter store
6663 : internal function. Return the final statement on success and set
6664 : *TYPE_OUT to the vector type being loaded or stored.
6665 :
6666 : This function only handles gathers and scatters that were recognized
6667 : as such from the outset (indicated by STMT_VINFO_GATHER_SCATTER_P). */
6668 :
6669 : static gimple *
6670 33321613 : vect_recog_gather_scatter_pattern (vec_info *vinfo,
6671 : stmt_vec_info stmt_info, tree *type_out)
6672 : {
6673 : /* Currently we only support this for loop vectorization. */
6674 33321613 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6675 4652571 : if (!loop_vinfo)
6676 : return NULL;
6677 :
6678 : /* Make sure that we're looking at a gather load or scatter store. */
6679 4652571 : data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
6680 4652571 : if (!dr || !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
6681 : return NULL;
6682 :
6683 : /* Get the boolean that controls whether the load or store happens.
6684 : This is null if the operation is unconditional. */
6685 106102 : tree mask = vect_get_load_store_mask (stmt_info);
6686 :
6687 : /* DR analysis nailed down the vector type for the access. */
6688 106102 : tree gs_vectype = STMT_VINFO_VECTYPE (stmt_info);
6689 :
6690 : /* Make sure that the target supports an appropriate internal
6691 : function for the gather/scatter operation. */
6692 106102 : gather_scatter_info gs_info;
6693 106102 : if (!vect_check_gather_scatter (stmt_info, gs_vectype, loop_vinfo, &gs_info)
6694 106102 : || gs_info.ifn == IFN_LAST)
6695 : return NULL;
6696 :
6697 : /* Convert the mask to the right form. */
6698 0 : if (mask)
6699 0 : mask = vect_convert_mask_for_vectype (mask, gs_vectype, stmt_info,
6700 : loop_vinfo);
6701 0 : else if (gs_info.ifn == IFN_MASK_SCATTER_STORE
6702 0 : || gs_info.ifn == IFN_MASK_GATHER_LOAD
6703 0 : || gs_info.ifn == IFN_MASK_LEN_SCATTER_STORE
6704 0 : || gs_info.ifn == IFN_MASK_LEN_GATHER_LOAD)
6705 0 : mask = build_int_cst (TREE_TYPE (truth_type_for (gs_vectype)), -1);
6706 :
6707 : /* Get the invariant base and non-invariant offset, converting the
6708 : latter to the same width as the vector elements. */
6709 0 : tree base = gs_info.base;
6710 0 : tree offset_type = TREE_TYPE (gs_info.offset_vectype);
6711 0 : tree offset = vect_add_conversion_to_pattern (vinfo, offset_type,
6712 : gs_info.offset, stmt_info);
6713 :
6714 : /* Build the new pattern statement. */
6715 0 : tree scale = size_int (gs_info.scale);
6716 0 : gcall *pattern_stmt;
6717 :
6718 0 : if (DR_IS_READ (dr))
6719 : {
6720 0 : tree zero = build_zero_cst (gs_info.element_type);
6721 0 : if (mask != NULL)
6722 : {
6723 0 : int elsval = MASK_LOAD_ELSE_ZERO;
6724 :
6725 0 : tree vec_els
6726 0 : = vect_get_mask_load_else (elsval, TREE_TYPE (gs_vectype));
6727 0 : pattern_stmt = gimple_build_call_internal (gs_info.ifn, 7, base,
6728 : gs_info.alias_ptr,
6729 : offset, scale, zero, mask,
6730 : vec_els);
6731 : }
6732 : else
6733 0 : pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5, base,
6734 : gs_info.alias_ptr,
6735 : offset, scale, zero);
6736 0 : tree lhs = gimple_get_lhs (stmt_info->stmt);
6737 0 : tree load_lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6738 0 : gimple_call_set_lhs (pattern_stmt, load_lhs);
6739 : }
6740 : else
6741 : {
6742 0 : tree rhs = vect_get_store_rhs (stmt_info);
6743 0 : if (mask != NULL)
6744 0 : pattern_stmt = gimple_build_call_internal (gs_info.ifn, 6,
6745 : base, gs_info.alias_ptr,
6746 : offset, scale, rhs, mask);
6747 : else
6748 0 : pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5,
6749 : base, gs_info.alias_ptr,
6750 : offset, scale, rhs);
6751 : }
6752 0 : gimple_call_set_nothrow (pattern_stmt, true);
6753 :
6754 : /* Copy across relevant vectorization info and associate DR with the
6755 : new pattern statement instead of the original statement. */
6756 0 : stmt_vec_info pattern_stmt_info = loop_vinfo->add_stmt (pattern_stmt);
6757 0 : loop_vinfo->move_dr (pattern_stmt_info, stmt_info);
6758 :
6759 0 : *type_out = gs_vectype;
6760 0 : vect_pattern_detected ("gather/scatter pattern", stmt_info->stmt);
6761 :
6762 0 : return pattern_stmt;
6763 : }
6764 :
6765 : /* Helper method of vect_recog_cond_store_pattern, checks to see if COND_ARG
6766 : is points to a load statement that reads the same data as that of
6767 : STORE_VINFO. */
6768 :
6769 : static bool
6770 35678 : vect_cond_store_pattern_same_ref (vec_info *vinfo,
6771 : stmt_vec_info store_vinfo, tree cond_arg)
6772 : {
6773 35678 : stmt_vec_info load_stmt_vinfo = vinfo->lookup_def (cond_arg);
6774 35678 : if (!load_stmt_vinfo
6775 20429 : || !STMT_VINFO_DATA_REF (load_stmt_vinfo)
6776 11970 : || DR_IS_WRITE (STMT_VINFO_DATA_REF (load_stmt_vinfo))
6777 47648 : || !same_data_refs (STMT_VINFO_DATA_REF (store_vinfo),
6778 : STMT_VINFO_DATA_REF (load_stmt_vinfo)))
6779 26374 : return false;
6780 :
6781 : return true;
6782 : }
6783 :
6784 : /* Function vect_recog_cond_store_pattern
6785 :
6786 : Try to find the following pattern:
6787 :
6788 : x = *_3;
6789 : c = a CMP b;
6790 : y = c ? t_20 : x;
6791 : *_3 = y;
6792 :
6793 : where the store of _3 happens on a conditional select on a value loaded
6794 : from the same location. In such case we can elide the initial load if
6795 : MASK_STORE is supported and instead only conditionally write out the result.
6796 :
6797 : The pattern produces for the above:
6798 :
6799 : c = a CMP b;
6800 : .MASK_STORE (_3, c, t_20)
6801 :
6802 : Input:
6803 :
6804 : * STMT_VINFO: The stmt from which the pattern search begins. In the
6805 : example, when this function is called with _3 then the search begins.
6806 :
6807 : Output:
6808 :
6809 : * TYPE_OUT: The type of the output of this pattern.
6810 :
6811 : * Return value: A new stmt that will be used to replace the sequence. */
6812 :
6813 : static gimple *
6814 33321613 : vect_recog_cond_store_pattern (vec_info *vinfo,
6815 : stmt_vec_info stmt_vinfo, tree *type_out)
6816 : {
6817 33321613 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6818 4652571 : if (!loop_vinfo)
6819 : return NULL;
6820 :
6821 4652571 : gimple *store_stmt = STMT_VINFO_STMT (stmt_vinfo);
6822 :
6823 : /* Needs to be a gimple store where we have DR info for. */
6824 4652571 : if (!STMT_VINFO_DATA_REF (stmt_vinfo)
6825 1113254 : || DR_IS_READ (STMT_VINFO_DATA_REF (stmt_vinfo))
6826 5044596 : || !gimple_store_p (store_stmt))
6827 : return NULL;
6828 :
6829 389637 : tree st_rhs = gimple_assign_rhs1 (store_stmt);
6830 :
6831 389637 : if (TREE_CODE (st_rhs) != SSA_NAME)
6832 : return NULL;
6833 :
6834 302701 : auto cond_vinfo = vinfo->lookup_def (st_rhs);
6835 :
6836 : /* If the condition isn't part of the loop then bool recog wouldn't have seen
6837 : it and so this transformation may not be valid. */
6838 302701 : if (!cond_vinfo)
6839 : return NULL;
6840 :
6841 284989 : cond_vinfo = vect_stmt_to_vectorize (cond_vinfo);
6842 284989 : gassign *cond_stmt = dyn_cast<gassign *> (STMT_VINFO_STMT (cond_vinfo));
6843 353508 : if (!cond_stmt || gimple_assign_rhs_code (cond_stmt) != COND_EXPR)
6844 : return NULL;
6845 :
6846 : /* Check if the else value matches the original loaded one. */
6847 18880 : bool invert = false;
6848 18880 : tree cmp_ls = gimple_arg (cond_stmt, 0);
6849 18880 : if (TREE_CODE (cmp_ls) != SSA_NAME)
6850 : return NULL;
6851 :
6852 18880 : tree cond_arg1 = gimple_arg (cond_stmt, 1);
6853 18880 : tree cond_arg2 = gimple_arg (cond_stmt, 2);
6854 :
6855 18880 : if (!vect_cond_store_pattern_same_ref (vinfo, stmt_vinfo, cond_arg2)
6856 18880 : && !(invert = vect_cond_store_pattern_same_ref (vinfo, stmt_vinfo,
6857 : cond_arg1)))
6858 : return NULL;
6859 :
6860 9304 : vect_pattern_detected ("vect_recog_cond_store_pattern", store_stmt);
6861 :
6862 9304 : tree scalar_type = TREE_TYPE (st_rhs);
6863 9304 : if (VECTOR_TYPE_P (scalar_type))
6864 : return NULL;
6865 :
6866 9304 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
6867 9304 : if (vectype == NULL_TREE)
6868 : return NULL;
6869 :
6870 9304 : machine_mode mask_mode;
6871 9304 : machine_mode vecmode = TYPE_MODE (vectype);
6872 1864 : if (!VECTOR_MODE_P (vecmode)
6873 9304 : || targetm.vectorize.conditional_operation_is_expensive (IFN_MASK_STORE)
6874 9304 : || !targetm.vectorize.get_mask_mode (vecmode).exists (&mask_mode)
6875 9304 : || !can_vec_mask_load_store_p (vecmode, mask_mode, false))
6876 : return NULL;
6877 :
6878 0 : tree base = DR_REF (STMT_VINFO_DATA_REF (stmt_vinfo));
6879 0 : if (may_be_nonaddressable_p (base))
6880 : return NULL;
6881 :
6882 : /* We need to use the false parameter of the conditional select. */
6883 0 : tree cond_store_arg = invert ? cond_arg2 : cond_arg1;
6884 0 : tree cond_load_arg = invert ? cond_arg1 : cond_arg2;
6885 0 : gimple *load_stmt = SSA_NAME_DEF_STMT (cond_load_arg);
6886 :
6887 : /* This is a rough estimation to check that there aren't any aliasing stores
6888 : in between the load and store. It's a bit strict, but for now it's good
6889 : enough. */
6890 0 : if (gimple_vuse (load_stmt) != gimple_vuse (store_stmt))
6891 : return NULL;
6892 :
6893 : /* If we have to invert the condition, i.e. use the true argument rather than
6894 : the false argument, we have to negate the mask. */
6895 0 : if (invert)
6896 : {
6897 0 : tree var = vect_recog_temp_ssa_var (boolean_type_node, NULL);
6898 :
6899 : /* Invert the mask using ^ 1. */
6900 0 : tree itype = TREE_TYPE (cmp_ls);
6901 0 : gassign *conv = gimple_build_assign (var, BIT_XOR_EXPR, cmp_ls,
6902 : build_int_cst (itype, 1));
6903 :
6904 0 : tree mask_vec_type = get_mask_type_for_scalar_type (vinfo, itype);
6905 0 : append_pattern_def_seq (vinfo, stmt_vinfo, conv, mask_vec_type, itype);
6906 0 : cmp_ls= var;
6907 : }
6908 :
6909 0 : if (TREE_CODE (base) != MEM_REF)
6910 0 : base = build_fold_addr_expr (base);
6911 :
6912 0 : tree ptr = build_int_cst (reference_alias_ptr_type (base),
6913 0 : get_object_alignment (base));
6914 :
6915 : /* Convert the mask to the right form. */
6916 0 : tree mask = vect_convert_mask_for_vectype (cmp_ls, vectype, stmt_vinfo,
6917 : vinfo);
6918 :
6919 0 : gcall *call
6920 0 : = gimple_build_call_internal (IFN_MASK_STORE, 4, base, ptr, mask,
6921 : cond_store_arg);
6922 0 : gimple_set_location (call, gimple_location (store_stmt));
6923 :
6924 : /* Copy across relevant vectorization info and associate DR with the
6925 : new pattern statement instead of the original statement. */
6926 0 : stmt_vec_info pattern_stmt_info = loop_vinfo->add_stmt (call);
6927 0 : loop_vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
6928 :
6929 0 : *type_out = vectype;
6930 0 : return call;
6931 : }
6932 :
6933 : /* Return true if TYPE is a non-boolean integer type. These are the types
6934 : that we want to consider for narrowing. */
6935 :
6936 : static bool
6937 68189777 : vect_narrowable_type_p (tree type)
6938 : {
6939 68189777 : return INTEGRAL_TYPE_P (type) && !VECT_SCALAR_BOOLEAN_TYPE_P (type);
6940 : }
6941 :
6942 : /* Return true if the operation given by CODE can be truncated to N bits
6943 : when only N bits of the output are needed. This is only true if bit N+1
6944 : of the inputs has no effect on the low N bits of the result. */
6945 :
6946 : static bool
6947 16854173 : vect_truncatable_operation_p (tree_code code)
6948 : {
6949 16854173 : switch (code)
6950 : {
6951 : case NEGATE_EXPR:
6952 : case PLUS_EXPR:
6953 : case MINUS_EXPR:
6954 : case MULT_EXPR:
6955 : case BIT_NOT_EXPR:
6956 : case BIT_AND_EXPR:
6957 : case BIT_IOR_EXPR:
6958 : case BIT_XOR_EXPR:
6959 : case COND_EXPR:
6960 : return true;
6961 :
6962 6574486 : default:
6963 6574486 : return false;
6964 : }
6965 : }
6966 :
6967 : /* Record that STMT_INFO could be changed from operating on TYPE to
6968 : operating on a type with the precision and sign given by PRECISION
6969 : and SIGN respectively. PRECISION is an arbitrary bit precision;
6970 : it might not be a whole number of bytes. */
6971 :
6972 : static void
6973 2593438 : vect_set_operation_type (stmt_vec_info stmt_info, tree type,
6974 : unsigned int precision, signop sign)
6975 : {
6976 : /* Round the precision up to a whole number of bytes. */
6977 2593438 : precision = vect_element_precision (precision);
6978 2593438 : if (precision < TYPE_PRECISION (type)
6979 2593438 : && (!stmt_info->operation_precision
6980 41072 : || stmt_info->operation_precision > precision))
6981 : {
6982 1687708 : stmt_info->operation_precision = precision;
6983 1687708 : stmt_info->operation_sign = sign;
6984 : }
6985 2593438 : }
6986 :
6987 : /* Record that STMT_INFO only requires MIN_INPUT_PRECISION from its
6988 : non-boolean inputs, all of which have type TYPE. MIN_INPUT_PRECISION
6989 : is an arbitrary bit precision; it might not be a whole number of bytes. */
6990 :
6991 : static void
6992 12174087 : vect_set_min_input_precision (stmt_vec_info stmt_info, tree type,
6993 : unsigned int min_input_precision)
6994 : {
6995 : /* This operation in isolation only requires the inputs to have
6996 : MIN_INPUT_PRECISION of precision, However, that doesn't mean
6997 : that MIN_INPUT_PRECISION is a natural precision for the chain
6998 : as a whole. E.g. consider something like:
6999 :
7000 : unsigned short *x, *y;
7001 : *y = ((*x & 0xf0) >> 4) | (*y << 4);
7002 :
7003 : The right shift can be done on unsigned chars, and only requires the
7004 : result of "*x & 0xf0" to be done on unsigned chars. But taking that
7005 : approach would mean turning a natural chain of single-vector unsigned
7006 : short operations into one that truncates "*x" and then extends
7007 : "(*x & 0xf0) >> 4", with two vectors for each unsigned short
7008 : operation and one vector for each unsigned char operation.
7009 : This would be a significant pessimization.
7010 :
7011 : Instead only propagate the maximum of this precision and the precision
7012 : required by the users of the result. This means that we don't pessimize
7013 : the case above but continue to optimize things like:
7014 :
7015 : unsigned char *y;
7016 : unsigned short *x;
7017 : *y = ((*x & 0xf0) >> 4) | (*y << 4);
7018 :
7019 : Here we would truncate two vectors of *x to a single vector of
7020 : unsigned chars and use single-vector unsigned char operations for
7021 : everything else, rather than doing two unsigned short copies of
7022 : "(*x & 0xf0) >> 4" and then truncating the result. */
7023 12174087 : min_input_precision = MAX (min_input_precision,
7024 : stmt_info->min_output_precision);
7025 :
7026 12174087 : if (min_input_precision < TYPE_PRECISION (type)
7027 12174087 : && (!stmt_info->min_input_precision
7028 65669 : || stmt_info->min_input_precision > min_input_precision))
7029 630507 : stmt_info->min_input_precision = min_input_precision;
7030 12174087 : }
7031 :
7032 : /* Subroutine of vect_determine_min_output_precision. Return true if
7033 : we can calculate a reduced number of output bits for STMT_INFO,
7034 : whose result is LHS. */
7035 :
7036 : static bool
7037 15615212 : vect_determine_min_output_precision_1 (vec_info *vinfo,
7038 : stmt_vec_info stmt_info, tree lhs)
7039 : {
7040 : /* Take the maximum precision required by users of the result. */
7041 15615212 : unsigned int precision = 0;
7042 15615212 : imm_use_iterator iter;
7043 15615212 : use_operand_p use;
7044 16710708 : FOR_EACH_IMM_USE_FAST (use, iter, lhs)
7045 : {
7046 16409134 : gimple *use_stmt = USE_STMT (use);
7047 16409134 : if (is_gimple_debug (use_stmt))
7048 790559 : continue;
7049 15618575 : stmt_vec_info use_stmt_info = vinfo->lookup_stmt (use_stmt);
7050 15618575 : if (!use_stmt_info || !use_stmt_info->min_input_precision)
7051 : return false;
7052 : /* The input precision recorded for COND_EXPRs applies only to the
7053 : "then" and "else" values. */
7054 305540 : gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
7055 273877 : if (assign
7056 273877 : && gimple_assign_rhs_code (assign) == COND_EXPR
7057 603 : && use->use != gimple_assign_rhs2_ptr (assign)
7058 603 : && use->use != gimple_assign_rhs3_ptr (assign))
7059 : return false;
7060 1097205 : precision = MAX (precision, use_stmt_info->min_input_precision);
7061 15313638 : }
7062 :
7063 301574 : if (dump_enabled_p ())
7064 6146 : dump_printf_loc (MSG_NOTE, vect_location,
7065 : "only the low %d bits of %T are significant\n",
7066 : precision, lhs);
7067 301574 : stmt_info->min_output_precision = precision;
7068 301574 : return true;
7069 : }
7070 :
7071 : /* Calculate min_output_precision for STMT_INFO. */
7072 :
7073 : static void
7074 40181430 : vect_determine_min_output_precision (vec_info *vinfo, stmt_vec_info stmt_info)
7075 : {
7076 : /* We're only interested in statements with a narrowable result. */
7077 40181430 : tree lhs = gimple_get_lhs (stmt_info->stmt);
7078 40181430 : if (!lhs
7079 31723485 : || TREE_CODE (lhs) != SSA_NAME
7080 67052768 : || !vect_narrowable_type_p (TREE_TYPE (lhs)))
7081 : return;
7082 :
7083 15615212 : if (!vect_determine_min_output_precision_1 (vinfo, stmt_info, lhs))
7084 15313638 : stmt_info->min_output_precision = TYPE_PRECISION (TREE_TYPE (lhs));
7085 : }
7086 :
7087 : /* Use range information to decide whether STMT (described by STMT_INFO)
7088 : could be done in a narrower type. This is effectively a forward
7089 : propagation, since it uses context-independent information that applies
7090 : to all users of an SSA name. */
7091 :
7092 : static void
7093 22937658 : vect_determine_precisions_from_range (stmt_vec_info stmt_info, gassign *stmt)
7094 : {
7095 22937658 : tree lhs = gimple_assign_lhs (stmt);
7096 22937658 : if (!lhs || TREE_CODE (lhs) != SSA_NAME)
7097 20463444 : return;
7098 :
7099 18380781 : tree type = TREE_TYPE (lhs);
7100 18380781 : if (!vect_narrowable_type_p (type))
7101 : return;
7102 :
7103 : /* First see whether we have any useful range information for the result. */
7104 12094988 : unsigned int precision = TYPE_PRECISION (type);
7105 12094988 : signop sign = TYPE_SIGN (type);
7106 12094988 : wide_int min_value, max_value;
7107 12094988 : if (!vect_get_range_info (lhs, &min_value, &max_value))
7108 : return;
7109 :
7110 5793934 : tree_code code = gimple_assign_rhs_code (stmt);
7111 5793934 : unsigned int nops = gimple_num_ops (stmt);
7112 :
7113 5793934 : if (!vect_truncatable_operation_p (code))
7114 : {
7115 : /* Handle operations that can be computed in type T if all inputs
7116 : and outputs can be represented in type T. Also handle left and
7117 : right shifts, where (in addition) the maximum shift amount must
7118 : be less than the number of bits in T. */
7119 2149386 : bool is_shift;
7120 2149386 : switch (code)
7121 : {
7122 : case LSHIFT_EXPR:
7123 : case RSHIFT_EXPR:
7124 : is_shift = true;
7125 : break;
7126 :
7127 311773 : case ABS_EXPR:
7128 311773 : case MIN_EXPR:
7129 311773 : case MAX_EXPR:
7130 311773 : case TRUNC_DIV_EXPR:
7131 311773 : case CEIL_DIV_EXPR:
7132 311773 : case FLOOR_DIV_EXPR:
7133 311773 : case ROUND_DIV_EXPR:
7134 311773 : case EXACT_DIV_EXPR:
7135 : /* Modulus is excluded because it is typically calculated by doing
7136 : a division, for which minimum signed / -1 isn't representable in
7137 : the original signed type. We could take the division range into
7138 : account instead, if handling modulus ever becomes important. */
7139 311773 : is_shift = false;
7140 311773 : break;
7141 :
7142 : default:
7143 : return;
7144 : }
7145 1423163 : for (unsigned int i = 1; i < nops; ++i)
7146 : {
7147 1106077 : tree op = gimple_op (stmt, i);
7148 1106077 : wide_int op_min_value, op_max_value;
7149 1106077 : if (TREE_CODE (op) == INTEGER_CST)
7150 : {
7151 324512 : unsigned int op_precision = TYPE_PRECISION (TREE_TYPE (op));
7152 324512 : op_min_value = op_max_value = wi::to_wide (op, op_precision);
7153 : }
7154 781565 : else if (TREE_CODE (op) == SSA_NAME)
7155 : {
7156 781565 : if (!vect_get_range_info (op, &op_min_value, &op_max_value))
7157 : return;
7158 : }
7159 : else
7160 : return;
7161 :
7162 718527 : if (is_shift && i == 2)
7163 : {
7164 : /* There needs to be one more bit than the maximum shift amount.
7165 :
7166 : If the maximum shift amount is already 1 less than PRECISION
7167 : then we can't narrow the shift further. Dealing with that
7168 : case first ensures that we can safely use an unsigned range
7169 : below.
7170 :
7171 : op_min_value isn't relevant, since shifts by negative amounts
7172 : are UB. */
7173 208494 : if (wi::geu_p (op_max_value, precision - 1))
7174 : return;
7175 183123 : unsigned int min_bits = op_max_value.to_uhwi () + 1;
7176 :
7177 : /* As explained below, we can convert a signed shift into an
7178 : unsigned shift if the sign bit is always clear. At this
7179 : point we've already processed the ranges of the output and
7180 : the first input. */
7181 183123 : auto op_sign = sign;
7182 183123 : if (sign == SIGNED && !wi::neg_p (min_value))
7183 : op_sign = UNSIGNED;
7184 366246 : op_min_value = wide_int::from (wi::min_value (min_bits, op_sign),
7185 183123 : precision, op_sign);
7186 366246 : op_max_value = wide_int::from (wi::max_value (min_bits, op_sign),
7187 183123 : precision, op_sign);
7188 : }
7189 693156 : min_value = wi::min (min_value, op_min_value, sign);
7190 693156 : max_value = wi::max (max_value, op_max_value, sign);
7191 1106077 : }
7192 : }
7193 :
7194 : /* Try to switch signed types for unsigned types if we can.
7195 : This is better for two reasons. First, unsigned ops tend
7196 : to be cheaper than signed ops. Second, it means that we can
7197 : handle things like:
7198 :
7199 : signed char c;
7200 : int res = (int) c & 0xff00; // range [0x0000, 0xff00]
7201 :
7202 : as:
7203 :
7204 : signed char c;
7205 : unsigned short res_1 = (unsigned short) c & 0xff00;
7206 : int res = (int) res_1;
7207 :
7208 : where the intermediate result res_1 has unsigned rather than
7209 : signed type. */
7210 3961634 : if (sign == SIGNED && !wi::neg_p (min_value))
7211 : sign = UNSIGNED;
7212 :
7213 : /* See what precision is required for MIN_VALUE and MAX_VALUE. */
7214 3961634 : unsigned int precision1 = wi::min_precision (min_value, sign);
7215 3961634 : unsigned int precision2 = wi::min_precision (max_value, sign);
7216 3961634 : unsigned int value_precision = MAX (precision1, precision2);
7217 3961634 : if (value_precision >= precision)
7218 : return;
7219 :
7220 2474214 : if (dump_enabled_p ())
7221 115796 : dump_printf_loc (MSG_NOTE, vect_location, "can narrow to %s:%d"
7222 : " without loss of precision: %G",
7223 : sign == SIGNED ? "signed" : "unsigned",
7224 : value_precision, (gimple *) stmt);
7225 :
7226 2474214 : vect_set_operation_type (stmt_info, type, value_precision, sign);
7227 2474214 : vect_set_min_input_precision (stmt_info, type, value_precision);
7228 12094988 : }
7229 :
7230 : /* Use information about the users of STMT's result to decide whether
7231 : STMT (described by STMT_INFO) could be done in a narrower type.
7232 : This is effectively a backward propagation. */
7233 :
7234 : static void
7235 22937658 : vect_determine_precisions_from_users (stmt_vec_info stmt_info, gassign *stmt)
7236 : {
7237 22937658 : tree_code code = gimple_assign_rhs_code (stmt);
7238 22937658 : unsigned int opno = (code == COND_EXPR ? 2 : 1);
7239 22937658 : tree type = TREE_TYPE (gimple_op (stmt, opno));
7240 22937658 : if (!vect_narrowable_type_p (type))
7241 13237785 : return;
7242 :
7243 14160528 : unsigned int precision = TYPE_PRECISION (type);
7244 14160528 : unsigned int operation_precision, min_input_precision;
7245 14160528 : switch (code)
7246 : {
7247 2610736 : CASE_CONVERT:
7248 : /* Only the bits that contribute to the output matter. Don't change
7249 : the precision of the operation itself. */
7250 2610736 : operation_precision = precision;
7251 2610736 : min_input_precision = stmt_info->min_output_precision;
7252 2610736 : break;
7253 :
7254 489553 : case LSHIFT_EXPR:
7255 489553 : case RSHIFT_EXPR:
7256 489553 : {
7257 489553 : tree shift = gimple_assign_rhs2 (stmt);
7258 489553 : unsigned int min_const_shift, max_const_shift;
7259 489553 : wide_int min_shift, max_shift;
7260 489553 : if (TREE_CODE (shift) == SSA_NAME
7261 108121 : && vect_get_range_info (shift, &min_shift, &max_shift)
7262 83518 : && wi::ge_p (min_shift, 0, TYPE_SIGN (TREE_TYPE (shift)))
7263 570302 : && wi::lt_p (max_shift, TYPE_PRECISION (type),
7264 80749 : TYPE_SIGN (TREE_TYPE (shift))))
7265 : {
7266 72674 : min_const_shift = min_shift.to_uhwi ();
7267 72674 : max_const_shift = max_shift.to_uhwi ();
7268 : }
7269 416879 : else if (TREE_CODE (shift) == INTEGER_CST
7270 798311 : && wi::ltu_p (wi::to_widest (shift), precision))
7271 381324 : min_const_shift = max_const_shift = TREE_INT_CST_LOW (shift);
7272 : else
7273 35555 : return;
7274 453998 : if (code == LSHIFT_EXPR)
7275 : {
7276 : /* Avoid creating an undefined shift.
7277 :
7278 : ??? We could instead use min_output_precision as-is and
7279 : optimize out-of-range shifts to zero. However, only
7280 : degenerate testcases shift away all their useful input data,
7281 : and it isn't natural to drop input operations in the middle
7282 : of vectorization. This sort of thing should really be
7283 : handled before vectorization. */
7284 110113 : operation_precision = MAX (stmt_info->min_output_precision,
7285 : max_const_shift + 1);
7286 : /* We need CONST_SHIFT fewer bits of the input. */
7287 110113 : min_input_precision = (MAX (operation_precision, max_const_shift)
7288 : - min_const_shift);
7289 : }
7290 : else
7291 : {
7292 : /* We need CONST_SHIFT extra bits to do the operation. */
7293 343885 : operation_precision = (stmt_info->min_output_precision
7294 : + max_const_shift);
7295 343885 : min_input_precision = operation_precision;
7296 : }
7297 453998 : break;
7298 489553 : }
7299 :
7300 11060239 : default:
7301 11060239 : if (vect_truncatable_operation_p (code))
7302 : {
7303 : /* Input bit N has no effect on output bits N-1 and lower. */
7304 6635139 : operation_precision = stmt_info->min_output_precision;
7305 6635139 : min_input_precision = operation_precision;
7306 6635139 : break;
7307 : }
7308 : return;
7309 : }
7310 :
7311 9699873 : if (operation_precision < precision)
7312 : {
7313 119224 : if (dump_enabled_p ())
7314 2859 : dump_printf_loc (MSG_NOTE, vect_location, "can narrow to %s:%d"
7315 : " without affecting users: %G",
7316 2859 : TYPE_UNSIGNED (type) ? "unsigned" : "signed",
7317 : operation_precision, (gimple *) stmt);
7318 238448 : vect_set_operation_type (stmt_info, type, operation_precision,
7319 119224 : TYPE_SIGN (type));
7320 : }
7321 9699873 : vect_set_min_input_precision (stmt_info, type, min_input_precision);
7322 : }
7323 :
7324 : /* Return true if the statement described by STMT_INFO sets a boolean
7325 : SSA_NAME and if we know how to vectorize this kind of statement using
7326 : vector mask types. */
7327 :
7328 : static bool
7329 41286103 : possible_vector_mask_operation_p (stmt_vec_info stmt_info)
7330 : {
7331 41286103 : tree lhs = gimple_get_lhs (stmt_info->stmt);
7332 41286103 : tree_code code = ERROR_MARK;
7333 41286103 : gassign *assign = NULL;
7334 41286103 : gcond *cond = NULL;
7335 :
7336 41286103 : if ((assign = dyn_cast <gassign *> (stmt_info->stmt)))
7337 23727248 : code = gimple_assign_rhs_code (assign);
7338 17558855 : else if ((cond = dyn_cast <gcond *> (stmt_info->stmt)))
7339 : {
7340 5595524 : lhs = gimple_cond_lhs (cond);
7341 5595524 : code = gimple_cond_code (cond);
7342 : }
7343 :
7344 41286103 : if (!lhs
7345 38328335 : || TREE_CODE (lhs) != SSA_NAME
7346 74723448 : || !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)))
7347 : return false;
7348 :
7349 2256607 : if (code != ERROR_MARK)
7350 : {
7351 1994183 : switch (code)
7352 : {
7353 : CASE_CONVERT:
7354 : case SSA_NAME:
7355 : case BIT_NOT_EXPR:
7356 : case BIT_IOR_EXPR:
7357 : case BIT_XOR_EXPR:
7358 : case BIT_AND_EXPR:
7359 : return true;
7360 :
7361 1564578 : default:
7362 1564578 : return TREE_CODE_CLASS (code) == tcc_comparison;
7363 : }
7364 : }
7365 262424 : else if (is_a <gphi *> (stmt_info->stmt))
7366 147541 : return true;
7367 : return false;
7368 : }
7369 :
7370 : /* If STMT_INFO sets a boolean SSA_NAME, see whether we should use
7371 : a vector mask type instead of a normal vector type. Record the
7372 : result in STMT_INFO->mask_precision. Returns true when the
7373 : precision changed. */
7374 :
7375 : static bool
7376 41286103 : vect_determine_mask_precision (vec_info *vinfo, stmt_vec_info stmt_info)
7377 : {
7378 41286103 : if (!possible_vector_mask_operation_p (stmt_info))
7379 : return false;
7380 :
7381 : /* If at least one boolean input uses a vector mask type,
7382 : pick the mask type with the narrowest elements.
7383 :
7384 : ??? This is the traditional behavior. It should always produce
7385 : the smallest number of operations, but isn't necessarily the
7386 : optimal choice. For example, if we have:
7387 :
7388 : a = b & c
7389 :
7390 : where:
7391 :
7392 : - the user of a wants it to have a mask type for 16-bit elements (M16)
7393 : - b also uses M16
7394 : - c uses a mask type for 8-bit elements (M8)
7395 :
7396 : then picking M8 gives:
7397 :
7398 : - 1 M16->M8 pack for b
7399 : - 1 M8 AND for a
7400 : - 2 M8->M16 unpacks for the user of a
7401 :
7402 : whereas picking M16 would have given:
7403 :
7404 : - 2 M8->M16 unpacks for c
7405 : - 2 M16 ANDs for a
7406 :
7407 : The number of operations are equal, but M16 would have given
7408 : a shorter dependency chain and allowed more ILP. */
7409 2088817 : unsigned int precision = ~0U;
7410 2088817 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
7411 :
7412 : /* If the statement compares two values that shouldn't use vector masks,
7413 : try comparing the values as normal scalars instead. */
7414 2088817 : tree_code code = ERROR_MARK;
7415 2088817 : tree op0_type;
7416 2088817 : unsigned int nops = -1;
7417 2088817 : unsigned int ops_start = 0;
7418 :
7419 2088817 : if (gassign *assign = dyn_cast <gassign *> (stmt))
7420 : {
7421 1362257 : code = gimple_assign_rhs_code (assign);
7422 1362257 : op0_type = TREE_TYPE (gimple_assign_rhs1 (assign));
7423 1362257 : nops = gimple_num_ops (assign);
7424 1362257 : ops_start = 1;
7425 : }
7426 726560 : else if (gcond *cond = dyn_cast <gcond *> (stmt))
7427 : {
7428 579019 : code = gimple_cond_code (cond);
7429 579019 : op0_type = TREE_TYPE (gimple_cond_lhs (cond));
7430 579019 : nops = 2;
7431 579019 : ops_start = 0;
7432 : }
7433 :
7434 1941276 : if (code != ERROR_MARK)
7435 : {
7436 5778538 : for (unsigned int i = ops_start; i < nops; ++i)
7437 : {
7438 3837262 : tree rhs = gimple_op (stmt, i);
7439 3837262 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (rhs)))
7440 1875534 : continue;
7441 :
7442 1961728 : stmt_vec_info def_stmt_info = vinfo->lookup_def (rhs);
7443 1961728 : if (!def_stmt_info)
7444 : /* Don't let external or constant operands influence the choice.
7445 : We can convert them to whichever vector type we pick. */
7446 595387 : continue;
7447 :
7448 1366341 : if (def_stmt_info->mask_precision)
7449 : {
7450 1137406 : if (precision > def_stmt_info->mask_precision)
7451 3837262 : precision = def_stmt_info->mask_precision;
7452 : }
7453 : }
7454 :
7455 1941276 : if (precision == ~0U
7456 1565970 : && TREE_CODE_CLASS (code) == tcc_comparison)
7457 : {
7458 1355966 : scalar_mode mode;
7459 1355966 : tree vectype, mask_type;
7460 1355966 : if (is_a <scalar_mode> (TYPE_MODE (op0_type), &mode)
7461 : /* Do not allow this to set vinfo->vector_mode, this might
7462 : disrupt the result for the next iteration. */
7463 1355966 : && (vectype = get_related_vectype_for_scalar_type
7464 1627607 : (vinfo->vector_mode, op0_type))
7465 1184899 : && (mask_type = truth_type_for (vectype))
7466 1184899 : && expand_vec_cmp_expr_p (vectype, mask_type, code))
7467 1826516 : precision = GET_MODE_BITSIZE (mode);
7468 : }
7469 : }
7470 : else
7471 : {
7472 147541 : gphi *phi = as_a <gphi *> (stmt_info->stmt);
7473 757960 : for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
7474 : {
7475 462878 : tree rhs = gimple_phi_arg_def (phi, i);
7476 :
7477 462878 : stmt_vec_info def_stmt_info = vinfo->lookup_def (rhs);
7478 462878 : if (!def_stmt_info)
7479 : /* Don't let external or constant operands influence the choice.
7480 : We can convert them to whichever vector type we pick. */
7481 305109 : continue;
7482 :
7483 157769 : if (def_stmt_info->mask_precision)
7484 : {
7485 131986 : if (precision > def_stmt_info->mask_precision)
7486 462878 : precision = def_stmt_info->mask_precision;
7487 : }
7488 : }
7489 : }
7490 :
7491 2088817 : if (stmt_info->mask_precision != precision)
7492 : {
7493 1958724 : if (dump_enabled_p ())
7494 : {
7495 8182 : if (precision == ~0U)
7496 1797 : dump_printf_loc (MSG_NOTE, vect_location,
7497 : "using normal nonmask vectors for %G",
7498 : stmt_info->stmt);
7499 : else
7500 6385 : dump_printf_loc (MSG_NOTE, vect_location,
7501 : "using boolean precision %d for %G",
7502 : precision, stmt_info->stmt);
7503 : }
7504 :
7505 : /* ??? We'd like to assert stmt_info->mask_precision == 0
7506 : || stmt_info->mask_precision > precision, thus that we only
7507 : decrease mask precisions throughout iteration, but the
7508 : tcc_comparison handling above means for comparisons of bools
7509 : we start with 8 but might increase in case the bools get mask
7510 : precision on their own. */
7511 1958724 : stmt_info->mask_precision = precision;
7512 1958724 : return true;
7513 : }
7514 : return false;
7515 : }
7516 :
7517 : /* Handle vect_determine_precisions for STMT_INFO, given that we
7518 : have already done so for the users of its result. */
7519 :
7520 : void
7521 40181430 : vect_determine_stmt_precisions (vec_info *vinfo, stmt_vec_info stmt_info)
7522 : {
7523 40181430 : vect_determine_min_output_precision (vinfo, stmt_info);
7524 40181430 : if (gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt))
7525 : {
7526 22937658 : vect_determine_precisions_from_range (stmt_info, stmt);
7527 22937658 : vect_determine_precisions_from_users (stmt_info, stmt);
7528 : }
7529 40181430 : }
7530 :
7531 : /* Walk backwards through the vectorizable region to determine the
7532 : values of these fields:
7533 :
7534 : - min_output_precision
7535 : - min_input_precision
7536 : - operation_precision
7537 : - operation_sign. */
7538 :
7539 : void
7540 1137300 : vect_determine_precisions (vec_info *vinfo)
7541 : {
7542 1137300 : basic_block *bbs = vinfo->bbs;
7543 1137300 : unsigned int nbbs = vinfo->nbbs;
7544 :
7545 1157649 : DUMP_VECT_SCOPE ("vect_determine_precisions");
7546 :
7547 : /* For mask precisions we have to iterate since otherwise we do not
7548 : get reduction PHI precision correct. For now do this only for
7549 : loop vectorization. */
7550 1208835 : bool changed;
7551 1208835 : do
7552 : {
7553 1208835 : changed = false;
7554 13594271 : for (unsigned int i = 0; i < nbbs; i++)
7555 : {
7556 12385436 : basic_block bb = bbs[i];
7557 12385436 : for (auto gsi = gsi_start_phis (bb);
7558 20010117 : !gsi_end_p (gsi); gsi_next (&gsi))
7559 : {
7560 7624681 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi.phi ());
7561 7624681 : if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
7562 7428771 : changed |= vect_determine_mask_precision (vinfo, stmt_info);
7563 : }
7564 134669278 : for (auto gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
7565 : {
7566 109898406 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (gsi));
7567 109898406 : if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
7568 33857332 : changed |= vect_determine_mask_precision (vinfo, stmt_info);
7569 : }
7570 : }
7571 : }
7572 2346135 : while (changed && is_a <loop_vec_info> (vinfo));
7573 :
7574 13358009 : for (unsigned int i = 0; i < nbbs; i++)
7575 : {
7576 12220709 : basic_block bb = bbs[nbbs - i - 1];
7577 142260036 : for (auto gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
7578 : {
7579 108229317 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (gsi));
7580 108229317 : if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
7581 32959348 : vect_determine_stmt_precisions (vinfo, stmt_info);
7582 : }
7583 19638701 : for (auto gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
7584 : {
7585 7417992 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi.phi ());
7586 7417992 : if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
7587 7222082 : vect_determine_stmt_precisions (vinfo, stmt_info);
7588 : }
7589 : }
7590 1137300 : }
7591 :
7592 : typedef gimple *(*vect_recog_func_ptr) (vec_info *, stmt_vec_info, tree *);
7593 :
7594 : struct vect_recog_func
7595 : {
7596 : vect_recog_func_ptr fn;
7597 : const char *name;
7598 : };
7599 :
7600 : /* Note that ordering matters - the first pattern matching on a stmt is
7601 : taken which means usually the more complex one needs to precede the
7602 : less comples onex (widen_sum only after dot_prod or sad for example). */
7603 : static vect_recog_func vect_vect_recog_func_ptrs[] = {
7604 : { vect_recog_bitfield_ref_pattern, "bitfield_ref" },
7605 : { vect_recog_bit_insert_pattern, "bit_insert" },
7606 : { vect_recog_abd_pattern, "abd" },
7607 : { vect_recog_over_widening_pattern, "over_widening" },
7608 : /* Must come after over_widening, which narrows the shift as much as
7609 : possible beforehand. */
7610 : { vect_recog_average_pattern, "average" },
7611 : { vect_recog_cond_expr_convert_pattern, "cond_expr_convert" },
7612 : { vect_recog_mulhs_pattern, "mult_high" },
7613 : { vect_recog_cast_forwprop_pattern, "cast_forwprop" },
7614 : { vect_recog_widen_mult_pattern, "widen_mult" },
7615 : { vect_recog_dot_prod_pattern, "dot_prod" },
7616 : { vect_recog_sad_pattern, "sad" },
7617 : { vect_recog_widen_sum_pattern, "widen_sum" },
7618 : { vect_recog_pow_pattern, "pow" },
7619 : { vect_recog_popcount_clz_ctz_ffs_pattern, "popcount_clz_ctz_ffs" },
7620 : { vect_recog_ctz_ffs_pattern, "ctz_ffs" },
7621 : { vect_recog_widen_shift_pattern, "widen_shift" },
7622 : { vect_recog_rotate_pattern, "rotate" },
7623 : { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" },
7624 : { vect_recog_divmod_pattern, "divmod" },
7625 : { vect_recog_mod_var_pattern, "modvar" },
7626 : { vect_recog_mult_pattern, "mult" },
7627 : { vect_recog_sat_add_pattern, "sat_add" },
7628 : { vect_recog_sat_sub_pattern, "sat_sub" },
7629 : { vect_recog_sat_trunc_pattern, "sat_trunc" },
7630 : { vect_recog_gcond_pattern, "gcond" },
7631 : { vect_recog_bool_pattern, "bool" },
7632 : /* This must come before mask conversion, and includes the parts
7633 : of mask conversion that are needed for gather and scatter
7634 : internal functions. */
7635 : { vect_recog_gather_scatter_pattern, "gather_scatter" },
7636 : { vect_recog_cond_store_pattern, "cond_store" },
7637 : { vect_recog_mask_conversion_pattern, "mask_conversion" },
7638 : { vect_recog_widen_plus_pattern, "widen_plus" },
7639 : { vect_recog_widen_minus_pattern, "widen_minus" },
7640 : { vect_recog_widen_abd_pattern, "widen_abd" },
7641 : /* These must come after the double widening ones. */
7642 : };
7643 :
7644 : /* Mark statements that are involved in a pattern. */
7645 :
7646 : void
7647 1094504 : vect_mark_pattern_stmts (vec_info *vinfo,
7648 : stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
7649 : tree pattern_vectype)
7650 : {
7651 1094504 : stmt_vec_info orig_stmt_info_saved = orig_stmt_info;
7652 1094504 : gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
7653 :
7654 1094504 : gimple *orig_pattern_stmt = NULL;
7655 1094504 : if (is_pattern_stmt_p (orig_stmt_info))
7656 : {
7657 : /* We're replacing a statement in an existing pattern definition
7658 : sequence. */
7659 11872 : orig_pattern_stmt = orig_stmt_info->stmt;
7660 11872 : if (dump_enabled_p ())
7661 993 : dump_printf_loc (MSG_NOTE, vect_location,
7662 : "replacing earlier pattern %G", orig_pattern_stmt);
7663 :
7664 : /* To keep the book-keeping simple, just swap the lhs of the
7665 : old and new statements, so that the old one has a valid but
7666 : unused lhs. */
7667 11872 : tree old_lhs = gimple_get_lhs (orig_pattern_stmt);
7668 11872 : gimple_set_lhs (orig_pattern_stmt, gimple_get_lhs (pattern_stmt));
7669 11872 : gimple_set_lhs (pattern_stmt, old_lhs);
7670 :
7671 11872 : if (dump_enabled_p ())
7672 993 : dump_printf_loc (MSG_NOTE, vect_location, "with %G", pattern_stmt);
7673 :
7674 : /* Switch to the statement that ORIG replaces. */
7675 11872 : orig_stmt_info = STMT_VINFO_RELATED_STMT (orig_stmt_info);
7676 :
7677 : /* We shouldn't be replacing the main pattern statement. */
7678 11872 : gcc_assert (STMT_VINFO_RELATED_STMT (orig_stmt_info)->stmt
7679 : != orig_pattern_stmt);
7680 : }
7681 :
7682 1094504 : if (def_seq)
7683 2400906 : for (gimple_stmt_iterator si = gsi_start (def_seq);
7684 2400906 : !gsi_end_p (si); gsi_next (&si))
7685 : {
7686 1444943 : if (dump_enabled_p ())
7687 27967 : dump_printf_loc (MSG_NOTE, vect_location,
7688 : "extra pattern stmt: %G", gsi_stmt (si));
7689 1444943 : stmt_vec_info pattern_stmt_info
7690 1444943 : = vect_init_pattern_stmt (vinfo, gsi_stmt (si),
7691 : orig_stmt_info, pattern_vectype);
7692 : /* Stmts in the def sequence are not vectorizable cycle or
7693 : induction defs, instead they should all be vect_internal_def
7694 : feeding the main pattern stmt which retains this def type. */
7695 1444943 : STMT_VINFO_DEF_TYPE (pattern_stmt_info) = vect_internal_def;
7696 : }
7697 :
7698 1094504 : if (orig_pattern_stmt)
7699 : {
7700 11872 : vect_init_pattern_stmt (vinfo, pattern_stmt,
7701 : orig_stmt_info, pattern_vectype);
7702 :
7703 : /* Insert all the new pattern statements before the original one. */
7704 11872 : gimple_seq *orig_def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
7705 11872 : gimple_stmt_iterator gsi = gsi_for_stmt (orig_pattern_stmt,
7706 : orig_def_seq);
7707 11872 : gsi_insert_seq_before_without_update (&gsi, def_seq, GSI_SAME_STMT);
7708 11872 : gsi_insert_before_without_update (&gsi, pattern_stmt, GSI_SAME_STMT);
7709 :
7710 : /* Remove the pattern statement that this new pattern replaces. */
7711 11872 : gsi_remove (&gsi, false);
7712 : }
7713 : else
7714 1082632 : vect_set_pattern_stmt (vinfo,
7715 : pattern_stmt, orig_stmt_info, pattern_vectype);
7716 :
7717 : /* For any conditionals mark them as vect_condition_def. */
7718 1094504 : if (is_a <gcond *> (pattern_stmt))
7719 385919 : STMT_VINFO_DEF_TYPE (STMT_VINFO_RELATED_STMT (orig_stmt_info)) = vect_condition_def;
7720 :
7721 : /* Transfer reduction path info to the pattern. */
7722 1094504 : if (STMT_VINFO_REDUC_IDX (orig_stmt_info_saved) != -1)
7723 : {
7724 16320 : gimple_match_op op;
7725 16320 : if (!gimple_extract_op (orig_stmt_info_saved->stmt, &op))
7726 0 : gcc_unreachable ();
7727 16320 : tree lookfor = op.ops[STMT_VINFO_REDUC_IDX (orig_stmt_info)];
7728 : /* Search the pattern def sequence and the main pattern stmt. Note
7729 : we may have inserted all into a containing pattern def sequence
7730 : so the following is a bit awkward. */
7731 16320 : gimple_stmt_iterator si;
7732 16320 : gimple *s;
7733 16320 : if (def_seq)
7734 : {
7735 15113 : si = gsi_start (def_seq);
7736 15113 : s = gsi_stmt (si);
7737 15113 : gsi_next (&si);
7738 : }
7739 : else
7740 : {
7741 : si = gsi_none ();
7742 16320 : s = pattern_stmt;
7743 : }
7744 34438 : do
7745 : {
7746 34438 : bool found = false;
7747 34438 : if (gimple_extract_op (s, &op))
7748 : {
7749 84050 : for (unsigned i = 0; i < op.num_ops; ++i)
7750 65926 : if (op.ops[i] == lookfor)
7751 : {
7752 16314 : STMT_VINFO_REDUC_IDX (vinfo->lookup_stmt (s)) = i;
7753 16314 : lookfor = gimple_get_lhs (s);
7754 16314 : found = true;
7755 16314 : break;
7756 : }
7757 : /* Try harder to find a mid-entry into an earlier pattern
7758 : sequence. Likewise an entry to a stmt skipping a conversion
7759 : on an input. This means that the initial 'lookfor' was
7760 : bogus. */
7761 16314 : if (!found)
7762 : {
7763 39442 : for (unsigned i = 0; i < op.num_ops; ++i)
7764 21324 : if (TREE_CODE (op.ops[i]) == SSA_NAME)
7765 18124 : if (auto def = vinfo->lookup_def (op.ops[i]))
7766 17930 : if (vect_is_reduction (def)
7767 17930 : || (is_a <gphi *> (def->stmt)
7768 6 : && STMT_VINFO_REDUC_DEF (def) != NULL))
7769 : {
7770 6 : STMT_VINFO_REDUC_IDX (vinfo->lookup_stmt (s)) = i;
7771 6 : lookfor = gimple_get_lhs (s);
7772 6 : found = true;
7773 6 : break;
7774 : }
7775 : }
7776 : }
7777 34438 : if (s == pattern_stmt)
7778 : {
7779 16320 : if (!found && dump_enabled_p ())
7780 0 : dump_printf_loc (MSG_NOTE, vect_location,
7781 : "failed to update reduction index.\n");
7782 16320 : break;
7783 : }
7784 18118 : if (gsi_end_p (si))
7785 : s = pattern_stmt;
7786 : else
7787 : {
7788 3005 : s = gsi_stmt (si);
7789 3005 : if (s == pattern_stmt)
7790 : /* Found the end inside a bigger pattern def seq. */
7791 : si = gsi_none ();
7792 : else
7793 3005 : gsi_next (&si);
7794 : }
7795 : } while (1);
7796 : }
7797 1094504 : }
7798 :
7799 : /* Function vect_pattern_recog_1
7800 :
7801 : Input:
7802 : PATTERN_RECOG_FUNC: A pointer to a function that detects a certain
7803 : computation pattern.
7804 : STMT_INFO: A stmt from which the pattern search should start.
7805 :
7806 : If PATTERN_RECOG_FUNC successfully detected the pattern, it creates
7807 : a sequence of statements that has the same functionality and can be
7808 : used to replace STMT_INFO. It returns the last statement in the sequence
7809 : and adds any earlier statements to STMT_INFO's STMT_VINFO_PATTERN_DEF_SEQ.
7810 : PATTERN_RECOG_FUNC also sets *TYPE_OUT to the vector type of the final
7811 : statement, having first checked that the target supports the new operation
7812 : in that type.
7813 :
7814 : This function also does some bookkeeping, as explained in the documentation
7815 : for vect_recog_pattern. */
7816 :
7817 : static void
7818 1074238366 : vect_pattern_recog_1 (vec_info *vinfo,
7819 : const vect_recog_func &recog_func, stmt_vec_info stmt_info)
7820 : {
7821 1074238366 : gimple *pattern_stmt;
7822 1074238366 : tree pattern_vectype;
7823 :
7824 : /* If this statement has already been replaced with pattern statements,
7825 : leave the original statement alone, since the first match wins.
7826 : Instead try to match against the definition statements that feed
7827 : the main pattern statement. */
7828 1074238366 : if (STMT_VINFO_IN_PATTERN_P (stmt_info))
7829 : {
7830 13923238 : gimple_stmt_iterator gsi;
7831 13923238 : for (gsi = gsi_start (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
7832 33462468 : !gsi_end_p (gsi); gsi_next (&gsi))
7833 19539230 : vect_pattern_recog_1 (vinfo, recog_func,
7834 : vinfo->lookup_stmt (gsi_stmt (gsi)));
7835 : return;
7836 : }
7837 :
7838 1060315128 : gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
7839 1060315128 : pattern_stmt = recog_func.fn (vinfo, stmt_info, &pattern_vectype);
7840 1060315128 : if (!pattern_stmt)
7841 : {
7842 : /* Clear any half-formed pattern definition sequence. */
7843 1059220624 : STMT_VINFO_PATTERN_DEF_SEQ (stmt_info) = NULL;
7844 1059220624 : return;
7845 : }
7846 :
7847 : /* Found a vectorizable pattern. */
7848 1094504 : if (dump_enabled_p ())
7849 20605 : dump_printf_loc (MSG_NOTE, vect_location,
7850 : "%s pattern recognized: %G",
7851 20605 : recog_func.name, pattern_stmt);
7852 :
7853 : /* Mark the stmts that are involved in the pattern. */
7854 1094504 : vect_mark_pattern_stmts (vinfo, stmt_info, pattern_stmt, pattern_vectype);
7855 : }
7856 :
7857 :
7858 : /* Function vect_pattern_recog
7859 :
7860 : Input:
7861 : LOOP_VINFO - a struct_loop_info of a loop in which we want to look for
7862 : computation idioms.
7863 :
7864 : Output - for each computation idiom that is detected we create a new stmt
7865 : that provides the same functionality and that can be vectorized. We
7866 : also record some information in the struct_stmt_info of the relevant
7867 : stmts, as explained below:
7868 :
7869 : At the entry to this function we have the following stmts, with the
7870 : following initial value in the STMT_VINFO fields:
7871 :
7872 : stmt in_pattern_p related_stmt vec_stmt
7873 : S1: a_i = .... - - -
7874 : S2: a_2 = ..use(a_i).. - - -
7875 : S3: a_1 = ..use(a_2).. - - -
7876 : S4: a_0 = ..use(a_1).. - - -
7877 : S5: ... = ..use(a_0).. - - -
7878 :
7879 : Say the sequence {S1,S2,S3,S4} was detected as a pattern that can be
7880 : represented by a single stmt. We then:
7881 : - create a new stmt S6 equivalent to the pattern (the stmt is not
7882 : inserted into the code)
7883 : - fill in the STMT_VINFO fields as follows:
7884 :
7885 : in_pattern_p related_stmt vec_stmt
7886 : S1: a_i = .... - - -
7887 : S2: a_2 = ..use(a_i).. - - -
7888 : S3: a_1 = ..use(a_2).. - - -
7889 : S4: a_0 = ..use(a_1).. true S6 -
7890 : '---> S6: a_new = .... - S4 -
7891 : S5: ... = ..use(a_0).. - - -
7892 :
7893 : (the last stmt in the pattern (S4) and the new pattern stmt (S6) point
7894 : to each other through the RELATED_STMT field).
7895 :
7896 : S6 will be marked as relevant in vect_mark_stmts_to_be_vectorized instead
7897 : of S4 because it will replace all its uses. Stmts {S1,S2,S3} will
7898 : remain irrelevant unless used by stmts other than S4.
7899 :
7900 : If vectorization succeeds, vect_transform_stmt will skip over {S1,S2,S3}
7901 : (because they are marked as irrelevant). It will vectorize S6, and record
7902 : a pointer to the new vector stmt VS6 from S6 (as usual).
7903 : S4 will be skipped, and S5 will be vectorized as usual:
7904 :
7905 : in_pattern_p related_stmt vec_stmt
7906 : S1: a_i = .... - - -
7907 : S2: a_2 = ..use(a_i).. - - -
7908 : S3: a_1 = ..use(a_2).. - - -
7909 : > VS6: va_new = .... - - -
7910 : S4: a_0 = ..use(a_1).. true S6 VS6
7911 : '---> S6: a_new = .... - S4 VS6
7912 : > VS5: ... = ..vuse(va_new).. - - -
7913 : S5: ... = ..use(a_0).. - - -
7914 :
7915 : DCE could then get rid of {S1,S2,S3,S4,S5} (if their defs are not used
7916 : elsewhere), and we'll end up with:
7917 :
7918 : VS6: va_new = ....
7919 : VS5: ... = ..vuse(va_new)..
7920 :
7921 : In case of more than one pattern statements, e.g., widen-mult with
7922 : intermediate type:
7923 :
7924 : S1 a_t = ;
7925 : S2 a_T = (TYPE) a_t;
7926 : '--> S3: a_it = (interm_type) a_t;
7927 : S4 prod_T = a_T * CONST;
7928 : '--> S5: prod_T' = a_it w* CONST;
7929 :
7930 : there may be other users of a_T outside the pattern. In that case S2 will
7931 : be marked as relevant (as well as S3), and both S2 and S3 will be analyzed
7932 : and vectorized. The vector stmt VS2 will be recorded in S2, and VS3 will
7933 : be recorded in S3. */
7934 :
7935 : void
7936 1137300 : vect_pattern_recog (vec_info *vinfo)
7937 : {
7938 1137300 : basic_block *bbs = vinfo->bbs;
7939 1137300 : unsigned int nbbs = vinfo->nbbs;
7940 :
7941 1137300 : vect_determine_precisions (vinfo);
7942 :
7943 1137300 : DUMP_VECT_SCOPE ("vect_pattern_recog");
7944 :
7945 : /* Scan through the stmts in the region, applying the pattern recognition
7946 : functions starting at each stmt visited. */
7947 13358009 : for (unsigned i = 0; i < nbbs; i++)
7948 : {
7949 12220709 : basic_block bb = bbs[i];
7950 :
7951 132670735 : for (auto si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
7952 : {
7953 108229317 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (si));
7954 :
7955 108229317 : if (!stmt_info || !STMT_VINFO_VECTORIZABLE (stmt_info))
7956 75269969 : continue;
7957 :
7958 : /* Scan over all generic vect_recog_xxx_pattern functions. */
7959 1087658484 : for (const auto &func_ptr : vect_vect_recog_func_ptrs)
7960 1054699136 : vect_pattern_recog_1 (vinfo, func_ptr,
7961 : stmt_info);
7962 : }
7963 : }
7964 :
7965 : /* After this no more add_stmt calls are allowed. */
7966 1137300 : vinfo->stmt_vec_info_ro = true;
7967 1137300 : }
7968 :
7969 : /* Build a GIMPLE_ASSIGN or GIMPLE_CALL with the tree_code,
7970 : or internal_fn contained in ch, respectively. */
7971 : gimple *
7972 170738 : vect_gimple_build (tree lhs, code_helper ch, tree op0, tree op1)
7973 : {
7974 170738 : gcc_assert (op0 != NULL_TREE);
7975 170738 : if (ch.is_tree_code ())
7976 170738 : return gimple_build_assign (lhs, (tree_code) ch, op0, op1);
7977 :
7978 0 : gcc_assert (ch.is_internal_fn ());
7979 0 : gimple* stmt = gimple_build_call_internal (as_internal_fn ((combined_fn) ch),
7980 : op1 == NULL_TREE ? 1 : 2,
7981 : op0, op1);
7982 0 : gimple_call_set_lhs (stmt, lhs);
7983 0 : return stmt;
7984 : }
|
