Line data Source code
1 : /* Analysis Utilities for Loop Vectorization.
2 : Copyright (C) 2006-2026 Free Software Foundation, Inc.
3 : Contributed by Dorit Nuzman <dorit@il.ibm.com>
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "gimple-iterator.h"
29 : #include "gimple-fold.h"
30 : #include "ssa.h"
31 : #include "expmed.h"
32 : #include "optabs-tree.h"
33 : #include "insn-config.h"
34 : #include "recog.h" /* FIXME: for insn_data */
35 : #include "fold-const.h"
36 : #include "stor-layout.h"
37 : #include "tree-eh.h"
38 : #include "gimplify.h"
39 : #include "gimple-iterator.h"
40 : #include "gimple-fold.h"
41 : #include "gimplify-me.h"
42 : #include "cfgloop.h"
43 : #include "tree-vectorizer.h"
44 : #include "dumpfile.h"
45 : #include "builtins.h"
46 : #include "internal-fn.h"
47 : #include "case-cfn-macros.h"
48 : #include "fold-const-call.h"
49 : #include "attribs.h"
50 : #include "cgraph.h"
51 : #include "omp-simd-clone.h"
52 : #include "predict.h"
53 : #include "tree-vector-builder.h"
54 : #include "tree-ssa-loop-ivopts.h"
55 : #include "vec-perm-indices.h"
56 : #include "gimple-range.h"
57 : #include "alias.h"
58 :
59 :
60 : /* TODO: Note the vectorizer still builds COND_EXPRs with GENERIC compares
61 : in the first operand. Disentangling this is future work, the
62 : IL is properly transferred to VEC_COND_EXPRs with separate compares. */
63 :
64 :
65 : /* Return true if we have a useful VR_RANGE range for VAR, storing it
66 : in *MIN_VALUE and *MAX_VALUE if so. Note the range in the dump files. */
67 :
68 : bool
69 12677907 : vect_get_range_info (tree var, wide_int *min_value, wide_int *max_value)
70 : {
71 12677907 : int_range_max vr;
72 12677907 : tree vr_min, vr_max;
73 25355814 : get_range_query (cfun)->range_of_expr (vr, var);
74 12677907 : if (vr.undefined_p ())
75 71 : vr.set_varying (TREE_TYPE (var));
76 12677907 : value_range_kind vr_type = get_legacy_range (vr, vr_min, vr_max);
77 12677907 : *min_value = wi::to_wide (vr_min);
78 12677907 : *max_value = wi::to_wide (vr_max);
79 12677907 : wide_int nonzero = get_nonzero_bits (var);
80 12677907 : signop sgn = TYPE_SIGN (TREE_TYPE (var));
81 12677907 : if (intersect_range_with_nonzero_bits (vr_type, min_value, max_value,
82 : nonzero, sgn) == VR_RANGE)
83 : {
84 6242903 : if (dump_enabled_p ())
85 : {
86 89388 : dump_generic_expr_loc (MSG_NOTE, vect_location, TDF_SLIM, var);
87 89388 : dump_printf (MSG_NOTE, " has range [");
88 89388 : dump_hex (MSG_NOTE, *min_value);
89 89388 : dump_printf (MSG_NOTE, ", ");
90 89388 : dump_hex (MSG_NOTE, *max_value);
91 89388 : dump_printf (MSG_NOTE, "]\n");
92 : }
93 : return true;
94 : }
95 : else
96 : {
97 6435004 : if (dump_enabled_p ())
98 : {
99 68168 : dump_generic_expr_loc (MSG_NOTE, vect_location, TDF_SLIM, var);
100 68168 : dump_printf (MSG_NOTE, " has no range info\n");
101 : }
102 : return false;
103 : }
104 12677907 : }
105 :
106 : /* Report that we've found an instance of pattern PATTERN in
107 : statement STMT. */
108 :
109 : static void
110 1331468 : vect_pattern_detected (const char *name, gimple *stmt)
111 : {
112 1331468 : if (dump_enabled_p ())
113 25560 : dump_printf_loc (MSG_NOTE, vect_location, "%s: detected: %G", name, stmt);
114 1331468 : }
115 :
116 : /* Associate pattern statement PATTERN_STMT with ORIG_STMT_INFO and
117 : return the pattern statement's stmt_vec_info. Set its vector type to
118 : VECTYPE if it doesn't have one already. */
119 :
120 : static stmt_vec_info
121 2523145 : vect_init_pattern_stmt (vec_info *vinfo, gimple *pattern_stmt,
122 : stmt_vec_info orig_stmt_info, tree vectype)
123 : {
124 2523145 : stmt_vec_info pattern_stmt_info = vinfo->lookup_stmt (pattern_stmt);
125 2523145 : if (pattern_stmt_info == NULL)
126 1343327 : pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
127 2523145 : gimple_set_bb (pattern_stmt, gimple_bb (orig_stmt_info->stmt));
128 :
129 2523145 : pattern_stmt_info->pattern_stmt_p = true;
130 2523145 : STMT_VINFO_RELATED_STMT (pattern_stmt_info) = orig_stmt_info;
131 2523145 : STMT_VINFO_DEF_TYPE (pattern_stmt_info)
132 2523145 : = STMT_VINFO_DEF_TYPE (orig_stmt_info);
133 2523145 : if (!STMT_VINFO_VECTYPE (pattern_stmt_info))
134 : {
135 2247005 : gcc_assert (!vectype
136 : || is_a <gcond *> (pattern_stmt)
137 : || (VECTOR_BOOLEAN_TYPE_P (vectype)
138 : == vect_use_mask_type_p (orig_stmt_info)));
139 1353980 : STMT_VINFO_VECTYPE (pattern_stmt_info) = vectype;
140 1353980 : pattern_stmt_info->mask_precision = orig_stmt_info->mask_precision;
141 : }
142 2523145 : return pattern_stmt_info;
143 : }
144 :
145 : /* Set the pattern statement of ORIG_STMT_INFO to PATTERN_STMT.
146 : Also set the vector type of PATTERN_STMT to VECTYPE, if it doesn't
147 : have one already. */
148 :
149 : static void
150 1083582 : vect_set_pattern_stmt (vec_info *vinfo, gimple *pattern_stmt,
151 : stmt_vec_info orig_stmt_info, tree vectype)
152 : {
153 1083582 : STMT_VINFO_IN_PATTERN_P (orig_stmt_info) = true;
154 1083582 : STMT_VINFO_RELATED_STMT (orig_stmt_info)
155 0 : = vect_init_pattern_stmt (vinfo, pattern_stmt, orig_stmt_info, vectype);
156 0 : }
157 :
158 : /* Add NEW_STMT to STMT_INFO's pattern definition statements. If VECTYPE
159 : is nonnull, record that NEW_STMT's vector type is VECTYPE, which might
160 : be different from the vector type of the final pattern statement.
161 : If VECTYPE is a mask type, SCALAR_TYPE_FOR_MASK is the scalar type
162 : from which it was derived. */
163 :
164 : static inline void
165 1398832 : append_pattern_def_seq (vec_info *vinfo,
166 : stmt_vec_info stmt_info, gimple *new_stmt,
167 : tree vectype = NULL_TREE,
168 : tree scalar_type_for_mask = NULL_TREE)
169 : {
170 2120940 : gcc_assert (!scalar_type_for_mask
171 : == (!vectype || !VECTOR_BOOLEAN_TYPE_P (vectype)));
172 1398832 : if (vectype)
173 : {
174 1169380 : stmt_vec_info new_stmt_info = vinfo->add_stmt (new_stmt);
175 1169380 : STMT_VINFO_VECTYPE (new_stmt_info) = vectype;
176 1169380 : if (scalar_type_for_mask)
177 676724 : new_stmt_info->mask_precision
178 1353448 : = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (scalar_type_for_mask));
179 : }
180 1398832 : gimple_seq_add_stmt_without_update (&STMT_VINFO_PATTERN_DEF_SEQ (stmt_info),
181 : new_stmt);
182 1398832 : }
183 :
184 :
185 : /* Add NEW_STMT to VINFO's invariant pattern definition statements. These
186 : statements are not vectorized but are materialized as scalar in the loop
187 : preheader. */
188 :
189 : static inline void
190 1391 : append_inv_pattern_def_seq (vec_info *vinfo, gimple *new_stmt)
191 : {
192 1391 : gimple_seq_add_stmt_without_update (&vinfo->inv_pattern_def_seq, new_stmt);
193 : }
194 :
195 : /* The caller wants to perform new operations on vect_external variable
196 : VAR, so that the result of the operations would also be vect_external.
197 : Return the edge on which the operations can be performed, if one exists.
198 : Return null if the operations should instead be treated as part of
199 : the pattern that needs them. */
200 :
201 : static edge
202 8657 : vect_get_external_def_edge (vec_info *vinfo, tree var)
203 : {
204 8657 : edge e = NULL;
205 8657 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
206 : {
207 819 : e = loop_preheader_edge (loop_vinfo->loop);
208 819 : if (!SSA_NAME_IS_DEFAULT_DEF (var))
209 : {
210 623 : basic_block bb = gimple_bb (SSA_NAME_DEF_STMT (var));
211 623 : if (bb == NULL
212 623 : || !dominated_by_p (CDI_DOMINATORS, e->dest, bb))
213 : e = NULL;
214 : }
215 : }
216 8657 : return e;
217 : }
218 :
219 : /* Return true if the target supports a vector version of CODE,
220 : where CODE is known to map to a direct optab with the given SUBTYPE.
221 : ITYPE specifies the type of (some of) the scalar inputs and OTYPE
222 : specifies the type of the scalar result.
223 :
224 : If CODE allows the inputs and outputs to have different type
225 : (such as for WIDEN_SUM_EXPR), it is the input mode rather
226 : than the output mode that determines the appropriate target pattern.
227 : Operand 0 of the target pattern then specifies the mode that the output
228 : must have.
229 :
230 : When returning true, set *VECOTYPE_OUT to the vector version of OTYPE.
231 : Also set *VECITYPE_OUT to the vector version of ITYPE if VECITYPE_OUT
232 : is nonnull. */
233 :
234 : static bool
235 817 : vect_supportable_direct_optab_p (vec_info *vinfo, tree otype, tree_code code,
236 : tree itype, tree *vecotype_out,
237 : tree *vecitype_out = NULL,
238 : enum optab_subtype subtype = optab_default)
239 : {
240 817 : tree vecitype = get_vectype_for_scalar_type (vinfo, itype);
241 817 : if (!vecitype)
242 : return false;
243 :
244 815 : tree vecotype = get_vectype_for_scalar_type (vinfo, otype);
245 815 : if (!vecotype)
246 : return false;
247 :
248 815 : optab optab = optab_for_tree_code (code, vecitype, subtype);
249 815 : if (!optab)
250 : return false;
251 :
252 815 : insn_code icode = optab_handler (optab, TYPE_MODE (vecitype));
253 815 : if (icode == CODE_FOR_nothing
254 815 : || insn_data[icode].operand[0].mode != TYPE_MODE (vecotype))
255 : return false;
256 :
257 421 : *vecotype_out = vecotype;
258 421 : if (vecitype_out)
259 412 : *vecitype_out = vecitype;
260 : return true;
261 : }
262 :
263 : /* Return true if the target supports a vector version of CODE,
264 : where CODE is known to map to a conversion optab with the given SUBTYPE.
265 : ITYPE specifies the type of (some of) the scalar inputs and OTYPE
266 : specifies the type of the scalar result.
267 :
268 : When returning true, set *VECOTYPE_OUT to the vector version of OTYPE.
269 : Also set *VECITYPE_OUT to the vector version of ITYPE if VECITYPE_OUT
270 : is nonnull. */
271 :
272 : static bool
273 5659 : vect_supportable_conv_optab_p (vec_info *vinfo, tree otype, tree_code code,
274 : tree itype, tree *vecotype_out,
275 : tree *vecitype_out = NULL,
276 : enum optab_subtype subtype = optab_default)
277 : {
278 5659 : tree vecitype = get_vectype_for_scalar_type (vinfo, itype);
279 5659 : tree vecotype = get_vectype_for_scalar_type (vinfo, otype);
280 5659 : if (!vecitype || !vecotype)
281 : return false;
282 :
283 5119 : if (!directly_supported_p (code, vecotype, vecitype, subtype))
284 : return false;
285 :
286 720 : *vecotype_out = vecotype;
287 720 : if (vecitype_out)
288 720 : *vecitype_out = vecitype;
289 : return true;
290 : }
291 :
292 : /* Round bit precision PRECISION up to a full element. */
293 :
294 : static unsigned int
295 3322075 : vect_element_precision (unsigned int precision)
296 : {
297 0 : precision = 1 << ceil_log2 (precision);
298 4978620 : return MAX (precision, BITS_PER_UNIT);
299 : }
300 :
301 : /* If OP is defined by a statement that's being considered for vectorization,
302 : return information about that statement, otherwise return NULL. */
303 :
304 : static stmt_vec_info
305 376534 : vect_get_internal_def (vec_info *vinfo, tree op)
306 : {
307 376534 : stmt_vec_info def_stmt_info = vinfo->lookup_def (op);
308 376534 : if (def_stmt_info
309 362782 : && STMT_VINFO_DEF_TYPE (def_stmt_info) == vect_internal_def)
310 344760 : return vect_stmt_to_vectorize (def_stmt_info);
311 : return NULL;
312 : }
313 :
314 : /* Holds information about an input operand after some sign changes
315 : and type promotions have been peeled away. */
316 : class vect_unpromoted_value {
317 : public:
318 : vect_unpromoted_value ();
319 :
320 : void set_op (tree, vect_def_type, stmt_vec_info = NULL);
321 :
322 : /* The value obtained after peeling away zero or more casts. */
323 : tree op;
324 :
325 : /* The type of OP. */
326 : tree type;
327 :
328 : /* The definition type of OP. */
329 : vect_def_type dt;
330 :
331 : /* If OP is the result of peeling at least one cast, and if the cast
332 : of OP itself is a vectorizable statement, CASTER identifies that
333 : statement, otherwise it is null. */
334 : stmt_vec_info caster;
335 : };
336 :
337 309866580 : inline vect_unpromoted_value::vect_unpromoted_value ()
338 309866580 : : op (NULL_TREE),
339 309866580 : type (NULL_TREE),
340 309866580 : dt (vect_uninitialized_def),
341 3299277 : caster (NULL)
342 : {
343 : }
344 :
345 : /* Set the operand to OP_IN, its definition type to DT_IN, and the
346 : statement that casts it to CASTER_IN. */
347 :
348 : inline void
349 11743384 : vect_unpromoted_value::set_op (tree op_in, vect_def_type dt_in,
350 : stmt_vec_info caster_in)
351 : {
352 11743384 : op = op_in;
353 11743384 : type = TREE_TYPE (op);
354 11743384 : dt = dt_in;
355 11743384 : caster = caster_in;
356 11743384 : }
357 :
358 : /* If OP is a vectorizable SSA name, strip a sequence of integer conversions
359 : to reach some vectorizable inner operand OP', continuing as long as it
360 : is possible to convert OP' back to OP using a possible sign change
361 : followed by a possible promotion P. Return this OP', or null if OP is
362 : not a vectorizable SSA name. If there is a promotion P, describe its
363 : input in UNPROM, otherwise describe OP' in UNPROM. If SINGLE_USE_P
364 : is nonnull, set *SINGLE_USE_P to false if any of the SSA names involved
365 : have more than one user.
366 :
367 : A successful return means that it is possible to go from OP' to OP
368 : via UNPROM. The cast from OP' to UNPROM is at most a sign change,
369 : whereas the cast from UNPROM to OP might be a promotion, a sign
370 : change, or a nop.
371 :
372 : E.g. say we have:
373 :
374 : signed short *ptr = ...;
375 : signed short C = *ptr;
376 : unsigned short B = (unsigned short) C; // sign change
377 : signed int A = (signed int) B; // unsigned promotion
378 : ...possible other uses of A...
379 : unsigned int OP = (unsigned int) A; // sign change
380 :
381 : In this case it's possible to go directly from C to OP using:
382 :
383 : OP = (unsigned int) (unsigned short) C;
384 : +------------+ +--------------+
385 : promotion sign change
386 :
387 : so OP' would be C. The input to the promotion is B, so UNPROM
388 : would describe B. */
389 :
390 : static tree
391 8667428 : vect_look_through_possible_promotion (vec_info *vinfo, tree op,
392 : vect_unpromoted_value *unprom,
393 : bool *single_use_p = NULL)
394 : {
395 8667428 : tree op_type = TREE_TYPE (op);
396 8667428 : if (!INTEGRAL_TYPE_P (op_type))
397 : return NULL_TREE;
398 :
399 8616215 : tree res = NULL_TREE;
400 8616215 : unsigned int orig_precision = TYPE_PRECISION (op_type);
401 8616215 : unsigned int min_precision = orig_precision;
402 8616215 : stmt_vec_info caster = NULL;
403 10327323 : while (TREE_CODE (op) == SSA_NAME && INTEGRAL_TYPE_P (op_type))
404 : {
405 : /* See whether OP is simple enough to vectorize. */
406 10094494 : stmt_vec_info def_stmt_info;
407 10094494 : gimple *def_stmt;
408 10094494 : vect_def_type dt;
409 10094494 : if (!vect_is_simple_use (op, vinfo, &dt, &def_stmt_info, &def_stmt))
410 : break;
411 :
412 : /* If OP is the input of a demotion, skip over it to see whether
413 : OP is itself the result of a promotion. If so, the combined
414 : effect of the promotion and the demotion might fit the required
415 : pattern, otherwise neither operation fits.
416 :
417 : This copes with cases such as the result of an arithmetic
418 : operation being truncated before being stored, and where that
419 : arithmetic operation has been recognized as an over-widened one. */
420 10077675 : if (TYPE_PRECISION (op_type) <= min_precision)
421 : {
422 : /* Use OP as the UNPROM described above if we haven't yet
423 : found a promotion, or if using the new input preserves the
424 : sign of the previous promotion. */
425 9943481 : if (!res
426 1456898 : || TYPE_PRECISION (unprom->type) == orig_precision
427 38399 : || TYPE_SIGN (unprom->type) == TYPE_SIGN (op_type)
428 9978985 : || (TYPE_UNSIGNED (op_type)
429 24892 : && TYPE_PRECISION (op_type) < TYPE_PRECISION (unprom->type)))
430 : {
431 9908413 : unprom->set_op (op, dt, caster);
432 9908413 : min_precision = TYPE_PRECISION (op_type);
433 : }
434 : /* Stop if we've already seen a promotion and if this
435 : conversion does more than change the sign. */
436 35068 : else if (TYPE_PRECISION (op_type)
437 35068 : != TYPE_PRECISION (unprom->type))
438 : break;
439 :
440 : /* The sequence now extends to OP. */
441 : res = op;
442 : }
443 :
444 : /* See whether OP is defined by a cast. Record it as CASTER if
445 : the cast is potentially vectorizable. */
446 10077627 : if (!def_stmt)
447 : break;
448 9864178 : caster = def_stmt_info;
449 :
450 : /* Ignore pattern statements, since we don't link uses for them. */
451 9864178 : if (caster
452 9864178 : && single_use_p
453 2004405 : && !STMT_VINFO_RELATED_STMT (caster)
454 11721061 : && !has_single_use (res))
455 1108560 : *single_use_p = false;
456 :
457 14635028 : gassign *assign = dyn_cast <gassign *> (def_stmt);
458 6251642 : if (!assign || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def_stmt)))
459 : break;
460 :
461 : /* Continue with the input to the cast. */
462 1711108 : op = gimple_assign_rhs1 (def_stmt);
463 1711108 : op_type = TREE_TYPE (op);
464 : }
465 : return res;
466 : }
467 :
468 : /* OP is an integer operand to an operation that returns TYPE, and we
469 : want to treat the operation as a widening one. So far we can treat
470 : it as widening from *COMMON_TYPE.
471 :
472 : Return true if OP is suitable for such a widening operation,
473 : either widening from *COMMON_TYPE or from some supertype of it.
474 : Update *COMMON_TYPE to the supertype in the latter case.
475 :
476 : SHIFT_P is true if OP is a shift amount. */
477 :
478 : static bool
479 324330 : vect_joust_widened_integer (tree type, bool shift_p, tree op,
480 : tree *common_type)
481 : {
482 : /* Calculate the minimum precision required by OP, without changing
483 : the sign of either operand. */
484 324330 : unsigned int precision;
485 324330 : if (shift_p)
486 : {
487 13389 : if (!wi::leu_p (wi::to_widest (op), TYPE_PRECISION (type) / 2))
488 : return false;
489 10798 : precision = TREE_INT_CST_LOW (op);
490 : }
491 : else
492 : {
493 310941 : precision = wi::min_precision (wi::to_widest (op),
494 310941 : TYPE_SIGN (*common_type));
495 310941 : if (precision * 2 > TYPE_PRECISION (type))
496 : return false;
497 : }
498 :
499 : /* If OP requires a wider type, switch to that type. The checks
500 : above ensure that this is still narrower than the result. */
501 306888 : precision = vect_element_precision (precision);
502 306888 : if (TYPE_PRECISION (*common_type) < precision)
503 6558 : *common_type = build_nonstandard_integer_type
504 6558 : (precision, TYPE_UNSIGNED (*common_type));
505 : return true;
506 : }
507 :
508 : /* Return true if the common supertype of NEW_TYPE and *COMMON_TYPE
509 : is narrower than type, storing the supertype in *COMMON_TYPE if so. */
510 :
511 : static bool
512 48599 : vect_joust_widened_type (tree type, tree new_type, tree *common_type)
513 : {
514 48599 : if (types_compatible_p (*common_type, new_type))
515 : return true;
516 :
517 : /* See if *COMMON_TYPE can hold all values of NEW_TYPE. */
518 4002 : if ((TYPE_PRECISION (new_type) < TYPE_PRECISION (*common_type))
519 4002 : && (TYPE_UNSIGNED (new_type) || !TYPE_UNSIGNED (*common_type)))
520 : return true;
521 :
522 : /* See if NEW_TYPE can hold all values of *COMMON_TYPE. */
523 3230 : if (TYPE_PRECISION (*common_type) < TYPE_PRECISION (new_type)
524 3230 : && (TYPE_UNSIGNED (*common_type) || !TYPE_UNSIGNED (new_type)))
525 : {
526 376 : *common_type = new_type;
527 376 : return true;
528 : }
529 :
530 : /* We have mismatched signs, with the signed type being
531 : no wider than the unsigned type. In this case we need
532 : a wider signed type. */
533 2854 : unsigned int precision = MAX (TYPE_PRECISION (*common_type),
534 : TYPE_PRECISION (new_type));
535 2854 : precision *= 2;
536 :
537 2854 : if (precision * 2 > TYPE_PRECISION (type))
538 : return false;
539 :
540 40 : *common_type = build_nonstandard_integer_type (precision, false);
541 40 : return true;
542 : }
543 :
544 : /* Check whether STMT_INFO can be viewed as a tree of integer operations
545 : in which each node either performs CODE or WIDENED_CODE, and where
546 : each leaf operand is narrower than the result of STMT_INFO. MAX_NOPS
547 : specifies the maximum number of leaf operands. SHIFT_P says whether
548 : CODE and WIDENED_CODE are some sort of shift.
549 :
550 : If STMT_INFO is such a tree, return the number of leaf operands
551 : and describe them in UNPROM[0] onwards. Also set *COMMON_TYPE
552 : to a type that (a) is narrower than the result of STMT_INFO and
553 : (b) can hold all leaf operand values.
554 :
555 : If SUBTYPE then allow that the signs of the operands
556 : may differ in signs but not in precision. SUBTYPE is updated to reflect
557 : this.
558 :
559 : Return 0 if STMT_INFO isn't such a tree, or if no such COMMON_TYPE
560 : exists. */
561 :
562 : static unsigned int
563 130334545 : vect_widened_op_tree (vec_info *vinfo, stmt_vec_info stmt_info, tree_code code,
564 : code_helper widened_code, bool shift_p,
565 : unsigned int max_nops,
566 : vect_unpromoted_value *unprom, tree *common_type,
567 : enum optab_subtype *subtype = NULL)
568 : {
569 : /* Check for an integer operation with the right code. */
570 130334545 : gimple* stmt = stmt_info->stmt;
571 130334545 : if (!(is_gimple_assign (stmt) || is_gimple_call (stmt)))
572 : return 0;
573 :
574 106988453 : code_helper rhs_code;
575 106988453 : if (is_gimple_assign (stmt))
576 91628071 : rhs_code = gimple_assign_rhs_code (stmt);
577 15360382 : else if (is_gimple_call (stmt))
578 15360382 : rhs_code = gimple_call_combined_fn (stmt);
579 : else
580 : return 0;
581 :
582 106988453 : if (rhs_code != code
583 106988453 : && rhs_code != widened_code)
584 : return 0;
585 :
586 6780043 : tree lhs = gimple_get_lhs (stmt);
587 6780043 : tree type = TREE_TYPE (lhs);
588 6780043 : if (!INTEGRAL_TYPE_P (type))
589 : return 0;
590 :
591 : /* Assume that both operands will be leaf operands. */
592 5899588 : max_nops -= 2;
593 :
594 : /* Check the operands. */
595 5899588 : unsigned int next_op = 0;
596 6705179 : for (unsigned int i = 0; i < 2; ++i)
597 : {
598 6352083 : vect_unpromoted_value *this_unprom = &unprom[next_op];
599 6352083 : unsigned int nops = 1;
600 6352083 : tree op = gimple_arg (stmt, i);
601 6352083 : if (i == 1 && TREE_CODE (op) == INTEGER_CST)
602 : {
603 : /* We already have a common type from earlier operands.
604 : Update it to account for OP. */
605 324330 : this_unprom->set_op (op, vect_constant_def);
606 324330 : if (!vect_joust_widened_integer (type, shift_p, op, common_type))
607 : return 0;
608 : }
609 : else
610 : {
611 : /* Only allow shifts by constants. */
612 6027753 : if (shift_p && i == 1)
613 : return 0;
614 :
615 6021661 : if (rhs_code != code)
616 : {
617 : /* If rhs_code is widened_code, don't look through further
618 : possible promotions, there is a promotion already embedded
619 : in the WIDEN_*_EXPR. */
620 1710 : if (TREE_CODE (op) != SSA_NAME
621 1710 : || !INTEGRAL_TYPE_P (TREE_TYPE (op)))
622 0 : return 0;
623 :
624 1710 : stmt_vec_info def_stmt_info;
625 1710 : gimple *def_stmt;
626 1710 : vect_def_type dt;
627 1710 : if (!vect_is_simple_use (op, vinfo, &dt, &def_stmt_info,
628 : &def_stmt))
629 : return 0;
630 1710 : this_unprom->set_op (op, dt, NULL);
631 : }
632 6019951 : else if (!vect_look_through_possible_promotion (vinfo, op,
633 : this_unprom))
634 : return 0;
635 :
636 5896356 : if (TYPE_PRECISION (this_unprom->type) == TYPE_PRECISION (type))
637 : {
638 : /* The operand isn't widened. If STMT_INFO has the code
639 : for an unwidened operation, recursively check whether
640 : this operand is a node of the tree. */
641 5391072 : if (rhs_code != code
642 5391072 : || max_nops == 0
643 5391541 : || this_unprom->dt != vect_internal_def)
644 : return 0;
645 :
646 : /* Give back the leaf slot allocated above now that we're
647 : not treating this as a leaf operand. */
648 469 : max_nops += 1;
649 :
650 : /* Recursively process the definition of the operand. */
651 469 : stmt_vec_info def_stmt_info
652 469 : = vect_get_internal_def (vinfo, this_unprom->op);
653 :
654 469 : nops = vect_widened_op_tree (vinfo, def_stmt_info, code,
655 : widened_code, shift_p, max_nops,
656 : this_unprom, common_type,
657 : subtype);
658 469 : if (nops == 0)
659 : return 0;
660 :
661 311 : max_nops -= nops;
662 : }
663 : else
664 : {
665 : /* Make sure that the operand is narrower than the result. */
666 505284 : if (TYPE_PRECISION (this_unprom->type) * 2
667 505284 : > TYPE_PRECISION (type))
668 : return 0;
669 :
670 : /* Update COMMON_TYPE for the new operand. */
671 500921 : if (i == 0)
672 452322 : *common_type = this_unprom->type;
673 48599 : else if (!vect_joust_widened_type (type, this_unprom->type,
674 : common_type))
675 : {
676 2814 : if (subtype)
677 : {
678 : /* See if we can sign extend the smaller type. */
679 285 : if (TYPE_PRECISION (this_unprom->type)
680 285 : > TYPE_PRECISION (*common_type))
681 27 : *common_type = this_unprom->type;
682 285 : *subtype = optab_vector_mixed_sign;
683 : }
684 : else
685 : return 0;
686 : }
687 : }
688 : }
689 805591 : next_op += nops;
690 : }
691 : return next_op;
692 : }
693 :
694 : /* Helper to return a new temporary for pattern of TYPE for STMT. If STMT
695 : is NULL, the caller must set SSA_NAME_DEF_STMT for the returned SSA var. */
696 :
697 : static tree
698 2143527 : vect_recog_temp_ssa_var (tree type, gimple *stmt = NULL)
699 : {
700 0 : return make_temp_ssa_name (type, stmt, "patt");
701 : }
702 :
703 : /* STMT2_INFO describes a type conversion that could be split into STMT1
704 : followed by a version of STMT2_INFO that takes NEW_RHS as its first
705 : input. Try to do this using pattern statements, returning true on
706 : success. */
707 :
708 : static bool
709 30792 : vect_split_statement (vec_info *vinfo, stmt_vec_info stmt2_info, tree new_rhs,
710 : gimple *stmt1, tree vectype)
711 : {
712 30792 : if (is_pattern_stmt_p (stmt2_info))
713 : {
714 : /* STMT2_INFO is part of a pattern. Get the statement to which
715 : the pattern is attached. */
716 454 : stmt_vec_info orig_stmt2_info = STMT_VINFO_RELATED_STMT (stmt2_info);
717 454 : vect_init_pattern_stmt (vinfo, stmt1, orig_stmt2_info, vectype);
718 :
719 454 : if (dump_enabled_p ())
720 19 : dump_printf_loc (MSG_NOTE, vect_location,
721 : "Splitting pattern statement: %G", stmt2_info->stmt);
722 :
723 : /* Since STMT2_INFO is a pattern statement, we can change it
724 : in-situ without worrying about changing the code for the
725 : containing block. */
726 454 : gimple_assign_set_rhs1 (stmt2_info->stmt, new_rhs);
727 :
728 454 : if (dump_enabled_p ())
729 : {
730 19 : dump_printf_loc (MSG_NOTE, vect_location, "into: %G", stmt1);
731 19 : dump_printf_loc (MSG_NOTE, vect_location, "and: %G",
732 : stmt2_info->stmt);
733 : }
734 :
735 454 : gimple_seq *def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt2_info);
736 454 : if (STMT_VINFO_RELATED_STMT (orig_stmt2_info) == stmt2_info)
737 : /* STMT2_INFO is the actual pattern statement. Add STMT1
738 : to the end of the definition sequence. */
739 451 : gimple_seq_add_stmt_without_update (def_seq, stmt1);
740 : else
741 : {
742 : /* STMT2_INFO belongs to the definition sequence. Insert STMT1
743 : before it. */
744 3 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt2_info->stmt, def_seq);
745 3 : gsi_insert_before_without_update (&gsi, stmt1, GSI_SAME_STMT);
746 : }
747 : return true;
748 : }
749 : else
750 : {
751 : /* STMT2_INFO doesn't yet have a pattern. Try to create a
752 : two-statement pattern now. */
753 30338 : gcc_assert (!STMT_VINFO_RELATED_STMT (stmt2_info));
754 30338 : tree lhs_type = TREE_TYPE (gimple_get_lhs (stmt2_info->stmt));
755 30338 : tree lhs_vectype = get_vectype_for_scalar_type (vinfo, lhs_type);
756 30338 : if (!lhs_vectype)
757 : return false;
758 :
759 30338 : if (dump_enabled_p ())
760 1937 : dump_printf_loc (MSG_NOTE, vect_location,
761 : "Splitting statement: %G", stmt2_info->stmt);
762 :
763 : /* Add STMT1 as a singleton pattern definition sequence. */
764 30338 : gimple_seq *def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (stmt2_info);
765 30338 : vect_init_pattern_stmt (vinfo, stmt1, stmt2_info, vectype);
766 30338 : gimple_seq_add_stmt_without_update (def_seq, stmt1);
767 :
768 : /* Build the second of the two pattern statements. */
769 30338 : tree new_lhs = vect_recog_temp_ssa_var (lhs_type, NULL);
770 30338 : gassign *new_stmt2 = gimple_build_assign (new_lhs, NOP_EXPR, new_rhs);
771 30338 : vect_set_pattern_stmt (vinfo, new_stmt2, stmt2_info, lhs_vectype);
772 :
773 30338 : if (dump_enabled_p ())
774 : {
775 1937 : dump_printf_loc (MSG_NOTE, vect_location,
776 : "into pattern statements: %G", stmt1);
777 1937 : dump_printf_loc (MSG_NOTE, vect_location, "and: %G",
778 : (gimple *) new_stmt2);
779 : }
780 :
781 : return true;
782 : }
783 : }
784 :
785 : /* Look for the following pattern
786 : X = x[i]
787 : Y = y[i]
788 : DIFF = X - Y
789 : DAD = ABS_EXPR<DIFF>
790 :
791 : ABS_STMT should point to a statement of code ABS_EXPR or ABSU_EXPR.
792 : HALF_TYPE and UNPROM will be set should the statement be found to
793 : be a widened operation.
794 : DIFF_STMT will be set to the MINUS_EXPR
795 : statement that precedes the ABS_STMT if it is a MINUS_EXPR..
796 : */
797 : static bool
798 22525876 : vect_recog_absolute_difference (vec_info *vinfo, gassign *abs_stmt,
799 : tree *half_type,
800 : vect_unpromoted_value unprom[2],
801 : gassign **diff_stmt)
802 : {
803 22525876 : if (!abs_stmt)
804 : return false;
805 :
806 : /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
807 : inside the loop (in case we are analyzing an outer-loop). */
808 22525876 : enum tree_code code = gimple_assign_rhs_code (abs_stmt);
809 22525876 : if (code != ABS_EXPR && code != ABSU_EXPR)
810 : return false;
811 :
812 34089 : tree abs_oprnd = gimple_assign_rhs1 (abs_stmt);
813 34089 : tree abs_type = TREE_TYPE (abs_oprnd);
814 34089 : if (!abs_oprnd)
815 : return false;
816 26710 : if (!ANY_INTEGRAL_TYPE_P (abs_type)
817 7633 : || TYPE_OVERFLOW_WRAPS (abs_type)
818 41553 : || TYPE_UNSIGNED (abs_type))
819 : return false;
820 :
821 : /* Peel off conversions from the ABS input. This can involve sign
822 : changes (e.g. from an unsigned subtraction to a signed ABS input)
823 : or signed promotion, but it can't include unsigned promotion.
824 : (Note that ABS of an unsigned promotion should have been folded
825 : away before now anyway.) */
826 7464 : vect_unpromoted_value unprom_diff;
827 7464 : abs_oprnd = vect_look_through_possible_promotion (vinfo, abs_oprnd,
828 : &unprom_diff);
829 7464 : if (!abs_oprnd)
830 : return false;
831 7210 : if (TYPE_PRECISION (unprom_diff.type) != TYPE_PRECISION (abs_type)
832 7210 : && TYPE_UNSIGNED (unprom_diff.type))
833 : return false;
834 :
835 : /* We then detect if the operand of abs_expr is defined by a minus_expr. */
836 7210 : stmt_vec_info diff_stmt_vinfo = vect_get_internal_def (vinfo, abs_oprnd);
837 7210 : if (!diff_stmt_vinfo)
838 : return false;
839 :
840 7002 : gassign *diff = dyn_cast <gassign *> (STMT_VINFO_STMT (diff_stmt_vinfo));
841 7002 : if (diff_stmt && diff
842 5803 : && gimple_assign_rhs_code (diff) == MINUS_EXPR
843 9008 : && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (abs_oprnd)))
844 278 : *diff_stmt = diff;
845 :
846 : /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
847 : inside the loop (in case we are analyzing an outer-loop). */
848 7002 : if (vect_widened_op_tree (vinfo, diff_stmt_vinfo,
849 7002 : MINUS_EXPR, IFN_VEC_WIDEN_MINUS,
850 : false, 2, unprom, half_type))
851 2324 : return true;
852 :
853 : return false;
854 : }
855 :
856 : /* Convert UNPROM to TYPE and return the result, adding new statements
857 : to STMT_INFO's pattern definition statements if no better way is
858 : available. VECTYPE is the vector form of TYPE.
859 :
860 : If SUBTYPE then convert the type based on the subtype. */
861 :
862 : static tree
863 495717 : vect_convert_input (vec_info *vinfo, stmt_vec_info stmt_info, tree type,
864 : vect_unpromoted_value *unprom, tree vectype,
865 : enum optab_subtype subtype = optab_default)
866 : {
867 : /* Update the type if the signs differ. */
868 495717 : if (subtype == optab_vector_mixed_sign)
869 : {
870 322 : gcc_assert (!TYPE_UNSIGNED (type));
871 322 : if (TYPE_UNSIGNED (TREE_TYPE (unprom->op)))
872 : {
873 161 : type = unsigned_type_for (type);
874 161 : vectype = unsigned_type_for (vectype);
875 : }
876 : }
877 :
878 : /* Check for a no-op conversion. */
879 495717 : if (types_compatible_p (type, TREE_TYPE (unprom->op)))
880 171460 : return unprom->op;
881 :
882 : /* Allow the caller to create constant vect_unpromoted_values. */
883 324257 : if (TREE_CODE (unprom->op) == INTEGER_CST)
884 200471 : return wide_int_to_tree (type, wi::to_widest (unprom->op));
885 :
886 123786 : tree input = unprom->op;
887 123786 : if (unprom->caster)
888 : {
889 65775 : tree lhs = gimple_get_lhs (unprom->caster->stmt);
890 65775 : tree lhs_type = TREE_TYPE (lhs);
891 :
892 : /* If the result of the existing cast is the right width, use it
893 : instead of the source of the cast. */
894 65775 : if (TYPE_PRECISION (lhs_type) == TYPE_PRECISION (type))
895 : input = lhs;
896 : /* If the precision we want is between the source and result
897 : precisions of the existing cast, try splitting the cast into
898 : two and tapping into a mid-way point. */
899 63672 : else if (TYPE_PRECISION (lhs_type) > TYPE_PRECISION (type)
900 63672 : && TYPE_PRECISION (type) > TYPE_PRECISION (unprom->type))
901 : {
902 : /* In order to preserve the semantics of the original cast,
903 : give the mid-way point the same signedness as the input value.
904 :
905 : It would be possible to use a signed type here instead if
906 : TYPE is signed and UNPROM->TYPE is unsigned, but that would
907 : make the sign of the midtype sensitive to the order in
908 : which we process the statements, since the signedness of
909 : TYPE is the signedness required by just one of possibly
910 : many users. Also, unsigned promotions are usually as cheap
911 : as or cheaper than signed ones, so it's better to keep an
912 : unsigned promotion. */
913 30792 : tree midtype = build_nonstandard_integer_type
914 30792 : (TYPE_PRECISION (type), TYPE_UNSIGNED (unprom->type));
915 30792 : tree vec_midtype = get_vectype_for_scalar_type (vinfo, midtype);
916 30792 : if (vec_midtype)
917 : {
918 30792 : input = vect_recog_temp_ssa_var (midtype, NULL);
919 30792 : gassign *new_stmt = gimple_build_assign (input, NOP_EXPR,
920 : unprom->op);
921 30792 : if (!vect_split_statement (vinfo, unprom->caster, input, new_stmt,
922 : vec_midtype))
923 0 : append_pattern_def_seq (vinfo, stmt_info,
924 : new_stmt, vec_midtype);
925 : }
926 : }
927 :
928 : /* See if we can reuse an existing result. */
929 65775 : if (types_compatible_p (type, TREE_TYPE (input)))
930 : return input;
931 : }
932 :
933 : /* We need a new conversion statement. */
934 101176 : tree new_op = vect_recog_temp_ssa_var (type, NULL);
935 101176 : gassign *new_stmt = gimple_build_assign (new_op, NOP_EXPR, input);
936 :
937 : /* If OP is an external value, see if we can insert the new statement
938 : on an incoming edge. */
939 101176 : if (input == unprom->op && unprom->dt == vect_external_def)
940 8642 : if (edge e = vect_get_external_def_edge (vinfo, input))
941 : {
942 804 : basic_block new_bb = gsi_insert_on_edge_immediate (e, new_stmt);
943 804 : gcc_assert (!new_bb);
944 : return new_op;
945 : }
946 :
947 : /* As a (common) last resort, add the statement to the pattern itself. */
948 100372 : append_pattern_def_seq (vinfo, stmt_info, new_stmt, vectype);
949 100372 : return new_op;
950 : }
951 :
952 : /* Invoke vect_convert_input for N elements of UNPROM and store the
953 : result in the corresponding elements of RESULT.
954 :
955 : If SUBTYPE then convert the type based on the subtype. */
956 :
957 : static void
958 251818 : vect_convert_inputs (vec_info *vinfo, stmt_vec_info stmt_info, unsigned int n,
959 : tree *result, tree type, vect_unpromoted_value *unprom,
960 : tree vectype, enum optab_subtype subtype = optab_default)
961 : {
962 747514 : for (unsigned int i = 0; i < n; ++i)
963 : {
964 : unsigned int j;
965 739095 : for (j = 0; j < i; ++j)
966 243878 : if (unprom[j].op == unprom[i].op)
967 : break;
968 :
969 495696 : if (j < i)
970 479 : result[i] = result[j];
971 : else
972 495217 : result[i] = vect_convert_input (vinfo, stmt_info,
973 495217 : type, &unprom[i], vectype, subtype);
974 : }
975 251818 : }
976 :
977 : /* The caller has created a (possibly empty) sequence of pattern definition
978 : statements followed by a single statement PATTERN_STMT. Cast the result
979 : of this final statement to TYPE. If a new statement is needed, add
980 : PATTERN_STMT to the end of STMT_INFO's pattern definition statements
981 : and return the new statement, otherwise return PATTERN_STMT as-is.
982 : VECITYPE is the vector form of PATTERN_STMT's result type. */
983 :
984 : static gimple *
985 278380 : vect_convert_output (vec_info *vinfo, stmt_vec_info stmt_info, tree type,
986 : gimple *pattern_stmt, tree vecitype)
987 : {
988 278380 : tree lhs = gimple_get_lhs (pattern_stmt);
989 278380 : if (!types_compatible_p (type, TREE_TYPE (lhs)))
990 : {
991 247720 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vecitype);
992 247720 : tree cast_var = vect_recog_temp_ssa_var (type, NULL);
993 247720 : pattern_stmt = gimple_build_assign (cast_var, NOP_EXPR, lhs);
994 : }
995 278380 : return pattern_stmt;
996 : }
997 :
998 : /* Return true if STMT_VINFO describes a reduction for which reassociation
999 : is allowed. If STMT_INFO is part of a group, assume that it's part of
1000 : a reduction chain and optimistically assume that all statements
1001 : except the last allow reassociation.
1002 : Also require it to have code CODE and to be a reduction
1003 : in the outermost loop. When returning true, store the operands in
1004 : *OP0_OUT and *OP1_OUT. */
1005 :
1006 : static bool
1007 97233276 : vect_reassociating_reduction_p (vec_info *vinfo,
1008 : stmt_vec_info stmt_info, tree_code code,
1009 : tree *op0_out, tree *op1_out)
1010 : {
1011 97233276 : loop_vec_info loop_info = dyn_cast <loop_vec_info> (vinfo);
1012 13693005 : if (!loop_info)
1013 : return false;
1014 :
1015 13693005 : gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
1016 14950548 : if (!assign || gimple_assign_rhs_code (assign) != code)
1017 : return false;
1018 :
1019 : /* We don't allow changing the order of the computation in the inner-loop
1020 : when doing outer-loop vectorization. */
1021 2668226 : class loop *loop = LOOP_VINFO_LOOP (loop_info);
1022 2668226 : if (loop && nested_in_vect_loop_p (loop, stmt_info))
1023 : return false;
1024 :
1025 2614760 : if (!vect_is_reduction (stmt_info))
1026 : return false;
1027 :
1028 171482 : if (needs_fold_left_reduction_p (TREE_TYPE (gimple_assign_lhs (assign)),
1029 171482 : code))
1030 : return false;
1031 :
1032 159617 : *op0_out = gimple_assign_rhs1 (assign);
1033 159617 : *op1_out = gimple_assign_rhs2 (assign);
1034 159617 : if (commutative_tree_code (code) && STMT_VINFO_REDUC_IDX (stmt_info) == 0)
1035 63780 : std::swap (*op0_out, *op1_out);
1036 : return true;
1037 : }
1038 :
1039 : /* Return true iff the target has a vector optab implementing the operation
1040 : CODE on type VECTYPE with SUBTYPE. */
1041 :
1042 : static bool
1043 871321 : target_has_vecop_for_code (tree_code code, tree vectype,
1044 : enum optab_subtype subtype = optab_vector)
1045 : {
1046 871321 : optab voptab = optab_for_tree_code (code, vectype, subtype);
1047 871321 : return voptab && can_implement_p (voptab, TYPE_MODE (vectype));
1048 : }
1049 :
1050 : /* match.pd function to match
1051 : (cond (cmp@3 a b) (convert@1 c) (convert@2 d))
1052 : with conditions:
1053 : 1) @1, @2, c, d, a, b are all integral type.
1054 : 2) There's single_use for both @1 and @2.
1055 : 3) a, c have same precision.
1056 : 4) c and @1 have different precision.
1057 : 5) c, d are the same type or they can differ in sign when convert is
1058 : truncation.
1059 :
1060 : record a and c and d and @3. */
1061 :
1062 : extern bool gimple_cond_expr_convert_p (tree, tree*, tree (*)(tree));
1063 :
1064 : /* Function vect_recog_cond_expr_convert
1065 :
1066 : Try to find the following pattern:
1067 :
1068 : TYPE_AB A,B;
1069 : TYPE_CD C,D;
1070 : TYPE_E E;
1071 : TYPE_E op_true = (TYPE_E) A;
1072 : TYPE_E op_false = (TYPE_E) B;
1073 :
1074 : E = C cmp D ? op_true : op_false;
1075 :
1076 : where
1077 : TYPE_PRECISION (TYPE_E) != TYPE_PRECISION (TYPE_CD);
1078 : TYPE_PRECISION (TYPE_AB) == TYPE_PRECISION (TYPE_CD);
1079 : single_use of op_true and op_false.
1080 : TYPE_AB could differ in sign when (TYPE_E) A is a truncation.
1081 :
1082 : Input:
1083 :
1084 : * STMT_VINFO: The stmt from which the pattern search begins.
1085 : here it starts with E = c cmp D ? op_true : op_false;
1086 :
1087 : Output:
1088 :
1089 : TYPE1 E' = C cmp D ? A : B;
1090 : TYPE3 E = (TYPE3) E';
1091 :
1092 : There may extra nop_convert for A or B to handle different signness.
1093 :
1094 : * TYPE_OUT: The vector type of the output of this pattern.
1095 :
1096 : * Return value: A new stmt that will be used to replace the sequence of
1097 : stmts that constitute the pattern. In this case it will be:
1098 : E = (TYPE3)E';
1099 : E' = C cmp D ? A : B; is recorded in pattern definition statements; */
1100 :
1101 : static gimple *
1102 32488896 : vect_recog_cond_expr_convert_pattern (vec_info *vinfo,
1103 : stmt_vec_info stmt_vinfo, tree *type_out)
1104 : {
1105 32488896 : gassign *last_stmt = dyn_cast <gassign *> (stmt_vinfo->stmt);
1106 22620496 : tree lhs, match[4], temp, type, new_lhs, op2, op1;
1107 22620496 : gimple *cond_stmt;
1108 22620496 : gimple *pattern_stmt;
1109 55109392 : enum tree_code code = NOP_EXPR;
1110 :
1111 22620496 : if (!last_stmt)
1112 : return NULL;
1113 :
1114 22620496 : lhs = gimple_assign_lhs (last_stmt);
1115 :
1116 : /* Find E = C cmp D ? (TYPE3) A ? (TYPE3) B;
1117 : TYPE_PRECISION (A) == TYPE_PRECISION (C). */
1118 22620496 : if (!gimple_cond_expr_convert_p (lhs, &match[0], NULL))
1119 : return NULL;
1120 :
1121 29 : if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (lhs)))
1122 20 : code = INTEGRAL_TYPE_P (TREE_TYPE (match[1])) ? FLOAT_EXPR : CONVERT_EXPR;
1123 9 : else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (match[1])))
1124 0 : code = FIX_TRUNC_EXPR;
1125 :
1126 29 : op1 = match[1];
1127 29 : op2 = match[2];
1128 29 : type = TREE_TYPE (op1);
1129 : /* When op1/op2 is REAL_CST, the conversion must be CONVERT_EXPR from
1130 : SCALAR_FLOAT_TYPE_P which is restricted in gimple_cond_expr_convert_p.
1131 : Otherwise, the conversion could be FLOAT_EXPR, FIX_TRUNC_EXPR
1132 : or CONVERT_EXPR. */
1133 29 : if (TREE_CODE (op1) == REAL_CST)
1134 : {
1135 20 : op1 = const_unop (CONVERT_EXPR, TREE_TYPE (op2), op1);
1136 20 : type = TREE_TYPE (op2);
1137 20 : if (op1 == NULL_TREE)
1138 : return NULL;
1139 : }
1140 9 : else if (TREE_CODE (op2) == REAL_CST)
1141 : {
1142 0 : op2 = const_unop (FLOAT_EXPR, TREE_TYPE (op1), op2);
1143 0 : if (op2 == NULL_TREE)
1144 : return NULL;
1145 : }
1146 9 : else if (code == NOP_EXPR)
1147 : {
1148 9 : if (TYPE_SIGN (type) != TYPE_SIGN (TREE_TYPE (match[2])))
1149 : {
1150 9 : op2 = vect_recog_temp_ssa_var (type, NULL);
1151 9 : gimple* nop_stmt = gimple_build_assign (op2, NOP_EXPR, match[2]);
1152 9 : append_pattern_def_seq (vinfo, stmt_vinfo, nop_stmt);
1153 : }
1154 : }
1155 :
1156 29 : vect_pattern_detected ("vect_recog_cond_expr_convert_pattern", last_stmt);
1157 :
1158 29 : temp = vect_recog_temp_ssa_var (type, NULL);
1159 29 : cond_stmt = gimple_build_assign (temp, build3 (COND_EXPR, type, match[3],
1160 : op1, op2));
1161 29 : append_pattern_def_seq (vinfo, stmt_vinfo, cond_stmt);
1162 29 : new_lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
1163 29 : pattern_stmt = gimple_build_assign (new_lhs, code, temp);
1164 29 : *type_out = NULL_TREE;
1165 :
1166 29 : if (dump_enabled_p ())
1167 20 : dump_printf_loc (MSG_NOTE, vect_location,
1168 : "created pattern stmt: %G", pattern_stmt);
1169 : return pattern_stmt;
1170 : }
1171 :
1172 : /* Function vect_recog_dot_prod_pattern
1173 :
1174 : Try to find the following pattern:
1175 :
1176 : type1a x_t
1177 : type1b y_t;
1178 : TYPE1 prod;
1179 : TYPE2 sum = init;
1180 : loop:
1181 : sum_0 = phi <init, sum_1>
1182 : S1 x_t = ...
1183 : S2 y_t = ...
1184 : S3 x_T = (TYPE1) x_t;
1185 : S4 y_T = (TYPE1) y_t;
1186 : S5 prod = x_T * y_T;
1187 : [S6 prod = (TYPE2) prod; #optional]
1188 : S7 sum_1 = prod + sum_0;
1189 :
1190 : where 'TYPE1' is exactly double the size of type 'type1a' and 'type1b',
1191 : the sign of 'TYPE1' must be one of 'type1a' or 'type1b' but the sign of
1192 : 'type1a' and 'type1b' can differ.
1193 :
1194 : Input:
1195 :
1196 : * STMT_VINFO: The stmt from which the pattern search begins. In the
1197 : example, when this function is called with S7, the pattern {S3,S4,S5,S6,S7}
1198 : will be detected.
1199 :
1200 : Output:
1201 :
1202 : * TYPE_OUT: The type of the output of this pattern.
1203 :
1204 : * Return value: A new stmt that will be used to replace the sequence of
1205 : stmts that constitute the pattern. In this case it will be:
1206 : WIDEN_DOT_PRODUCT <x_t, y_t, sum_0>
1207 :
1208 : Note: The dot-prod idiom is a widening reduction pattern that is
1209 : vectorized without preserving all the intermediate results. It
1210 : produces only N/2 (widened) results (by summing up pairs of
1211 : intermediate results) rather than all N results. Therefore, we
1212 : cannot allow this pattern when we want to get all the results and in
1213 : the correct order (as is the case when this computation is in an
1214 : inner-loop nested in an outer-loop that us being vectorized). */
1215 :
1216 : static gimple *
1217 32411706 : vect_recog_dot_prod_pattern (vec_info *vinfo,
1218 : stmt_vec_info stmt_vinfo, tree *type_out)
1219 : {
1220 32411706 : tree oprnd0, oprnd1;
1221 32411706 : gimple *last_stmt = stmt_vinfo->stmt;
1222 32411706 : tree type, half_type;
1223 32411706 : gimple *pattern_stmt;
1224 32411706 : tree var;
1225 :
1226 : /* Look for the following pattern
1227 : DX = (TYPE1) X;
1228 : DY = (TYPE1) Y;
1229 : DPROD = DX * DY;
1230 : DDPROD = (TYPE2) DPROD;
1231 : sum_1 = DDPROD + sum_0;
1232 : In which
1233 : - DX is double the size of X
1234 : - DY is double the size of Y
1235 : - DX, DY, DPROD all have the same type but the sign
1236 : between X, Y and DPROD can differ.
1237 : - sum is the same size of DPROD or bigger
1238 : - sum has been recognized as a reduction variable.
1239 :
1240 : This is equivalent to:
1241 : DPROD = X w* Y; #widen mult
1242 : sum_1 = DPROD w+ sum_0; #widen summation
1243 : or
1244 : DPROD = X w* Y; #widen mult
1245 : sum_1 = DPROD + sum_0; #summation
1246 : */
1247 :
1248 : /* Starting from LAST_STMT, follow the defs of its uses in search
1249 : of the above pattern. */
1250 :
1251 32411706 : if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, PLUS_EXPR,
1252 : &oprnd0, &oprnd1))
1253 : return NULL;
1254 :
1255 53823 : type = TREE_TYPE (gimple_get_lhs (last_stmt));
1256 :
1257 53823 : vect_unpromoted_value unprom_mult;
1258 53823 : oprnd0 = vect_look_through_possible_promotion (vinfo, oprnd0, &unprom_mult);
1259 :
1260 : /* So far so good. Since last_stmt was detected as a (summation) reduction,
1261 : we know that oprnd1 is the reduction variable (defined by a loop-header
1262 : phi), and oprnd0 is an ssa-name defined by a stmt in the loop body.
1263 : Left to check that oprnd0 is defined by a (widen_)mult_expr */
1264 53823 : if (!oprnd0)
1265 : return NULL;
1266 :
1267 36540 : stmt_vec_info mult_vinfo = vect_get_internal_def (vinfo, oprnd0);
1268 36540 : if (!mult_vinfo)
1269 : return NULL;
1270 :
1271 : /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
1272 : inside the loop (in case we are analyzing an outer-loop). */
1273 107085 : vect_unpromoted_value unprom0[2];
1274 35695 : enum optab_subtype subtype = optab_vector;
1275 35695 : if (!vect_widened_op_tree (vinfo, mult_vinfo, MULT_EXPR, WIDEN_MULT_EXPR,
1276 : false, 2, unprom0, &half_type, &subtype))
1277 : return NULL;
1278 :
1279 : /* If there are two widening operations, make sure they agree on the sign
1280 : of the extension. The result of an optab_vector_mixed_sign operation
1281 : is signed; otherwise, the result has the same sign as the operands. */
1282 1371 : if (TYPE_PRECISION (unprom_mult.type) != TYPE_PRECISION (type)
1283 2100 : && (subtype == optab_vector_mixed_sign
1284 729 : ? TYPE_UNSIGNED (unprom_mult.type)
1285 526 : : TYPE_SIGN (unprom_mult.type) != TYPE_SIGN (half_type)))
1286 : return NULL;
1287 :
1288 1290 : vect_pattern_detected ("vect_recog_dot_prod_pattern", last_stmt);
1289 :
1290 : /* If the inputs have mixed signs, canonicalize on using the signed
1291 : input type for analysis. This also helps when emulating mixed-sign
1292 : operations using signed operations. */
1293 1290 : if (subtype == optab_vector_mixed_sign)
1294 240 : half_type = signed_type_for (half_type);
1295 :
1296 1290 : tree half_vectype;
1297 1290 : if (!vect_supportable_conv_optab_p (vinfo, type, DOT_PROD_EXPR, half_type,
1298 : type_out, &half_vectype, subtype))
1299 : {
1300 : /* We can emulate a mixed-sign dot-product using a sequence of
1301 : signed dot-products; see vect_emulate_mixed_dot_prod for details. */
1302 585 : if (subtype != optab_vector_mixed_sign
1303 585 : || !vect_supportable_conv_optab_p (vinfo, signed_type_for (type),
1304 : DOT_PROD_EXPR, half_type,
1305 : type_out, &half_vectype,
1306 : optab_vector))
1307 : return NULL;
1308 :
1309 15 : *type_out = signed_or_unsigned_type_for (TYPE_UNSIGNED (type),
1310 : *type_out);
1311 : }
1312 :
1313 : /* Get the inputs in the appropriate types. */
1314 720 : tree mult_oprnd[2];
1315 720 : vect_convert_inputs (vinfo, stmt_vinfo, 2, mult_oprnd, half_type,
1316 : unprom0, half_vectype, subtype);
1317 :
1318 720 : var = vect_recog_temp_ssa_var (type, NULL);
1319 720 : pattern_stmt = gimple_build_assign (var, DOT_PROD_EXPR,
1320 : mult_oprnd[0], mult_oprnd[1], oprnd1);
1321 :
1322 720 : return pattern_stmt;
1323 : }
1324 :
1325 :
1326 : /* Function vect_recog_sad_pattern
1327 :
1328 : Try to find the following Sum of Absolute Difference (SAD) pattern:
1329 :
1330 : type x_t, y_t;
1331 : signed TYPE1 diff, abs_diff;
1332 : TYPE2 sum = init;
1333 : loop:
1334 : sum_0 = phi <init, sum_1>
1335 : S1 x_t = ...
1336 : S2 y_t = ...
1337 : S3 x_T = (TYPE1) x_t;
1338 : S4 y_T = (TYPE1) y_t;
1339 : S5 diff = x_T - y_T;
1340 : S6 abs_diff = ABS_EXPR <diff>;
1341 : [S7 abs_diff = (TYPE2) abs_diff; #optional]
1342 : S8 sum_1 = abs_diff + sum_0;
1343 :
1344 : where 'TYPE1' is at least double the size of type 'type', and 'TYPE2' is the
1345 : same size of 'TYPE1' or bigger. This is a special case of a reduction
1346 : computation.
1347 :
1348 : Input:
1349 :
1350 : * STMT_VINFO: The stmt from which the pattern search begins. In the
1351 : example, when this function is called with S8, the pattern
1352 : {S3,S4,S5,S6,S7,S8} will be detected.
1353 :
1354 : Output:
1355 :
1356 : * TYPE_OUT: The type of the output of this pattern.
1357 :
1358 : * Return value: A new stmt that will be used to replace the sequence of
1359 : stmts that constitute the pattern. In this case it will be:
1360 : SAD_EXPR <x_t, y_t, sum_0>
1361 : */
1362 :
1363 : static gimple *
1364 32410991 : vect_recog_sad_pattern (vec_info *vinfo,
1365 : stmt_vec_info stmt_vinfo, tree *type_out)
1366 : {
1367 32410991 : gimple *last_stmt = stmt_vinfo->stmt;
1368 32410991 : tree half_type;
1369 :
1370 : /* Look for the following pattern
1371 : DX = (TYPE1) X;
1372 : DY = (TYPE1) Y;
1373 : DDIFF = DX - DY;
1374 : DAD = ABS_EXPR <DDIFF>;
1375 : DDPROD = (TYPE2) DPROD;
1376 : sum_1 = DAD + sum_0;
1377 : In which
1378 : - DX is at least double the size of X
1379 : - DY is at least double the size of Y
1380 : - DX, DY, DDIFF, DAD all have the same type
1381 : - sum is the same size of DAD or bigger
1382 : - sum has been recognized as a reduction variable.
1383 :
1384 : This is equivalent to:
1385 : DDIFF = X w- Y; #widen sub
1386 : DAD = ABS_EXPR <DDIFF>;
1387 : sum_1 = DAD w+ sum_0; #widen summation
1388 : or
1389 : DDIFF = X w- Y; #widen sub
1390 : DAD = ABS_EXPR <DDIFF>;
1391 : sum_1 = DAD + sum_0; #summation
1392 : */
1393 :
1394 : /* Starting from LAST_STMT, follow the defs of its uses in search
1395 : of the above pattern. */
1396 :
1397 32410991 : tree plus_oprnd0, plus_oprnd1;
1398 32410991 : if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, PLUS_EXPR,
1399 : &plus_oprnd0, &plus_oprnd1))
1400 : return NULL;
1401 :
1402 53103 : tree sum_type = TREE_TYPE (gimple_get_lhs (last_stmt));
1403 :
1404 : /* Any non-truncating sequence of conversions is OK here, since
1405 : with a successful match, the result of the ABS(U) is known to fit
1406 : within the nonnegative range of the result type. (It cannot be the
1407 : negative of the minimum signed value due to the range of the widening
1408 : MINUS_EXPR.) */
1409 53103 : vect_unpromoted_value unprom_abs;
1410 53103 : plus_oprnd0 = vect_look_through_possible_promotion (vinfo, plus_oprnd0,
1411 : &unprom_abs);
1412 :
1413 : /* So far so good. Since last_stmt was detected as a (summation) reduction,
1414 : we know that plus_oprnd1 is the reduction variable (defined by a loop-header
1415 : phi), and plus_oprnd0 is an ssa-name defined by a stmt in the loop body.
1416 : Then check that plus_oprnd0 is defined by an abs_expr. */
1417 :
1418 53103 : if (!plus_oprnd0)
1419 : return NULL;
1420 :
1421 35820 : stmt_vec_info abs_stmt_vinfo = vect_get_internal_def (vinfo, plus_oprnd0);
1422 35820 : if (!abs_stmt_vinfo)
1423 : return NULL;
1424 :
1425 : /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
1426 : inside the loop (in case we are analyzing an outer-loop). */
1427 34975 : gassign *abs_stmt = dyn_cast <gassign *> (abs_stmt_vinfo->stmt);
1428 104925 : vect_unpromoted_value unprom[2];
1429 :
1430 34975 : if (!abs_stmt)
1431 : {
1432 367 : gcall *abd_stmt = dyn_cast <gcall *> (abs_stmt_vinfo->stmt);
1433 302 : if (!abd_stmt
1434 302 : || !gimple_call_internal_p (abd_stmt)
1435 0 : || gimple_call_num_args (abd_stmt) != 2)
1436 : return NULL;
1437 :
1438 0 : tree abd_oprnd0 = gimple_call_arg (abd_stmt, 0);
1439 0 : tree abd_oprnd1 = gimple_call_arg (abd_stmt, 1);
1440 :
1441 0 : if (gimple_call_internal_fn (abd_stmt) == IFN_ABD
1442 0 : || gimple_call_internal_fn (abd_stmt) == IFN_VEC_WIDEN_ABD)
1443 : {
1444 0 : unprom[0].op = abd_oprnd0;
1445 0 : unprom[0].type = TREE_TYPE (abd_oprnd0);
1446 0 : unprom[1].op = abd_oprnd1;
1447 0 : unprom[1].type = TREE_TYPE (abd_oprnd1);
1448 : }
1449 : else
1450 : return NULL;
1451 :
1452 0 : half_type = unprom[0].type;
1453 : }
1454 34608 : else if (!vect_recog_absolute_difference (vinfo, abs_stmt, &half_type,
1455 : unprom, NULL))
1456 : return NULL;
1457 :
1458 806 : vect_pattern_detected ("vect_recog_sad_pattern", last_stmt);
1459 :
1460 806 : tree half_vectype;
1461 806 : if (!vect_supportable_direct_optab_p (vinfo, sum_type, SAD_EXPR, half_type,
1462 : type_out, &half_vectype))
1463 : return NULL;
1464 :
1465 : /* Get the inputs to the SAD_EXPR in the appropriate types. */
1466 412 : tree sad_oprnd[2];
1467 412 : vect_convert_inputs (vinfo, stmt_vinfo, 2, sad_oprnd, half_type,
1468 : unprom, half_vectype);
1469 :
1470 412 : tree var = vect_recog_temp_ssa_var (sum_type, NULL);
1471 412 : gimple *pattern_stmt = gimple_build_assign (var, SAD_EXPR, sad_oprnd[0],
1472 : sad_oprnd[1], plus_oprnd1);
1473 :
1474 412 : return pattern_stmt;
1475 : }
1476 :
1477 : /* Function vect_recog_abd_pattern
1478 :
1479 : Try to find the following ABsolute Difference (ABD) or
1480 : widening ABD (WIDEN_ABD) pattern:
1481 :
1482 : TYPE1 x;
1483 : TYPE2 y;
1484 : TYPE3 x_cast = (TYPE3) x; // widening or no-op
1485 : TYPE3 y_cast = (TYPE3) y; // widening or no-op
1486 : TYPE3 diff = x_cast - y_cast;
1487 : TYPE4 diff_cast = (TYPE4) diff; // widening or no-op
1488 : TYPE5 abs = ABS(U)_EXPR <diff_cast>;
1489 :
1490 : WIDEN_ABD exists to optimize the case where TYPE4 is at least
1491 : twice as wide as TYPE3.
1492 :
1493 : Input:
1494 :
1495 : * STMT_VINFO: The stmt from which the pattern search begins
1496 :
1497 : Output:
1498 :
1499 : * TYPE_OUT: The type of the output of this pattern
1500 :
1501 : * Return value: A new stmt that will be used to replace the sequence of
1502 : stmts that constitute the pattern, principally:
1503 : out = IFN_ABD (x, y)
1504 : out = IFN_WIDEN_ABD (x, y)
1505 : */
1506 :
1507 : static gimple *
1508 32359510 : vect_recog_abd_pattern (vec_info *vinfo,
1509 : stmt_vec_info stmt_vinfo, tree *type_out)
1510 : {
1511 32359510 : gassign *last_stmt = dyn_cast <gassign *> (STMT_VINFO_STMT (stmt_vinfo));
1512 22491268 : if (!last_stmt)
1513 : return NULL;
1514 :
1515 22491268 : tree out_type = TREE_TYPE (gimple_assign_lhs (last_stmt));
1516 :
1517 67473804 : vect_unpromoted_value unprom[2];
1518 22491268 : gassign *diff_stmt = NULL;
1519 22491268 : tree abd_in_type;
1520 22491268 : if (!vect_recog_absolute_difference (vinfo, last_stmt, &abd_in_type,
1521 : unprom, &diff_stmt))
1522 : {
1523 : /* We cannot try further without having a non-widening MINUS. */
1524 22489750 : if (!diff_stmt)
1525 : return NULL;
1526 :
1527 278 : unprom[0].op = gimple_assign_rhs1 (diff_stmt);
1528 278 : unprom[1].op = gimple_assign_rhs2 (diff_stmt);
1529 278 : abd_in_type = signed_type_for (out_type);
1530 : }
1531 :
1532 1796 : tree abd_out_type = abd_in_type;
1533 :
1534 1796 : tree vectype_in = get_vectype_for_scalar_type (vinfo, abd_in_type);
1535 1796 : if (!vectype_in)
1536 : return NULL;
1537 :
1538 1777 : internal_fn ifn = IFN_ABD;
1539 1777 : tree vectype_out = vectype_in;
1540 :
1541 1777 : if (TYPE_PRECISION (out_type) >= TYPE_PRECISION (abd_in_type) * 2
1542 1777 : && stmt_vinfo->min_output_precision >= TYPE_PRECISION (abd_in_type) * 2)
1543 : {
1544 1409 : tree mid_type
1545 1409 : = build_nonstandard_integer_type (TYPE_PRECISION (abd_in_type) * 2,
1546 1409 : TYPE_UNSIGNED (abd_in_type));
1547 1409 : tree mid_vectype = get_vectype_for_scalar_type (vinfo, mid_type);
1548 :
1549 1409 : code_helper dummy_code;
1550 1409 : int dummy_int;
1551 1409 : auto_vec<tree> dummy_vec;
1552 1409 : if (mid_vectype
1553 1409 : && supportable_widening_operation (IFN_VEC_WIDEN_ABD,
1554 : mid_vectype, vectype_in, false,
1555 : &dummy_code, &dummy_code,
1556 : &dummy_int, &dummy_vec))
1557 : {
1558 0 : ifn = IFN_VEC_WIDEN_ABD;
1559 0 : abd_out_type = mid_type;
1560 0 : vectype_out = mid_vectype;
1561 : }
1562 1409 : }
1563 :
1564 1409 : if (ifn == IFN_ABD
1565 1777 : && !direct_internal_fn_supported_p (ifn, vectype_in,
1566 : OPTIMIZE_FOR_SPEED))
1567 : return NULL;
1568 :
1569 0 : vect_pattern_detected ("vect_recog_abd_pattern", last_stmt);
1570 :
1571 0 : tree abd_oprnds[2];
1572 0 : vect_convert_inputs (vinfo, stmt_vinfo, 2, abd_oprnds,
1573 : abd_in_type, unprom, vectype_in);
1574 :
1575 0 : *type_out = get_vectype_for_scalar_type (vinfo, out_type);
1576 :
1577 0 : tree abd_result = vect_recog_temp_ssa_var (abd_out_type, NULL);
1578 0 : gcall *abd_stmt = gimple_build_call_internal (ifn, 2,
1579 : abd_oprnds[0], abd_oprnds[1]);
1580 0 : gimple_call_set_lhs (abd_stmt, abd_result);
1581 0 : gimple_set_location (abd_stmt, gimple_location (last_stmt));
1582 :
1583 0 : gimple *stmt = abd_stmt;
1584 0 : if (TYPE_PRECISION (abd_in_type) == TYPE_PRECISION (abd_out_type)
1585 0 : && TYPE_PRECISION (abd_out_type) < TYPE_PRECISION (out_type)
1586 0 : && !TYPE_UNSIGNED (abd_out_type))
1587 : {
1588 0 : tree unsign = unsigned_type_for (abd_out_type);
1589 0 : stmt = vect_convert_output (vinfo, stmt_vinfo, unsign, stmt, vectype_out);
1590 0 : vectype_out = get_vectype_for_scalar_type (vinfo, unsign);
1591 : }
1592 :
1593 0 : return vect_convert_output (vinfo, stmt_vinfo, out_type, stmt, vectype_out);
1594 : }
1595 :
1596 : /* Recognize an operation that performs ORIG_CODE on widened inputs,
1597 : so that it can be treated as though it had the form:
1598 :
1599 : A_TYPE a;
1600 : B_TYPE b;
1601 : HALF_TYPE a_cast = (HALF_TYPE) a; // possible no-op
1602 : HALF_TYPE b_cast = (HALF_TYPE) b; // possible no-op
1603 : | RES_TYPE a_extend = (RES_TYPE) a_cast; // promotion from HALF_TYPE
1604 : | RES_TYPE b_extend = (RES_TYPE) b_cast; // promotion from HALF_TYPE
1605 : | RES_TYPE res = a_extend ORIG_CODE b_extend;
1606 :
1607 : Try to replace the pattern with:
1608 :
1609 : A_TYPE a;
1610 : B_TYPE b;
1611 : HALF_TYPE a_cast = (HALF_TYPE) a; // possible no-op
1612 : HALF_TYPE b_cast = (HALF_TYPE) b; // possible no-op
1613 : | EXT_TYPE ext = a_cast WIDE_CODE b_cast;
1614 : | RES_TYPE res = (EXT_TYPE) ext; // possible no-op
1615 :
1616 : where EXT_TYPE is wider than HALF_TYPE but has the same signedness.
1617 :
1618 : SHIFT_P is true if ORIG_CODE and WIDE_CODE are shifts. NAME is the
1619 : name of the pattern being matched, for dump purposes. */
1620 :
1621 : static gimple *
1622 130250926 : vect_recog_widen_op_pattern (vec_info *vinfo,
1623 : stmt_vec_info last_stmt_info, tree *type_out,
1624 : tree_code orig_code, code_helper wide_code,
1625 : bool shift_p, const char *name)
1626 : {
1627 130250926 : gimple *last_stmt = last_stmt_info->stmt;
1628 :
1629 390752778 : vect_unpromoted_value unprom[2];
1630 130250926 : tree half_type;
1631 130250926 : if (!vect_widened_op_tree (vinfo, last_stmt_info, orig_code, orig_code,
1632 : shift_p, 2, unprom, &half_type))
1633 :
1634 : return NULL;
1635 :
1636 : /* Pattern detected. */
1637 339325 : vect_pattern_detected (name, last_stmt);
1638 :
1639 339325 : tree type = TREE_TYPE (gimple_get_lhs (last_stmt));
1640 339325 : tree itype = type;
1641 339325 : if (TYPE_PRECISION (type) != TYPE_PRECISION (half_type) * 2
1642 339325 : || TYPE_UNSIGNED (type) != TYPE_UNSIGNED (half_type))
1643 229646 : itype = build_nonstandard_integer_type (TYPE_PRECISION (half_type) * 2,
1644 229646 : TYPE_UNSIGNED (half_type));
1645 :
1646 : /* Check target support */
1647 339325 : tree vectype = get_vectype_for_scalar_type (vinfo, half_type);
1648 339325 : tree vecitype = get_vectype_for_scalar_type (vinfo, itype);
1649 339325 : tree ctype = itype;
1650 339325 : tree vecctype = vecitype;
1651 339325 : if (orig_code == MINUS_EXPR
1652 6472 : && TYPE_UNSIGNED (itype)
1653 343893 : && TYPE_PRECISION (type) > TYPE_PRECISION (itype))
1654 : {
1655 : /* Subtraction is special, even if half_type is unsigned and no matter
1656 : whether type is signed or unsigned, if type is wider than itype,
1657 : we need to sign-extend from the widening operation result to the
1658 : result type.
1659 : Consider half_type unsigned char, operand 1 0xfe, operand 2 0xff,
1660 : itype unsigned short and type either int or unsigned int.
1661 : Widened (unsigned short) 0xfe - (unsigned short) 0xff is
1662 : (unsigned short) 0xffff, but for type int we want the result -1
1663 : and for type unsigned int 0xffffffff rather than 0xffff. */
1664 790 : ctype = build_nonstandard_integer_type (TYPE_PRECISION (itype), 0);
1665 790 : vecctype = get_vectype_for_scalar_type (vinfo, ctype);
1666 : }
1667 :
1668 339325 : code_helper dummy_code;
1669 339325 : int dummy_int;
1670 339325 : auto_vec<tree> dummy_vec;
1671 339325 : if (!vectype
1672 339325 : || !vecitype
1673 261457 : || !vecctype
1674 600782 : || !supportable_widening_operation (wide_code, vecitype, vectype, true,
1675 : &dummy_code, &dummy_code,
1676 : &dummy_int, &dummy_vec))
1677 : return NULL;
1678 :
1679 113352 : *type_out = get_vectype_for_scalar_type (vinfo, type);
1680 113352 : if (!*type_out)
1681 : return NULL;
1682 :
1683 113352 : tree oprnd[2];
1684 113352 : vect_convert_inputs (vinfo, last_stmt_info,
1685 : 2, oprnd, half_type, unprom, vectype);
1686 :
1687 113352 : tree var = vect_recog_temp_ssa_var (itype, NULL);
1688 113352 : gimple *pattern_stmt = vect_gimple_build (var, wide_code, oprnd[0], oprnd[1]);
1689 :
1690 113352 : if (vecctype != vecitype)
1691 0 : pattern_stmt = vect_convert_output (vinfo, last_stmt_info, ctype,
1692 : pattern_stmt, vecitype);
1693 :
1694 113352 : return vect_convert_output (vinfo, last_stmt_info,
1695 113352 : type, pattern_stmt, vecctype);
1696 339325 : }
1697 :
1698 : /* Try to detect multiplication on widened inputs, converting MULT_EXPR
1699 : to WIDEN_MULT_EXPR. See vect_recog_widen_op_pattern for details. */
1700 :
1701 : static gimple *
1702 32439615 : vect_recog_widen_mult_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
1703 : tree *type_out)
1704 : {
1705 32439615 : return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
1706 32439615 : MULT_EXPR, WIDEN_MULT_EXPR, false,
1707 32439615 : "vect_recog_widen_mult_pattern");
1708 : }
1709 :
1710 : /* Try to detect addition on widened inputs, converting PLUS_EXPR
1711 : to IFN_VEC_WIDEN_PLUS. See vect_recog_widen_op_pattern for details. */
1712 :
1713 : static gimple *
1714 32700095 : vect_recog_widen_plus_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
1715 : tree *type_out)
1716 : {
1717 32700095 : return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
1718 32700095 : PLUS_EXPR, IFN_VEC_WIDEN_PLUS,
1719 32700095 : false, "vect_recog_widen_plus_pattern");
1720 : }
1721 :
1722 : /* Try to detect subtraction on widened inputs, converting MINUS_EXPR
1723 : to IFN_VEC_WIDEN_MINUS. See vect_recog_widen_op_pattern for details. */
1724 : static gimple *
1725 32700095 : vect_recog_widen_minus_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
1726 : tree *type_out)
1727 : {
1728 32700095 : return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
1729 32700095 : MINUS_EXPR, IFN_VEC_WIDEN_MINUS,
1730 32700095 : false, "vect_recog_widen_minus_pattern");
1731 : }
1732 :
1733 : /* Try to detect abd on widened inputs, converting IFN_ABD
1734 : to IFN_VEC_WIDEN_ABD. */
1735 : static gimple *
1736 32700095 : vect_recog_widen_abd_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
1737 : tree *type_out)
1738 : {
1739 32700095 : gassign *last_stmt = dyn_cast <gassign *> (STMT_VINFO_STMT (stmt_vinfo));
1740 30743562 : if (!last_stmt || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (last_stmt)))
1741 : return NULL;
1742 :
1743 3250883 : tree last_rhs = gimple_assign_rhs1 (last_stmt);
1744 :
1745 3250883 : tree in_type = TREE_TYPE (last_rhs);
1746 3250883 : tree out_type = TREE_TYPE (gimple_assign_lhs (last_stmt));
1747 3250883 : if (!INTEGRAL_TYPE_P (in_type)
1748 2840338 : || !INTEGRAL_TYPE_P (out_type)
1749 2723224 : || TYPE_PRECISION (in_type) * 2 != TYPE_PRECISION (out_type)
1750 3911706 : || !TYPE_UNSIGNED (in_type))
1751 : return NULL;
1752 :
1753 240192 : vect_unpromoted_value unprom;
1754 240192 : tree op = vect_look_through_possible_promotion (vinfo, last_rhs, &unprom);
1755 240192 : if (!op || TYPE_PRECISION (TREE_TYPE (op)) != TYPE_PRECISION (in_type))
1756 : return NULL;
1757 :
1758 237602 : stmt_vec_info abd_pattern_vinfo = vect_get_internal_def (vinfo, op);
1759 237602 : if (!abd_pattern_vinfo)
1760 : return NULL;
1761 :
1762 212395 : gcall *abd_stmt = dyn_cast <gcall *> (STMT_VINFO_STMT (abd_pattern_vinfo));
1763 9129 : if (!abd_stmt
1764 9129 : || !gimple_call_internal_p (abd_stmt)
1765 265 : || gimple_call_internal_fn (abd_stmt) != IFN_ABD)
1766 : return NULL;
1767 :
1768 0 : tree vectype_in = get_vectype_for_scalar_type (vinfo, in_type);
1769 0 : tree vectype_out = get_vectype_for_scalar_type (vinfo, out_type);
1770 :
1771 0 : code_helper dummy_code;
1772 0 : int dummy_int;
1773 0 : auto_vec<tree> dummy_vec;
1774 0 : if (!supportable_widening_operation (IFN_VEC_WIDEN_ABD, vectype_out,
1775 : vectype_in, false,
1776 : &dummy_code, &dummy_code,
1777 : &dummy_int, &dummy_vec))
1778 : return NULL;
1779 :
1780 0 : vect_pattern_detected ("vect_recog_widen_abd_pattern", last_stmt);
1781 :
1782 0 : *type_out = vectype_out;
1783 :
1784 0 : tree abd_oprnd0 = gimple_call_arg (abd_stmt, 0);
1785 0 : tree abd_oprnd1 = gimple_call_arg (abd_stmt, 1);
1786 0 : tree widen_abd_result = vect_recog_temp_ssa_var (out_type, NULL);
1787 0 : gcall *widen_abd_stmt = gimple_build_call_internal (IFN_VEC_WIDEN_ABD, 2,
1788 : abd_oprnd0, abd_oprnd1);
1789 0 : gimple_call_set_lhs (widen_abd_stmt, widen_abd_result);
1790 0 : gimple_set_location (widen_abd_stmt, gimple_location (last_stmt));
1791 0 : return widen_abd_stmt;
1792 0 : }
1793 :
1794 : /* Function vect_recog_ctz_ffs_pattern
1795 :
1796 : Try to find the following pattern:
1797 :
1798 : TYPE1 A;
1799 : TYPE1 B;
1800 :
1801 : B = __builtin_ctz{,l,ll} (A);
1802 :
1803 : or
1804 :
1805 : B = __builtin_ffs{,l,ll} (A);
1806 :
1807 : Input:
1808 :
1809 : * STMT_VINFO: The stmt from which the pattern search begins.
1810 : here it starts with B = __builtin_* (A);
1811 :
1812 : Output:
1813 :
1814 : * TYPE_OUT: The vector type of the output of this pattern.
1815 :
1816 : * Return value: A new stmt that will be used to replace the sequence of
1817 : stmts that constitute the pattern, using clz or popcount builtins. */
1818 :
1819 : static gimple *
1820 32410928 : vect_recog_ctz_ffs_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
1821 : tree *type_out)
1822 : {
1823 32410928 : gimple *call_stmt = stmt_vinfo->stmt;
1824 32410928 : gimple *pattern_stmt;
1825 32410928 : tree rhs_oprnd, rhs_type, lhs_oprnd, lhs_type, vec_type, vec_rhs_type;
1826 32410928 : tree new_var;
1827 32410928 : internal_fn ifn = IFN_LAST, ifnnew = IFN_LAST;
1828 32410928 : bool defined_at_zero = true, defined_at_zero_new = false;
1829 32410928 : int val = 0, val_new = 0, val_cmp = 0;
1830 32410928 : int prec;
1831 32410928 : int sub = 0, add = 0;
1832 32410928 : location_t loc;
1833 :
1834 32410928 : if (!is_gimple_call (call_stmt))
1835 : return NULL;
1836 :
1837 3843289 : if (gimple_call_num_args (call_stmt) != 1
1838 3843289 : && gimple_call_num_args (call_stmt) != 2)
1839 : return NULL;
1840 :
1841 2189162 : rhs_oprnd = gimple_call_arg (call_stmt, 0);
1842 2189162 : rhs_type = TREE_TYPE (rhs_oprnd);
1843 2189162 : lhs_oprnd = gimple_call_lhs (call_stmt);
1844 2189162 : if (!lhs_oprnd)
1845 : return NULL;
1846 1048543 : lhs_type = TREE_TYPE (lhs_oprnd);
1847 1048543 : if (!INTEGRAL_TYPE_P (lhs_type)
1848 340956 : || !INTEGRAL_TYPE_P (rhs_type)
1849 47314 : || !type_has_mode_precision_p (rhs_type)
1850 1094270 : || TREE_CODE (rhs_oprnd) != SSA_NAME)
1851 : return NULL;
1852 :
1853 32552 : switch (gimple_call_combined_fn (call_stmt))
1854 : {
1855 1975 : CASE_CFN_CTZ:
1856 1975 : ifn = IFN_CTZ;
1857 1975 : if (!gimple_call_internal_p (call_stmt)
1858 1975 : || gimple_call_num_args (call_stmt) != 2)
1859 : defined_at_zero = false;
1860 : else
1861 121 : val = tree_to_shwi (gimple_call_arg (call_stmt, 1));
1862 : break;
1863 : CASE_CFN_FFS:
1864 : ifn = IFN_FFS;
1865 : break;
1866 : default:
1867 : return NULL;
1868 : }
1869 :
1870 2210 : prec = TYPE_PRECISION (rhs_type);
1871 2210 : loc = gimple_location (call_stmt);
1872 :
1873 2210 : vec_type = get_vectype_for_scalar_type (vinfo, lhs_type);
1874 2210 : if (!vec_type)
1875 : return NULL;
1876 :
1877 2204 : vec_rhs_type = get_vectype_for_scalar_type (vinfo, rhs_type);
1878 2204 : if (!vec_rhs_type)
1879 : return NULL;
1880 :
1881 : /* Do it only if the backend doesn't have ctz<vector_mode>2 or
1882 : ffs<vector_mode>2 pattern but does have clz<vector_mode>2 or
1883 : popcount<vector_mode>2. */
1884 1941 : if (!vec_type
1885 1941 : || direct_internal_fn_supported_p (ifn, vec_rhs_type,
1886 : OPTIMIZE_FOR_SPEED))
1887 : return NULL;
1888 :
1889 1941 : if (ifn == IFN_FFS
1890 1941 : && direct_internal_fn_supported_p (IFN_CTZ, vec_rhs_type,
1891 : OPTIMIZE_FOR_SPEED))
1892 : {
1893 0 : ifnnew = IFN_CTZ;
1894 0 : defined_at_zero_new
1895 0 : = CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (rhs_type),
1896 : val_new) == 2;
1897 : }
1898 1941 : else if (direct_internal_fn_supported_p (IFN_CLZ, vec_rhs_type,
1899 : OPTIMIZE_FOR_SPEED))
1900 : {
1901 166 : ifnnew = IFN_CLZ;
1902 166 : defined_at_zero_new
1903 166 : = CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (rhs_type),
1904 : val_new) == 2;
1905 : }
1906 166 : if ((ifnnew == IFN_LAST
1907 166 : || (defined_at_zero && !defined_at_zero_new))
1908 1775 : && direct_internal_fn_supported_p (IFN_POPCOUNT, vec_rhs_type,
1909 : OPTIMIZE_FOR_SPEED))
1910 : {
1911 : ifnnew = IFN_POPCOUNT;
1912 : defined_at_zero_new = true;
1913 : val_new = prec;
1914 : }
1915 1815 : if (ifnnew == IFN_LAST)
1916 : return NULL;
1917 :
1918 292 : vect_pattern_detected ("vec_recog_ctz_ffs_pattern", call_stmt);
1919 :
1920 292 : val_cmp = val_new;
1921 292 : if ((ifnnew == IFN_CLZ
1922 292 : && defined_at_zero
1923 106 : && defined_at_zero_new
1924 106 : && val == prec
1925 54 : && val_new == prec)
1926 238 : || (ifnnew == IFN_POPCOUNT && ifn == IFN_CTZ))
1927 : {
1928 137 : if (vect_is_reduction (stmt_vinfo))
1929 : return NULL;
1930 :
1931 : /* .CTZ (X) = PREC - .CLZ ((X - 1) & ~X)
1932 : .CTZ (X) = .POPCOUNT ((X - 1) & ~X). */
1933 137 : if (ifnnew == IFN_CLZ)
1934 54 : sub = prec;
1935 137 : val_cmp = prec;
1936 :
1937 137 : if (!TYPE_UNSIGNED (rhs_type))
1938 : {
1939 12 : rhs_type = unsigned_type_for (rhs_type);
1940 12 : vec_rhs_type = get_vectype_for_scalar_type (vinfo, rhs_type);
1941 12 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
1942 12 : pattern_stmt = gimple_build_assign (new_var, NOP_EXPR, rhs_oprnd);
1943 12 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
1944 : vec_rhs_type);
1945 12 : rhs_oprnd = new_var;
1946 : }
1947 :
1948 137 : tree m1 = vect_recog_temp_ssa_var (rhs_type, NULL);
1949 137 : pattern_stmt = gimple_build_assign (m1, PLUS_EXPR, rhs_oprnd,
1950 : build_int_cst (rhs_type, -1));
1951 137 : gimple_set_location (pattern_stmt, loc);
1952 137 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
1953 :
1954 137 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
1955 137 : pattern_stmt = gimple_build_assign (new_var, BIT_NOT_EXPR, rhs_oprnd);
1956 137 : gimple_set_location (pattern_stmt, loc);
1957 137 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
1958 137 : rhs_oprnd = new_var;
1959 :
1960 137 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
1961 137 : pattern_stmt = gimple_build_assign (new_var, BIT_AND_EXPR,
1962 : m1, rhs_oprnd);
1963 137 : gimple_set_location (pattern_stmt, loc);
1964 137 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
1965 137 : rhs_oprnd = new_var;
1966 137 : }
1967 155 : else if (ifnnew == IFN_CLZ)
1968 : {
1969 112 : if (vect_is_reduction (stmt_vinfo))
1970 : return NULL;
1971 :
1972 : /* .CTZ (X) = (PREC - 1) - .CLZ (X & -X)
1973 : .FFS (X) = PREC - .CLZ (X & -X). */
1974 106 : sub = prec - (ifn == IFN_CTZ);
1975 106 : val_cmp = sub - val_new;
1976 :
1977 106 : tree neg = vect_recog_temp_ssa_var (rhs_type, NULL);
1978 106 : pattern_stmt = gimple_build_assign (neg, NEGATE_EXPR, rhs_oprnd);
1979 106 : gimple_set_location (pattern_stmt, loc);
1980 106 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
1981 :
1982 106 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
1983 106 : pattern_stmt = gimple_build_assign (new_var, BIT_AND_EXPR,
1984 : rhs_oprnd, neg);
1985 106 : gimple_set_location (pattern_stmt, loc);
1986 106 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
1987 106 : rhs_oprnd = new_var;
1988 : }
1989 43 : else if (ifnnew == IFN_POPCOUNT)
1990 : {
1991 43 : if (vect_is_reduction (stmt_vinfo))
1992 : return NULL;
1993 :
1994 : /* .CTZ (X) = PREC - .POPCOUNT (X | -X)
1995 : .FFS (X) = (PREC + 1) - .POPCOUNT (X | -X). */
1996 43 : sub = prec + (ifn == IFN_FFS);
1997 43 : val_cmp = sub;
1998 :
1999 43 : tree neg = vect_recog_temp_ssa_var (rhs_type, NULL);
2000 43 : pattern_stmt = gimple_build_assign (neg, NEGATE_EXPR, rhs_oprnd);
2001 43 : gimple_set_location (pattern_stmt, loc);
2002 43 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2003 :
2004 43 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
2005 43 : pattern_stmt = gimple_build_assign (new_var, BIT_IOR_EXPR,
2006 : rhs_oprnd, neg);
2007 43 : gimple_set_location (pattern_stmt, loc);
2008 43 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2009 43 : rhs_oprnd = new_var;
2010 : }
2011 0 : else if (ifnnew == IFN_CTZ)
2012 : {
2013 : /* .FFS (X) = .CTZ (X) + 1. */
2014 0 : add = 1;
2015 0 : val_cmp++;
2016 :
2017 0 : if (vect_is_reduction (stmt_vinfo)
2018 0 : && defined_at_zero
2019 0 : && (!defined_at_zero_new || val != val_cmp))
2020 : return NULL;
2021 : }
2022 :
2023 : /* Create B = .IFNNEW (A). */
2024 286 : new_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2025 286 : if ((ifnnew == IFN_CLZ || ifnnew == IFN_CTZ) && defined_at_zero_new)
2026 160 : pattern_stmt
2027 160 : = gimple_build_call_internal (ifnnew, 2, rhs_oprnd,
2028 : build_int_cst (integer_type_node,
2029 160 : val_new));
2030 : else
2031 126 : pattern_stmt = gimple_build_call_internal (ifnnew, 1, rhs_oprnd);
2032 286 : gimple_call_set_lhs (pattern_stmt, new_var);
2033 286 : gimple_set_location (pattern_stmt, loc);
2034 286 : *type_out = vec_type;
2035 :
2036 286 : if (sub)
2037 : {
2038 203 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2039 203 : tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2040 203 : pattern_stmt = gimple_build_assign (ret_var, MINUS_EXPR,
2041 203 : build_int_cst (lhs_type, sub),
2042 : new_var);
2043 203 : gimple_set_location (pattern_stmt, loc);
2044 203 : new_var = ret_var;
2045 : }
2046 83 : else if (add)
2047 : {
2048 0 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2049 0 : tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2050 0 : pattern_stmt = gimple_build_assign (ret_var, PLUS_EXPR, new_var,
2051 0 : build_int_cst (lhs_type, add));
2052 0 : gimple_set_location (pattern_stmt, loc);
2053 0 : new_var = ret_var;
2054 : }
2055 :
2056 286 : if (defined_at_zero
2057 210 : && (!defined_at_zero_new || val != val_cmp))
2058 : {
2059 43 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2060 43 : tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2061 43 : rhs_oprnd = gimple_call_arg (call_stmt, 0);
2062 43 : rhs_type = TREE_TYPE (rhs_oprnd);
2063 43 : tree cmp = vect_recog_temp_ssa_var (boolean_type_node, NULL);
2064 43 : pattern_stmt = gimple_build_assign (cmp, NE_EXPR, rhs_oprnd,
2065 : build_zero_cst (rhs_type));
2066 43 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
2067 : truth_type_for (vec_type), rhs_type);
2068 43 : pattern_stmt = gimple_build_assign (ret_var, COND_EXPR, cmp,
2069 : new_var,
2070 43 : build_int_cst (lhs_type, val));
2071 : }
2072 :
2073 286 : if (dump_enabled_p ())
2074 36 : dump_printf_loc (MSG_NOTE, vect_location,
2075 : "created pattern stmt: %G", pattern_stmt);
2076 :
2077 : return pattern_stmt;
2078 : }
2079 :
2080 : /* Function vect_recog_popcount_clz_ctz_ffs_pattern
2081 :
2082 : Try to find the following pattern:
2083 :
2084 : UTYPE1 A;
2085 : TYPE1 B;
2086 : UTYPE2 temp_in;
2087 : TYPE3 temp_out;
2088 : temp_in = (UTYPE2)A;
2089 :
2090 : temp_out = __builtin_popcount{,l,ll} (temp_in);
2091 : B = (TYPE1) temp_out;
2092 :
2093 : TYPE2 may or may not be equal to TYPE3.
2094 : i.e. TYPE2 is equal to TYPE3 for __builtin_popcount
2095 : i.e. TYPE2 is not equal to TYPE3 for __builtin_popcountll
2096 :
2097 : Input:
2098 :
2099 : * STMT_VINFO: The stmt from which the pattern search begins.
2100 : here it starts with B = (TYPE1) temp_out;
2101 :
2102 : Output:
2103 :
2104 : * TYPE_OUT: The vector type of the output of this pattern.
2105 :
2106 : * Return value: A new stmt that will be used to replace the sequence of
2107 : stmts that constitute the pattern. In this case it will be:
2108 : B = .POPCOUNT (A);
2109 :
2110 : Similarly for clz, ctz and ffs.
2111 : */
2112 :
2113 : static gimple *
2114 32410562 : vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
2115 : stmt_vec_info stmt_vinfo,
2116 : tree *type_out)
2117 : {
2118 32410562 : gassign *last_stmt = dyn_cast <gassign *> (stmt_vinfo->stmt);
2119 22542038 : gimple *call_stmt, *pattern_stmt;
2120 22542038 : tree rhs_oprnd, rhs_origin, lhs_oprnd, lhs_type, vec_type, new_var;
2121 54952600 : internal_fn ifn = IFN_LAST;
2122 54952600 : int addend = 0;
2123 :
2124 : /* Find B = (TYPE1) temp_out. */
2125 22542038 : if (!last_stmt)
2126 : return NULL;
2127 22542038 : tree_code code = gimple_assign_rhs_code (last_stmt);
2128 22542038 : if (!CONVERT_EXPR_CODE_P (code))
2129 : return NULL;
2130 :
2131 3102283 : lhs_oprnd = gimple_assign_lhs (last_stmt);
2132 3102283 : lhs_type = TREE_TYPE (lhs_oprnd);
2133 3102283 : if (!INTEGRAL_TYPE_P (lhs_type))
2134 : return NULL;
2135 :
2136 2822074 : rhs_oprnd = gimple_assign_rhs1 (last_stmt);
2137 2822074 : if (TREE_CODE (rhs_oprnd) != SSA_NAME
2138 2822074 : || !has_single_use (rhs_oprnd))
2139 : return NULL;
2140 1436081 : call_stmt = SSA_NAME_DEF_STMT (rhs_oprnd);
2141 :
2142 : /* Find temp_out = __builtin_popcount{,l,ll} (temp_in); */
2143 1436081 : if (!is_gimple_call (call_stmt))
2144 : return NULL;
2145 59690 : switch (gimple_call_combined_fn (call_stmt))
2146 : {
2147 : int val;
2148 : CASE_CFN_POPCOUNT:
2149 : ifn = IFN_POPCOUNT;
2150 : break;
2151 2375 : CASE_CFN_CLZ:
2152 2375 : ifn = IFN_CLZ;
2153 : /* Punt if call result is unsigned and defined value at zero
2154 : is negative, as the negative value doesn't extend correctly. */
2155 2375 : if (TYPE_UNSIGNED (TREE_TYPE (rhs_oprnd))
2156 0 : && gimple_call_internal_p (call_stmt)
2157 2375 : && CLZ_DEFINED_VALUE_AT_ZERO
2158 : (SCALAR_INT_TYPE_MODE (TREE_TYPE (rhs_oprnd)), val) == 2
2159 2375 : && val < 0)
2160 : return NULL;
2161 : break;
2162 740 : CASE_CFN_CTZ:
2163 740 : ifn = IFN_CTZ;
2164 : /* Punt if call result is unsigned and defined value at zero
2165 : is negative, as the negative value doesn't extend correctly. */
2166 740 : if (TYPE_UNSIGNED (TREE_TYPE (rhs_oprnd))
2167 0 : && gimple_call_internal_p (call_stmt)
2168 740 : && CTZ_DEFINED_VALUE_AT_ZERO
2169 : (SCALAR_INT_TYPE_MODE (TREE_TYPE (rhs_oprnd)), val) == 2
2170 740 : && val < 0)
2171 : return NULL;
2172 : break;
2173 57 : CASE_CFN_FFS:
2174 57 : ifn = IFN_FFS;
2175 57 : break;
2176 : default:
2177 : return NULL;
2178 : }
2179 :
2180 3474 : if (gimple_call_num_args (call_stmt) != 1
2181 3474 : && gimple_call_num_args (call_stmt) != 2)
2182 : return NULL;
2183 :
2184 3474 : rhs_oprnd = gimple_call_arg (call_stmt, 0);
2185 3474 : vect_unpromoted_value unprom_diff;
2186 3474 : rhs_origin
2187 3474 : = vect_look_through_possible_promotion (vinfo, rhs_oprnd, &unprom_diff);
2188 :
2189 3474 : if (!rhs_origin)
2190 : return NULL;
2191 :
2192 : /* Input and output of .POPCOUNT should be same-precision integer. */
2193 3464 : if (TYPE_PRECISION (unprom_diff.type) != TYPE_PRECISION (lhs_type))
2194 : return NULL;
2195 :
2196 : /* Also A should be unsigned or same precision as temp_in, otherwise
2197 : different builtins/internal functions have different behaviors. */
2198 1757 : if (TYPE_PRECISION (unprom_diff.type)
2199 1757 : != TYPE_PRECISION (TREE_TYPE (rhs_oprnd)))
2200 264 : switch (ifn)
2201 : {
2202 95 : case IFN_POPCOUNT:
2203 : /* For popcount require zero extension, which doesn't add any
2204 : further bits to the count. */
2205 95 : if (!TYPE_UNSIGNED (unprom_diff.type))
2206 : return NULL;
2207 : break;
2208 109 : case IFN_CLZ:
2209 : /* clzll (x) == clz (x) + 32 for unsigned x != 0, so ok
2210 : if it is undefined at zero or if it matches also for the
2211 : defined value there. */
2212 109 : if (!TYPE_UNSIGNED (unprom_diff.type))
2213 : return NULL;
2214 109 : if (!type_has_mode_precision_p (lhs_type)
2215 109 : || !type_has_mode_precision_p (TREE_TYPE (rhs_oprnd)))
2216 : return NULL;
2217 109 : addend = (TYPE_PRECISION (TREE_TYPE (rhs_oprnd))
2218 109 : - TYPE_PRECISION (lhs_type));
2219 109 : if (gimple_call_internal_p (call_stmt)
2220 109 : && gimple_call_num_args (call_stmt) == 2)
2221 : {
2222 0 : int val1, val2;
2223 0 : val1 = tree_to_shwi (gimple_call_arg (call_stmt, 1));
2224 0 : int d2
2225 0 : = CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (lhs_type),
2226 : val2);
2227 0 : if (d2 != 2 || val1 != val2 + addend)
2228 : return NULL;
2229 : }
2230 : break;
2231 40 : case IFN_CTZ:
2232 : /* ctzll (x) == ctz (x) for unsigned or signed x != 0, so ok
2233 : if it is undefined at zero or if it matches also for the
2234 : defined value there. */
2235 40 : if (gimple_call_internal_p (call_stmt)
2236 40 : && gimple_call_num_args (call_stmt) == 2)
2237 : {
2238 0 : int val1, val2;
2239 0 : val1 = tree_to_shwi (gimple_call_arg (call_stmt, 1));
2240 0 : int d2
2241 0 : = CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (lhs_type),
2242 : val2);
2243 0 : if (d2 != 2 || val1 != val2)
2244 : return NULL;
2245 : }
2246 : break;
2247 : case IFN_FFS:
2248 : /* ffsll (x) == ffs (x) for unsigned or signed x. */
2249 : break;
2250 : default:
2251 : gcc_unreachable ();
2252 : }
2253 :
2254 1757 : vec_type = get_vectype_for_scalar_type (vinfo, lhs_type);
2255 : /* Do it only if the backend has popcount<vector_mode>2 etc. pattern. */
2256 1757 : if (!vec_type)
2257 : return NULL;
2258 :
2259 1628 : bool supported
2260 1628 : = direct_internal_fn_supported_p (ifn, vec_type, OPTIMIZE_FOR_SPEED);
2261 : /* We cannot rely on the reduction check in vect_recog_ctz_ffs_pattern
2262 : because we do not set up the temporary pattern stmt finally for
2263 : the recursive analysis. */
2264 1628 : if (!supported && vect_is_reduction (stmt_vinfo))
2265 : return NULL;
2266 1612 : if (!supported)
2267 1435 : switch (ifn)
2268 : {
2269 : case IFN_POPCOUNT:
2270 : case IFN_CLZ:
2271 : return NULL;
2272 57 : case IFN_FFS:
2273 : /* vect_recog_ctz_ffs_pattern can implement ffs using ctz. */
2274 57 : if (direct_internal_fn_supported_p (IFN_CTZ, vec_type,
2275 : OPTIMIZE_FOR_SPEED))
2276 : break;
2277 : /* FALLTHRU */
2278 531 : case IFN_CTZ:
2279 : /* vect_recog_ctz_ffs_pattern can implement ffs or ctz using
2280 : clz or popcount. */
2281 531 : if (direct_internal_fn_supported_p (IFN_CLZ, vec_type,
2282 : OPTIMIZE_FOR_SPEED))
2283 : break;
2284 471 : if (direct_internal_fn_supported_p (IFN_POPCOUNT, vec_type,
2285 : OPTIMIZE_FOR_SPEED))
2286 : break;
2287 : return NULL;
2288 : default:
2289 : gcc_unreachable ();
2290 : }
2291 :
2292 318 : vect_pattern_detected ("vec_recog_popcount_clz_ctz_ffs_pattern",
2293 : call_stmt);
2294 :
2295 : /* Create B = .POPCOUNT (A). */
2296 318 : new_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2297 318 : tree arg2 = NULL_TREE;
2298 318 : int val;
2299 318 : if (ifn == IFN_CLZ
2300 368 : && CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (lhs_type),
2301 : val) == 2)
2302 48 : arg2 = build_int_cst (integer_type_node, val);
2303 270 : else if (ifn == IFN_CTZ
2304 363 : && CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (lhs_type),
2305 : val) == 2)
2306 93 : arg2 = build_int_cst (integer_type_node, val);
2307 318 : if (arg2)
2308 141 : pattern_stmt = gimple_build_call_internal (ifn, 2, unprom_diff.op, arg2);
2309 : else
2310 177 : pattern_stmt = gimple_build_call_internal (ifn, 1, unprom_diff.op);
2311 318 : gimple_call_set_lhs (pattern_stmt, new_var);
2312 318 : gimple_set_location (pattern_stmt, gimple_location (last_stmt));
2313 318 : *type_out = vec_type;
2314 :
2315 318 : if (dump_enabled_p ())
2316 24 : dump_printf_loc (MSG_NOTE, vect_location,
2317 : "created pattern stmt: %G", pattern_stmt);
2318 :
2319 318 : if (addend)
2320 : {
2321 12 : gcc_assert (supported);
2322 12 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2323 12 : tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2324 12 : pattern_stmt = gimple_build_assign (ret_var, PLUS_EXPR, new_var,
2325 12 : build_int_cst (lhs_type, addend));
2326 : }
2327 306 : else if (!supported)
2328 : {
2329 141 : stmt_vec_info new_stmt_info = vinfo->add_stmt (pattern_stmt);
2330 141 : STMT_VINFO_VECTYPE (new_stmt_info) = vec_type;
2331 141 : pattern_stmt
2332 141 : = vect_recog_ctz_ffs_pattern (vinfo, new_stmt_info, type_out);
2333 141 : if (pattern_stmt == NULL)
2334 : return NULL;
2335 141 : if (gimple_seq seq = STMT_VINFO_PATTERN_DEF_SEQ (new_stmt_info))
2336 : {
2337 141 : gimple_seq *pseq = &STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo);
2338 141 : gimple_seq_add_seq_without_update (pseq, seq);
2339 : }
2340 : }
2341 : return pattern_stmt;
2342 : }
2343 :
2344 : /* Function vect_recog_pow_pattern
2345 :
2346 : Try to find the following pattern:
2347 :
2348 : x = POW (y, N);
2349 :
2350 : with POW being one of pow, powf, powi, powif and N being
2351 : either 2 or 0.5.
2352 :
2353 : Input:
2354 :
2355 : * STMT_VINFO: The stmt from which the pattern search begins.
2356 :
2357 : Output:
2358 :
2359 : * TYPE_OUT: The type of the output of this pattern.
2360 :
2361 : * Return value: A new stmt that will be used to replace the sequence of
2362 : stmts that constitute the pattern. In this case it will be:
2363 : x = x * x
2364 : or
2365 : x = sqrt (x)
2366 : */
2367 :
2368 : static gimple *
2369 32410579 : vect_recog_pow_pattern (vec_info *vinfo,
2370 : stmt_vec_info stmt_vinfo, tree *type_out)
2371 : {
2372 32410579 : gimple *last_stmt = stmt_vinfo->stmt;
2373 32410579 : tree base, exp;
2374 32410579 : gimple *stmt;
2375 32410579 : tree var;
2376 :
2377 32410579 : if (!is_gimple_call (last_stmt) || gimple_call_lhs (last_stmt) == NULL)
2378 : return NULL;
2379 :
2380 1539777 : switch (gimple_call_combined_fn (last_stmt))
2381 : {
2382 517 : CASE_CFN_POW:
2383 517 : CASE_CFN_POWI:
2384 517 : break;
2385 :
2386 : default:
2387 : return NULL;
2388 : }
2389 :
2390 517 : base = gimple_call_arg (last_stmt, 0);
2391 517 : exp = gimple_call_arg (last_stmt, 1);
2392 517 : if (TREE_CODE (exp) != REAL_CST
2393 462 : && TREE_CODE (exp) != INTEGER_CST)
2394 : {
2395 462 : if (flag_unsafe_math_optimizations
2396 37 : && TREE_CODE (base) == REAL_CST
2397 465 : && gimple_call_builtin_p (last_stmt, BUILT_IN_NORMAL))
2398 : {
2399 3 : combined_fn log_cfn;
2400 3 : built_in_function exp_bfn;
2401 3 : switch (DECL_FUNCTION_CODE (gimple_call_fndecl (last_stmt)))
2402 : {
2403 : case BUILT_IN_POW:
2404 : log_cfn = CFN_BUILT_IN_LOG;
2405 : exp_bfn = BUILT_IN_EXP;
2406 : break;
2407 0 : case BUILT_IN_POWF:
2408 0 : log_cfn = CFN_BUILT_IN_LOGF;
2409 0 : exp_bfn = BUILT_IN_EXPF;
2410 0 : break;
2411 0 : case BUILT_IN_POWL:
2412 0 : log_cfn = CFN_BUILT_IN_LOGL;
2413 0 : exp_bfn = BUILT_IN_EXPL;
2414 0 : break;
2415 : default:
2416 : return NULL;
2417 : }
2418 3 : tree logc = fold_const_call (log_cfn, TREE_TYPE (base), base);
2419 3 : tree exp_decl = builtin_decl_implicit (exp_bfn);
2420 : /* Optimize pow (C, x) as exp (log (C) * x). Normally match.pd
2421 : does that, but if C is a power of 2, we want to use
2422 : exp2 (log2 (C) * x) in the non-vectorized version, but for
2423 : vectorization we don't have vectorized exp2. */
2424 3 : if (logc
2425 3 : && TREE_CODE (logc) == REAL_CST
2426 3 : && exp_decl
2427 6 : && lookup_attribute ("omp declare simd",
2428 3 : DECL_ATTRIBUTES (exp_decl)))
2429 : {
2430 3 : cgraph_node *node = cgraph_node::get_create (exp_decl);
2431 3 : if (node->simd_clones == NULL)
2432 : {
2433 2 : if (targetm.simd_clone.compute_vecsize_and_simdlen == NULL
2434 2 : || node->definition)
2435 : return NULL;
2436 2 : expand_simd_clones (node);
2437 2 : if (node->simd_clones == NULL)
2438 : return NULL;
2439 : }
2440 3 : *type_out = get_vectype_for_scalar_type (vinfo, TREE_TYPE (base));
2441 3 : if (!*type_out)
2442 : return NULL;
2443 3 : tree def = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
2444 3 : gimple *g = gimple_build_assign (def, MULT_EXPR, exp, logc);
2445 3 : append_pattern_def_seq (vinfo, stmt_vinfo, g);
2446 3 : tree res = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
2447 3 : g = gimple_build_call (exp_decl, 1, def);
2448 3 : gimple_call_set_lhs (g, res);
2449 3 : return g;
2450 : }
2451 : }
2452 :
2453 : return NULL;
2454 : }
2455 :
2456 : /* We now have a pow or powi builtin function call with a constant
2457 : exponent. */
2458 :
2459 : /* Catch squaring. */
2460 55 : if ((tree_fits_shwi_p (exp)
2461 0 : && tree_to_shwi (exp) == 2)
2462 55 : || (TREE_CODE (exp) == REAL_CST
2463 55 : && real_equal (&TREE_REAL_CST (exp), &dconst2)))
2464 : {
2465 11 : if (!vect_supportable_direct_optab_p (vinfo, TREE_TYPE (base), MULT_EXPR,
2466 11 : TREE_TYPE (base), type_out))
2467 : return NULL;
2468 :
2469 9 : var = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
2470 9 : stmt = gimple_build_assign (var, MULT_EXPR, base, base);
2471 9 : return stmt;
2472 : }
2473 :
2474 : /* Catch square root. */
2475 44 : if (TREE_CODE (exp) == REAL_CST
2476 44 : && real_equal (&TREE_REAL_CST (exp), &dconsthalf))
2477 : {
2478 10 : *type_out = get_vectype_for_scalar_type (vinfo, TREE_TYPE (base));
2479 10 : if (*type_out
2480 10 : && direct_internal_fn_supported_p (IFN_SQRT, *type_out,
2481 : OPTIMIZE_FOR_SPEED))
2482 : {
2483 8 : gcall *stmt = gimple_build_call_internal (IFN_SQRT, 1, base);
2484 8 : var = vect_recog_temp_ssa_var (TREE_TYPE (base), stmt);
2485 8 : gimple_call_set_lhs (stmt, var);
2486 8 : gimple_call_set_nothrow (stmt, true);
2487 8 : return stmt;
2488 : }
2489 : }
2490 :
2491 : return NULL;
2492 : }
2493 :
2494 :
2495 : /* Function vect_recog_widen_sum_pattern
2496 :
2497 : Try to find the following pattern:
2498 :
2499 : type x_t;
2500 : TYPE x_T, sum = init;
2501 : loop:
2502 : sum_0 = phi <init, sum_1>
2503 : S1 x_t = *p;
2504 : S2 x_T = (TYPE) x_t;
2505 : S3 sum_1 = x_T + sum_0;
2506 :
2507 : where type 'TYPE' is at least double the size of type 'type', i.e - we're
2508 : summing elements of type 'type' into an accumulator of type 'TYPE'. This is
2509 : a special case of a reduction computation.
2510 :
2511 : Input:
2512 :
2513 : * STMT_VINFO: The stmt from which the pattern search begins. In the example,
2514 : when this function is called with S3, the pattern {S2,S3} will be detected.
2515 :
2516 : Output:
2517 :
2518 : * TYPE_OUT: The type of the output of this pattern.
2519 :
2520 : * Return value: A new stmt that will be used to replace the sequence of
2521 : stmts that constitute the pattern. In this case it will be:
2522 : WIDEN_SUM <x_t, sum_0>
2523 :
2524 : Note: The widening-sum idiom is a widening reduction pattern that is
2525 : vectorized without preserving all the intermediate results. It
2526 : produces only N/2 (widened) results (by summing up pairs of
2527 : intermediate results) rather than all N results. Therefore, we
2528 : cannot allow this pattern when we want to get all the results and in
2529 : the correct order (as is the case when this computation is in an
2530 : inner-loop nested in an outer-loop that us being vectorized). */
2531 :
2532 : static gimple *
2533 32410579 : vect_recog_widen_sum_pattern (vec_info *vinfo,
2534 : stmt_vec_info stmt_vinfo, tree *type_out)
2535 : {
2536 32410579 : gimple *last_stmt = stmt_vinfo->stmt;
2537 32410579 : tree oprnd0, oprnd1;
2538 32410579 : tree type;
2539 32410579 : gimple *pattern_stmt;
2540 32410579 : tree var;
2541 :
2542 : /* Look for the following pattern
2543 : DX = (TYPE) X;
2544 : sum_1 = DX + sum_0;
2545 : In which DX is at least double the size of X, and sum_1 has been
2546 : recognized as a reduction variable.
2547 : */
2548 :
2549 : /* Starting from LAST_STMT, follow the defs of its uses in search
2550 : of the above pattern. */
2551 :
2552 32410579 : if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, PLUS_EXPR,
2553 : &oprnd0, &oprnd1)
2554 52691 : || TREE_CODE (oprnd0) != SSA_NAME
2555 32463001 : || !vinfo->lookup_def (oprnd0))
2556 : return NULL;
2557 :
2558 52357 : type = TREE_TYPE (gimple_get_lhs (last_stmt));
2559 :
2560 : /* So far so good. Since last_stmt was detected as a (summation) reduction,
2561 : we know that oprnd1 is the reduction variable (defined by a loop-header
2562 : phi), and oprnd0 is an ssa-name defined by a stmt in the loop body.
2563 : Left to check that oprnd0 is defined by a cast from type 'type' to type
2564 : 'TYPE'. */
2565 :
2566 52357 : vect_unpromoted_value unprom0;
2567 52357 : if (!vect_look_through_possible_promotion (vinfo, oprnd0, &unprom0)
2568 52357 : || TYPE_PRECISION (unprom0.type) * 2 > TYPE_PRECISION (type))
2569 : return NULL;
2570 :
2571 2310 : vect_pattern_detected ("vect_recog_widen_sum_pattern", last_stmt);
2572 :
2573 : /* If the exact input type is not supported, widen it to an intermediate
2574 : type before adding it to the accumulator. Start with the narrowest
2575 : type in order to retain the largest vectorization factor. */
2576 2310 : tree input_type = unprom0.type;
2577 2310 : tree input_vectype = NULL_TREE;
2578 :
2579 4275 : while (!vect_supportable_conv_optab_p (vinfo, type, WIDEN_SUM_EXPR,
2580 : input_type, type_out, &input_vectype))
2581 : {
2582 4275 : unsigned int precision
2583 4275 : = vect_element_precision (TYPE_PRECISION (input_type) + 1);
2584 :
2585 4275 : if (precision > TYPE_PRECISION (type) / 2)
2586 : return NULL;
2587 :
2588 1965 : input_type = build_nonstandard_integer_type
2589 1965 : (precision, TYPE_UNSIGNED (unprom0.type));
2590 : }
2591 :
2592 0 : tree input = vect_convert_input (vinfo, stmt_vinfo, input_type, &unprom0,
2593 : input_vectype);
2594 :
2595 0 : var = vect_recog_temp_ssa_var (type, NULL);
2596 0 : pattern_stmt = gimple_build_assign (var, WIDEN_SUM_EXPR, input, oprnd1);
2597 :
2598 0 : return pattern_stmt;
2599 : }
2600 :
2601 : /* Function vect_recog_bitfield_ref_pattern
2602 :
2603 : Try to find the following pattern:
2604 :
2605 : bf_value = BIT_FIELD_REF (container, bitsize, bitpos);
2606 : result = (type_out) bf_value;
2607 :
2608 : or
2609 :
2610 : if (BIT_FIELD_REF (container, bitsize, bitpos) `cmp` <constant>)
2611 :
2612 : where type_out is a non-bitfield type, that is to say, it's precision matches
2613 : 2^(TYPE_SIZE(type_out) - (TYPE_UNSIGNED (type_out) ? 1 : 2)).
2614 :
2615 : Input:
2616 :
2617 : * STMT_VINFO: The stmt from which the pattern search begins.
2618 : here it starts with:
2619 : result = (type_out) bf_value;
2620 :
2621 : or
2622 :
2623 : if (BIT_FIELD_REF (container, bitsize, bitpos) `cmp` <constant>)
2624 :
2625 : Output:
2626 :
2627 : * TYPE_OUT: The vector type of the output of this pattern.
2628 :
2629 : * Return value: A new stmt that will be used to replace the sequence of
2630 : stmts that constitute the pattern. If the precision of type_out is bigger
2631 : than the precision type of _1 we perform the widening before the shifting,
2632 : since the new precision will be large enough to shift the value and moving
2633 : widening operations up the statement chain enables the generation of
2634 : widening loads. If we are widening and the operation after the pattern is
2635 : an addition then we mask first and shift later, to enable the generation of
2636 : shifting adds. In the case of narrowing we will always mask first, shift
2637 : last and then perform a narrowing operation. This will enable the
2638 : generation of narrowing shifts.
2639 :
2640 : Widening with mask first, shift later:
2641 : container = (type_out) container;
2642 : masked = container & (((1 << bitsize) - 1) << bitpos);
2643 : result = masked >> bitpos;
2644 :
2645 : Widening with shift first, mask last:
2646 : container = (type_out) container;
2647 : shifted = container >> bitpos;
2648 : result = shifted & ((1 << bitsize) - 1);
2649 :
2650 : Narrowing:
2651 : masked = container & (((1 << bitsize) - 1) << bitpos);
2652 : result = masked >> bitpos;
2653 : result = (type_out) result;
2654 :
2655 : If the bitfield is signed and it's wider than type_out, we need to
2656 : keep the result sign-extended:
2657 : container = (type) container;
2658 : masked = container << (prec - bitsize - bitpos);
2659 : result = (type_out) (masked >> (prec - bitsize));
2660 :
2661 : Here type is the signed variant of the wider of type_out and the type
2662 : of container.
2663 :
2664 : The shifting is always optional depending on whether bitpos != 0.
2665 :
2666 : When the original bitfield was inside a gcond then an new gcond is also
2667 : generated with the newly `result` as the operand to the comparison.
2668 :
2669 : */
2670 :
2671 : static gimple *
2672 32356018 : vect_recog_bitfield_ref_pattern (vec_info *vinfo, stmt_vec_info stmt_info,
2673 : tree *type_out)
2674 : {
2675 32356018 : gimple *bf_stmt = NULL;
2676 32356018 : tree lhs = NULL_TREE;
2677 32356018 : tree ret_type = NULL_TREE;
2678 32356018 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
2679 32356018 : if (gcond *cond_stmt = dyn_cast <gcond *> (stmt))
2680 : {
2681 5426315 : tree op = gimple_cond_lhs (cond_stmt);
2682 5426315 : if (TREE_CODE (op) != SSA_NAME)
2683 : return NULL;
2684 5426015 : bf_stmt = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (op));
2685 5426015 : if (TREE_CODE (gimple_cond_rhs (cond_stmt)) != INTEGER_CST)
2686 : return NULL;
2687 : }
2688 26929703 : else if (is_gimple_assign (stmt)
2689 22487155 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
2690 29952600 : && TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME)
2691 : {
2692 2981337 : gimple *second_stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmt));
2693 2981337 : bf_stmt = dyn_cast <gassign *> (second_stmt);
2694 2981337 : lhs = gimple_assign_lhs (stmt);
2695 2981337 : ret_type = TREE_TYPE (lhs);
2696 : }
2697 :
2698 6500380 : if (!bf_stmt
2699 6500380 : || gimple_assign_rhs_code (bf_stmt) != BIT_FIELD_REF)
2700 : return NULL;
2701 :
2702 16780 : tree bf_ref = gimple_assign_rhs1 (bf_stmt);
2703 16780 : tree container = TREE_OPERAND (bf_ref, 0);
2704 16780 : ret_type = ret_type ? ret_type : TREE_TYPE (container);
2705 :
2706 16780 : if (!bit_field_offset (bf_ref).is_constant ()
2707 16780 : || !bit_field_size (bf_ref).is_constant ()
2708 16780 : || !tree_fits_uhwi_p (TYPE_SIZE (TREE_TYPE (container))))
2709 : return NULL;
2710 :
2711 33182 : if (!INTEGRAL_TYPE_P (TREE_TYPE (bf_ref))
2712 16778 : || !INTEGRAL_TYPE_P (TREE_TYPE (container))
2713 18955 : || TYPE_MODE (TREE_TYPE (container)) == E_BLKmode)
2714 : return NULL;
2715 :
2716 2175 : gimple *use_stmt, *pattern_stmt;
2717 2175 : use_operand_p use_p;
2718 2175 : bool shift_first = true;
2719 2175 : tree container_type = TREE_TYPE (container);
2720 2175 : tree vectype = get_vectype_for_scalar_type (vinfo, container_type);
2721 :
2722 : /* Calculate shift_n before the adjustments for widening loads, otherwise
2723 : the container may change and we have to consider offset change for
2724 : widening loads on big endianness. The shift_n calculated here can be
2725 : independent of widening. */
2726 2175 : unsigned HOST_WIDE_INT shift_n = bit_field_offset (bf_ref).to_constant ();
2727 2175 : unsigned HOST_WIDE_INT mask_width = bit_field_size (bf_ref).to_constant ();
2728 2175 : unsigned HOST_WIDE_INT prec = tree_to_uhwi (TYPE_SIZE (container_type));
2729 2175 : if (BYTES_BIG_ENDIAN)
2730 : shift_n = prec - shift_n - mask_width;
2731 :
2732 2175 : bool ref_sext = (!TYPE_UNSIGNED (TREE_TYPE (bf_ref)) &&
2733 1400 : TYPE_PRECISION (ret_type) > mask_width);
2734 2175 : bool load_widen = (TYPE_PRECISION (TREE_TYPE (container)) <
2735 2175 : TYPE_PRECISION (ret_type));
2736 :
2737 : /* We move the conversion earlier if the loaded type is smaller than the
2738 : return type to enable the use of widening loads. And if we need a
2739 : sign extension, we need to convert the loaded value early to a signed
2740 : type as well. */
2741 2175 : if (ref_sext || load_widen)
2742 : {
2743 947 : tree type = load_widen ? ret_type : container_type;
2744 947 : if (ref_sext)
2745 908 : type = gimple_signed_type (type);
2746 947 : pattern_stmt = gimple_build_assign (vect_recog_temp_ssa_var (type),
2747 : NOP_EXPR, container);
2748 947 : container = gimple_get_lhs (pattern_stmt);
2749 947 : container_type = TREE_TYPE (container);
2750 947 : prec = tree_to_uhwi (TYPE_SIZE (container_type));
2751 947 : vectype = get_vectype_for_scalar_type (vinfo, container_type);
2752 947 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2753 : }
2754 1228 : else if (!useless_type_conversion_p (TREE_TYPE (container), ret_type))
2755 : /* If we are doing the conversion last then also delay the shift as we may
2756 : be able to combine the shift and conversion in certain cases. */
2757 : shift_first = false;
2758 :
2759 : /* If the only use of the result of this BIT_FIELD_REF + CONVERT is a
2760 : PLUS_EXPR then do the shift last as some targets can combine the shift and
2761 : add into a single instruction. */
2762 1416 : if (lhs && !is_pattern_stmt_p (stmt_info)
2763 3591 : && single_imm_use (lhs, &use_p, &use_stmt))
2764 : {
2765 1049 : if (gimple_code (use_stmt) == GIMPLE_ASSIGN
2766 1049 : && gimple_assign_rhs_code (use_stmt) == PLUS_EXPR)
2767 : shift_first = false;
2768 : }
2769 :
2770 : /* If we don't have to shift we only generate the mask, so just fix the
2771 : code-path to shift_first. */
2772 2175 : if (shift_n == 0)
2773 765 : shift_first = true;
2774 :
2775 2175 : tree result;
2776 2175 : if (shift_first && !ref_sext)
2777 : {
2778 515 : tree shifted = container;
2779 515 : if (shift_n)
2780 : {
2781 65 : pattern_stmt
2782 65 : = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
2783 : RSHIFT_EXPR, container,
2784 65 : build_int_cst (sizetype, shift_n));
2785 65 : shifted = gimple_assign_lhs (pattern_stmt);
2786 65 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2787 : }
2788 :
2789 515 : tree mask = wide_int_to_tree (container_type,
2790 515 : wi::mask (mask_width, false, prec));
2791 :
2792 515 : pattern_stmt
2793 515 : = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
2794 : BIT_AND_EXPR, shifted, mask);
2795 515 : result = gimple_assign_lhs (pattern_stmt);
2796 : }
2797 : else
2798 : {
2799 1660 : tree temp = vect_recog_temp_ssa_var (container_type);
2800 1660 : if (!ref_sext)
2801 : {
2802 752 : tree mask = wide_int_to_tree (container_type,
2803 752 : wi::shifted_mask (shift_n,
2804 : mask_width,
2805 : false, prec));
2806 752 : pattern_stmt = gimple_build_assign (temp, BIT_AND_EXPR,
2807 : container, mask);
2808 : }
2809 : else
2810 : {
2811 908 : HOST_WIDE_INT shl = prec - shift_n - mask_width;
2812 908 : shift_n += shl;
2813 908 : pattern_stmt = gimple_build_assign (temp, LSHIFT_EXPR,
2814 : container,
2815 : build_int_cst (sizetype,
2816 908 : shl));
2817 : }
2818 :
2819 1660 : tree masked = gimple_assign_lhs (pattern_stmt);
2820 1660 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2821 1660 : pattern_stmt
2822 1660 : = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
2823 : RSHIFT_EXPR, masked,
2824 1660 : build_int_cst (sizetype, shift_n));
2825 1660 : result = gimple_assign_lhs (pattern_stmt);
2826 : }
2827 :
2828 2175 : if (!useless_type_conversion_p (TREE_TYPE (result), ret_type))
2829 : {
2830 1444 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2831 1444 : pattern_stmt
2832 1444 : = gimple_build_assign (vect_recog_temp_ssa_var (ret_type),
2833 : NOP_EXPR, result);
2834 : }
2835 :
2836 2175 : if (!lhs)
2837 : {
2838 759 : if (!vectype)
2839 : return NULL;
2840 :
2841 621 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2842 621 : vectype = truth_type_for (vectype);
2843 :
2844 : /* FIXME: This part extracts the boolean value out of the bitfield in the
2845 : same way as vect_recog_gcond_pattern does. However because
2846 : patterns cannot match the same root twice, when we handle and
2847 : lower the bitfield in the gcond, vect_recog_gcond_pattern can't
2848 : apply anymore. We should really fix it so that we don't need to
2849 : duplicate transformations like these. */
2850 621 : tree new_lhs = vect_recog_temp_ssa_var (boolean_type_node, NULL);
2851 621 : gcond *cond_stmt = dyn_cast <gcond *> (stmt_info->stmt);
2852 621 : tree cond_cst = gimple_cond_rhs (cond_stmt);
2853 621 : gimple *new_stmt
2854 621 : = gimple_build_assign (new_lhs, gimple_cond_code (cond_stmt),
2855 : gimple_get_lhs (pattern_stmt),
2856 : fold_convert (container_type, cond_cst));
2857 621 : append_pattern_def_seq (vinfo, stmt_info, new_stmt, vectype, container_type);
2858 621 : pattern_stmt
2859 621 : = gimple_build_cond (NE_EXPR, new_lhs,
2860 621 : build_zero_cst (TREE_TYPE (new_lhs)),
2861 : NULL_TREE, NULL_TREE);
2862 : }
2863 :
2864 2037 : *type_out = STMT_VINFO_VECTYPE (stmt_info);
2865 2037 : vect_pattern_detected ("bitfield_ref pattern", stmt_info->stmt);
2866 :
2867 2037 : return pattern_stmt;
2868 : }
2869 :
2870 : /* Function vect_recog_bit_insert_pattern
2871 :
2872 : Try to find the following pattern:
2873 :
2874 : written = BIT_INSERT_EXPR (container, value, bitpos);
2875 :
2876 : Input:
2877 :
2878 : * STMT_VINFO: The stmt we want to replace.
2879 :
2880 : Output:
2881 :
2882 : * TYPE_OUT: The vector type of the output of this pattern.
2883 :
2884 : * Return value: A new stmt that will be used to replace the sequence of
2885 : stmts that constitute the pattern. In this case it will be:
2886 : value = (container_type) value; // Make sure
2887 : shifted = value << bitpos; // Shift value into place
2888 : masked = shifted & (mask << bitpos); // Mask off the non-relevant bits in
2889 : // the 'to-write value'.
2890 : cleared = container & ~(mask << bitpos); // Clearing the bits we want to
2891 : // write to from the value we want
2892 : // to write to.
2893 : written = cleared | masked; // Write bits.
2894 :
2895 :
2896 : where mask = ((1 << TYPE_PRECISION (value)) - 1), a mask to keep the number of
2897 : bits corresponding to the real size of the bitfield value we are writing to.
2898 : The shifting is always optional depending on whether bitpos != 0.
2899 :
2900 : */
2901 :
2902 : static gimple *
2903 32359070 : vect_recog_bit_insert_pattern (vec_info *vinfo, stmt_vec_info stmt_info,
2904 : tree *type_out)
2905 : {
2906 32359070 : gassign *bf_stmt = dyn_cast <gassign *> (stmt_info->stmt);
2907 30023741 : if (!bf_stmt || gimple_assign_rhs_code (bf_stmt) != BIT_INSERT_EXPR)
2908 : return NULL;
2909 :
2910 672 : tree container = gimple_assign_rhs1 (bf_stmt);
2911 672 : tree value = gimple_assign_rhs2 (bf_stmt);
2912 672 : tree shift = gimple_assign_rhs3 (bf_stmt);
2913 :
2914 672 : tree bf_type = TREE_TYPE (value);
2915 672 : tree container_type = TREE_TYPE (container);
2916 :
2917 672 : if (!INTEGRAL_TYPE_P (container_type)
2918 672 : || !tree_fits_uhwi_p (TYPE_SIZE (container_type)))
2919 : return NULL;
2920 :
2921 500 : gimple *pattern_stmt;
2922 :
2923 500 : vect_unpromoted_value unprom;
2924 500 : unprom.set_op (value, vect_internal_def);
2925 500 : value = vect_convert_input (vinfo, stmt_info, container_type, &unprom,
2926 : get_vectype_for_scalar_type (vinfo,
2927 : container_type));
2928 :
2929 500 : unsigned HOST_WIDE_INT mask_width = TYPE_PRECISION (bf_type);
2930 500 : unsigned HOST_WIDE_INT prec = tree_to_uhwi (TYPE_SIZE (container_type));
2931 500 : unsigned HOST_WIDE_INT shift_n = tree_to_uhwi (shift);
2932 500 : if (BYTES_BIG_ENDIAN)
2933 : {
2934 : shift_n = prec - shift_n - mask_width;
2935 : shift = build_int_cst (TREE_TYPE (shift), shift_n);
2936 : }
2937 :
2938 500 : if (!useless_type_conversion_p (TREE_TYPE (value), container_type))
2939 : {
2940 0 : pattern_stmt =
2941 0 : gimple_build_assign (vect_recog_temp_ssa_var (container_type),
2942 : NOP_EXPR, value);
2943 0 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt);
2944 0 : value = gimple_get_lhs (pattern_stmt);
2945 : }
2946 :
2947 : /* Shift VALUE into place. */
2948 500 : tree shifted = value;
2949 500 : if (shift_n)
2950 : {
2951 249 : gimple_seq stmts = NULL;
2952 249 : shifted
2953 249 : = gimple_build (&stmts, LSHIFT_EXPR, container_type, value, shift);
2954 249 : if (!gimple_seq_empty_p (stmts))
2955 112 : append_pattern_def_seq (vinfo, stmt_info,
2956 : gimple_seq_first_stmt (stmts));
2957 : }
2958 :
2959 500 : tree mask_t
2960 500 : = wide_int_to_tree (container_type,
2961 500 : wi::shifted_mask (shift_n, mask_width, false, prec));
2962 :
2963 : /* Clear bits we don't want to write back from SHIFTED. */
2964 500 : gimple_seq stmts = NULL;
2965 500 : tree masked = gimple_build (&stmts, BIT_AND_EXPR, container_type, shifted,
2966 : mask_t);
2967 500 : if (!gimple_seq_empty_p (stmts))
2968 : {
2969 110 : pattern_stmt = gimple_seq_first_stmt (stmts);
2970 110 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt);
2971 : }
2972 :
2973 : /* Mask off the bits in the container that we are to write to. */
2974 500 : mask_t = wide_int_to_tree (container_type,
2975 500 : wi::shifted_mask (shift_n, mask_width, true, prec));
2976 500 : tree cleared = vect_recog_temp_ssa_var (container_type);
2977 500 : pattern_stmt = gimple_build_assign (cleared, BIT_AND_EXPR, container, mask_t);
2978 500 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt);
2979 :
2980 : /* Write MASKED into CLEARED. */
2981 500 : pattern_stmt
2982 500 : = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
2983 : BIT_IOR_EXPR, cleared, masked);
2984 :
2985 500 : *type_out = STMT_VINFO_VECTYPE (stmt_info);
2986 500 : vect_pattern_detected ("bit_insert pattern", stmt_info->stmt);
2987 :
2988 500 : return pattern_stmt;
2989 : }
2990 :
2991 :
2992 : /* Recognize cases in which an operation is performed in one type WTYPE
2993 : but could be done more efficiently in a narrower type NTYPE. For example,
2994 : if we have:
2995 :
2996 : ATYPE a; // narrower than NTYPE
2997 : BTYPE b; // narrower than NTYPE
2998 : WTYPE aw = (WTYPE) a;
2999 : WTYPE bw = (WTYPE) b;
3000 : WTYPE res = aw + bw; // only uses of aw and bw
3001 :
3002 : then it would be more efficient to do:
3003 :
3004 : NTYPE an = (NTYPE) a;
3005 : NTYPE bn = (NTYPE) b;
3006 : NTYPE resn = an + bn;
3007 : WTYPE res = (WTYPE) resn;
3008 :
3009 : Other situations include things like:
3010 :
3011 : ATYPE a; // NTYPE or narrower
3012 : WTYPE aw = (WTYPE) a;
3013 : WTYPE res = aw + b;
3014 :
3015 : when only "(NTYPE) res" is significant. In that case it's more efficient
3016 : to truncate "b" and do the operation on NTYPE instead:
3017 :
3018 : NTYPE an = (NTYPE) a;
3019 : NTYPE bn = (NTYPE) b; // truncation
3020 : NTYPE resn = an + bn;
3021 : WTYPE res = (WTYPE) resn;
3022 :
3023 : All users of "res" should then use "resn" instead, making the final
3024 : statement dead (not marked as relevant). The final statement is still
3025 : needed to maintain the type correctness of the IR.
3026 :
3027 : vect_determine_precisions has already determined the minimum
3028 : precision of the operation and the minimum precision required
3029 : by users of the result. */
3030 :
3031 : static gimple *
3032 32359510 : vect_recog_over_widening_pattern (vec_info *vinfo,
3033 : stmt_vec_info last_stmt_info, tree *type_out)
3034 : {
3035 32359510 : gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
3036 22491268 : if (!last_stmt)
3037 : return NULL;
3038 :
3039 : /* See whether we have found that this operation can be done on a
3040 : narrower type without changing its semantics. */
3041 22491268 : unsigned int new_precision = last_stmt_info->operation_precision;
3042 22491268 : if (!new_precision)
3043 : return NULL;
3044 :
3045 1666578 : tree lhs = gimple_assign_lhs (last_stmt);
3046 1666578 : tree type = TREE_TYPE (lhs);
3047 1666578 : tree_code code = gimple_assign_rhs_code (last_stmt);
3048 :
3049 : /* Punt for reductions where we don't handle the type conversions. */
3050 1666578 : if (vect_is_reduction (last_stmt_info))
3051 : return NULL;
3052 :
3053 : /* Keep the first operand of a COND_EXPR as-is: only the other two
3054 : operands are interesting. */
3055 1658250 : unsigned int first_op = (code == COND_EXPR ? 2 : 1);
3056 :
3057 : /* Check the operands. */
3058 1658250 : unsigned int nops = gimple_num_ops (last_stmt) - first_op;
3059 1658250 : auto_vec <vect_unpromoted_value, 3> unprom (nops);
3060 1658250 : unprom.quick_grow_cleared (nops);
3061 1658250 : unsigned int min_precision = 0;
3062 1658250 : bool single_use_p = false;
3063 6612736 : for (unsigned int i = 0; i < nops; ++i)
3064 : {
3065 3297941 : tree op = gimple_op (last_stmt, first_op + i);
3066 3297941 : if (TREE_CODE (op) == INTEGER_CST)
3067 1508431 : unprom[i].set_op (op, vect_constant_def);
3068 1789510 : else if (TREE_CODE (op) == SSA_NAME)
3069 : {
3070 1789510 : bool op_single_use_p = true;
3071 1789510 : if (!vect_look_through_possible_promotion (vinfo, op, &unprom[i],
3072 : &op_single_use_p))
3073 1705 : return NULL;
3074 : /* If:
3075 :
3076 : (1) N bits of the result are needed;
3077 : (2) all inputs are widened from M<N bits; and
3078 : (3) one operand OP is a single-use SSA name
3079 :
3080 : we can shift the M->N widening from OP to the output
3081 : without changing the number or type of extensions involved.
3082 : This then reduces the number of copies of STMT_INFO.
3083 :
3084 : If instead of (3) more than one operand is a single-use SSA name,
3085 : shifting the extension to the output is even more of a win.
3086 :
3087 : If instead:
3088 :
3089 : (1) N bits of the result are needed;
3090 : (2) one operand OP2 is widened from M2<N bits;
3091 : (3) another operand OP1 is widened from M1<M2 bits; and
3092 : (4) both OP1 and OP2 are single-use
3093 :
3094 : the choice is between:
3095 :
3096 : (a) truncating OP2 to M1, doing the operation on M1,
3097 : and then widening the result to N
3098 :
3099 : (b) widening OP1 to M2, doing the operation on M2, and then
3100 : widening the result to N
3101 :
3102 : Both shift the M2->N widening of the inputs to the output.
3103 : (a) additionally shifts the M1->M2 widening to the output;
3104 : it requires fewer copies of STMT_INFO but requires an extra
3105 : M2->M1 truncation.
3106 :
3107 : Which is better will depend on the complexity and cost of
3108 : STMT_INFO, which is hard to predict at this stage. However,
3109 : a clear tie-breaker in favor of (b) is the fact that the
3110 : truncation in (a) increases the length of the operation chain.
3111 :
3112 : If instead of (4) only one of OP1 or OP2 is single-use,
3113 : (b) is still a win over doing the operation in N bits:
3114 : it still shifts the M2->N widening on the single-use operand
3115 : to the output and reduces the number of STMT_INFO copies.
3116 :
3117 : If neither operand is single-use then operating on fewer than
3118 : N bits might lead to more extensions overall. Whether it does
3119 : or not depends on global information about the vectorization
3120 : region, and whether that's a good trade-off would again
3121 : depend on the complexity and cost of the statements involved,
3122 : as well as things like register pressure that are not normally
3123 : modelled at this stage. We therefore ignore these cases
3124 : and just optimize the clear single-use wins above.
3125 :
3126 : Thus we take the maximum precision of the unpromoted operands
3127 : and record whether any operand is single-use. */
3128 1787805 : if (unprom[i].dt == vect_internal_def)
3129 : {
3130 1029835 : min_precision = MAX (min_precision,
3131 : TYPE_PRECISION (unprom[i].type));
3132 1029835 : single_use_p |= op_single_use_p;
3133 : }
3134 : }
3135 : else
3136 : return NULL;
3137 : }
3138 :
3139 : /* Although the operation could be done in operation_precision, we have
3140 : to balance that against introducing extra truncations or extensions.
3141 : Calculate the minimum precision that can be handled efficiently.
3142 :
3143 : The loop above determined that the operation could be handled
3144 : efficiently in MIN_PRECISION if SINGLE_USE_P; this would shift an
3145 : extension from the inputs to the output without introducing more
3146 : instructions, and would reduce the number of instructions required
3147 : for STMT_INFO itself.
3148 :
3149 : vect_determine_precisions has also determined that the result only
3150 : needs min_output_precision bits. Truncating by a factor of N times
3151 : requires a tree of N - 1 instructions, so if TYPE is N times wider
3152 : than min_output_precision, doing the operation in TYPE and truncating
3153 : the result requires N + (N - 1) = 2N - 1 instructions per output vector.
3154 : In contrast:
3155 :
3156 : - truncating the input to a unary operation and doing the operation
3157 : in the new type requires at most N - 1 + 1 = N instructions per
3158 : output vector
3159 :
3160 : - doing the same for a binary operation requires at most
3161 : (N - 1) * 2 + 1 = 2N - 1 instructions per output vector
3162 :
3163 : Both unary and binary operations require fewer instructions than
3164 : this if the operands were extended from a suitable truncated form.
3165 : Thus there is usually nothing to lose by doing operations in
3166 : min_output_precision bits, but there can be something to gain. */
3167 1656545 : if (!single_use_p)
3168 1305617 : min_precision = last_stmt_info->min_output_precision;
3169 : else
3170 350928 : min_precision = MIN (min_precision, last_stmt_info->min_output_precision);
3171 :
3172 : /* Apply the minimum efficient precision we just calculated. */
3173 1656545 : if (new_precision < min_precision)
3174 : new_precision = min_precision;
3175 1656545 : new_precision = vect_element_precision (new_precision);
3176 1656545 : if (new_precision >= TYPE_PRECISION (type))
3177 : return NULL;
3178 :
3179 155955 : vect_pattern_detected ("vect_recog_over_widening_pattern", last_stmt);
3180 :
3181 155955 : *type_out = get_vectype_for_scalar_type (vinfo, type);
3182 155955 : if (!*type_out)
3183 : return NULL;
3184 :
3185 : /* We've found a viable pattern. Get the new type of the operation. */
3186 137888 : bool unsigned_p = (last_stmt_info->operation_sign == UNSIGNED);
3187 137888 : tree new_type = build_nonstandard_integer_type (new_precision, unsigned_p);
3188 :
3189 : /* If we're truncating an operation, we need to make sure that we
3190 : don't introduce new undefined overflow. The codes tested here are
3191 : a subset of those accepted by vect_truncatable_operation_p. */
3192 137888 : tree op_type = new_type;
3193 137888 : if (TYPE_OVERFLOW_UNDEFINED (new_type)
3194 178306 : && (code == PLUS_EXPR || code == MINUS_EXPR || code == MULT_EXPR))
3195 27701 : op_type = build_nonstandard_integer_type (new_precision, true);
3196 :
3197 137888 : tree new_vectype = get_vectype_for_scalar_type (vinfo, new_type);
3198 137888 : tree op_vectype = get_vectype_for_scalar_type (vinfo, op_type);
3199 137888 : if (!new_vectype || !op_vectype)
3200 : return NULL;
3201 :
3202 : /* Verify we can handle the new operation. For shifts and rotates
3203 : apply heuristic of whether we are likely facing vector-vector or
3204 : vector-scalar operation. Since we are eventually expecting that
3205 : a later pattern might eventually want to rewrite an unsupported
3206 : into a supported case error on that side in case the original
3207 : operation was not supported either or this is a binary operation
3208 : and the 2nd operand is constant. */
3209 137888 : if (code == RSHIFT_EXPR || code == LSHIFT_EXPR || code == RROTATE_EXPR)
3210 : {
3211 29158 : if (!target_has_vecop_for_code (code, op_vectype, optab_vector)
3212 28099 : && ((unprom[1].dt != vect_external_def
3213 27687 : && unprom[1].dt != vect_constant_def)
3214 19228 : || !target_has_vecop_for_code (code, op_vectype, optab_scalar))
3215 38058 : && !(!target_has_vecop_for_code (code, *type_out, optab_vector)
3216 7802 : && ((unprom[1].dt != vect_external_def
3217 7802 : || unprom[1].dt != vect_constant_def)
3218 : || !target_has_vecop_for_code (code, *type_out,
3219 : optab_scalar))))
3220 : return NULL;
3221 : }
3222 108730 : else if (!target_has_vecop_for_code (code, op_vectype, optab_vector)
3223 108730 : && (target_has_vecop_for_code (code, *type_out, optab_vector)
3224 29 : && !(nops == 2 && unprom[1].dt == vect_constant_def)))
3225 : return NULL;
3226 :
3227 136780 : if (dump_enabled_p ())
3228 4341 : dump_printf_loc (MSG_NOTE, vect_location, "demoting %T to %T\n",
3229 : type, new_type);
3230 :
3231 : /* Calculate the rhs operands for an operation on OP_TYPE. */
3232 136780 : tree ops[3] = {};
3233 137009 : for (unsigned int i = 1; i < first_op; ++i)
3234 229 : ops[i - 1] = gimple_op (last_stmt, i);
3235 136780 : vect_convert_inputs (vinfo, last_stmt_info, nops, &ops[first_op - 1],
3236 136780 : op_type, &unprom[0], op_vectype);
3237 :
3238 : /* Use the operation to produce a result of type OP_TYPE. */
3239 136780 : tree new_var = vect_recog_temp_ssa_var (op_type, NULL);
3240 136780 : gimple *pattern_stmt = gimple_build_assign (new_var, code,
3241 : ops[0], ops[1], ops[2]);
3242 136780 : gimple_set_location (pattern_stmt, gimple_location (last_stmt));
3243 :
3244 136780 : if (dump_enabled_p ())
3245 4341 : dump_printf_loc (MSG_NOTE, vect_location,
3246 : "created pattern stmt: %G", pattern_stmt);
3247 :
3248 : /* Convert back to the original signedness, if OP_TYPE is different
3249 : from NEW_TYPE. */
3250 136780 : if (op_type != new_type)
3251 27694 : pattern_stmt = vect_convert_output (vinfo, last_stmt_info, new_type,
3252 : pattern_stmt, op_vectype);
3253 :
3254 : /* Promote the result to the original type. */
3255 136780 : pattern_stmt = vect_convert_output (vinfo, last_stmt_info, type,
3256 : pattern_stmt, new_vectype);
3257 :
3258 136780 : return pattern_stmt;
3259 1658250 : }
3260 :
3261 : /* Recognize the following patterns:
3262 :
3263 : ATYPE a; // narrower than TYPE
3264 : BTYPE b; // narrower than TYPE
3265 :
3266 : 1) Multiply high with scaling
3267 : TYPE res = ((TYPE) a * (TYPE) b) >> c;
3268 : Here, c is bitsize (TYPE) / 2 - 1.
3269 :
3270 : 2) ... or also with rounding
3271 : TYPE res = (((TYPE) a * (TYPE) b) >> d + 1) >> 1;
3272 : Here, d is bitsize (TYPE) / 2 - 2.
3273 :
3274 : 3) Normal multiply high
3275 : TYPE res = ((TYPE) a * (TYPE) b) >> e;
3276 : Here, e is bitsize (TYPE) / 2.
3277 :
3278 : where only the bottom half of res is used. */
3279 :
3280 : static gimple *
3281 32488905 : vect_recog_mulhs_pattern (vec_info *vinfo,
3282 : stmt_vec_info last_stmt_info, tree *type_out)
3283 : {
3284 : /* Check for a right shift. */
3285 32488905 : gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
3286 22620505 : if (!last_stmt
3287 22620505 : || gimple_assign_rhs_code (last_stmt) != RSHIFT_EXPR)
3288 : return NULL;
3289 :
3290 : /* Check that the shift result is wider than the users of the
3291 : result need (i.e. that narrowing would be a natural choice). */
3292 352374 : tree lhs_type = TREE_TYPE (gimple_assign_lhs (last_stmt));
3293 352374 : unsigned int target_precision
3294 352374 : = vect_element_precision (last_stmt_info->min_output_precision);
3295 352374 : if (!INTEGRAL_TYPE_P (lhs_type)
3296 352374 : || target_precision >= TYPE_PRECISION (lhs_type))
3297 : return NULL;
3298 :
3299 : /* Look through any change in sign on the outer shift input. */
3300 61401 : vect_unpromoted_value unprom_rshift_input;
3301 61401 : tree rshift_input = vect_look_through_possible_promotion
3302 61401 : (vinfo, gimple_assign_rhs1 (last_stmt), &unprom_rshift_input);
3303 61401 : if (!rshift_input
3304 61401 : || TYPE_PRECISION (TREE_TYPE (rshift_input))
3305 60803 : != TYPE_PRECISION (lhs_type))
3306 : return NULL;
3307 :
3308 : /* Get the definition of the shift input. */
3309 56508 : stmt_vec_info rshift_input_stmt_info
3310 56508 : = vect_get_internal_def (vinfo, rshift_input);
3311 56508 : if (!rshift_input_stmt_info)
3312 : return NULL;
3313 51845 : gassign *rshift_input_stmt
3314 51845 : = dyn_cast <gassign *> (rshift_input_stmt_info->stmt);
3315 48512 : if (!rshift_input_stmt)
3316 : return NULL;
3317 :
3318 48512 : stmt_vec_info mulh_stmt_info;
3319 48512 : tree scale_term;
3320 48512 : bool rounding_p = false;
3321 :
3322 : /* Check for the presence of the rounding term. */
3323 55758 : if (gimple_assign_rhs_code (rshift_input_stmt) == PLUS_EXPR)
3324 : {
3325 : /* Check that the outer shift was by 1. */
3326 18686 : if (!integer_onep (gimple_assign_rhs2 (last_stmt)))
3327 9276 : return NULL;
3328 :
3329 : /* Check that the second operand of the PLUS_EXPR is 1. */
3330 1310 : if (!integer_onep (gimple_assign_rhs2 (rshift_input_stmt)))
3331 : return NULL;
3332 :
3333 : /* Look through any change in sign on the addition input. */
3334 110 : vect_unpromoted_value unprom_plus_input;
3335 110 : tree plus_input = vect_look_through_possible_promotion
3336 110 : (vinfo, gimple_assign_rhs1 (rshift_input_stmt), &unprom_plus_input);
3337 110 : if (!plus_input
3338 110 : || TYPE_PRECISION (TREE_TYPE (plus_input))
3339 110 : != TYPE_PRECISION (TREE_TYPE (rshift_input)))
3340 : return NULL;
3341 :
3342 : /* Get the definition of the multiply-high-scale part. */
3343 110 : stmt_vec_info plus_input_stmt_info
3344 110 : = vect_get_internal_def (vinfo, plus_input);
3345 110 : if (!plus_input_stmt_info)
3346 : return NULL;
3347 110 : gassign *plus_input_stmt
3348 9386 : = dyn_cast <gassign *> (plus_input_stmt_info->stmt);
3349 110 : if (!plus_input_stmt
3350 110 : || gimple_assign_rhs_code (plus_input_stmt) != RSHIFT_EXPR)
3351 : return NULL;
3352 :
3353 : /* Look through any change in sign on the scaling input. */
3354 67 : vect_unpromoted_value unprom_scale_input;
3355 67 : tree scale_input = vect_look_through_possible_promotion
3356 67 : (vinfo, gimple_assign_rhs1 (plus_input_stmt), &unprom_scale_input);
3357 67 : if (!scale_input
3358 67 : || TYPE_PRECISION (TREE_TYPE (scale_input))
3359 67 : != TYPE_PRECISION (TREE_TYPE (plus_input)))
3360 : return NULL;
3361 :
3362 : /* Get the definition of the multiply-high part. */
3363 67 : mulh_stmt_info = vect_get_internal_def (vinfo, scale_input);
3364 67 : if (!mulh_stmt_info)
3365 : return NULL;
3366 :
3367 : /* Get the scaling term. */
3368 67 : scale_term = gimple_assign_rhs2 (plus_input_stmt);
3369 67 : rounding_p = true;
3370 : }
3371 : else
3372 : {
3373 39169 : mulh_stmt_info = rshift_input_stmt_info;
3374 39169 : scale_term = gimple_assign_rhs2 (last_stmt);
3375 : }
3376 :
3377 : /* Check that the scaling factor is constant. */
3378 39236 : if (TREE_CODE (scale_term) != INTEGER_CST)
3379 : return NULL;
3380 :
3381 : /* Check whether the scaling input term can be seen as two widened
3382 : inputs multiplied together. */
3383 114753 : vect_unpromoted_value unprom_mult[2];
3384 38251 : tree new_type;
3385 38251 : unsigned int nops
3386 38251 : = vect_widened_op_tree (vinfo, mulh_stmt_info, MULT_EXPR, WIDEN_MULT_EXPR,
3387 : false, 2, unprom_mult, &new_type);
3388 38251 : if (nops != 2)
3389 : return NULL;
3390 :
3391 : /* Adjust output precision. */
3392 8858 : if (TYPE_PRECISION (new_type) < target_precision)
3393 0 : new_type = build_nonstandard_integer_type
3394 0 : (target_precision, TYPE_UNSIGNED (new_type));
3395 :
3396 8858 : unsigned mult_precision = TYPE_PRECISION (new_type);
3397 8858 : internal_fn ifn;
3398 : /* Check that the scaling factor is expected. Instead of
3399 : target_precision, we should use the one that we actually
3400 : use for internal function. */
3401 8858 : if (rounding_p)
3402 : {
3403 : /* Check pattern 2). */
3404 134 : if (wi::to_widest (scale_term) + mult_precision + 2
3405 201 : != TYPE_PRECISION (lhs_type))
3406 : return NULL;
3407 :
3408 : ifn = IFN_MULHRS;
3409 : }
3410 : else
3411 : {
3412 : /* Check for pattern 1). */
3413 17582 : if (wi::to_widest (scale_term) + mult_precision + 1
3414 26373 : == TYPE_PRECISION (lhs_type))
3415 : ifn = IFN_MULHS;
3416 : /* Check for pattern 3). */
3417 8757 : else if (wi::to_widest (scale_term) + mult_precision
3418 17514 : == TYPE_PRECISION (lhs_type))
3419 : ifn = IFN_MULH;
3420 : else
3421 : return NULL;
3422 : }
3423 :
3424 8793 : vect_pattern_detected ("vect_recog_mulhs_pattern", last_stmt);
3425 :
3426 : /* Check for target support. */
3427 8793 : tree new_vectype = get_vectype_for_scalar_type (vinfo, new_type);
3428 8793 : if (!new_vectype
3429 15500 : || !direct_internal_fn_supported_p
3430 6707 : (ifn, new_vectype, OPTIMIZE_FOR_SPEED))
3431 : return NULL;
3432 :
3433 : /* The result is cast back to LHS_TYPE, a cast that the over-widening
3434 : machinery then removes. LHS_TYPE need not have a vector type, as for the
3435 : 128-bit product of a 64-bit high-part multiply, so leave *TYPE_OUT null
3436 : rather than giving up. */
3437 144 : *type_out = get_vectype_for_scalar_type (vinfo, lhs_type);
3438 :
3439 : /* Generate the IFN_MULHRS call. */
3440 144 : tree new_var = vect_recog_temp_ssa_var (new_type, NULL);
3441 144 : tree new_ops[2];
3442 144 : vect_convert_inputs (vinfo, last_stmt_info, 2, new_ops, new_type,
3443 : unprom_mult, new_vectype);
3444 144 : gcall *mulhrs_stmt
3445 144 : = gimple_build_call_internal (ifn, 2, new_ops[0], new_ops[1]);
3446 144 : gimple_call_set_lhs (mulhrs_stmt, new_var);
3447 144 : gimple_set_location (mulhrs_stmt, gimple_location (last_stmt));
3448 :
3449 144 : if (dump_enabled_p ())
3450 0 : dump_printf_loc (MSG_NOTE, vect_location,
3451 : "created pattern stmt: %G", (gimple *) mulhrs_stmt);
3452 :
3453 144 : return vect_convert_output (vinfo, last_stmt_info, lhs_type,
3454 144 : mulhrs_stmt, new_vectype);
3455 : }
3456 :
3457 : /* Recognize the patterns:
3458 :
3459 : ATYPE a; // narrower than TYPE
3460 : BTYPE b; // narrower than TYPE
3461 : (1) TYPE avg = ((TYPE) a + (TYPE) b) >> 1;
3462 : or (2) TYPE avg = ((TYPE) a + (TYPE) b + 1) >> 1;
3463 :
3464 : where only the bottom half of avg is used. Try to transform them into:
3465 :
3466 : (1) NTYPE avg' = .AVG_FLOOR ((NTYPE) a, (NTYPE) b);
3467 : or (2) NTYPE avg' = .AVG_CEIL ((NTYPE) a, (NTYPE) b);
3468 :
3469 : followed by:
3470 :
3471 : TYPE avg = (TYPE) avg';
3472 :
3473 : where NTYPE is no wider than half of TYPE. Since only the bottom half
3474 : of avg is used, all or part of the cast of avg' should become redundant.
3475 :
3476 : If there is no target support available, generate code to distribute rshift
3477 : over plus and add a carry. */
3478 :
3479 : static gimple *
3480 32487271 : vect_recog_average_pattern (vec_info *vinfo,
3481 : stmt_vec_info last_stmt_info, tree *type_out)
3482 : {
3483 : /* Check for a shift right by one bit. */
3484 32487271 : gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
3485 22619029 : if (!last_stmt
3486 22619029 : || gimple_assign_rhs_code (last_stmt) != RSHIFT_EXPR
3487 352280 : || !integer_onep (gimple_assign_rhs2 (last_stmt)))
3488 : return NULL;
3489 :
3490 : /* Check that the shift result is wider than the users of the
3491 : result need (i.e. that narrowing would be a natural choice). */
3492 57072 : tree lhs = gimple_assign_lhs (last_stmt);
3493 57072 : tree type = TREE_TYPE (lhs);
3494 57072 : unsigned int target_precision
3495 57072 : = vect_element_precision (last_stmt_info->min_output_precision);
3496 57072 : if (!INTEGRAL_TYPE_P (type) || target_precision >= TYPE_PRECISION (type))
3497 : return NULL;
3498 :
3499 : /* Look through any change in sign on the shift input. */
3500 2210 : tree rshift_rhs = gimple_assign_rhs1 (last_stmt);
3501 2210 : vect_unpromoted_value unprom_plus;
3502 2210 : rshift_rhs = vect_look_through_possible_promotion (vinfo, rshift_rhs,
3503 : &unprom_plus);
3504 2210 : if (!rshift_rhs
3505 2210 : || TYPE_PRECISION (TREE_TYPE (rshift_rhs)) != TYPE_PRECISION (type))
3506 : return NULL;
3507 :
3508 : /* Get the definition of the shift input. */
3509 2208 : stmt_vec_info plus_stmt_info = vect_get_internal_def (vinfo, rshift_rhs);
3510 2208 : if (!plus_stmt_info)
3511 : return NULL;
3512 :
3513 : /* Check whether the shift input can be seen as a tree of additions on
3514 : 2 or 3 widened inputs.
3515 :
3516 : Note that the pattern should be a win even if the result of one or
3517 : more additions is reused elsewhere: if the pattern matches, we'd be
3518 : replacing 2N RSHIFT_EXPRs and N VEC_PACK_*s with N IFN_AVG_*s. */
3519 2202 : internal_fn ifn = IFN_AVG_FLOOR;
3520 8808 : vect_unpromoted_value unprom[3];
3521 2202 : tree new_type;
3522 2202 : unsigned int nops = vect_widened_op_tree (vinfo, plus_stmt_info, PLUS_EXPR,
3523 2202 : IFN_VEC_WIDEN_PLUS, false, 3,
3524 : unprom, &new_type);
3525 2202 : if (nops == 0)
3526 : return NULL;
3527 907 : if (nops == 3)
3528 : {
3529 : /* Check that one operand is 1. */
3530 : unsigned int i;
3531 987 : for (i = 0; i < 3; ++i)
3532 933 : if (integer_onep (unprom[i].op))
3533 : break;
3534 311 : if (i == 3)
3535 : return NULL;
3536 : /* Throw away the 1 operand and keep the other two. */
3537 257 : if (i < 2)
3538 0 : unprom[i] = unprom[2];
3539 : ifn = IFN_AVG_CEIL;
3540 : }
3541 :
3542 853 : vect_pattern_detected ("vect_recog_average_pattern", last_stmt);
3543 :
3544 : /* We know that:
3545 :
3546 : (a) the operation can be viewed as:
3547 :
3548 : TYPE widened0 = (TYPE) UNPROM[0];
3549 : TYPE widened1 = (TYPE) UNPROM[1];
3550 : TYPE tmp1 = widened0 + widened1 {+ 1};
3551 : TYPE tmp2 = tmp1 >> 1; // LAST_STMT_INFO
3552 :
3553 : (b) the first two statements are equivalent to:
3554 :
3555 : TYPE widened0 = (TYPE) (NEW_TYPE) UNPROM[0];
3556 : TYPE widened1 = (TYPE) (NEW_TYPE) UNPROM[1];
3557 :
3558 : (c) vect_recog_over_widening_pattern has already tried to narrow TYPE
3559 : where sensible;
3560 :
3561 : (d) all the operations can be performed correctly at twice the width of
3562 : NEW_TYPE, due to the nature of the average operation; and
3563 :
3564 : (e) users of the result of the right shift need only TARGET_PRECISION
3565 : bits, where TARGET_PRECISION is no more than half of TYPE's
3566 : precision.
3567 :
3568 : Under these circumstances, the only situation in which NEW_TYPE
3569 : could be narrower than TARGET_PRECISION is if widened0, widened1
3570 : and an addition result are all used more than once. Thus we can
3571 : treat any widening of UNPROM[0] and UNPROM[1] to TARGET_PRECISION
3572 : as "free", whereas widening the result of the average instruction
3573 : from NEW_TYPE to TARGET_PRECISION would be a new operation. It's
3574 : therefore better not to go narrower than TARGET_PRECISION. */
3575 853 : if (TYPE_PRECISION (new_type) < target_precision)
3576 0 : new_type = build_nonstandard_integer_type (target_precision,
3577 0 : TYPE_UNSIGNED (new_type));
3578 :
3579 : /* Check for target support. */
3580 853 : tree new_vectype = get_vectype_for_scalar_type (vinfo, new_type);
3581 853 : if (!new_vectype)
3582 : return NULL;
3583 :
3584 853 : bool fallback_p = false;
3585 :
3586 853 : if (direct_internal_fn_supported_p (ifn, new_vectype, OPTIMIZE_FOR_SPEED))
3587 : ;
3588 695 : else if (TYPE_UNSIGNED (new_type)
3589 256 : && optab_for_tree_code (RSHIFT_EXPR, new_vectype, optab_scalar)
3590 256 : && optab_for_tree_code (PLUS_EXPR, new_vectype, optab_default)
3591 256 : && optab_for_tree_code (BIT_IOR_EXPR, new_vectype, optab_default)
3592 951 : && optab_for_tree_code (BIT_AND_EXPR, new_vectype, optab_default))
3593 : fallback_p = true;
3594 : else
3595 : return NULL;
3596 :
3597 : /* The IR requires a valid vector type for the cast result, even though
3598 : it's likely to be discarded. */
3599 414 : *type_out = get_vectype_for_scalar_type (vinfo, type);
3600 414 : if (!*type_out)
3601 : return NULL;
3602 :
3603 410 : tree new_var = vect_recog_temp_ssa_var (new_type, NULL);
3604 410 : tree new_ops[2];
3605 410 : vect_convert_inputs (vinfo, last_stmt_info, 2, new_ops, new_type,
3606 : unprom, new_vectype);
3607 :
3608 410 : if (fallback_p)
3609 : {
3610 : /* As a fallback, generate code for following sequence:
3611 :
3612 : shifted_op0 = new_ops[0] >> 1;
3613 : shifted_op1 = new_ops[1] >> 1;
3614 : sum_of_shifted = shifted_op0 + shifted_op1;
3615 : unmasked_carry = new_ops[0] and/or new_ops[1];
3616 : carry = unmasked_carry & 1;
3617 : new_var = sum_of_shifted + carry;
3618 : */
3619 :
3620 252 : tree one_cst = build_one_cst (new_type);
3621 252 : gassign *g;
3622 :
3623 252 : tree shifted_op0 = vect_recog_temp_ssa_var (new_type, NULL);
3624 252 : g = gimple_build_assign (shifted_op0, RSHIFT_EXPR, new_ops[0], one_cst);
3625 252 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3626 :
3627 252 : tree shifted_op1 = vect_recog_temp_ssa_var (new_type, NULL);
3628 252 : g = gimple_build_assign (shifted_op1, RSHIFT_EXPR, new_ops[1], one_cst);
3629 252 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3630 :
3631 252 : tree sum_of_shifted = vect_recog_temp_ssa_var (new_type, NULL);
3632 252 : g = gimple_build_assign (sum_of_shifted, PLUS_EXPR,
3633 : shifted_op0, shifted_op1);
3634 252 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3635 :
3636 252 : tree unmasked_carry = vect_recog_temp_ssa_var (new_type, NULL);
3637 252 : tree_code c = (ifn == IFN_AVG_CEIL) ? BIT_IOR_EXPR : BIT_AND_EXPR;
3638 252 : g = gimple_build_assign (unmasked_carry, c, new_ops[0], new_ops[1]);
3639 252 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3640 :
3641 252 : tree carry = vect_recog_temp_ssa_var (new_type, NULL);
3642 252 : g = gimple_build_assign (carry, BIT_AND_EXPR, unmasked_carry, one_cst);
3643 252 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3644 :
3645 252 : g = gimple_build_assign (new_var, PLUS_EXPR, sum_of_shifted, carry);
3646 252 : return vect_convert_output (vinfo, last_stmt_info, type, g, new_vectype);
3647 : }
3648 :
3649 : /* Generate the IFN_AVG* call. */
3650 158 : gcall *average_stmt = gimple_build_call_internal (ifn, 2, new_ops[0],
3651 : new_ops[1]);
3652 158 : gimple_call_set_lhs (average_stmt, new_var);
3653 158 : gimple_set_location (average_stmt, gimple_location (last_stmt));
3654 :
3655 158 : if (dump_enabled_p ())
3656 31 : dump_printf_loc (MSG_NOTE, vect_location,
3657 : "created pattern stmt: %G", (gimple *) average_stmt);
3658 :
3659 158 : return vect_convert_output (vinfo, last_stmt_info,
3660 158 : type, average_stmt, new_vectype);
3661 : }
3662 :
3663 : /* Recognize cases in which the input to a cast is wider than its
3664 : output, and the input is fed by a widening operation. Fold this
3665 : by removing the unnecessary intermediate widening. E.g.:
3666 :
3667 : unsigned char a;
3668 : unsigned int b = (unsigned int) a;
3669 : unsigned short c = (unsigned short) b;
3670 :
3671 : -->
3672 :
3673 : unsigned short c = (unsigned short) a;
3674 :
3675 : Although this is rare in input IR, it is an expected side-effect
3676 : of the over-widening pattern above.
3677 :
3678 : This is beneficial also for integer-to-float conversions, if the
3679 : widened integer has more bits than the float, and if the unwidened
3680 : input doesn't. */
3681 :
3682 : static gimple *
3683 32488905 : vect_recog_cast_forwprop_pattern (vec_info *vinfo,
3684 : stmt_vec_info last_stmt_info, tree *type_out)
3685 : {
3686 : /* Check for a cast, including an integer-to-float conversion. */
3687 32488905 : gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
3688 22620361 : if (!last_stmt)
3689 : return NULL;
3690 22620361 : tree_code code = gimple_assign_rhs_code (last_stmt);
3691 22620361 : if (!CONVERT_EXPR_CODE_P (code) && code != FLOAT_EXPR)
3692 : return NULL;
3693 :
3694 : /* Make sure that the rhs is a scalar with a natural bitsize. */
3695 3277834 : tree lhs = gimple_assign_lhs (last_stmt);
3696 3277834 : if (!lhs)
3697 : return NULL;
3698 3277834 : tree lhs_type = TREE_TYPE (lhs);
3699 3277834 : scalar_mode lhs_mode;
3700 3257141 : if (VECT_SCALAR_BOOLEAN_TYPE_P (lhs_type)
3701 6533239 : || !is_a <scalar_mode> (TYPE_MODE (lhs_type), &lhs_mode))
3702 : return NULL;
3703 :
3704 : /* Check for a narrowing operation (from a vector point of view). */
3705 3251680 : tree rhs = gimple_assign_rhs1 (last_stmt);
3706 3251680 : tree rhs_type = TREE_TYPE (rhs);
3707 3251680 : if (!INTEGRAL_TYPE_P (rhs_type)
3708 2843346 : || VECT_SCALAR_BOOLEAN_TYPE_P (rhs_type)
3709 8777924 : || TYPE_PRECISION (rhs_type) <= GET_MODE_BITSIZE (lhs_mode))
3710 : return NULL;
3711 :
3712 : /* Try to find an unpromoted input. */
3713 383766 : vect_unpromoted_value unprom;
3714 383766 : if (!vect_look_through_possible_promotion (vinfo, rhs, &unprom)
3715 383766 : || TYPE_PRECISION (unprom.type) >= TYPE_PRECISION (rhs_type))
3716 : return NULL;
3717 :
3718 : /* If the bits above RHS_TYPE matter, make sure that they're the
3719 : same when extending from UNPROM as they are when extending from RHS. */
3720 49410 : if (!INTEGRAL_TYPE_P (lhs_type)
3721 49410 : && TYPE_SIGN (rhs_type) != TYPE_SIGN (unprom.type))
3722 : return NULL;
3723 :
3724 : /* We can get the same result by casting UNPROM directly, to avoid
3725 : the unnecessary widening and narrowing. */
3726 49290 : vect_pattern_detected ("vect_recog_cast_forwprop_pattern", last_stmt);
3727 :
3728 49290 : *type_out = get_vectype_for_scalar_type (vinfo, lhs_type);
3729 49290 : if (!*type_out)
3730 : return NULL;
3731 :
3732 49290 : tree new_var = vect_recog_temp_ssa_var (lhs_type, NULL);
3733 49290 : gimple *pattern_stmt = gimple_build_assign (new_var, code, unprom.op);
3734 49290 : gimple_set_location (pattern_stmt, gimple_location (last_stmt));
3735 :
3736 49290 : return pattern_stmt;
3737 : }
3738 :
3739 : /* Try to detect a shift left of a widened input, converting LSHIFT_EXPR
3740 : to WIDEN_LSHIFT_EXPR. See vect_recog_widen_op_pattern for details. */
3741 :
3742 : static gimple *
3743 32411121 : vect_recog_widen_shift_pattern (vec_info *vinfo,
3744 : stmt_vec_info last_stmt_info, tree *type_out)
3745 : {
3746 32411121 : return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
3747 32411121 : LSHIFT_EXPR, WIDEN_LSHIFT_EXPR, true,
3748 32411121 : "vect_recog_widen_shift_pattern");
3749 : }
3750 :
3751 : /* Detect a rotate pattern wouldn't be otherwise vectorized:
3752 :
3753 : type a_t, b_t, c_t;
3754 :
3755 : S0 a_t = b_t r<< c_t;
3756 :
3757 : Input/Output:
3758 :
3759 : * STMT_VINFO: The stmt from which the pattern search begins,
3760 : i.e. the shift/rotate stmt. The original stmt (S0) is replaced
3761 : with a sequence:
3762 :
3763 : S1 d_t = -c_t;
3764 : S2 e_t = d_t & (B - 1);
3765 : S3 f_t = b_t << c_t;
3766 : S4 g_t = b_t >> e_t;
3767 : S0 a_t = f_t | g_t;
3768 :
3769 : where B is element bitsize of type.
3770 :
3771 : Output:
3772 :
3773 : * TYPE_OUT: The type of the output of this pattern.
3774 :
3775 : * Return value: A new stmt that will be used to replace the rotate
3776 : S0 stmt. */
3777 :
3778 : static gimple *
3779 32411121 : vect_recog_rotate_pattern (vec_info *vinfo,
3780 : stmt_vec_info stmt_vinfo, tree *type_out)
3781 : {
3782 32411121 : gimple *last_stmt = stmt_vinfo->stmt;
3783 32411121 : tree oprnd0, oprnd1, lhs, var, var1, var2, vectype, type, stype, def, def2;
3784 32411121 : gimple *pattern_stmt, *def_stmt;
3785 32411121 : enum tree_code rhs_code;
3786 32411121 : enum vect_def_type dt;
3787 32411121 : optab optab1, optab2;
3788 32411121 : edge ext_def = NULL;
3789 32411121 : bool bswap16_p = false;
3790 :
3791 32411121 : if (is_gimple_assign (last_stmt))
3792 : {
3793 22542527 : rhs_code = gimple_assign_rhs_code (last_stmt);
3794 22542527 : switch (rhs_code)
3795 : {
3796 8535 : case LROTATE_EXPR:
3797 8535 : case RROTATE_EXPR:
3798 8535 : break;
3799 : default:
3800 : return NULL;
3801 : }
3802 :
3803 8535 : lhs = gimple_assign_lhs (last_stmt);
3804 8535 : oprnd0 = gimple_assign_rhs1 (last_stmt);
3805 8535 : type = TREE_TYPE (oprnd0);
3806 8535 : oprnd1 = gimple_assign_rhs2 (last_stmt);
3807 : }
3808 9868594 : else if (gimple_call_builtin_p (last_stmt, BUILT_IN_BSWAP16))
3809 : {
3810 : /* __builtin_bswap16 (x) is another form of x r>> 8.
3811 : The vectorizer has bswap support, but only if the argument isn't
3812 : promoted. */
3813 206 : lhs = gimple_call_lhs (last_stmt);
3814 206 : oprnd0 = gimple_call_arg (last_stmt, 0);
3815 206 : type = TREE_TYPE (oprnd0);
3816 206 : if (!lhs
3817 206 : || TYPE_PRECISION (TREE_TYPE (lhs)) != 16
3818 206 : || TYPE_PRECISION (type) <= 16
3819 0 : || TREE_CODE (oprnd0) != SSA_NAME
3820 206 : || BITS_PER_UNIT != 8)
3821 206 : return NULL;
3822 :
3823 0 : stmt_vec_info def_stmt_info;
3824 0 : if (!vect_is_simple_use (oprnd0, vinfo, &dt, &def_stmt_info, &def_stmt))
3825 : return NULL;
3826 :
3827 0 : if (dt != vect_internal_def)
3828 : return NULL;
3829 :
3830 0 : if (gimple_assign_cast_p (def_stmt))
3831 : {
3832 0 : def = gimple_assign_rhs1 (def_stmt);
3833 0 : if (INTEGRAL_TYPE_P (TREE_TYPE (def))
3834 0 : && TYPE_PRECISION (TREE_TYPE (def)) == 16)
3835 : oprnd0 = def;
3836 : }
3837 :
3838 0 : type = TREE_TYPE (lhs);
3839 0 : vectype = get_vectype_for_scalar_type (vinfo, type);
3840 0 : if (vectype == NULL_TREE)
3841 : return NULL;
3842 :
3843 0 : if (tree char_vectype = get_same_sized_vectype (char_type_node, vectype))
3844 : {
3845 : /* The encoding uses one stepped pattern for each byte in the
3846 : 16-bit word. */
3847 0 : vec_perm_builder elts (TYPE_VECTOR_SUBPARTS (char_vectype), 2, 3);
3848 0 : for (unsigned i = 0; i < 3; ++i)
3849 0 : for (unsigned j = 0; j < 2; ++j)
3850 0 : elts.quick_push ((i + 1) * 2 - j - 1);
3851 :
3852 0 : vec_perm_indices indices (elts, 1,
3853 0 : TYPE_VECTOR_SUBPARTS (char_vectype));
3854 0 : machine_mode vmode = TYPE_MODE (char_vectype);
3855 0 : if (can_vec_perm_const_p (vmode, vmode, indices))
3856 : {
3857 : /* vectorizable_bswap can handle the __builtin_bswap16 if we
3858 : undo the argument promotion. */
3859 0 : if (!useless_type_conversion_p (type, TREE_TYPE (oprnd0)))
3860 : {
3861 0 : def = vect_recog_temp_ssa_var (type, NULL);
3862 0 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd0);
3863 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
3864 0 : oprnd0 = def;
3865 : }
3866 :
3867 : /* Pattern detected. */
3868 0 : vect_pattern_detected ("vect_recog_rotate_pattern", last_stmt);
3869 :
3870 0 : *type_out = vectype;
3871 :
3872 : /* Pattern supported. Create a stmt to be used to replace the
3873 : pattern, with the unpromoted argument. */
3874 0 : var = vect_recog_temp_ssa_var (type, NULL);
3875 0 : pattern_stmt = gimple_build_call (gimple_call_fndecl (last_stmt),
3876 : 1, oprnd0);
3877 0 : gimple_call_set_lhs (pattern_stmt, var);
3878 0 : gimple_call_set_fntype (as_a <gcall *> (pattern_stmt),
3879 : gimple_call_fntype (last_stmt));
3880 0 : return pattern_stmt;
3881 : }
3882 0 : }
3883 :
3884 0 : oprnd1 = build_int_cst (integer_type_node, 8);
3885 0 : rhs_code = LROTATE_EXPR;
3886 0 : bswap16_p = true;
3887 : }
3888 : else
3889 : return NULL;
3890 :
3891 8535 : if (TREE_CODE (oprnd0) != SSA_NAME
3892 8415 : || !INTEGRAL_TYPE_P (type)
3893 16629 : || TYPE_PRECISION (TREE_TYPE (lhs)) != TYPE_PRECISION (type))
3894 : return NULL;
3895 :
3896 8094 : stmt_vec_info def_stmt_info;
3897 8094 : if (!vect_is_simple_use (oprnd1, vinfo, &dt, &def_stmt_info, &def_stmt))
3898 : return NULL;
3899 :
3900 8094 : if (dt != vect_internal_def
3901 7027 : && dt != vect_constant_def
3902 25 : && dt != vect_external_def)
3903 : return NULL;
3904 :
3905 8088 : vectype = get_vectype_for_scalar_type (vinfo, type);
3906 8088 : if (vectype == NULL_TREE)
3907 : return NULL;
3908 :
3909 : /* If vector/vector or vector/scalar rotate is supported by the target,
3910 : don't do anything here. */
3911 7793 : optab1 = optab_for_tree_code (rhs_code, vectype, optab_vector);
3912 7793 : if (optab1
3913 7793 : && can_implement_p (optab1, TYPE_MODE (vectype)))
3914 : {
3915 564 : use_rotate:
3916 564 : if (bswap16_p)
3917 : {
3918 0 : if (!useless_type_conversion_p (type, TREE_TYPE (oprnd0)))
3919 : {
3920 0 : def = vect_recog_temp_ssa_var (type, NULL);
3921 0 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd0);
3922 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
3923 0 : oprnd0 = def;
3924 : }
3925 :
3926 : /* Pattern detected. */
3927 0 : vect_pattern_detected ("vect_recog_rotate_pattern", last_stmt);
3928 :
3929 0 : *type_out = vectype;
3930 :
3931 : /* Pattern supported. Create a stmt to be used to replace the
3932 : pattern. */
3933 0 : var = vect_recog_temp_ssa_var (type, NULL);
3934 0 : pattern_stmt = gimple_build_assign (var, LROTATE_EXPR, oprnd0,
3935 : oprnd1);
3936 0 : return pattern_stmt;
3937 : }
3938 : return NULL;
3939 : }
3940 :
3941 7769 : if (is_a <bb_vec_info> (vinfo) || dt != vect_internal_def)
3942 : {
3943 7685 : optab2 = optab_for_tree_code (rhs_code, vectype, optab_scalar);
3944 7685 : if (optab2
3945 7685 : && can_implement_p (optab2, TYPE_MODE (vectype)))
3946 540 : goto use_rotate;
3947 : }
3948 :
3949 : /* We may not use a reduction operand twice. */
3950 7229 : if (vect_is_reduction (stmt_vinfo))
3951 : return NULL;
3952 :
3953 7208 : tree utype = unsigned_type_for (type);
3954 7208 : tree uvectype = get_vectype_for_scalar_type (vinfo, utype);
3955 7208 : if (!uvectype)
3956 : return NULL;
3957 :
3958 : /* If vector/vector or vector/scalar shifts aren't supported by the target,
3959 : don't do anything here either. */
3960 7208 : optab1 = optab_for_tree_code (LSHIFT_EXPR, uvectype, optab_vector);
3961 7208 : optab2 = optab_for_tree_code (RSHIFT_EXPR, uvectype, optab_vector);
3962 7208 : if (!optab1
3963 7208 : || !can_implement_p (optab1, TYPE_MODE (uvectype))
3964 746 : || !optab2
3965 7954 : || !can_implement_p (optab2, TYPE_MODE (uvectype)))
3966 : {
3967 6462 : if (! is_a <bb_vec_info> (vinfo) && dt == vect_internal_def)
3968 : return NULL;
3969 6399 : optab1 = optab_for_tree_code (LSHIFT_EXPR, uvectype, optab_scalar);
3970 6399 : optab2 = optab_for_tree_code (RSHIFT_EXPR, uvectype, optab_scalar);
3971 6399 : if (!optab1
3972 6399 : || !can_implement_p (optab1, TYPE_MODE (uvectype))
3973 4785 : || !optab2
3974 11184 : || !can_implement_p (optab2, TYPE_MODE (uvectype)))
3975 : return NULL;
3976 : }
3977 :
3978 5531 : *type_out = vectype;
3979 :
3980 5531 : if (!useless_type_conversion_p (utype, TREE_TYPE (oprnd0)))
3981 : {
3982 52 : def = vect_recog_temp_ssa_var (utype, NULL);
3983 52 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd0);
3984 52 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
3985 52 : oprnd0 = def;
3986 : }
3987 :
3988 5531 : if (dt == vect_external_def && TREE_CODE (oprnd1) == SSA_NAME)
3989 15 : ext_def = vect_get_external_def_edge (vinfo, oprnd1);
3990 :
3991 5531 : def = NULL_TREE;
3992 5531 : scalar_int_mode mode = SCALAR_INT_TYPE_MODE (utype);
3993 5531 : if (dt != vect_internal_def || TYPE_MODE (TREE_TYPE (oprnd1)) == mode)
3994 : def = oprnd1;
3995 124 : else if (def_stmt && gimple_assign_cast_p (def_stmt))
3996 : {
3997 96 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
3998 96 : if (TYPE_MODE (TREE_TYPE (rhs1)) == mode
3999 96 : && TYPE_PRECISION (TREE_TYPE (rhs1))
4000 0 : == TYPE_PRECISION (type))
4001 : def = rhs1;
4002 : }
4003 :
4004 5407 : if (def == NULL_TREE)
4005 : {
4006 124 : def = vect_recog_temp_ssa_var (utype, NULL);
4007 124 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd1);
4008 124 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
4009 : }
4010 5531 : stype = TREE_TYPE (def);
4011 :
4012 5531 : if (TREE_CODE (def) == INTEGER_CST)
4013 : {
4014 4953 : if (!tree_fits_uhwi_p (def)
4015 4953 : || tree_to_uhwi (def) >= GET_MODE_PRECISION (mode)
4016 9906 : || integer_zerop (def))
4017 : return NULL;
4018 4953 : def2 = build_int_cst (stype,
4019 4953 : GET_MODE_PRECISION (mode) - tree_to_uhwi (def));
4020 : }
4021 : else
4022 : {
4023 578 : tree vecstype = get_vectype_for_scalar_type (vinfo, stype);
4024 :
4025 578 : if (vecstype == NULL_TREE)
4026 : return NULL;
4027 578 : def2 = vect_recog_temp_ssa_var (stype, NULL);
4028 578 : def_stmt = gimple_build_assign (def2, NEGATE_EXPR, def);
4029 578 : if (ext_def)
4030 : {
4031 15 : basic_block new_bb
4032 15 : = gsi_insert_on_edge_immediate (ext_def, def_stmt);
4033 15 : gcc_assert (!new_bb);
4034 : }
4035 : else
4036 563 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecstype);
4037 :
4038 578 : def2 = vect_recog_temp_ssa_var (stype, NULL);
4039 578 : tree mask = build_int_cst (stype, GET_MODE_PRECISION (mode) - 1);
4040 578 : def_stmt = gimple_build_assign (def2, BIT_AND_EXPR,
4041 : gimple_assign_lhs (def_stmt), mask);
4042 578 : if (ext_def)
4043 : {
4044 15 : basic_block new_bb
4045 15 : = gsi_insert_on_edge_immediate (ext_def, def_stmt);
4046 15 : gcc_assert (!new_bb);
4047 : }
4048 : else
4049 563 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecstype);
4050 : }
4051 :
4052 5531 : var1 = vect_recog_temp_ssa_var (utype, NULL);
4053 10751 : def_stmt = gimple_build_assign (var1, rhs_code == LROTATE_EXPR
4054 : ? LSHIFT_EXPR : RSHIFT_EXPR,
4055 : oprnd0, def);
4056 5531 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
4057 :
4058 5531 : var2 = vect_recog_temp_ssa_var (utype, NULL);
4059 10751 : def_stmt = gimple_build_assign (var2, rhs_code == LROTATE_EXPR
4060 : ? RSHIFT_EXPR : LSHIFT_EXPR,
4061 : oprnd0, def2);
4062 5531 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
4063 :
4064 : /* Pattern detected. */
4065 5531 : vect_pattern_detected ("vect_recog_rotate_pattern", last_stmt);
4066 :
4067 : /* Pattern supported. Create a stmt to be used to replace the pattern. */
4068 5531 : var = vect_recog_temp_ssa_var (utype, NULL);
4069 5531 : pattern_stmt = gimple_build_assign (var, BIT_IOR_EXPR, var1, var2);
4070 :
4071 5531 : if (!useless_type_conversion_p (type, utype))
4072 : {
4073 52 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, uvectype);
4074 52 : tree result = vect_recog_temp_ssa_var (type, NULL);
4075 52 : pattern_stmt = gimple_build_assign (result, NOP_EXPR, var);
4076 : }
4077 : return pattern_stmt;
4078 : }
4079 :
4080 : /* Detect a vector by vector shift pattern that wouldn't be otherwise
4081 : vectorized:
4082 :
4083 : type a_t;
4084 : TYPE b_T, res_T;
4085 :
4086 : S1 a_t = ;
4087 : S2 b_T = ;
4088 : S3 res_T = b_T op a_t;
4089 :
4090 : where type 'TYPE' is a type with different size than 'type',
4091 : and op is <<, >> or rotate.
4092 :
4093 : Also detect cases:
4094 :
4095 : type a_t;
4096 : TYPE b_T, c_T, res_T;
4097 :
4098 : S0 c_T = ;
4099 : S1 a_t = (type) c_T;
4100 : S2 b_T = ;
4101 : S3 res_T = b_T op a_t;
4102 :
4103 : Input/Output:
4104 :
4105 : * STMT_VINFO: The stmt from which the pattern search begins,
4106 : i.e. the shift/rotate stmt. The original stmt (S3) is replaced
4107 : with a shift/rotate which has same type on both operands, in the
4108 : second case just b_T op c_T, in the first case with added cast
4109 : from a_t to c_T in STMT_VINFO_PATTERN_DEF_SEQ.
4110 :
4111 : Output:
4112 :
4113 : * TYPE_OUT: The type of the output of this pattern.
4114 :
4115 : * Return value: A new stmt that will be used to replace the shift/rotate
4116 : S3 stmt. */
4117 :
4118 : static gimple *
4119 32418006 : vect_recog_vector_vector_shift_pattern (vec_info *vinfo,
4120 : stmt_vec_info stmt_vinfo,
4121 : tree *type_out)
4122 : {
4123 32418006 : gimple *last_stmt = stmt_vinfo->stmt;
4124 32418006 : tree oprnd0, oprnd1, lhs, var;
4125 32418006 : gimple *pattern_stmt;
4126 32418006 : enum tree_code rhs_code;
4127 :
4128 32418006 : if (!is_gimple_assign (last_stmt))
4129 : return NULL;
4130 :
4131 22549412 : rhs_code = gimple_assign_rhs_code (last_stmt);
4132 22549412 : switch (rhs_code)
4133 : {
4134 502352 : case LSHIFT_EXPR:
4135 502352 : case RSHIFT_EXPR:
4136 502352 : case LROTATE_EXPR:
4137 502352 : case RROTATE_EXPR:
4138 502352 : break;
4139 : default:
4140 : return NULL;
4141 : }
4142 :
4143 502352 : lhs = gimple_assign_lhs (last_stmt);
4144 502352 : oprnd0 = gimple_assign_rhs1 (last_stmt);
4145 502352 : oprnd1 = gimple_assign_rhs2 (last_stmt);
4146 502352 : if (TREE_CODE (oprnd1) != SSA_NAME
4147 110794 : || TYPE_MODE (TREE_TYPE (oprnd0)) == TYPE_MODE (TREE_TYPE (oprnd1))
4148 49431 : || !INTEGRAL_TYPE_P (TREE_TYPE (oprnd0))
4149 48926 : || !type_has_mode_precision_p (TREE_TYPE (oprnd1))
4150 551278 : || TYPE_PRECISION (TREE_TYPE (lhs))
4151 48926 : != TYPE_PRECISION (TREE_TYPE (oprnd0)))
4152 : return NULL;
4153 :
4154 48926 : stmt_vec_info def_vinfo = vinfo->lookup_def (oprnd1);
4155 48926 : if (!def_vinfo || STMT_VINFO_DEF_TYPE (def_vinfo) == vect_external_def)
4156 : return NULL;
4157 :
4158 46076 : def_vinfo = vect_stmt_to_vectorize (def_vinfo);
4159 1138 : gcc_assert (def_vinfo);
4160 :
4161 46076 : *type_out = get_vectype_for_scalar_type (vinfo, TREE_TYPE (oprnd0));
4162 46076 : if (*type_out == NULL_TREE)
4163 : return NULL;
4164 :
4165 33314 : tree def = NULL_TREE;
4166 33314 : gassign *def_stmt = dyn_cast <gassign *> (def_vinfo->stmt);
4167 19510 : if (def_stmt && gimple_assign_cast_p (def_stmt))
4168 : {
4169 5483 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
4170 5483 : if (TYPE_MODE (TREE_TYPE (rhs1)) == TYPE_MODE (TREE_TYPE (oprnd0))
4171 5483 : && TYPE_PRECISION (TREE_TYPE (rhs1))
4172 1273 : == TYPE_PRECISION (TREE_TYPE (oprnd0)))
4173 : {
4174 1273 : if (TYPE_PRECISION (TREE_TYPE (oprnd1))
4175 1273 : >= TYPE_PRECISION (TREE_TYPE (rhs1)))
4176 : def = rhs1;
4177 : else
4178 : {
4179 1186 : tree mask
4180 1186 : = build_low_bits_mask (TREE_TYPE (rhs1),
4181 1186 : TYPE_PRECISION (TREE_TYPE (oprnd1)));
4182 1186 : def = vect_recog_temp_ssa_var (TREE_TYPE (rhs1), NULL);
4183 1186 : def_stmt = gimple_build_assign (def, BIT_AND_EXPR, rhs1, mask);
4184 1186 : tree vecstype = get_vectype_for_scalar_type (vinfo,
4185 1186 : TREE_TYPE (rhs1));
4186 1186 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecstype);
4187 : }
4188 : }
4189 : }
4190 :
4191 1273 : if (def == NULL_TREE)
4192 : {
4193 32041 : def = vect_recog_temp_ssa_var (TREE_TYPE (oprnd0), NULL);
4194 32041 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd1);
4195 32041 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
4196 : }
4197 :
4198 : /* Pattern detected. */
4199 33314 : vect_pattern_detected ("vect_recog_vector_vector_shift_pattern", last_stmt);
4200 :
4201 : /* Pattern supported. Create a stmt to be used to replace the pattern. */
4202 33314 : var = vect_recog_temp_ssa_var (TREE_TYPE (oprnd0), NULL);
4203 33314 : pattern_stmt = gimple_build_assign (var, rhs_code, oprnd0, def);
4204 :
4205 33314 : return pattern_stmt;
4206 : }
4207 :
4208 : /* Verify that the target has optabs of VECTYPE to perform all the steps
4209 : needed by the multiplication-by-immediate synthesis algorithm described by
4210 : ALG and VAR. If SYNTH_SHIFT_P is true ensure that vector addition is
4211 : present. Return true iff the target supports all the steps. */
4212 :
4213 : static bool
4214 301211 : target_supports_mult_synth_alg (struct algorithm *alg, mult_variant var,
4215 : tree vectype, bool synth_shift_p)
4216 : {
4217 301211 : if (alg->op[0] != alg_zero && alg->op[0] != alg_m)
4218 : return false;
4219 :
4220 301211 : bool supports_vminus = target_has_vecop_for_code (MINUS_EXPR, vectype);
4221 301211 : bool supports_vplus = target_has_vecop_for_code (PLUS_EXPR, vectype);
4222 :
4223 301211 : if (var == negate_variant
4224 301211 : && !target_has_vecop_for_code (NEGATE_EXPR, vectype))
4225 : return false;
4226 :
4227 : /* If we must synthesize shifts with additions make sure that vector
4228 : addition is available. */
4229 300607 : if ((var == add_variant || synth_shift_p) && !supports_vplus)
4230 : return false;
4231 :
4232 146587 : for (int i = 1; i < alg->ops; i++)
4233 : {
4234 110047 : switch (alg->op[i])
4235 : {
4236 : case alg_shift:
4237 : break;
4238 26976 : case alg_add_t_m2:
4239 26976 : case alg_add_t2_m:
4240 26976 : case alg_add_factor:
4241 26976 : if (!supports_vplus)
4242 : return false;
4243 : break;
4244 17119 : case alg_sub_t_m2:
4245 17119 : case alg_sub_t2_m:
4246 17119 : case alg_sub_factor:
4247 17119 : if (!supports_vminus)
4248 : return false;
4249 : break;
4250 : case alg_unknown:
4251 : case alg_m:
4252 : case alg_zero:
4253 : case alg_impossible:
4254 : return false;
4255 0 : default:
4256 0 : gcc_unreachable ();
4257 : }
4258 : }
4259 :
4260 : return true;
4261 : }
4262 :
4263 : /* Synthesize a left shift of OP by AMNT bits using a series of additions and
4264 : putting the final result in DEST. Append all statements but the last into
4265 : VINFO. Return the last statement. */
4266 :
4267 : static gimple *
4268 0 : synth_lshift_by_additions (vec_info *vinfo,
4269 : tree dest, tree op, HOST_WIDE_INT amnt,
4270 : stmt_vec_info stmt_info, tree vectype)
4271 : {
4272 0 : HOST_WIDE_INT i;
4273 0 : tree itype = TREE_TYPE (op);
4274 0 : tree prev_res = op;
4275 0 : gcc_assert (amnt >= 0);
4276 0 : for (i = 0; i < amnt; i++)
4277 : {
4278 0 : tree tmp_var = (i < amnt - 1) ? vect_recog_temp_ssa_var (itype, NULL)
4279 : : dest;
4280 0 : gimple *stmt
4281 0 : = gimple_build_assign (tmp_var, PLUS_EXPR, prev_res, prev_res);
4282 0 : prev_res = tmp_var;
4283 0 : if (i < amnt - 1)
4284 0 : append_pattern_def_seq (vinfo, stmt_info, stmt, vectype);
4285 : else
4286 0 : return stmt;
4287 : }
4288 0 : gcc_unreachable ();
4289 : return NULL;
4290 : }
4291 :
4292 : /* Helper for vect_synth_mult_by_constant. Apply a binary operation
4293 : CODE to operands OP1 and OP2, creating a new temporary SSA var in
4294 : the process if necessary. Append the resulting assignment statements
4295 : to the sequence in STMT_VINFO. Return the SSA variable that holds the
4296 : result of the binary operation. If SYNTH_SHIFT_P is true synthesize
4297 : left shifts using additions. */
4298 :
4299 : static tree
4300 43777 : apply_binop_and_append_stmt (vec_info *vinfo,
4301 : tree_code code, tree op1, tree op2,
4302 : stmt_vec_info stmt_vinfo, tree vectype,
4303 : bool synth_shift_p)
4304 : {
4305 43777 : if (integer_zerop (op2)
4306 43777 : && (code == LSHIFT_EXPR
4307 38001 : || code == PLUS_EXPR))
4308 : {
4309 38001 : gcc_assert (TREE_CODE (op1) == SSA_NAME);
4310 : return op1;
4311 : }
4312 :
4313 5776 : gimple *stmt;
4314 5776 : tree itype = TREE_TYPE (op1);
4315 5776 : tree tmp_var = vect_recog_temp_ssa_var (itype, NULL);
4316 :
4317 5776 : if (code == LSHIFT_EXPR
4318 5776 : && synth_shift_p)
4319 : {
4320 0 : stmt = synth_lshift_by_additions (vinfo, tmp_var, op1,
4321 0 : TREE_INT_CST_LOW (op2), stmt_vinfo,
4322 : vectype);
4323 0 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4324 0 : return tmp_var;
4325 : }
4326 :
4327 5776 : stmt = gimple_build_assign (tmp_var, code, op1, op2);
4328 5776 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4329 5776 : return tmp_var;
4330 : }
4331 :
4332 : /* Synthesize a multiplication of OP by an INTEGER_CST VAL using shifts
4333 : and simple arithmetic operations to be vectorized. Record the statements
4334 : produced in STMT_VINFO and return the last statement in the sequence or
4335 : NULL if it's not possible to synthesize such a multiplication.
4336 : This function mirrors the behavior of expand_mult_const in expmed.cc but
4337 : works on tree-ssa form. */
4338 :
4339 : static gimple *
4340 304568 : vect_synth_mult_by_constant (vec_info *vinfo, tree op, tree val,
4341 : stmt_vec_info stmt_vinfo)
4342 : {
4343 304568 : tree itype = TREE_TYPE (op);
4344 304568 : machine_mode mode = TYPE_MODE (itype);
4345 304568 : struct algorithm alg;
4346 304568 : mult_variant variant;
4347 304568 : if (!tree_fits_shwi_p (val))
4348 : return NULL;
4349 :
4350 : /* Multiplication synthesis by shifts, adds and subs can introduce
4351 : signed overflow where the original operation didn't. Perform the
4352 : operations on an unsigned type and cast back to avoid this.
4353 : In the future we may want to relax this for synthesis algorithms
4354 : that we can prove do not cause unexpected overflow. */
4355 301226 : bool cast_to_unsigned_p = !TYPE_OVERFLOW_WRAPS (itype);
4356 :
4357 59034 : tree multtype = cast_to_unsigned_p ? unsigned_type_for (itype) : itype;
4358 301226 : tree vectype = get_vectype_for_scalar_type (vinfo, multtype);
4359 301226 : if (!vectype)
4360 : return NULL;
4361 :
4362 : /* Targets that don't support vector shifts but support vector additions
4363 : can synthesize shifts that way. */
4364 301226 : bool synth_shift_p = !vect_supportable_shift (vinfo, LSHIFT_EXPR, multtype);
4365 :
4366 301226 : HOST_WIDE_INT hwval = tree_to_shwi (val);
4367 : /* Use MAX_COST here as we don't want to limit the sequence on rtx costs.
4368 : The vectorizer's benefit analysis will decide whether it's beneficial
4369 : to do this. */
4370 602256 : bool possible = choose_mult_variant (VECTOR_MODE_P (TYPE_MODE (vectype))
4371 301030 : ? TYPE_MODE (vectype) : mode,
4372 : hwval, &alg, &variant, MAX_COST);
4373 301226 : if (!possible)
4374 : return NULL;
4375 :
4376 301226 : if (vect_is_reduction (stmt_vinfo))
4377 : {
4378 26 : int op_uses = alg.op[0] != alg_zero;
4379 45 : for (int i = 1; i < alg.ops; i++)
4380 32 : switch (alg.op[i])
4381 : {
4382 4 : case alg_add_t_m2:
4383 4 : case alg_sub_t_m2:
4384 4 : if (synth_shift_p && alg.log[i])
4385 : return NULL;
4386 : else
4387 4 : op_uses++;
4388 4 : break;
4389 0 : case alg_add_t2_m:
4390 0 : case alg_sub_t2_m:
4391 0 : op_uses++;
4392 : /* Fallthru. */
4393 28 : case alg_shift:
4394 28 : if (synth_shift_p && alg.log[i])
4395 : return NULL;
4396 : break;
4397 : case alg_add_factor:
4398 : case alg_sub_factor:
4399 : return NULL;
4400 : default:
4401 : break;
4402 : }
4403 13 : if (variant == add_variant)
4404 0 : op_uses++;
4405 : /* When we'll synthesize more than a single use of the reduction
4406 : operand the reduction constraints are violated. Avoid this
4407 : situation. */
4408 13 : if (op_uses > 1)
4409 : return NULL;
4410 : }
4411 :
4412 301211 : if (!target_supports_mult_synth_alg (&alg, variant, vectype, synth_shift_p))
4413 : return NULL;
4414 :
4415 36540 : tree accumulator;
4416 :
4417 : /* Clear out the sequence of statements so we can populate it below. */
4418 36540 : gimple *stmt = NULL;
4419 :
4420 36540 : if (cast_to_unsigned_p)
4421 : {
4422 12157 : tree tmp_op = vect_recog_temp_ssa_var (multtype, NULL);
4423 12157 : stmt = gimple_build_assign (tmp_op, CONVERT_EXPR, op);
4424 12157 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4425 12157 : op = tmp_op;
4426 : }
4427 :
4428 36540 : if (alg.op[0] == alg_zero)
4429 205 : accumulator = build_int_cst (multtype, 0);
4430 : else
4431 : accumulator = op;
4432 :
4433 36540 : bool needs_fixup = (variant == negate_variant)
4434 36540 : || (variant == add_variant);
4435 :
4436 145984 : for (int i = 1; i < alg.ops; i++)
4437 : {
4438 109444 : tree shft_log = build_int_cst (multtype, alg.log[i]);
4439 109444 : tree accum_tmp = vect_recog_temp_ssa_var (multtype, NULL);
4440 109444 : tree tmp_var = NULL_TREE;
4441 :
4442 109444 : switch (alg.op[i])
4443 : {
4444 65667 : case alg_shift:
4445 65667 : if (synth_shift_p)
4446 0 : stmt
4447 0 : = synth_lshift_by_additions (vinfo, accum_tmp, accumulator,
4448 0 : alg.log[i], stmt_vinfo, vectype);
4449 : else
4450 65667 : stmt = gimple_build_assign (accum_tmp, LSHIFT_EXPR, accumulator,
4451 : shft_log);
4452 : break;
4453 22065 : case alg_add_t_m2:
4454 22065 : tmp_var
4455 22065 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, op, shft_log,
4456 : stmt_vinfo, vectype, synth_shift_p);
4457 22065 : stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, accumulator,
4458 : tmp_var);
4459 22065 : break;
4460 16139 : case alg_sub_t_m2:
4461 16139 : tmp_var = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, op,
4462 : shft_log, stmt_vinfo,
4463 : vectype, synth_shift_p);
4464 : /* In some algorithms the first step involves zeroing the
4465 : accumulator. If subtracting from such an accumulator
4466 : just emit the negation directly. */
4467 16139 : if (integer_zerop (accumulator))
4468 205 : stmt = gimple_build_assign (accum_tmp, NEGATE_EXPR, tmp_var);
4469 : else
4470 15934 : stmt = gimple_build_assign (accum_tmp, MINUS_EXPR, accumulator,
4471 : tmp_var);
4472 : break;
4473 0 : case alg_add_t2_m:
4474 0 : tmp_var
4475 0 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
4476 : shft_log, stmt_vinfo, vectype,
4477 : synth_shift_p);
4478 0 : stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, tmp_var, op);
4479 0 : break;
4480 0 : case alg_sub_t2_m:
4481 0 : tmp_var
4482 0 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
4483 : shft_log, stmt_vinfo, vectype,
4484 : synth_shift_p);
4485 0 : stmt = gimple_build_assign (accum_tmp, MINUS_EXPR, tmp_var, op);
4486 0 : break;
4487 4787 : case alg_add_factor:
4488 4787 : tmp_var
4489 4787 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
4490 : shft_log, stmt_vinfo, vectype,
4491 : synth_shift_p);
4492 4787 : stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, accumulator,
4493 : tmp_var);
4494 4787 : break;
4495 786 : case alg_sub_factor:
4496 786 : tmp_var
4497 786 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
4498 : shft_log, stmt_vinfo, vectype,
4499 : synth_shift_p);
4500 786 : stmt = gimple_build_assign (accum_tmp, MINUS_EXPR, tmp_var,
4501 : accumulator);
4502 786 : break;
4503 0 : default:
4504 0 : gcc_unreachable ();
4505 : }
4506 : /* We don't want to append the last stmt in the sequence to stmt_vinfo
4507 : but rather return it directly. */
4508 :
4509 109444 : if ((i < alg.ops - 1) || needs_fixup || cast_to_unsigned_p)
4510 85362 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4511 109444 : accumulator = accum_tmp;
4512 : }
4513 36540 : if (variant == negate_variant)
4514 : {
4515 441 : tree accum_tmp = vect_recog_temp_ssa_var (multtype, NULL);
4516 441 : stmt = gimple_build_assign (accum_tmp, NEGATE_EXPR, accumulator);
4517 441 : accumulator = accum_tmp;
4518 441 : if (cast_to_unsigned_p)
4519 150 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4520 : }
4521 36099 : else if (variant == add_variant)
4522 : {
4523 101 : tree accum_tmp = vect_recog_temp_ssa_var (multtype, NULL);
4524 101 : stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, accumulator, op);
4525 101 : accumulator = accum_tmp;
4526 101 : if (cast_to_unsigned_p)
4527 91 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4528 : }
4529 : /* Move back to a signed if needed. */
4530 36239 : if (cast_to_unsigned_p)
4531 : {
4532 12157 : tree accum_tmp = vect_recog_temp_ssa_var (itype, NULL);
4533 12157 : stmt = gimple_build_assign (accum_tmp, CONVERT_EXPR, accumulator);
4534 : }
4535 :
4536 : return stmt;
4537 : }
4538 :
4539 : /* Detect multiplication by constant and convert it into a sequence of
4540 : shifts and additions, subtractions, negations. We reuse the
4541 : choose_mult_variant algorithms from expmed.cc
4542 :
4543 : Input/Output:
4544 :
4545 : STMT_VINFO: The stmt from which the pattern search begins,
4546 : i.e. the mult stmt.
4547 :
4548 : Output:
4549 :
4550 : * TYPE_OUT: The type of the output of this pattern.
4551 :
4552 : * Return value: A new stmt that will be used to replace
4553 : the multiplication. */
4554 :
4555 : static gimple *
4556 32605732 : vect_recog_mult_pattern (vec_info *vinfo,
4557 : stmt_vec_info stmt_vinfo, tree *type_out)
4558 : {
4559 32605732 : gimple *last_stmt = stmt_vinfo->stmt;
4560 32605732 : tree oprnd0, oprnd1, vectype, itype;
4561 32605732 : gimple *pattern_stmt;
4562 :
4563 32605732 : if (!is_gimple_assign (last_stmt))
4564 : return NULL;
4565 :
4566 22737138 : if (gimple_assign_rhs_code (last_stmt) != MULT_EXPR)
4567 : return NULL;
4568 :
4569 1529097 : oprnd0 = gimple_assign_rhs1 (last_stmt);
4570 1529097 : oprnd1 = gimple_assign_rhs2 (last_stmt);
4571 1529097 : itype = TREE_TYPE (oprnd0);
4572 :
4573 1529097 : if (TREE_CODE (oprnd0) != SSA_NAME
4574 1529024 : || TREE_CODE (oprnd1) != INTEGER_CST
4575 909674 : || !INTEGRAL_TYPE_P (itype)
4576 2438771 : || !type_has_mode_precision_p (itype))
4577 : return NULL;
4578 :
4579 909619 : vectype = get_vectype_for_scalar_type (vinfo, itype);
4580 909619 : if (vectype == NULL_TREE)
4581 : return NULL;
4582 :
4583 : /* If the target can handle vectorized multiplication natively,
4584 : don't attempt to optimize this. */
4585 739000 : optab mul_optab = optab_for_tree_code (MULT_EXPR, vectype, optab_default);
4586 739000 : if (mul_optab != unknown_optab
4587 739000 : && can_implement_p (mul_optab, TYPE_MODE (vectype)))
4588 : return NULL;
4589 :
4590 304568 : pattern_stmt = vect_synth_mult_by_constant (vinfo,
4591 : oprnd0, oprnd1, stmt_vinfo);
4592 304568 : if (!pattern_stmt)
4593 : return NULL;
4594 :
4595 : /* Pattern detected. */
4596 36540 : vect_pattern_detected ("vect_recog_mult_pattern", last_stmt);
4597 :
4598 36540 : *type_out = vectype;
4599 :
4600 36540 : return pattern_stmt;
4601 : }
4602 :
4603 : extern bool gimple_unsigned_integer_sat_add (tree, tree*, tree (*)(tree));
4604 : extern bool gimple_unsigned_integer_sat_sub (tree, tree*, tree (*)(tree));
4605 : extern bool gimple_unsigned_integer_sat_trunc (tree, tree*, tree (*)(tree));
4606 :
4607 : extern bool gimple_unsigned_integer_narrow_clip (tree, tree*, tree (*)(tree));
4608 :
4609 : extern bool gimple_signed_integer_sat_add (tree, tree*, tree (*)(tree));
4610 : extern bool gimple_signed_integer_sat_sub (tree, tree*, tree (*)(tree));
4611 : extern bool gimple_signed_integer_sat_trunc (tree, tree*, tree (*)(tree));
4612 :
4613 : static gimple *
4614 301 : vect_recog_build_binary_gimple_stmt (vec_info *vinfo, stmt_vec_info stmt_info,
4615 : internal_fn fn, tree *type_out,
4616 : tree lhs, tree op_0, tree op_1)
4617 : {
4618 301 : tree itype = TREE_TYPE (op_0);
4619 301 : tree otype = TREE_TYPE (lhs);
4620 301 : tree v_itype = get_vectype_for_scalar_type (vinfo, itype);
4621 301 : tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
4622 :
4623 301 : if (v_itype != NULL_TREE && v_otype != NULL_TREE
4624 301 : && direct_internal_fn_supported_p (fn, v_itype, OPTIMIZE_FOR_BOTH))
4625 : {
4626 97 : gcall *call = gimple_build_call_internal (fn, 2, op_0, op_1);
4627 97 : tree in_ssa = vect_recog_temp_ssa_var (itype, NULL);
4628 :
4629 97 : gimple_call_set_lhs (call, in_ssa);
4630 97 : gimple_call_set_nothrow (call, /* nothrow_p */ false);
4631 97 : gimple_set_location (call, gimple_location (STMT_VINFO_STMT (stmt_info)));
4632 :
4633 97 : *type_out = v_otype;
4634 :
4635 97 : if (types_compatible_p (itype, otype))
4636 : return call;
4637 : else
4638 : {
4639 0 : append_pattern_def_seq (vinfo, stmt_info, call, v_itype);
4640 0 : tree out_ssa = vect_recog_temp_ssa_var (otype, NULL);
4641 :
4642 0 : return gimple_build_assign (out_ssa, NOP_EXPR, in_ssa);
4643 : }
4644 : }
4645 :
4646 : return NULL;
4647 : }
4648 :
4649 : /*
4650 : * Try to detect saturation add pattern (SAT_ADD), aka below gimple:
4651 : * _7 = _4 + _6;
4652 : * _8 = _4 > _7;
4653 : * _9 = (long unsigned int) _8;
4654 : * _10 = -_9;
4655 : * _12 = _7 | _10;
4656 : *
4657 : * And then simplified to
4658 : * _12 = .SAT_ADD (_4, _6);
4659 : */
4660 :
4661 : static gimple *
4662 32677860 : vect_recog_sat_add_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
4663 : tree *type_out)
4664 : {
4665 32677860 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
4666 :
4667 32677860 : if (!is_gimple_assign (last_stmt))
4668 : return NULL;
4669 :
4670 22809266 : tree ops[2];
4671 22809266 : tree lhs = gimple_assign_lhs (last_stmt);
4672 :
4673 22809266 : if (gimple_unsigned_integer_sat_add (lhs, ops, NULL)
4674 22809266 : || gimple_signed_integer_sat_add (lhs, ops, NULL))
4675 : {
4676 63 : if (TREE_CODE (ops[1]) == INTEGER_CST)
4677 13 : ops[1] = fold_convert (TREE_TYPE (ops[0]), ops[1]);
4678 :
4679 63 : gimple *stmt = vect_recog_build_binary_gimple_stmt (vinfo, stmt_vinfo,
4680 : IFN_SAT_ADD, type_out,
4681 : lhs, ops[0], ops[1]);
4682 63 : if (stmt)
4683 : {
4684 44 : vect_pattern_detected ("vect_recog_sat_add_pattern", last_stmt);
4685 44 : return stmt;
4686 : }
4687 : }
4688 :
4689 : return NULL;
4690 : }
4691 :
4692 : /*
4693 : * Try to transform the truncation for .SAT_SUB pattern, mostly occurs in
4694 : * the benchmark zip. Aka:
4695 : *
4696 : * unsigned int _1;
4697 : * unsigned int _2;
4698 : * unsigned short int _4;
4699 : * _9 = (unsigned short int).SAT_SUB (_1, _2);
4700 : *
4701 : * if _1 is known to be in the range of unsigned short int. For example
4702 : * there is a def _1 = (unsigned short int)_4. Then we can transform the
4703 : * truncation to:
4704 : *
4705 : * _3 = (unsigned short int) MIN (65535, _2); // aka _3 = .SAT_TRUNC (_2);
4706 : * _9 = .SAT_SUB (_4, _3);
4707 : *
4708 : * Then, we can better vectorized code and avoid the unnecessary narrowing
4709 : * stmt during vectorization with below stmt(s).
4710 : *
4711 : * _3 = .SAT_TRUNC(_2); // SI => HI
4712 : * _9 = .SAT_SUB (_4, _3);
4713 : */
4714 : static void
4715 238 : vect_recog_sat_sub_pattern_transform (vec_info *vinfo,
4716 : stmt_vec_info stmt_vinfo,
4717 : tree lhs, tree *ops)
4718 : {
4719 238 : tree otype = TREE_TYPE (lhs);
4720 238 : tree itype = TREE_TYPE (ops[0]);
4721 238 : unsigned itype_prec = TYPE_PRECISION (itype);
4722 238 : unsigned otype_prec = TYPE_PRECISION (otype);
4723 :
4724 238 : if (types_compatible_p (otype, itype) || otype_prec >= itype_prec)
4725 238 : return;
4726 :
4727 0 : tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
4728 0 : tree v_itype = get_vectype_for_scalar_type (vinfo, itype);
4729 0 : tree_pair v_pair = tree_pair (v_otype, v_itype);
4730 :
4731 0 : if (v_otype == NULL_TREE || v_itype == NULL_TREE
4732 0 : || !direct_internal_fn_supported_p (IFN_SAT_TRUNC, v_pair,
4733 : OPTIMIZE_FOR_BOTH))
4734 : return;
4735 :
4736 : /* 1. Find the _4 and update ops[0] as above example. */
4737 0 : vect_unpromoted_value unprom;
4738 0 : tree tmp = vect_look_through_possible_promotion (vinfo, ops[0], &unprom);
4739 :
4740 0 : if (tmp == NULL_TREE || TYPE_PRECISION (unprom.type) != otype_prec)
4741 : return;
4742 :
4743 0 : ops[0] = tmp;
4744 :
4745 : /* 2. Generate _3 = .SAT_TRUNC (_2) and update ops[1] as above example. */
4746 0 : tree trunc_lhs_ssa = vect_recog_temp_ssa_var (otype, NULL);
4747 0 : gcall *call = gimple_build_call_internal (IFN_SAT_TRUNC, 1, ops[1]);
4748 :
4749 0 : gimple_call_set_lhs (call, trunc_lhs_ssa);
4750 0 : gimple_call_set_nothrow (call, /* nothrow_p */ false);
4751 0 : append_pattern_def_seq (vinfo, stmt_vinfo, call, v_otype);
4752 :
4753 0 : ops[1] = trunc_lhs_ssa;
4754 : }
4755 :
4756 : /*
4757 : * Try to detect saturation sub pattern (SAT_ADD), aka below gimple:
4758 : * Unsigned:
4759 : * _7 = _1 >= _2;
4760 : * _8 = _1 - _2;
4761 : * _10 = (long unsigned int) _7;
4762 : * _9 = _8 * _10;
4763 : *
4764 : * And then simplified to
4765 : * _9 = .SAT_SUB (_1, _2);
4766 : *
4767 : * Signed:
4768 : * x.0_4 = (unsigned char) x_16;
4769 : * y.1_5 = (unsigned char) y_18;
4770 : * _6 = x.0_4 - y.1_5;
4771 : * minus_19 = (int8_t) _6;
4772 : * _7 = x_16 ^ y_18;
4773 : * _8 = x_16 ^ minus_19;
4774 : * _44 = _7 < 0;
4775 : * _23 = x_16 < 0;
4776 : * _24 = (signed char) _23;
4777 : * _58 = (unsigned char) _24;
4778 : * _59 = -_58;
4779 : * _25 = (signed char) _59;
4780 : * _26 = _25 ^ 127;
4781 : * _42 = _8 < 0;
4782 : * _41 = _42 & _44;
4783 : * iftmp.2_11 = _41 ? _26 : minus_19;
4784 : *
4785 : * And then simplified to
4786 : * iftmp.2_11 = .SAT_SUB (x_16, y_18);
4787 : */
4788 :
4789 : static gimple *
4790 32677816 : vect_recog_sat_sub_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
4791 : tree *type_out)
4792 : {
4793 32677816 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
4794 :
4795 32677816 : if (!is_gimple_assign (last_stmt))
4796 : return NULL;
4797 :
4798 22809222 : tree ops[2];
4799 22809222 : tree lhs = gimple_assign_lhs (last_stmt);
4800 :
4801 22809222 : if (gimple_unsigned_integer_sat_sub (lhs, ops, NULL)
4802 22809222 : || gimple_signed_integer_sat_sub (lhs, ops, NULL))
4803 : {
4804 238 : vect_recog_sat_sub_pattern_transform (vinfo, stmt_vinfo, lhs, ops);
4805 238 : gimple *stmt = vect_recog_build_binary_gimple_stmt (vinfo, stmt_vinfo,
4806 : IFN_SAT_SUB, type_out,
4807 : lhs, ops[0], ops[1]);
4808 238 : if (stmt)
4809 : {
4810 53 : vect_pattern_detected ("vect_recog_sat_sub_pattern", last_stmt);
4811 53 : return stmt;
4812 : }
4813 : }
4814 :
4815 : return NULL;
4816 : }
4817 :
4818 : /*
4819 : * Try to detect saturation truncation pattern (SAT_TRUNC), aka below gimple:
4820 : * overflow_5 = x_4(D) > 4294967295;
4821 : * _1 = (unsigned int) x_4(D);
4822 : * _2 = (unsigned int) overflow_5;
4823 : * _3 = -_2;
4824 : * _6 = _1 | _3;
4825 : *
4826 : * And then simplified to
4827 : * _6 = .SAT_TRUNC (x_4(D));
4828 : */
4829 :
4830 : static gimple *
4831 32677763 : vect_recog_sat_trunc_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
4832 : tree *type_out)
4833 : {
4834 32677763 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
4835 :
4836 32677763 : if (!is_gimple_assign (last_stmt))
4837 : return NULL;
4838 :
4839 22809169 : tree ops[1];
4840 22809169 : tree lhs = gimple_assign_lhs (last_stmt);
4841 22809169 : tree otype = TREE_TYPE (lhs);
4842 :
4843 22809169 : if ((gimple_unsigned_integer_narrow_clip (lhs, ops, NULL))
4844 22809169 : && type_has_mode_precision_p (otype))
4845 : {
4846 16 : tree itype = TREE_TYPE (ops[0]);
4847 16 : tree v_itype = get_vectype_for_scalar_type (vinfo, itype);
4848 16 : tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
4849 16 : internal_fn fn = IFN_SAT_TRUNC;
4850 :
4851 16 : if (v_itype != NULL_TREE && v_otype != NULL_TREE
4852 32 : && direct_internal_fn_supported_p (fn, tree_pair (v_otype, v_itype),
4853 : OPTIMIZE_FOR_BOTH))
4854 : {
4855 0 : tree temp = vect_recog_temp_ssa_var (itype, NULL);
4856 0 : gimple * max_stmt = gimple_build_assign (temp, build2 (MAX_EXPR, itype, build_zero_cst(itype), ops[0]));
4857 0 : append_pattern_def_seq (vinfo, stmt_vinfo, max_stmt, v_itype);
4858 :
4859 0 : gcall *call = gimple_build_call_internal (fn, 1, temp);
4860 0 : tree out_ssa = vect_recog_temp_ssa_var (otype, NULL);
4861 :
4862 0 : gimple_call_set_lhs (call, out_ssa);
4863 0 : gimple_call_set_nothrow (call, /* nothrow_p */ false);
4864 0 : gimple_set_location (call, gimple_location (last_stmt));
4865 :
4866 0 : *type_out = v_otype;
4867 :
4868 0 : return call;
4869 : }
4870 :
4871 : }
4872 :
4873 22809169 : if ((gimple_unsigned_integer_sat_trunc (lhs, ops, NULL)
4874 22808846 : || gimple_signed_integer_sat_trunc (lhs, ops, NULL))
4875 22809169 : && type_has_mode_precision_p (otype))
4876 : {
4877 311 : tree itype = TREE_TYPE (ops[0]);
4878 311 : tree v_itype = get_vectype_for_scalar_type (vinfo, itype);
4879 311 : tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
4880 311 : internal_fn fn = IFN_SAT_TRUNC;
4881 :
4882 305 : if (v_itype != NULL_TREE && v_otype != NULL_TREE
4883 616 : && direct_internal_fn_supported_p (fn, tree_pair (v_otype, v_itype),
4884 : OPTIMIZE_FOR_BOTH))
4885 : {
4886 0 : gcall *call = gimple_build_call_internal (fn, 1, ops[0]);
4887 0 : tree out_ssa = vect_recog_temp_ssa_var (otype, NULL);
4888 :
4889 0 : gimple_call_set_lhs (call, out_ssa);
4890 0 : gimple_call_set_nothrow (call, /* nothrow_p */ false);
4891 0 : gimple_set_location (call, gimple_location (last_stmt));
4892 :
4893 0 : *type_out = v_otype;
4894 :
4895 0 : return call;
4896 : }
4897 : }
4898 :
4899 : return NULL;
4900 : }
4901 :
4902 :
4903 : /* Function add_code_for_floorceilround_divmod
4904 : A helper function to add compensation code for implementing FLOOR_MOD_EXPR,
4905 : FLOOR_DIV_EXPR, CEIL_MOD_EXPR, CEIL_DIV_EXPR, ROUND_MOD_EXPR and
4906 : ROUND_DIV_EXPR
4907 : The quotient and remainder are needed for implemented these operators.
4908 : FLOOR cases
4909 : r = x %[fl] y; r = x/[fl] y;
4910 : is
4911 : r = x % y; if (r && (x ^ y) < 0) r += y;
4912 : r = x % y; d = x/y; if (r && (x ^ y) < 0) d--; Respectively
4913 : Produce following sequence
4914 : v0 = x^y
4915 : v1 = -r
4916 : v2 = r | -r
4917 : v3 = v0 & v2
4918 : v4 = v3 < 0
4919 : if (floor_mod)
4920 : v5 = v4 ? y : 0
4921 : v6 = r + v5
4922 : if (floor_div)
4923 : v5 = v4 ? 1 : 0
4924 : v6 = d - 1
4925 : Similar sequences of vector instructions are produces for following cases
4926 : CEIL cases
4927 : r = x %[cl] y; r = x/[cl] y;
4928 : is
4929 : r = x % y; if (r && (x ^ y) >= 0) r -= y;
4930 : r = x % y; if (r) r -= y; (unsigned)
4931 : r = x % y; d = x/y; if (r && (x ^ y) >= 0) d++;
4932 : r = x % y; d = x/y; if (r) d++; (unsigned)
4933 : ROUND cases
4934 : r = x %[rd] y; r = x/[rd] y;
4935 : is
4936 : r = x % y; if (r > ((y-1)/2)) if ((x ^ y) >= 0) r -= y; else r += y;
4937 : r = x % y; if (r > ((y-1)/2)) r -= y; (unsigned)
4938 : r = x % y; d = x/y; if (r > ((y-1)/2)) if ((x ^ y) >= 0) d++; else d--;
4939 : r = x % y; d = x/y; if (r > ((y-1)/2)) d++; (unsigned)
4940 : Inputs:
4941 : VECTYPE: Vector type of the operands
4942 : STMT_VINFO: Statement where pattern begins
4943 : RHS_CODE: Should either be FLOOR_MOD_EXPR or FLOOR_DIV_EXPR
4944 : Q: The quotient of division
4945 : R: Remainder of division
4946 : OPRDN0/OPRND1: Actual operands involved
4947 : ITYPE: tree type of oprnd0
4948 : Output:
4949 : NULL if vectorization not possible
4950 : Gimple statement based on rhs_code
4951 : */
4952 : static gimple *
4953 411 : add_code_for_floorceilround_divmod (tree vectype, vec_info *vinfo,
4954 : stmt_vec_info stmt_vinfo,
4955 : enum tree_code rhs_code, tree q, tree r,
4956 : tree oprnd0, tree oprnd1, tree itype)
4957 : {
4958 411 : gimple *def_stmt;
4959 411 : tree mask_vectype = truth_type_for (vectype);
4960 411 : if (!mask_vectype)
4961 : return NULL;
4962 411 : tree bool_cond;
4963 411 : bool unsigned_p = TYPE_UNSIGNED (itype);
4964 :
4965 411 : switch (rhs_code)
4966 : {
4967 375 : case FLOOR_MOD_EXPR:
4968 375 : case FLOOR_DIV_EXPR:
4969 375 : case CEIL_MOD_EXPR:
4970 375 : case CEIL_DIV_EXPR:
4971 375 : {
4972 375 : if (!target_has_vecop_for_code (NEGATE_EXPR, vectype)
4973 351 : || !target_has_vecop_for_code (BIT_XOR_EXPR, vectype)
4974 351 : || !target_has_vecop_for_code (BIT_IOR_EXPR, vectype)
4975 351 : || !target_has_vecop_for_code (PLUS_EXPR, vectype)
4976 351 : || !target_has_vecop_for_code (MINUS_EXPR, vectype)
4977 351 : || !expand_vec_cmp_expr_p (vectype, mask_vectype, LT_EXPR)
4978 599 : || !expand_vec_cond_expr_p (vectype, mask_vectype))
4979 : return NULL;
4980 224 : if (unsigned_p)
4981 : {
4982 18 : gcc_assert (rhs_code == CEIL_MOD_EXPR || rhs_code == CEIL_DIV_EXPR);
4983 :
4984 18 : if (!expand_vec_cmp_expr_p (vectype, mask_vectype, GT_EXPR))
4985 : return NULL;
4986 18 : bool is_mod = rhs_code == CEIL_MOD_EXPR;
4987 : // r > 0
4988 18 : bool_cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
4989 18 : def_stmt = gimple_build_assign (bool_cond, GT_EXPR, r,
4990 : build_int_cst (itype, 0));
4991 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
4992 : itype);
4993 :
4994 : // (r > 0) ? y : 0 (mod)
4995 : // (r > 0) ? 1 : 0 (ceil)
4996 18 : tree extr_cond = vect_recog_temp_ssa_var (itype, NULL);
4997 18 : def_stmt
4998 27 : = gimple_build_assign (extr_cond, COND_EXPR, bool_cond,
4999 9 : is_mod ? oprnd1 : build_int_cst (itype, 1),
5000 : build_int_cst (itype, 0));
5001 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5002 :
5003 : // r -= (r > 0) ? y : 0 (mod)
5004 : // d += (x^y < 0 && r) ? -1 : 0 (ceil)
5005 18 : tree result = vect_recog_temp_ssa_var (itype, NULL);
5006 27 : return gimple_build_assign (result, is_mod ? MINUS_EXPR : PLUS_EXPR,
5007 18 : is_mod ? r : q, extr_cond);
5008 : }
5009 : else
5010 : {
5011 206 : bool ceil_p
5012 206 : = (rhs_code == CEIL_MOD_EXPR || rhs_code == CEIL_DIV_EXPR);
5013 206 : if (ceil_p && !target_has_vecop_for_code (BIT_NOT_EXPR, vectype))
5014 : return NULL;
5015 : // x ^ y
5016 206 : tree xort = vect_recog_temp_ssa_var (itype, NULL);
5017 206 : def_stmt = gimple_build_assign (xort, BIT_XOR_EXPR, oprnd0, oprnd1);
5018 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5019 :
5020 206 : tree cond_reg = xort;
5021 : // ~(x ^ y) (ceil)
5022 206 : if (ceil_p)
5023 : {
5024 18 : cond_reg = vect_recog_temp_ssa_var (itype, NULL);
5025 18 : def_stmt = gimple_build_assign (cond_reg, BIT_NOT_EXPR, xort);
5026 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5027 : }
5028 :
5029 : // -r
5030 206 : tree negate_r = vect_recog_temp_ssa_var (itype, NULL);
5031 206 : def_stmt = gimple_build_assign (negate_r, NEGATE_EXPR, r);
5032 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5033 :
5034 : // r | -r , sign bit is set if r!=0
5035 206 : tree r_or_negr = vect_recog_temp_ssa_var (itype, NULL);
5036 206 : def_stmt
5037 206 : = gimple_build_assign (r_or_negr, BIT_IOR_EXPR, r, negate_r);
5038 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5039 :
5040 : // (x ^ y) & (r | -r)
5041 : // ~(x ^ y) & (r | -r) (ceil)
5042 206 : tree r_or_negr_and_xor = vect_recog_temp_ssa_var (itype, NULL);
5043 206 : def_stmt = gimple_build_assign (r_or_negr_and_xor, BIT_AND_EXPR,
5044 : r_or_negr, cond_reg);
5045 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5046 :
5047 : // (x ^ y) & (r | -r) < 0 which is equivalent to (x^y < 0 && r!=0)
5048 206 : bool_cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5049 206 : def_stmt
5050 206 : = gimple_build_assign (bool_cond, LT_EXPR, r_or_negr_and_xor,
5051 : build_int_cst (itype, 0));
5052 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5053 : itype);
5054 :
5055 : // (x^y < 0 && r) ? y : 0 (mod)
5056 : // (x^y < 0 && r) ? -1 : 0 (div)
5057 206 : bool is_mod
5058 206 : = (rhs_code == FLOOR_MOD_EXPR || rhs_code == CEIL_MOD_EXPR);
5059 206 : tree extr_cond = vect_recog_temp_ssa_var (itype, NULL);
5060 246 : def_stmt = gimple_build_assign (extr_cond, COND_EXPR, bool_cond,
5061 : is_mod ? oprnd1
5062 40 : : build_int_cst (itype, -1),
5063 : build_int_cst (itype, 0));
5064 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5065 :
5066 : // r += (x ^ y < 0 && r) ? y : 0 (floor mod)
5067 : // d += (x^y < 0 && r) ? -1 : 0 (floor div)
5068 : // r -= (x ^ y < 0 && r) ? y : 0 (ceil mod)
5069 : // d -= (x^y < 0 && r) ? -1 : 0 (ceil div)
5070 206 : tree result = vect_recog_temp_ssa_var (itype, NULL);
5071 412 : return gimple_build_assign (result,
5072 206 : (rhs_code == FLOOR_MOD_EXPR
5073 206 : || rhs_code == FLOOR_DIV_EXPR)
5074 : ? PLUS_EXPR
5075 : : MINUS_EXPR,
5076 206 : is_mod ? r : q, extr_cond);
5077 : }
5078 : }
5079 36 : case ROUND_MOD_EXPR:
5080 36 : case ROUND_DIV_EXPR:
5081 36 : {
5082 36 : if (!target_has_vecop_for_code (BIT_AND_EXPR, vectype)
5083 36 : || !target_has_vecop_for_code (PLUS_EXPR, vectype)
5084 36 : || !expand_vec_cmp_expr_p (vectype, mask_vectype, LT_EXPR)
5085 36 : || !expand_vec_cmp_expr_p (vectype, mask_vectype, GT_EXPR)
5086 72 : || !expand_vec_cond_expr_p (vectype, mask_vectype))
5087 : return NULL;
5088 :
5089 36 : bool is_mod = rhs_code == ROUND_MOD_EXPR;
5090 36 : HOST_WIDE_INT d = TREE_INT_CST_LOW (oprnd1);
5091 36 : unsigned HOST_WIDE_INT abs_d
5092 : = (d >= 0 ? (unsigned HOST_WIDE_INT) d : -(unsigned HOST_WIDE_INT) d);
5093 36 : unsigned HOST_WIDE_INT mid_d = (abs_d - 1) >> 1;
5094 36 : if (!unsigned_p)
5095 : {
5096 : // check availability of abs expression for vector
5097 18 : if (!target_has_vecop_for_code (ABS_EXPR, vectype))
5098 : return NULL;
5099 : // abs (r)
5100 18 : tree abs_r = vect_recog_temp_ssa_var (itype, NULL);
5101 18 : def_stmt = gimple_build_assign (abs_r, ABS_EXPR, r);
5102 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5103 :
5104 : // abs (r) > (abs (y-1) >> 1)
5105 18 : tree round_p = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5106 18 : def_stmt = gimple_build_assign (round_p, GT_EXPR, abs_r,
5107 18 : build_int_cst (itype, mid_d));
5108 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5109 : itype);
5110 :
5111 : // x ^ y
5112 18 : tree cond_reg = vect_recog_temp_ssa_var (itype, NULL);
5113 18 : def_stmt
5114 18 : = gimple_build_assign (cond_reg, BIT_XOR_EXPR, oprnd0, oprnd1);
5115 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5116 :
5117 : // x ^ y < 0
5118 18 : bool_cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5119 18 : def_stmt = gimple_build_assign (bool_cond, LT_EXPR, cond_reg,
5120 : build_int_cst (itype, 0));
5121 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5122 : itype);
5123 :
5124 : // x ^ y < 0 ? y : -y (mod)
5125 : // x ^ y < 0 ? -1 : 1 (div)
5126 18 : tree val1 = vect_recog_temp_ssa_var (itype, NULL);
5127 18 : def_stmt
5128 36 : = gimple_build_assign (val1, COND_EXPR, bool_cond,
5129 27 : build_int_cst (itype, is_mod ? d : -1),
5130 18 : build_int_cst (itype, is_mod ? -d : 1));
5131 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5132 18 : int precision = TYPE_PRECISION (itype);
5133 18 : wide_int wmask = wi::mask (precision, false, precision);
5134 :
5135 : // abs (r) > (abs (y-1) >> 1) ? 0xffffffff : 0
5136 18 : tree val2 = vect_recog_temp_ssa_var (itype, NULL);
5137 36 : def_stmt = gimple_build_assign (val2, COND_EXPR, round_p,
5138 18 : wide_int_to_tree (itype, wmask),
5139 : build_int_cst (itype, 0));
5140 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5141 :
5142 18 : tree fval = vect_recog_temp_ssa_var (itype, NULL);
5143 18 : def_stmt = gimple_build_assign (fval, BIT_AND_EXPR, val1, val2);
5144 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5145 :
5146 18 : tree result = vect_recog_temp_ssa_var (itype, NULL);
5147 27 : return gimple_build_assign (result, PLUS_EXPR, is_mod ? r : q,
5148 : fval);
5149 18 : }
5150 : else
5151 : {
5152 : // r > (y-1 >> 1)
5153 18 : tree round_p = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5154 18 : def_stmt = gimple_build_assign (round_p, GT_EXPR, r,
5155 18 : build_int_cst (itype, mid_d));
5156 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5157 : itype);
5158 :
5159 : // (r > (y-1)>>1) ? -d : 1
5160 18 : tree val2 = vect_recog_temp_ssa_var (itype, NULL);
5161 18 : def_stmt
5162 36 : = gimple_build_assign (val2, COND_EXPR, round_p,
5163 18 : build_int_cst (itype, is_mod ? -d : 1),
5164 : build_int_cst (itype, 0));
5165 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5166 :
5167 18 : tree result = vect_recog_temp_ssa_var (itype, NULL);
5168 27 : return gimple_build_assign (result, PLUS_EXPR, is_mod ? r : q,
5169 18 : val2);
5170 : }
5171 : }
5172 : default:
5173 : return NULL;
5174 : }
5175 : }
5176 :
5177 : /* Detect a signed division by a constant that wouldn't be
5178 : otherwise vectorized:
5179 :
5180 : type a_t, b_t;
5181 :
5182 : S1 a_t = b_t / N;
5183 :
5184 : where type 'type' is an integral type and N is a constant.
5185 :
5186 : Similarly handle modulo by a constant:
5187 :
5188 : S4 a_t = b_t % N;
5189 :
5190 : Input/Output:
5191 :
5192 : * STMT_VINFO: The stmt from which the pattern search begins,
5193 : i.e. the division stmt. S1 is replaced by if N is a power
5194 : of two constant and type is signed:
5195 : S3 y_t = b_t < 0 ? N - 1 : 0;
5196 : S2 x_t = b_t + y_t;
5197 : S1' a_t = x_t >> log2 (N);
5198 :
5199 : S4 is replaced if N is a power of two constant and
5200 : type is signed by (where *_T temporaries have unsigned type):
5201 : S9 y_T = b_t < 0 ? -1U : 0U;
5202 : S8 z_T = y_T >> (sizeof (type_t) * CHAR_BIT - log2 (N));
5203 : S7 z_t = (type) z_T;
5204 : S6 w_t = b_t + z_t;
5205 : S5 x_t = w_t & (N - 1);
5206 : S4' a_t = x_t - z_t;
5207 :
5208 : Output:
5209 :
5210 : * TYPE_OUT: The type of the output of this pattern.
5211 :
5212 : * Return value: A new stmt that will be used to replace the division
5213 : S1 or modulo S4 stmt. */
5214 :
5215 : static gimple *
5216 32417919 : vect_recog_divmod_pattern (vec_info *vinfo,
5217 : stmt_vec_info stmt_vinfo, tree *type_out)
5218 : {
5219 32417919 : gimple *last_stmt = stmt_vinfo->stmt;
5220 32417919 : tree oprnd0, oprnd1, vectype, itype, cond;
5221 32417919 : gimple *pattern_stmt = NULL;
5222 32417919 : gimple *def_stmt = NULL;
5223 32417919 : enum tree_code rhs_code;
5224 32417919 : optab optab;
5225 32417919 : tree q, cst;
5226 32417919 : int prec;
5227 :
5228 32417919 : if (!is_gimple_assign (last_stmt)
5229 : /* The pattern will disrupt the reduction chain with multiple uses. */
5230 32417919 : || vect_is_reduction (stmt_vinfo))
5231 : return NULL;
5232 :
5233 22436571 : rhs_code = gimple_assign_rhs_code (last_stmt);
5234 22436571 : switch (rhs_code)
5235 : {
5236 293668 : case TRUNC_DIV_EXPR:
5237 293668 : case EXACT_DIV_EXPR:
5238 293668 : case TRUNC_MOD_EXPR:
5239 293668 : case FLOOR_MOD_EXPR:
5240 293668 : case FLOOR_DIV_EXPR:
5241 293668 : case CEIL_MOD_EXPR:
5242 293668 : case CEIL_DIV_EXPR:
5243 293668 : case ROUND_MOD_EXPR:
5244 293668 : case ROUND_DIV_EXPR:
5245 293668 : break;
5246 : default:
5247 : return NULL;
5248 : }
5249 :
5250 293668 : oprnd0 = gimple_assign_rhs1 (last_stmt);
5251 293668 : oprnd1 = gimple_assign_rhs2 (last_stmt);
5252 293668 : itype = TREE_TYPE (oprnd0);
5253 293668 : if (TREE_CODE (oprnd0) != SSA_NAME
5254 275099 : || TREE_CODE (oprnd1) != INTEGER_CST
5255 164019 : || TREE_CODE (itype) != INTEGER_TYPE
5256 457687 : || !type_has_mode_precision_p (itype))
5257 : return NULL;
5258 :
5259 164019 : scalar_int_mode itype_mode = SCALAR_INT_TYPE_MODE (itype);
5260 164019 : vectype = get_vectype_for_scalar_type (vinfo, itype);
5261 164019 : if (vectype == NULL_TREE)
5262 : return NULL;
5263 :
5264 131046 : if (optimize_bb_for_size_p (gimple_bb (last_stmt)))
5265 : {
5266 : /* If the target can handle vectorized division or modulo natively,
5267 : don't attempt to optimize this, since native division is likely
5268 : to give smaller code. */
5269 2375 : optab = optab_for_tree_code (rhs_code, vectype, optab_default);
5270 2375 : if (optab != unknown_optab
5271 2375 : && can_implement_p (optab, TYPE_MODE (vectype)))
5272 : return NULL;
5273 : }
5274 :
5275 131046 : prec = TYPE_PRECISION (itype);
5276 :
5277 262092 : bool is_flclrd_moddiv_p
5278 131046 : = rhs_code == FLOOR_MOD_EXPR || rhs_code == FLOOR_DIV_EXPR
5279 : || rhs_code == CEIL_MOD_EXPR || rhs_code == CEIL_DIV_EXPR
5280 130446 : || rhs_code == ROUND_MOD_EXPR || rhs_code == ROUND_DIV_EXPR;
5281 131046 : if (integer_pow2p (oprnd1))
5282 : {
5283 78294 : if ((TYPE_UNSIGNED (itype)
5284 36 : && (rhs_code == FLOOR_MOD_EXPR || rhs_code == FLOOR_DIV_EXPR))
5285 78330 : || tree_int_cst_sgn (oprnd1) != 1)
5286 : return NULL;
5287 :
5288 : /* Pattern detected. */
5289 78294 : vect_pattern_detected ("vect_recog_divmod_pattern", last_stmt);
5290 :
5291 78294 : *type_out = vectype;
5292 :
5293 : /* Check if the target supports this internal function. */
5294 78294 : internal_fn ifn = IFN_DIV_POW2;
5295 78294 : if (!TYPE_UNSIGNED (itype)
5296 78294 : && direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
5297 : {
5298 0 : tree shift = build_int_cst (itype, tree_log2 (oprnd1));
5299 :
5300 0 : tree var_div = vect_recog_temp_ssa_var (itype, NULL);
5301 0 : gimple *div_stmt = gimple_build_call_internal (ifn, 2, oprnd0, shift);
5302 0 : gimple_call_set_lhs (div_stmt, var_div);
5303 0 : if (rhs_code == TRUNC_MOD_EXPR || is_flclrd_moddiv_p)
5304 : {
5305 0 : append_pattern_def_seq (vinfo, stmt_vinfo, div_stmt);
5306 0 : tree t1 = vect_recog_temp_ssa_var (itype, NULL);
5307 0 : def_stmt
5308 0 : = gimple_build_assign (t1, LSHIFT_EXPR, var_div, shift);
5309 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5310 0 : tree r = vect_recog_temp_ssa_var (itype, NULL);
5311 0 : pattern_stmt = gimple_build_assign (r, MINUS_EXPR, oprnd0, t1);
5312 0 : if (is_flclrd_moddiv_p)
5313 : {
5314 0 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5315 0 : pattern_stmt
5316 0 : = add_code_for_floorceilround_divmod (vectype, vinfo,
5317 : stmt_vinfo, rhs_code,
5318 : var_div, r, oprnd0,
5319 : oprnd1, itype);
5320 0 : if (pattern_stmt == NULL)
5321 : return NULL;
5322 : }
5323 : }
5324 : else
5325 : pattern_stmt = div_stmt;
5326 0 : gimple_set_location (pattern_stmt, gimple_location (last_stmt));
5327 :
5328 0 : return pattern_stmt;
5329 : }
5330 :
5331 78294 : cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5332 78294 : def_stmt = gimple_build_assign (cond, LT_EXPR, oprnd0,
5333 : build_int_cst (itype, 0));
5334 78294 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt,
5335 : truth_type_for (vectype), itype);
5336 78294 : tree div_result = NULL_TREE;
5337 78294 : if (rhs_code == TRUNC_DIV_EXPR
5338 78294 : || rhs_code == EXACT_DIV_EXPR
5339 : || rhs_code == FLOOR_DIV_EXPR
5340 2689 : || rhs_code == CEIL_DIV_EXPR
5341 2530 : || rhs_code == ROUND_DIV_EXPR)
5342 : {
5343 75776 : tree var = vect_recog_temp_ssa_var (itype, NULL);
5344 75776 : tree shift;
5345 75776 : def_stmt
5346 75776 : = gimple_build_assign (var, COND_EXPR, cond,
5347 : fold_build2 (MINUS_EXPR, itype, oprnd1,
5348 : build_int_cst (itype, 1)),
5349 : build_int_cst (itype, 0));
5350 75776 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5351 75776 : var = vect_recog_temp_ssa_var (itype, NULL);
5352 75776 : def_stmt
5353 75776 : = gimple_build_assign (var, PLUS_EXPR, oprnd0,
5354 : gimple_assign_lhs (def_stmt));
5355 75776 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5356 :
5357 75776 : shift = build_int_cst (itype, tree_log2 (oprnd1));
5358 75776 : div_result = vect_recog_temp_ssa_var (itype, NULL);
5359 75776 : pattern_stmt
5360 75776 : = gimple_build_assign (div_result, RSHIFT_EXPR, var, shift);
5361 : }
5362 78294 : if (rhs_code == TRUNC_MOD_EXPR || is_flclrd_moddiv_p)
5363 : {
5364 2689 : if (rhs_code == FLOOR_DIV_EXPR
5365 : || rhs_code == CEIL_DIV_EXPR
5366 2689 : || rhs_code == ROUND_DIV_EXPR)
5367 171 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5368 :
5369 2689 : tree signmask;
5370 2689 : if (compare_tree_int (oprnd1, 2) == 0)
5371 : {
5372 1305 : signmask = vect_recog_temp_ssa_var (itype, NULL);
5373 1305 : def_stmt = gimple_build_assign (signmask, COND_EXPR, cond,
5374 : build_int_cst (itype, 1),
5375 : build_int_cst (itype, 0));
5376 1305 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5377 : }
5378 : else
5379 : {
5380 1384 : tree utype
5381 1384 : = build_nonstandard_integer_type (prec, 1);
5382 1384 : tree vecutype = get_vectype_for_scalar_type (vinfo, utype);
5383 1384 : tree shift
5384 1384 : = build_int_cst (utype, GET_MODE_BITSIZE (itype_mode)
5385 1384 : - tree_log2 (oprnd1));
5386 1384 : tree var = vect_recog_temp_ssa_var (utype, NULL);
5387 :
5388 1384 : def_stmt = gimple_build_assign (var, COND_EXPR, cond,
5389 : build_int_cst (utype, -1),
5390 : build_int_cst (utype, 0));
5391 1384 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecutype);
5392 1384 : var = vect_recog_temp_ssa_var (utype, NULL);
5393 1384 : def_stmt = gimple_build_assign (var, RSHIFT_EXPR,
5394 : gimple_assign_lhs (def_stmt),
5395 : shift);
5396 1384 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecutype);
5397 1384 : signmask = vect_recog_temp_ssa_var (itype, NULL);
5398 1384 : def_stmt
5399 1384 : = gimple_build_assign (signmask, NOP_EXPR, var);
5400 1384 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5401 : }
5402 2689 : def_stmt
5403 2689 : = gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL),
5404 : PLUS_EXPR, oprnd0, signmask);
5405 2689 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5406 2689 : def_stmt
5407 2689 : = gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL),
5408 : BIT_AND_EXPR, gimple_assign_lhs (def_stmt),
5409 : fold_build2 (MINUS_EXPR, itype, oprnd1,
5410 : build_int_cst (itype, 1)));
5411 2689 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5412 :
5413 2689 : tree r = vect_recog_temp_ssa_var (itype, NULL);
5414 2689 : pattern_stmt
5415 2689 : = gimple_build_assign (r, MINUS_EXPR, gimple_assign_lhs (def_stmt),
5416 : signmask);
5417 2689 : if (is_flclrd_moddiv_p)
5418 : {
5419 265 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5420 265 : pattern_stmt
5421 265 : = add_code_for_floorceilround_divmod (vectype, vinfo,
5422 : stmt_vinfo, rhs_code,
5423 : div_result, r, oprnd0,
5424 : oprnd1, itype);
5425 265 : if (pattern_stmt == NULL)
5426 : return NULL;
5427 : }
5428 : }
5429 :
5430 78143 : return pattern_stmt;
5431 : }
5432 :
5433 52752 : if ((cst = uniform_integer_cst_p (oprnd1))
5434 52752 : && TYPE_UNSIGNED (itype)
5435 : && rhs_code == TRUNC_DIV_EXPR
5436 29914 : && vectype
5437 70780 : && targetm.vectorize.preferred_div_as_shifts_over_mult (vectype))
5438 : {
5439 : /* We can use the relationship:
5440 :
5441 : x // N == ((x+N+2) // (N+1) + x) // (N+1) for 0 <= x < N(N+3)
5442 :
5443 : to optimize cases where N+1 is a power of 2, and where // (N+1)
5444 : is therefore a shift right. When operating in modes that are
5445 : multiples of a byte in size, there are two cases:
5446 :
5447 : (1) N(N+3) is not representable, in which case the question
5448 : becomes whether the replacement expression overflows.
5449 : It is enough to test that x+N+2 does not overflow,
5450 : i.e. that x < MAX-(N+1).
5451 :
5452 : (2) N(N+3) is representable, in which case it is the (only)
5453 : bound that we need to check.
5454 :
5455 : ??? For now we just handle the case where // (N+1) is a shift
5456 : right by half the precision, since some architectures can
5457 : optimize the associated addition and shift combinations
5458 : into single instructions. */
5459 :
5460 12204 : auto wcst = wi::to_wide (cst);
5461 12204 : int pow = wi::exact_log2 (wcst + 1);
5462 12204 : if (pow == prec / 2)
5463 : {
5464 : /* Check that no overflow will occur. If we don't have range
5465 : information we can't perform the optimization. */
5466 :
5467 472 : int_range_max r;
5468 944 : if (get_range_query (cfun)->range_of_expr (r, oprnd0, last_stmt)
5469 472 : && !r.undefined_p ())
5470 : {
5471 472 : wide_int max = r.upper_bound ();
5472 472 : wide_int one = wi::shwi (1, prec);
5473 472 : wide_int adder = wi::add (one, wi::lshift (one, pow));
5474 472 : wi::overflow_type ovf;
5475 472 : wi::add (max, adder, UNSIGNED, &ovf);
5476 472 : if (ovf == wi::OVF_NONE)
5477 : {
5478 305 : *type_out = vectype;
5479 305 : tree tadder = wide_int_to_tree (itype, adder);
5480 305 : tree rshift = wide_int_to_tree (itype, pow);
5481 :
5482 305 : tree new_lhs1 = vect_recog_temp_ssa_var (itype, NULL);
5483 305 : gassign *patt1
5484 305 : = gimple_build_assign (new_lhs1, PLUS_EXPR, oprnd0, tadder);
5485 305 : append_pattern_def_seq (vinfo, stmt_vinfo, patt1, vectype);
5486 :
5487 305 : tree new_lhs2 = vect_recog_temp_ssa_var (itype, NULL);
5488 305 : patt1 = gimple_build_assign (new_lhs2, RSHIFT_EXPR, new_lhs1,
5489 : rshift);
5490 305 : append_pattern_def_seq (vinfo, stmt_vinfo, patt1, vectype);
5491 :
5492 305 : tree new_lhs3 = vect_recog_temp_ssa_var (itype, NULL);
5493 305 : patt1 = gimple_build_assign (new_lhs3, PLUS_EXPR, new_lhs2,
5494 : oprnd0);
5495 305 : append_pattern_def_seq (vinfo, stmt_vinfo, patt1, vectype);
5496 :
5497 305 : tree new_lhs4 = vect_recog_temp_ssa_var (itype, NULL);
5498 305 : pattern_stmt = gimple_build_assign (new_lhs4, RSHIFT_EXPR,
5499 : new_lhs3, rshift);
5500 :
5501 305 : return pattern_stmt;
5502 : }
5503 472 : }
5504 472 : }
5505 : }
5506 :
5507 52447 : if (prec > HOST_BITS_PER_WIDE_INT
5508 52447 : || integer_zerop (oprnd1))
5509 : return NULL;
5510 :
5511 52156 : if (!can_mult_highpart_p (TYPE_MODE (vectype), TYPE_UNSIGNED (itype)))
5512 : return NULL;
5513 :
5514 13760 : if (TYPE_UNSIGNED (itype))
5515 : {
5516 8926 : unsigned HOST_WIDE_INT mh, ml;
5517 8926 : int pre_shift, post_shift;
5518 8926 : unsigned HOST_WIDE_INT d = (TREE_INT_CST_LOW (oprnd1)
5519 8926 : & GET_MODE_MASK (itype_mode));
5520 8926 : tree t1, t2, t3, t4;
5521 :
5522 8926 : if (d >= (HOST_WIDE_INT_1U << (prec - 1)))
5523 : /* FIXME: Can transform this into oprnd0 >= oprnd1 ? 1 : 0. */
5524 23 : return NULL;
5525 :
5526 : /* Find a suitable multiplier and right shift count instead of
5527 : directly dividing by D. */
5528 8903 : mh = choose_multiplier (d, prec, prec, &ml, &post_shift);
5529 :
5530 : /* If the suggested multiplier is more than PREC bits, we can do better
5531 : for even divisors, using an initial right shift. */
5532 8903 : if (mh != 0 && (d & 1) == 0)
5533 : {
5534 248 : pre_shift = ctz_or_zero (d);
5535 248 : mh = choose_multiplier (d >> pre_shift, prec, prec - pre_shift,
5536 : &ml, &post_shift);
5537 248 : gcc_assert (!mh);
5538 : }
5539 : else
5540 : pre_shift = 0;
5541 :
5542 651 : if (mh != 0)
5543 : {
5544 651 : if (post_shift - 1 >= prec)
5545 : return NULL;
5546 :
5547 : /* t1 = oprnd0 h* ml;
5548 : t2 = oprnd0 - t1;
5549 : t3 = t2 >> 1;
5550 : t4 = t1 + t3;
5551 : q = t4 >> (post_shift - 1); */
5552 651 : t1 = vect_recog_temp_ssa_var (itype, NULL);
5553 651 : def_stmt = gimple_build_assign (t1, MULT_HIGHPART_EXPR, oprnd0,
5554 651 : build_int_cst (itype, ml));
5555 651 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5556 :
5557 651 : t2 = vect_recog_temp_ssa_var (itype, NULL);
5558 651 : def_stmt
5559 651 : = gimple_build_assign (t2, MINUS_EXPR, oprnd0, t1);
5560 651 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5561 :
5562 651 : t3 = vect_recog_temp_ssa_var (itype, NULL);
5563 651 : def_stmt
5564 651 : = gimple_build_assign (t3, RSHIFT_EXPR, t2, integer_one_node);
5565 651 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5566 :
5567 651 : t4 = vect_recog_temp_ssa_var (itype, NULL);
5568 651 : def_stmt
5569 651 : = gimple_build_assign (t4, PLUS_EXPR, t1, t3);
5570 :
5571 651 : if (post_shift != 1)
5572 : {
5573 651 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5574 :
5575 651 : q = vect_recog_temp_ssa_var (itype, NULL);
5576 651 : pattern_stmt
5577 651 : = gimple_build_assign (q, RSHIFT_EXPR, t4,
5578 651 : build_int_cst (itype, post_shift - 1));
5579 : }
5580 : else
5581 : {
5582 : q = t4;
5583 : pattern_stmt = def_stmt;
5584 : }
5585 : }
5586 : else
5587 : {
5588 8252 : if (pre_shift >= prec || post_shift >= prec)
5589 : return NULL;
5590 :
5591 : /* t1 = oprnd0 >> pre_shift;
5592 : t2 = t1 h* ml;
5593 : q = t2 >> post_shift; */
5594 8252 : if (pre_shift)
5595 : {
5596 248 : t1 = vect_recog_temp_ssa_var (itype, NULL);
5597 248 : def_stmt
5598 248 : = gimple_build_assign (t1, RSHIFT_EXPR, oprnd0,
5599 248 : build_int_cst (NULL, pre_shift));
5600 248 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5601 : }
5602 : else
5603 : t1 = oprnd0;
5604 :
5605 8252 : t2 = vect_recog_temp_ssa_var (itype, NULL);
5606 8252 : def_stmt = gimple_build_assign (t2, MULT_HIGHPART_EXPR, t1,
5607 8252 : build_int_cst (itype, ml));
5608 :
5609 8252 : if (post_shift)
5610 : {
5611 8242 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5612 :
5613 8242 : q = vect_recog_temp_ssa_var (itype, NULL);
5614 8242 : def_stmt
5615 8242 : = gimple_build_assign (q, RSHIFT_EXPR, t2,
5616 8242 : build_int_cst (itype, post_shift));
5617 : }
5618 : else
5619 : q = t2;
5620 :
5621 : pattern_stmt = def_stmt;
5622 : }
5623 : }
5624 : else
5625 : {
5626 4834 : unsigned HOST_WIDE_INT ml;
5627 4834 : int post_shift;
5628 4834 : HOST_WIDE_INT d = TREE_INT_CST_LOW (oprnd1);
5629 4834 : unsigned HOST_WIDE_INT abs_d;
5630 4834 : bool add = false;
5631 4834 : tree t1, t2, t3, t4;
5632 :
5633 : /* Give up for -1. */
5634 4834 : if (d == -1)
5635 0 : return NULL;
5636 :
5637 : /* Since d might be INT_MIN, we have to cast to
5638 : unsigned HOST_WIDE_INT before negating to avoid
5639 : undefined signed overflow. */
5640 4834 : abs_d = (d >= 0
5641 4834 : ? (unsigned HOST_WIDE_INT) d
5642 : : - (unsigned HOST_WIDE_INT) d);
5643 :
5644 : /* n rem d = n rem -d */
5645 4834 : if (rhs_code == TRUNC_MOD_EXPR && d < 0)
5646 : {
5647 0 : d = abs_d;
5648 0 : oprnd1 = build_int_cst (itype, abs_d);
5649 : }
5650 4834 : if (HOST_BITS_PER_WIDE_INT >= prec
5651 4834 : && abs_d == HOST_WIDE_INT_1U << (prec - 1))
5652 : /* This case is not handled correctly below. */
5653 : return NULL;
5654 :
5655 4834 : choose_multiplier (abs_d, prec, prec - 1, &ml, &post_shift);
5656 4834 : if (ml >= HOST_WIDE_INT_1U << (prec - 1))
5657 : {
5658 1623 : add = true;
5659 1623 : ml |= HOST_WIDE_INT_M1U << (prec - 1);
5660 : }
5661 4834 : if (post_shift >= prec)
5662 : return NULL;
5663 :
5664 : /* t1 = oprnd0 h* ml; */
5665 4834 : t1 = vect_recog_temp_ssa_var (itype, NULL);
5666 4834 : def_stmt = gimple_build_assign (t1, MULT_HIGHPART_EXPR, oprnd0,
5667 4834 : build_int_cst (itype, ml));
5668 :
5669 4834 : if (add)
5670 : {
5671 : /* t2 = t1 + oprnd0; */
5672 1623 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5673 1623 : t2 = vect_recog_temp_ssa_var (itype, NULL);
5674 1623 : def_stmt = gimple_build_assign (t2, PLUS_EXPR, t1, oprnd0);
5675 : }
5676 : else
5677 : t2 = t1;
5678 :
5679 4834 : if (post_shift)
5680 : {
5681 : /* t3 = t2 >> post_shift; */
5682 4019 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5683 4019 : t3 = vect_recog_temp_ssa_var (itype, NULL);
5684 4019 : def_stmt = gimple_build_assign (t3, RSHIFT_EXPR, t2,
5685 4019 : build_int_cst (itype, post_shift));
5686 : }
5687 : else
5688 : t3 = t2;
5689 :
5690 4834 : int msb = 1;
5691 4834 : int_range_max r;
5692 9668 : get_range_query (cfun)->range_of_expr (r, oprnd0);
5693 4834 : if (!r.varying_p () && !r.undefined_p ())
5694 : {
5695 2975 : if (!wi::neg_p (r.lower_bound (), TYPE_SIGN (itype)))
5696 : msb = 0;
5697 743 : else if (wi::neg_p (r.upper_bound (), TYPE_SIGN (itype)))
5698 : msb = -1;
5699 : }
5700 :
5701 2232 : if (msb == 0 && d >= 0)
5702 : {
5703 : /* q = t3; */
5704 : q = t3;
5705 : pattern_stmt = def_stmt;
5706 : }
5707 : else
5708 : {
5709 : /* t4 = oprnd0 >> (prec - 1);
5710 : or if we know from VRP that oprnd0 >= 0
5711 : t4 = 0;
5712 : or if we know from VRP that oprnd0 < 0
5713 : t4 = -1; */
5714 2662 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5715 2662 : t4 = vect_recog_temp_ssa_var (itype, NULL);
5716 2662 : if (msb != 1)
5717 68 : def_stmt = gimple_build_assign (t4, INTEGER_CST,
5718 68 : build_int_cst (itype, msb));
5719 : else
5720 2594 : def_stmt = gimple_build_assign (t4, RSHIFT_EXPR, oprnd0,
5721 2594 : build_int_cst (itype, prec - 1));
5722 2662 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5723 :
5724 : /* q = t3 - t4; or q = t4 - t3; */
5725 2662 : q = vect_recog_temp_ssa_var (itype, NULL);
5726 5140 : pattern_stmt = gimple_build_assign (q, MINUS_EXPR, d < 0 ? t4 : t3,
5727 : d < 0 ? t3 : t4);
5728 : }
5729 4834 : }
5730 :
5731 13737 : if (rhs_code == TRUNC_MOD_EXPR || is_flclrd_moddiv_p)
5732 : {
5733 6223 : tree r, t1;
5734 :
5735 : /* We divided. Now finish by:
5736 : t1 = q * oprnd1;
5737 : r = oprnd0 - t1; */
5738 6223 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5739 :
5740 6223 : t1 = vect_recog_temp_ssa_var (itype, NULL);
5741 6223 : def_stmt = gimple_build_assign (t1, MULT_EXPR, q, oprnd1);
5742 6223 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5743 :
5744 6223 : r = vect_recog_temp_ssa_var (itype, NULL);
5745 6223 : pattern_stmt = gimple_build_assign (r, MINUS_EXPR, oprnd0, t1);
5746 :
5747 6223 : if (is_flclrd_moddiv_p)
5748 : {
5749 146 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5750 146 : pattern_stmt
5751 146 : = add_code_for_floorceilround_divmod (vectype, vinfo, stmt_vinfo,
5752 : rhs_code, q, r, oprnd0, oprnd1,
5753 : itype);
5754 146 : if (pattern_stmt == NULL)
5755 : return NULL;
5756 : }
5757 : }
5758 :
5759 : /* Pattern detected. */
5760 13737 : vect_pattern_detected ("vect_recog_divmod_pattern", last_stmt);
5761 :
5762 13737 : *type_out = vectype;
5763 13737 : return pattern_stmt;
5764 : }
5765 :
5766 : /* Detects pattern with a modulo operation (S1) where both arguments
5767 : are variables of integral type.
5768 : The statement is replaced by division, multiplication, and subtraction.
5769 : The last statement (S4) is returned.
5770 :
5771 : Example:
5772 : S1 c_t = a_t % b_t;
5773 :
5774 : is replaced by
5775 : S2 x_t = a_t / b_t;
5776 : S3 y_t = x_t * b_t;
5777 : S4 z_t = a_t - y_t; */
5778 :
5779 : static gimple *
5780 32605732 : vect_recog_mod_var_pattern (vec_info *vinfo,
5781 : stmt_vec_info stmt_vinfo, tree *type_out)
5782 : {
5783 32605732 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
5784 32605732 : tree oprnd0, oprnd1, vectype, itype;
5785 32605732 : gimple *pattern_stmt, *def_stmt;
5786 32605732 : enum tree_code rhs_code;
5787 :
5788 32605732 : if (!is_gimple_assign (last_stmt) || vect_is_reduction (stmt_vinfo))
5789 : return NULL;
5790 :
5791 22624384 : rhs_code = gimple_assign_rhs_code (last_stmt);
5792 22624384 : if (rhs_code != TRUNC_MOD_EXPR)
5793 : return NULL;
5794 :
5795 78671 : oprnd0 = gimple_assign_rhs1 (last_stmt);
5796 78671 : oprnd1 = gimple_assign_rhs2 (last_stmt);
5797 78671 : itype = TREE_TYPE (oprnd0);
5798 78671 : if (TREE_CODE (oprnd0) != SSA_NAME
5799 69872 : || TREE_CODE (oprnd1) != SSA_NAME
5800 53199 : || TREE_CODE (itype) != INTEGER_TYPE)
5801 : return NULL;
5802 :
5803 53096 : vectype = get_vectype_for_scalar_type (vinfo, itype);
5804 :
5805 53096 : if (!vectype
5806 44397 : || target_has_vecop_for_code (TRUNC_MOD_EXPR, vectype)
5807 44397 : || !target_has_vecop_for_code (TRUNC_DIV_EXPR, vectype)
5808 0 : || !target_has_vecop_for_code (MULT_EXPR, vectype)
5809 53096 : || !target_has_vecop_for_code (MINUS_EXPR, vectype))
5810 : return NULL;
5811 :
5812 0 : tree q, tmp, r;
5813 0 : q = vect_recog_temp_ssa_var (itype, NULL);
5814 0 : def_stmt = gimple_build_assign (q, TRUNC_DIV_EXPR, oprnd0, oprnd1);
5815 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vectype);
5816 :
5817 0 : tmp = vect_recog_temp_ssa_var (itype, NULL);
5818 0 : def_stmt = gimple_build_assign (tmp, MULT_EXPR, q, oprnd1);
5819 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vectype);
5820 :
5821 0 : r = vect_recog_temp_ssa_var (itype, NULL);
5822 0 : pattern_stmt = gimple_build_assign (r, MINUS_EXPR, oprnd0, tmp);
5823 :
5824 : /* Pattern detected. */
5825 0 : *type_out = vectype;
5826 0 : vect_pattern_detected ("vect_recog_mod_var_pattern", last_stmt);
5827 :
5828 0 : return pattern_stmt;
5829 : }
5830 :
5831 :
5832 : /* Return the proper type for converting bool VAR into
5833 : an integer value or NULL_TREE if no such type exists.
5834 : The type is chosen so that the converted value has the
5835 : same number of elements as VAR's vector type. */
5836 :
5837 : static tree
5838 4828541 : integer_type_for_mask (tree var, vec_info *vinfo, vect_def_type *dt = nullptr)
5839 : {
5840 4828541 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (var)))
5841 : return NULL_TREE;
5842 :
5843 2220569 : stmt_vec_info def_stmt_info = vinfo->lookup_def (var);
5844 2220569 : if (dt)
5845 : {
5846 409018 : if (!def_stmt_info)
5847 3398 : *dt = vect_external_def;
5848 : else
5849 405620 : *dt = STMT_VINFO_DEF_TYPE (def_stmt_info);
5850 : }
5851 409018 : if (!def_stmt_info
5852 2125724 : || STMT_VINFO_DEF_TYPE (def_stmt_info) == vect_external_def
5853 3937275 : || !vect_use_mask_type_p (def_stmt_info))
5854 : return NULL_TREE;
5855 :
5856 1392369 : return build_nonstandard_integer_type (def_stmt_info->mask_precision, 1);
5857 : }
5858 :
5859 : /* Function vect_recog_gcond_pattern
5860 :
5861 : Try to find pattern like following:
5862 :
5863 : if (a op b)
5864 :
5865 : where operator 'op' is not != and convert it to an adjusted boolean pattern
5866 :
5867 : mask = a op b
5868 : if (mask != 0)
5869 :
5870 : and set the mask type on MASK.
5871 :
5872 : Input:
5873 :
5874 : * STMT_VINFO: The stmt at the end from which the pattern
5875 : search begins, i.e. cast of a bool to
5876 : an integer type.
5877 :
5878 : Output:
5879 :
5880 : * TYPE_OUT: The type of the output of this pattern.
5881 :
5882 : * Return value: A new stmt that will be used to replace the pattern. */
5883 :
5884 : static gimple *
5885 32677763 : vect_recog_gcond_pattern (vec_info *vinfo,
5886 : stmt_vec_info stmt_vinfo, tree *type_out)
5887 : {
5888 : /* Currently we only support this for loop vectorization and when multiple
5889 : exits. */
5890 32677763 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
5891 4585908 : if (!loop_vinfo || !LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
5892 : return NULL;
5893 :
5894 1700353 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
5895 1700353 : gcond* cond = NULL;
5896 1700353 : if (!(cond = dyn_cast <gcond *> (last_stmt)))
5897 : return NULL;
5898 :
5899 398289 : auto lhs = gimple_cond_lhs (cond);
5900 398289 : auto rhs = gimple_cond_rhs (cond);
5901 398289 : auto code = gimple_cond_code (cond);
5902 :
5903 398289 : tree scalar_type = TREE_TYPE (lhs);
5904 398289 : if (VECTOR_TYPE_P (scalar_type))
5905 : return NULL;
5906 :
5907 : /* If the input is a boolean then try to figure out the precision that the
5908 : vector type should use. We cannot use the scalar precision as this would
5909 : later mismatch. This is similar to what recog_bool does. */
5910 398289 : if (VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type))
5911 : {
5912 10363 : if (tree stype = integer_type_for_mask (lhs, vinfo))
5913 398289 : scalar_type = stype;
5914 : }
5915 :
5916 398289 : tree vectype = get_mask_type_for_scalar_type (vinfo, scalar_type);
5917 398289 : if (vectype == NULL_TREE)
5918 : return NULL;
5919 :
5920 378085 : tree new_lhs = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5921 378085 : gimple *new_stmt = gimple_build_assign (new_lhs, code, lhs, rhs);
5922 378085 : append_pattern_def_seq (vinfo, stmt_vinfo, new_stmt, vectype, scalar_type);
5923 :
5924 378085 : gimple *pattern_stmt
5925 378085 : = gimple_build_cond (NE_EXPR, new_lhs,
5926 378085 : build_int_cst (TREE_TYPE (new_lhs), 0),
5927 : NULL_TREE, NULL_TREE);
5928 378085 : *type_out = vectype;
5929 378085 : vect_pattern_detected ("vect_recog_gcond_pattern", last_stmt);
5930 378085 : return pattern_stmt;
5931 : }
5932 :
5933 :
5934 : /* A helper for vect_recog_mask_conversion_pattern. Build
5935 : conversion of MASK to a type suitable for masking VECTYPE.
5936 : Built statement gets required vectype and is appended to
5937 : a pattern sequence of STMT_VINFO.
5938 : If VECTYPE is a mask type, SCALAR_TYPE_FOR_MASK is the scalar type
5939 : from which it was derived.
5940 :
5941 : Return converted mask. */
5942 :
5943 : static tree
5944 145106 : build_mask_conversion (vec_info *vinfo,
5945 : tree mask, tree vectype, stmt_vec_info stmt_vinfo,
5946 : tree scalar_type_for_mask = NULL_TREE)
5947 : {
5948 145106 : gimple *stmt;
5949 145106 : tree masktype, tmp;
5950 :
5951 234466 : gcc_assert (!scalar_type_for_mask == !VECTOR_BOOLEAN_TYPE_P (vectype));
5952 :
5953 145106 : masktype = truth_type_for (vectype);
5954 145106 : tmp = vect_recog_temp_ssa_var (TREE_TYPE (masktype), NULL);
5955 145106 : stmt = gimple_build_assign (tmp, CONVERT_EXPR, mask);
5956 234466 : append_pattern_def_seq (vinfo, stmt_vinfo,
5957 : stmt, masktype,
5958 : scalar_type_for_mask
5959 89360 : ? scalar_type_for_mask : TREE_TYPE (vectype));
5960 :
5961 145106 : return tmp;
5962 : }
5963 :
5964 :
5965 : /* Return MASK if MASK is suitable for masking an operation on vectors
5966 : of type VECTYPE, otherwise convert it into such a form and return
5967 : the result. Associate any conversion statements with STMT_INFO's
5968 : pattern. If VECTYPE is a mask type, SCALAR_TYPE_FOR_MASK is the scalar
5969 : type from which it was derived. */
5970 :
5971 : static tree
5972 78730 : vect_convert_mask_for_vectype (tree mask, tree vectype,
5973 : stmt_vec_info stmt_info, vec_info *vinfo,
5974 : tree scalar_type_for_mask = NULL_TREE)
5975 : {
5976 78730 : tree mask_type = integer_type_for_mask (mask, vinfo);
5977 78730 : if (mask_type)
5978 : {
5979 78730 : tree mask_vectype = get_mask_type_for_scalar_type (vinfo, mask_type);
5980 78730 : if (mask_vectype
5981 157460 : && maybe_ne (TYPE_VECTOR_SUBPARTS (vectype),
5982 95926 : TYPE_VECTOR_SUBPARTS (mask_vectype)))
5983 61534 : mask = build_mask_conversion (vinfo, mask, vectype, stmt_info,
5984 : scalar_type_for_mask);
5985 : }
5986 78730 : return mask;
5987 : }
5988 :
5989 :
5990 : /* Function vect_recog_bool_pattern
5991 :
5992 : Try to find pattern like following:
5993 :
5994 : bool a_b, b_b, c_b, d_b, e_b;
5995 : TYPE f_T;
5996 : loop:
5997 : S1 a_b = x1 CMP1 y1;
5998 : S2 b_b = x2 CMP2 y2;
5999 : S3 c_b = a_b & b_b;
6000 : S4 d_b = x3 CMP3 y3;
6001 : S5 e_b = c_b | d_b;
6002 : S6 f_T = (TYPE) e_b;
6003 :
6004 : where type 'TYPE' is an integral type. Or a similar pattern
6005 : ending in
6006 :
6007 : S6 f_Y = e_b ? r_Y : s_Y;
6008 :
6009 : as results from if-conversion of a complex condition.
6010 :
6011 : Input:
6012 :
6013 : * STMT_VINFO: The stmt at the end from which the pattern
6014 : search begins, i.e. cast of a bool to
6015 : an integer type.
6016 :
6017 : Output:
6018 :
6019 : * TYPE_OUT: The type of the output of this pattern.
6020 :
6021 : * Return value: A new stmt that will be used to replace the pattern.
6022 :
6023 : Assuming size of TYPE is the same as size of all comparisons
6024 : (otherwise some casts would be added where needed), the above
6025 : sequence we create related pattern stmts:
6026 : S1' a_T = x1 CMP1 y1 ? 1 : 0;
6027 : S3' c_T = x2 CMP2 y2 ? a_T : 0;
6028 : S4' d_T = x3 CMP3 y3 ? 1 : 0;
6029 : S5' e_T = c_T | d_T;
6030 : S6' f_T = e_T;
6031 :
6032 : Instead of the above S3' we could emit:
6033 : S2' b_T = x2 CMP2 y2 ? 1 : 0;
6034 : S3' c_T = a_T | b_T;
6035 : but the above is more efficient. */
6036 :
6037 : static gimple *
6038 32677763 : vect_recog_bool_pattern (vec_info *vinfo,
6039 : stmt_vec_info stmt_vinfo, tree *type_out)
6040 : {
6041 32677763 : gimple *last_stmt = stmt_vinfo->stmt;
6042 32677763 : enum tree_code rhs_code;
6043 32677763 : tree var, lhs, rhs, vectype;
6044 32677763 : gimple *pattern_stmt;
6045 :
6046 32677763 : if (!is_gimple_assign (last_stmt))
6047 : return NULL;
6048 :
6049 23187254 : var = gimple_assign_rhs1 (last_stmt);
6050 23187254 : lhs = gimple_assign_lhs (last_stmt);
6051 23187254 : rhs_code = gimple_assign_rhs_code (last_stmt);
6052 :
6053 23187254 : if (rhs_code == VIEW_CONVERT_EXPR)
6054 193541 : var = TREE_OPERAND (var, 0);
6055 :
6056 23187254 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (var)))
6057 : return NULL;
6058 :
6059 781725 : hash_set<gimple *> bool_stmts;
6060 :
6061 781725 : if (CONVERT_EXPR_CODE_P (rhs_code)
6062 : || rhs_code == VIEW_CONVERT_EXPR
6063 : || rhs_code == FLOAT_EXPR)
6064 : {
6065 176650 : if (! (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
6066 2423 : || SCALAR_FLOAT_TYPE_P (TREE_TYPE (lhs)))
6067 174761 : || VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)))
6068 : return NULL;
6069 81962 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6070 :
6071 81962 : tree type = integer_type_for_mask (var, vinfo);
6072 81962 : tree cst0, cst1, tmp;
6073 :
6074 81962 : if (!type)
6075 : return NULL;
6076 :
6077 : /* We may directly use cond with narrowed type to avoid multiple cond
6078 : exprs with following result packing and perform single cond with
6079 : packed mask instead. In case of widening we better make cond first
6080 : and then extract results. */
6081 43026 : if (TYPE_MODE (type) == TYPE_MODE (TREE_TYPE (lhs)))
6082 29925 : type = TREE_TYPE (lhs);
6083 :
6084 43026 : cst0 = build_int_cst (type, 0);
6085 43026 : cst1 = build_int_cst (type, 1);
6086 43026 : tmp = vect_recog_temp_ssa_var (type, NULL);
6087 43026 : pattern_stmt = gimple_build_assign (tmp, COND_EXPR, var, cst1, cst0);
6088 :
6089 43026 : if (!useless_type_conversion_p (type, TREE_TYPE (lhs)))
6090 : {
6091 13101 : tree new_vectype = get_vectype_for_scalar_type (vinfo, type);
6092 13101 : append_pattern_def_seq (vinfo, stmt_vinfo,
6093 : pattern_stmt, new_vectype);
6094 :
6095 13101 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6096 13101 : pattern_stmt
6097 25894 : = gimple_build_assign (lhs, (rhs_code == FLOAT_EXPR
6098 : ? FLOAT_EXPR : CONVERT_EXPR), tmp);
6099 : }
6100 :
6101 43026 : *type_out = vectype;
6102 43026 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6103 :
6104 43026 : return pattern_stmt;
6105 : }
6106 : else if (rhs_code == COND_EXPR
6107 212816 : && TREE_CODE (var) == SSA_NAME)
6108 : {
6109 212816 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6110 212816 : if (vectype == NULL_TREE)
6111 : return NULL;
6112 :
6113 : /* Build a scalar type for the boolean result that when
6114 : vectorized matches the vector type of the result in
6115 : size and number of elements. */
6116 197512 : unsigned prec
6117 197512 : = vector_element_size (tree_to_poly_uint64 (TYPE_SIZE (vectype)),
6118 : TYPE_VECTOR_SUBPARTS (vectype));
6119 :
6120 197512 : tree type
6121 395024 : = build_nonstandard_integer_type (prec,
6122 197512 : TYPE_UNSIGNED (TREE_TYPE (var)));
6123 197512 : if (get_vectype_for_scalar_type (vinfo, type) == NULL_TREE)
6124 : return NULL;
6125 :
6126 197512 : enum vect_def_type dt;
6127 197512 : if (integer_type_for_mask (var, vinfo))
6128 : return NULL;
6129 36608 : else if (TREE_CODE (TREE_TYPE (var)) == BOOLEAN_TYPE
6130 36608 : && vect_is_simple_use (var, vinfo, &dt)
6131 36608 : && (dt == vect_external_def
6132 36601 : || dt == vect_constant_def))
6133 : {
6134 : /* If the condition is already a boolean then manually convert it to a
6135 : mask of the given integer type but don't set a vectype. */
6136 1391 : tree lhs_ivar = vect_recog_temp_ssa_var (type, NULL);
6137 1391 : pattern_stmt = gimple_build_assign (lhs_ivar, COND_EXPR, var,
6138 : build_all_ones_cst (type),
6139 : build_zero_cst (type));
6140 1391 : append_inv_pattern_def_seq (vinfo, pattern_stmt);
6141 1391 : var = lhs_ivar;
6142 : }
6143 :
6144 36608 : tree lhs_var = vect_recog_temp_ssa_var (boolean_type_node, NULL);
6145 36608 : pattern_stmt = gimple_build_assign (lhs_var, NE_EXPR, var,
6146 36608 : build_zero_cst (TREE_TYPE (var)));
6147 :
6148 36608 : tree new_vectype = get_mask_type_for_scalar_type (vinfo, TREE_TYPE (var));
6149 36608 : if (!new_vectype)
6150 : return NULL;
6151 :
6152 36608 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, new_vectype,
6153 36608 : TREE_TYPE (var));
6154 :
6155 36608 : lhs_var = vect_convert_mask_for_vectype (lhs_var, vectype, stmt_vinfo,
6156 : vinfo);
6157 :
6158 36608 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6159 36608 : pattern_stmt
6160 36608 : = gimple_build_assign (lhs, COND_EXPR, lhs_var,
6161 : gimple_assign_rhs2 (last_stmt),
6162 : gimple_assign_rhs3 (last_stmt));
6163 36608 : *type_out = vectype;
6164 36608 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6165 :
6166 36608 : return pattern_stmt;
6167 : }
6168 480209 : else if (rhs_code == BIT_NOT_EXPR
6169 480209 : && !vect_use_mask_type_p (stmt_vinfo))
6170 : {
6171 : /* When we have a bool data inversion rewrite that to an XOR to
6172 : cope with the fact that we'll use a wider vector element type. */
6173 9457 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6174 9457 : pattern_stmt
6175 9457 : = gimple_build_assign (lhs, BIT_XOR_EXPR, var,
6176 9457 : build_all_ones_cst (TREE_TYPE (var)));
6177 9457 : *type_out = NULL_TREE;
6178 9457 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6179 :
6180 9457 : return pattern_stmt;
6181 : }
6182 470752 : else if ((rhs_code == BIT_XOR_EXPR
6183 : || rhs_code == BIT_AND_EXPR
6184 470752 : || rhs_code == BIT_IOR_EXPR)
6185 376358 : && TREE_CODE (var) == SSA_NAME)
6186 : {
6187 376358 : tree rhs2 = gimple_assign_rhs2 (last_stmt);
6188 376358 : if (TREE_CODE (rhs2) != SSA_NAME)
6189 : return NULL;
6190 376358 : tree lhs_type = integer_type_for_mask (lhs, vinfo);
6191 376358 : if (!lhs_type)
6192 : return NULL;
6193 204509 : vectype = get_mask_type_for_scalar_type (vinfo, lhs_type);
6194 204509 : if (!vectype)
6195 : return NULL;
6196 204509 : vect_def_type dt1, dt2;
6197 204509 : tree rhs1_type = integer_type_for_mask (var, vinfo, &dt1);
6198 204509 : tree rhs2_type = integer_type_for_mask (rhs2, vinfo, &dt2);
6199 204509 : if ((rhs1_type || dt1 == vect_external_def)
6200 189425 : && (rhs2_type || dt2 == vect_external_def))
6201 : return NULL;
6202 : /* When one input is a mask and the other is not create a pattern
6203 : stmt sequence that creates a mask for the non-mask input and
6204 : convert it to one suitable for the output mask used. */
6205 37689 : if (rhs1_type && !rhs2_type)
6206 : {
6207 22605 : tree rhs1_vectype = get_mask_type_for_scalar_type (vinfo, rhs1_type);
6208 22605 : if (!rhs1_vectype)
6209 : return NULL;
6210 22605 : tree rhs2_vectype = get_vectype_for_scalar_type (vinfo,
6211 22605 : TREE_TYPE (rhs2));
6212 22605 : if (!rhs2_vectype)
6213 : return NULL;
6214 22605 : tree new_vectype = truth_type_for (rhs2_vectype);
6215 22605 : tree tem = vect_recog_temp_ssa_var (TREE_TYPE (new_vectype), NULL);
6216 22605 : pattern_stmt = gimple_build_assign (tem, NE_EXPR, rhs2,
6217 : build_zero_cst
6218 22605 : (TREE_TYPE (rhs2)));
6219 22605 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
6220 22605 : new_vectype, TREE_TYPE (new_vectype));
6221 22605 : rhs2 = vect_convert_mask_for_vectype (tem, rhs1_vectype,
6222 : stmt_vinfo, vinfo, rhs1_type);
6223 : }
6224 15084 : else if (!rhs1_type && rhs2_type)
6225 : {
6226 15084 : tree rhs2_vectype = get_mask_type_for_scalar_type (vinfo, rhs2_type);
6227 15084 : if (!rhs2_vectype)
6228 : return NULL;
6229 15084 : tree rhs1_vectype = get_vectype_for_scalar_type (vinfo,
6230 15084 : TREE_TYPE (var));
6231 15084 : if (!rhs1_vectype)
6232 : return NULL;
6233 15084 : tree new_vectype = truth_type_for (rhs1_vectype);
6234 15084 : tree tem = vect_recog_temp_ssa_var (TREE_TYPE (new_vectype), NULL);
6235 15084 : pattern_stmt = gimple_build_assign (tem, NE_EXPR, var,
6236 : build_zero_cst
6237 15084 : (TREE_TYPE (var)));
6238 15084 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
6239 15084 : new_vectype, TREE_TYPE (new_vectype));
6240 15084 : var = vect_convert_mask_for_vectype (tem, rhs2_vectype,
6241 : stmt_vinfo, vinfo, rhs2_type);
6242 : }
6243 37689 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6244 37689 : pattern_stmt = gimple_build_assign (lhs, rhs_code, var, rhs2);
6245 37689 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6246 37689 : *type_out = vectype;
6247 37689 : return pattern_stmt;
6248 : }
6249 94394 : else if (rhs_code == SSA_NAME
6250 25708 : && STMT_VINFO_DATA_REF (stmt_vinfo))
6251 : {
6252 7807 : stmt_vec_info pattern_stmt_info;
6253 7807 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6254 7807 : if (!vectype || !VECTOR_MODE_P (TYPE_MODE (vectype)))
6255 : return NULL;
6256 :
6257 7807 : tree type = integer_type_for_mask (var, vinfo);
6258 7807 : if (!type)
6259 : return NULL;
6260 :
6261 4433 : var = vect_convert_mask_for_vectype (var, vectype, stmt_vinfo, vinfo);
6262 :
6263 4433 : tree cst0 = build_int_cst (TREE_TYPE (vectype), 0);
6264 4433 : tree cst1 = build_int_cst (TREE_TYPE (vectype), 1);
6265 4433 : rhs = vect_recog_temp_ssa_var (TREE_TYPE (vectype), NULL);
6266 4433 : pattern_stmt = gimple_build_assign (rhs, COND_EXPR, var, cst1, cst0);
6267 4433 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vectype);
6268 :
6269 4433 : lhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (vectype), lhs);
6270 4433 : pattern_stmt = gimple_build_assign (lhs, SSA_NAME, rhs);
6271 4433 : pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
6272 4433 : vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
6273 4433 : *type_out = vectype;
6274 4433 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6275 :
6276 4433 : return pattern_stmt;
6277 : }
6278 : else
6279 : return NULL;
6280 781725 : }
6281 :
6282 :
6283 : /* Function vect_recog_mask_conversion_pattern
6284 :
6285 : Try to find statements which require boolean type
6286 : conversion. Additional conversion statements are
6287 : added to handle such cases. For example:
6288 :
6289 : bool m_1, m_2, m_3;
6290 : int i_4, i_5;
6291 : double d_6, d_7;
6292 : char c_1, c_2, c_3;
6293 :
6294 : S1 m_1 = i_4 > i_5;
6295 : S2 m_2 = d_6 < d_7;
6296 : S3 m_3 = m_1 & m_2;
6297 : S4 c_1 = m_3 ? c_2 : c_3;
6298 :
6299 : Will be transformed into:
6300 :
6301 : S1 m_1 = i_4 > i_5;
6302 : S2 m_2 = d_6 < d_7;
6303 : S3'' m_2' = (_Bool[bitsize=32])m_2
6304 : S3' m_3' = m_1 & m_2';
6305 : S4'' m_3'' = (_Bool[bitsize=8])m_3'
6306 : S4' c_1' = m_3'' ? c_2 : c_3; */
6307 :
6308 : static gimple *
6309 32700051 : vect_recog_mask_conversion_pattern (vec_info *vinfo,
6310 : stmt_vec_info stmt_vinfo, tree *type_out)
6311 : {
6312 32700051 : gimple *last_stmt = stmt_vinfo->stmt;
6313 32700051 : enum tree_code rhs_code;
6314 32700051 : tree lhs = NULL_TREE, rhs1, rhs2, tmp, rhs1_type, rhs2_type;
6315 32700051 : tree vectype1, vectype2;
6316 32700051 : stmt_vec_info pattern_stmt_info;
6317 :
6318 : /* Check for MASK_LOAD and MASK_STORE as well as COND_OP calls requiring mask
6319 : conversion. */
6320 32700051 : if (is_gimple_call (last_stmt)
6321 32700051 : && gimple_call_internal_p (last_stmt))
6322 : {
6323 128300 : gcall *pattern_stmt;
6324 :
6325 128300 : internal_fn ifn = gimple_call_internal_fn (last_stmt);
6326 128300 : int mask_argno = internal_fn_mask_index (ifn);
6327 128300 : if (mask_argno < 0)
6328 : return NULL;
6329 :
6330 15099 : bool store_p = internal_store_fn_p (ifn);
6331 15099 : bool load_p = internal_store_fn_p (ifn);
6332 15099 : if (store_p)
6333 : {
6334 2488 : int rhs_index = internal_fn_stored_value_index (ifn);
6335 2488 : tree rhs = gimple_call_arg (last_stmt, rhs_index);
6336 2488 : vectype1 = get_vectype_for_scalar_type (vinfo, TREE_TYPE (rhs));
6337 : }
6338 : else
6339 : {
6340 12611 : lhs = gimple_call_lhs (last_stmt);
6341 12611 : if (!lhs)
6342 : return NULL;
6343 12611 : vectype1 = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6344 : }
6345 :
6346 15099 : if (!vectype1)
6347 : return NULL;
6348 :
6349 14779 : tree mask_arg = gimple_call_arg (last_stmt, mask_argno);
6350 14779 : tree mask_arg_type = integer_type_for_mask (mask_arg, vinfo);
6351 14779 : if (mask_arg_type)
6352 : {
6353 13138 : vectype2 = get_mask_type_for_scalar_type (vinfo, mask_arg_type);
6354 :
6355 13138 : if (!vectype2
6356 13138 : || known_eq (TYPE_VECTOR_SUBPARTS (vectype1),
6357 : TYPE_VECTOR_SUBPARTS (vectype2)))
6358 : return NULL;
6359 : }
6360 1641 : else if (store_p || load_p)
6361 : return NULL;
6362 :
6363 6220 : tmp = build_mask_conversion (vinfo, mask_arg, vectype1, stmt_vinfo);
6364 :
6365 6220 : auto_vec<tree, 8> args;
6366 6220 : unsigned int nargs = gimple_call_num_args (last_stmt);
6367 6220 : args.safe_grow (nargs, true);
6368 37320 : for (unsigned int i = 0; i < nargs; ++i)
6369 24880 : args[i] = ((int) i == mask_argno
6370 24880 : ? tmp
6371 18660 : : gimple_call_arg (last_stmt, i));
6372 6220 : pattern_stmt = gimple_build_call_internal_vec (ifn, args);
6373 :
6374 6220 : if (!store_p)
6375 : {
6376 5948 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6377 5948 : gimple_call_set_lhs (pattern_stmt, lhs);
6378 : }
6379 :
6380 5948 : if (load_p || store_p)
6381 272 : gimple_call_set_nothrow (pattern_stmt, true);
6382 :
6383 6220 : pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
6384 6220 : if (STMT_VINFO_DATA_REF (stmt_vinfo))
6385 1924 : vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
6386 :
6387 6220 : *type_out = vectype1;
6388 6220 : vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
6389 :
6390 6220 : return pattern_stmt;
6391 6220 : }
6392 :
6393 32571751 : if (!is_gimple_assign (last_stmt))
6394 : return NULL;
6395 :
6396 23209542 : gimple *pattern_stmt;
6397 23209542 : lhs = gimple_assign_lhs (last_stmt);
6398 23209542 : rhs1 = gimple_assign_rhs1 (last_stmt);
6399 23209542 : rhs_code = gimple_assign_rhs_code (last_stmt);
6400 :
6401 : /* Check for cond expression requiring mask conversion. */
6402 23209542 : if (rhs_code == COND_EXPR)
6403 : {
6404 193878 : vectype1 = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6405 :
6406 193878 : gcc_assert (! COMPARISON_CLASS_P (rhs1));
6407 193878 : if (TREE_CODE (rhs1) == SSA_NAME)
6408 : {
6409 193878 : rhs1_type = integer_type_for_mask (rhs1, vinfo);
6410 193878 : if (!rhs1_type)
6411 : return NULL;
6412 : }
6413 : else
6414 : return NULL;
6415 :
6416 180726 : vectype2 = get_mask_type_for_scalar_type (vinfo, rhs1_type);
6417 :
6418 180726 : if (!vectype1 || !vectype2)
6419 : return NULL;
6420 :
6421 : /* Continue if a conversion is needed. Also continue if we have
6422 : a comparison whose vector type would normally be different from
6423 : VECTYPE2 when considered in isolation. In that case we'll
6424 : replace the comparison with an SSA name (so that we can record
6425 : its vector type) and behave as though the comparison was an SSA
6426 : name from the outset. */
6427 178574 : if (known_eq (TYPE_VECTOR_SUBPARTS (vectype1),
6428 : TYPE_VECTOR_SUBPARTS (vectype2)))
6429 : return NULL;
6430 :
6431 45571 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1),
6432 91142 : TYPE_VECTOR_SUBPARTS (vectype2)))
6433 45571 : tmp = build_mask_conversion (vinfo, rhs1, vectype1, stmt_vinfo);
6434 : else
6435 : tmp = rhs1;
6436 :
6437 45571 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6438 45571 : pattern_stmt = gimple_build_assign (lhs, COND_EXPR, tmp,
6439 : gimple_assign_rhs2 (last_stmt),
6440 : gimple_assign_rhs3 (last_stmt));
6441 :
6442 45571 : *type_out = vectype1;
6443 45571 : vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
6444 :
6445 45571 : return pattern_stmt;
6446 : }
6447 :
6448 : /* Now check for binary boolean operations requiring conversion for
6449 : one of operands. */
6450 23015664 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)))
6451 : return NULL;
6452 :
6453 1912418 : if (rhs_code != BIT_IOR_EXPR
6454 : && rhs_code != BIT_XOR_EXPR
6455 1912418 : && rhs_code != BIT_AND_EXPR
6456 1573749 : && TREE_CODE_CLASS (rhs_code) != tcc_comparison)
6457 : return NULL;
6458 :
6459 1729067 : rhs2 = gimple_assign_rhs2 (last_stmt);
6460 :
6461 1729067 : rhs1_type = integer_type_for_mask (rhs1, vinfo);
6462 1729067 : rhs2_type = integer_type_for_mask (rhs2, vinfo);
6463 :
6464 1729067 : if (!rhs1_type || !rhs2_type
6465 1729067 : || TYPE_PRECISION (rhs1_type) == TYPE_PRECISION (rhs2_type))
6466 : return NULL;
6467 :
6468 31781 : if (TYPE_PRECISION (rhs1_type) < TYPE_PRECISION (rhs2_type))
6469 : {
6470 24428 : vectype1 = get_mask_type_for_scalar_type (vinfo, rhs1_type);
6471 24428 : if (!vectype1)
6472 : return NULL;
6473 24428 : rhs2 = build_mask_conversion (vinfo, rhs2, vectype1, stmt_vinfo,
6474 : rhs2_type);
6475 : }
6476 : else
6477 : {
6478 7353 : vectype1 = get_mask_type_for_scalar_type (vinfo, rhs2_type);
6479 7353 : if (!vectype1)
6480 : return NULL;
6481 7353 : rhs1 = build_mask_conversion (vinfo, rhs1, vectype1, stmt_vinfo,
6482 : rhs2_type);
6483 : }
6484 :
6485 31781 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6486 31781 : pattern_stmt = gimple_build_assign (lhs, rhs_code, rhs1, rhs2);
6487 :
6488 31781 : *type_out = vectype1;
6489 31781 : vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
6490 :
6491 31781 : return pattern_stmt;
6492 : }
6493 :
6494 : /* STMT_INFO is a load or store. If the load or store is conditional, return
6495 : the boolean condition under which it occurs, otherwise return null. */
6496 :
6497 : static tree
6498 102582 : vect_get_load_store_mask (stmt_vec_info stmt_info)
6499 : {
6500 102582 : if (gassign *def_assign = dyn_cast <gassign *> (stmt_info->stmt))
6501 : {
6502 101109 : gcc_assert (gimple_assign_single_p (def_assign));
6503 : return NULL_TREE;
6504 : }
6505 :
6506 1473 : if (gcall *def_call = dyn_cast <gcall *> (stmt_info->stmt))
6507 : {
6508 1473 : internal_fn ifn = gimple_call_internal_fn (def_call);
6509 1473 : int mask_index = internal_fn_mask_index (ifn);
6510 1473 : return gimple_call_arg (def_call, mask_index);
6511 : }
6512 :
6513 0 : gcc_unreachable ();
6514 : }
6515 :
6516 : /* Return the equivalent of:
6517 :
6518 : fold_convert (TYPE, VALUE)
6519 :
6520 : with the expectation that the operation will be vectorized.
6521 : If new statements are needed, add them as pattern statements
6522 : to STMT_INFO. */
6523 :
6524 : static tree
6525 0 : vect_add_conversion_to_pattern (vec_info *vinfo,
6526 : tree type, tree value, stmt_vec_info stmt_info)
6527 : {
6528 0 : if (useless_type_conversion_p (type, TREE_TYPE (value)))
6529 : return value;
6530 :
6531 0 : tree new_value = vect_recog_temp_ssa_var (type, NULL);
6532 0 : gassign *conversion = gimple_build_assign (new_value, CONVERT_EXPR, value);
6533 0 : append_pattern_def_seq (vinfo, stmt_info, conversion,
6534 : get_vectype_for_scalar_type (vinfo, type));
6535 0 : return new_value;
6536 : }
6537 :
6538 : /* Try to convert STMT_INFO into a call to a gather load or scatter store
6539 : internal function. Return the final statement on success and set
6540 : *TYPE_OUT to the vector type being loaded or stored.
6541 :
6542 : This function only handles gathers and scatters that were recognized
6543 : as such from the outset (indicated by STMT_VINFO_GATHER_SCATTER_P). */
6544 :
6545 : static gimple *
6546 32700051 : vect_recog_gather_scatter_pattern (vec_info *vinfo,
6547 : stmt_vec_info stmt_info, tree *type_out)
6548 : {
6549 : /* Currently we only support this for loop vectorization. */
6550 32700051 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6551 4598524 : if (!loop_vinfo)
6552 : return NULL;
6553 :
6554 : /* Make sure that we're looking at a gather load or scatter store. */
6555 4598524 : data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
6556 4598524 : if (!dr || !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
6557 : return NULL;
6558 :
6559 : /* Get the boolean that controls whether the load or store happens.
6560 : This is null if the operation is unconditional. */
6561 102582 : tree mask = vect_get_load_store_mask (stmt_info);
6562 :
6563 : /* DR analysis nailed down the vector type for the access. */
6564 102582 : tree gs_vectype = STMT_VINFO_VECTYPE (stmt_info);
6565 :
6566 : /* Make sure that the target supports an appropriate internal
6567 : function for the gather/scatter operation. */
6568 102582 : gather_scatter_info gs_info;
6569 102582 : if (!vect_check_gather_scatter (stmt_info, gs_vectype, loop_vinfo, &gs_info)
6570 102582 : || gs_info.ifn == IFN_LAST)
6571 : return NULL;
6572 :
6573 : /* Convert the mask to the right form. */
6574 0 : if (mask)
6575 0 : mask = vect_convert_mask_for_vectype (mask, gs_vectype, stmt_info,
6576 : loop_vinfo);
6577 0 : else if (gs_info.ifn == IFN_MASK_SCATTER_STORE
6578 0 : || gs_info.ifn == IFN_MASK_GATHER_LOAD
6579 0 : || gs_info.ifn == IFN_MASK_LEN_SCATTER_STORE
6580 0 : || gs_info.ifn == IFN_MASK_LEN_GATHER_LOAD)
6581 0 : mask = build_int_cst (TREE_TYPE (truth_type_for (gs_vectype)), -1);
6582 :
6583 : /* Get the invariant base and non-invariant offset, converting the
6584 : latter to the same width as the vector elements. */
6585 0 : tree base = gs_info.base;
6586 0 : tree offset_type = TREE_TYPE (gs_info.offset_vectype);
6587 0 : tree offset = vect_add_conversion_to_pattern (vinfo, offset_type,
6588 : gs_info.offset, stmt_info);
6589 :
6590 : /* Build the new pattern statement. */
6591 0 : tree scale = size_int (gs_info.scale);
6592 0 : gcall *pattern_stmt;
6593 :
6594 0 : if (DR_IS_READ (dr))
6595 : {
6596 0 : tree zero = build_zero_cst (gs_info.element_type);
6597 0 : if (mask != NULL)
6598 : {
6599 0 : int elsval = MASK_LOAD_ELSE_ZERO;
6600 :
6601 0 : tree vec_els
6602 0 : = vect_get_mask_load_else (elsval, TREE_TYPE (gs_vectype));
6603 0 : pattern_stmt = gimple_build_call_internal (gs_info.ifn, 7, base,
6604 : gs_info.alias_ptr,
6605 : offset, scale, zero, mask,
6606 : vec_els);
6607 : }
6608 : else
6609 0 : pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5, base,
6610 : gs_info.alias_ptr,
6611 : offset, scale, zero);
6612 0 : tree lhs = gimple_get_lhs (stmt_info->stmt);
6613 0 : tree load_lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6614 0 : gimple_call_set_lhs (pattern_stmt, load_lhs);
6615 : }
6616 : else
6617 : {
6618 0 : tree rhs = vect_get_store_rhs (stmt_info);
6619 0 : if (mask != NULL)
6620 0 : pattern_stmt = gimple_build_call_internal (gs_info.ifn, 6,
6621 : base, gs_info.alias_ptr,
6622 : offset, scale, rhs, mask);
6623 : else
6624 0 : pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5,
6625 : base, gs_info.alias_ptr,
6626 : offset, scale, rhs);
6627 : }
6628 0 : gimple_call_set_nothrow (pattern_stmt, true);
6629 :
6630 : /* Copy across relevant vectorization info and associate DR with the
6631 : new pattern statement instead of the original statement. */
6632 0 : stmt_vec_info pattern_stmt_info = loop_vinfo->add_stmt (pattern_stmt);
6633 0 : loop_vinfo->move_dr (pattern_stmt_info, stmt_info);
6634 :
6635 0 : *type_out = gs_vectype;
6636 0 : vect_pattern_detected ("gather/scatter pattern", stmt_info->stmt);
6637 :
6638 0 : return pattern_stmt;
6639 : }
6640 :
6641 : /* Helper method of vect_recog_cond_store_pattern, checks to see if COND_ARG
6642 : is points to a load statement that reads the same data as that of
6643 : STORE_VINFO. */
6644 :
6645 : static bool
6646 35615 : vect_cond_store_pattern_same_ref (vec_info *vinfo,
6647 : stmt_vec_info store_vinfo, tree cond_arg)
6648 : {
6649 35615 : stmt_vec_info load_stmt_vinfo = vinfo->lookup_def (cond_arg);
6650 35615 : if (!load_stmt_vinfo
6651 20549 : || !STMT_VINFO_DATA_REF (load_stmt_vinfo)
6652 12349 : || DR_IS_WRITE (STMT_VINFO_DATA_REF (load_stmt_vinfo))
6653 47964 : || !same_data_refs (STMT_VINFO_DATA_REF (store_vinfo),
6654 : STMT_VINFO_DATA_REF (load_stmt_vinfo)))
6655 26328 : return false;
6656 :
6657 : return true;
6658 : }
6659 :
6660 : /* Function vect_recog_cond_store_pattern
6661 :
6662 : Try to find the following pattern:
6663 :
6664 : x = *_3;
6665 : c = a CMP b;
6666 : y = c ? t_20 : x;
6667 : *_3 = y;
6668 :
6669 : where the store of _3 happens on a conditional select on a value loaded
6670 : from the same location. In such case we can elide the initial load if
6671 : MASK_STORE is supported and instead only conditionally write out the result.
6672 :
6673 : The pattern produces for the above:
6674 :
6675 : c = a CMP b;
6676 : .MASK_STORE (_3, c, t_20)
6677 :
6678 : Input:
6679 :
6680 : * STMT_VINFO: The stmt from which the pattern search begins. In the
6681 : example, when this function is called with _3 then the search begins.
6682 :
6683 : Output:
6684 :
6685 : * TYPE_OUT: The type of the output of this pattern.
6686 :
6687 : * Return value: A new stmt that will be used to replace the sequence. */
6688 :
6689 : static gimple *
6690 32700051 : vect_recog_cond_store_pattern (vec_info *vinfo,
6691 : stmt_vec_info stmt_vinfo, tree *type_out)
6692 : {
6693 32700051 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6694 4598524 : if (!loop_vinfo)
6695 : return NULL;
6696 :
6697 4598524 : gimple *store_stmt = STMT_VINFO_STMT (stmt_vinfo);
6698 :
6699 : /* Needs to be a gimple store where we have DR info for. */
6700 4598524 : if (!STMT_VINFO_DATA_REF (stmt_vinfo)
6701 1102335 : || DR_IS_READ (STMT_VINFO_DATA_REF (stmt_vinfo))
6702 4987266 : || !gimple_store_p (store_stmt))
6703 : return NULL;
6704 :
6705 386354 : tree st_rhs = gimple_assign_rhs1 (store_stmt);
6706 :
6707 386354 : if (TREE_CODE (st_rhs) != SSA_NAME)
6708 : return NULL;
6709 :
6710 300289 : auto cond_vinfo = vinfo->lookup_def (st_rhs);
6711 :
6712 : /* If the condition isn't part of the loop then bool recog wouldn't have seen
6713 : it and so this transformation may not be valid. */
6714 300289 : if (!cond_vinfo)
6715 : return NULL;
6716 :
6717 282701 : cond_vinfo = vect_stmt_to_vectorize (cond_vinfo);
6718 282701 : gassign *cond_stmt = dyn_cast<gassign *> (STMT_VINFO_STMT (cond_vinfo));
6719 351296 : if (!cond_stmt || gimple_assign_rhs_code (cond_stmt) != COND_EXPR)
6720 : return NULL;
6721 :
6722 : /* Check if the else value matches the original loaded one. */
6723 18843 : bool invert = false;
6724 18843 : tree cmp_ls = gimple_arg (cond_stmt, 0);
6725 18843 : if (TREE_CODE (cmp_ls) != SSA_NAME)
6726 : return NULL;
6727 :
6728 18843 : tree cond_arg1 = gimple_arg (cond_stmt, 1);
6729 18843 : tree cond_arg2 = gimple_arg (cond_stmt, 2);
6730 :
6731 18843 : if (!vect_cond_store_pattern_same_ref (vinfo, stmt_vinfo, cond_arg2)
6732 18843 : && !(invert = vect_cond_store_pattern_same_ref (vinfo, stmt_vinfo,
6733 : cond_arg1)))
6734 : return NULL;
6735 :
6736 9287 : vect_pattern_detected ("vect_recog_cond_store_pattern", store_stmt);
6737 :
6738 9287 : tree scalar_type = TREE_TYPE (st_rhs);
6739 9287 : if (VECTOR_TYPE_P (scalar_type))
6740 : return NULL;
6741 :
6742 9287 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
6743 9287 : if (vectype == NULL_TREE)
6744 : return NULL;
6745 :
6746 9287 : machine_mode mask_mode;
6747 9287 : machine_mode vecmode = TYPE_MODE (vectype);
6748 1858 : if (!VECTOR_MODE_P (vecmode)
6749 9287 : || targetm.vectorize.conditional_operation_is_expensive (IFN_MASK_STORE)
6750 9287 : || !targetm.vectorize.get_mask_mode (vecmode).exists (&mask_mode)
6751 9287 : || !can_vec_mask_load_store_p (vecmode, mask_mode, false))
6752 : return NULL;
6753 :
6754 0 : tree base = DR_REF (STMT_VINFO_DATA_REF (stmt_vinfo));
6755 0 : if (may_be_nonaddressable_p (base))
6756 : return NULL;
6757 :
6758 : /* We need to use the false parameter of the conditional select. */
6759 0 : tree cond_store_arg = invert ? cond_arg2 : cond_arg1;
6760 0 : tree cond_load_arg = invert ? cond_arg1 : cond_arg2;
6761 0 : gimple *load_stmt = SSA_NAME_DEF_STMT (cond_load_arg);
6762 :
6763 : /* This is a rough estimation to check that there aren't any aliasing stores
6764 : in between the load and store. It's a bit strict, but for now it's good
6765 : enough. */
6766 0 : if (gimple_vuse (load_stmt) != gimple_vuse (store_stmt))
6767 : return NULL;
6768 :
6769 : /* If we have to invert the condition, i.e. use the true argument rather than
6770 : the false argument, we have to negate the mask. */
6771 0 : if (invert)
6772 : {
6773 0 : tree var = vect_recog_temp_ssa_var (boolean_type_node, NULL);
6774 :
6775 : /* Invert the mask using ^ 1. */
6776 0 : tree itype = TREE_TYPE (cmp_ls);
6777 0 : gassign *conv = gimple_build_assign (var, BIT_XOR_EXPR, cmp_ls,
6778 : build_int_cst (itype, 1));
6779 :
6780 0 : tree mask_vec_type = get_mask_type_for_scalar_type (vinfo, itype);
6781 0 : append_pattern_def_seq (vinfo, stmt_vinfo, conv, mask_vec_type, itype);
6782 0 : cmp_ls= var;
6783 : }
6784 :
6785 0 : if (TREE_CODE (base) != MEM_REF)
6786 0 : base = build_fold_addr_expr (base);
6787 :
6788 0 : tree ptr = build_int_cst (reference_alias_ptr_type (base),
6789 0 : get_object_alignment (base));
6790 :
6791 : /* Convert the mask to the right form. */
6792 0 : tree mask = vect_convert_mask_for_vectype (cmp_ls, vectype, stmt_vinfo,
6793 : vinfo);
6794 :
6795 0 : gcall *call
6796 0 : = gimple_build_call_internal (IFN_MASK_STORE, 4, base, ptr, mask,
6797 : cond_store_arg);
6798 0 : gimple_set_location (call, gimple_location (store_stmt));
6799 :
6800 : /* Copy across relevant vectorization info and associate DR with the
6801 : new pattern statement instead of the original statement. */
6802 0 : stmt_vec_info pattern_stmt_info = loop_vinfo->add_stmt (call);
6803 0 : loop_vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
6804 :
6805 0 : *type_out = vectype;
6806 0 : return call;
6807 : }
6808 :
6809 : /* Return true if TYPE is a non-boolean integer type. These are the types
6810 : that we want to consider for narrowing. */
6811 :
6812 : static bool
6813 66792842 : vect_narrowable_type_p (tree type)
6814 : {
6815 66792842 : return INTEGRAL_TYPE_P (type) && !VECT_SCALAR_BOOLEAN_TYPE_P (type);
6816 : }
6817 :
6818 : /* Return true if the operation given by CODE can be truncated to N bits
6819 : when only N bits of the output are needed. This is only true if bit N+1
6820 : of the inputs has no effect on the low N bits of the result. */
6821 :
6822 : static bool
6823 16609249 : vect_truncatable_operation_p (tree_code code)
6824 : {
6825 16609249 : switch (code)
6826 : {
6827 : case NEGATE_EXPR:
6828 : case PLUS_EXPR:
6829 : case MINUS_EXPR:
6830 : case MULT_EXPR:
6831 : case BIT_NOT_EXPR:
6832 : case BIT_AND_EXPR:
6833 : case BIT_IOR_EXPR:
6834 : case BIT_XOR_EXPR:
6835 : case COND_EXPR:
6836 : return true;
6837 :
6838 6521717 : default:
6839 6521717 : return false;
6840 : }
6841 : }
6842 :
6843 : /* Record that STMT_INFO could be changed from operating on TYPE to
6844 : operating on a type with the precision and sign given by PRECISION
6845 : and SIGN respectively. PRECISION is an arbitrary bit precision;
6846 : it might not be a whole number of bytes. */
6847 :
6848 : static void
6849 2601466 : vect_set_operation_type (stmt_vec_info stmt_info, tree type,
6850 : unsigned int precision, signop sign)
6851 : {
6852 : /* Round the precision up to a whole number of bytes. */
6853 2601466 : precision = vect_element_precision (precision);
6854 2601466 : if (precision < TYPE_PRECISION (type)
6855 2601466 : && (!stmt_info->operation_precision
6856 41418 : || stmt_info->operation_precision > precision))
6857 : {
6858 1674162 : stmt_info->operation_precision = precision;
6859 1674162 : stmt_info->operation_sign = sign;
6860 : }
6861 2601466 : }
6862 :
6863 : /* Record that STMT_INFO only requires MIN_INPUT_PRECISION from its
6864 : non-boolean inputs, all of which have type TYPE. MIN_INPUT_PRECISION
6865 : is an arbitrary bit precision; it might not be a whole number of bytes. */
6866 :
6867 : static void
6868 11917447 : vect_set_min_input_precision (stmt_vec_info stmt_info, tree type,
6869 : unsigned int min_input_precision)
6870 : {
6871 : /* This operation in isolation only requires the inputs to have
6872 : MIN_INPUT_PRECISION of precision, However, that doesn't mean
6873 : that MIN_INPUT_PRECISION is a natural precision for the chain
6874 : as a whole. E.g. consider something like:
6875 :
6876 : unsigned short *x, *y;
6877 : *y = ((*x & 0xf0) >> 4) | (*y << 4);
6878 :
6879 : The right shift can be done on unsigned chars, and only requires the
6880 : result of "*x & 0xf0" to be done on unsigned chars. But taking that
6881 : approach would mean turning a natural chain of single-vector unsigned
6882 : short operations into one that truncates "*x" and then extends
6883 : "(*x & 0xf0) >> 4", with two vectors for each unsigned short
6884 : operation and one vector for each unsigned char operation.
6885 : This would be a significant pessimization.
6886 :
6887 : Instead only propagate the maximum of this precision and the precision
6888 : required by the users of the result. This means that we don't pessimize
6889 : the case above but continue to optimize things like:
6890 :
6891 : unsigned char *y;
6892 : unsigned short *x;
6893 : *y = ((*x & 0xf0) >> 4) | (*y << 4);
6894 :
6895 : Here we would truncate two vectors of *x to a single vector of
6896 : unsigned chars and use single-vector unsigned char operations for
6897 : everything else, rather than doing two unsigned short copies of
6898 : "(*x & 0xf0) >> 4" and then truncating the result. */
6899 11917447 : min_input_precision = MAX (min_input_precision,
6900 : stmt_info->min_output_precision);
6901 :
6902 11917447 : if (min_input_precision < TYPE_PRECISION (type)
6903 11917447 : && (!stmt_info->min_input_precision
6904 64583 : || stmt_info->min_input_precision > min_input_precision))
6905 610389 : stmt_info->min_input_precision = min_input_precision;
6906 11917447 : }
6907 :
6908 : /* Subroutine of vect_determine_min_output_precision. Return true if
6909 : we can calculate a reduced number of output bits for STMT_INFO,
6910 : whose result is LHS. */
6911 :
6912 : static bool
6913 15264069 : vect_determine_min_output_precision_1 (vec_info *vinfo,
6914 : stmt_vec_info stmt_info, tree lhs)
6915 : {
6916 : /* Take the maximum precision required by users of the result. */
6917 15264069 : unsigned int precision = 0;
6918 15264069 : imm_use_iterator iter;
6919 15264069 : use_operand_p use;
6920 16361773 : FOR_EACH_IMM_USE_FAST (use, iter, lhs)
6921 : {
6922 16064652 : gimple *use_stmt = USE_STMT (use);
6923 16064652 : if (is_gimple_debug (use_stmt))
6924 797167 : continue;
6925 15267485 : stmt_vec_info use_stmt_info = vinfo->lookup_stmt (use_stmt);
6926 15267485 : if (!use_stmt_info || !use_stmt_info->min_input_precision)
6927 : return false;
6928 : /* The input precision recorded for COND_EXPRs applies only to the
6929 : "then" and "else" values. */
6930 301121 : gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
6931 270314 : if (assign
6932 270314 : && gimple_assign_rhs_code (assign) == COND_EXPR
6933 584 : && use->use != gimple_assign_rhs2_ptr (assign)
6934 584 : && use->use != gimple_assign_rhs3_ptr (assign))
6935 : return false;
6936 1099445 : precision = MAX (precision, use_stmt_info->min_input_precision);
6937 14966948 : }
6938 :
6939 297121 : if (dump_enabled_p ())
6940 5914 : dump_printf_loc (MSG_NOTE, vect_location,
6941 : "only the low %d bits of %T are significant\n",
6942 : precision, lhs);
6943 297121 : stmt_info->min_output_precision = precision;
6944 297121 : return true;
6945 : }
6946 :
6947 : /* Calculate min_output_precision for STMT_INFO. */
6948 :
6949 : static void
6950 39438166 : vect_determine_min_output_precision (vec_info *vinfo, stmt_vec_info stmt_info)
6951 : {
6952 : /* We're only interested in statements with a narrowable result. */
6953 39438166 : tree lhs = gimple_get_lhs (stmt_info->stmt);
6954 39438166 : if (!lhs
6955 31108778 : || TREE_CODE (lhs) != SSA_NAME
6956 65757769 : || !vect_narrowable_type_p (TREE_TYPE (lhs)))
6957 : return;
6958 :
6959 15264069 : if (!vect_determine_min_output_precision_1 (vinfo, stmt_info, lhs))
6960 14966948 : stmt_info->min_output_precision = TYPE_PRECISION (TREE_TYPE (lhs));
6961 : }
6962 :
6963 : /* Use range information to decide whether STMT (described by STMT_INFO)
6964 : could be done in a narrower type. This is effectively a forward
6965 : propagation, since it uses context-independent information that applies
6966 : to all users of an SSA name. */
6967 :
6968 : static void
6969 22487155 : vect_determine_precisions_from_range (stmt_vec_info stmt_info, gassign *stmt)
6970 : {
6971 22487155 : tree lhs = gimple_assign_lhs (stmt);
6972 22487155 : if (!lhs || TREE_CODE (lhs) != SSA_NAME)
6973 20002280 : return;
6974 :
6975 17986084 : tree type = TREE_TYPE (lhs);
6976 17986084 : if (!vect_narrowable_type_p (type))
6977 : return;
6978 :
6979 : /* First see whether we have any useful range information for the result. */
6980 11792326 : unsigned int precision = TYPE_PRECISION (type);
6981 11792326 : signop sign = TYPE_SIGN (type);
6982 11792326 : wide_int min_value, max_value;
6983 11792326 : if (!vect_get_range_info (lhs, &min_value, &max_value))
6984 : return;
6985 :
6986 5747741 : tree_code code = gimple_assign_rhs_code (stmt);
6987 5747741 : unsigned int nops = gimple_num_ops (stmt);
6988 :
6989 5747741 : if (!vect_truncatable_operation_p (code))
6990 : {
6991 : /* Handle operations that can be computed in type T if all inputs
6992 : and outputs can be represented in type T. Also handle left and
6993 : right shifts, where (in addition) the maximum shift amount must
6994 : be less than the number of bits in T. */
6995 2127828 : bool is_shift;
6996 2127828 : switch (code)
6997 : {
6998 : case LSHIFT_EXPR:
6999 : case RSHIFT_EXPR:
7000 : is_shift = true;
7001 : break;
7002 :
7003 312554 : case ABS_EXPR:
7004 312554 : case MIN_EXPR:
7005 312554 : case MAX_EXPR:
7006 312554 : case TRUNC_DIV_EXPR:
7007 312554 : case CEIL_DIV_EXPR:
7008 312554 : case FLOOR_DIV_EXPR:
7009 312554 : case ROUND_DIV_EXPR:
7010 312554 : case EXACT_DIV_EXPR:
7011 : /* Modulus is excluded because it is typically calculated by doing
7012 : a division, for which minimum signed / -1 isn't representable in
7013 : the original signed type. We could take the division range into
7014 : account instead, if handling modulus ever becomes important. */
7015 312554 : is_shift = false;
7016 312554 : break;
7017 :
7018 : default:
7019 : return;
7020 : }
7021 1419489 : for (unsigned int i = 1; i < nops; ++i)
7022 : {
7023 1095982 : tree op = gimple_op (stmt, i);
7024 1095982 : wide_int op_min_value, op_max_value;
7025 1095982 : if (TREE_CODE (op) == INTEGER_CST)
7026 : {
7027 317863 : unsigned int op_precision = TYPE_PRECISION (TREE_TYPE (op));
7028 317863 : op_min_value = op_max_value = wi::to_wide (op, op_precision);
7029 : }
7030 778119 : else if (TREE_CODE (op) == SSA_NAME)
7031 : {
7032 778119 : if (!vect_get_range_info (op, &op_min_value, &op_max_value))
7033 : return;
7034 : }
7035 : else
7036 : return;
7037 :
7038 730143 : if (is_shift && i == 2)
7039 : {
7040 : /* There needs to be one more bit than the maximum shift amount.
7041 :
7042 : If the maximum shift amount is already 1 less than PRECISION
7043 : then we can't narrow the shift further. Dealing with that
7044 : case first ensures that we can safely use an unsigned range
7045 : below.
7046 :
7047 : op_min_value isn't relevant, since shifts by negative amounts
7048 : are UB. */
7049 204808 : if (wi::geu_p (op_max_value, precision - 1))
7050 : return;
7051 179660 : unsigned int min_bits = op_max_value.to_uhwi () + 1;
7052 :
7053 : /* As explained below, we can convert a signed shift into an
7054 : unsigned shift if the sign bit is always clear. At this
7055 : point we've already processed the ranges of the output and
7056 : the first input. */
7057 179660 : auto op_sign = sign;
7058 179660 : if (sign == SIGNED && !wi::neg_p (min_value))
7059 : op_sign = UNSIGNED;
7060 359320 : op_min_value = wide_int::from (wi::min_value (min_bits, op_sign),
7061 179660 : precision, op_sign);
7062 359320 : op_max_value = wide_int::from (wi::max_value (min_bits, op_sign),
7063 179660 : precision, op_sign);
7064 : }
7065 704995 : min_value = wi::min (min_value, op_min_value, sign);
7066 704995 : max_value = wi::max (max_value, op_max_value, sign);
7067 1095982 : }
7068 : }
7069 :
7070 : /* Try to switch signed types for unsigned types if we can.
7071 : This is better for two reasons. First, unsigned ops tend
7072 : to be cheaper than signed ops. Second, it means that we can
7073 : handle things like:
7074 :
7075 : signed char c;
7076 : int res = (int) c & 0xff00; // range [0x0000, 0xff00]
7077 :
7078 : as:
7079 :
7080 : signed char c;
7081 : unsigned short res_1 = (unsigned short) c & 0xff00;
7082 : int res = (int) res_1;
7083 :
7084 : where the intermediate result res_1 has unsigned rather than
7085 : signed type. */
7086 3943420 : if (sign == SIGNED && !wi::neg_p (min_value))
7087 : sign = UNSIGNED;
7088 :
7089 : /* See what precision is required for MIN_VALUE and MAX_VALUE. */
7090 3943420 : unsigned int precision1 = wi::min_precision (min_value, sign);
7091 3943420 : unsigned int precision2 = wi::min_precision (max_value, sign);
7092 3943420 : unsigned int value_precision = MAX (precision1, precision2);
7093 3943420 : if (value_precision >= precision)
7094 : return;
7095 :
7096 2484875 : if (dump_enabled_p ())
7097 112070 : dump_printf_loc (MSG_NOTE, vect_location, "can narrow to %s:%d"
7098 : " without loss of precision: %G",
7099 : sign == SIGNED ? "signed" : "unsigned",
7100 : value_precision, (gimple *) stmt);
7101 :
7102 2484875 : vect_set_operation_type (stmt_info, type, value_precision, sign);
7103 2484875 : vect_set_min_input_precision (stmt_info, type, value_precision);
7104 11792326 : }
7105 :
7106 : /* Use information about the users of STMT's result to decide whether
7107 : STMT (described by STMT_INFO) could be done in a narrower type.
7108 : This is effectively a backward propagation. */
7109 :
7110 : static void
7111 22487155 : vect_determine_precisions_from_users (stmt_vec_info stmt_info, gassign *stmt)
7112 : {
7113 22487155 : tree_code code = gimple_assign_rhs_code (stmt);
7114 22487155 : unsigned int opno = (code == COND_EXPR ? 2 : 1);
7115 22487155 : tree type = TREE_TYPE (gimple_op (stmt, opno));
7116 22487155 : if (!vect_narrowable_type_p (type))
7117 13054583 : return;
7118 :
7119 13862019 : unsigned int precision = TYPE_PRECISION (type);
7120 13862019 : unsigned int operation_precision, min_input_precision;
7121 13862019 : switch (code)
7122 : {
7123 2527774 : CASE_CONVERT:
7124 : /* Only the bits that contribute to the output matter. Don't change
7125 : the precision of the operation itself. */
7126 2527774 : operation_precision = precision;
7127 2527774 : min_input_precision = stmt_info->min_output_precision;
7128 2527774 : break;
7129 :
7130 472737 : case LSHIFT_EXPR:
7131 472737 : case RSHIFT_EXPR:
7132 472737 : {
7133 472737 : tree shift = gimple_assign_rhs2 (stmt);
7134 472737 : unsigned int min_const_shift, max_const_shift;
7135 472737 : wide_int min_shift, max_shift;
7136 472737 : if (TREE_CODE (shift) == SSA_NAME
7137 107329 : && vect_get_range_info (shift, &min_shift, &max_shift)
7138 82853 : && wi::ge_p (min_shift, 0, TYPE_SIGN (TREE_TYPE (shift)))
7139 552831 : && wi::lt_p (max_shift, TYPE_PRECISION (type),
7140 80094 : TYPE_SIGN (TREE_TYPE (shift))))
7141 : {
7142 71879 : min_const_shift = min_shift.to_uhwi ();
7143 71879 : max_const_shift = max_shift.to_uhwi ();
7144 : }
7145 400858 : else if (TREE_CODE (shift) == INTEGER_CST
7146 766266 : && wi::ltu_p (wi::to_widest (shift), precision))
7147 365300 : min_const_shift = max_const_shift = TREE_INT_CST_LOW (shift);
7148 : else
7149 35558 : return;
7150 437179 : if (code == LSHIFT_EXPR)
7151 : {
7152 : /* Avoid creating an undefined shift.
7153 :
7154 : ??? We could instead use min_output_precision as-is and
7155 : optimize out-of-range shifts to zero. However, only
7156 : degenerate testcases shift away all their useful input data,
7157 : and it isn't natural to drop input operations in the middle
7158 : of vectorization. This sort of thing should really be
7159 : handled before vectorization. */
7160 109701 : operation_precision = MAX (stmt_info->min_output_precision,
7161 : max_const_shift + 1);
7162 : /* We need CONST_SHIFT fewer bits of the input. */
7163 109701 : min_input_precision = (MAX (operation_precision, max_const_shift)
7164 : - min_const_shift);
7165 : }
7166 : else
7167 : {
7168 : /* We need CONST_SHIFT extra bits to do the operation. */
7169 327478 : operation_precision = (stmt_info->min_output_precision
7170 : + max_const_shift);
7171 327478 : min_input_precision = operation_precision;
7172 : }
7173 437179 : break;
7174 472737 : }
7175 :
7176 10861508 : default:
7177 10861508 : if (vect_truncatable_operation_p (code))
7178 : {
7179 : /* Input bit N has no effect on output bits N-1 and lower. */
7180 6467619 : operation_precision = stmt_info->min_output_precision;
7181 6467619 : min_input_precision = operation_precision;
7182 6467619 : break;
7183 : }
7184 : return;
7185 : }
7186 :
7187 9432572 : if (operation_precision < precision)
7188 : {
7189 116591 : if (dump_enabled_p ())
7190 2803 : dump_printf_loc (MSG_NOTE, vect_location, "can narrow to %s:%d"
7191 : " without affecting users: %G",
7192 2803 : TYPE_UNSIGNED (type) ? "unsigned" : "signed",
7193 : operation_precision, (gimple *) stmt);
7194 233182 : vect_set_operation_type (stmt_info, type, operation_precision,
7195 116591 : TYPE_SIGN (type));
7196 : }
7197 9432572 : vect_set_min_input_precision (stmt_info, type, min_input_precision);
7198 : }
7199 :
7200 : /* Return true if the statement described by STMT_INFO sets a boolean
7201 : SSA_NAME and if we know how to vectorize this kind of statement using
7202 : vector mask types. */
7203 :
7204 : static bool
7205 40536692 : possible_vector_mask_operation_p (stmt_vec_info stmt_info)
7206 : {
7207 40536692 : tree lhs = gimple_get_lhs (stmt_info->stmt);
7208 40536692 : tree_code code = ERROR_MARK;
7209 40536692 : gassign *assign = NULL;
7210 40536692 : gcond *cond = NULL;
7211 :
7212 40536692 : if ((assign = dyn_cast <gassign *> (stmt_info->stmt)))
7213 23271578 : code = gimple_assign_rhs_code (assign);
7214 17265114 : else if ((cond = dyn_cast <gcond *> (stmt_info->stmt)))
7215 : {
7216 5515896 : lhs = gimple_cond_lhs (cond);
7217 5515896 : code = gimple_cond_code (cond);
7218 : }
7219 :
7220 40536692 : if (!lhs
7221 37628504 : || TREE_CODE (lhs) != SSA_NAME
7222 73337265 : || !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)))
7223 : return false;
7224 :
7225 2276633 : if (code != ERROR_MARK)
7226 : {
7227 2017214 : switch (code)
7228 : {
7229 : CASE_CONVERT:
7230 : case SSA_NAME:
7231 : case BIT_NOT_EXPR:
7232 : case BIT_IOR_EXPR:
7233 : case BIT_XOR_EXPR:
7234 : case BIT_AND_EXPR:
7235 : return true;
7236 :
7237 1575336 : default:
7238 1575336 : return TREE_CODE_CLASS (code) == tcc_comparison;
7239 : }
7240 : }
7241 259419 : else if (is_a <gphi *> (stmt_info->stmt))
7242 148059 : return true;
7243 : return false;
7244 : }
7245 :
7246 : /* If STMT_INFO sets a boolean SSA_NAME, see whether we should use
7247 : a vector mask type instead of a normal vector type. Record the
7248 : result in STMT_INFO->mask_precision. Returns true when the
7249 : precision changed. */
7250 :
7251 : static bool
7252 40536692 : vect_determine_mask_precision (vec_info *vinfo, stmt_vec_info stmt_info)
7253 : {
7254 40536692 : if (!possible_vector_mask_operation_p (stmt_info))
7255 : return false;
7256 :
7257 : /* If at least one boolean input uses a vector mask type,
7258 : pick the mask type with the narrowest elements.
7259 :
7260 : ??? This is the traditional behavior. It should always produce
7261 : the smallest number of operations, but isn't necessarily the
7262 : optimal choice. For example, if we have:
7263 :
7264 : a = b & c
7265 :
7266 : where:
7267 :
7268 : - the user of a wants it to have a mask type for 16-bit elements (M16)
7269 : - b also uses M16
7270 : - c uses a mask type for 8-bit elements (M8)
7271 :
7272 : then picking M8 gives:
7273 :
7274 : - 1 M16->M8 pack for b
7275 : - 1 M8 AND for a
7276 : - 2 M8->M16 unpacks for the user of a
7277 :
7278 : whereas picking M16 would have given:
7279 :
7280 : - 2 M8->M16 unpacks for c
7281 : - 2 M16 ANDs for a
7282 :
7283 : The number of operations are equal, but M16 would have given
7284 : a shorter dependency chain and allowed more ILP. */
7285 2113611 : unsigned int precision = ~0U;
7286 2113611 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
7287 :
7288 : /* If the statement compares two values that shouldn't use vector masks,
7289 : try comparing the values as normal scalars instead. */
7290 2113611 : tree_code code = ERROR_MARK;
7291 2113611 : tree op0_type;
7292 2113611 : unsigned int nops = -1;
7293 2113611 : unsigned int ops_start = 0;
7294 :
7295 2113611 : if (gassign *assign = dyn_cast <gassign *> (stmt))
7296 : {
7297 1392532 : code = gimple_assign_rhs_code (assign);
7298 1392532 : op0_type = TREE_TYPE (gimple_assign_rhs1 (assign));
7299 1392532 : nops = gimple_num_ops (assign);
7300 1392532 : ops_start = 1;
7301 : }
7302 721079 : else if (gcond *cond = dyn_cast <gcond *> (stmt))
7303 : {
7304 573020 : code = gimple_cond_code (cond);
7305 573020 : op0_type = TREE_TYPE (gimple_cond_lhs (cond));
7306 573020 : nops = 2;
7307 573020 : ops_start = 0;
7308 : }
7309 :
7310 1965552 : if (code != ERROR_MARK)
7311 : {
7312 5852502 : for (unsigned int i = ops_start; i < nops; ++i)
7313 : {
7314 3886950 : tree rhs = gimple_op (stmt, i);
7315 3886950 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (rhs)))
7316 1911352 : continue;
7317 :
7318 1975598 : stmt_vec_info def_stmt_info = vinfo->lookup_def (rhs);
7319 1975598 : if (!def_stmt_info)
7320 : /* Don't let external or constant operands influence the choice.
7321 : We can convert them to whichever vector type we pick. */
7322 589320 : continue;
7323 :
7324 1386278 : if (def_stmt_info->mask_precision)
7325 : {
7326 1162927 : if (precision > def_stmt_info->mask_precision)
7327 3886950 : precision = def_stmt_info->mask_precision;
7328 : }
7329 : }
7330 :
7331 1965552 : if (precision == ~0U
7332 1572238 : && TREE_CODE_CLASS (code) == tcc_comparison)
7333 : {
7334 1363560 : scalar_mode mode;
7335 1363560 : tree vectype, mask_type;
7336 1363560 : if (is_a <scalar_mode> (TYPE_MODE (op0_type), &mode)
7337 : /* Do not allow this to set vinfo->vector_mode, this might
7338 : disrupt the result for the next iteration. */
7339 1363560 : && (vectype = get_related_vectype_for_scalar_type
7340 1630269 : (vinfo->vector_mode, op0_type))
7341 1195014 : && (mask_type = truth_type_for (vectype))
7342 1195014 : && expand_vec_cmp_expr_p (vectype, mask_type, code))
7343 1856610 : precision = GET_MODE_BITSIZE (mode);
7344 : }
7345 : }
7346 : else
7347 : {
7348 148059 : gphi *phi = as_a <gphi *> (stmt_info->stmt);
7349 759257 : for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
7350 : {
7351 463139 : tree rhs = gimple_phi_arg_def (phi, i);
7352 :
7353 463139 : stmt_vec_info def_stmt_info = vinfo->lookup_def (rhs);
7354 463139 : if (!def_stmt_info)
7355 : /* Don't let external or constant operands influence the choice.
7356 : We can convert them to whichever vector type we pick. */
7357 302783 : continue;
7358 :
7359 160356 : if (def_stmt_info->mask_precision)
7360 : {
7361 134981 : if (precision > def_stmt_info->mask_precision)
7362 463139 : precision = def_stmt_info->mask_precision;
7363 : }
7364 : }
7365 : }
7366 :
7367 2113611 : if (stmt_info->mask_precision != precision)
7368 : {
7369 1984204 : if (dump_enabled_p ())
7370 : {
7371 7839 : if (precision == ~0U)
7372 1746 : dump_printf_loc (MSG_NOTE, vect_location,
7373 : "using normal nonmask vectors for %G",
7374 : stmt_info->stmt);
7375 : else
7376 6093 : dump_printf_loc (MSG_NOTE, vect_location,
7377 : "using boolean precision %d for %G",
7378 : precision, stmt_info->stmt);
7379 : }
7380 :
7381 : /* ??? We'd like to assert stmt_info->mask_precision == 0
7382 : || stmt_info->mask_precision > precision, thus that we only
7383 : decrease mask precisions throughout iteration, but the
7384 : tcc_comparison handling above means for comparisons of bools
7385 : we start with 8 but might increase in case the bools get mask
7386 : precision on their own. */
7387 1984204 : stmt_info->mask_precision = precision;
7388 1984204 : return true;
7389 : }
7390 : return false;
7391 : }
7392 :
7393 : /* Handle vect_determine_precisions for STMT_INFO, given that we
7394 : have already done so for the users of its result. */
7395 :
7396 : void
7397 39438166 : vect_determine_stmt_precisions (vec_info *vinfo, stmt_vec_info stmt_info)
7398 : {
7399 39438166 : vect_determine_min_output_precision (vinfo, stmt_info);
7400 39438166 : if (gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt))
7401 : {
7402 22487155 : vect_determine_precisions_from_range (stmt_info, stmt);
7403 22487155 : vect_determine_precisions_from_users (stmt_info, stmt);
7404 : }
7405 39438166 : }
7406 :
7407 : /* Walk backwards through the vectorizable region to determine the
7408 : values of these fields:
7409 :
7410 : - min_output_precision
7411 : - min_input_precision
7412 : - operation_precision
7413 : - operation_sign. */
7414 :
7415 : void
7416 1120591 : vect_determine_precisions (vec_info *vinfo)
7417 : {
7418 1120591 : basic_block *bbs = vinfo->bbs;
7419 1120591 : unsigned int nbbs = vinfo->nbbs;
7420 :
7421 1140204 : DUMP_VECT_SCOPE ("vect_determine_precisions");
7422 :
7423 : /* For mask precisions we have to iterate since otherwise we do not
7424 : get reduction PHI precision correct. For now do this only for
7425 : loop vectorization. */
7426 1191609 : bool changed;
7427 1191609 : do
7428 : {
7429 1191609 : changed = false;
7430 13408968 : for (unsigned int i = 0; i < nbbs; i++)
7431 : {
7432 12217359 : basic_block bb = bbs[i];
7433 12217359 : for (auto gsi = gsi_start_phis (bb);
7434 19703525 : !gsi_end_p (gsi); gsi_next (&gsi))
7435 : {
7436 7486166 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi.phi ());
7437 7486166 : if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
7438 7288070 : changed |= vect_determine_mask_precision (vinfo, stmt_info);
7439 : }
7440 131887158 : for (auto gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
7441 : {
7442 107452440 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (gsi));
7443 107452440 : if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
7444 33248622 : changed |= vect_determine_mask_precision (vinfo, stmt_info);
7445 : }
7446 : }
7447 : }
7448 2312200 : while (changed && is_a <loop_vec_info> (vinfo));
7449 :
7450 13174335 : for (unsigned int i = 0; i < nbbs; i++)
7451 : {
7452 12053744 : basic_block bb = bbs[nbbs - i - 1];
7453 139347776 : for (auto gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
7454 : {
7455 105784608 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (gsi));
7456 105784608 : if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
7457 32356018 : vect_determine_stmt_precisions (vinfo, stmt_info);
7458 : }
7459 19333988 : for (auto gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
7460 : {
7461 7280244 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi.phi ());
7462 7280244 : if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
7463 7082148 : vect_determine_stmt_precisions (vinfo, stmt_info);
7464 : }
7465 : }
7466 1120591 : }
7467 :
7468 : typedef gimple *(*vect_recog_func_ptr) (vec_info *, stmt_vec_info, tree *);
7469 :
7470 : struct vect_recog_func
7471 : {
7472 : vect_recog_func_ptr fn;
7473 : const char *name;
7474 : };
7475 :
7476 : /* Note that ordering matters - the first pattern matching on a stmt is
7477 : taken which means usually the more complex one needs to precede the
7478 : less comples onex (widen_sum only after dot_prod or sad for example). */
7479 : static vect_recog_func vect_vect_recog_func_ptrs[] = {
7480 : { vect_recog_bitfield_ref_pattern, "bitfield_ref" },
7481 : { vect_recog_bit_insert_pattern, "bit_insert" },
7482 : { vect_recog_abd_pattern, "abd" },
7483 : { vect_recog_over_widening_pattern, "over_widening" },
7484 : /* Must come after over_widening, which narrows the shift as much as
7485 : possible beforehand. */
7486 : { vect_recog_average_pattern, "average" },
7487 : { vect_recog_cond_expr_convert_pattern, "cond_expr_convert" },
7488 : { vect_recog_mulhs_pattern, "mult_high" },
7489 : { vect_recog_cast_forwprop_pattern, "cast_forwprop" },
7490 : { vect_recog_widen_mult_pattern, "widen_mult" },
7491 : { vect_recog_dot_prod_pattern, "dot_prod" },
7492 : { vect_recog_sad_pattern, "sad" },
7493 : { vect_recog_widen_sum_pattern, "widen_sum" },
7494 : { vect_recog_pow_pattern, "pow" },
7495 : { vect_recog_popcount_clz_ctz_ffs_pattern, "popcount_clz_ctz_ffs" },
7496 : { vect_recog_ctz_ffs_pattern, "ctz_ffs" },
7497 : { vect_recog_widen_shift_pattern, "widen_shift" },
7498 : { vect_recog_rotate_pattern, "rotate" },
7499 : { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" },
7500 : { vect_recog_divmod_pattern, "divmod" },
7501 : { vect_recog_mod_var_pattern, "modvar" },
7502 : { vect_recog_mult_pattern, "mult" },
7503 : { vect_recog_sat_add_pattern, "sat_add" },
7504 : { vect_recog_sat_sub_pattern, "sat_sub" },
7505 : { vect_recog_sat_trunc_pattern, "sat_trunc" },
7506 : { vect_recog_gcond_pattern, "gcond" },
7507 : { vect_recog_bool_pattern, "bool" },
7508 : /* This must come before mask conversion, and includes the parts
7509 : of mask conversion that are needed for gather and scatter
7510 : internal functions. */
7511 : { vect_recog_gather_scatter_pattern, "gather_scatter" },
7512 : { vect_recog_cond_store_pattern, "cond_store" },
7513 : { vect_recog_mask_conversion_pattern, "mask_conversion" },
7514 : { vect_recog_widen_plus_pattern, "widen_plus" },
7515 : { vect_recog_widen_minus_pattern, "widen_minus" },
7516 : { vect_recog_widen_abd_pattern, "widen_abd" },
7517 : /* These must come after the double widening ones. */
7518 : };
7519 :
7520 : /* Mark statements that are involved in a pattern. */
7521 :
7522 : void
7523 1064694 : vect_mark_pattern_stmts (vec_info *vinfo,
7524 : stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
7525 : tree pattern_vectype)
7526 : {
7527 1064694 : stmt_vec_info orig_stmt_info_saved = orig_stmt_info;
7528 1064694 : gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
7529 :
7530 1064694 : gimple *orig_pattern_stmt = NULL;
7531 1064694 : if (is_pattern_stmt_p (orig_stmt_info))
7532 : {
7533 : /* We're replacing a statement in an existing pattern definition
7534 : sequence. */
7535 11450 : orig_pattern_stmt = orig_stmt_info->stmt;
7536 11450 : if (dump_enabled_p ())
7537 666 : dump_printf_loc (MSG_NOTE, vect_location,
7538 : "replacing earlier pattern %G", orig_pattern_stmt);
7539 :
7540 : /* To keep the book-keeping simple, just swap the lhs of the
7541 : old and new statements, so that the old one has a valid but
7542 : unused lhs. */
7543 11450 : tree old_lhs = gimple_get_lhs (orig_pattern_stmt);
7544 11450 : gimple_set_lhs (orig_pattern_stmt, gimple_get_lhs (pattern_stmt));
7545 11450 : gimple_set_lhs (pattern_stmt, old_lhs);
7546 :
7547 11450 : if (dump_enabled_p ())
7548 666 : dump_printf_loc (MSG_NOTE, vect_location, "with %G", pattern_stmt);
7549 :
7550 : /* Switch to the statement that ORIG replaces. */
7551 11450 : orig_stmt_info = STMT_VINFO_RELATED_STMT (orig_stmt_info);
7552 :
7553 : /* We shouldn't be replacing the main pattern statement. */
7554 11450 : gcc_assert (STMT_VINFO_RELATED_STMT (orig_stmt_info)->stmt
7555 : != orig_pattern_stmt);
7556 : }
7557 :
7558 1064694 : if (def_seq)
7559 2324791 : for (gimple_stmt_iterator si = gsi_start (def_seq);
7560 2324791 : !gsi_end_p (si); gsi_next (&si))
7561 : {
7562 1397321 : if (dump_enabled_p ())
7563 24936 : dump_printf_loc (MSG_NOTE, vect_location,
7564 : "extra pattern stmt: %G", gsi_stmt (si));
7565 1397321 : stmt_vec_info pattern_stmt_info
7566 1397321 : = vect_init_pattern_stmt (vinfo, gsi_stmt (si),
7567 : orig_stmt_info, pattern_vectype);
7568 : /* Stmts in the def sequence are not vectorizable cycle or
7569 : induction defs, instead they should all be vect_internal_def
7570 : feeding the main pattern stmt which retains this def type. */
7571 1397321 : STMT_VINFO_DEF_TYPE (pattern_stmt_info) = vect_internal_def;
7572 : }
7573 :
7574 1064694 : if (orig_pattern_stmt)
7575 : {
7576 11450 : vect_init_pattern_stmt (vinfo, pattern_stmt,
7577 : orig_stmt_info, pattern_vectype);
7578 :
7579 : /* Insert all the new pattern statements before the original one. */
7580 11450 : gimple_seq *orig_def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
7581 11450 : gimple_stmt_iterator gsi = gsi_for_stmt (orig_pattern_stmt,
7582 : orig_def_seq);
7583 11450 : gsi_insert_seq_before_without_update (&gsi, def_seq, GSI_SAME_STMT);
7584 11450 : gsi_insert_before_without_update (&gsi, pattern_stmt, GSI_SAME_STMT);
7585 :
7586 : /* Remove the pattern statement that this new pattern replaces. */
7587 11450 : gsi_remove (&gsi, false);
7588 : }
7589 : else
7590 1053244 : vect_set_pattern_stmt (vinfo,
7591 : pattern_stmt, orig_stmt_info, pattern_vectype);
7592 :
7593 : /* For any conditionals mark them as vect_condition_def. */
7594 1064694 : if (is_a <gcond *> (pattern_stmt))
7595 378706 : STMT_VINFO_DEF_TYPE (STMT_VINFO_RELATED_STMT (orig_stmt_info)) = vect_condition_def;
7596 :
7597 : /* Transfer reduction path info to the pattern. */
7598 1064694 : if (STMT_VINFO_REDUC_IDX (orig_stmt_info_saved) != -1)
7599 : {
7600 16046 : gimple_match_op op;
7601 16046 : if (!gimple_extract_op (orig_stmt_info_saved->stmt, &op))
7602 0 : gcc_unreachable ();
7603 16046 : tree lookfor = op.ops[STMT_VINFO_REDUC_IDX (orig_stmt_info)];
7604 : /* Search the pattern def sequence and the main pattern stmt. Note
7605 : we may have inserted all into a containing pattern def sequence
7606 : so the following is a bit awkward. */
7607 16046 : gimple_stmt_iterator si;
7608 16046 : gimple *s;
7609 16046 : if (def_seq)
7610 : {
7611 14905 : si = gsi_start (def_seq);
7612 14905 : s = gsi_stmt (si);
7613 14905 : gsi_next (&si);
7614 : }
7615 : else
7616 : {
7617 : si = gsi_none ();
7618 16046 : s = pattern_stmt;
7619 : }
7620 33764 : do
7621 : {
7622 33764 : bool found = false;
7623 33764 : if (gimple_extract_op (s, &op))
7624 : {
7625 82246 : for (unsigned i = 0; i < op.num_ops; ++i)
7626 64528 : if (op.ops[i] == lookfor)
7627 : {
7628 16046 : STMT_VINFO_REDUC_IDX (vinfo->lookup_stmt (s)) = i;
7629 16046 : lookfor = gimple_get_lhs (s);
7630 16046 : found = true;
7631 16046 : break;
7632 : }
7633 : /* Try harder to find a mid-entry into an earlier pattern
7634 : sequence. Likewise an entry to a stmt skipping a conversion
7635 : on an input. This means that the initial 'lookfor' was
7636 : bogus. */
7637 16046 : if (!found)
7638 : {
7639 38443 : for (unsigned i = 0; i < op.num_ops; ++i)
7640 20725 : if (TREE_CODE (op.ops[i]) == SSA_NAME)
7641 17718 : if (auto def = vinfo->lookup_def (op.ops[i]))
7642 17525 : if (vect_is_reduction (def)
7643 17525 : || (is_a <gphi *> (def->stmt)
7644 0 : && STMT_VINFO_REDUC_DEF (def) != NULL))
7645 : {
7646 0 : STMT_VINFO_REDUC_IDX (vinfo->lookup_stmt (s)) = i;
7647 0 : lookfor = gimple_get_lhs (s);
7648 0 : found = true;
7649 0 : break;
7650 : }
7651 : }
7652 : }
7653 33764 : if (s == pattern_stmt)
7654 : {
7655 16046 : if (!found && dump_enabled_p ())
7656 0 : dump_printf_loc (MSG_NOTE, vect_location,
7657 : "failed to update reduction index.\n");
7658 16046 : break;
7659 : }
7660 17718 : if (gsi_end_p (si))
7661 : s = pattern_stmt;
7662 : else
7663 : {
7664 2813 : s = gsi_stmt (si);
7665 2813 : if (s == pattern_stmt)
7666 : /* Found the end inside a bigger pattern def seq. */
7667 : si = gsi_none ();
7668 : else
7669 2813 : gsi_next (&si);
7670 : }
7671 : } while (1);
7672 : }
7673 1064694 : }
7674 :
7675 : /* Function vect_pattern_recog_1
7676 :
7677 : Input:
7678 : PATTERN_RECOG_FUNC: A pointer to a function that detects a certain
7679 : computation pattern.
7680 : STMT_INFO: A stmt from which the pattern search should start.
7681 :
7682 : If PATTERN_RECOG_FUNC successfully detected the pattern, it creates
7683 : a sequence of statements that has the same functionality and can be
7684 : used to replace STMT_INFO. It returns the last statement in the sequence
7685 : and adds any earlier statements to STMT_INFO's STMT_VINFO_PATTERN_DEF_SEQ.
7686 : PATTERN_RECOG_FUNC also sets *TYPE_OUT to the vector type of the final
7687 : statement, having first checked that the target supports the new operation
7688 : in that type.
7689 :
7690 : This function also does some bookkeeping, as explained in the documentation
7691 : for vect_recog_pattern. */
7692 :
7693 : static void
7694 1054164684 : vect_pattern_recog_1 (vec_info *vinfo,
7695 : const vect_recog_func &recog_func, stmt_vec_info stmt_info)
7696 : {
7697 1054164684 : gimple *pattern_stmt;
7698 1054164684 : tree pattern_vectype;
7699 :
7700 : /* If this statement has already been replaced with pattern statements,
7701 : leave the original statement alone, since the first match wins.
7702 : Instead try to match against the definition statements that feed
7703 : the main pattern statement. */
7704 1054164684 : if (STMT_VINFO_IN_PATTERN_P (stmt_info))
7705 : {
7706 13412746 : gimple_stmt_iterator gsi;
7707 13412746 : for (gsi = gsi_start (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
7708 32184854 : !gsi_end_p (gsi); gsi_next (&gsi))
7709 18772108 : vect_pattern_recog_1 (vinfo, recog_func,
7710 : vinfo->lookup_stmt (gsi_stmt (gsi)));
7711 : return;
7712 : }
7713 :
7714 1040751938 : gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
7715 1040751938 : pattern_stmt = recog_func.fn (vinfo, stmt_info, &pattern_vectype);
7716 1040751938 : if (!pattern_stmt)
7717 : {
7718 : /* Clear any half-formed pattern definition sequence. */
7719 1039687244 : STMT_VINFO_PATTERN_DEF_SEQ (stmt_info) = NULL;
7720 1039687244 : return;
7721 : }
7722 :
7723 : /* Found a vectorizable pattern. */
7724 1064694 : if (dump_enabled_p ())
7725 19196 : dump_printf_loc (MSG_NOTE, vect_location,
7726 : "%s pattern recognized: %G",
7727 19196 : recog_func.name, pattern_stmt);
7728 :
7729 : /* Mark the stmts that are involved in the pattern. */
7730 1064694 : vect_mark_pattern_stmts (vinfo, stmt_info, pattern_stmt, pattern_vectype);
7731 : }
7732 :
7733 :
7734 : /* Function vect_pattern_recog
7735 :
7736 : Input:
7737 : LOOP_VINFO - a struct_loop_info of a loop in which we want to look for
7738 : computation idioms.
7739 :
7740 : Output - for each computation idiom that is detected we create a new stmt
7741 : that provides the same functionality and that can be vectorized. We
7742 : also record some information in the struct_stmt_info of the relevant
7743 : stmts, as explained below:
7744 :
7745 : At the entry to this function we have the following stmts, with the
7746 : following initial value in the STMT_VINFO fields:
7747 :
7748 : stmt in_pattern_p related_stmt vec_stmt
7749 : S1: a_i = .... - - -
7750 : S2: a_2 = ..use(a_i).. - - -
7751 : S3: a_1 = ..use(a_2).. - - -
7752 : S4: a_0 = ..use(a_1).. - - -
7753 : S5: ... = ..use(a_0).. - - -
7754 :
7755 : Say the sequence {S1,S2,S3,S4} was detected as a pattern that can be
7756 : represented by a single stmt. We then:
7757 : - create a new stmt S6 equivalent to the pattern (the stmt is not
7758 : inserted into the code)
7759 : - fill in the STMT_VINFO fields as follows:
7760 :
7761 : in_pattern_p related_stmt vec_stmt
7762 : S1: a_i = .... - - -
7763 : S2: a_2 = ..use(a_i).. - - -
7764 : S3: a_1 = ..use(a_2).. - - -
7765 : S4: a_0 = ..use(a_1).. true S6 -
7766 : '---> S6: a_new = .... - S4 -
7767 : S5: ... = ..use(a_0).. - - -
7768 :
7769 : (the last stmt in the pattern (S4) and the new pattern stmt (S6) point
7770 : to each other through the RELATED_STMT field).
7771 :
7772 : S6 will be marked as relevant in vect_mark_stmts_to_be_vectorized instead
7773 : of S4 because it will replace all its uses. Stmts {S1,S2,S3} will
7774 : remain irrelevant unless used by stmts other than S4.
7775 :
7776 : If vectorization succeeds, vect_transform_stmt will skip over {S1,S2,S3}
7777 : (because they are marked as irrelevant). It will vectorize S6, and record
7778 : a pointer to the new vector stmt VS6 from S6 (as usual).
7779 : S4 will be skipped, and S5 will be vectorized as usual:
7780 :
7781 : in_pattern_p related_stmt vec_stmt
7782 : S1: a_i = .... - - -
7783 : S2: a_2 = ..use(a_i).. - - -
7784 : S3: a_1 = ..use(a_2).. - - -
7785 : > VS6: va_new = .... - - -
7786 : S4: a_0 = ..use(a_1).. true S6 VS6
7787 : '---> S6: a_new = .... - S4 VS6
7788 : > VS5: ... = ..vuse(va_new).. - - -
7789 : S5: ... = ..use(a_0).. - - -
7790 :
7791 : DCE could then get rid of {S1,S2,S3,S4,S5} (if their defs are not used
7792 : elsewhere), and we'll end up with:
7793 :
7794 : VS6: va_new = ....
7795 : VS5: ... = ..vuse(va_new)..
7796 :
7797 : In case of more than one pattern statements, e.g., widen-mult with
7798 : intermediate type:
7799 :
7800 : S1 a_t = ;
7801 : S2 a_T = (TYPE) a_t;
7802 : '--> S3: a_it = (interm_type) a_t;
7803 : S4 prod_T = a_T * CONST;
7804 : '--> S5: prod_T' = a_it w* CONST;
7805 :
7806 : there may be other users of a_T outside the pattern. In that case S2 will
7807 : be marked as relevant (as well as S3), and both S2 and S3 will be analyzed
7808 : and vectorized. The vector stmt VS2 will be recorded in S2, and VS3 will
7809 : be recorded in S3. */
7810 :
7811 : void
7812 1120591 : vect_pattern_recog (vec_info *vinfo)
7813 : {
7814 1120591 : basic_block *bbs = vinfo->bbs;
7815 1120591 : unsigned int nbbs = vinfo->nbbs;
7816 :
7817 1120591 : vect_determine_precisions (vinfo);
7818 :
7819 1120591 : DUMP_VECT_SCOPE ("vect_pattern_recog");
7820 :
7821 : /* Scan through the stmts in the region, applying the pattern recognition
7822 : functions starting at each stmt visited. */
7823 13174335 : for (unsigned i = 0; i < nbbs; i++)
7824 : {
7825 12053744 : basic_block bb = bbs[i];
7826 :
7827 129892096 : for (auto si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
7828 : {
7829 105784608 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (si));
7830 :
7831 105784608 : if (!stmt_info || !STMT_VINFO_VECTORIZABLE (stmt_info))
7832 73428590 : continue;
7833 :
7834 : /* Scan over all generic vect_recog_xxx_pattern functions. */
7835 1067748594 : for (const auto &func_ptr : vect_vect_recog_func_ptrs)
7836 1035392576 : vect_pattern_recog_1 (vinfo, func_ptr,
7837 : stmt_info);
7838 : }
7839 : }
7840 :
7841 : /* After this no more add_stmt calls are allowed. */
7842 1120591 : vinfo->stmt_vec_info_ro = true;
7843 1120591 : }
7844 :
7845 : /* Build a GIMPLE_ASSIGN or GIMPLE_CALL with the tree_code,
7846 : or internal_fn contained in ch, respectively. */
7847 : gimple *
7848 167452 : vect_gimple_build (tree lhs, code_helper ch, tree op0, tree op1)
7849 : {
7850 167452 : gcc_assert (op0 != NULL_TREE);
7851 167452 : if (ch.is_tree_code ())
7852 167452 : return gimple_build_assign (lhs, (tree_code) ch, op0, op1);
7853 :
7854 0 : gcc_assert (ch.is_internal_fn ());
7855 0 : gimple* stmt = gimple_build_call_internal (as_internal_fn ((combined_fn) ch),
7856 : op1 == NULL_TREE ? 1 : 2,
7857 : op0, op1);
7858 0 : gimple_call_set_lhs (stmt, lhs);
7859 0 : return stmt;
7860 : }
|
