Line data Source code
1 : /* Analysis Utilities for Loop Vectorization.
2 : Copyright (C) 2006-2026 Free Software Foundation, Inc.
3 : Contributed by Dorit Nuzman <dorit@il.ibm.com>
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "gimple-iterator.h"
29 : #include "gimple-fold.h"
30 : #include "ssa.h"
31 : #include "expmed.h"
32 : #include "optabs-tree.h"
33 : #include "insn-config.h"
34 : #include "recog.h" /* FIXME: for insn_data */
35 : #include "fold-const.h"
36 : #include "stor-layout.h"
37 : #include "tree-eh.h"
38 : #include "gimplify.h"
39 : #include "gimplify-me.h"
40 : #include "cfgloop.h"
41 : #include "tree-vectorizer.h"
42 : #include "dumpfile.h"
43 : #include "builtins.h"
44 : #include "internal-fn.h"
45 : #include "case-cfn-macros.h"
46 : #include "fold-const-call.h"
47 : #include "attribs.h"
48 : #include "cgraph.h"
49 : #include "omp-simd-clone.h"
50 : #include "predict.h"
51 : #include "tree-vector-builder.h"
52 : #include "tree-ssa-loop-ivopts.h"
53 : #include "vec-perm-indices.h"
54 : #include "gimple-range.h"
55 : #include "alias.h"
56 :
57 :
58 : /* TODO: Note the vectorizer still builds COND_EXPRs with GENERIC compares
59 : in the first operand. Disentangling this is future work, the
60 : IL is properly transferred to VEC_COND_EXPRs with separate compares. */
61 :
62 :
63 : /* Return true if we have a useful VR_RANGE range for VAR, storing it
64 : in *MIN_VALUE and *MAX_VALUE if so. Note the range in the dump files. */
65 :
66 : bool
67 13065450 : vect_get_range_info (tree var, wide_int *min_value, wide_int *max_value)
68 : {
69 13065450 : int_range_max vr;
70 13065450 : tree vr_min, vr_max;
71 26130900 : get_range_query (cfun)->range_of_expr (vr, var);
72 13065450 : if (vr.undefined_p ())
73 53 : vr.set_varying (TREE_TYPE (var));
74 13065450 : value_range_kind vr_type = get_legacy_range (vr, vr_min, vr_max);
75 13065450 : *min_value = wi::to_wide (vr_min);
76 13065450 : *max_value = wi::to_wide (vr_max);
77 13065450 : wide_int nonzero = get_nonzero_bits (var);
78 13065450 : signop sgn = TYPE_SIGN (TREE_TYPE (var));
79 13065450 : if (intersect_range_with_nonzero_bits (vr_type, min_value, max_value,
80 : nonzero, sgn) == VR_RANGE)
81 : {
82 6336518 : if (dump_enabled_p ())
83 : {
84 94805 : dump_generic_expr_loc (MSG_NOTE, vect_location, TDF_SLIM, var);
85 94805 : dump_printf (MSG_NOTE, " has range [");
86 94805 : dump_hex (MSG_NOTE, *min_value);
87 94805 : dump_printf (MSG_NOTE, ", ");
88 94805 : dump_hex (MSG_NOTE, *max_value);
89 94805 : dump_printf (MSG_NOTE, "]\n");
90 : }
91 : return true;
92 : }
93 : else
94 : {
95 6728932 : if (dump_enabled_p ())
96 : {
97 71326 : dump_generic_expr_loc (MSG_NOTE, vect_location, TDF_SLIM, var);
98 71326 : dump_printf (MSG_NOTE, " has no range info\n");
99 : }
100 : return false;
101 : }
102 13065450 : }
103 :
104 : /* Report that we've found an instance of pattern PATTERN in
105 : statement STMT. */
106 :
107 : static void
108 1398518 : vect_pattern_detected (const char *name, gimple *stmt)
109 : {
110 1398518 : if (dump_enabled_p ())
111 28002 : dump_printf_loc (MSG_NOTE, vect_location, "%s: detected: %G", name, stmt);
112 1398518 : }
113 :
114 : /* Associate pattern statement PATTERN_STMT with ORIG_STMT_INFO and
115 : return the pattern statement's stmt_vec_info. Set its vector type to
116 : VECTYPE if it doesn't have one already. */
117 :
118 : static stmt_vec_info
119 2443389 : vect_init_pattern_stmt (vec_info *vinfo, gimple *pattern_stmt,
120 : stmt_vec_info orig_stmt_info, tree vectype)
121 : {
122 2443389 : stmt_vec_info pattern_stmt_info = vinfo->lookup_stmt (pattern_stmt);
123 2443389 : if (pattern_stmt_info == NULL)
124 1280724 : pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
125 2443389 : gimple_set_bb (pattern_stmt, gimple_bb (orig_stmt_info->stmt));
126 :
127 2443389 : pattern_stmt_info->pattern_stmt_p = true;
128 2443389 : STMT_VINFO_RELATED_STMT (pattern_stmt_info) = orig_stmt_info;
129 2443389 : STMT_VINFO_DEF_TYPE (pattern_stmt_info)
130 2443389 : = STMT_VINFO_DEF_TYPE (orig_stmt_info);
131 2443389 : if (!STMT_VINFO_VECTYPE (pattern_stmt_info))
132 : {
133 2106658 : gcc_assert (!vectype
134 : || is_a <gcond *> (pattern_stmt)
135 : || (VECTOR_BOOLEAN_TYPE_P (vectype)
136 : == vect_use_mask_type_p (orig_stmt_info)));
137 1291495 : STMT_VINFO_VECTYPE (pattern_stmt_info) = vectype;
138 1291495 : pattern_stmt_info->mask_precision = orig_stmt_info->mask_precision;
139 1291495 : if (dump_enabled_p ())
140 : {
141 27576 : if (pattern_stmt_info->mask_precision == ~0U)
142 63 : dump_printf_loc (MSG_NOTE, vect_location,
143 : "using normal nonmask vectors for %G",
144 : pattern_stmt_info->stmt);
145 : else
146 27513 : dump_printf_loc (MSG_NOTE, vect_location,
147 : "using boolean precision %d for %G",
148 : pattern_stmt_info->mask_precision,
149 : pattern_stmt_info->stmt);
150 : }
151 : }
152 2443389 : return pattern_stmt_info;
153 : }
154 :
155 : /* Set the pattern statement of ORIG_STMT_INFO to PATTERN_STMT.
156 : Also set the vector type of PATTERN_STMT to VECTYPE, if it doesn't
157 : have one already. */
158 :
159 : static void
160 1121138 : vect_set_pattern_stmt (vec_info *vinfo, gimple *pattern_stmt,
161 : stmt_vec_info orig_stmt_info, tree vectype)
162 : {
163 1121138 : STMT_VINFO_IN_PATTERN_P (orig_stmt_info) = true;
164 1121138 : STMT_VINFO_RELATED_STMT (orig_stmt_info)
165 0 : = vect_init_pattern_stmt (vinfo, pattern_stmt, orig_stmt_info, vectype);
166 0 : }
167 :
168 : /* Add NEW_STMT to STMT_INFO's pattern definition statements. If VECTYPE
169 : is nonnull, record that NEW_STMT's vector type is VECTYPE, which might
170 : be different from the vector type of the final pattern statement.
171 : If VECTYPE is a mask type, SCALAR_TYPE_FOR_MASK is the scalar type
172 : from which it was derived. */
173 :
174 : static inline void
175 1282927 : append_pattern_def_seq (vec_info *vinfo,
176 : stmt_vec_info stmt_info, gimple *new_stmt,
177 : tree vectype = NULL_TREE,
178 : tree scalar_type_for_mask = NULL_TREE)
179 : {
180 1930536 : gcc_assert (!scalar_type_for_mask
181 : == (!vectype || !VECTOR_BOOLEAN_TYPE_P (vectype)));
182 1282927 : if (vectype)
183 : {
184 1152124 : stmt_vec_info new_stmt_info = vinfo->add_stmt (new_stmt);
185 1152124 : STMT_VINFO_VECTYPE (new_stmt_info) = vectype;
186 1152124 : if (scalar_type_for_mask)
187 : {
188 635318 : new_stmt_info->mask_precision
189 635318 : = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (scalar_type_for_mask));
190 635318 : if (dump_enabled_p ())
191 : {
192 6889 : if (new_stmt_info->mask_precision == ~0U)
193 : dump_printf_loc (MSG_NOTE, vect_location,
194 : "using normal nonmask vectors for %G",
195 : new_stmt_info->stmt);
196 : else
197 6889 : dump_printf_loc (MSG_NOTE, vect_location,
198 : "using boolean precision %d for %G",
199 : new_stmt_info->mask_precision,
200 : new_stmt_info->stmt);
201 : }
202 : }
203 : }
204 1282927 : gimple_seq_add_stmt_without_update (&STMT_VINFO_PATTERN_DEF_SEQ (stmt_info),
205 : new_stmt);
206 1282927 : }
207 :
208 :
209 : /* Add NEW_STMT to VINFO's invariant pattern definition statements. These
210 : statements are not vectorized but are materialized as scalar in the loop
211 : preheader. */
212 :
213 : static inline void
214 1366 : append_inv_pattern_def_seq (vec_info *vinfo, gimple *new_stmt)
215 : {
216 1366 : gimple_seq_add_stmt_without_update (&vinfo->inv_pattern_def_seq, new_stmt);
217 : }
218 :
219 : /* The caller wants to perform new operations on vect_external variable
220 : VAR, so that the result of the operations would also be vect_external.
221 : Return the edge on which the operations can be performed, if one exists.
222 : Return null if the operations should instead be treated as part of
223 : the pattern that needs them. */
224 :
225 : static edge
226 9713 : vect_get_external_def_edge (vec_info *vinfo, tree var)
227 : {
228 9713 : edge e = NULL;
229 9713 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
230 : {
231 977 : e = loop_preheader_edge (loop_vinfo->loop);
232 977 : if (!SSA_NAME_IS_DEFAULT_DEF (var))
233 : {
234 773 : basic_block bb = gimple_bb (SSA_NAME_DEF_STMT (var));
235 773 : if (bb == NULL
236 773 : || !dominated_by_p (CDI_DOMINATORS, e->dest, bb))
237 : e = NULL;
238 : }
239 : }
240 9713 : return e;
241 : }
242 :
243 : /* Return true if the target supports a vector version of CODE,
244 : where CODE is known to map to a direct optab with the given SUBTYPE.
245 : ITYPE specifies the type of (some of) the scalar inputs and OTYPE
246 : specifies the type of the scalar result.
247 :
248 : If CODE allows the inputs and outputs to have different type
249 : (such as for WIDEN_SUM_EXPR), it is the input mode rather
250 : than the output mode that determines the appropriate target pattern.
251 : Operand 0 of the target pattern then specifies the mode that the output
252 : must have.
253 :
254 : When returning true, set *VECOTYPE_OUT to the vector version of OTYPE.
255 : Also set *VECITYPE_OUT to the vector version of ITYPE if VECITYPE_OUT
256 : is nonnull. */
257 :
258 : static bool
259 84 : vect_supportable_direct_optab_p (vec_info *vinfo, tree otype, tree_code code,
260 : tree itype, tree *vecotype_out,
261 : tree *vecitype_out = NULL,
262 : enum optab_subtype subtype = optab_default)
263 : {
264 84 : tree vecitype = get_vectype_for_scalar_type (vinfo, itype);
265 84 : if (!vecitype)
266 : return false;
267 :
268 82 : tree vecotype = get_vectype_for_scalar_type (vinfo, otype);
269 82 : if (!vecotype)
270 : return false;
271 :
272 82 : optab optab = optab_for_tree_code (code, vecitype, subtype);
273 82 : if (!optab)
274 : return false;
275 :
276 82 : insn_code icode = optab_handler (optab, TYPE_MODE (vecitype));
277 82 : if (icode == CODE_FOR_nothing
278 82 : || insn_data[icode].operand[0].mode != TYPE_MODE (vecotype))
279 : return false;
280 :
281 31 : *vecotype_out = vecotype;
282 31 : if (vecitype_out)
283 0 : *vecitype_out = vecitype;
284 : return true;
285 : }
286 :
287 : /* Return true if the target supports a vector version of CODE,
288 : where CODE is known to map to a conversion optab with the given SUBTYPE.
289 : ITYPE specifies the type of (some of) the scalar inputs and OTYPE
290 : specifies the type of the scalar result.
291 :
292 : When returning true, set *VECOTYPE_OUT to the vector version of OTYPE.
293 : Also set *VECITYPE_OUT to the vector version of ITYPE if VECITYPE_OUT
294 : is nonnull. */
295 :
296 : static bool
297 6694 : vect_supportable_conv_optab_p (vec_info *vinfo, tree otype, tree_code code,
298 : tree itype, tree *vecotype_out,
299 : tree *vecitype_out = NULL,
300 : enum optab_subtype subtype = optab_default)
301 : {
302 6694 : tree vecitype = get_vectype_for_scalar_type (vinfo, itype);
303 6694 : tree vecotype = get_vectype_for_scalar_type (vinfo, otype);
304 6694 : if (!vecitype || !vecotype)
305 : return false;
306 :
307 6156 : if (!directly_supported_p (code, vecotype, vecitype, subtype))
308 : return false;
309 :
310 1192 : *vecotype_out = vecotype;
311 1192 : if (vecitype_out)
312 1192 : *vecitype_out = vecitype;
313 : return true;
314 : }
315 :
316 : /* Round bit precision PRECISION up to a full element. */
317 :
318 : static unsigned int
319 3669826 : vect_element_precision (unsigned int precision)
320 : {
321 277327 : precision = 1 << ceil_log2 (precision);
322 5357138 : return MAX (precision, BITS_PER_UNIT);
323 : }
324 :
325 : /* If OP is defined by a statement that's being considered for vectorization,
326 : return information about that statement, otherwise return NULL. */
327 :
328 : static stmt_vec_info
329 422597 : vect_get_internal_def (vec_info *vinfo, tree op)
330 : {
331 422597 : stmt_vec_info def_stmt_info = vinfo->lookup_def (op);
332 422597 : if (def_stmt_info
333 408619 : && STMT_VINFO_DEF_TYPE (def_stmt_info) == vect_internal_def)
334 390352 : return vect_stmt_to_vectorize (def_stmt_info);
335 : return NULL;
336 : }
337 :
338 : /* Holds information about an input operand after some sign changes
339 : and type promotions have been peeled away. */
340 : class vect_unpromoted_value {
341 : public:
342 : vect_unpromoted_value ();
343 :
344 : void set_op (tree, vect_def_type, stmt_vec_info = NULL);
345 :
346 : /* The value obtained after peeling away zero or more casts. */
347 : tree op;
348 :
349 : /* The type of OP. */
350 : tree type;
351 :
352 : /* The definition type of OP. */
353 : vect_def_type dt;
354 :
355 : /* If OP is the result of peeling at least one cast, and if the cast
356 : of OP itself is a vectorizable statement, CASTER identifies that
357 : statement, otherwise it is null. */
358 : stmt_vec_info caster;
359 : };
360 :
361 315722518 : inline vect_unpromoted_value::vect_unpromoted_value ()
362 315722518 : : op (NULL_TREE),
363 315722518 : type (NULL_TREE),
364 315722518 : dt (vect_uninitialized_def),
365 3360877 : caster (NULL)
366 : {
367 : }
368 :
369 : /* Set the operand to OP_IN, its definition type to DT_IN, and the
370 : statement that casts it to CASTER_IN. */
371 :
372 : inline void
373 12051193 : vect_unpromoted_value::set_op (tree op_in, vect_def_type dt_in,
374 : stmt_vec_info caster_in)
375 : {
376 12051193 : op = op_in;
377 12051193 : type = TREE_TYPE (op);
378 12051193 : dt = dt_in;
379 12051193 : caster = caster_in;
380 12051193 : }
381 :
382 : /* If OP is a vectorizable SSA name, strip a sequence of integer conversions
383 : to reach some vectorizable inner operand OP', continuing as long as it
384 : is possible to convert OP' back to OP using a possible sign change
385 : followed by a possible promotion P. Return this OP', or null if OP is
386 : not a vectorizable SSA name. If there is a promotion P, describe its
387 : input in UNPROM, otherwise describe OP' in UNPROM. If SINGLE_USE_P
388 : is nonnull, set *SINGLE_USE_P to false if any of the SSA names involved
389 : have more than one user.
390 :
391 : A successful return means that it is possible to go from OP' to OP
392 : via UNPROM. The cast from OP' to UNPROM is at most a sign change,
393 : whereas the cast from UNPROM to OP might be a promotion, a sign
394 : change, or a nop.
395 :
396 : E.g. say we have:
397 :
398 : signed short *ptr = ...;
399 : signed short C = *ptr;
400 : unsigned short B = (unsigned short) C; // sign change
401 : signed int A = (signed int) B; // unsigned promotion
402 : ...possible other uses of A...
403 : unsigned int OP = (unsigned int) A; // sign change
404 :
405 : In this case it's possible to go directly from C to OP using:
406 :
407 : OP = (unsigned int) (unsigned short) C;
408 : +------------+ +--------------+
409 : promotion sign change
410 :
411 : so OP' would be C. The input to the promotion is B, so UNPROM
412 : would describe B. */
413 :
414 : static tree
415 8833480 : vect_look_through_possible_promotion (vec_info *vinfo, tree op,
416 : vect_unpromoted_value *unprom,
417 : bool *single_use_p = NULL)
418 : {
419 8833480 : tree op_type = TREE_TYPE (op);
420 8833480 : if (!INTEGRAL_TYPE_P (op_type))
421 : return NULL_TREE;
422 :
423 8799131 : tree res = NULL_TREE;
424 8799131 : unsigned int orig_precision = TYPE_PRECISION (op_type);
425 8799131 : unsigned int min_precision = orig_precision;
426 8799131 : stmt_vec_info caster = NULL;
427 10572439 : while (TREE_CODE (op) == SSA_NAME && INTEGRAL_TYPE_P (op_type))
428 : {
429 : /* See whether OP is simple enough to vectorize. */
430 10344116 : stmt_vec_info def_stmt_info;
431 10344116 : gimple *def_stmt;
432 10344116 : vect_def_type dt;
433 10344116 : if (!vect_is_simple_use (op, vinfo, &dt, &def_stmt_info, &def_stmt))
434 : break;
435 :
436 : /* If OP is the input of a demotion, skip over it to see whether
437 : OP is itself the result of a promotion. If so, the combined
438 : effect of the promotion and the demotion might fit the required
439 : pattern, otherwise neither operation fits.
440 :
441 : This copes with cases such as the result of an arithmetic
442 : operation being truncated before being stored, and where that
443 : arithmetic operation has been recognized as an over-widened one. */
444 10326659 : if (TYPE_PRECISION (op_type) <= min_precision)
445 : {
446 : /* Use OP as the UNPROM described above if we haven't yet
447 : found a promotion, or if using the new input preserves the
448 : sign of the previous promotion. */
449 10191051 : if (!res
450 1512300 : || TYPE_PRECISION (unprom->type) == orig_precision
451 40519 : || TYPE_SIGN (unprom->type) == TYPE_SIGN (op_type)
452 10228118 : || (TYPE_UNSIGNED (op_type)
453 25925 : && TYPE_PRECISION (op_type) < TYPE_PRECISION (unprom->type)))
454 : {
455 10154443 : unprom->set_op (op, dt, caster);
456 10154443 : min_precision = TYPE_PRECISION (op_type);
457 : }
458 : /* Stop if we've already seen a promotion and if this
459 : conversion does more than change the sign. */
460 36608 : else if (TYPE_PRECISION (op_type)
461 36608 : != TYPE_PRECISION (unprom->type))
462 : break;
463 :
464 : /* The sequence now extends to OP. */
465 : res = op;
466 : }
467 :
468 : /* See whether OP is defined by a cast. Record it as CASTER if
469 : the cast is potentially vectorizable. */
470 10326629 : if (!def_stmt)
471 : break;
472 10104502 : caster = def_stmt_info;
473 :
474 : /* Ignore pattern statements, since we don't link uses for them. */
475 10104502 : if (caster
476 10104502 : && single_use_p
477 2044253 : && !STMT_VINFO_RELATED_STMT (caster)
478 11992854 : && !has_single_use (res))
479 1129218 : *single_use_p = false;
480 :
481 14960653 : gassign *assign = dyn_cast <gassign *> (def_stmt);
482 6389845 : if (!assign || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def_stmt)))
483 : break;
484 :
485 : /* Continue with the input to the cast. */
486 1773308 : op = gimple_assign_rhs1 (def_stmt);
487 1773308 : op_type = TREE_TYPE (op);
488 : }
489 : return res;
490 : }
491 :
492 : /* OP is an integer operand to an operation that returns TYPE, and we
493 : want to treat the operation as a widening one. So far we can treat
494 : it as widening from *COMMON_TYPE.
495 :
496 : Return true if OP is suitable for such a widening operation,
497 : either widening from *COMMON_TYPE or from some supertype of it.
498 : Update *COMMON_TYPE to the supertype in the latter case.
499 :
500 : SHIFT_P is true if OP is a shift amount. */
501 :
502 : static bool
503 357414 : vect_joust_widened_integer (tree type, bool shift_p, tree op,
504 : tree *common_type)
505 : {
506 : /* Calculate the minimum precision required by OP, without changing
507 : the sign of either operand. */
508 357414 : unsigned int precision;
509 357414 : if (shift_p)
510 : {
511 13493 : if (!wi::leu_p (wi::to_widest (op), TYPE_PRECISION (type) / 2))
512 : return false;
513 10875 : precision = TREE_INT_CST_LOW (op);
514 : }
515 : else
516 : {
517 343921 : precision = wi::min_precision (wi::to_widest (op),
518 343921 : TYPE_SIGN (*common_type));
519 343921 : if (precision * 2 > TYPE_PRECISION (type))
520 : return false;
521 : }
522 :
523 : /* If OP requires a wider type, switch to that type. The checks
524 : above ensure that this is still narrower than the result. */
525 338183 : precision = vect_element_precision (precision);
526 338183 : if (TYPE_PRECISION (*common_type) < precision)
527 6829 : *common_type = build_nonstandard_integer_type
528 6829 : (precision, TYPE_UNSIGNED (*common_type));
529 : return true;
530 : }
531 :
532 : /* Return true if the common supertype of NEW_TYPE and *COMMON_TYPE
533 : is narrower than type, storing the supertype in *COMMON_TYPE if so. */
534 :
535 : static bool
536 51987 : vect_joust_widened_type (tree type, tree new_type, tree *common_type)
537 : {
538 51987 : if (types_compatible_p (*common_type, new_type))
539 : return true;
540 :
541 : /* See if *COMMON_TYPE can hold all values of NEW_TYPE. */
542 4819 : if ((TYPE_PRECISION (new_type) < TYPE_PRECISION (*common_type))
543 4819 : && (TYPE_UNSIGNED (new_type) || !TYPE_UNSIGNED (*common_type)))
544 : return true;
545 :
546 : /* See if NEW_TYPE can hold all values of *COMMON_TYPE. */
547 3858 : if (TYPE_PRECISION (*common_type) < TYPE_PRECISION (new_type)
548 3858 : && (TYPE_UNSIGNED (*common_type) || !TYPE_UNSIGNED (new_type)))
549 : {
550 932 : *common_type = new_type;
551 932 : return true;
552 : }
553 :
554 : /* We have mismatched signs, with the signed type being
555 : no wider than the unsigned type. In this case we need
556 : a wider signed type. */
557 2926 : unsigned int precision = MAX (TYPE_PRECISION (*common_type),
558 : TYPE_PRECISION (new_type));
559 2926 : precision *= 2;
560 :
561 2926 : if (precision * 2 > TYPE_PRECISION (type))
562 : return false;
563 :
564 40 : *common_type = build_nonstandard_integer_type (precision, false);
565 40 : return true;
566 : }
567 :
568 : /* If the range of UNPROM->OP at STMT fits in a narrower element type than
569 : UNPROM->TYPE, change UNPROM->TYPE to that type and return true. */
570 :
571 : static bool
572 278597 : vect_narrow_unprom_to_range (vect_unpromoted_value *unprom, gimple *stmt)
573 : {
574 278597 : int_range_max r;
575 557194 : get_range_query (cfun)->range_of_expr (r, unprom->op, stmt);
576 278597 : if (r.undefined_p ())
577 : return false;
578 :
579 277327 : signop sgn = TYPE_SIGN (unprom->type);
580 277327 : unsigned int precision
581 831981 : = vect_element_precision (MAX (wi::min_precision (r.lower_bound (), sgn),
582 : wi::min_precision (r.upper_bound (), sgn)));
583 277327 : if (precision >= TYPE_PRECISION (unprom->type))
584 : return false;
585 :
586 35814 : unprom->type = build_nonstandard_integer_type (precision, sgn == UNSIGNED);
587 35814 : return true;
588 278597 : }
589 :
590 : /* Check whether STMT_INFO can be viewed as a tree of integer operations
591 : in which each node either performs CODE or WIDENED_CODE, and where
592 : each leaf operand is narrower than the result of STMT_INFO. MAX_NOPS
593 : specifies the maximum number of leaf operands. SHIFT_P says whether
594 : CODE and WIDENED_CODE are some sort of shift.
595 :
596 : If STMT_INFO is such a tree, return the number of leaf operands
597 : and describe them in UNPROM[0] onwards. Also set *COMMON_TYPE
598 : to a type that (a) is narrower than the result of STMT_INFO and
599 : (b) can hold all leaf operand values.
600 :
601 : If SUBTYPE then allow that the signs of the operands
602 : may differ in signs but not in precision. SUBTYPE is updated to reflect
603 : this.
604 :
605 : Return 0 if STMT_INFO isn't such a tree, or if no such COMMON_TYPE
606 : exists. */
607 :
608 : static unsigned int
609 132595124 : vect_widened_op_tree (vec_info *vinfo, stmt_vec_info stmt_info, tree_code code,
610 : code_helper widened_code, bool shift_p,
611 : unsigned int max_nops,
612 : vect_unpromoted_value *unprom, tree *common_type,
613 : enum optab_subtype *subtype = NULL)
614 : {
615 : /* Check for an integer operation with the right code. */
616 132595124 : gimple* stmt = stmt_info->stmt;
617 132595124 : if (!(is_gimple_assign (stmt) || is_gimple_call (stmt)))
618 : return 0;
619 :
620 109123395 : code_helper rhs_code;
621 109123395 : if (is_gimple_assign (stmt))
622 93757397 : rhs_code = gimple_assign_rhs_code (stmt);
623 15365998 : else if (is_gimple_call (stmt))
624 15365998 : rhs_code = gimple_call_combined_fn (stmt);
625 : else
626 : return 0;
627 :
628 109123395 : if (rhs_code != code
629 109123395 : && rhs_code != widened_code)
630 : return 0;
631 :
632 6895255 : tree lhs = gimple_get_lhs (stmt);
633 6895255 : tree type = TREE_TYPE (lhs);
634 6895255 : if (!INTEGRAL_TYPE_P (type))
635 : return 0;
636 :
637 : /* Assume that both operands will be leaf operands. */
638 6003006 : max_nops -= 2;
639 :
640 : /* Check the operands. */
641 6003006 : unsigned int next_op = 0;
642 6896820 : for (unsigned int i = 0; i < 2; ++i)
643 : {
644 6509131 : vect_unpromoted_value *this_unprom = &unprom[next_op];
645 6509131 : unsigned int nops = 1;
646 6509131 : tree op = gimple_arg (stmt, i);
647 6509131 : if (i == 1 && TREE_CODE (op) == INTEGER_CST)
648 : {
649 : /* We already have a common type from earlier operands.
650 : Update it to account for OP. */
651 357414 : this_unprom->set_op (op, vect_constant_def);
652 357414 : if (!vect_joust_widened_integer (type, shift_p, op, common_type))
653 : return 0;
654 : }
655 : else
656 : {
657 : /* Only allow shifts by constants. */
658 6151717 : if (shift_p && i == 1)
659 : return 0;
660 :
661 6145576 : if (rhs_code != code)
662 : {
663 : /* If rhs_code is widened_code, don't look through further
664 : possible promotions, there is a promotion already embedded
665 : in the WIDEN_*_EXPR. */
666 1711 : if (TREE_CODE (op) != SSA_NAME
667 1711 : || !INTEGRAL_TYPE_P (TREE_TYPE (op)))
668 0 : return 0;
669 :
670 1711 : stmt_vec_info def_stmt_info;
671 1711 : gimple *def_stmt;
672 1711 : vect_def_type dt;
673 1711 : if (!vect_is_simple_use (op, vinfo, &dt, &def_stmt_info,
674 : &def_stmt))
675 : return 0;
676 1711 : this_unprom->set_op (op, dt, NULL);
677 : }
678 6143865 : else if (!vect_look_through_possible_promotion (vinfo, op,
679 : this_unprom))
680 : return 0;
681 :
682 6029741 : if (TYPE_PRECISION (this_unprom->type) == TYPE_PRECISION (type)
683 6308338 : && !(this_unprom->dt == vect_external_def
684 278597 : && vect_narrow_unprom_to_range (this_unprom, stmt)))
685 : {
686 : /* The operand isn't widened. If STMT_INFO has the code
687 : for an unwidened operation, recursively check whether
688 : this operand is a node of the tree. */
689 5467519 : if (rhs_code != code
690 5467519 : || max_nops == 0
691 5468070 : || this_unprom->dt != vect_internal_def)
692 : return 0;
693 :
694 : /* Give back the leaf slot allocated above now that we're
695 : not treating this as a leaf operand. */
696 551 : max_nops += 1;
697 :
698 : /* Recursively process the definition of the operand. */
699 551 : stmt_vec_info def_stmt_info
700 551 : = vect_get_internal_def (vinfo, this_unprom->op);
701 :
702 551 : nops = vect_widened_op_tree (vinfo, def_stmt_info, code,
703 : widened_code, shift_p, max_nops,
704 : this_unprom, common_type,
705 : subtype);
706 551 : if (nops == 0)
707 : return 0;
708 :
709 367 : max_nops -= nops;
710 : }
711 : else
712 : {
713 : /* Make sure that the operand is narrower than the result. */
714 562222 : if (TYPE_PRECISION (this_unprom->type) * 2
715 562222 : > TYPE_PRECISION (type))
716 : return 0;
717 :
718 : /* Update COMMON_TYPE for the new operand. */
719 557859 : if (i == 0)
720 505872 : *common_type = this_unprom->type;
721 51987 : else if (!vect_joust_widened_type (type, this_unprom->type,
722 : common_type))
723 : {
724 2886 : if (subtype)
725 : {
726 : /* See if we can sign extend the smaller type. */
727 291 : if (TYPE_PRECISION (this_unprom->type)
728 291 : > TYPE_PRECISION (*common_type))
729 27 : *common_type = this_unprom->type;
730 291 : *subtype = optab_vector_mixed_sign;
731 : }
732 : else
733 : return 0;
734 : }
735 : }
736 : }
737 893814 : next_op += nops;
738 : }
739 : return next_op;
740 : }
741 :
742 : /* Helper to return a new temporary for pattern of TYPE for STMT. If STMT
743 : is NULL, the caller must set SSA_NAME_DEF_STMT for the returned SSA var. */
744 :
745 : static tree
746 2051897 : vect_recog_temp_ssa_var (tree type, gimple *stmt = NULL)
747 : {
748 0 : return make_temp_ssa_name (type, stmt, "patt");
749 : }
750 :
751 : /* STMT2_INFO describes a type conversion that could be split into STMT1
752 : followed by a version of STMT2_INFO that takes NEW_RHS as its first
753 : input. Try to do this using pattern statements, returning true on
754 : success. */
755 :
756 : static bool
757 29005 : vect_split_statement (vec_info *vinfo, stmt_vec_info stmt2_info, tree new_rhs,
758 : gimple *stmt1, tree vectype)
759 : {
760 29005 : if (is_pattern_stmt_p (stmt2_info))
761 : {
762 : /* STMT2_INFO is part of a pattern. Get the statement to which
763 : the pattern is attached. */
764 493 : stmt_vec_info orig_stmt2_info = STMT_VINFO_RELATED_STMT (stmt2_info);
765 493 : vect_init_pattern_stmt (vinfo, stmt1, orig_stmt2_info, vectype);
766 :
767 493 : if (dump_enabled_p ())
768 19 : dump_printf_loc (MSG_NOTE, vect_location,
769 : "Splitting pattern statement: %G", stmt2_info->stmt);
770 :
771 : /* Since STMT2_INFO is a pattern statement, we can change it
772 : in-situ without worrying about changing the code for the
773 : containing block. */
774 493 : gimple_assign_set_rhs1 (stmt2_info->stmt, new_rhs);
775 :
776 493 : if (dump_enabled_p ())
777 : {
778 19 : dump_printf_loc (MSG_NOTE, vect_location, "into: %G", stmt1);
779 19 : dump_printf_loc (MSG_NOTE, vect_location, "and: %G",
780 : stmt2_info->stmt);
781 : }
782 :
783 493 : gimple_seq *def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt2_info);
784 493 : if (STMT_VINFO_RELATED_STMT (orig_stmt2_info) == stmt2_info)
785 : /* STMT2_INFO is the actual pattern statement. Add STMT1
786 : to the end of the definition sequence. */
787 484 : gimple_seq_add_stmt_without_update (def_seq, stmt1);
788 : else
789 : {
790 : /* STMT2_INFO belongs to the definition sequence. Insert STMT1
791 : before it. */
792 9 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt2_info->stmt, def_seq);
793 9 : gsi_insert_before_without_update (&gsi, stmt1, GSI_SAME_STMT);
794 : }
795 : return true;
796 : }
797 : else
798 : {
799 : /* STMT2_INFO doesn't yet have a pattern. Try to create a
800 : two-statement pattern now. */
801 28512 : gcc_assert (!STMT_VINFO_RELATED_STMT (stmt2_info));
802 28512 : tree lhs_type = TREE_TYPE (gimple_get_lhs (stmt2_info->stmt));
803 28512 : tree lhs_vectype = get_vectype_for_scalar_type (vinfo, lhs_type);
804 28512 : if (!lhs_vectype)
805 : return false;
806 :
807 28512 : if (dump_enabled_p ())
808 2059 : dump_printf_loc (MSG_NOTE, vect_location,
809 : "Splitting statement: %G", stmt2_info->stmt);
810 :
811 : /* Add STMT1 as a singleton pattern definition sequence. */
812 28512 : gimple_seq *def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (stmt2_info);
813 28512 : vect_init_pattern_stmt (vinfo, stmt1, stmt2_info, vectype);
814 28512 : gimple_seq_add_stmt_without_update (def_seq, stmt1);
815 :
816 : /* Build the second of the two pattern statements. */
817 28512 : tree new_lhs = vect_recog_temp_ssa_var (lhs_type, NULL);
818 28512 : gassign *new_stmt2 = gimple_build_assign (new_lhs, NOP_EXPR, new_rhs);
819 28512 : vect_set_pattern_stmt (vinfo, new_stmt2, stmt2_info, lhs_vectype);
820 :
821 28512 : if (dump_enabled_p ())
822 : {
823 2059 : dump_printf_loc (MSG_NOTE, vect_location,
824 : "into pattern statements: %G", stmt1);
825 2059 : dump_printf_loc (MSG_NOTE, vect_location, "and: %G",
826 : (gimple *) new_stmt2);
827 : }
828 :
829 : return true;
830 : }
831 : }
832 :
833 : /* Look for the following pattern
834 : X = x[i]
835 : Y = y[i]
836 : DIFF = X - Y
837 : DAD = ABS_EXPR<DIFF>
838 :
839 : ABS_STMT should point to a statement of code ABS_EXPR or ABSU_EXPR.
840 : HALF_TYPE and UNPROM will be set should the statement be found to
841 : be a widened operation.
842 : DIFF_STMT will be set to the MINUS_EXPR
843 : statement that precedes the ABS_STMT if it is a MINUS_EXPR..
844 : */
845 : static bool
846 23132316 : vect_recog_absolute_difference (vec_info *vinfo, gassign *abs_stmt,
847 : tree *half_type,
848 : vect_unpromoted_value unprom[2],
849 : gassign **diff_stmt)
850 : {
851 23132316 : if (!abs_stmt)
852 : return false;
853 :
854 : /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
855 : inside the loop (in case we are analyzing an outer-loop). */
856 23132316 : enum tree_code code = gimple_assign_rhs_code (abs_stmt);
857 23132316 : if (code != ABS_EXPR && code != ABSU_EXPR)
858 : return false;
859 :
860 34421 : tree abs_oprnd = gimple_assign_rhs1 (abs_stmt);
861 34421 : tree abs_type = TREE_TYPE (abs_oprnd);
862 34421 : if (!abs_oprnd)
863 : return false;
864 27008 : if (!ANY_INTEGRAL_TYPE_P (abs_type)
865 7664 : || TYPE_OVERFLOW_WRAPS (abs_type)
866 41925 : || TYPE_UNSIGNED (abs_type))
867 : return false;
868 :
869 : /* Peel off conversions from the ABS input. This can involve sign
870 : changes (e.g. from an unsigned subtraction to a signed ABS input)
871 : or signed promotion, but it can't include unsigned promotion.
872 : (Note that ABS of an unsigned promotion should have been folded
873 : away before now anyway.) */
874 7504 : vect_unpromoted_value unprom_diff;
875 7504 : abs_oprnd = vect_look_through_possible_promotion (vinfo, abs_oprnd,
876 : &unprom_diff);
877 7504 : if (!abs_oprnd)
878 : return false;
879 7253 : if (TYPE_PRECISION (unprom_diff.type) != TYPE_PRECISION (abs_type)
880 7253 : && TYPE_UNSIGNED (unprom_diff.type))
881 : return false;
882 :
883 : /* We then detect if the operand of abs_expr is defined by a minus_expr. */
884 7253 : stmt_vec_info diff_stmt_vinfo = vect_get_internal_def (vinfo, abs_oprnd);
885 7253 : if (!diff_stmt_vinfo)
886 : return false;
887 :
888 7041 : gassign *diff = dyn_cast <gassign *> (STMT_VINFO_STMT (diff_stmt_vinfo));
889 7041 : if (diff_stmt && diff
890 5799 : && gimple_assign_rhs_code (diff) == MINUS_EXPR
891 8894 : && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (abs_oprnd)))
892 251 : *diff_stmt = diff;
893 :
894 : /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
895 : inside the loop (in case we are analyzing an outer-loop). */
896 7041 : if (vect_widened_op_tree (vinfo, diff_stmt_vinfo,
897 7041 : MINUS_EXPR, IFN_VEC_WIDEN_MINUS,
898 : false, 2, unprom, half_type))
899 2408 : return true;
900 :
901 : return false;
902 : }
903 :
904 : /* Convert UNPROM to TYPE and return the result, adding new statements
905 : to STMT_INFO's pattern definition statements if no better way is
906 : available. VECTYPE is the vector form of TYPE.
907 :
908 : If SUBTYPE then convert the type based on the subtype. */
909 :
910 : static tree
911 520266 : vect_convert_input (vec_info *vinfo, stmt_vec_info stmt_info, tree type,
912 : vect_unpromoted_value *unprom, tree vectype,
913 : enum optab_subtype subtype = optab_default)
914 : {
915 : /* Update the type if the signs differ. */
916 520266 : if (subtype == optab_vector_mixed_sign)
917 : {
918 328 : gcc_assert (!TYPE_UNSIGNED (type));
919 328 : if (TYPE_UNSIGNED (TREE_TYPE (unprom->op)))
920 : {
921 164 : type = unsigned_type_for (type);
922 164 : vectype = unsigned_type_for (vectype);
923 : }
924 : }
925 :
926 : /* Check for a no-op conversion. */
927 520266 : if (types_compatible_p (type, TREE_TYPE (unprom->op)))
928 183984 : return unprom->op;
929 :
930 : /* Allow the caller to create constant vect_unpromoted_values. */
931 336282 : if (TREE_CODE (unprom->op) == INTEGER_CST)
932 208606 : return wide_int_to_tree (type, wi::to_widest (unprom->op));
933 :
934 127676 : tree input = unprom->op;
935 127676 : if (unprom->caster)
936 : {
937 67498 : tree lhs = gimple_get_lhs (unprom->caster->stmt);
938 67498 : tree lhs_type = TREE_TYPE (lhs);
939 :
940 : /* If the result of the existing cast is the right width, use it
941 : instead of the source of the cast. */
942 67498 : if (TYPE_PRECISION (lhs_type) == TYPE_PRECISION (type))
943 : input = lhs;
944 : /* If the precision we want is between the source and result
945 : precisions of the existing cast, try splitting the cast into
946 : two and tapping into a mid-way point. */
947 65213 : else if (TYPE_PRECISION (lhs_type) > TYPE_PRECISION (type)
948 65213 : && TYPE_PRECISION (type) > TYPE_PRECISION (unprom->type))
949 : {
950 : /* In order to preserve the semantics of the original cast,
951 : give the mid-way point the same signedness as the input value.
952 :
953 : It would be possible to use a signed type here instead if
954 : TYPE is signed and UNPROM->TYPE is unsigned, but that would
955 : make the sign of the midtype sensitive to the order in
956 : which we process the statements, since the signedness of
957 : TYPE is the signedness required by just one of possibly
958 : many users. Also, unsigned promotions are usually as cheap
959 : as or cheaper than signed ones, so it's better to keep an
960 : unsigned promotion. */
961 29005 : tree midtype = build_nonstandard_integer_type
962 29005 : (TYPE_PRECISION (type), TYPE_UNSIGNED (unprom->type));
963 29005 : tree vec_midtype = get_vectype_for_scalar_type (vinfo, midtype);
964 29005 : if (vec_midtype)
965 : {
966 29005 : input = vect_recog_temp_ssa_var (midtype, NULL);
967 29005 : gassign *new_stmt = gimple_build_assign (input, NOP_EXPR,
968 : unprom->op);
969 29005 : if (!vect_split_statement (vinfo, unprom->caster, input, new_stmt,
970 : vec_midtype))
971 0 : append_pattern_def_seq (vinfo, stmt_info,
972 : new_stmt, vec_midtype);
973 : }
974 : }
975 :
976 : /* See if we can reuse an existing result. */
977 67498 : if (types_compatible_p (type, TREE_TYPE (input)))
978 : return input;
979 : }
980 :
981 : /* We need a new conversion statement. */
982 106771 : tree new_op = vect_recog_temp_ssa_var (type, NULL);
983 106771 : gassign *new_stmt = gimple_build_assign (new_op, NOP_EXPR, input);
984 :
985 : /* If OP is an external value, see if we can insert the new statement
986 : on an incoming edge. */
987 106771 : if (input == unprom->op && unprom->dt == vect_external_def)
988 9698 : if (edge e = vect_get_external_def_edge (vinfo, input))
989 : {
990 962 : basic_block new_bb = gsi_insert_on_edge_immediate (e, new_stmt);
991 962 : gcc_assert (!new_bb);
992 : return new_op;
993 : }
994 :
995 : /* As a (common) last resort, add the statement to the pattern itself. */
996 105809 : append_pattern_def_seq (vinfo, stmt_info, new_stmt, vectype);
997 105809 : return new_op;
998 : }
999 :
1000 : /* Invoke vect_convert_input for N elements of UNPROM and store the
1001 : result in the corresponding elements of RESULT.
1002 :
1003 : If SUBTYPE then convert the type based on the subtype. */
1004 :
1005 : static void
1006 264230 : vect_convert_inputs (vec_info *vinfo, stmt_vec_info stmt_info, unsigned int n,
1007 : tree *result, tree type, vect_unpromoted_value *unprom,
1008 : tree vectype, enum optab_subtype subtype = optab_default)
1009 : {
1010 784577 : for (unsigned int i = 0; i < n; ++i)
1011 : {
1012 : unsigned int j;
1013 775875 : for (j = 0; j < i; ++j)
1014 256117 : if (unprom[j].op == unprom[i].op)
1015 : break;
1016 :
1017 520347 : if (j < i)
1018 589 : result[i] = result[j];
1019 : else
1020 519758 : result[i] = vect_convert_input (vinfo, stmt_info,
1021 519758 : type, &unprom[i], vectype, subtype);
1022 : }
1023 264230 : }
1024 :
1025 : /* The caller has created a (possibly empty) sequence of pattern definition
1026 : statements followed by a single statement PATTERN_STMT. Cast the result
1027 : of this final statement to TYPE. If a new statement is needed, add
1028 : PATTERN_STMT to the end of STMT_INFO's pattern definition statements
1029 : and return the new statement, otherwise return PATTERN_STMT as-is.
1030 : VECITYPE is the vector form of PATTERN_STMT's result type. */
1031 :
1032 : static gimple *
1033 291127 : vect_convert_output (vec_info *vinfo, stmt_vec_info stmt_info, tree type,
1034 : gimple *pattern_stmt, tree vecitype)
1035 : {
1036 291127 : tree lhs = gimple_get_lhs (pattern_stmt);
1037 291127 : if (!types_compatible_p (type, TREE_TYPE (lhs)))
1038 : {
1039 259595 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vecitype);
1040 259595 : tree cast_var = vect_recog_temp_ssa_var (type, NULL);
1041 259595 : pattern_stmt = gimple_build_assign (cast_var, NOP_EXPR, lhs);
1042 : }
1043 291127 : return pattern_stmt;
1044 : }
1045 :
1046 : /* Return true if STMT_VINFO describes a reduction for which reassociation
1047 : is allowed. If STMT_INFO is part of a group, assume that it's part of
1048 : a reduction chain and optimistically assume that all statements
1049 : except the last allow reassociation.
1050 : Also require it to have code CODE and to be a reduction
1051 : in the outermost loop. When returning true, store the operands in
1052 : *OP0_OUT and *OP1_OUT. */
1053 :
1054 : static bool
1055 99163344 : vect_reassociating_reduction_p (vec_info *vinfo,
1056 : stmt_vec_info stmt_info, tree_code code,
1057 : tree *op0_out, tree *op1_out)
1058 : {
1059 99163344 : loop_vec_info loop_info = dyn_cast <loop_vec_info> (vinfo);
1060 13989699 : if (!loop_info)
1061 : return false;
1062 :
1063 13989699 : gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
1064 15255672 : if (!assign || gimple_assign_rhs_code (assign) != code)
1065 : return false;
1066 :
1067 : /* We don't allow changing the order of the computation in the inner-loop
1068 : when doing outer-loop vectorization. */
1069 2718632 : class loop *loop = LOOP_VINFO_LOOP (loop_info);
1070 2718632 : if (loop && nested_in_vect_loop_p (loop, stmt_info))
1071 : return false;
1072 :
1073 2664608 : if (!vect_is_reduction (stmt_info))
1074 : return false;
1075 :
1076 174821 : if (needs_fold_left_reduction_p (TREE_TYPE (gimple_assign_lhs (assign)),
1077 174821 : code))
1078 : return false;
1079 :
1080 161234 : *op0_out = gimple_assign_rhs1 (assign);
1081 161234 : *op1_out = gimple_assign_rhs2 (assign);
1082 161234 : if (commutative_tree_code (code) && STMT_VINFO_REDUC_IDX (stmt_info) == 0)
1083 64149 : std::swap (*op0_out, *op1_out);
1084 : return true;
1085 : }
1086 :
1087 : /* Return true iff the target has a vector optab implementing the operation
1088 : CODE on type VECTYPE with SUBTYPE. */
1089 :
1090 : static bool
1091 908694 : target_has_vecop_for_code (tree_code code, tree vectype,
1092 : enum optab_subtype subtype = optab_vector)
1093 : {
1094 908694 : optab voptab = optab_for_tree_code (code, vectype, subtype);
1095 908694 : return voptab && can_implement_p (voptab, TYPE_MODE (vectype));
1096 : }
1097 :
1098 : /* match.pd function to match
1099 : (cond (cmp@3 a b) (convert@1 c) (convert@2 d))
1100 : with conditions:
1101 : 1) @1, @2, c, d, a, b are all integral type.
1102 : 2) There's single_use for both @1 and @2.
1103 : 3) a, c have same precision.
1104 : 4) c and @1 have different precision.
1105 : 5) c, d are the same type or they can differ in sign when convert is
1106 : truncation.
1107 :
1108 : record a and c and d and @3. */
1109 :
1110 : extern bool gimple_cond_expr_convert_p (tree, tree*, tree (*)(tree));
1111 :
1112 : /* Function vect_recog_cond_expr_convert
1113 :
1114 : Try to find the following pattern:
1115 :
1116 : TYPE_AB A,B;
1117 : TYPE_CD C,D;
1118 : TYPE_E E;
1119 : TYPE_E op_true = (TYPE_E) A;
1120 : TYPE_E op_false = (TYPE_E) B;
1121 :
1122 : E = C cmp D ? op_true : op_false;
1123 :
1124 : where
1125 : TYPE_PRECISION (TYPE_E) != TYPE_PRECISION (TYPE_CD);
1126 : TYPE_PRECISION (TYPE_AB) == TYPE_PRECISION (TYPE_CD);
1127 : single_use of op_true and op_false.
1128 : TYPE_AB could differ in sign when (TYPE_E) A is a truncation.
1129 :
1130 : Input:
1131 :
1132 : * STMT_VINFO: The stmt from which the pattern search begins.
1133 : here it starts with E = c cmp D ? op_true : op_false;
1134 :
1135 : Output:
1136 :
1137 : TYPE1 E' = C cmp D ? A : B;
1138 : TYPE3 E = (TYPE3) E';
1139 :
1140 : There may extra nop_convert for A or B to handle different signness.
1141 :
1142 : * TYPE_OUT: The vector type of the output of this pattern.
1143 :
1144 : * Return value: A new stmt that will be used to replace the sequence of
1145 : stmts that constitute the pattern. In this case it will be:
1146 : E = (TYPE3)E';
1147 : E' = C cmp D ? A : B; is recorded in pattern definition statements; */
1148 :
1149 : static gimple *
1150 33136945 : vect_recog_cond_expr_convert_pattern (vec_info *vinfo,
1151 : stmt_vec_info stmt_vinfo, tree *type_out)
1152 : {
1153 33136945 : gassign *last_stmt = dyn_cast <gassign *> (stmt_vinfo->stmt);
1154 23234096 : tree lhs, match[4], temp, type, new_lhs, op2, op1;
1155 23234096 : gimple *cond_stmt;
1156 23234096 : gimple *pattern_stmt;
1157 56371041 : enum tree_code code = NOP_EXPR;
1158 :
1159 23234096 : if (!last_stmt)
1160 : return NULL;
1161 :
1162 23234096 : lhs = gimple_assign_lhs (last_stmt);
1163 :
1164 : /* Find E = C cmp D ? (TYPE3) A ? (TYPE3) B;
1165 : TYPE_PRECISION (A) == TYPE_PRECISION (C). */
1166 23234096 : if (!gimple_cond_expr_convert_p (lhs, &match[0], NULL))
1167 : return NULL;
1168 :
1169 44 : if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (lhs)))
1170 20 : code = INTEGRAL_TYPE_P (TREE_TYPE (match[1])) ? FLOAT_EXPR : CONVERT_EXPR;
1171 24 : else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (match[1])))
1172 0 : code = FIX_TRUNC_EXPR;
1173 :
1174 44 : op1 = match[1];
1175 44 : op2 = match[2];
1176 44 : type = TREE_TYPE (op1);
1177 : /* When op1/op2 is REAL_CST, the conversion must be CONVERT_EXPR from
1178 : SCALAR_FLOAT_TYPE_P which is restricted in gimple_cond_expr_convert_p.
1179 : Otherwise, the conversion could be FLOAT_EXPR, FIX_TRUNC_EXPR
1180 : or CONVERT_EXPR. */
1181 44 : if (TREE_CODE (op1) == REAL_CST)
1182 : {
1183 20 : op1 = const_unop (CONVERT_EXPR, TREE_TYPE (op2), op1);
1184 20 : type = TREE_TYPE (op2);
1185 20 : if (op1 == NULL_TREE)
1186 : return NULL;
1187 : }
1188 24 : else if (TREE_CODE (op2) == REAL_CST)
1189 : {
1190 0 : op2 = const_unop (FLOAT_EXPR, TREE_TYPE (op1), op2);
1191 0 : if (op2 == NULL_TREE)
1192 : return NULL;
1193 : }
1194 24 : else if (code == NOP_EXPR)
1195 : {
1196 24 : if (TYPE_SIGN (type) != TYPE_SIGN (TREE_TYPE (match[2])))
1197 : {
1198 9 : op2 = vect_recog_temp_ssa_var (type, NULL);
1199 9 : gimple* nop_stmt = gimple_build_assign (op2, NOP_EXPR, match[2]);
1200 9 : append_pattern_def_seq (vinfo, stmt_vinfo, nop_stmt);
1201 : }
1202 : }
1203 :
1204 44 : vect_pattern_detected ("vect_recog_cond_expr_convert_pattern", last_stmt);
1205 :
1206 44 : temp = vect_recog_temp_ssa_var (type, NULL);
1207 44 : cond_stmt = gimple_build_assign (temp, build3 (COND_EXPR, type, match[3],
1208 : op1, op2));
1209 44 : append_pattern_def_seq (vinfo, stmt_vinfo, cond_stmt);
1210 44 : new_lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
1211 44 : pattern_stmt = gimple_build_assign (new_lhs, code, temp);
1212 44 : *type_out = NULL_TREE;
1213 :
1214 44 : if (dump_enabled_p ())
1215 24 : dump_printf_loc (MSG_NOTE, vect_location,
1216 : "created pattern stmt: %G", pattern_stmt);
1217 : return pattern_stmt;
1218 : }
1219 :
1220 : /* Function vect_recog_dot_prod_pattern
1221 :
1222 : Try to find the following pattern:
1223 :
1224 : type1a x_t
1225 : type1b y_t;
1226 : TYPE1 prod;
1227 : TYPE2 sum = init;
1228 : loop:
1229 : sum_0 = phi <init, sum_1>
1230 : S1 x_t = ...
1231 : S2 y_t = ...
1232 : S3 x_T = (TYPE1) x_t;
1233 : S4 y_T = (TYPE1) y_t;
1234 : S5 prod = x_T * y_T;
1235 : [S6 prod = (TYPE2) prod; #optional]
1236 : S7 sum_1 = prod + sum_0;
1237 :
1238 : For integer reductions, 'TYPE1' is exactly double the size of 'type1a'
1239 : and 'type1b', and its sign must match one of them. Floating-point
1240 : reductions require matching input types at most half the width of TYPE1,
1241 : with the inputs converted before multiplication, and TYPE2 the same as TYPE1.
1242 :
1243 : Input:
1244 :
1245 : * STMT_VINFO: The stmt from which the pattern search begins. In the
1246 : example, when this function is called with S7, the pattern {S3,S4,S5,S6,S7}
1247 : will be detected.
1248 :
1249 : Output:
1250 :
1251 : * TYPE_OUT: The type of the output of this pattern.
1252 :
1253 : * Return value: A new stmt that will be used to replace the sequence of
1254 : stmts that constitute the pattern. In this case it will be:
1255 : WIDEN_DOT_PRODUCT <x_t, y_t, sum_0>
1256 :
1257 : Note: The dot-prod idiom is a widening reduction pattern that is
1258 : vectorized without preserving all the intermediate results. It
1259 : produces only N/2 (widened) results (by summing up pairs of
1260 : intermediate results) rather than all N results. Therefore, we
1261 : cannot allow this pattern when we want to get all the results and in
1262 : the correct order (as is the case when this computation is in an
1263 : inner-loop nested in an outer-loop that us being vectorized). */
1264 :
1265 : static gimple *
1266 33055086 : vect_recog_dot_prod_pattern (vec_info *vinfo,
1267 : stmt_vec_info stmt_vinfo, tree *type_out)
1268 : {
1269 33055086 : tree oprnd0, oprnd1;
1270 33055086 : gimple *last_stmt = stmt_vinfo->stmt;
1271 33055086 : tree type, half_type;
1272 33055086 : gimple *pattern_stmt;
1273 33055086 : tree var;
1274 :
1275 : /* Look for the following pattern
1276 : DX = (TYPE1) X;
1277 : DY = (TYPE1) Y;
1278 : DPROD = DX * DY;
1279 : DDPROD = (TYPE2) DPROD;
1280 : sum_1 = DDPROD + sum_0;
1281 : In which
1282 : - DX and DY are at least twice as wide as X and Y (exactly twice for
1283 : integers).
1284 : - DX, DY, DPROD all have the same type but the sign
1285 : between X, Y and DPROD can differ.
1286 : - sum is the same size of DPROD or bigger
1287 : - sum has been recognized as a reduction variable.
1288 :
1289 : This is equivalent to:
1290 : DPROD = X w* Y; #widen mult
1291 : sum_1 = DPROD w+ sum_0; #widen summation
1292 : or
1293 : DPROD = X w* Y; #widen mult
1294 : sum_1 = DPROD + sum_0; #summation
1295 : */
1296 :
1297 : /* Starting from LAST_STMT, follow the defs of its uses in search
1298 : of the above pattern. */
1299 :
1300 33055086 : if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, PLUS_EXPR,
1301 : &oprnd0, &oprnd1))
1302 : return NULL;
1303 :
1304 54386 : type = TREE_TYPE (gimple_get_lhs (last_stmt));
1305 :
1306 : /* So far so good. Since LAST_STMT is a summation reduction, OPRND1 is
1307 : the reduction variable defined by a loop-header phi. OPRND0 is the
1308 : value added to it, which must come from a multiplication. For integer
1309 : reductions, first look through a possible promotion of OPRND0. */
1310 54386 : bool float_p = SCALAR_FLOAT_TYPE_P (type);
1311 54386 : enum optab_subtype subtype = optab_vector;
1312 54386 : vect_unpromoted_value unprom_mult;
1313 54386 : if (!float_p)
1314 37336 : oprnd0 = vect_look_through_possible_promotion (vinfo, oprnd0, &unprom_mult);
1315 54386 : if (!oprnd0 || TREE_CODE (oprnd0) != SSA_NAME)
1316 : return NULL;
1317 :
1318 54088 : stmt_vec_info mult_vinfo = vect_get_internal_def (vinfo, oprnd0);
1319 54088 : if (!mult_vinfo)
1320 : return NULL;
1321 :
1322 : /* FORNOW. We could continue following the def-use chain if this statement
1323 : is a phi inside the loop, as when analyzing an outer loop. */
1324 159678 : vect_unpromoted_value unprom0[2];
1325 53226 : if (float_p)
1326 : {
1327 : /* Require multiplication of two directly promoted floating-point
1328 : inputs. A product rounded before promotion is not equivalent. */
1329 17038 : gassign *mult = dyn_cast <gassign *> (mult_vinfo->stmt);
1330 17268 : if (!mult || gimple_assign_rhs_code (mult) != MULT_EXPR)
1331 : return NULL;
1332 :
1333 16772 : for (unsigned int i = 0; i < 2; ++i)
1334 : {
1335 16756 : tree op = gimple_op (mult, i + 1);
1336 16756 : stmt_vec_info cast_info
1337 16756 : = (TREE_CODE (op) == SSA_NAME
1338 16756 : ? vect_get_internal_def (vinfo, op) : NULL);
1339 16726 : gassign *cast
1340 16726 : = cast_info ? dyn_cast <gassign *> (cast_info->stmt) : NULL;
1341 33293 : if (!cast || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (cast)))
1342 16694 : return NULL;
1343 :
1344 62 : op = gimple_assign_rhs1 (cast);
1345 62 : tree op_type = TREE_TYPE (op);
1346 62 : vect_def_type dt;
1347 62 : if (TREE_CODE (op) != SSA_NAME
1348 62 : || !SCALAR_FLOAT_TYPE_P (op_type)
1349 62 : || TYPE_PRECISION (op_type) * 2 > TYPE_PRECISION (type)
1350 124 : || !vect_is_simple_use (op, vinfo, &dt))
1351 : return NULL;
1352 62 : unprom0[i].set_op (op, dt, cast_info);
1353 : }
1354 16 : half_type = unprom0[0].type;
1355 16 : if (!types_compatible_p (half_type, unprom0[1].type))
1356 : return NULL;
1357 : }
1358 36188 : else if (!vect_widened_op_tree (vinfo, mult_vinfo, MULT_EXPR, WIDEN_MULT_EXPR,
1359 : false, 2, unprom0, &half_type, &subtype))
1360 : return NULL;
1361 :
1362 : /* If there are two integer widening operations, make sure they agree on
1363 : the sign of the extension. The result of an optab_vector_mixed_sign
1364 : operation is signed; otherwise, it has the same sign as the operands. */
1365 1386 : if (!float_p
1366 1370 : && TYPE_PRECISION (unprom_mult.type) != TYPE_PRECISION (type)
1367 2127 : && (subtype == optab_vector_mixed_sign
1368 741 : ? TYPE_UNSIGNED (unprom_mult.type)
1369 532 : : TYPE_SIGN (unprom_mult.type) != TYPE_SIGN (half_type)))
1370 : return NULL;
1371 :
1372 1305 : vect_pattern_detected ("vect_recog_dot_prod_pattern", last_stmt);
1373 :
1374 : /* If the inputs have mixed signs, canonicalize on using the signed
1375 : input type for analysis. This also helps when emulating mixed-sign
1376 : operations using signed operations. */
1377 1305 : if (subtype == optab_vector_mixed_sign)
1378 246 : half_type = signed_type_for (half_type);
1379 :
1380 1305 : tree half_vectype;
1381 1305 : if (!vect_supportable_conv_optab_p (vinfo, type, DOT_PROD_EXPR, half_type,
1382 : type_out, &half_vectype, subtype))
1383 : {
1384 : /* We can emulate a mixed-sign dot-product using a sequence of
1385 : signed dot-products; see vect_emulate_mixed_dot_prod for details. */
1386 588 : if (subtype != optab_vector_mixed_sign
1387 588 : || !vect_supportable_conv_optab_p (vinfo, signed_type_for (type),
1388 : DOT_PROD_EXPR, half_type,
1389 : type_out, &half_vectype,
1390 : optab_vector))
1391 : return NULL;
1392 :
1393 15 : *type_out = signed_or_unsigned_type_for (TYPE_UNSIGNED (type),
1394 : *type_out);
1395 : }
1396 :
1397 : /* Get the inputs in the appropriate types. */
1398 732 : tree mult_oprnd[2];
1399 732 : vect_convert_inputs (vinfo, stmt_vinfo, 2, mult_oprnd, half_type,
1400 : unprom0, half_vectype, subtype);
1401 :
1402 732 : var = vect_recog_temp_ssa_var (type, NULL);
1403 732 : pattern_stmt = gimple_build_assign (var, DOT_PROD_EXPR,
1404 : mult_oprnd[0], mult_oprnd[1], oprnd1);
1405 :
1406 732 : return pattern_stmt;
1407 : }
1408 :
1409 :
1410 : /* Function vect_recog_sad_pattern
1411 :
1412 : Try to find the following Sum of Absolute Difference (SAD) pattern:
1413 :
1414 : type x_t, y_t;
1415 : signed TYPE1 diff, abs_diff;
1416 : TYPE2 sum = init;
1417 : loop:
1418 : sum_0 = phi <init, sum_1>
1419 : S1 x_t = ...
1420 : S2 y_t = ...
1421 : S3 x_T = (TYPE1) x_t;
1422 : S4 y_T = (TYPE1) y_t;
1423 : S5 diff = x_T - y_T;
1424 : S6 abs_diff = ABS_EXPR <diff>;
1425 : [S7 abs_diff = (TYPE2) abs_diff; #optional]
1426 : S8 sum_1 = abs_diff + sum_0;
1427 :
1428 : where 'TYPE1' is at least double the size of type 'type', and 'TYPE2' is the
1429 : same size of 'TYPE1' or bigger. This is a special case of a reduction
1430 : computation.
1431 :
1432 : Input:
1433 :
1434 : * STMT_VINFO: The stmt from which the pattern search begins. In the
1435 : example, when this function is called with S8, the pattern
1436 : {S3,S4,S5,S6,S7,S8} will be detected.
1437 :
1438 : Output:
1439 :
1440 : * TYPE_OUT: The type of the output of this pattern.
1441 :
1442 : * Return value: A new stmt that will be used to replace the sequence of
1443 : stmts that constitute the pattern. In this case it will be:
1444 : SAD_EXPR <x_t, y_t, sum_0>
1445 : */
1446 :
1447 : static gimple *
1448 33054359 : vect_recog_sad_pattern (vec_info *vinfo,
1449 : stmt_vec_info stmt_vinfo, tree *type_out)
1450 : {
1451 33054359 : gimple *last_stmt = stmt_vinfo->stmt;
1452 33054359 : tree half_type;
1453 :
1454 : /* Look for the following pattern
1455 : DX = (TYPE1) X;
1456 : DY = (TYPE1) Y;
1457 : DDIFF = DX - DY;
1458 : DAD = ABS_EXPR <DDIFF>;
1459 : DDPROD = (TYPE2) DPROD;
1460 : sum_1 = DAD + sum_0;
1461 : In which
1462 : - DX is at least double the size of X
1463 : - DY is at least double the size of Y
1464 : - DX, DY, DDIFF, DAD all have the same type
1465 : - sum is the same size of DAD or bigger
1466 : - sum has been recognized as a reduction variable.
1467 :
1468 : This is equivalent to:
1469 : DDIFF = X w- Y; #widen sub
1470 : DAD = ABS_EXPR <DDIFF>;
1471 : sum_1 = DAD w+ sum_0; #widen summation
1472 : or
1473 : DDIFF = X w- Y; #widen sub
1474 : DAD = ABS_EXPR <DDIFF>;
1475 : sum_1 = DAD + sum_0; #summation
1476 : */
1477 :
1478 : /* Starting from LAST_STMT, follow the defs of its uses in search
1479 : of the above pattern. */
1480 :
1481 33054359 : tree plus_oprnd0, plus_oprnd1;
1482 33054359 : if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, PLUS_EXPR,
1483 : &plus_oprnd0, &plus_oprnd1))
1484 : return NULL;
1485 :
1486 53654 : tree sum_type = TREE_TYPE (gimple_get_lhs (last_stmt));
1487 :
1488 : /* Any non-truncating sequence of conversions is OK here, since
1489 : with a successful match, the result of the ABS(U) is known to fit
1490 : within the nonnegative range of the result type. (It cannot be the
1491 : negative of the minimum signed value due to the range of the widening
1492 : MINUS_EXPR.) */
1493 53654 : vect_unpromoted_value unprom_abs;
1494 53654 : plus_oprnd0 = vect_look_through_possible_promotion (vinfo, plus_oprnd0,
1495 : &unprom_abs);
1496 :
1497 : /* So far so good. Since last_stmt was detected as a (summation) reduction,
1498 : we know that plus_oprnd1 is the reduction variable (defined by a loop-header
1499 : phi), and plus_oprnd0 is an ssa-name defined by a stmt in the loop body.
1500 : Then check that plus_oprnd0 is defined by an abs_expr. */
1501 :
1502 53654 : if (!plus_oprnd0)
1503 : return NULL;
1504 :
1505 36306 : stmt_vec_info abs_stmt_vinfo = vect_get_internal_def (vinfo, plus_oprnd0);
1506 36306 : if (!abs_stmt_vinfo)
1507 : return NULL;
1508 :
1509 : /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
1510 : inside the loop (in case we are analyzing an outer-loop). */
1511 35456 : gassign *abs_stmt = dyn_cast <gassign *> (abs_stmt_vinfo->stmt);
1512 106368 : vect_unpromoted_value unprom[2];
1513 :
1514 35456 : if (!abs_stmt)
1515 : {
1516 367 : gcall *abd_stmt = dyn_cast <gcall *> (abs_stmt_vinfo->stmt);
1517 302 : if (!abd_stmt
1518 302 : || !gimple_call_internal_p (abd_stmt)
1519 0 : || gimple_call_num_args (abd_stmt) != 2)
1520 : return NULL;
1521 :
1522 0 : tree abd_oprnd0 = gimple_call_arg (abd_stmt, 0);
1523 0 : tree abd_oprnd1 = gimple_call_arg (abd_stmt, 1);
1524 :
1525 0 : if (gimple_call_internal_fn (abd_stmt) == IFN_ABD
1526 0 : || gimple_call_internal_fn (abd_stmt) == IFN_VEC_WIDEN_ABD)
1527 : {
1528 0 : unprom[0].op = abd_oprnd0;
1529 0 : unprom[0].type = TREE_TYPE (abd_oprnd0);
1530 0 : unprom[1].op = abd_oprnd1;
1531 0 : unprom[1].type = TREE_TYPE (abd_oprnd1);
1532 : }
1533 : else
1534 : return NULL;
1535 :
1536 0 : half_type = unprom[0].type;
1537 : }
1538 35089 : else if (!vect_recog_absolute_difference (vinfo, abs_stmt, &half_type,
1539 : unprom, NULL))
1540 : return NULL;
1541 :
1542 854 : vect_pattern_detected ("vect_recog_sad_pattern", last_stmt);
1543 :
1544 854 : tree half_vectype;
1545 854 : if (!vect_supportable_conv_optab_p (vinfo, sum_type, SAD_EXPR, half_type,
1546 : type_out, &half_vectype))
1547 : return NULL;
1548 :
1549 : /* Get the inputs to the SAD_EXPR in the appropriate types. */
1550 460 : tree sad_oprnd[2];
1551 460 : vect_convert_inputs (vinfo, stmt_vinfo, 2, sad_oprnd, half_type,
1552 : unprom, half_vectype);
1553 :
1554 460 : tree var = vect_recog_temp_ssa_var (sum_type, NULL);
1555 460 : gimple *pattern_stmt = gimple_build_assign (var, SAD_EXPR, sad_oprnd[0],
1556 : sad_oprnd[1], plus_oprnd1);
1557 :
1558 460 : return pattern_stmt;
1559 : }
1560 :
1561 : /* Function vect_recog_abd_pattern
1562 :
1563 : Try to find the following ABsolute Difference (ABD) or
1564 : widening ABD (WIDEN_ABD) pattern:
1565 :
1566 : TYPE1 x;
1567 : TYPE2 y;
1568 : TYPE3 x_cast = (TYPE3) x; // widening or no-op
1569 : TYPE3 y_cast = (TYPE3) y; // widening or no-op
1570 : TYPE3 diff = x_cast - y_cast;
1571 : TYPE4 diff_cast = (TYPE4) diff; // widening or no-op
1572 : TYPE5 abs = ABS(U)_EXPR <diff_cast>;
1573 :
1574 : WIDEN_ABD exists to optimize the case where TYPE4 is at least
1575 : twice as wide as TYPE3.
1576 :
1577 : Input:
1578 :
1579 : * STMT_VINFO: The stmt from which the pattern search begins
1580 :
1581 : Output:
1582 :
1583 : * TYPE_OUT: The type of the output of this pattern
1584 :
1585 : * Return value: A new stmt that will be used to replace the sequence of
1586 : stmts that constitute the pattern, principally:
1587 : out = IFN_ABD (x, y)
1588 : out = IFN_WIDEN_ABD (x, y)
1589 : */
1590 :
1591 : static gimple *
1592 32999870 : vect_recog_abd_pattern (vec_info *vinfo,
1593 : stmt_vec_info stmt_vinfo, tree *type_out)
1594 : {
1595 32999870 : gassign *last_stmt = dyn_cast <gassign *> (STMT_VINFO_STMT (stmt_vinfo));
1596 23097227 : if (!last_stmt)
1597 : return NULL;
1598 :
1599 23097227 : tree out_type = TREE_TYPE (gimple_assign_lhs (last_stmt));
1600 :
1601 69291681 : vect_unpromoted_value unprom[2];
1602 23097227 : gassign *diff_stmt = NULL;
1603 23097227 : tree abd_in_type;
1604 23097227 : if (!vect_recog_absolute_difference (vinfo, last_stmt, &abd_in_type,
1605 : unprom, &diff_stmt))
1606 : {
1607 : /* We cannot try further without having a non-widening MINUS. */
1608 23095673 : if (!diff_stmt)
1609 : return NULL;
1610 :
1611 251 : unprom[0].op = gimple_assign_rhs1 (diff_stmt);
1612 251 : unprom[1].op = gimple_assign_rhs2 (diff_stmt);
1613 251 : abd_in_type = signed_type_for (out_type);
1614 : }
1615 :
1616 1805 : tree abd_out_type = abd_in_type;
1617 :
1618 1805 : tree vectype_in = get_vectype_for_scalar_type (vinfo, abd_in_type);
1619 1805 : if (!vectype_in)
1620 : return NULL;
1621 :
1622 1787 : internal_fn ifn = IFN_ABD;
1623 1787 : tree vectype_out = vectype_in;
1624 :
1625 1787 : if (TYPE_PRECISION (out_type) >= TYPE_PRECISION (abd_in_type) * 2
1626 1787 : && stmt_vinfo->min_output_precision >= TYPE_PRECISION (abd_in_type) * 2)
1627 : {
1628 1445 : tree mid_type
1629 1445 : = build_nonstandard_integer_type (TYPE_PRECISION (abd_in_type) * 2,
1630 1445 : TYPE_UNSIGNED (abd_in_type));
1631 1445 : tree mid_vectype = get_vectype_for_scalar_type (vinfo, mid_type);
1632 :
1633 1445 : code_helper dummy_code;
1634 1445 : int dummy_int;
1635 1445 : auto_vec<tree> dummy_vec;
1636 1445 : if (mid_vectype
1637 1445 : && supportable_widening_operation (IFN_VEC_WIDEN_ABD,
1638 : mid_vectype, vectype_in, false,
1639 : &dummy_code, &dummy_code,
1640 : &dummy_int, &dummy_vec))
1641 : {
1642 0 : ifn = IFN_VEC_WIDEN_ABD;
1643 0 : abd_out_type = mid_type;
1644 0 : vectype_out = mid_vectype;
1645 : }
1646 1445 : }
1647 :
1648 1445 : if (ifn == IFN_ABD
1649 1787 : && !direct_internal_fn_supported_p (ifn, vectype_in,
1650 : OPTIMIZE_FOR_SPEED))
1651 : return NULL;
1652 :
1653 0 : vect_pattern_detected ("vect_recog_abd_pattern", last_stmt);
1654 :
1655 0 : tree abd_oprnds[2];
1656 0 : vect_convert_inputs (vinfo, stmt_vinfo, 2, abd_oprnds,
1657 : abd_in_type, unprom, vectype_in);
1658 :
1659 0 : *type_out = get_vectype_for_scalar_type (vinfo, out_type);
1660 :
1661 0 : tree abd_result = vect_recog_temp_ssa_var (abd_out_type, NULL);
1662 0 : gcall *abd_stmt = gimple_build_call_internal (ifn, 2,
1663 : abd_oprnds[0], abd_oprnds[1]);
1664 0 : gimple_call_set_lhs (abd_stmt, abd_result);
1665 0 : gimple_set_location (abd_stmt, gimple_location (last_stmt));
1666 :
1667 0 : gimple *stmt = abd_stmt;
1668 0 : if (TYPE_PRECISION (abd_in_type) == TYPE_PRECISION (abd_out_type)
1669 0 : && TYPE_PRECISION (abd_out_type) < TYPE_PRECISION (out_type)
1670 0 : && !TYPE_UNSIGNED (abd_out_type))
1671 : {
1672 0 : tree unsign = unsigned_type_for (abd_out_type);
1673 0 : stmt = vect_convert_output (vinfo, stmt_vinfo, unsign, stmt, vectype_out);
1674 0 : vectype_out = get_vectype_for_scalar_type (vinfo, unsign);
1675 : }
1676 :
1677 0 : return vect_convert_output (vinfo, stmt_vinfo, out_type, stmt, vectype_out);
1678 : }
1679 :
1680 : /* Recognize an operation that performs ORIG_CODE on widened inputs,
1681 : so that it can be treated as though it had the form:
1682 :
1683 : A_TYPE a;
1684 : B_TYPE b;
1685 : HALF_TYPE a_cast = (HALF_TYPE) a; // possible no-op
1686 : HALF_TYPE b_cast = (HALF_TYPE) b; // possible no-op
1687 : | RES_TYPE a_extend = (RES_TYPE) a_cast; // promotion from HALF_TYPE
1688 : | RES_TYPE b_extend = (RES_TYPE) b_cast; // promotion from HALF_TYPE
1689 : | RES_TYPE res = a_extend ORIG_CODE b_extend;
1690 :
1691 : Try to replace the pattern with:
1692 :
1693 : A_TYPE a;
1694 : B_TYPE b;
1695 : HALF_TYPE a_cast = (HALF_TYPE) a; // possible no-op
1696 : HALF_TYPE b_cast = (HALF_TYPE) b; // possible no-op
1697 : | EXT_TYPE ext = a_cast WIDE_CODE b_cast;
1698 : | RES_TYPE res = (EXT_TYPE) ext; // possible no-op
1699 :
1700 : where EXT_TYPE is wider than HALF_TYPE but has the same signedness.
1701 :
1702 : SHIFT_P is true if ORIG_CODE and WIDE_CODE are shifts. NAME is the
1703 : name of the pattern being matched, for dump purposes. */
1704 :
1705 : static gimple *
1706 132507878 : vect_recog_widen_op_pattern (vec_info *vinfo,
1707 : stmt_vec_info last_stmt_info, tree *type_out,
1708 : tree_code orig_code, code_helper wide_code,
1709 : bool shift_p, const char *name)
1710 : {
1711 132507878 : gimple *last_stmt = last_stmt_info->stmt;
1712 :
1713 397523634 : vect_unpromoted_value unprom[2];
1714 132507878 : tree half_type;
1715 132507878 : if (!vect_widened_op_tree (vinfo, last_stmt_info, orig_code, orig_code,
1716 : shift_p, 2, unprom, &half_type))
1717 :
1718 : return NULL;
1719 :
1720 : /* Pattern detected. */
1721 371775 : vect_pattern_detected (name, last_stmt);
1722 :
1723 371775 : tree type = TREE_TYPE (gimple_get_lhs (last_stmt));
1724 371775 : tree itype = type;
1725 371775 : if (TYPE_PRECISION (type) != TYPE_PRECISION (half_type) * 2
1726 371775 : || TYPE_UNSIGNED (type) != TYPE_UNSIGNED (half_type))
1727 257108 : itype = build_nonstandard_integer_type (TYPE_PRECISION (half_type) * 2,
1728 257108 : TYPE_UNSIGNED (half_type));
1729 :
1730 : /* Check target support */
1731 371775 : tree vectype = get_vectype_for_scalar_type (vinfo, half_type);
1732 371775 : tree vecitype = get_vectype_for_scalar_type (vinfo, itype);
1733 371775 : tree ctype = itype;
1734 371775 : tree vecctype = vecitype;
1735 371775 : if (orig_code == MINUS_EXPR
1736 8951 : && TYPE_UNSIGNED (itype)
1737 378695 : && TYPE_PRECISION (type) > TYPE_PRECISION (itype))
1738 : {
1739 : /* Subtraction is special, even if half_type is unsigned and no matter
1740 : whether type is signed or unsigned, if type is wider than itype,
1741 : we need to sign-extend from the widening operation result to the
1742 : result type.
1743 : Consider half_type unsigned char, operand 1 0xfe, operand 2 0xff,
1744 : itype unsigned short and type either int or unsigned int.
1745 : Widened (unsigned short) 0xfe - (unsigned short) 0xff is
1746 : (unsigned short) 0xffff, but for type int we want the result -1
1747 : and for type unsigned int 0xffffffff rather than 0xffff. */
1748 3066 : ctype = build_nonstandard_integer_type (TYPE_PRECISION (itype), 0);
1749 3066 : vecctype = get_vectype_for_scalar_type (vinfo, ctype);
1750 : }
1751 :
1752 371775 : code_helper dummy_code;
1753 371775 : int dummy_int;
1754 371775 : auto_vec<tree> dummy_vec;
1755 371775 : if (!vectype
1756 371775 : || !vecitype
1757 288808 : || !vecctype
1758 660583 : || !supportable_widening_operation (wide_code, vecitype, vectype, true,
1759 : &dummy_code, &dummy_code,
1760 : &dummy_int, &dummy_vec))
1761 : return NULL;
1762 :
1763 117484 : *type_out = get_vectype_for_scalar_type (vinfo, type);
1764 117484 : if (!*type_out)
1765 : return NULL;
1766 :
1767 117484 : tree oprnd[2];
1768 117484 : vect_convert_inputs (vinfo, last_stmt_info,
1769 : 2, oprnd, half_type, unprom, vectype);
1770 :
1771 117484 : tree var = vect_recog_temp_ssa_var (itype, NULL);
1772 117484 : gimple *pattern_stmt = vect_gimple_build (var, wide_code, oprnd[0], oprnd[1]);
1773 :
1774 117484 : if (vecctype != vecitype)
1775 0 : pattern_stmt = vect_convert_output (vinfo, last_stmt_info, ctype,
1776 : pattern_stmt, vecitype);
1777 :
1778 117484 : return vect_convert_output (vinfo, last_stmt_info,
1779 117484 : type, pattern_stmt, vecctype);
1780 371775 : }
1781 :
1782 : /* Try to detect multiplication on widened inputs, converting MULT_EXPR
1783 : to WIDEN_MULT_EXPR. See vect_recog_widen_op_pattern for details. */
1784 :
1785 : static gimple *
1786 33083170 : vect_recog_widen_mult_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
1787 : tree *type_out)
1788 : {
1789 33083170 : return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
1790 33083170 : MULT_EXPR, WIDEN_MULT_EXPR, false,
1791 33083170 : "vect_recog_widen_mult_pattern");
1792 : }
1793 :
1794 : /* Try to detect addition on widened inputs, converting PLUS_EXPR
1795 : to IFN_VEC_WIDEN_PLUS. See vect_recog_widen_op_pattern for details. */
1796 :
1797 : static gimple *
1798 33185090 : vect_recog_widen_plus_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
1799 : tree *type_out)
1800 : {
1801 33185090 : return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
1802 33185090 : PLUS_EXPR, IFN_VEC_WIDEN_PLUS,
1803 33185090 : false, "vect_recog_widen_plus_pattern");
1804 : }
1805 :
1806 : /* Try to detect subtraction on widened inputs, converting MINUS_EXPR
1807 : to IFN_VEC_WIDEN_MINUS. See vect_recog_widen_op_pattern for details. */
1808 : static gimple *
1809 33185090 : vect_recog_widen_minus_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
1810 : tree *type_out)
1811 : {
1812 33185090 : return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
1813 33185090 : MINUS_EXPR, IFN_VEC_WIDEN_MINUS,
1814 33185090 : false, "vect_recog_widen_minus_pattern");
1815 : }
1816 :
1817 : /* Try to detect abd on widened inputs, converting IFN_ABD
1818 : to IFN_VEC_WIDEN_ABD. */
1819 : static gimple *
1820 33185090 : vect_recog_widen_abd_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
1821 : tree *type_out)
1822 : {
1823 33185090 : gassign *last_stmt = dyn_cast <gassign *> (STMT_VINFO_STMT (stmt_vinfo));
1824 31392613 : if (!last_stmt || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (last_stmt)))
1825 : return NULL;
1826 :
1827 3370826 : tree last_rhs = gimple_assign_rhs1 (last_stmt);
1828 :
1829 3370826 : tree in_type = TREE_TYPE (last_rhs);
1830 3370826 : tree out_type = TREE_TYPE (gimple_assign_lhs (last_stmt));
1831 3370826 : if (!INTEGRAL_TYPE_P (in_type)
1832 2947713 : || !INTEGRAL_TYPE_P (out_type)
1833 2819196 : || TYPE_PRECISION (in_type) * 2 != TYPE_PRECISION (out_type)
1834 4050393 : || !TYPE_UNSIGNED (in_type))
1835 : return NULL;
1836 :
1837 248498 : vect_unpromoted_value unprom;
1838 248498 : tree op = vect_look_through_possible_promotion (vinfo, last_rhs, &unprom);
1839 248498 : if (!op || TYPE_PRECISION (TREE_TYPE (op)) != TYPE_PRECISION (in_type))
1840 : return NULL;
1841 :
1842 245941 : stmt_vec_info abd_pattern_vinfo = vect_get_internal_def (vinfo, op);
1843 245941 : if (!abd_pattern_vinfo)
1844 : return NULL;
1845 :
1846 220365 : gcall *abd_stmt = dyn_cast <gcall *> (STMT_VINFO_STMT (abd_pattern_vinfo));
1847 9292 : if (!abd_stmt
1848 9292 : || !gimple_call_internal_p (abd_stmt)
1849 320 : || gimple_call_internal_fn (abd_stmt) != IFN_ABD)
1850 : return NULL;
1851 :
1852 0 : tree vectype_in = get_vectype_for_scalar_type (vinfo, in_type);
1853 0 : tree vectype_out = get_vectype_for_scalar_type (vinfo, out_type);
1854 :
1855 0 : code_helper dummy_code;
1856 0 : int dummy_int;
1857 0 : auto_vec<tree> dummy_vec;
1858 0 : if (!supportable_widening_operation (IFN_VEC_WIDEN_ABD, vectype_out,
1859 : vectype_in, false,
1860 : &dummy_code, &dummy_code,
1861 : &dummy_int, &dummy_vec))
1862 : return NULL;
1863 :
1864 0 : vect_pattern_detected ("vect_recog_widen_abd_pattern", last_stmt);
1865 :
1866 0 : *type_out = vectype_out;
1867 :
1868 0 : tree abd_oprnd0 = gimple_call_arg (abd_stmt, 0);
1869 0 : tree abd_oprnd1 = gimple_call_arg (abd_stmt, 1);
1870 0 : tree widen_abd_result = vect_recog_temp_ssa_var (out_type, NULL);
1871 0 : gcall *widen_abd_stmt = gimple_build_call_internal (IFN_VEC_WIDEN_ABD, 2,
1872 : abd_oprnd0, abd_oprnd1);
1873 0 : gimple_call_set_lhs (widen_abd_stmt, widen_abd_result);
1874 0 : gimple_set_location (widen_abd_stmt, gimple_location (last_stmt));
1875 0 : return widen_abd_stmt;
1876 0 : }
1877 :
1878 : /* Function vect_recog_ctz_ffs_pattern
1879 :
1880 : Try to find the following pattern:
1881 :
1882 : TYPE1 A;
1883 : TYPE1 B;
1884 :
1885 : B = __builtin_ctz{,l,ll} (A);
1886 :
1887 : or
1888 :
1889 : B = __builtin_ffs{,l,ll} (A);
1890 :
1891 : Input:
1892 :
1893 : * STMT_VINFO: The stmt from which the pattern search begins.
1894 : here it starts with B = __builtin_* (A);
1895 :
1896 : Output:
1897 :
1898 : * TYPE_OUT: The vector type of the output of this pattern.
1899 :
1900 : * Return value: A new stmt that will be used to replace the sequence of
1901 : stmts that constitute the pattern, using clz or popcount builtins. */
1902 :
1903 : static gimple *
1904 33054288 : vect_recog_ctz_ffs_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
1905 : tree *type_out)
1906 : {
1907 33054288 : gimple *call_stmt = stmt_vinfo->stmt;
1908 33054288 : gimple *pattern_stmt;
1909 33054288 : tree rhs_oprnd, rhs_type, lhs_oprnd, lhs_type, vec_type, vec_rhs_type;
1910 33054288 : tree new_var;
1911 33054288 : internal_fn ifn = IFN_LAST, ifnnew = IFN_LAST;
1912 33054288 : bool defined_at_zero = true, defined_at_zero_new = false;
1913 33054288 : int val = 0, val_new = 0, val_cmp = 0;
1914 33054288 : int prec;
1915 33054288 : int sub = 0, add = 0;
1916 33054288 : location_t loc;
1917 :
1918 33054288 : if (!is_gimple_call (call_stmt))
1919 : return NULL;
1920 :
1921 3844734 : if (gimple_call_num_args (call_stmt) != 1
1922 3844734 : && gimple_call_num_args (call_stmt) != 2)
1923 : return NULL;
1924 :
1925 2180799 : rhs_oprnd = gimple_call_arg (call_stmt, 0);
1926 2180799 : rhs_type = TREE_TYPE (rhs_oprnd);
1927 2180799 : lhs_oprnd = gimple_call_lhs (call_stmt);
1928 2180799 : if (!lhs_oprnd)
1929 : return NULL;
1930 1069052 : lhs_type = TREE_TYPE (lhs_oprnd);
1931 1069052 : if (!INTEGRAL_TYPE_P (lhs_type)
1932 357330 : || !INTEGRAL_TYPE_P (rhs_type)
1933 56128 : || !type_has_mode_precision_p (rhs_type)
1934 1122171 : || TREE_CODE (rhs_oprnd) != SSA_NAME)
1935 : return NULL;
1936 :
1937 39211 : switch (gimple_call_combined_fn (call_stmt))
1938 : {
1939 2075 : CASE_CFN_CTZ:
1940 2075 : ifn = IFN_CTZ;
1941 2075 : if (!gimple_call_internal_p (call_stmt)
1942 2075 : || gimple_call_num_args (call_stmt) != 2)
1943 : defined_at_zero = false;
1944 : else
1945 139 : val = tree_to_shwi (gimple_call_arg (call_stmt, 1));
1946 : break;
1947 : CASE_CFN_FFS:
1948 : ifn = IFN_FFS;
1949 : break;
1950 : default:
1951 : return NULL;
1952 : }
1953 :
1954 2310 : prec = TYPE_PRECISION (rhs_type);
1955 2310 : loc = gimple_location (call_stmt);
1956 :
1957 2310 : vec_type = get_vectype_for_scalar_type (vinfo, lhs_type);
1958 2310 : if (!vec_type)
1959 : return NULL;
1960 :
1961 2304 : vec_rhs_type = get_vectype_for_scalar_type (vinfo, rhs_type);
1962 2304 : if (!vec_rhs_type)
1963 : return NULL;
1964 :
1965 : /* Do it only if the backend doesn't have ctz<vector_mode>2 or
1966 : ffs<vector_mode>2 pattern but does have clz<vector_mode>2 or
1967 : popcount<vector_mode>2. */
1968 2038 : if (!vec_type
1969 2038 : || direct_internal_fn_supported_p (ifn, vec_rhs_type,
1970 : OPTIMIZE_FOR_SPEED))
1971 : return NULL;
1972 :
1973 2038 : if (ifn == IFN_FFS
1974 2038 : && direct_internal_fn_supported_p (IFN_CTZ, vec_rhs_type,
1975 : OPTIMIZE_FOR_SPEED))
1976 : {
1977 0 : ifnnew = IFN_CTZ;
1978 0 : defined_at_zero_new
1979 0 : = CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_rhs_type),
1980 : val_new) == 2;
1981 : }
1982 2038 : else if (direct_internal_fn_supported_p (IFN_CLZ, vec_rhs_type,
1983 : OPTIMIZE_FOR_SPEED))
1984 : {
1985 166 : ifnnew = IFN_CLZ;
1986 166 : defined_at_zero_new
1987 166 : = CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_rhs_type),
1988 : val_new) == 2;
1989 : }
1990 166 : if ((ifnnew == IFN_LAST
1991 166 : || (defined_at_zero && !defined_at_zero_new))
1992 1872 : && direct_internal_fn_supported_p (IFN_POPCOUNT, vec_rhs_type,
1993 : OPTIMIZE_FOR_SPEED))
1994 : {
1995 : ifnnew = IFN_POPCOUNT;
1996 : defined_at_zero_new = true;
1997 : val_new = prec;
1998 : }
1999 1912 : if (ifnnew == IFN_LAST)
2000 : return NULL;
2001 :
2002 : /* Only .FFS (X) = .CTZ (X) + 1 can be part of a reduction, and only
2003 : if the result for zero needs no select. */
2004 292 : if (vect_is_reduction (stmt_vinfo)
2005 292 : && (ifnnew != IFN_CTZ
2006 0 : || (defined_at_zero
2007 0 : && (!defined_at_zero_new || val != val_new + 1))))
2008 : return NULL;
2009 :
2010 286 : vect_pattern_detected ("vec_recog_ctz_ffs_pattern", call_stmt);
2011 :
2012 : /* .CLZ, .CTZ and .POPCOUNT take an unsigned argument. */
2013 286 : if (!TYPE_UNSIGNED (rhs_type))
2014 : {
2015 51 : rhs_type = unsigned_type_for (rhs_type);
2016 51 : vec_rhs_type = get_vectype_for_scalar_type (vinfo, rhs_type);
2017 51 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
2018 51 : pattern_stmt = gimple_build_assign (new_var, NOP_EXPR, rhs_oprnd);
2019 51 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2020 51 : rhs_oprnd = new_var;
2021 : }
2022 :
2023 286 : val_cmp = val_new;
2024 286 : if ((ifnnew == IFN_CLZ
2025 286 : && defined_at_zero
2026 106 : && defined_at_zero_new
2027 106 : && val == prec
2028 54 : && val_new == prec)
2029 232 : || (ifnnew == IFN_POPCOUNT && ifn == IFN_CTZ))
2030 : {
2031 : /* .CTZ (X) = PREC - .CLZ ((X - 1) & ~X)
2032 : .CTZ (X) = .POPCOUNT ((X - 1) & ~X). */
2033 : if (ifnnew == IFN_CLZ)
2034 : sub = prec;
2035 137 : val_cmp = prec;
2036 :
2037 137 : tree m1 = vect_recog_temp_ssa_var (rhs_type, NULL);
2038 137 : pattern_stmt = gimple_build_assign (m1, PLUS_EXPR, rhs_oprnd,
2039 : build_int_cst (rhs_type, -1));
2040 137 : gimple_set_location (pattern_stmt, loc);
2041 137 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2042 :
2043 137 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
2044 137 : pattern_stmt = gimple_build_assign (new_var, BIT_NOT_EXPR, rhs_oprnd);
2045 137 : gimple_set_location (pattern_stmt, loc);
2046 137 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2047 137 : rhs_oprnd = new_var;
2048 :
2049 137 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
2050 137 : pattern_stmt = gimple_build_assign (new_var, BIT_AND_EXPR,
2051 : m1, rhs_oprnd);
2052 137 : gimple_set_location (pattern_stmt, loc);
2053 137 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2054 137 : rhs_oprnd = new_var;
2055 137 : }
2056 149 : else if (ifnnew == IFN_CLZ)
2057 : {
2058 : /* .CTZ (X) = (PREC - 1) - .CLZ (X & -X)
2059 : .FFS (X) = PREC - .CLZ (X & -X). */
2060 106 : sub = prec - (ifn == IFN_CTZ);
2061 106 : val_cmp = sub - val_new;
2062 :
2063 106 : tree neg = vect_recog_temp_ssa_var (rhs_type, NULL);
2064 106 : pattern_stmt = gimple_build_assign (neg, NEGATE_EXPR, rhs_oprnd);
2065 106 : gimple_set_location (pattern_stmt, loc);
2066 106 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2067 :
2068 106 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
2069 106 : pattern_stmt = gimple_build_assign (new_var, BIT_AND_EXPR,
2070 : rhs_oprnd, neg);
2071 106 : gimple_set_location (pattern_stmt, loc);
2072 106 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2073 106 : rhs_oprnd = new_var;
2074 : }
2075 43 : else if (ifnnew == IFN_POPCOUNT)
2076 : {
2077 : /* .CTZ (X) = PREC - .POPCOUNT (X | -X)
2078 : .FFS (X) = (PREC + 1) - .POPCOUNT (X | -X). */
2079 43 : sub = prec + (ifn == IFN_FFS);
2080 43 : val_cmp = sub;
2081 :
2082 43 : tree neg = vect_recog_temp_ssa_var (rhs_type, NULL);
2083 43 : pattern_stmt = gimple_build_assign (neg, NEGATE_EXPR, rhs_oprnd);
2084 43 : gimple_set_location (pattern_stmt, loc);
2085 43 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2086 :
2087 43 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
2088 43 : pattern_stmt = gimple_build_assign (new_var, BIT_IOR_EXPR,
2089 : rhs_oprnd, neg);
2090 43 : gimple_set_location (pattern_stmt, loc);
2091 43 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2092 43 : rhs_oprnd = new_var;
2093 : }
2094 0 : else if (ifnnew == IFN_CTZ)
2095 : {
2096 : /* .FFS (X) = .CTZ (X) + 1. */
2097 0 : add = 1;
2098 0 : val_cmp++;
2099 : }
2100 :
2101 : /* Create B = .IFNNEW (A). */
2102 286 : new_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2103 286 : if ((ifnnew == IFN_CLZ || ifnnew == IFN_CTZ) && defined_at_zero_new)
2104 160 : pattern_stmt
2105 160 : = gimple_build_call_internal (ifnnew, 2, rhs_oprnd,
2106 : build_int_cst (integer_type_node,
2107 160 : val_new));
2108 : else
2109 126 : pattern_stmt = gimple_build_call_internal (ifnnew, 1, rhs_oprnd);
2110 286 : gimple_call_set_lhs (pattern_stmt, new_var);
2111 286 : gimple_set_location (pattern_stmt, loc);
2112 286 : *type_out = vec_type;
2113 :
2114 286 : if (sub)
2115 : {
2116 203 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2117 203 : tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2118 203 : pattern_stmt = gimple_build_assign (ret_var, MINUS_EXPR,
2119 203 : build_int_cst (lhs_type, sub),
2120 : new_var);
2121 203 : gimple_set_location (pattern_stmt, loc);
2122 203 : new_var = ret_var;
2123 : }
2124 83 : else if (add)
2125 : {
2126 0 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2127 0 : tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2128 0 : pattern_stmt = gimple_build_assign (ret_var, PLUS_EXPR, new_var,
2129 0 : build_int_cst (lhs_type, add));
2130 0 : gimple_set_location (pattern_stmt, loc);
2131 0 : new_var = ret_var;
2132 : }
2133 :
2134 286 : if (defined_at_zero
2135 210 : && (!defined_at_zero_new || val != val_cmp))
2136 : {
2137 43 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2138 43 : tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2139 43 : rhs_oprnd = gimple_call_arg (call_stmt, 0);
2140 43 : rhs_type = TREE_TYPE (rhs_oprnd);
2141 43 : tree cmp = vect_recog_temp_ssa_var (boolean_type_node, NULL);
2142 43 : pattern_stmt = gimple_build_assign (cmp, NE_EXPR, rhs_oprnd,
2143 : build_zero_cst (rhs_type));
2144 43 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
2145 : truth_type_for (vec_type), rhs_type);
2146 43 : pattern_stmt = gimple_build_assign (ret_var, COND_EXPR, cmp,
2147 : new_var,
2148 43 : build_int_cst (lhs_type, val));
2149 : }
2150 :
2151 286 : if (dump_enabled_p ())
2152 36 : dump_printf_loc (MSG_NOTE, vect_location,
2153 : "created pattern stmt: %G", pattern_stmt);
2154 :
2155 : return pattern_stmt;
2156 : }
2157 :
2158 : /* Function vect_recog_popcount_clz_ctz_ffs_pattern
2159 :
2160 : Try to find the following pattern:
2161 :
2162 : UTYPE1 A;
2163 : TYPE1 B;
2164 : UTYPE2 temp_in;
2165 : TYPE3 temp_out;
2166 : temp_in = (UTYPE2)A;
2167 :
2168 : temp_out = __builtin_popcount{,l,ll} (temp_in);
2169 : B = (TYPE1) temp_out;
2170 :
2171 : TYPE2 may or may not be equal to TYPE3.
2172 : i.e. TYPE2 is equal to TYPE3 for __builtin_popcount
2173 : i.e. TYPE2 is not equal to TYPE3 for __builtin_popcountll
2174 :
2175 : Input:
2176 :
2177 : * STMT_VINFO: The stmt from which the pattern search begins.
2178 : here it starts with B = (TYPE1) temp_out;
2179 :
2180 : Output:
2181 :
2182 : * TYPE_OUT: The vector type of the output of this pattern.
2183 :
2184 : * Return value: A new stmt that will be used to replace the sequence of
2185 : stmts that constitute the pattern. In this case it will be:
2186 : B = .POPCOUNT (A);
2187 :
2188 : Similarly for clz, ctz and ffs.
2189 : */
2190 :
2191 : static gimple *
2192 33053890 : vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
2193 : stmt_vec_info stmt_vinfo,
2194 : tree *type_out)
2195 : {
2196 33053890 : gassign *last_stmt = dyn_cast <gassign *> (stmt_vinfo->stmt);
2197 23146502 : gimple *call_stmt, *pattern_stmt;
2198 23146502 : tree rhs_oprnd, rhs_origin, lhs_oprnd, lhs_type, vec_type, new_var;
2199 56200392 : internal_fn ifn = IFN_LAST;
2200 56200392 : int addend = 0;
2201 :
2202 : /* Find B = (TYPE1) temp_out. */
2203 23146502 : if (!last_stmt)
2204 : return NULL;
2205 23146502 : tree_code code = gimple_assign_rhs_code (last_stmt);
2206 23146502 : if (!CONVERT_EXPR_CODE_P (code))
2207 : return NULL;
2208 :
2209 3213684 : lhs_oprnd = gimple_assign_lhs (last_stmt);
2210 3213684 : lhs_type = TREE_TYPE (lhs_oprnd);
2211 3213684 : if (!INTEGRAL_TYPE_P (lhs_type))
2212 : return NULL;
2213 :
2214 2922385 : rhs_oprnd = gimple_assign_rhs1 (last_stmt);
2215 2922385 : if (TREE_CODE (rhs_oprnd) != SSA_NAME
2216 2922385 : || !has_single_use (rhs_oprnd))
2217 : return NULL;
2218 1490258 : call_stmt = SSA_NAME_DEF_STMT (rhs_oprnd);
2219 :
2220 : /* Find temp_out = __builtin_popcount{,l,ll} (temp_in); */
2221 1490258 : if (!is_gimple_call (call_stmt))
2222 : return NULL;
2223 60958 : switch (gimple_call_combined_fn (call_stmt))
2224 : {
2225 : int val;
2226 : CASE_CFN_POPCOUNT:
2227 : ifn = IFN_POPCOUNT;
2228 : break;
2229 2412 : CASE_CFN_CLZ:
2230 2412 : ifn = IFN_CLZ;
2231 : /* Punt if call result is unsigned and defined value at zero
2232 : is negative, as the negative value doesn't extend correctly. */
2233 2412 : if (TYPE_UNSIGNED (TREE_TYPE (rhs_oprnd))
2234 0 : && gimple_call_internal_p (call_stmt)
2235 2412 : && CLZ_DEFINED_VALUE_AT_ZERO
2236 : (SCALAR_INT_TYPE_MODE (TREE_TYPE (rhs_oprnd)), val) == 2
2237 2412 : && val < 0)
2238 : return NULL;
2239 : break;
2240 765 : CASE_CFN_CTZ:
2241 765 : ifn = IFN_CTZ;
2242 : /* Punt if call result is unsigned and defined value at zero
2243 : is negative, as the negative value doesn't extend correctly. */
2244 765 : if (TYPE_UNSIGNED (TREE_TYPE (rhs_oprnd))
2245 0 : && gimple_call_internal_p (call_stmt)
2246 765 : && CTZ_DEFINED_VALUE_AT_ZERO
2247 : (SCALAR_INT_TYPE_MODE (TREE_TYPE (rhs_oprnd)), val) == 2
2248 765 : && val < 0)
2249 : return NULL;
2250 : break;
2251 57 : CASE_CFN_FFS:
2252 57 : ifn = IFN_FFS;
2253 57 : break;
2254 : default:
2255 : return NULL;
2256 : }
2257 :
2258 3588 : if (gimple_call_num_args (call_stmt) != 1
2259 3588 : && gimple_call_num_args (call_stmt) != 2)
2260 : return NULL;
2261 :
2262 3588 : rhs_oprnd = gimple_call_arg (call_stmt, 0);
2263 3588 : vect_unpromoted_value unprom_diff;
2264 3588 : rhs_origin
2265 3588 : = vect_look_through_possible_promotion (vinfo, rhs_oprnd, &unprom_diff);
2266 :
2267 3588 : if (!rhs_origin)
2268 : return NULL;
2269 :
2270 : /* Input and output of .POPCOUNT should be same-precision integer. */
2271 3578 : if (TYPE_PRECISION (unprom_diff.type) != TYPE_PRECISION (lhs_type))
2272 : return NULL;
2273 :
2274 1867 : vec_type = get_vectype_for_scalar_type (vinfo, lhs_type);
2275 1867 : if (!vec_type || !VECTOR_MODE_P (TYPE_MODE (vec_type)))
2276 : return NULL;
2277 :
2278 : /* Also A should be unsigned or same precision as temp_in, otherwise
2279 : different builtins/internal functions have different behaviors. */
2280 1735 : if (TYPE_PRECISION (unprom_diff.type)
2281 1735 : != TYPE_PRECISION (TREE_TYPE (rhs_oprnd)))
2282 319 : switch (ifn)
2283 : {
2284 147 : case IFN_POPCOUNT:
2285 : /* For popcount require zero extension, which doesn't add any
2286 : further bits to the count. */
2287 147 : if (!TYPE_UNSIGNED (unprom_diff.type))
2288 : return NULL;
2289 : break;
2290 109 : case IFN_CLZ:
2291 : /* clzll (x) == clz (x) + 32 for unsigned x != 0. */
2292 109 : if (!TYPE_UNSIGNED (unprom_diff.type))
2293 : return NULL;
2294 109 : if (!type_has_mode_precision_p (lhs_type)
2295 109 : || !type_has_mode_precision_p (TREE_TYPE (rhs_oprnd)))
2296 : return NULL;
2297 109 : addend = (TYPE_PRECISION (TREE_TYPE (rhs_oprnd))
2298 109 : - TYPE_PRECISION (lhs_type));
2299 109 : break;
2300 : case IFN_CTZ:
2301 : /* ctzll (x) == ctz (x) for unsigned or signed x != 0. */
2302 : break;
2303 : case IFN_FFS:
2304 : /* ffsll (x) == ffs (x) for unsigned or signed x. */
2305 : break;
2306 : default:
2307 : gcc_unreachable ();
2308 : }
2309 :
2310 : /* For zero the pattern gives the value of the target for VEC_TYPE plus
2311 : ADDEND. If the call has another value at zero, select it. */
2312 1735 : tree zero_val = NULL_TREE;
2313 1735 : if ((ifn == IFN_CLZ || ifn == IFN_CTZ)
2314 1383 : && gimple_call_internal_p (call_stmt)
2315 1887 : && gimple_call_num_args (call_stmt) == 2)
2316 : {
2317 138 : int val;
2318 138 : int d = (ifn == IFN_CLZ
2319 138 : ? CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val)
2320 174 : : CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val));
2321 138 : if (d != 2
2322 138 : || tree_to_shwi (gimple_call_arg (call_stmt, 1)) != val + addend)
2323 94 : zero_val = gimple_call_arg (call_stmt, 1);
2324 : }
2325 :
2326 : /* Do it only if the backend has popcount<vector_mode>2 etc. pattern. */
2327 1735 : bool supported
2328 1735 : = direct_internal_fn_supported_p (ifn, vec_type, OPTIMIZE_FOR_SPEED);
2329 : /* We cannot rely on the reduction check in vect_recog_ctz_ffs_pattern
2330 : because we do not set up the temporary pattern stmt finally for
2331 : the recursive analysis. */
2332 1735 : if (!supported && vect_is_reduction (stmt_vinfo))
2333 : return NULL;
2334 1719 : if (!supported)
2335 1542 : switch (ifn)
2336 : {
2337 : case IFN_POPCOUNT:
2338 : case IFN_CLZ:
2339 : return NULL;
2340 57 : case IFN_FFS:
2341 : /* vect_recog_ctz_ffs_pattern can implement ffs using ctz. */
2342 57 : if (direct_internal_fn_supported_p (IFN_CTZ, vec_type,
2343 : OPTIMIZE_FOR_SPEED))
2344 : break;
2345 : /* FALLTHRU */
2346 552 : case IFN_CTZ:
2347 : /* vect_recog_ctz_ffs_pattern can implement ffs or ctz using
2348 : clz or popcount. */
2349 552 : if (direct_internal_fn_supported_p (IFN_CLZ, vec_type,
2350 : OPTIMIZE_FOR_SPEED))
2351 : break;
2352 492 : if (direct_internal_fn_supported_p (IFN_POPCOUNT, vec_type,
2353 : OPTIMIZE_FOR_SPEED))
2354 : break;
2355 : return NULL;
2356 : default:
2357 : gcc_unreachable ();
2358 : }
2359 :
2360 318 : vect_pattern_detected ("vec_recog_popcount_clz_ctz_ffs_pattern",
2361 : call_stmt);
2362 :
2363 318 : tree orig_lhs_type = lhs_type;
2364 318 : if (!type_has_mode_precision_p (lhs_type))
2365 0 : lhs_type = TREE_TYPE (vec_type);
2366 318 : tree arg_type = (lhs_type != orig_lhs_type
2367 318 : ? lhs_type : TREE_TYPE (unprom_diff.op));
2368 : /* .CLZ, .CTZ and .POPCOUNT take an unsigned argument. */
2369 318 : if (ifn != IFN_FFS && !TYPE_UNSIGNED (arg_type))
2370 40 : arg_type = unsigned_type_for (arg_type);
2371 318 : gimple *cast_stmt = NULL;
2372 318 : tree arg_vectype = vec_type;
2373 318 : if (arg_type != TREE_TYPE (unprom_diff.op))
2374 : {
2375 40 : arg_vectype = get_vectype_for_scalar_type (vinfo, arg_type);
2376 40 : if (!arg_vectype)
2377 : return NULL;
2378 40 : cast_stmt
2379 40 : = gimple_build_assign (vect_recog_temp_ssa_var (arg_type, NULL),
2380 : NOP_EXPR, unprom_diff.op);
2381 40 : unprom_diff.op = gimple_assign_lhs (cast_stmt);
2382 : }
2383 :
2384 : /* Create B = .POPCOUNT (A). */
2385 318 : new_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2386 318 : tree arg2 = NULL_TREE;
2387 318 : int val;
2388 318 : if (ifn == IFN_CLZ
2389 368 : && CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type),
2390 : val) == 2)
2391 48 : arg2 = build_int_cst (integer_type_node, val);
2392 270 : else if (ifn == IFN_CTZ
2393 363 : && CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type),
2394 : val) == 2)
2395 93 : arg2 = build_int_cst (integer_type_node, val);
2396 : /* vect_recog_ctz_ffs_pattern selects the value at zero itself. */
2397 318 : if (zero_val && !supported)
2398 : {
2399 : arg2 = zero_val;
2400 : zero_val = NULL_TREE;
2401 : }
2402 318 : if (arg2)
2403 141 : pattern_stmt = gimple_build_call_internal (ifn, 2, unprom_diff.op, arg2);
2404 : else
2405 177 : pattern_stmt = gimple_build_call_internal (ifn, 1, unprom_diff.op);
2406 318 : gimple_call_set_lhs (pattern_stmt, new_var);
2407 318 : gimple_set_location (pattern_stmt, gimple_location (last_stmt));
2408 318 : *type_out = vec_type;
2409 :
2410 318 : if (dump_enabled_p ())
2411 24 : dump_printf_loc (MSG_NOTE, vect_location,
2412 : "created pattern stmt: %G", pattern_stmt);
2413 :
2414 318 : if (cast_stmt && supported)
2415 28 : append_pattern_def_seq (vinfo, stmt_vinfo, cast_stmt, arg_vectype);
2416 :
2417 318 : if (addend)
2418 : {
2419 12 : gcc_assert (supported);
2420 12 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2421 12 : tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2422 12 : pattern_stmt = gimple_build_assign (ret_var, PLUS_EXPR, new_var,
2423 12 : build_int_cst (lhs_type, addend));
2424 : }
2425 306 : else if (!supported)
2426 : {
2427 141 : stmt_vec_info new_stmt_info = vinfo->add_stmt (pattern_stmt);
2428 141 : STMT_VINFO_VECTYPE (new_stmt_info) = vec_type;
2429 141 : pattern_stmt
2430 141 : = vect_recog_ctz_ffs_pattern (vinfo, new_stmt_info, type_out);
2431 141 : if (pattern_stmt == NULL)
2432 : return NULL;
2433 141 : if (cast_stmt)
2434 12 : append_pattern_def_seq (vinfo, stmt_vinfo, cast_stmt, arg_vectype);
2435 141 : if (gimple_seq seq = STMT_VINFO_PATTERN_DEF_SEQ (new_stmt_info))
2436 : {
2437 141 : gimple_seq *pseq = &STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo);
2438 141 : gimple_seq_add_seq_without_update (pseq, seq);
2439 : }
2440 : }
2441 :
2442 318 : if (zero_val)
2443 : {
2444 0 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2445 0 : tree op = unprom_diff.op;
2446 0 : tree cmp = vect_recog_temp_ssa_var (boolean_type_node, NULL);
2447 0 : gimple *cmp_stmt = gimple_build_assign (cmp, NE_EXPR, op,
2448 0 : build_zero_cst (TREE_TYPE (op)));
2449 0 : append_pattern_def_seq (vinfo, stmt_vinfo, cmp_stmt,
2450 0 : truth_type_for (vec_type), TREE_TYPE (op));
2451 0 : tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2452 0 : pattern_stmt = gimple_build_assign (ret_var, COND_EXPR, cmp,
2453 : gimple_get_lhs (pattern_stmt),
2454 : fold_convert (lhs_type, zero_val));
2455 : }
2456 :
2457 318 : if (lhs_type != orig_lhs_type)
2458 : {
2459 0 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2460 0 : tree ret_var = vect_recog_temp_ssa_var (orig_lhs_type, NULL);
2461 0 : pattern_stmt = gimple_build_assign (ret_var, NOP_EXPR,
2462 : gimple_get_lhs (pattern_stmt));
2463 : }
2464 : return pattern_stmt;
2465 : }
2466 :
2467 : /* Function vect_recog_pow_pattern
2468 :
2469 : Try to find the following pattern:
2470 :
2471 : x = POW (y, N);
2472 :
2473 : with POW being one of pow, powf, powi, powif and N being
2474 : either 2 or 0.5.
2475 :
2476 : Input:
2477 :
2478 : * STMT_VINFO: The stmt from which the pattern search begins.
2479 :
2480 : Output:
2481 :
2482 : * TYPE_OUT: The type of the output of this pattern.
2483 :
2484 : * Return value: A new stmt that will be used to replace the sequence of
2485 : stmts that constitute the pattern. In this case it will be:
2486 : x = x * x
2487 : or
2488 : x = sqrt (x)
2489 : */
2490 :
2491 : static gimple *
2492 33053899 : vect_recog_pow_pattern (vec_info *vinfo,
2493 : stmt_vec_info stmt_vinfo, tree *type_out)
2494 : {
2495 33053899 : gimple *last_stmt = stmt_vinfo->stmt;
2496 33053899 : tree base, exp;
2497 33053899 : gimple *stmt;
2498 33053899 : tree var;
2499 :
2500 33053899 : if (!is_gimple_call (last_stmt) || gimple_call_lhs (last_stmt) == NULL)
2501 : return NULL;
2502 :
2503 1563683 : switch (gimple_call_combined_fn (last_stmt))
2504 : {
2505 550 : CASE_CFN_POW:
2506 550 : CASE_CFN_POWI:
2507 550 : break;
2508 :
2509 : default:
2510 : return NULL;
2511 : }
2512 :
2513 550 : base = gimple_call_arg (last_stmt, 0);
2514 550 : exp = gimple_call_arg (last_stmt, 1);
2515 550 : if (TREE_CODE (exp) != REAL_CST
2516 495 : && TREE_CODE (exp) != INTEGER_CST)
2517 : {
2518 495 : if (flag_unsafe_math_optimizations
2519 57 : && TREE_CODE (base) == REAL_CST
2520 498 : && gimple_call_builtin_p (last_stmt, BUILT_IN_NORMAL))
2521 : {
2522 3 : combined_fn log_cfn;
2523 3 : built_in_function exp_bfn;
2524 3 : switch (DECL_FUNCTION_CODE (gimple_call_fndecl (last_stmt)))
2525 : {
2526 : case BUILT_IN_POW:
2527 : log_cfn = CFN_BUILT_IN_LOG;
2528 : exp_bfn = BUILT_IN_EXP;
2529 : break;
2530 0 : case BUILT_IN_POWF:
2531 0 : log_cfn = CFN_BUILT_IN_LOGF;
2532 0 : exp_bfn = BUILT_IN_EXPF;
2533 0 : break;
2534 0 : case BUILT_IN_POWL:
2535 0 : log_cfn = CFN_BUILT_IN_LOGL;
2536 0 : exp_bfn = BUILT_IN_EXPL;
2537 0 : break;
2538 : default:
2539 : return NULL;
2540 : }
2541 3 : tree logc = fold_const_call (log_cfn, TREE_TYPE (base), base);
2542 3 : tree exp_decl = builtin_decl_implicit (exp_bfn);
2543 : /* Optimize pow (C, x) as exp (log (C) * x). Normally match.pd
2544 : does that, but if C is a power of 2, we want to use
2545 : exp2 (log2 (C) * x) in the non-vectorized version, but for
2546 : vectorization we don't have vectorized exp2. */
2547 3 : if (logc
2548 3 : && TREE_CODE (logc) == REAL_CST
2549 3 : && exp_decl
2550 6 : && lookup_attribute ("omp declare simd",
2551 3 : DECL_ATTRIBUTES (exp_decl)))
2552 : {
2553 3 : cgraph_node *node = cgraph_node::get_create (exp_decl);
2554 3 : if (node->simd_clones == NULL)
2555 : {
2556 2 : if (targetm.simd_clone.compute_vecsize_and_simdlen == NULL
2557 2 : || node->definition)
2558 : return NULL;
2559 2 : expand_simd_clones (node);
2560 2 : if (node->simd_clones == NULL)
2561 : return NULL;
2562 : }
2563 3 : *type_out = get_vectype_for_scalar_type (vinfo, TREE_TYPE (base));
2564 3 : if (!*type_out)
2565 : return NULL;
2566 3 : tree def = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
2567 3 : gimple *g = gimple_build_assign (def, MULT_EXPR, exp, logc);
2568 3 : append_pattern_def_seq (vinfo, stmt_vinfo, g);
2569 3 : tree res = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
2570 3 : g = gimple_build_call (exp_decl, 1, def);
2571 3 : gimple_call_set_lhs (g, res);
2572 3 : return g;
2573 : }
2574 : }
2575 :
2576 : return NULL;
2577 : }
2578 :
2579 : /* We now have a pow or powi builtin function call with a constant
2580 : exponent. */
2581 :
2582 : /* Catch squaring. */
2583 55 : if ((tree_fits_shwi_p (exp)
2584 0 : && tree_to_shwi (exp) == 2)
2585 55 : || (TREE_CODE (exp) == REAL_CST
2586 55 : && real_equal (&TREE_REAL_CST (exp), &dconst2)))
2587 : {
2588 11 : if (!vect_supportable_direct_optab_p (vinfo, TREE_TYPE (base), MULT_EXPR,
2589 11 : TREE_TYPE (base), type_out))
2590 : return NULL;
2591 :
2592 9 : var = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
2593 9 : stmt = gimple_build_assign (var, MULT_EXPR, base, base);
2594 9 : return stmt;
2595 : }
2596 :
2597 : /* Catch square root. Transforming pow(x,0.5) to sqrt(x) is unsafe
2598 : for signed zeroes and negative infinity.
2599 : pow(-0,0.5) = +0, while sqrt(-0) = -0.
2600 : pow(-Inf,0.5) = +Inf, while sqrt(-Inf) is NaN. */
2601 44 : if (TREE_CODE (exp) == REAL_CST
2602 44 : && real_equal (&TREE_REAL_CST (exp), &dconsthalf)
2603 10 : && !tree_expr_maybe_real_minus_zero_p (base)
2604 44 : && (!tree_expr_maybe_infinite_p (base)
2605 0 : || tree_expr_nonnegative_p (base)))
2606 : {
2607 0 : *type_out = get_vectype_for_scalar_type (vinfo, TREE_TYPE (base));
2608 0 : if (*type_out
2609 0 : && direct_internal_fn_supported_p (IFN_SQRT, *type_out,
2610 : OPTIMIZE_FOR_SPEED))
2611 : {
2612 0 : gcall *stmt = gimple_build_call_internal (IFN_SQRT, 1, base);
2613 0 : var = vect_recog_temp_ssa_var (TREE_TYPE (base), stmt);
2614 0 : gimple_call_set_lhs (stmt, var);
2615 0 : gimple_call_set_nothrow (stmt, true);
2616 0 : return stmt;
2617 : }
2618 : }
2619 :
2620 : return NULL;
2621 : }
2622 :
2623 :
2624 : /* Function vect_recog_widen_sum_pattern
2625 :
2626 : Try to find the following pattern:
2627 :
2628 : type x_t;
2629 : TYPE x_T, sum = init;
2630 : loop:
2631 : sum_0 = phi <init, sum_1>
2632 : S1 x_t = *p;
2633 : S2 x_T = (TYPE) x_t;
2634 : S3 sum_1 = x_T + sum_0;
2635 :
2636 : where type 'TYPE' is at least double the size of type 'type', i.e - we're
2637 : summing elements of type 'type' into an accumulator of type 'TYPE'. This is
2638 : a special case of a reduction computation.
2639 :
2640 : Input:
2641 :
2642 : * STMT_VINFO: The stmt from which the pattern search begins. In the example,
2643 : when this function is called with S3, the pattern {S2,S3} will be detected.
2644 :
2645 : Output:
2646 :
2647 : * TYPE_OUT: The type of the output of this pattern.
2648 :
2649 : * Return value: A new stmt that will be used to replace the sequence of
2650 : stmts that constitute the pattern. In this case it will be:
2651 : WIDEN_SUM <x_t, sum_0>
2652 :
2653 : Note: The widening-sum idiom is a widening reduction pattern that is
2654 : vectorized without preserving all the intermediate results. It
2655 : produces only N/2 (widened) results (by summing up pairs of
2656 : intermediate results) rather than all N results. Therefore, we
2657 : cannot allow this pattern when we want to get all the results and in
2658 : the correct order (as is the case when this computation is in an
2659 : inner-loop nested in an outer-loop that us being vectorized). */
2660 :
2661 : static gimple *
2662 33053899 : vect_recog_widen_sum_pattern (vec_info *vinfo,
2663 : stmt_vec_info stmt_vinfo, tree *type_out)
2664 : {
2665 33053899 : gimple *last_stmt = stmt_vinfo->stmt;
2666 33053899 : tree oprnd0, oprnd1;
2667 33053899 : tree type;
2668 33053899 : gimple *pattern_stmt;
2669 33053899 : tree var;
2670 :
2671 : /* Look for the following pattern
2672 : DX = (TYPE) X;
2673 : sum_1 = DX + sum_0;
2674 : In which DX is at least double the size of X, and sum_1 has been
2675 : recognized as a reduction variable.
2676 : */
2677 :
2678 : /* Starting from LAST_STMT, follow the defs of its uses in search
2679 : of the above pattern. */
2680 :
2681 33053899 : if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, PLUS_EXPR,
2682 : &oprnd0, &oprnd1)
2683 53194 : || TREE_CODE (oprnd0) != SSA_NAME
2684 33106822 : || !vinfo->lookup_def (oprnd0))
2685 : return NULL;
2686 :
2687 52846 : type = TREE_TYPE (gimple_get_lhs (last_stmt));
2688 :
2689 : /* So far so good. Since last_stmt was detected as a (summation) reduction,
2690 : we know that oprnd1 is the reduction variable (defined by a loop-header
2691 : phi), and oprnd0 is an ssa-name defined by a stmt in the loop body.
2692 : Left to check that oprnd0 is defined by a cast from type 'type' to type
2693 : 'TYPE'. */
2694 :
2695 52846 : vect_unpromoted_value unprom0;
2696 52846 : if (!vect_look_through_possible_promotion (vinfo, oprnd0, &unprom0)
2697 52846 : || TYPE_PRECISION (unprom0.type) * 2 > TYPE_PRECISION (type))
2698 : return NULL;
2699 :
2700 2372 : vect_pattern_detected ("vect_recog_widen_sum_pattern", last_stmt);
2701 :
2702 : /* If the exact input type is not supported, widen it to an intermediate
2703 : type before adding it to the accumulator. Start with the narrowest
2704 : type in order to retain the largest vectorization factor. */
2705 2372 : tree input_type = unprom0.type;
2706 2372 : tree input_vectype = NULL_TREE;
2707 :
2708 4438 : while (!vect_supportable_conv_optab_p (vinfo, type, WIDEN_SUM_EXPR,
2709 : input_type, type_out, &input_vectype))
2710 : {
2711 4438 : unsigned int precision
2712 4438 : = vect_element_precision (TYPE_PRECISION (input_type) + 1);
2713 :
2714 4438 : if (precision > TYPE_PRECISION (type) / 2)
2715 : return NULL;
2716 :
2717 2066 : input_type = build_nonstandard_integer_type
2718 2066 : (precision, TYPE_UNSIGNED (unprom0.type));
2719 : }
2720 :
2721 0 : tree input = vect_convert_input (vinfo, stmt_vinfo, input_type, &unprom0,
2722 : input_vectype);
2723 :
2724 0 : var = vect_recog_temp_ssa_var (type, NULL);
2725 0 : pattern_stmt = gimple_build_assign (var, WIDEN_SUM_EXPR, input, oprnd1);
2726 :
2727 0 : return pattern_stmt;
2728 : }
2729 :
2730 : /* Function vect_recog_bitfield_ref_pattern
2731 :
2732 : Try to find the following pattern:
2733 :
2734 : bf_value = BIT_FIELD_REF (container, bitsize, bitpos);
2735 : result = (type_out) bf_value;
2736 :
2737 : or
2738 :
2739 : if (BIT_FIELD_REF (container, bitsize, bitpos) `cmp` <constant>)
2740 :
2741 : where type_out is a non-bitfield type, that is to say, it's precision matches
2742 : 2^(TYPE_SIZE(type_out) - (TYPE_UNSIGNED (type_out) ? 1 : 2)).
2743 :
2744 : Input:
2745 :
2746 : * STMT_VINFO: The stmt from which the pattern search begins.
2747 : here it starts with:
2748 : result = (type_out) bf_value;
2749 :
2750 : or
2751 :
2752 : if (BIT_FIELD_REF (container, bitsize, bitpos) `cmp` <constant>)
2753 :
2754 : Output:
2755 :
2756 : * TYPE_OUT: The vector type of the output of this pattern.
2757 :
2758 : * Return value: A new stmt that will be used to replace the sequence of
2759 : stmts that constitute the pattern. If the precision of type_out is bigger
2760 : than the precision type of _1 we perform the widening before the shifting,
2761 : since the new precision will be large enough to shift the value and moving
2762 : widening operations up the statement chain enables the generation of
2763 : widening loads. If we are widening and the operation after the pattern is
2764 : an addition then we mask first and shift later, to enable the generation of
2765 : shifting adds. In the case of narrowing we will always mask first, shift
2766 : last and then perform a narrowing operation. This will enable the
2767 : generation of narrowing shifts.
2768 :
2769 : Widening with mask first, shift later:
2770 : container = (type_out) container;
2771 : masked = container & (((1 << bitsize) - 1) << bitpos);
2772 : result = masked >> bitpos;
2773 :
2774 : Widening with shift first, mask last:
2775 : container = (type_out) container;
2776 : shifted = container >> bitpos;
2777 : result = shifted & ((1 << bitsize) - 1);
2778 :
2779 : Narrowing:
2780 : masked = container & (((1 << bitsize) - 1) << bitpos);
2781 : result = masked >> bitpos;
2782 : result = (type_out) result;
2783 :
2784 : If the bitfield is signed and it's wider than type_out, we need to
2785 : keep the result sign-extended:
2786 : container = (type) container;
2787 : masked = container << (prec - bitsize - bitpos);
2788 : result = (type_out) (masked >> (prec - bitsize));
2789 :
2790 : Here type is the signed variant of the wider of type_out and the type
2791 : of container.
2792 :
2793 : The shifting is always optional depending on whether bitpos != 0.
2794 :
2795 : When the original bitfield was inside a gcond then an new gcond is also
2796 : generated with the newly `result` as the operand to the comparison.
2797 :
2798 : */
2799 :
2800 : static gimple *
2801 32996364 : vect_recog_bitfield_ref_pattern (vec_info *vinfo, stmt_vec_info stmt_info,
2802 : tree *type_out)
2803 : {
2804 32996364 : gimple *bf_stmt = NULL;
2805 32996364 : tree lhs = NULL_TREE;
2806 32996364 : tree ret_type = NULL_TREE;
2807 32996364 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
2808 32996364 : if (gcond *cond_stmt = dyn_cast <gcond *> (stmt))
2809 : {
2810 5460563 : tree op = gimple_cond_lhs (cond_stmt);
2811 5460563 : if (TREE_CODE (op) != SSA_NAME)
2812 : return NULL;
2813 5460254 : bf_stmt = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (op));
2814 5460254 : if (TREE_CODE (gimple_cond_rhs (cond_stmt)) != INTEGER_CST)
2815 : return NULL;
2816 : }
2817 27535801 : else if (is_gimple_assign (stmt)
2818 23093102 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
2819 30668353 : && TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME)
2820 : {
2821 3089984 : gimple *second_stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmt));
2822 3089984 : bf_stmt = dyn_cast <gassign *> (second_stmt);
2823 3089984 : lhs = gimple_assign_lhs (stmt);
2824 3089984 : ret_type = TREE_TYPE (lhs);
2825 : }
2826 :
2827 6620822 : if (!bf_stmt
2828 6620822 : || gimple_assign_rhs_code (bf_stmt) != BIT_FIELD_REF)
2829 : return NULL;
2830 :
2831 17399 : tree bf_ref = gimple_assign_rhs1 (bf_stmt);
2832 17399 : tree container = TREE_OPERAND (bf_ref, 0);
2833 17399 : ret_type = ret_type ? ret_type : TREE_TYPE (container);
2834 :
2835 17399 : if (!bit_field_offset (bf_ref).is_constant ()
2836 17399 : || !bit_field_size (bf_ref).is_constant ()
2837 17399 : || !tree_fits_uhwi_p (TYPE_SIZE (TREE_TYPE (container))))
2838 : return NULL;
2839 :
2840 34420 : if (!INTEGRAL_TYPE_P (TREE_TYPE (bf_ref))
2841 17397 : || !INTEGRAL_TYPE_P (TREE_TYPE (container))
2842 19572 : || TYPE_MODE (TREE_TYPE (container)) == E_BLKmode)
2843 : return NULL;
2844 :
2845 2173 : gimple *use_stmt, *pattern_stmt;
2846 2173 : use_operand_p use_p;
2847 2173 : bool shift_first = true;
2848 2173 : tree container_type = TREE_TYPE (container);
2849 2173 : tree vectype = get_vectype_for_scalar_type (vinfo, container_type);
2850 :
2851 : /* Calculate shift_n before the adjustments for widening loads, otherwise
2852 : the container may change and we have to consider offset change for
2853 : widening loads on big endianness. The shift_n calculated here can be
2854 : independent of widening. */
2855 2173 : unsigned HOST_WIDE_INT shift_n = bit_field_offset (bf_ref).to_constant ();
2856 2173 : unsigned HOST_WIDE_INT mask_width = bit_field_size (bf_ref).to_constant ();
2857 2173 : unsigned HOST_WIDE_INT prec = tree_to_uhwi (TYPE_SIZE (container_type));
2858 2173 : if (BYTES_BIG_ENDIAN)
2859 : shift_n = prec - shift_n - mask_width;
2860 :
2861 2173 : bool ref_sext = (!TYPE_UNSIGNED (TREE_TYPE (bf_ref)) &&
2862 1400 : TYPE_PRECISION (ret_type) > mask_width);
2863 2173 : bool load_widen = (TYPE_PRECISION (TREE_TYPE (container)) <
2864 2173 : TYPE_PRECISION (ret_type));
2865 :
2866 : /* We move the conversion earlier if the loaded type is smaller than the
2867 : return type to enable the use of widening loads. And if we need a
2868 : sign extension, we need to convert the loaded value early to a signed
2869 : type as well. */
2870 2173 : if (ref_sext || load_widen)
2871 : {
2872 947 : tree type = load_widen ? ret_type : container_type;
2873 947 : if (ref_sext)
2874 908 : type = gimple_signed_type (type);
2875 947 : pattern_stmt = gimple_build_assign (vect_recog_temp_ssa_var (type),
2876 : NOP_EXPR, container);
2877 947 : container = gimple_get_lhs (pattern_stmt);
2878 947 : container_type = TREE_TYPE (container);
2879 947 : prec = tree_to_uhwi (TYPE_SIZE (container_type));
2880 947 : vectype = get_vectype_for_scalar_type (vinfo, container_type);
2881 947 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2882 : }
2883 1226 : else if (!useless_type_conversion_p (TREE_TYPE (container), ret_type))
2884 : /* If we are doing the conversion last then also delay the shift as we may
2885 : be able to combine the shift and conversion in certain cases. */
2886 : shift_first = false;
2887 :
2888 : /* If the only use of the result of this BIT_FIELD_REF + CONVERT is a
2889 : PLUS_EXPR then do the shift last as some targets can combine the shift and
2890 : add into a single instruction. */
2891 1416 : if (lhs && !is_pattern_stmt_p (stmt_info)
2892 3589 : && single_imm_use (lhs, &use_p, &use_stmt))
2893 : {
2894 1049 : if (gimple_code (use_stmt) == GIMPLE_ASSIGN
2895 1049 : && gimple_assign_rhs_code (use_stmt) == PLUS_EXPR)
2896 : shift_first = false;
2897 : }
2898 :
2899 : /* If we don't have to shift we only generate the mask, so just fix the
2900 : code-path to shift_first. */
2901 2173 : if (shift_n == 0)
2902 763 : shift_first = true;
2903 :
2904 2173 : tree result;
2905 2173 : if (shift_first && !ref_sext)
2906 : {
2907 513 : tree shifted = container;
2908 513 : if (shift_n)
2909 : {
2910 65 : pattern_stmt
2911 65 : = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
2912 : RSHIFT_EXPR, container,
2913 65 : build_int_cst (sizetype, shift_n));
2914 65 : shifted = gimple_assign_lhs (pattern_stmt);
2915 65 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2916 : }
2917 :
2918 513 : tree mask = wide_int_to_tree (container_type,
2919 513 : wi::mask (mask_width, false, prec));
2920 :
2921 513 : pattern_stmt
2922 513 : = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
2923 : BIT_AND_EXPR, shifted, mask);
2924 513 : result = gimple_assign_lhs (pattern_stmt);
2925 : }
2926 : else
2927 : {
2928 1660 : tree temp = vect_recog_temp_ssa_var (container_type);
2929 1660 : if (!ref_sext)
2930 : {
2931 752 : tree mask = wide_int_to_tree (container_type,
2932 752 : wi::shifted_mask (shift_n,
2933 : mask_width,
2934 : false, prec));
2935 752 : pattern_stmt = gimple_build_assign (temp, BIT_AND_EXPR,
2936 : container, mask);
2937 : }
2938 : else
2939 : {
2940 908 : HOST_WIDE_INT shl = prec - shift_n - mask_width;
2941 908 : shift_n += shl;
2942 908 : pattern_stmt = gimple_build_assign (temp, LSHIFT_EXPR,
2943 : container,
2944 : build_int_cst (sizetype,
2945 908 : shl));
2946 : }
2947 :
2948 1660 : tree masked = gimple_assign_lhs (pattern_stmt);
2949 1660 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2950 1660 : pattern_stmt
2951 1660 : = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
2952 : RSHIFT_EXPR, masked,
2953 1660 : build_int_cst (sizetype, shift_n));
2954 1660 : result = gimple_assign_lhs (pattern_stmt);
2955 : }
2956 :
2957 2173 : if (!useless_type_conversion_p (TREE_TYPE (result), ret_type))
2958 : {
2959 1444 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2960 1444 : pattern_stmt
2961 1444 : = gimple_build_assign (vect_recog_temp_ssa_var (ret_type),
2962 : NOP_EXPR, result);
2963 : }
2964 :
2965 2173 : if (!lhs)
2966 : {
2967 757 : if (!vectype)
2968 : return NULL;
2969 :
2970 619 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2971 619 : vectype = truth_type_for (vectype);
2972 :
2973 : /* FIXME: This part extracts the boolean value out of the bitfield in the
2974 : same way as vect_recog_gcond_pattern does. However because
2975 : patterns cannot match the same root twice, when we handle and
2976 : lower the bitfield in the gcond, vect_recog_gcond_pattern can't
2977 : apply anymore. We should really fix it so that we don't need to
2978 : duplicate transformations like these. */
2979 619 : tree new_lhs = vect_recog_temp_ssa_var (boolean_type_node, NULL);
2980 619 : gcond *cond_stmt = dyn_cast <gcond *> (stmt_info->stmt);
2981 619 : tree cond_cst = gimple_cond_rhs (cond_stmt);
2982 619 : gimple *new_stmt
2983 619 : = gimple_build_assign (new_lhs, gimple_cond_code (cond_stmt),
2984 : gimple_get_lhs (pattern_stmt),
2985 : fold_convert (container_type, cond_cst));
2986 619 : append_pattern_def_seq (vinfo, stmt_info, new_stmt, vectype, container_type);
2987 619 : pattern_stmt
2988 619 : = gimple_build_cond (NE_EXPR, new_lhs,
2989 619 : build_zero_cst (TREE_TYPE (new_lhs)),
2990 : NULL_TREE, NULL_TREE);
2991 : }
2992 :
2993 2035 : *type_out = STMT_VINFO_VECTYPE (stmt_info);
2994 2035 : vect_pattern_detected ("bitfield_ref pattern", stmt_info->stmt);
2995 :
2996 2035 : return pattern_stmt;
2997 : }
2998 :
2999 : /* Function vect_recog_bit_insert_pattern
3000 :
3001 : Try to find the following pattern:
3002 :
3003 : written = BIT_INSERT_EXPR (container, value, bitpos);
3004 :
3005 : Input:
3006 :
3007 : * STMT_VINFO: The stmt we want to replace.
3008 :
3009 : Output:
3010 :
3011 : * TYPE_OUT: The vector type of the output of this pattern.
3012 :
3013 : * Return value: A new stmt that will be used to replace the sequence of
3014 : stmts that constitute the pattern. In this case it will be:
3015 : value = (container_type) value; // Make sure
3016 : shifted = value << bitpos; // Shift value into place
3017 : masked = shifted & (mask << bitpos); // Mask off the non-relevant bits in
3018 : // the 'to-write value'.
3019 : cleared = container & ~(mask << bitpos); // Clearing the bits we want to
3020 : // write to from the value we want
3021 : // to write to.
3022 : written = cleared | masked; // Write bits.
3023 :
3024 :
3025 : where mask = ((1 << TYPE_PRECISION (value)) - 1), a mask to keep the number of
3026 : bits corresponding to the real size of the bitfield value we are writing to.
3027 : The shifting is always optional depending on whether bitpos != 0.
3028 :
3029 : */
3030 :
3031 : static gimple *
3032 32999414 : vect_recog_bit_insert_pattern (vec_info *vinfo, stmt_vec_info stmt_info,
3033 : tree *type_out)
3034 : {
3035 32999414 : gassign *bf_stmt = dyn_cast <gassign *> (stmt_info->stmt);
3036 30820529 : if (!bf_stmt || gimple_assign_rhs_code (bf_stmt) != BIT_INSERT_EXPR)
3037 : return NULL;
3038 :
3039 690 : tree container = gimple_assign_rhs1 (bf_stmt);
3040 690 : tree value = gimple_assign_rhs2 (bf_stmt);
3041 690 : tree shift = gimple_assign_rhs3 (bf_stmt);
3042 :
3043 690 : tree bf_type = TREE_TYPE (value);
3044 690 : tree container_type = TREE_TYPE (container);
3045 :
3046 690 : if (!INTEGRAL_TYPE_P (container_type)
3047 690 : || !tree_fits_uhwi_p (TYPE_SIZE (container_type)))
3048 : return NULL;
3049 :
3050 508 : gimple *pattern_stmt;
3051 :
3052 508 : vect_unpromoted_value unprom;
3053 508 : unprom.set_op (value, vect_internal_def);
3054 508 : value = vect_convert_input (vinfo, stmt_info, container_type, &unprom,
3055 : get_vectype_for_scalar_type (vinfo,
3056 : container_type));
3057 :
3058 508 : unsigned HOST_WIDE_INT mask_width = TYPE_PRECISION (bf_type);
3059 508 : unsigned HOST_WIDE_INT prec = tree_to_uhwi (TYPE_SIZE (container_type));
3060 508 : unsigned HOST_WIDE_INT shift_n = tree_to_uhwi (shift);
3061 508 : if (BYTES_BIG_ENDIAN)
3062 : {
3063 : shift_n = prec - shift_n - mask_width;
3064 : shift = build_int_cst (TREE_TYPE (shift), shift_n);
3065 : }
3066 :
3067 508 : if (!useless_type_conversion_p (TREE_TYPE (value), container_type))
3068 : {
3069 0 : pattern_stmt =
3070 0 : gimple_build_assign (vect_recog_temp_ssa_var (container_type),
3071 : NOP_EXPR, value);
3072 0 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt);
3073 0 : value = gimple_get_lhs (pattern_stmt);
3074 : }
3075 :
3076 : /* Shift VALUE into place. */
3077 508 : tree shifted = value;
3078 508 : if (shift_n)
3079 : {
3080 249 : gimple_seq stmts = NULL;
3081 249 : shifted
3082 249 : = gimple_build (&stmts, LSHIFT_EXPR, container_type, value, shift);
3083 249 : if (!gimple_seq_empty_p (stmts))
3084 112 : append_pattern_def_seq (vinfo, stmt_info,
3085 : gimple_seq_first_stmt (stmts));
3086 : }
3087 :
3088 508 : tree mask_t
3089 508 : = wide_int_to_tree (container_type,
3090 508 : wi::shifted_mask (shift_n, mask_width, false, prec));
3091 :
3092 : /* Clear bits we don't want to write back from SHIFTED. */
3093 508 : gimple_seq stmts = NULL;
3094 508 : tree masked = gimple_build (&stmts, BIT_AND_EXPR, container_type, shifted,
3095 : mask_t);
3096 508 : if (!gimple_seq_empty_p (stmts))
3097 : {
3098 118 : pattern_stmt = gimple_seq_first_stmt (stmts);
3099 118 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt);
3100 : }
3101 :
3102 : /* Mask off the bits in the container that we are to write to. */
3103 508 : mask_t = wide_int_to_tree (container_type,
3104 508 : wi::shifted_mask (shift_n, mask_width, true, prec));
3105 508 : tree cleared = vect_recog_temp_ssa_var (container_type);
3106 508 : pattern_stmt = gimple_build_assign (cleared, BIT_AND_EXPR, container, mask_t);
3107 508 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt);
3108 :
3109 : /* Write MASKED into CLEARED. */
3110 508 : pattern_stmt
3111 508 : = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
3112 : BIT_IOR_EXPR, cleared, masked);
3113 :
3114 508 : *type_out = STMT_VINFO_VECTYPE (stmt_info);
3115 508 : vect_pattern_detected ("bit_insert pattern", stmt_info->stmt);
3116 :
3117 508 : return pattern_stmt;
3118 : }
3119 :
3120 :
3121 : /* Recognize cases in which an operation is performed in one type WTYPE
3122 : but could be done more efficiently in a narrower type NTYPE. For example,
3123 : if we have:
3124 :
3125 : ATYPE a; // narrower than NTYPE
3126 : BTYPE b; // narrower than NTYPE
3127 : WTYPE aw = (WTYPE) a;
3128 : WTYPE bw = (WTYPE) b;
3129 : WTYPE res = aw + bw; // only uses of aw and bw
3130 :
3131 : then it would be more efficient to do:
3132 :
3133 : NTYPE an = (NTYPE) a;
3134 : NTYPE bn = (NTYPE) b;
3135 : NTYPE resn = an + bn;
3136 : WTYPE res = (WTYPE) resn;
3137 :
3138 : Other situations include things like:
3139 :
3140 : ATYPE a; // NTYPE or narrower
3141 : WTYPE aw = (WTYPE) a;
3142 : WTYPE res = aw + b;
3143 :
3144 : when only "(NTYPE) res" is significant. In that case it's more efficient
3145 : to truncate "b" and do the operation on NTYPE instead:
3146 :
3147 : NTYPE an = (NTYPE) a;
3148 : NTYPE bn = (NTYPE) b; // truncation
3149 : NTYPE resn = an + bn;
3150 : WTYPE res = (WTYPE) resn;
3151 :
3152 : All users of "res" should then use "resn" instead, making the final
3153 : statement dead (not marked as relevant). The final statement is still
3154 : needed to maintain the type correctness of the IR.
3155 :
3156 : vect_determine_precisions has already determined the minimum
3157 : precision of the operation and the minimum precision required
3158 : by users of the result. */
3159 :
3160 : static gimple *
3161 32999870 : vect_recog_over_widening_pattern (vec_info *vinfo,
3162 : stmt_vec_info last_stmt_info, tree *type_out)
3163 : {
3164 32999870 : gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
3165 23097227 : if (!last_stmt)
3166 : return NULL;
3167 :
3168 : /* See whether we have found that this operation can be done on a
3169 : narrower type without changing its semantics. */
3170 23097227 : unsigned int new_precision = last_stmt_info->operation_precision;
3171 23097227 : if (!new_precision)
3172 : return NULL;
3173 :
3174 1697544 : tree lhs = gimple_assign_lhs (last_stmt);
3175 1697544 : tree type = TREE_TYPE (lhs);
3176 1697544 : tree_code code = gimple_assign_rhs_code (last_stmt);
3177 :
3178 : /* Punt for reductions where we don't handle the type conversions. */
3179 1697544 : if (vect_is_reduction (last_stmt_info))
3180 : return NULL;
3181 :
3182 : /* Keep the first operand of a COND_EXPR as-is: only the other two
3183 : operands are interesting. */
3184 1689197 : unsigned int first_op = (code == COND_EXPR ? 2 : 1);
3185 :
3186 : /* Check the operands. */
3187 1689197 : unsigned int nops = gimple_num_ops (last_stmt) - first_op;
3188 1689197 : auto_vec <vect_unpromoted_value, 3> unprom (nops);
3189 1689197 : unprom.quick_grow_cleared (nops);
3190 1689197 : unsigned int min_precision = 0;
3191 1689197 : bool single_use_p = false;
3192 6735864 : for (unsigned int i = 0; i < nops; ++i)
3193 : {
3194 3359355 : tree op = gimple_op (last_stmt, first_op + i);
3195 3359355 : if (TREE_CODE (op) == INTEGER_CST)
3196 1537055 : unprom[i].set_op (op, vect_constant_def);
3197 1822300 : else if (TREE_CODE (op) == SSA_NAME)
3198 : {
3199 1822300 : bool op_single_use_p = true;
3200 1822300 : if (!vect_look_through_possible_promotion (vinfo, op, &unprom[i],
3201 : &op_single_use_p))
3202 1885 : return NULL;
3203 : /* If:
3204 :
3205 : (1) N bits of the result are needed;
3206 : (2) all inputs are widened from M<N bits; and
3207 : (3) one operand OP is a single-use SSA name
3208 :
3209 : we can shift the M->N widening from OP to the output
3210 : without changing the number or type of extensions involved.
3211 : This then reduces the number of copies of STMT_INFO.
3212 :
3213 : If instead of (3) more than one operand is a single-use SSA name,
3214 : shifting the extension to the output is even more of a win.
3215 :
3216 : If instead:
3217 :
3218 : (1) N bits of the result are needed;
3219 : (2) one operand OP2 is widened from M2<N bits;
3220 : (3) another operand OP1 is widened from M1<M2 bits; and
3221 : (4) both OP1 and OP2 are single-use
3222 :
3223 : the choice is between:
3224 :
3225 : (a) truncating OP2 to M1, doing the operation on M1,
3226 : and then widening the result to N
3227 :
3228 : (b) widening OP1 to M2, doing the operation on M2, and then
3229 : widening the result to N
3230 :
3231 : Both shift the M2->N widening of the inputs to the output.
3232 : (a) additionally shifts the M1->M2 widening to the output;
3233 : it requires fewer copies of STMT_INFO but requires an extra
3234 : M2->M1 truncation.
3235 :
3236 : Which is better will depend on the complexity and cost of
3237 : STMT_INFO, which is hard to predict at this stage. However,
3238 : a clear tie-breaker in favor of (b) is the fact that the
3239 : truncation in (a) increases the length of the operation chain.
3240 :
3241 : If instead of (4) only one of OP1 or OP2 is single-use,
3242 : (b) is still a win over doing the operation in N bits:
3243 : it still shifts the M2->N widening on the single-use operand
3244 : to the output and reduces the number of STMT_INFO copies.
3245 :
3246 : If neither operand is single-use then operating on fewer than
3247 : N bits might lead to more extensions overall. Whether it does
3248 : or not depends on global information about the vectorization
3249 : region, and whether that's a good trade-off would again
3250 : depend on the complexity and cost of the statements involved,
3251 : as well as things like register pressure that are not normally
3252 : modelled at this stage. We therefore ignore these cases
3253 : and just optimize the clear single-use wins above.
3254 :
3255 : Thus we take the maximum precision of the unpromoted operands
3256 : and record whether any operand is single-use. */
3257 1820415 : if (unprom[i].dt == vect_internal_def)
3258 : {
3259 1048758 : min_precision = MAX (min_precision,
3260 : TYPE_PRECISION (unprom[i].type));
3261 1048758 : single_use_p |= op_single_use_p;
3262 : }
3263 : }
3264 : else
3265 : return NULL;
3266 : }
3267 :
3268 : /* Although the operation could be done in operation_precision, we have
3269 : to balance that against introducing extra truncations or extensions.
3270 : Calculate the minimum precision that can be handled efficiently.
3271 :
3272 : The loop above determined that the operation could be handled
3273 : efficiently in MIN_PRECISION if SINGLE_USE_P; this would shift an
3274 : extension from the inputs to the output without introducing more
3275 : instructions, and would reduce the number of instructions required
3276 : for STMT_INFO itself.
3277 :
3278 : vect_determine_precisions has also determined that the result only
3279 : needs min_output_precision bits. Truncating by a factor of N times
3280 : requires a tree of N - 1 instructions, so if TYPE is N times wider
3281 : than min_output_precision, doing the operation in TYPE and truncating
3282 : the result requires N + (N - 1) = 2N - 1 instructions per output vector.
3283 : In contrast:
3284 :
3285 : - truncating the input to a unary operation and doing the operation
3286 : in the new type requires at most N - 1 + 1 = N instructions per
3287 : output vector
3288 :
3289 : - doing the same for a binary operation requires at most
3290 : (N - 1) * 2 + 1 = 2N - 1 instructions per output vector
3291 :
3292 : Both unary and binary operations require fewer instructions than
3293 : this if the operands were extended from a suitable truncated form.
3294 : Thus there is usually nothing to lose by doing operations in
3295 : min_output_precision bits, but there can be something to gain. */
3296 1687312 : if (!single_use_p)
3297 1328141 : min_precision = last_stmt_info->min_output_precision;
3298 : else
3299 359171 : min_precision = MIN (min_precision, last_stmt_info->min_output_precision);
3300 :
3301 : /* Apply the minimum efficient precision we just calculated. */
3302 1687312 : if (new_precision < min_precision)
3303 : new_precision = min_precision;
3304 1687312 : new_precision = vect_element_precision (new_precision);
3305 1687312 : if (new_precision >= TYPE_PRECISION (type))
3306 : return NULL;
3307 :
3308 161026 : vect_pattern_detected ("vect_recog_over_widening_pattern", last_stmt);
3309 :
3310 161026 : *type_out = get_vectype_for_scalar_type (vinfo, type);
3311 161026 : if (!*type_out)
3312 : return NULL;
3313 :
3314 : /* We've found a viable pattern. Get the new type of the operation. */
3315 141212 : bool unsigned_p = (last_stmt_info->operation_sign == UNSIGNED);
3316 141212 : tree new_type = build_nonstandard_integer_type (new_precision, unsigned_p);
3317 :
3318 : /* If we're truncating an operation, we need to make sure that we
3319 : don't introduce new undefined overflow. The codes tested here are
3320 : a subset of those accepted by vect_truncatable_operation_p. */
3321 141212 : tree op_type = new_type;
3322 141212 : if (TYPE_OVERFLOW_UNDEFINED (new_type)
3323 182273 : && (code == PLUS_EXPR || code == MINUS_EXPR || code == MULT_EXPR))
3324 28096 : op_type = build_nonstandard_integer_type (new_precision, true);
3325 :
3326 141212 : tree new_vectype = get_vectype_for_scalar_type (vinfo, new_type);
3327 141212 : tree op_vectype = get_vectype_for_scalar_type (vinfo, op_type);
3328 141212 : if (!new_vectype || !op_vectype)
3329 : return NULL;
3330 :
3331 : /* Verify we can handle the new operation. For shifts and rotates
3332 : apply heuristic of whether we are likely facing vector-vector or
3333 : vector-scalar operation. Since we are eventually expecting that
3334 : a later pattern might eventually want to rewrite an unsupported
3335 : into a supported case error on that side in case the original
3336 : operation was not supported either or this is a binary operation
3337 : and the 2nd operand is constant. */
3338 141212 : if (code == RSHIFT_EXPR || code == LSHIFT_EXPR || code == RROTATE_EXPR)
3339 : {
3340 29693 : if (!target_has_vecop_for_code (code, op_vectype, optab_vector)
3341 28616 : && ((unprom[1].dt != vect_external_def
3342 28198 : && unprom[1].dt != vect_constant_def)
3343 19630 : || !target_has_vecop_for_code (code, op_vectype, optab_scalar))
3344 38702 : && !(!target_has_vecop_for_code (code, *type_out, optab_vector)
3345 7911 : && ((unprom[1].dt != vect_external_def
3346 7911 : || unprom[1].dt != vect_constant_def)
3347 : || !target_has_vecop_for_code (code, *type_out,
3348 : optab_scalar))))
3349 : return NULL;
3350 : }
3351 111519 : else if (!target_has_vecop_for_code (code, op_vectype, optab_vector)
3352 111519 : && (target_has_vecop_for_code (code, *type_out, optab_vector)
3353 29 : && !(nops == 2 && unprom[1].dt == vect_constant_def)))
3354 : return NULL;
3355 :
3356 140104 : if (dump_enabled_p ())
3357 4691 : dump_printf_loc (MSG_NOTE, vect_location, "demoting %T to %T\n",
3358 : type, new_type);
3359 :
3360 : /* Calculate the rhs operands for an operation on OP_TYPE. */
3361 140104 : tree ops[3] = {};
3362 140377 : for (unsigned int i = 1; i < first_op; ++i)
3363 273 : ops[i - 1] = gimple_op (last_stmt, i);
3364 140104 : vect_convert_inputs (vinfo, last_stmt_info, nops, &ops[first_op - 1],
3365 140104 : op_type, &unprom[0], op_vectype);
3366 :
3367 : /* Use the operation to produce a result of type OP_TYPE. */
3368 140104 : tree new_var = vect_recog_temp_ssa_var (op_type, NULL);
3369 140104 : gimple *pattern_stmt = gimple_build_assign (new_var, code,
3370 : ops[0], ops[1], ops[2]);
3371 140104 : gimple_set_location (pattern_stmt, gimple_location (last_stmt));
3372 :
3373 140104 : if (dump_enabled_p ())
3374 4691 : dump_printf_loc (MSG_NOTE, vect_location,
3375 : "created pattern stmt: %G", pattern_stmt);
3376 :
3377 : /* Convert back to the original signedness, if OP_TYPE is different
3378 : from NEW_TYPE. */
3379 140104 : if (op_type != new_type)
3380 28089 : pattern_stmt = vect_convert_output (vinfo, last_stmt_info, new_type,
3381 : pattern_stmt, op_vectype);
3382 :
3383 : /* Promote the result to the original type. */
3384 140104 : pattern_stmt = vect_convert_output (vinfo, last_stmt_info, type,
3385 : pattern_stmt, new_vectype);
3386 :
3387 140104 : return pattern_stmt;
3388 1689197 : }
3389 :
3390 : /* Recognize the following patterns:
3391 :
3392 : ATYPE a; // narrower than TYPE
3393 : BTYPE b; // narrower than TYPE
3394 :
3395 : 1) Multiply high with scaling
3396 : TYPE res = ((TYPE) a * (TYPE) b) >> c;
3397 : Here, c is bitsize (TYPE) / 2 - 1.
3398 :
3399 : 2) ... or also with rounding
3400 : TYPE res = (((TYPE) a * (TYPE) b) >> d + 1) >> 1;
3401 : Here, d is bitsize (TYPE) / 2 - 2.
3402 :
3403 : 3) Normal multiply high
3404 : TYPE res = ((TYPE) a * (TYPE) b) >> e;
3405 : Here, e is bitsize (TYPE) / 2.
3406 :
3407 : where only the bottom half of res is used. */
3408 :
3409 : static gimple *
3410 33136954 : vect_recog_mulhs_pattern (vec_info *vinfo,
3411 : stmt_vec_info last_stmt_info, tree *type_out)
3412 : {
3413 : /* Check for a right shift. */
3414 33136954 : gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
3415 23234105 : if (!last_stmt
3416 23234105 : || gimple_assign_rhs_code (last_stmt) != RSHIFT_EXPR)
3417 : return NULL;
3418 :
3419 : /* Check that the shift result is wider than the users of the
3420 : result need (i.e. that narrowing would be a natural choice). */
3421 373839 : tree lhs_type = TREE_TYPE (gimple_assign_lhs (last_stmt));
3422 373839 : unsigned int target_precision
3423 373839 : = vect_element_precision (last_stmt_info->min_output_precision);
3424 373839 : if (!INTEGRAL_TYPE_P (lhs_type)
3425 373839 : || target_precision >= TYPE_PRECISION (lhs_type))
3426 : return NULL;
3427 :
3428 : /* Look through any change in sign on the outer shift input. */
3429 64238 : vect_unpromoted_value unprom_rshift_input;
3430 64238 : tree rshift_input = vect_look_through_possible_promotion
3431 64238 : (vinfo, gimple_assign_rhs1 (last_stmt), &unprom_rshift_input);
3432 64238 : if (!rshift_input
3433 64238 : || TYPE_PRECISION (TREE_TYPE (rshift_input))
3434 63612 : != TYPE_PRECISION (lhs_type))
3435 : return NULL;
3436 :
3437 : /* Get the definition of the shift input. */
3438 59302 : stmt_vec_info rshift_input_stmt_info
3439 59302 : = vect_get_internal_def (vinfo, rshift_input);
3440 59302 : if (!rshift_input_stmt_info)
3441 : return NULL;
3442 54570 : gassign *rshift_input_stmt
3443 54570 : = dyn_cast <gassign *> (rshift_input_stmt_info->stmt);
3444 51128 : if (!rshift_input_stmt)
3445 : return NULL;
3446 :
3447 51128 : stmt_vec_info mulh_stmt_info;
3448 51128 : tree scale_term;
3449 51128 : bool rounding_p = false;
3450 :
3451 : /* Check for the presence of the rounding term. */
3452 59008 : if (gimple_assign_rhs_code (rshift_input_stmt) == PLUS_EXPR)
3453 : {
3454 : /* Check that the outer shift was by 1. */
3455 17990 : if (!integer_onep (gimple_assign_rhs2 (last_stmt)))
3456 8928 : return NULL;
3457 :
3458 : /* Check that the second operand of the PLUS_EXPR is 1. */
3459 514 : if (!integer_onep (gimple_assign_rhs2 (rshift_input_stmt)))
3460 : return NULL;
3461 :
3462 : /* Look through any change in sign on the addition input. */
3463 108 : vect_unpromoted_value unprom_plus_input;
3464 108 : tree plus_input = vect_look_through_possible_promotion
3465 108 : (vinfo, gimple_assign_rhs1 (rshift_input_stmt), &unprom_plus_input);
3466 108 : if (!plus_input
3467 108 : || TYPE_PRECISION (TREE_TYPE (plus_input))
3468 108 : != TYPE_PRECISION (TREE_TYPE (rshift_input)))
3469 : return NULL;
3470 :
3471 : /* Get the definition of the multiply-high-scale part. */
3472 95 : stmt_vec_info plus_input_stmt_info
3473 95 : = vect_get_internal_def (vinfo, plus_input);
3474 95 : if (!plus_input_stmt_info)
3475 : return NULL;
3476 95 : gassign *plus_input_stmt
3477 9023 : = dyn_cast <gassign *> (plus_input_stmt_info->stmt);
3478 95 : if (!plus_input_stmt
3479 95 : || gimple_assign_rhs_code (plus_input_stmt) != RSHIFT_EXPR)
3480 : return NULL;
3481 :
3482 : /* Look through any change in sign on the scaling input. */
3483 67 : vect_unpromoted_value unprom_scale_input;
3484 67 : tree scale_input = vect_look_through_possible_promotion
3485 67 : (vinfo, gimple_assign_rhs1 (plus_input_stmt), &unprom_scale_input);
3486 67 : if (!scale_input
3487 67 : || TYPE_PRECISION (TREE_TYPE (scale_input))
3488 67 : != TYPE_PRECISION (TREE_TYPE (plus_input)))
3489 : return NULL;
3490 :
3491 : /* Get the definition of the multiply-high part. */
3492 67 : mulh_stmt_info = vect_get_internal_def (vinfo, scale_input);
3493 67 : if (!mulh_stmt_info)
3494 : return NULL;
3495 :
3496 : /* Get the scaling term. */
3497 67 : scale_term = gimple_assign_rhs2 (plus_input_stmt);
3498 67 : rounding_p = true;
3499 : }
3500 : else
3501 : {
3502 42133 : mulh_stmt_info = rshift_input_stmt_info;
3503 42133 : scale_term = gimple_assign_rhs2 (last_stmt);
3504 : }
3505 :
3506 : /* Check that the scaling factor is constant. */
3507 42200 : if (TREE_CODE (scale_term) != INTEGER_CST)
3508 : return NULL;
3509 :
3510 : /* Check whether the scaling input term can be seen as two widened
3511 : inputs multiplied together. */
3512 123612 : vect_unpromoted_value unprom_mult[2];
3513 41204 : tree new_type;
3514 41204 : unsigned int nops
3515 41204 : = vect_widened_op_tree (vinfo, mulh_stmt_info, MULT_EXPR, WIDEN_MULT_EXPR,
3516 : false, 2, unprom_mult, &new_type);
3517 41204 : if (nops != 2)
3518 : return NULL;
3519 :
3520 : /* Adjust output precision. */
3521 10797 : if (TYPE_PRECISION (new_type) < target_precision)
3522 0 : new_type = build_nonstandard_integer_type
3523 0 : (target_precision, TYPE_UNSIGNED (new_type));
3524 :
3525 10797 : unsigned mult_precision = TYPE_PRECISION (new_type);
3526 10797 : internal_fn ifn;
3527 : /* Check that the scaling factor is expected. Instead of
3528 : target_precision, we should use the one that we actually
3529 : use for internal function. */
3530 10797 : if (rounding_p)
3531 : {
3532 : /* Check pattern 2). */
3533 134 : if (wi::to_widest (scale_term) + mult_precision + 2
3534 201 : != TYPE_PRECISION (lhs_type))
3535 : return NULL;
3536 :
3537 : ifn = IFN_MULHRS;
3538 : }
3539 : else
3540 : {
3541 : /* Check for pattern 1). */
3542 21460 : if (wi::to_widest (scale_term) + mult_precision + 1
3543 32190 : == TYPE_PRECISION (lhs_type))
3544 : ifn = IFN_MULHS;
3545 : /* Check for pattern 3). */
3546 10696 : else if (wi::to_widest (scale_term) + mult_precision
3547 21392 : == TYPE_PRECISION (lhs_type))
3548 : ifn = IFN_MULH;
3549 : else
3550 : return NULL;
3551 : }
3552 :
3553 10708 : vect_pattern_detected ("vect_recog_mulhs_pattern", last_stmt);
3554 :
3555 : /* Check for target support. */
3556 10708 : tree new_vectype = get_vectype_for_scalar_type (vinfo, new_type);
3557 10708 : if (!new_vectype
3558 18070 : || !direct_internal_fn_supported_p
3559 7362 : (ifn, new_vectype, OPTIMIZE_FOR_SPEED))
3560 : return NULL;
3561 :
3562 : /* The result is cast back to LHS_TYPE, a cast that the over-widening
3563 : machinery then removes. LHS_TYPE need not have a vector type, as for the
3564 : 128-bit product of a 64-bit high-part multiply, so leave *TYPE_OUT null
3565 : rather than giving up. */
3566 4551 : *type_out = get_vectype_for_scalar_type (vinfo, lhs_type);
3567 :
3568 : /* Generate the IFN_MULHRS call. */
3569 4551 : tree new_var = vect_recog_temp_ssa_var (new_type, NULL);
3570 4551 : tree new_ops[2];
3571 4551 : vect_convert_inputs (vinfo, last_stmt_info, 2, new_ops, new_type,
3572 : unprom_mult, new_vectype);
3573 4551 : gcall *mulhrs_stmt
3574 4551 : = gimple_build_call_internal (ifn, 2, new_ops[0], new_ops[1]);
3575 4551 : gimple_call_set_lhs (mulhrs_stmt, new_var);
3576 4551 : gimple_set_location (mulhrs_stmt, gimple_location (last_stmt));
3577 :
3578 4551 : if (dump_enabled_p ())
3579 3 : dump_printf_loc (MSG_NOTE, vect_location,
3580 : "created pattern stmt: %G", (gimple *) mulhrs_stmt);
3581 :
3582 4551 : return vect_convert_output (vinfo, last_stmt_info, lhs_type,
3583 4551 : mulhrs_stmt, new_vectype);
3584 : }
3585 :
3586 : /* Recognize the patterns:
3587 :
3588 : ATYPE a; // narrower than TYPE
3589 : BTYPE b; // narrower than TYPE
3590 : (1) TYPE avg = ((TYPE) a + (TYPE) b) >> 1;
3591 : or (2) TYPE avg = ((TYPE) a + (TYPE) b + 1) >> 1;
3592 :
3593 : where only the bottom half of avg is used. Try to transform them into:
3594 :
3595 : (1) NTYPE avg' = .AVG_FLOOR ((NTYPE) a, (NTYPE) b);
3596 : or (2) NTYPE avg' = .AVG_CEIL ((NTYPE) a, (NTYPE) b);
3597 :
3598 : followed by:
3599 :
3600 : TYPE avg = (TYPE) avg';
3601 :
3602 : where NTYPE is no wider than half of TYPE. Since only the bottom half
3603 : of avg is used, all or part of the cast of avg' should become redundant.
3604 :
3605 : If there is no target support available, generate code to distribute rshift
3606 : over plus and add a carry. */
3607 :
3608 : static gimple *
3609 33132717 : vect_recog_average_pattern (vec_info *vinfo,
3610 : stmt_vec_info last_stmt_info, tree *type_out)
3611 : {
3612 : /* Check for a shift right by one bit. */
3613 33132717 : gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
3614 23230074 : if (!last_stmt
3615 23230074 : || gimple_assign_rhs_code (last_stmt) != RSHIFT_EXPR
3616 373352 : || !integer_onep (gimple_assign_rhs2 (last_stmt)))
3617 : return NULL;
3618 :
3619 : /* Check that the shift result is wider than the users of the
3620 : result need (i.e. that narrowing would be a natural choice). */
3621 58955 : tree lhs = gimple_assign_lhs (last_stmt);
3622 58955 : tree type = TREE_TYPE (lhs);
3623 58955 : unsigned int target_precision
3624 58955 : = vect_element_precision (last_stmt_info->min_output_precision);
3625 58955 : if (!INTEGRAL_TYPE_P (type) || target_precision >= TYPE_PRECISION (type))
3626 : return NULL;
3627 :
3628 : /* Look through any change in sign on the shift input. */
3629 2274 : tree rshift_rhs = gimple_assign_rhs1 (last_stmt);
3630 2274 : vect_unpromoted_value unprom_plus;
3631 2274 : rshift_rhs = vect_look_through_possible_promotion (vinfo, rshift_rhs,
3632 : &unprom_plus);
3633 2274 : if (!rshift_rhs
3634 2274 : || TYPE_PRECISION (TREE_TYPE (rshift_rhs)) != TYPE_PRECISION (type))
3635 : return NULL;
3636 :
3637 : /* Get the definition of the shift input. */
3638 2268 : stmt_vec_info plus_stmt_info = vect_get_internal_def (vinfo, rshift_rhs);
3639 2268 : if (!plus_stmt_info)
3640 : return NULL;
3641 :
3642 : /* Check whether the shift input can be seen as a tree of additions on
3643 : 2 or 3 widened inputs.
3644 :
3645 : Note that the pattern should be a win even if the result of one or
3646 : more additions is reused elsewhere: if the pattern matches, we'd be
3647 : replacing 2N RSHIFT_EXPRs and N VEC_PACK_*s with N IFN_AVG_*s. */
3648 2262 : internal_fn ifn = IFN_AVG_FLOOR;
3649 9048 : vect_unpromoted_value unprom[3];
3650 2262 : tree new_type;
3651 2262 : unsigned int nops = vect_widened_op_tree (vinfo, plus_stmt_info, PLUS_EXPR,
3652 2262 : IFN_VEC_WIDEN_PLUS, false, 3,
3653 : unprom, &new_type);
3654 2262 : if (nops == 0)
3655 : return NULL;
3656 972 : if (nops == 3)
3657 : {
3658 : /* Check that one operand is 1. */
3659 : unsigned int i;
3660 1155 : for (i = 0; i < 3; ++i)
3661 1101 : if (integer_onep (unprom[i].op))
3662 : break;
3663 367 : if (i == 3)
3664 : return NULL;
3665 : /* Throw away the 1 operand and keep the other two. */
3666 313 : if (i < 2)
3667 0 : unprom[i] = unprom[2];
3668 : ifn = IFN_AVG_CEIL;
3669 : }
3670 :
3671 918 : vect_pattern_detected ("vect_recog_average_pattern", last_stmt);
3672 :
3673 : /* We know that:
3674 :
3675 : (a) the operation can be viewed as:
3676 :
3677 : TYPE widened0 = (TYPE) UNPROM[0];
3678 : TYPE widened1 = (TYPE) UNPROM[1];
3679 : TYPE tmp1 = widened0 + widened1 {+ 1};
3680 : TYPE tmp2 = tmp1 >> 1; // LAST_STMT_INFO
3681 :
3682 : (b) the first two statements are equivalent to:
3683 :
3684 : TYPE widened0 = (TYPE) (NEW_TYPE) UNPROM[0];
3685 : TYPE widened1 = (TYPE) (NEW_TYPE) UNPROM[1];
3686 :
3687 : (c) vect_recog_over_widening_pattern has already tried to narrow TYPE
3688 : where sensible;
3689 :
3690 : (d) all the operations can be performed correctly at twice the width of
3691 : NEW_TYPE, due to the nature of the average operation; and
3692 :
3693 : (e) users of the result of the right shift need only TARGET_PRECISION
3694 : bits, where TARGET_PRECISION is no more than half of TYPE's
3695 : precision.
3696 :
3697 : Under these circumstances, the only situation in which NEW_TYPE
3698 : could be narrower than TARGET_PRECISION is if widened0, widened1
3699 : and an addition result are all used more than once. Thus we can
3700 : treat any widening of UNPROM[0] and UNPROM[1] to TARGET_PRECISION
3701 : as "free", whereas widening the result of the average instruction
3702 : from NEW_TYPE to TARGET_PRECISION would be a new operation. It's
3703 : therefore better not to go narrower than TARGET_PRECISION. */
3704 918 : if (TYPE_PRECISION (new_type) < target_precision)
3705 0 : new_type = build_nonstandard_integer_type (target_precision,
3706 0 : TYPE_UNSIGNED (new_type));
3707 :
3708 : /* Check for target support. */
3709 918 : tree new_vectype = get_vectype_for_scalar_type (vinfo, new_type);
3710 918 : if (!new_vectype)
3711 : return NULL;
3712 :
3713 918 : bool fallback_p = !direct_internal_fn_supported_p (ifn, new_vectype,
3714 918 : OPTIMIZE_FOR_SPEED);
3715 918 : if (fallback_p
3716 918 : && (!target_supports_op_p (new_vectype, RSHIFT_EXPR, optab_scalar)
3717 697 : || !target_supports_op_p (new_vectype, PLUS_EXPR, optab_default)
3718 697 : || !target_supports_op_p (new_vectype, BIT_AND_EXPR, optab_default)
3719 697 : || (ifn == IFN_AVG_CEIL
3720 105 : && !target_supports_op_p (new_vectype, BIT_IOR_EXPR,
3721 : optab_default))))
3722 : return NULL;
3723 :
3724 : /* The IR requires a valid vector type for the cast result, even though
3725 : it's likely to be discarded. */
3726 903 : *type_out = get_vectype_for_scalar_type (vinfo, type);
3727 903 : if (!*type_out)
3728 : return NULL;
3729 :
3730 899 : tree new_var = vect_recog_temp_ssa_var (new_type, NULL);
3731 899 : tree new_ops[2];
3732 899 : vect_convert_inputs (vinfo, last_stmt_info, 2, new_ops, new_type,
3733 : unprom, new_vectype);
3734 :
3735 899 : if (fallback_p)
3736 : {
3737 : /* As a fallback, generate code for following sequence:
3738 :
3739 : shifted_op0 = new_ops[0] >> 1;
3740 : shifted_op1 = new_ops[1] >> 1;
3741 : sum_of_shifted = shifted_op0 + shifted_op1;
3742 : unmasked_carry = new_ops[0] and/or new_ops[1];
3743 : carry = unmasked_carry & 1;
3744 : new_var = sum_of_shifted + carry;
3745 :
3746 : For signed types, arithmetic shifts round down and the carry is one
3747 : when both operands are odd (or either operand for IFN_AVG_CEIL). */
3748 :
3749 693 : tree one_cst = build_one_cst (new_type);
3750 693 : gassign *g;
3751 :
3752 693 : tree shifted_op0 = vect_recog_temp_ssa_var (new_type, NULL);
3753 693 : g = gimple_build_assign (shifted_op0, RSHIFT_EXPR, new_ops[0], one_cst);
3754 693 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3755 :
3756 693 : tree shifted_op1 = vect_recog_temp_ssa_var (new_type, NULL);
3757 693 : g = gimple_build_assign (shifted_op1, RSHIFT_EXPR, new_ops[1], one_cst);
3758 693 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3759 :
3760 693 : tree sum_of_shifted = vect_recog_temp_ssa_var (new_type, NULL);
3761 693 : g = gimple_build_assign (sum_of_shifted, PLUS_EXPR,
3762 : shifted_op0, shifted_op1);
3763 693 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3764 :
3765 693 : tree unmasked_carry = vect_recog_temp_ssa_var (new_type, NULL);
3766 693 : tree_code c = (ifn == IFN_AVG_CEIL) ? BIT_IOR_EXPR : BIT_AND_EXPR;
3767 693 : g = gimple_build_assign (unmasked_carry, c, new_ops[0], new_ops[1]);
3768 693 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3769 :
3770 693 : tree carry = vect_recog_temp_ssa_var (new_type, NULL);
3771 693 : g = gimple_build_assign (carry, BIT_AND_EXPR, unmasked_carry, one_cst);
3772 693 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3773 :
3774 693 : g = gimple_build_assign (new_var, PLUS_EXPR, sum_of_shifted, carry);
3775 693 : return vect_convert_output (vinfo, last_stmt_info, type, g, new_vectype);
3776 : }
3777 :
3778 : /* Generate the IFN_AVG* call. */
3779 206 : gcall *average_stmt = gimple_build_call_internal (ifn, 2, new_ops[0],
3780 : new_ops[1]);
3781 206 : gimple_call_set_lhs (average_stmt, new_var);
3782 206 : gimple_set_location (average_stmt, gimple_location (last_stmt));
3783 :
3784 206 : if (dump_enabled_p ())
3785 79 : dump_printf_loc (MSG_NOTE, vect_location,
3786 : "created pattern stmt: %G", (gimple *) average_stmt);
3787 :
3788 206 : return vect_convert_output (vinfo, last_stmt_info,
3789 206 : type, average_stmt, new_vectype);
3790 : }
3791 :
3792 : /* Recognize cases in which the input to a cast is wider than its
3793 : output, and the input is fed by a widening operation. Fold this
3794 : by removing the unnecessary intermediate widening. E.g.:
3795 :
3796 : unsigned char a;
3797 : unsigned int b = (unsigned int) a;
3798 : unsigned short c = (unsigned short) b;
3799 :
3800 : -->
3801 :
3802 : unsigned short c = (unsigned short) a;
3803 :
3804 : Although this is rare in input IR, it is an expected side-effect
3805 : of the over-widening pattern above.
3806 :
3807 : This is beneficial also for integer-to-float conversions, if the
3808 : widened integer has more bits than the float, and if the unwidened
3809 : input doesn't. */
3810 :
3811 : static gimple *
3812 33136954 : vect_recog_cast_forwprop_pattern (vec_info *vinfo,
3813 : stmt_vec_info last_stmt_info, tree *type_out)
3814 : {
3815 : /* Check for a cast, including an integer-to-float conversion. */
3816 33136954 : gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
3817 23229554 : if (!last_stmt)
3818 : return NULL;
3819 23229554 : tree_code code = gimple_assign_rhs_code (last_stmt);
3820 23229554 : if (!CONVERT_EXPR_CODE_P (code) && code != FLOAT_EXPR)
3821 : return NULL;
3822 :
3823 : /* Make sure that the rhs is a scalar with a natural bitsize. */
3824 3394286 : tree lhs = gimple_assign_lhs (last_stmt);
3825 3394286 : if (!lhs)
3826 : return NULL;
3827 3394286 : tree lhs_type = TREE_TYPE (lhs);
3828 3394286 : scalar_mode lhs_mode;
3829 3373277 : if (VECT_SCALAR_BOOLEAN_TYPE_P (lhs_type)
3830 6765773 : || !is_a <scalar_mode> (TYPE_MODE (lhs_type), &lhs_mode))
3831 : return NULL;
3832 :
3833 : /* Check for a narrowing operation (from a vector point of view). */
3834 3367685 : tree rhs = gimple_assign_rhs1 (last_stmt);
3835 3367685 : tree rhs_type = TREE_TYPE (rhs);
3836 3367685 : if (!INTEGRAL_TYPE_P (rhs_type)
3837 2946785 : || VECT_SCALAR_BOOLEAN_TYPE_P (rhs_type)
3838 9107751 : || TYPE_PRECISION (rhs_type) <= GET_MODE_BITSIZE (lhs_mode))
3839 : return NULL;
3840 :
3841 : /* Try to find an unpromoted input. */
3842 397129 : vect_unpromoted_value unprom;
3843 397129 : if (!vect_look_through_possible_promotion (vinfo, rhs, &unprom)
3844 397129 : || TYPE_PRECISION (unprom.type) >= TYPE_PRECISION (rhs_type))
3845 : return NULL;
3846 :
3847 : /* If the bits above RHS_TYPE matter, make sure that they're the
3848 : same when extending from UNPROM as they are when extending from RHS. */
3849 53904 : if (!INTEGRAL_TYPE_P (lhs_type)
3850 53904 : && TYPE_SIGN (rhs_type) != TYPE_SIGN (unprom.type))
3851 : return NULL;
3852 :
3853 : /* We can get the same result by casting UNPROM directly, to avoid
3854 : the unnecessary widening and narrowing. */
3855 53784 : vect_pattern_detected ("vect_recog_cast_forwprop_pattern", last_stmt);
3856 :
3857 53784 : *type_out = get_vectype_for_scalar_type (vinfo, lhs_type);
3858 53784 : if (!*type_out)
3859 : return NULL;
3860 :
3861 53784 : tree new_var = vect_recog_temp_ssa_var (lhs_type, NULL);
3862 53784 : gimple *pattern_stmt = gimple_build_assign (new_var, code, unprom.op);
3863 53784 : gimple_set_location (pattern_stmt, gimple_location (last_stmt));
3864 :
3865 53784 : return pattern_stmt;
3866 : }
3867 :
3868 : /* Try to detect a shift left of a widened input, converting LSHIFT_EXPR
3869 : to WIDEN_LSHIFT_EXPR. See vect_recog_widen_op_pattern for details. */
3870 :
3871 : static gimple *
3872 33054528 : vect_recog_widen_shift_pattern (vec_info *vinfo,
3873 : stmt_vec_info last_stmt_info, tree *type_out)
3874 : {
3875 33054528 : return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
3876 33054528 : LSHIFT_EXPR, WIDEN_LSHIFT_EXPR, true,
3877 33054528 : "vect_recog_widen_shift_pattern");
3878 : }
3879 :
3880 : /* Detect a rotate pattern wouldn't be otherwise vectorized:
3881 :
3882 : type a_t, b_t, c_t;
3883 :
3884 : S0 a_t = b_t r<< c_t;
3885 :
3886 : Input/Output:
3887 :
3888 : * STMT_VINFO: The stmt from which the pattern search begins,
3889 : i.e. the shift/rotate stmt. The original stmt (S0) is replaced
3890 : with a sequence:
3891 :
3892 : S1 d_t = -c_t;
3893 : S2 e_t = d_t & (B - 1);
3894 : S3 f_t = b_t << c_t;
3895 : S4 g_t = b_t >> e_t;
3896 : S0 a_t = f_t | g_t;
3897 :
3898 : where B is element bitsize of type.
3899 :
3900 : Output:
3901 :
3902 : * TYPE_OUT: The type of the output of this pattern.
3903 :
3904 : * Return value: A new stmt that will be used to replace the rotate
3905 : S0 stmt. */
3906 :
3907 : static gimple *
3908 33054528 : vect_recog_rotate_pattern (vec_info *vinfo,
3909 : stmt_vec_info stmt_vinfo, tree *type_out)
3910 : {
3911 33054528 : gimple *last_stmt = stmt_vinfo->stmt;
3912 33054528 : tree oprnd0, oprnd1, lhs, var, var1, var2, vectype, type, stype, def, def2;
3913 33054528 : gimple *pattern_stmt, *def_stmt;
3914 33054528 : enum tree_code rhs_code;
3915 33054528 : enum vect_def_type dt;
3916 33054528 : optab optab1, optab2;
3917 33054528 : edge ext_def = NULL;
3918 33054528 : bool bswap16_p = false;
3919 :
3920 33054528 : if (is_gimple_assign (last_stmt))
3921 : {
3922 23147070 : rhs_code = gimple_assign_rhs_code (last_stmt);
3923 23147070 : switch (rhs_code)
3924 : {
3925 8586 : case LROTATE_EXPR:
3926 8586 : case RROTATE_EXPR:
3927 8586 : break;
3928 : default:
3929 : return NULL;
3930 : }
3931 :
3932 8586 : lhs = gimple_assign_lhs (last_stmt);
3933 8586 : oprnd0 = gimple_assign_rhs1 (last_stmt);
3934 8586 : type = TREE_TYPE (oprnd0);
3935 8586 : oprnd1 = gimple_assign_rhs2 (last_stmt);
3936 : }
3937 9907458 : else if (gimple_call_builtin_p (last_stmt, BUILT_IN_BSWAP16))
3938 : {
3939 : /* __builtin_bswap16 (x) is another form of x r>> 8.
3940 : The vectorizer has bswap support, but only if the argument isn't
3941 : promoted. */
3942 206 : lhs = gimple_call_lhs (last_stmt);
3943 206 : oprnd0 = gimple_call_arg (last_stmt, 0);
3944 206 : type = TREE_TYPE (oprnd0);
3945 206 : if (!lhs
3946 206 : || TYPE_PRECISION (TREE_TYPE (lhs)) != 16
3947 206 : || TYPE_PRECISION (type) <= 16
3948 0 : || TREE_CODE (oprnd0) != SSA_NAME
3949 206 : || BITS_PER_UNIT != 8)
3950 206 : return NULL;
3951 :
3952 0 : stmt_vec_info def_stmt_info;
3953 0 : if (!vect_is_simple_use (oprnd0, vinfo, &dt, &def_stmt_info, &def_stmt))
3954 : return NULL;
3955 :
3956 0 : if (dt != vect_internal_def)
3957 : return NULL;
3958 :
3959 0 : if (gimple_assign_cast_p (def_stmt))
3960 : {
3961 0 : def = gimple_assign_rhs1 (def_stmt);
3962 0 : if (INTEGRAL_TYPE_P (TREE_TYPE (def))
3963 0 : && TYPE_PRECISION (TREE_TYPE (def)) == 16)
3964 : oprnd0 = def;
3965 : }
3966 :
3967 0 : type = TREE_TYPE (lhs);
3968 0 : vectype = get_vectype_for_scalar_type (vinfo, type);
3969 0 : if (vectype == NULL_TREE)
3970 : return NULL;
3971 :
3972 0 : if (tree char_vectype = get_same_sized_vectype (char_type_node, vectype))
3973 : {
3974 : /* The encoding uses one stepped pattern for each byte in the
3975 : 16-bit word. */
3976 0 : vec_perm_builder elts (TYPE_VECTOR_SUBPARTS (char_vectype), 2, 3);
3977 0 : for (unsigned i = 0; i < 3; ++i)
3978 0 : for (unsigned j = 0; j < 2; ++j)
3979 0 : elts.quick_push ((i + 1) * 2 - j - 1);
3980 :
3981 0 : vec_perm_indices indices (elts, 1,
3982 0 : TYPE_VECTOR_SUBPARTS (char_vectype));
3983 0 : machine_mode vmode = TYPE_MODE (char_vectype);
3984 0 : if (can_vec_perm_const_p (vmode, vmode, indices))
3985 : {
3986 : /* vectorizable_bswap can handle the __builtin_bswap16 if we
3987 : undo the argument promotion. */
3988 0 : if (!useless_type_conversion_p (type, TREE_TYPE (oprnd0)))
3989 : {
3990 0 : def = vect_recog_temp_ssa_var (type, NULL);
3991 0 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd0);
3992 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
3993 0 : oprnd0 = def;
3994 : }
3995 :
3996 : /* Pattern detected. */
3997 0 : vect_pattern_detected ("vect_recog_rotate_pattern", last_stmt);
3998 :
3999 0 : *type_out = vectype;
4000 :
4001 : /* Pattern supported. Create a stmt to be used to replace the
4002 : pattern, with the unpromoted argument. */
4003 0 : var = vect_recog_temp_ssa_var (type, NULL);
4004 0 : pattern_stmt = gimple_build_call (gimple_call_fndecl (last_stmt),
4005 : 1, oprnd0);
4006 0 : gimple_call_set_lhs (pattern_stmt, var);
4007 0 : gimple_call_set_fntype (as_a <gcall *> (pattern_stmt),
4008 : gimple_call_fntype (last_stmt));
4009 0 : return pattern_stmt;
4010 : }
4011 0 : }
4012 :
4013 0 : oprnd1 = build_int_cst (integer_type_node, 8);
4014 0 : rhs_code = LROTATE_EXPR;
4015 0 : bswap16_p = true;
4016 : }
4017 : else
4018 : return NULL;
4019 :
4020 8586 : if (TREE_CODE (oprnd0) != SSA_NAME
4021 8466 : || !INTEGRAL_TYPE_P (type)
4022 16731 : || TYPE_PRECISION (TREE_TYPE (lhs)) != TYPE_PRECISION (type))
4023 : return NULL;
4024 :
4025 8145 : stmt_vec_info def_stmt_info;
4026 8145 : if (!vect_is_simple_use (oprnd1, vinfo, &dt, &def_stmt_info, &def_stmt))
4027 : return NULL;
4028 :
4029 8145 : if (dt != vect_internal_def
4030 7078 : && dt != vect_constant_def
4031 25 : && dt != vect_external_def)
4032 : return NULL;
4033 :
4034 8139 : vectype = get_vectype_for_scalar_type (vinfo, type);
4035 8139 : if (vectype == NULL_TREE)
4036 : return NULL;
4037 :
4038 : /* If vector/vector or vector/scalar rotate is supported by the target,
4039 : don't do anything here. */
4040 7772 : optab1 = optab_for_tree_code (rhs_code, vectype, optab_vector);
4041 7772 : if (optab1
4042 7772 : && can_implement_p (optab1, TYPE_MODE (vectype)))
4043 : {
4044 564 : use_rotate:
4045 564 : if (bswap16_p)
4046 : {
4047 0 : if (!useless_type_conversion_p (type, TREE_TYPE (oprnd0)))
4048 : {
4049 0 : def = vect_recog_temp_ssa_var (type, NULL);
4050 0 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd0);
4051 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
4052 0 : oprnd0 = def;
4053 : }
4054 :
4055 : /* Pattern detected. */
4056 0 : vect_pattern_detected ("vect_recog_rotate_pattern", last_stmt);
4057 :
4058 0 : *type_out = vectype;
4059 :
4060 : /* Pattern supported. Create a stmt to be used to replace the
4061 : pattern. */
4062 0 : var = vect_recog_temp_ssa_var (type, NULL);
4063 0 : pattern_stmt = gimple_build_assign (var, LROTATE_EXPR, oprnd0,
4064 : oprnd1);
4065 0 : return pattern_stmt;
4066 : }
4067 : return NULL;
4068 : }
4069 :
4070 7748 : if (is_a <bb_vec_info> (vinfo) || dt != vect_internal_def)
4071 : {
4072 7664 : optab2 = optab_for_tree_code (rhs_code, vectype, optab_scalar);
4073 7664 : if (optab2
4074 7664 : && can_implement_p (optab2, TYPE_MODE (vectype)))
4075 540 : goto use_rotate;
4076 : }
4077 :
4078 : /* We may not use a reduction operand twice. */
4079 7208 : if (vect_is_reduction (stmt_vinfo))
4080 : return NULL;
4081 :
4082 7187 : tree utype = unsigned_type_for (type);
4083 7187 : tree uvectype = get_vectype_for_scalar_type (vinfo, utype);
4084 7187 : if (!uvectype)
4085 : return NULL;
4086 :
4087 : /* If vector/vector or vector/scalar shifts aren't supported by the target,
4088 : don't do anything here either. */
4089 7187 : optab1 = optab_for_tree_code (LSHIFT_EXPR, uvectype, optab_vector);
4090 7187 : optab2 = optab_for_tree_code (RSHIFT_EXPR, uvectype, optab_vector);
4091 7187 : if (!optab1
4092 7187 : || !can_implement_p (optab1, TYPE_MODE (uvectype))
4093 746 : || !optab2
4094 7933 : || !can_implement_p (optab2, TYPE_MODE (uvectype)))
4095 : {
4096 6441 : if (! is_a <bb_vec_info> (vinfo) && dt == vect_internal_def)
4097 : return NULL;
4098 6378 : optab1 = optab_for_tree_code (LSHIFT_EXPR, uvectype, optab_scalar);
4099 6378 : optab2 = optab_for_tree_code (RSHIFT_EXPR, uvectype, optab_scalar);
4100 6378 : if (!optab1
4101 6378 : || !can_implement_p (optab1, TYPE_MODE (uvectype))
4102 4728 : || !optab2
4103 11106 : || !can_implement_p (optab2, TYPE_MODE (uvectype)))
4104 : return NULL;
4105 : }
4106 :
4107 5474 : *type_out = vectype;
4108 :
4109 5474 : if (!useless_type_conversion_p (utype, TREE_TYPE (oprnd0)))
4110 : {
4111 58 : def = vect_recog_temp_ssa_var (utype, NULL);
4112 58 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd0);
4113 58 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
4114 58 : oprnd0 = def;
4115 : }
4116 :
4117 5474 : if (dt == vect_external_def && TREE_CODE (oprnd1) == SSA_NAME)
4118 15 : ext_def = vect_get_external_def_edge (vinfo, oprnd1);
4119 :
4120 5474 : def = NULL_TREE;
4121 5474 : scalar_int_mode mode = SCALAR_INT_TYPE_MODE (utype);
4122 5474 : if (dt != vect_internal_def || TYPE_MODE (TREE_TYPE (oprnd1)) == mode)
4123 : def = oprnd1;
4124 124 : else if (def_stmt && gimple_assign_cast_p (def_stmt))
4125 : {
4126 96 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
4127 96 : if (TYPE_MODE (TREE_TYPE (rhs1)) == mode
4128 96 : && TYPE_PRECISION (TREE_TYPE (rhs1))
4129 0 : == TYPE_PRECISION (type))
4130 : def = rhs1;
4131 : }
4132 :
4133 5350 : if (def == NULL_TREE)
4134 : {
4135 124 : def = vect_recog_temp_ssa_var (utype, NULL);
4136 124 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd1);
4137 124 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
4138 : }
4139 5474 : stype = TREE_TYPE (def);
4140 :
4141 5474 : if (TREE_CODE (def) == INTEGER_CST)
4142 : {
4143 4896 : if (!tree_fits_uhwi_p (def)
4144 4896 : || tree_to_uhwi (def) >= GET_MODE_PRECISION (mode)
4145 9792 : || integer_zerop (def))
4146 : return NULL;
4147 4896 : def2 = build_int_cst (stype,
4148 4896 : GET_MODE_PRECISION (mode) - tree_to_uhwi (def));
4149 : }
4150 : else
4151 : {
4152 578 : tree vecstype = get_vectype_for_scalar_type (vinfo, stype);
4153 :
4154 578 : if (vecstype == NULL_TREE)
4155 : return NULL;
4156 578 : def2 = vect_recog_temp_ssa_var (stype, NULL);
4157 578 : def_stmt = gimple_build_assign (def2, NEGATE_EXPR, def);
4158 578 : if (ext_def)
4159 : {
4160 15 : basic_block new_bb
4161 15 : = gsi_insert_on_edge_immediate (ext_def, def_stmt);
4162 15 : gcc_assert (!new_bb);
4163 : }
4164 : else
4165 563 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecstype);
4166 :
4167 578 : def2 = vect_recog_temp_ssa_var (stype, NULL);
4168 578 : tree mask = build_int_cst (stype, GET_MODE_PRECISION (mode) - 1);
4169 578 : def_stmt = gimple_build_assign (def2, BIT_AND_EXPR,
4170 : gimple_assign_lhs (def_stmt), mask);
4171 578 : if (ext_def)
4172 : {
4173 15 : basic_block new_bb
4174 15 : = gsi_insert_on_edge_immediate (ext_def, def_stmt);
4175 15 : gcc_assert (!new_bb);
4176 : }
4177 : else
4178 563 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecstype);
4179 : }
4180 :
4181 5474 : var1 = vect_recog_temp_ssa_var (utype, NULL);
4182 10637 : def_stmt = gimple_build_assign (var1, rhs_code == LROTATE_EXPR
4183 : ? LSHIFT_EXPR : RSHIFT_EXPR,
4184 : oprnd0, def);
4185 5474 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
4186 :
4187 5474 : var2 = vect_recog_temp_ssa_var (utype, NULL);
4188 10637 : def_stmt = gimple_build_assign (var2, rhs_code == LROTATE_EXPR
4189 : ? RSHIFT_EXPR : LSHIFT_EXPR,
4190 : oprnd0, def2);
4191 5474 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
4192 :
4193 : /* Pattern detected. */
4194 5474 : vect_pattern_detected ("vect_recog_rotate_pattern", last_stmt);
4195 :
4196 : /* Pattern supported. Create a stmt to be used to replace the pattern. */
4197 5474 : var = vect_recog_temp_ssa_var (utype, NULL);
4198 5474 : pattern_stmt = gimple_build_assign (var, BIT_IOR_EXPR, var1, var2);
4199 :
4200 5474 : if (!useless_type_conversion_p (type, utype))
4201 : {
4202 58 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, uvectype);
4203 58 : tree result = vect_recog_temp_ssa_var (type, NULL);
4204 58 : pattern_stmt = gimple_build_assign (result, NOP_EXPR, var);
4205 : }
4206 : return pattern_stmt;
4207 : }
4208 :
4209 : /* Detect a vector by vector shift pattern that wouldn't be otherwise
4210 : vectorized:
4211 :
4212 : type a_t;
4213 : TYPE b_T, res_T;
4214 :
4215 : S1 a_t = ;
4216 : S2 b_T = ;
4217 : S3 res_T = b_T op a_t;
4218 :
4219 : where type 'TYPE' is a type with different size than 'type',
4220 : and op is <<, >> or rotate.
4221 :
4222 : Also detect cases:
4223 :
4224 : type a_t;
4225 : TYPE b_T, c_T, res_T;
4226 :
4227 : S0 c_T = ;
4228 : S1 a_t = (type) c_T;
4229 : S2 b_T = ;
4230 : S3 res_T = b_T op a_t;
4231 :
4232 : Input/Output:
4233 :
4234 : * STMT_VINFO: The stmt from which the pattern search begins,
4235 : i.e. the shift/rotate stmt. The original stmt (S3) is replaced
4236 : with a shift/rotate which has same type on both operands, in the
4237 : second case just b_T op c_T, in the first case with added cast
4238 : from a_t to c_T in STMT_VINFO_PATTERN_DEF_SEQ.
4239 :
4240 : Output:
4241 :
4242 : * TYPE_OUT: The type of the output of this pattern.
4243 :
4244 : * Return value: A new stmt that will be used to replace the shift/rotate
4245 : S3 stmt. */
4246 :
4247 : static gimple *
4248 33061368 : vect_recog_vector_vector_shift_pattern (vec_info *vinfo,
4249 : stmt_vec_info stmt_vinfo,
4250 : tree *type_out)
4251 : {
4252 33061368 : gimple *last_stmt = stmt_vinfo->stmt;
4253 33061368 : tree oprnd0, oprnd1, lhs, var;
4254 33061368 : gimple *pattern_stmt;
4255 33061368 : enum tree_code rhs_code;
4256 :
4257 33061368 : if (!is_gimple_assign (last_stmt))
4258 : return NULL;
4259 :
4260 23153910 : rhs_code = gimple_assign_rhs_code (last_stmt);
4261 23153910 : switch (rhs_code)
4262 : {
4263 521116 : case LSHIFT_EXPR:
4264 521116 : case RSHIFT_EXPR:
4265 521116 : case LROTATE_EXPR:
4266 521116 : case RROTATE_EXPR:
4267 521116 : break;
4268 : default:
4269 : return NULL;
4270 : }
4271 :
4272 521116 : lhs = gimple_assign_lhs (last_stmt);
4273 521116 : oprnd0 = gimple_assign_rhs1 (last_stmt);
4274 521116 : oprnd1 = gimple_assign_rhs2 (last_stmt);
4275 521116 : if (TREE_CODE (oprnd1) != SSA_NAME
4276 112640 : || TYPE_MODE (TREE_TYPE (oprnd0)) == TYPE_MODE (TREE_TYPE (oprnd1))
4277 49988 : || !INTEGRAL_TYPE_P (TREE_TYPE (oprnd0))
4278 49367 : || !type_has_mode_precision_p (TREE_TYPE (oprnd1))
4279 570483 : || TYPE_PRECISION (TREE_TYPE (lhs))
4280 49367 : != TYPE_PRECISION (TREE_TYPE (oprnd0)))
4281 : return NULL;
4282 :
4283 49367 : stmt_vec_info def_vinfo = vinfo->lookup_def (oprnd1);
4284 49367 : if (!def_vinfo || STMT_VINFO_DEF_TYPE (def_vinfo) == vect_external_def)
4285 : return NULL;
4286 :
4287 46523 : def_vinfo = vect_stmt_to_vectorize (def_vinfo);
4288 1134 : gcc_assert (def_vinfo);
4289 :
4290 46523 : *type_out = get_vectype_for_scalar_type (vinfo, TREE_TYPE (oprnd0));
4291 46523 : if (*type_out == NULL_TREE)
4292 : return NULL;
4293 :
4294 33556 : tree def = NULL_TREE;
4295 33556 : gassign *def_stmt = dyn_cast <gassign *> (def_vinfo->stmt);
4296 19643 : if (def_stmt && gimple_assign_cast_p (def_stmt))
4297 : {
4298 5515 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
4299 5515 : if (TYPE_MODE (TREE_TYPE (rhs1)) == TYPE_MODE (TREE_TYPE (oprnd0))
4300 5515 : && TYPE_PRECISION (TREE_TYPE (rhs1))
4301 1296 : == TYPE_PRECISION (TREE_TYPE (oprnd0)))
4302 : {
4303 1296 : if (TYPE_PRECISION (TREE_TYPE (oprnd1))
4304 1296 : >= TYPE_PRECISION (TREE_TYPE (rhs1)))
4305 : def = rhs1;
4306 : else
4307 : {
4308 1194 : tree mask
4309 1194 : = build_low_bits_mask (TREE_TYPE (rhs1),
4310 1194 : TYPE_PRECISION (TREE_TYPE (oprnd1)));
4311 1194 : def = vect_recog_temp_ssa_var (TREE_TYPE (rhs1), NULL);
4312 1194 : def_stmt = gimple_build_assign (def, BIT_AND_EXPR, rhs1, mask);
4313 1194 : tree vecstype = get_vectype_for_scalar_type (vinfo,
4314 1194 : TREE_TYPE (rhs1));
4315 1194 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecstype);
4316 : }
4317 : }
4318 : }
4319 :
4320 1296 : if (def == NULL_TREE)
4321 : {
4322 32260 : def = vect_recog_temp_ssa_var (TREE_TYPE (oprnd0), NULL);
4323 32260 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd1);
4324 32260 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
4325 : }
4326 :
4327 : /* Pattern detected. */
4328 33556 : vect_pattern_detected ("vect_recog_vector_vector_shift_pattern", last_stmt);
4329 :
4330 : /* Pattern supported. Create a stmt to be used to replace the pattern. */
4331 33556 : var = vect_recog_temp_ssa_var (TREE_TYPE (oprnd0), NULL);
4332 33556 : pattern_stmt = gimple_build_assign (var, rhs_code, oprnd0, def);
4333 :
4334 33556 : return pattern_stmt;
4335 : }
4336 :
4337 : /* Verify that the target has optabs of VECTYPE to perform all the steps
4338 : needed by the multiplication-by-immediate synthesis algorithm described by
4339 : ALG and VAR. If SYNTH_SHIFT_P is true ensure that vector addition is
4340 : present. Return true iff the target supports all the steps. */
4341 :
4342 : static bool
4343 317185 : target_supports_mult_synth_alg (struct algorithm *alg, mult_variant var,
4344 : tree vectype, bool synth_shift_p)
4345 : {
4346 317185 : if (alg->op[0] != alg_zero && alg->op[0] != alg_m)
4347 : return false;
4348 :
4349 317185 : bool supports_vminus = target_has_vecop_for_code (MINUS_EXPR, vectype);
4350 317185 : bool supports_vplus = target_has_vecop_for_code (PLUS_EXPR, vectype);
4351 :
4352 317185 : if (var == negate_variant
4353 317185 : && !target_has_vecop_for_code (NEGATE_EXPR, vectype))
4354 : return false;
4355 :
4356 : /* If we must synthesize shifts with additions make sure that vector
4357 : addition is available. */
4358 316510 : if ((var == add_variant || synth_shift_p) && !supports_vplus)
4359 : return false;
4360 :
4361 153497 : for (int i = 1; i < alg->ops; i++)
4362 : {
4363 115125 : switch (alg->op[i])
4364 : {
4365 : case alg_shift:
4366 : break;
4367 28352 : case alg_add_t_m2:
4368 28352 : case alg_add_t2_m:
4369 28352 : case alg_add_factor:
4370 28352 : if (!supports_vplus)
4371 : return false;
4372 : break;
4373 18071 : case alg_sub_t_m2:
4374 18071 : case alg_sub_t2_m:
4375 18071 : case alg_sub_factor:
4376 18071 : if (!supports_vminus)
4377 : return false;
4378 : break;
4379 : case alg_unknown:
4380 : case alg_m:
4381 : case alg_zero:
4382 : case alg_impossible:
4383 : return false;
4384 0 : default:
4385 0 : gcc_unreachable ();
4386 : }
4387 : }
4388 :
4389 : return true;
4390 : }
4391 :
4392 : /* Synthesize a left shift of OP by AMNT bits using a series of additions and
4393 : putting the final result in DEST. Append all statements but the last into
4394 : VINFO. Return the last statement. */
4395 :
4396 : static gimple *
4397 0 : synth_lshift_by_additions (vec_info *vinfo,
4398 : tree dest, tree op, HOST_WIDE_INT amnt,
4399 : stmt_vec_info stmt_info, tree vectype)
4400 : {
4401 0 : HOST_WIDE_INT i;
4402 0 : tree itype = TREE_TYPE (op);
4403 0 : tree prev_res = op;
4404 0 : gcc_assert (amnt >= 0);
4405 0 : for (i = 0; i < amnt; i++)
4406 : {
4407 0 : tree tmp_var = (i < amnt - 1) ? vect_recog_temp_ssa_var (itype, NULL)
4408 : : dest;
4409 0 : gimple *stmt
4410 0 : = gimple_build_assign (tmp_var, PLUS_EXPR, prev_res, prev_res);
4411 0 : prev_res = tmp_var;
4412 0 : if (i < amnt - 1)
4413 0 : append_pattern_def_seq (vinfo, stmt_info, stmt, vectype);
4414 : else
4415 0 : return stmt;
4416 : }
4417 0 : gcc_unreachable ();
4418 : return NULL;
4419 : }
4420 :
4421 : /* Helper for vect_synth_mult_by_constant. Apply a binary operation
4422 : CODE to operands OP1 and OP2, creating a new temporary SSA var in
4423 : the process if necessary. Append the resulting assignment statements
4424 : to the sequence in STMT_VINFO. Return the SSA variable that holds the
4425 : result of the binary operation. If SYNTH_SHIFT_P is true synthesize
4426 : left shifts using additions. */
4427 :
4428 : static tree
4429 46105 : apply_binop_and_append_stmt (vec_info *vinfo,
4430 : tree_code code, tree op1, tree op2,
4431 : stmt_vec_info stmt_vinfo, tree vectype,
4432 : bool synth_shift_p)
4433 : {
4434 46105 : if (integer_zerop (op2)
4435 46105 : && (code == LSHIFT_EXPR
4436 40146 : || code == PLUS_EXPR))
4437 : {
4438 40146 : gcc_assert (TREE_CODE (op1) == SSA_NAME);
4439 : return op1;
4440 : }
4441 :
4442 5959 : gimple *stmt;
4443 5959 : tree itype = TREE_TYPE (op1);
4444 5959 : tree tmp_var = vect_recog_temp_ssa_var (itype, NULL);
4445 :
4446 5959 : if (code == LSHIFT_EXPR
4447 5959 : && synth_shift_p)
4448 : {
4449 0 : stmt = synth_lshift_by_additions (vinfo, tmp_var, op1,
4450 0 : TREE_INT_CST_LOW (op2), stmt_vinfo,
4451 : vectype);
4452 0 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4453 0 : return tmp_var;
4454 : }
4455 :
4456 5959 : stmt = gimple_build_assign (tmp_var, code, op1, op2);
4457 5959 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4458 5959 : return tmp_var;
4459 : }
4460 :
4461 : /* Synthesize a multiplication of OP by an INTEGER_CST VAL using shifts
4462 : and simple arithmetic operations to be vectorized. Record the statements
4463 : produced in STMT_VINFO and return the last statement in the sequence or
4464 : NULL if it's not possible to synthesize such a multiplication.
4465 : This function mirrors the behavior of expand_mult_const in expmed.cc but
4466 : works on tree-ssa form. */
4467 :
4468 : static gimple *
4469 320734 : vect_synth_mult_by_constant (vec_info *vinfo, tree op, tree val,
4470 : stmt_vec_info stmt_vinfo)
4471 : {
4472 320734 : tree itype = TREE_TYPE (op);
4473 320734 : machine_mode mode = TYPE_MODE (itype);
4474 320734 : struct algorithm alg;
4475 320734 : mult_variant variant;
4476 320734 : if (!tree_fits_shwi_p (val))
4477 : return NULL;
4478 :
4479 : /* Multiplication synthesis by shifts, adds and subs can introduce
4480 : signed overflow where the original operation didn't. Perform the
4481 : operations on an unsigned type and cast back to avoid this.
4482 : In the future we may want to relax this for synthesis algorithms
4483 : that we can prove do not cause unexpected overflow. */
4484 317200 : bool cast_to_unsigned_p = !TYPE_OVERFLOW_WRAPS (itype);
4485 :
4486 61142 : tree multtype = cast_to_unsigned_p ? unsigned_type_for (itype) : itype;
4487 317200 : tree vectype = get_vectype_for_scalar_type (vinfo, multtype);
4488 317200 : if (!vectype)
4489 : return NULL;
4490 :
4491 : /* Targets that don't support vector shifts but support vector additions
4492 : can synthesize shifts that way. */
4493 317200 : bool synth_shift_p = !vect_supportable_shift (vinfo, LSHIFT_EXPR, multtype);
4494 :
4495 317200 : HOST_WIDE_INT hwval = tree_to_shwi (val);
4496 : /* Use MAX_COST here as we don't want to limit the sequence on rtx costs.
4497 : The vectorizer's benefit analysis will decide whether it's beneficial
4498 : to do this. */
4499 634204 : bool possible = choose_mult_variant (VECTOR_MODE_P (TYPE_MODE (vectype))
4500 317004 : ? TYPE_MODE (vectype) : mode,
4501 : hwval, &alg, &variant, MAX_COST);
4502 317200 : if (!possible)
4503 : return NULL;
4504 :
4505 317200 : if (vect_is_reduction (stmt_vinfo))
4506 : {
4507 26 : int op_uses = alg.op[0] != alg_zero;
4508 45 : for (int i = 1; i < alg.ops; i++)
4509 32 : switch (alg.op[i])
4510 : {
4511 4 : case alg_add_t_m2:
4512 4 : case alg_sub_t_m2:
4513 4 : if (synth_shift_p && alg.log[i])
4514 : return NULL;
4515 : else
4516 4 : op_uses++;
4517 4 : break;
4518 0 : case alg_add_t2_m:
4519 0 : case alg_sub_t2_m:
4520 0 : op_uses++;
4521 : /* Fallthru. */
4522 28 : case alg_shift:
4523 28 : if (synth_shift_p && alg.log[i])
4524 : return NULL;
4525 : break;
4526 : case alg_add_factor:
4527 : case alg_sub_factor:
4528 : return NULL;
4529 : default:
4530 : break;
4531 : }
4532 13 : if (variant == add_variant)
4533 0 : op_uses++;
4534 : /* When we'll synthesize more than a single use of the reduction
4535 : operand the reduction constraints are violated. Avoid this
4536 : situation. */
4537 13 : if (op_uses > 1)
4538 : return NULL;
4539 : }
4540 :
4541 317185 : if (!target_supports_mult_synth_alg (&alg, variant, vectype, synth_shift_p))
4542 : return NULL;
4543 :
4544 38372 : tree accumulator;
4545 :
4546 : /* Clear out the sequence of statements so we can populate it below. */
4547 38372 : gimple *stmt = NULL;
4548 :
4549 38372 : if (cast_to_unsigned_p)
4550 : {
4551 12676 : tree tmp_op = vect_recog_temp_ssa_var (multtype, NULL);
4552 12676 : stmt = gimple_build_assign (tmp_op, CONVERT_EXPR, op);
4553 12676 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4554 12676 : op = tmp_op;
4555 : }
4556 :
4557 38372 : if (alg.op[0] == alg_zero)
4558 198 : accumulator = build_int_cst (multtype, 0);
4559 : else
4560 : accumulator = op;
4561 :
4562 38372 : bool needs_fixup = (variant == negate_variant)
4563 38372 : || (variant == add_variant);
4564 :
4565 152894 : for (int i = 1; i < alg.ops; i++)
4566 : {
4567 114522 : tree shft_log = build_int_cst (multtype, alg.log[i]);
4568 114522 : tree accum_tmp = vect_recog_temp_ssa_var (multtype, NULL);
4569 114522 : tree tmp_var = NULL_TREE;
4570 :
4571 114522 : switch (alg.op[i])
4572 : {
4573 68417 : case alg_shift:
4574 68417 : if (synth_shift_p)
4575 0 : stmt
4576 0 : = synth_lshift_by_additions (vinfo, accum_tmp, accumulator,
4577 0 : alg.log[i], stmt_vinfo, vectype);
4578 : else
4579 68417 : stmt = gimple_build_assign (accum_tmp, LSHIFT_EXPR, accumulator,
4580 : shft_log);
4581 : break;
4582 23358 : case alg_add_t_m2:
4583 23358 : tmp_var
4584 23358 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, op, shft_log,
4585 : stmt_vinfo, vectype, synth_shift_p);
4586 23358 : stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, accumulator,
4587 : tmp_var);
4588 23358 : break;
4589 16991 : case alg_sub_t_m2:
4590 16991 : tmp_var = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, op,
4591 : shft_log, stmt_vinfo,
4592 : vectype, synth_shift_p);
4593 : /* In some algorithms the first step involves zeroing the
4594 : accumulator. If subtracting from such an accumulator
4595 : just emit the negation directly. */
4596 16991 : if (integer_zerop (accumulator))
4597 198 : stmt = gimple_build_assign (accum_tmp, NEGATE_EXPR, tmp_var);
4598 : else
4599 16793 : stmt = gimple_build_assign (accum_tmp, MINUS_EXPR, accumulator,
4600 : tmp_var);
4601 : break;
4602 0 : case alg_add_t2_m:
4603 0 : tmp_var
4604 0 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
4605 : shft_log, stmt_vinfo, vectype,
4606 : synth_shift_p);
4607 0 : stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, tmp_var, op);
4608 0 : break;
4609 0 : case alg_sub_t2_m:
4610 0 : tmp_var
4611 0 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
4612 : shft_log, stmt_vinfo, vectype,
4613 : synth_shift_p);
4614 0 : stmt = gimple_build_assign (accum_tmp, MINUS_EXPR, tmp_var, op);
4615 0 : break;
4616 4870 : case alg_add_factor:
4617 4870 : tmp_var
4618 4870 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
4619 : shft_log, stmt_vinfo, vectype,
4620 : synth_shift_p);
4621 4870 : stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, accumulator,
4622 : tmp_var);
4623 4870 : break;
4624 886 : case alg_sub_factor:
4625 886 : tmp_var
4626 886 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
4627 : shft_log, stmt_vinfo, vectype,
4628 : synth_shift_p);
4629 886 : stmt = gimple_build_assign (accum_tmp, MINUS_EXPR, tmp_var,
4630 : accumulator);
4631 886 : break;
4632 0 : default:
4633 0 : gcc_unreachable ();
4634 : }
4635 : /* We don't want to append the last stmt in the sequence to stmt_vinfo
4636 : but rather return it directly. */
4637 :
4638 114522 : if ((i < alg.ops - 1) || needs_fixup || cast_to_unsigned_p)
4639 89156 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4640 114522 : accumulator = accum_tmp;
4641 : }
4642 38372 : if (variant == negate_variant)
4643 : {
4644 493 : tree accum_tmp = vect_recog_temp_ssa_var (multtype, NULL);
4645 493 : stmt = gimple_build_assign (accum_tmp, NEGATE_EXPR, accumulator);
4646 493 : accumulator = accum_tmp;
4647 493 : if (cast_to_unsigned_p)
4648 169 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4649 : }
4650 37879 : else if (variant == add_variant)
4651 : {
4652 97 : tree accum_tmp = vect_recog_temp_ssa_var (multtype, NULL);
4653 97 : stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, accumulator, op);
4654 97 : accumulator = accum_tmp;
4655 97 : if (cast_to_unsigned_p)
4656 91 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4657 : }
4658 : /* Move back to a signed if needed. */
4659 38042 : if (cast_to_unsigned_p)
4660 : {
4661 12676 : tree accum_tmp = vect_recog_temp_ssa_var (itype, NULL);
4662 12676 : stmt = gimple_build_assign (accum_tmp, CONVERT_EXPR, accumulator);
4663 : }
4664 :
4665 : return stmt;
4666 : }
4667 :
4668 : /* Detect multiplication by constant and convert it into a sequence of
4669 : shifts and additions, subtractions, negations. We reuse the
4670 : choose_mult_variant algorithms from expmed.cc
4671 :
4672 : Input/Output:
4673 :
4674 : STMT_VINFO: The stmt from which the pattern search begins,
4675 : i.e. the mult stmt.
4676 :
4677 : Output:
4678 :
4679 : * TYPE_OUT: The type of the output of this pattern.
4680 :
4681 : * Return value: A new stmt that will be used to replace
4682 : the multiplication. */
4683 :
4684 : static gimple *
4685 33083027 : vect_recog_mult_pattern (vec_info *vinfo,
4686 : stmt_vec_info stmt_vinfo, tree *type_out)
4687 : {
4688 33083027 : gimple *last_stmt = stmt_vinfo->stmt;
4689 33083027 : tree oprnd0, oprnd1, vectype, itype;
4690 33083027 : gimple *pattern_stmt;
4691 :
4692 33083027 : if (!is_gimple_assign (last_stmt))
4693 : return NULL;
4694 :
4695 23175569 : if (gimple_assign_rhs_code (last_stmt) != MULT_EXPR)
4696 : return NULL;
4697 :
4698 1583282 : oprnd0 = gimple_assign_rhs1 (last_stmt);
4699 1583282 : oprnd1 = gimple_assign_rhs2 (last_stmt);
4700 1583282 : itype = TREE_TYPE (oprnd0);
4701 :
4702 1583282 : if (TREE_CODE (oprnd0) != SSA_NAME
4703 1583215 : || TREE_CODE (oprnd1) != INTEGER_CST
4704 954416 : || !INTEGRAL_TYPE_P (itype)
4705 2537698 : || !type_has_mode_precision_p (itype))
4706 : return NULL;
4707 :
4708 954361 : vectype = get_vectype_for_scalar_type (vinfo, itype);
4709 954361 : if (vectype == NULL_TREE)
4710 : return NULL;
4711 :
4712 : /* If the target can handle vectorized multiplication natively,
4713 : don't attempt to optimize this. */
4714 776453 : optab mul_optab = optab_for_tree_code (MULT_EXPR, vectype, optab_default);
4715 776453 : if (mul_optab != unknown_optab
4716 776453 : && can_implement_p (mul_optab, TYPE_MODE (vectype)))
4717 : return NULL;
4718 :
4719 320734 : pattern_stmt = vect_synth_mult_by_constant (vinfo,
4720 : oprnd0, oprnd1, stmt_vinfo);
4721 320734 : if (!pattern_stmt)
4722 : return NULL;
4723 :
4724 : /* Pattern detected. */
4725 38372 : vect_pattern_detected ("vect_recog_mult_pattern", last_stmt);
4726 :
4727 38372 : *type_out = vectype;
4728 :
4729 38372 : return pattern_stmt;
4730 : }
4731 :
4732 : extern bool gimple_unsigned_integer_sat_add (tree, tree*, tree (*)(tree));
4733 : extern bool gimple_unsigned_integer_sat_sub (tree, tree*, tree (*)(tree));
4734 : extern bool gimple_unsigned_integer_sat_trunc (tree, tree*, tree (*)(tree));
4735 :
4736 : extern bool gimple_unsigned_integer_narrow_clip (tree, tree*, tree (*)(tree));
4737 :
4738 : extern bool gimple_signed_integer_sat_add (tree, tree*, tree (*)(tree));
4739 : extern bool gimple_signed_integer_sat_sub (tree, tree*, tree (*)(tree));
4740 : extern bool gimple_signed_integer_sat_trunc (tree, tree*, tree (*)(tree));
4741 :
4742 : static gimple *
4743 1269 : vect_recog_build_binary_gimple_stmt (vec_info *vinfo, stmt_vec_info stmt_info,
4744 : internal_fn fn, tree *type_out,
4745 : tree lhs, tree op_0, tree op_1)
4746 : {
4747 1269 : tree itype = TREE_TYPE (op_0);
4748 1269 : tree otype = TREE_TYPE (lhs);
4749 1269 : tree v_itype = get_vectype_for_scalar_type (vinfo, itype);
4750 1269 : tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
4751 :
4752 1269 : if (v_itype != NULL_TREE && v_otype != NULL_TREE
4753 1269 : && direct_internal_fn_supported_p (fn, v_itype, OPTIMIZE_FOR_BOTH))
4754 : {
4755 1011 : gcall *call = gimple_build_call_internal (fn, 2, op_0, op_1);
4756 1011 : tree in_ssa = vect_recog_temp_ssa_var (itype, NULL);
4757 :
4758 1011 : gimple_call_set_lhs (call, in_ssa);
4759 1011 : gimple_call_set_nothrow (call, /* nothrow_p */ false);
4760 1011 : gimple_set_location (call, gimple_location (STMT_VINFO_STMT (stmt_info)));
4761 :
4762 1011 : *type_out = v_otype;
4763 :
4764 1011 : if (types_compatible_p (itype, otype))
4765 : return call;
4766 : else
4767 : {
4768 0 : append_pattern_def_seq (vinfo, stmt_info, call, v_itype);
4769 0 : tree out_ssa = vect_recog_temp_ssa_var (otype, NULL);
4770 :
4771 0 : return gimple_build_assign (out_ssa, NOP_EXPR, in_ssa);
4772 : }
4773 : }
4774 :
4775 : return NULL;
4776 : }
4777 :
4778 : /*
4779 : * Try to detect saturation add pattern (SAT_ADD), aka below gimple:
4780 : * _7 = _4 + _6;
4781 : * _8 = _4 > _7;
4782 : * _9 = (long unsigned int) _8;
4783 : * _10 = -_9;
4784 : * _12 = _7 | _10;
4785 : *
4786 : * And then simplified to
4787 : * _12 = .SAT_ADD (_4, _6);
4788 : */
4789 :
4790 : static gimple *
4791 33157917 : vect_recog_sat_add_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
4792 : tree *type_out)
4793 : {
4794 33157917 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
4795 :
4796 33157917 : if (!is_gimple_assign (last_stmt))
4797 : return NULL;
4798 :
4799 23250459 : tree ops[2];
4800 23250459 : tree lhs = gimple_assign_lhs (last_stmt);
4801 :
4802 23250459 : if (gimple_unsigned_integer_sat_add (lhs, ops, NULL)
4803 23250459 : || gimple_signed_integer_sat_add (lhs, ops, NULL))
4804 : {
4805 : /* .SAT_ADD saturates in the type of its first operand, which must be
4806 : the type of the result. */
4807 831 : gcc_checking_assert (types_compatible_p (TREE_TYPE (lhs),
4808 : TREE_TYPE (ops[0])));
4809 831 : if (TREE_CODE (ops[1]) == INTEGER_CST)
4810 13 : ops[1] = fold_convert (TREE_TYPE (ops[0]), ops[1]);
4811 :
4812 831 : gimple *stmt = vect_recog_build_binary_gimple_stmt (vinfo, stmt_vinfo,
4813 : IFN_SAT_ADD, type_out,
4814 : lhs, ops[0], ops[1]);
4815 831 : if (stmt)
4816 : {
4817 796 : vect_pattern_detected ("vect_recog_sat_add_pattern", last_stmt);
4818 796 : return stmt;
4819 : }
4820 : }
4821 :
4822 : return NULL;
4823 : }
4824 :
4825 : /*
4826 : * Try to transform the truncation for .SAT_SUB pattern, mostly occurs in
4827 : * the benchmark zip. Aka:
4828 : *
4829 : * unsigned int _1;
4830 : * unsigned int _2;
4831 : * unsigned short int _4;
4832 : * _9 = (unsigned short int).SAT_SUB (_1, _2);
4833 : *
4834 : * if _1 is known to be in the range of unsigned short int. For example
4835 : * there is a def _1 = (unsigned short int)_4. Then we can transform the
4836 : * truncation to:
4837 : *
4838 : * _3 = (unsigned short int) MIN (65535, _2); // aka _3 = .SAT_TRUNC (_2);
4839 : * _9 = .SAT_SUB (_4, _3);
4840 : *
4841 : * Then, we can better vectorized code and avoid the unnecessary narrowing
4842 : * stmt during vectorization with below stmt(s).
4843 : *
4844 : * _3 = .SAT_TRUNC(_2); // SI => HI
4845 : * _9 = .SAT_SUB (_4, _3);
4846 : */
4847 : static void
4848 438 : vect_recog_sat_sub_pattern_transform (vec_info *vinfo,
4849 : stmt_vec_info stmt_vinfo,
4850 : tree lhs, tree *ops)
4851 : {
4852 438 : tree otype = TREE_TYPE (lhs);
4853 438 : tree itype = TREE_TYPE (ops[0]);
4854 438 : unsigned itype_prec = TYPE_PRECISION (itype);
4855 438 : unsigned otype_prec = TYPE_PRECISION (otype);
4856 :
4857 438 : if (types_compatible_p (otype, itype) || otype_prec >= itype_prec)
4858 438 : return;
4859 :
4860 0 : tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
4861 0 : tree v_itype = get_vectype_for_scalar_type (vinfo, itype);
4862 0 : tree_pair v_pair = tree_pair (v_otype, v_itype);
4863 :
4864 0 : if (v_otype == NULL_TREE || v_itype == NULL_TREE
4865 0 : || !direct_internal_fn_supported_p (IFN_SAT_TRUNC, v_pair,
4866 : OPTIMIZE_FOR_BOTH))
4867 : return;
4868 :
4869 : /* 1. Find the _4 and update ops[0] as above example. */
4870 0 : vect_unpromoted_value unprom;
4871 0 : tree tmp = vect_look_through_possible_promotion (vinfo, ops[0], &unprom);
4872 :
4873 0 : if (tmp == NULL_TREE || TYPE_PRECISION (unprom.type) != otype_prec)
4874 : return;
4875 :
4876 0 : ops[0] = tmp;
4877 :
4878 : /* 2. Generate _3 = .SAT_TRUNC (_2) and update ops[1] as above example. */
4879 0 : tree trunc_lhs_ssa = vect_recog_temp_ssa_var (otype, NULL);
4880 0 : gcall *call = gimple_build_call_internal (IFN_SAT_TRUNC, 1, ops[1]);
4881 :
4882 0 : gimple_call_set_lhs (call, trunc_lhs_ssa);
4883 0 : gimple_call_set_nothrow (call, /* nothrow_p */ false);
4884 0 : append_pattern_def_seq (vinfo, stmt_vinfo, call, v_otype);
4885 :
4886 0 : ops[1] = trunc_lhs_ssa;
4887 : }
4888 :
4889 : /*
4890 : * Try to detect saturation sub pattern (SAT_ADD), aka below gimple:
4891 : * Unsigned:
4892 : * _7 = _1 >= _2;
4893 : * _8 = _1 - _2;
4894 : * _10 = (long unsigned int) _7;
4895 : * _9 = _8 * _10;
4896 : *
4897 : * And then simplified to
4898 : * _9 = .SAT_SUB (_1, _2);
4899 : *
4900 : * Signed:
4901 : * x.0_4 = (unsigned char) x_16;
4902 : * y.1_5 = (unsigned char) y_18;
4903 : * _6 = x.0_4 - y.1_5;
4904 : * minus_19 = (int8_t) _6;
4905 : * _7 = x_16 ^ y_18;
4906 : * _8 = x_16 ^ minus_19;
4907 : * _44 = _7 < 0;
4908 : * _23 = x_16 < 0;
4909 : * _24 = (signed char) _23;
4910 : * _58 = (unsigned char) _24;
4911 : * _59 = -_58;
4912 : * _25 = (signed char) _59;
4913 : * _26 = _25 ^ 127;
4914 : * _42 = _8 < 0;
4915 : * _41 = _42 & _44;
4916 : * iftmp.2_11 = _41 ? _26 : minus_19;
4917 : *
4918 : * And then simplified to
4919 : * iftmp.2_11 = .SAT_SUB (x_16, y_18);
4920 : */
4921 :
4922 : static gimple *
4923 33157121 : vect_recog_sat_sub_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
4924 : tree *type_out)
4925 : {
4926 33157121 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
4927 :
4928 33157121 : if (!is_gimple_assign (last_stmt))
4929 : return NULL;
4930 :
4931 23249663 : tree ops[2];
4932 23249663 : tree lhs = gimple_assign_lhs (last_stmt);
4933 :
4934 23249663 : if (gimple_unsigned_integer_sat_sub (lhs, ops, NULL)
4935 23249663 : || gimple_signed_integer_sat_sub (lhs, ops, NULL))
4936 : {
4937 438 : vect_recog_sat_sub_pattern_transform (vinfo, stmt_vinfo, lhs, ops);
4938 438 : gimple *stmt = vect_recog_build_binary_gimple_stmt (vinfo, stmt_vinfo,
4939 : IFN_SAT_SUB, type_out,
4940 : lhs, ops[0], ops[1]);
4941 438 : if (stmt)
4942 : {
4943 215 : vect_pattern_detected ("vect_recog_sat_sub_pattern", last_stmt);
4944 215 : return stmt;
4945 : }
4946 : }
4947 :
4948 : return NULL;
4949 : }
4950 :
4951 : /* Return the vector type for ITYPE if the target supports the MAX_EXPR
4952 : and SAT_TRUNC sequence. */
4953 :
4954 : static tree
4955 73 : vect_sat_trunc_clip_input_vectype (vec_info *vinfo, tree v_otype,
4956 : tree itype)
4957 : {
4958 73 : tree v_itype;
4959 73 : if (!vect_supportable_direct_optab_p
4960 73 : (vinfo, itype, MAX_EXPR, itype, &v_itype)
4961 95 : || !direct_internal_fn_supported_p
4962 95 : (IFN_SAT_TRUNC, tree_pair (v_otype, v_itype),
4963 : OPTIMIZE_FOR_BOTH))
4964 : return NULL_TREE;
4965 :
4966 0 : return v_itype;
4967 : }
4968 :
4969 : /*
4970 : * Try to detect saturation truncation pattern (SAT_TRUNC), aka below gimple:
4971 : * overflow_5 = x_4(D) > 4294967295;
4972 : * _1 = (unsigned int) x_4(D);
4973 : * _2 = (unsigned int) overflow_5;
4974 : * _3 = -_2;
4975 : * _6 = _1 | _3;
4976 : *
4977 : * And then simplified to
4978 : * _6 = .SAT_TRUNC (x_4(D));
4979 : */
4980 :
4981 : static gimple *
4982 33156906 : vect_recog_sat_trunc_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
4983 : tree *type_out)
4984 : {
4985 33156906 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
4986 :
4987 33156906 : if (!is_gimple_assign (last_stmt))
4988 : return NULL;
4989 :
4990 23249448 : tree ops[1];
4991 23249448 : tree lhs = gimple_assign_lhs (last_stmt);
4992 23249448 : tree otype = TREE_TYPE (lhs);
4993 :
4994 23249448 : if ((gimple_unsigned_integer_narrow_clip (lhs, ops, NULL))
4995 23249448 : && type_has_mode_precision_p (otype))
4996 : {
4997 73 : tree input = ops[0];
4998 73 : tree itype = TREE_TYPE (input);
4999 73 : tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
5000 73 : internal_fn fn = IFN_SAT_TRUNC;
5001 73 : tree v_itype = NULL_TREE;
5002 :
5003 73 : if (v_otype != NULL_TREE)
5004 : {
5005 : /* Avoid widening an input only to narrow it again. A signed
5006 : promotion does not affect MAX (X, 0), so use the unpromoted
5007 : value when the target can narrow it directly to the output
5008 : type. */
5009 73 : vect_unpromoted_value unprom;
5010 73 : bool single_use_p = true;
5011 73 : if (vect_look_through_possible_promotion (vinfo, input, &unprom,
5012 : &single_use_p)
5013 73 : && single_use_p
5014 34 : && INTEGRAL_TYPE_P (unprom.type)
5015 34 : && TYPE_SIGN (unprom.type) == TYPE_SIGN (itype)
5016 34 : && type_has_mode_precision_p (unprom.type)
5017 34 : && TYPE_PRECISION (otype) < TYPE_PRECISION (unprom.type)
5018 107 : && TYPE_PRECISION (unprom.type) < TYPE_PRECISION (itype))
5019 : {
5020 0 : tree v_unprom_type
5021 0 : = vect_sat_trunc_clip_input_vectype (vinfo, v_otype,
5022 : unprom.type);
5023 0 : if (v_unprom_type != NULL_TREE)
5024 : {
5025 0 : input = unprom.op;
5026 0 : itype = unprom.type;
5027 0 : v_itype = v_unprom_type;
5028 : }
5029 : }
5030 :
5031 0 : if (v_itype == NULL_TREE)
5032 73 : v_itype
5033 73 : = vect_sat_trunc_clip_input_vectype (vinfo, v_otype, itype);
5034 :
5035 73 : if (v_itype != NULL_TREE)
5036 : {
5037 0 : tree temp = vect_recog_temp_ssa_var (itype, NULL);
5038 0 : gimple *max_stmt
5039 0 : = gimple_build_assign (temp, MAX_EXPR,
5040 : build_zero_cst (itype), input);
5041 0 : append_pattern_def_seq (vinfo, stmt_vinfo, max_stmt, v_itype);
5042 :
5043 0 : gcall *call = gimple_build_call_internal (fn, 1, temp);
5044 0 : tree out_ssa = vect_recog_temp_ssa_var (otype, NULL);
5045 :
5046 0 : gimple_call_set_lhs (call, out_ssa);
5047 0 : gimple_call_set_nothrow (call, false);
5048 0 : gimple_set_location (call, gimple_location (last_stmt));
5049 :
5050 0 : *type_out = v_otype;
5051 :
5052 0 : return call;
5053 : }
5054 : }
5055 : }
5056 :
5057 23249448 : if ((gimple_unsigned_integer_sat_trunc (lhs, ops, NULL)
5058 23249063 : || gimple_signed_integer_sat_trunc (lhs, ops, NULL))
5059 23249620 : && type_has_mode_precision_p (otype))
5060 : {
5061 545 : tree itype = TREE_TYPE (ops[0]);
5062 545 : tree v_itype = get_vectype_for_scalar_type (vinfo, itype);
5063 545 : tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
5064 545 : internal_fn fn = IFN_SAT_TRUNC;
5065 :
5066 537 : if (v_itype != NULL_TREE && v_otype != NULL_TREE
5067 1082 : && direct_internal_fn_supported_p (fn, tree_pair (v_otype, v_itype),
5068 : OPTIMIZE_FOR_BOTH))
5069 : {
5070 0 : gcall *call = gimple_build_call_internal (fn, 1, ops[0]);
5071 0 : tree out_ssa = vect_recog_temp_ssa_var (otype, NULL);
5072 :
5073 0 : gimple_call_set_lhs (call, out_ssa);
5074 0 : gimple_call_set_nothrow (call, /* nothrow_p */ false);
5075 0 : gimple_set_location (call, gimple_location (last_stmt));
5076 :
5077 0 : *type_out = v_otype;
5078 :
5079 0 : return call;
5080 : }
5081 : }
5082 :
5083 : return NULL;
5084 : }
5085 :
5086 :
5087 : /* Function add_code_for_floorceilround_divmod
5088 : A helper function to add compensation code for implementing FLOOR_MOD_EXPR,
5089 : FLOOR_DIV_EXPR, CEIL_MOD_EXPR, CEIL_DIV_EXPR, ROUND_MOD_EXPR and
5090 : ROUND_DIV_EXPR
5091 : The quotient and remainder are needed for implemented these operators.
5092 : FLOOR cases
5093 : r = x %[fl] y; r = x/[fl] y;
5094 : is
5095 : r = x % y; if (r && (x ^ y) < 0) r += y;
5096 : r = x % y; d = x/y; if (r && (x ^ y) < 0) d--; Respectively
5097 : Produce following sequence
5098 : v0 = x^y
5099 : v1 = -r
5100 : v2 = r | -r
5101 : v3 = v0 & v2
5102 : v4 = v3 < 0
5103 : if (floor_mod)
5104 : v5 = v4 ? y : 0
5105 : v6 = r + v5
5106 : if (floor_div)
5107 : v5 = v4 ? 1 : 0
5108 : v6 = d - 1
5109 : Similar sequences of vector instructions are produces for following cases
5110 : CEIL cases
5111 : r = x %[cl] y; r = x/[cl] y;
5112 : is
5113 : r = x % y; if (r && (x ^ y) >= 0) r -= y;
5114 : r = x % y; if (r) r -= y; (unsigned)
5115 : r = x % y; d = x/y; if (r && (x ^ y) >= 0) d++;
5116 : r = x % y; d = x/y; if (r) d++; (unsigned)
5117 : ROUND cases
5118 : r = x %[rd] y; r = x/[rd] y;
5119 : is
5120 : r = x % y; if (r > ((y-1)/2)) if ((x ^ y) >= 0) r -= y; else r += y;
5121 : r = x % y; if (r > ((y-1)/2)) r -= y; (unsigned)
5122 : r = x % y; d = x/y; if (r > ((y-1)/2)) if ((x ^ y) >= 0) d++; else d--;
5123 : r = x % y; d = x/y; if (r > ((y-1)/2)) d++; (unsigned)
5124 : Inputs:
5125 : VECTYPE: Vector type of the operands
5126 : STMT_VINFO: Statement where pattern begins
5127 : RHS_CODE: Should either be FLOOR_MOD_EXPR or FLOOR_DIV_EXPR
5128 : Q: The quotient of division
5129 : R: Remainder of division
5130 : OPRDN0/OPRND1: Actual operands involved
5131 : ITYPE: tree type of oprnd0
5132 : Output:
5133 : NULL if vectorization not possible
5134 : Gimple statement based on rhs_code
5135 : */
5136 : static gimple *
5137 418 : add_code_for_floorceilround_divmod (tree vectype, vec_info *vinfo,
5138 : stmt_vec_info stmt_vinfo,
5139 : enum tree_code rhs_code, tree q, tree r,
5140 : tree oprnd0, tree oprnd1, tree itype)
5141 : {
5142 418 : gimple *def_stmt;
5143 418 : tree mask_vectype = truth_type_for (vectype);
5144 418 : if (!mask_vectype)
5145 : return NULL;
5146 418 : tree bool_cond;
5147 418 : bool unsigned_p = TYPE_UNSIGNED (itype);
5148 :
5149 418 : switch (rhs_code)
5150 : {
5151 382 : case FLOOR_MOD_EXPR:
5152 382 : case FLOOR_DIV_EXPR:
5153 382 : case CEIL_MOD_EXPR:
5154 382 : case CEIL_DIV_EXPR:
5155 382 : {
5156 382 : if (!target_has_vecop_for_code (NEGATE_EXPR, vectype)
5157 358 : || !target_has_vecop_for_code (BIT_XOR_EXPR, vectype)
5158 358 : || !target_has_vecop_for_code (BIT_IOR_EXPR, vectype)
5159 358 : || !target_has_vecop_for_code (PLUS_EXPR, vectype)
5160 358 : || !target_has_vecop_for_code (MINUS_EXPR, vectype)
5161 358 : || !expand_vec_cmp_expr_p (vectype, mask_vectype, LT_EXPR)
5162 606 : || !expand_vec_cond_expr_p (vectype, mask_vectype))
5163 : return NULL;
5164 224 : if (unsigned_p)
5165 : {
5166 18 : gcc_assert (rhs_code == CEIL_MOD_EXPR || rhs_code == CEIL_DIV_EXPR);
5167 :
5168 18 : if (!expand_vec_cmp_expr_p (vectype, mask_vectype, GT_EXPR))
5169 : return NULL;
5170 18 : bool is_mod = rhs_code == CEIL_MOD_EXPR;
5171 : // r > 0
5172 18 : bool_cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5173 18 : def_stmt = gimple_build_assign (bool_cond, GT_EXPR, r,
5174 : build_int_cst (itype, 0));
5175 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5176 : itype);
5177 :
5178 : // (r > 0) ? y : 0 (mod)
5179 : // (r > 0) ? 1 : 0 (ceil)
5180 18 : tree extr_cond = vect_recog_temp_ssa_var (itype, NULL);
5181 18 : def_stmt
5182 27 : = gimple_build_assign (extr_cond, COND_EXPR, bool_cond,
5183 9 : is_mod ? oprnd1 : build_int_cst (itype, 1),
5184 : build_int_cst (itype, 0));
5185 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5186 :
5187 : // r -= (r > 0) ? y : 0 (mod)
5188 : // d += (x^y < 0 && r) ? -1 : 0 (ceil)
5189 18 : tree result = vect_recog_temp_ssa_var (itype, NULL);
5190 27 : return gimple_build_assign (result, is_mod ? MINUS_EXPR : PLUS_EXPR,
5191 18 : is_mod ? r : q, extr_cond);
5192 : }
5193 : else
5194 : {
5195 206 : bool ceil_p
5196 206 : = (rhs_code == CEIL_MOD_EXPR || rhs_code == CEIL_DIV_EXPR);
5197 206 : if (ceil_p && !target_has_vecop_for_code (BIT_NOT_EXPR, vectype))
5198 : return NULL;
5199 : // x ^ y
5200 206 : tree xort = vect_recog_temp_ssa_var (itype, NULL);
5201 206 : def_stmt = gimple_build_assign (xort, BIT_XOR_EXPR, oprnd0, oprnd1);
5202 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5203 :
5204 206 : tree cond_reg = xort;
5205 : // ~(x ^ y) (ceil)
5206 206 : if (ceil_p)
5207 : {
5208 18 : cond_reg = vect_recog_temp_ssa_var (itype, NULL);
5209 18 : def_stmt = gimple_build_assign (cond_reg, BIT_NOT_EXPR, xort);
5210 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5211 : }
5212 :
5213 : // -r
5214 206 : tree negate_r = vect_recog_temp_ssa_var (itype, NULL);
5215 206 : def_stmt = gimple_build_assign (negate_r, NEGATE_EXPR, r);
5216 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5217 :
5218 : // r | -r , sign bit is set if r!=0
5219 206 : tree r_or_negr = vect_recog_temp_ssa_var (itype, NULL);
5220 206 : def_stmt
5221 206 : = gimple_build_assign (r_or_negr, BIT_IOR_EXPR, r, negate_r);
5222 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5223 :
5224 : // (x ^ y) & (r | -r)
5225 : // ~(x ^ y) & (r | -r) (ceil)
5226 206 : tree r_or_negr_and_xor = vect_recog_temp_ssa_var (itype, NULL);
5227 206 : def_stmt = gimple_build_assign (r_or_negr_and_xor, BIT_AND_EXPR,
5228 : r_or_negr, cond_reg);
5229 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5230 :
5231 : // (x ^ y) & (r | -r) < 0 which is equivalent to (x^y < 0 && r!=0)
5232 206 : bool_cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5233 206 : def_stmt
5234 206 : = gimple_build_assign (bool_cond, LT_EXPR, r_or_negr_and_xor,
5235 : build_int_cst (itype, 0));
5236 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5237 : itype);
5238 :
5239 : // (x^y < 0 && r) ? y : 0 (mod)
5240 : // (x^y < 0 && r) ? -1 : 0 (div)
5241 206 : bool is_mod
5242 206 : = (rhs_code == FLOOR_MOD_EXPR || rhs_code == CEIL_MOD_EXPR);
5243 206 : tree extr_cond = vect_recog_temp_ssa_var (itype, NULL);
5244 246 : def_stmt = gimple_build_assign (extr_cond, COND_EXPR, bool_cond,
5245 : is_mod ? oprnd1
5246 40 : : build_int_cst (itype, -1),
5247 : build_int_cst (itype, 0));
5248 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5249 :
5250 : // r += (x ^ y < 0 && r) ? y : 0 (floor mod)
5251 : // d += (x^y < 0 && r) ? -1 : 0 (floor div)
5252 : // r -= (x ^ y < 0 && r) ? y : 0 (ceil mod)
5253 : // d -= (x^y < 0 && r) ? -1 : 0 (ceil div)
5254 206 : tree result = vect_recog_temp_ssa_var (itype, NULL);
5255 412 : return gimple_build_assign (result,
5256 206 : (rhs_code == FLOOR_MOD_EXPR
5257 206 : || rhs_code == FLOOR_DIV_EXPR)
5258 : ? PLUS_EXPR
5259 : : MINUS_EXPR,
5260 206 : is_mod ? r : q, extr_cond);
5261 : }
5262 : }
5263 36 : case ROUND_MOD_EXPR:
5264 36 : case ROUND_DIV_EXPR:
5265 36 : {
5266 36 : if (!target_has_vecop_for_code (BIT_AND_EXPR, vectype)
5267 36 : || !target_has_vecop_for_code (PLUS_EXPR, vectype)
5268 36 : || !expand_vec_cmp_expr_p (vectype, mask_vectype, LT_EXPR)
5269 36 : || !expand_vec_cmp_expr_p (vectype, mask_vectype, GT_EXPR)
5270 72 : || !expand_vec_cond_expr_p (vectype, mask_vectype))
5271 : return NULL;
5272 :
5273 36 : bool is_mod = rhs_code == ROUND_MOD_EXPR;
5274 36 : HOST_WIDE_INT d = TREE_INT_CST_LOW (oprnd1);
5275 36 : unsigned HOST_WIDE_INT abs_d
5276 : = (d >= 0 ? (unsigned HOST_WIDE_INT) d : -(unsigned HOST_WIDE_INT) d);
5277 36 : unsigned HOST_WIDE_INT mid_d = (abs_d - 1) >> 1;
5278 36 : if (!unsigned_p)
5279 : {
5280 : // check availability of abs expression for vector
5281 18 : if (!target_has_vecop_for_code (ABS_EXPR, vectype))
5282 : return NULL;
5283 : // abs (r)
5284 18 : tree abs_r = vect_recog_temp_ssa_var (itype, NULL);
5285 18 : def_stmt = gimple_build_assign (abs_r, ABS_EXPR, r);
5286 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5287 :
5288 : // abs (r) > (abs (y-1) >> 1)
5289 18 : tree round_p = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5290 18 : def_stmt = gimple_build_assign (round_p, GT_EXPR, abs_r,
5291 18 : build_int_cst (itype, mid_d));
5292 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5293 : itype);
5294 :
5295 : // x ^ y
5296 18 : tree cond_reg = vect_recog_temp_ssa_var (itype, NULL);
5297 18 : def_stmt
5298 18 : = gimple_build_assign (cond_reg, BIT_XOR_EXPR, oprnd0, oprnd1);
5299 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5300 :
5301 : // x ^ y < 0
5302 18 : bool_cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5303 18 : def_stmt = gimple_build_assign (bool_cond, LT_EXPR, cond_reg,
5304 : build_int_cst (itype, 0));
5305 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5306 : itype);
5307 :
5308 : // x ^ y < 0 ? y : -y (mod)
5309 : // x ^ y < 0 ? -1 : 1 (div)
5310 18 : tree val1 = vect_recog_temp_ssa_var (itype, NULL);
5311 18 : def_stmt
5312 36 : = gimple_build_assign (val1, COND_EXPR, bool_cond,
5313 27 : build_int_cst (itype, is_mod ? d : -1),
5314 18 : build_int_cst (itype, is_mod ? -d : 1));
5315 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5316 18 : int precision = TYPE_PRECISION (itype);
5317 18 : wide_int wmask = wi::mask (precision, false, precision);
5318 :
5319 : // abs (r) > (abs (y-1) >> 1) ? 0xffffffff : 0
5320 18 : tree val2 = vect_recog_temp_ssa_var (itype, NULL);
5321 36 : def_stmt = gimple_build_assign (val2, COND_EXPR, round_p,
5322 18 : wide_int_to_tree (itype, wmask),
5323 : build_int_cst (itype, 0));
5324 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5325 :
5326 18 : tree fval = vect_recog_temp_ssa_var (itype, NULL);
5327 18 : def_stmt = gimple_build_assign (fval, BIT_AND_EXPR, val1, val2);
5328 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5329 :
5330 18 : tree result = vect_recog_temp_ssa_var (itype, NULL);
5331 27 : return gimple_build_assign (result, PLUS_EXPR, is_mod ? r : q,
5332 : fval);
5333 18 : }
5334 : else
5335 : {
5336 : // r > (y-1 >> 1)
5337 18 : tree round_p = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5338 18 : def_stmt = gimple_build_assign (round_p, GT_EXPR, r,
5339 18 : build_int_cst (itype, mid_d));
5340 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5341 : itype);
5342 :
5343 : // (r > (y-1)>>1) ? -d : 1
5344 18 : tree val2 = vect_recog_temp_ssa_var (itype, NULL);
5345 18 : def_stmt
5346 36 : = gimple_build_assign (val2, COND_EXPR, round_p,
5347 18 : build_int_cst (itype, is_mod ? -d : 1),
5348 : build_int_cst (itype, 0));
5349 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5350 :
5351 18 : tree result = vect_recog_temp_ssa_var (itype, NULL);
5352 27 : return gimple_build_assign (result, PLUS_EXPR, is_mod ? r : q,
5353 18 : val2);
5354 : }
5355 : }
5356 : default:
5357 : return NULL;
5358 : }
5359 : }
5360 :
5361 : /* Detect a signed division by a constant that wouldn't be
5362 : otherwise vectorized:
5363 :
5364 : type a_t, b_t;
5365 :
5366 : S1 a_t = b_t / N;
5367 :
5368 : where type 'type' is an integral type and N is a constant.
5369 :
5370 : Similarly handle modulo by a constant:
5371 :
5372 : S4 a_t = b_t % N;
5373 :
5374 : Input/Output:
5375 :
5376 : * STMT_VINFO: The stmt from which the pattern search begins,
5377 : i.e. the division stmt. S1 is replaced by if N is a power
5378 : of two constant and type is signed:
5379 : S3 y_t = b_t < 0 ? N - 1 : 0;
5380 : S2 x_t = b_t + y_t;
5381 : S1' a_t = x_t >> log2 (N);
5382 :
5383 : S4 is replaced if N is a power of two constant and
5384 : type is signed by (where *_T temporaries have unsigned type):
5385 : S9 y_T = b_t < 0 ? -1U : 0U;
5386 : S8 z_T = y_T >> (sizeof (type_t) * CHAR_BIT - log2 (N));
5387 : S7 z_t = (type) z_T;
5388 : S6 w_t = b_t + z_t;
5389 : S5 x_t = w_t & (N - 1);
5390 : S4' a_t = x_t - z_t;
5391 :
5392 : Output:
5393 :
5394 : * TYPE_OUT: The type of the output of this pattern.
5395 :
5396 : * Return value: A new stmt that will be used to replace the division
5397 : S1 or modulo S4 stmt. */
5398 :
5399 : static gimple *
5400 33061266 : vect_recog_divmod_pattern (vec_info *vinfo,
5401 : stmt_vec_info stmt_vinfo, tree *type_out)
5402 : {
5403 33061266 : gimple *last_stmt = stmt_vinfo->stmt;
5404 33061266 : tree oprnd0, oprnd1, vectype, itype, cond;
5405 33061266 : gimple *pattern_stmt = NULL;
5406 33061266 : gimple *def_stmt = NULL;
5407 33061266 : enum tree_code rhs_code;
5408 33061266 : optab optab;
5409 33061266 : tree q, cst;
5410 33061266 : int prec;
5411 :
5412 33061266 : if (!is_gimple_assign (last_stmt)
5413 : /* The pattern will disrupt the reduction chain with multiple uses. */
5414 33061266 : || vect_is_reduction (stmt_vinfo))
5415 : return NULL;
5416 :
5417 23043376 : rhs_code = gimple_assign_rhs_code (last_stmt);
5418 23043376 : switch (rhs_code)
5419 : {
5420 294974 : case TRUNC_DIV_EXPR:
5421 294974 : case EXACT_DIV_EXPR:
5422 294974 : case TRUNC_MOD_EXPR:
5423 294974 : case FLOOR_MOD_EXPR:
5424 294974 : case FLOOR_DIV_EXPR:
5425 294974 : case CEIL_MOD_EXPR:
5426 294974 : case CEIL_DIV_EXPR:
5427 294974 : case ROUND_MOD_EXPR:
5428 294974 : case ROUND_DIV_EXPR:
5429 294974 : break;
5430 : default:
5431 : return NULL;
5432 : }
5433 :
5434 294974 : oprnd0 = gimple_assign_rhs1 (last_stmt);
5435 294974 : oprnd1 = gimple_assign_rhs2 (last_stmt);
5436 294974 : itype = TREE_TYPE (oprnd0);
5437 294974 : if (TREE_CODE (oprnd0) != SSA_NAME
5438 276241 : || TREE_CODE (oprnd1) != INTEGER_CST
5439 162859 : || TREE_CODE (itype) != INTEGER_TYPE
5440 457833 : || !type_has_mode_precision_p (itype))
5441 : return NULL;
5442 :
5443 162845 : scalar_int_mode itype_mode = SCALAR_INT_TYPE_MODE (itype);
5444 162845 : vectype = get_vectype_for_scalar_type (vinfo, itype);
5445 162845 : if (vectype == NULL_TREE)
5446 : return NULL;
5447 :
5448 131175 : if (optimize_bb_for_size_p (gimple_bb (last_stmt)))
5449 : {
5450 : /* If the target can handle vectorized division or modulo natively,
5451 : don't attempt to optimize this, since native division is likely
5452 : to give smaller code. */
5453 2264 : optab = optab_for_tree_code (rhs_code, vectype, optab_default);
5454 2264 : if (optab != unknown_optab
5455 2264 : && can_implement_p (optab, TYPE_MODE (vectype)))
5456 : return NULL;
5457 : }
5458 :
5459 131175 : prec = TYPE_PRECISION (itype);
5460 :
5461 262350 : bool is_flclrd_moddiv_p
5462 131175 : = rhs_code == FLOOR_MOD_EXPR || rhs_code == FLOOR_DIV_EXPR
5463 : || rhs_code == CEIL_MOD_EXPR || rhs_code == CEIL_DIV_EXPR
5464 130559 : || rhs_code == ROUND_MOD_EXPR || rhs_code == ROUND_DIV_EXPR;
5465 131175 : if (integer_pow2p (oprnd1))
5466 : {
5467 77659 : if ((TYPE_UNSIGNED (itype)
5468 36 : && (rhs_code == FLOOR_MOD_EXPR || rhs_code == FLOOR_DIV_EXPR))
5469 77695 : || tree_int_cst_sgn (oprnd1) != 1)
5470 : return NULL;
5471 :
5472 : /* Pattern detected. */
5473 77659 : vect_pattern_detected ("vect_recog_divmod_pattern", last_stmt);
5474 :
5475 77659 : *type_out = vectype;
5476 :
5477 : /* OPRND1 is a positive power of two here and the dividend of an
5478 : exact division is a multiple of it, so the right shift is exact
5479 : also for negative dividends and needs no rounding fixup. */
5480 77659 : if (rhs_code == EXACT_DIV_EXPR)
5481 : {
5482 58355 : tree shift = build_int_cst (itype, tree_log2 (oprnd1));
5483 58355 : pattern_stmt
5484 58355 : = gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL),
5485 : RSHIFT_EXPR, oprnd0, shift);
5486 58355 : gimple_set_location (pattern_stmt, gimple_location (last_stmt));
5487 58355 : return pattern_stmt;
5488 : }
5489 :
5490 : /* Check if the target supports this internal function. */
5491 19304 : internal_fn ifn = IFN_DIV_POW2;
5492 19304 : if (!TYPE_UNSIGNED (itype)
5493 19304 : && direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
5494 : {
5495 0 : tree shift = build_int_cst (itype, tree_log2 (oprnd1));
5496 :
5497 0 : tree var_div = vect_recog_temp_ssa_var (itype, NULL);
5498 0 : gimple *div_stmt = gimple_build_call_internal (ifn, 2, oprnd0, shift);
5499 0 : gimple_call_set_lhs (div_stmt, var_div);
5500 0 : if (rhs_code == TRUNC_MOD_EXPR || is_flclrd_moddiv_p)
5501 : {
5502 0 : append_pattern_def_seq (vinfo, stmt_vinfo, div_stmt);
5503 0 : tree t1 = vect_recog_temp_ssa_var (itype, NULL);
5504 0 : def_stmt
5505 0 : = gimple_build_assign (t1, LSHIFT_EXPR, var_div, shift);
5506 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5507 0 : tree r = vect_recog_temp_ssa_var (itype, NULL);
5508 0 : pattern_stmt = gimple_build_assign (r, MINUS_EXPR, oprnd0, t1);
5509 0 : if (is_flclrd_moddiv_p)
5510 : {
5511 0 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5512 0 : pattern_stmt
5513 0 : = add_code_for_floorceilround_divmod (vectype, vinfo,
5514 : stmt_vinfo, rhs_code,
5515 : var_div, r, oprnd0,
5516 : oprnd1, itype);
5517 0 : if (pattern_stmt == NULL)
5518 : return NULL;
5519 : }
5520 : }
5521 : else
5522 : pattern_stmt = div_stmt;
5523 0 : gimple_set_location (pattern_stmt, gimple_location (last_stmt));
5524 :
5525 0 : return pattern_stmt;
5526 : }
5527 :
5528 19304 : cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5529 19304 : def_stmt = gimple_build_assign (cond, LT_EXPR, oprnd0,
5530 : build_int_cst (itype, 0));
5531 19304 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt,
5532 : truth_type_for (vectype), itype);
5533 19304 : tree div_result = NULL_TREE;
5534 19304 : if (rhs_code == TRUNC_DIV_EXPR
5535 19304 : || rhs_code == FLOOR_DIV_EXPR
5536 2518 : || rhs_code == CEIL_DIV_EXPR
5537 2518 : || rhs_code == ROUND_DIV_EXPR)
5538 : {
5539 16829 : tree var = vect_recog_temp_ssa_var (itype, NULL);
5540 16829 : tree shift;
5541 16829 : def_stmt
5542 16829 : = gimple_build_assign (var, COND_EXPR, cond,
5543 : fold_build2 (MINUS_EXPR, itype, oprnd1,
5544 : build_int_cst (itype, 1)),
5545 : build_int_cst (itype, 0));
5546 16829 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5547 16829 : var = vect_recog_temp_ssa_var (itype, NULL);
5548 16829 : def_stmt
5549 16829 : = gimple_build_assign (var, PLUS_EXPR, oprnd0,
5550 : gimple_assign_lhs (def_stmt));
5551 16829 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5552 :
5553 16829 : shift = build_int_cst (itype, tree_log2 (oprnd1));
5554 16829 : div_result = vect_recog_temp_ssa_var (itype, NULL);
5555 16829 : pattern_stmt
5556 16829 : = gimple_build_assign (div_result, RSHIFT_EXPR, var, shift);
5557 : }
5558 19304 : if (rhs_code == TRUNC_MOD_EXPR || is_flclrd_moddiv_p)
5559 : {
5560 2653 : if (rhs_code == FLOOR_DIV_EXPR
5561 : || rhs_code == CEIL_DIV_EXPR
5562 2653 : || rhs_code == ROUND_DIV_EXPR)
5563 178 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5564 :
5565 2653 : tree signmask;
5566 2653 : if (compare_tree_int (oprnd1, 2) == 0)
5567 : {
5568 1343 : signmask = vect_recog_temp_ssa_var (itype, NULL);
5569 1343 : def_stmt = gimple_build_assign (signmask, COND_EXPR, cond,
5570 : build_int_cst (itype, 1),
5571 : build_int_cst (itype, 0));
5572 1343 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5573 : }
5574 : else
5575 : {
5576 1310 : tree utype
5577 1310 : = build_nonstandard_integer_type (prec, 1);
5578 1310 : tree vecutype = get_vectype_for_scalar_type (vinfo, utype);
5579 1310 : tree shift
5580 1310 : = build_int_cst (utype, GET_MODE_BITSIZE (itype_mode)
5581 1310 : - tree_log2 (oprnd1));
5582 1310 : tree var = vect_recog_temp_ssa_var (utype, NULL);
5583 :
5584 1310 : def_stmt = gimple_build_assign (var, COND_EXPR, cond,
5585 : build_int_cst (utype, -1),
5586 : build_int_cst (utype, 0));
5587 1310 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecutype);
5588 1310 : var = vect_recog_temp_ssa_var (utype, NULL);
5589 1310 : def_stmt = gimple_build_assign (var, RSHIFT_EXPR,
5590 : gimple_assign_lhs (def_stmt),
5591 : shift);
5592 1310 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecutype);
5593 1310 : signmask = vect_recog_temp_ssa_var (itype, NULL);
5594 1310 : def_stmt
5595 1310 : = gimple_build_assign (signmask, NOP_EXPR, var);
5596 1310 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5597 : }
5598 2653 : def_stmt
5599 2653 : = gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL),
5600 : PLUS_EXPR, oprnd0, signmask);
5601 2653 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5602 2653 : def_stmt
5603 2653 : = gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL),
5604 : BIT_AND_EXPR, gimple_assign_lhs (def_stmt),
5605 : fold_build2 (MINUS_EXPR, itype, oprnd1,
5606 : build_int_cst (itype, 1)));
5607 2653 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5608 :
5609 2653 : tree r = vect_recog_temp_ssa_var (itype, NULL);
5610 2653 : pattern_stmt
5611 2653 : = gimple_build_assign (r, MINUS_EXPR, gimple_assign_lhs (def_stmt),
5612 : signmask);
5613 2653 : if (is_flclrd_moddiv_p)
5614 : {
5615 272 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5616 272 : pattern_stmt
5617 272 : = add_code_for_floorceilround_divmod (vectype, vinfo,
5618 : stmt_vinfo, rhs_code,
5619 : div_result, r, oprnd0,
5620 : oprnd1, itype);
5621 272 : if (pattern_stmt == NULL)
5622 : return NULL;
5623 : }
5624 : }
5625 :
5626 19146 : return pattern_stmt;
5627 : }
5628 :
5629 53516 : if ((cst = uniform_integer_cst_p (oprnd1))
5630 53516 : && TYPE_UNSIGNED (itype)
5631 : && rhs_code == TRUNC_DIV_EXPR
5632 30758 : && vectype
5633 71786 : && targetm.vectorize.preferred_div_as_shifts_over_mult (vectype))
5634 : {
5635 : /* We can use the relationship:
5636 :
5637 : x // N == ((x+N+2) // (N+1) + x) // (N+1) for 0 <= x < N(N+3)
5638 :
5639 : to optimize cases where N+1 is a power of 2, and where // (N+1)
5640 : is therefore a shift right. When operating in modes that are
5641 : multiples of a byte in size, there are two cases:
5642 :
5643 : (1) N(N+3) is not representable, in which case the question
5644 : becomes whether the replacement expression overflows.
5645 : It is enough to test that x+N+2 does not overflow,
5646 : i.e. that x < MAX-(N+1).
5647 :
5648 : (2) N(N+3) is representable, in which case it is the (only)
5649 : bound that we need to check.
5650 :
5651 : ??? For now we just handle the case where // (N+1) is a shift
5652 : right by half the precision, since some architectures can
5653 : optimize the associated addition and shift combinations
5654 : into single instructions. */
5655 :
5656 7553 : auto wcst = wi::to_wide (cst);
5657 7553 : int pow = wi::exact_log2 (wcst + 1);
5658 7553 : if (pow == prec / 2)
5659 : {
5660 : /* Check that no overflow will occur. If we don't have range
5661 : information we can't perform the optimization. */
5662 :
5663 503 : int_range_max r;
5664 1006 : if (get_range_query (cfun)->range_of_expr (r, oprnd0, last_stmt)
5665 503 : && !r.undefined_p ())
5666 : {
5667 503 : wide_int max = r.upper_bound ();
5668 503 : wide_int one = wi::shwi (1, prec);
5669 503 : wide_int adder = wi::add (one, wi::lshift (one, pow));
5670 503 : wi::overflow_type ovf;
5671 503 : wi::add (max, adder, UNSIGNED, &ovf);
5672 503 : if (ovf == wi::OVF_NONE)
5673 : {
5674 321 : *type_out = vectype;
5675 321 : tree tadder = wide_int_to_tree (itype, adder);
5676 321 : tree rshift = wide_int_to_tree (itype, pow);
5677 :
5678 321 : tree new_lhs1 = vect_recog_temp_ssa_var (itype, NULL);
5679 321 : gassign *patt1
5680 321 : = gimple_build_assign (new_lhs1, PLUS_EXPR, oprnd0, tadder);
5681 321 : append_pattern_def_seq (vinfo, stmt_vinfo, patt1, vectype);
5682 :
5683 321 : tree new_lhs2 = vect_recog_temp_ssa_var (itype, NULL);
5684 321 : patt1 = gimple_build_assign (new_lhs2, RSHIFT_EXPR, new_lhs1,
5685 : rshift);
5686 321 : append_pattern_def_seq (vinfo, stmt_vinfo, patt1, vectype);
5687 :
5688 321 : tree new_lhs3 = vect_recog_temp_ssa_var (itype, NULL);
5689 321 : patt1 = gimple_build_assign (new_lhs3, PLUS_EXPR, new_lhs2,
5690 : oprnd0);
5691 321 : append_pattern_def_seq (vinfo, stmt_vinfo, patt1, vectype);
5692 :
5693 321 : tree new_lhs4 = vect_recog_temp_ssa_var (itype, NULL);
5694 321 : pattern_stmt = gimple_build_assign (new_lhs4, RSHIFT_EXPR,
5695 : new_lhs3, rshift);
5696 :
5697 321 : return pattern_stmt;
5698 : }
5699 503 : }
5700 503 : }
5701 : }
5702 :
5703 53195 : if (prec > HOST_BITS_PER_WIDE_INT
5704 53195 : || integer_zerop (oprnd1))
5705 : return NULL;
5706 :
5707 52898 : if (!can_mult_highpart_p (TYPE_MODE (vectype), TYPE_UNSIGNED (itype)))
5708 : return NULL;
5709 :
5710 22415 : if (TYPE_UNSIGNED (itype))
5711 : {
5712 17523 : unsigned HOST_WIDE_INT mh, ml;
5713 17523 : int pre_shift, post_shift;
5714 17523 : unsigned HOST_WIDE_INT d = (TREE_INT_CST_LOW (oprnd1)
5715 17523 : & GET_MODE_MASK (itype_mode));
5716 17523 : tree t1, t2, t3, t4;
5717 :
5718 17523 : if (d >= (HOST_WIDE_INT_1U << (prec - 1)))
5719 : /* FIXME: Can transform this into oprnd0 >= oprnd1 ? 1 : 0. */
5720 45 : return NULL;
5721 :
5722 : /* Find a suitable multiplier and right shift count instead of
5723 : directly dividing by D. */
5724 17478 : mh = choose_multiplier (d, prec, prec, &ml, &post_shift);
5725 :
5726 : /* If the suggested multiplier is more than PREC bits, we can do better
5727 : for even divisors, using an initial right shift. */
5728 17478 : if (mh != 0 && (d & 1) == 0)
5729 : {
5730 1551 : pre_shift = ctz_or_zero (d);
5731 1551 : mh = choose_multiplier (d >> pre_shift, prec, prec - pre_shift,
5732 : &ml, &post_shift);
5733 1551 : gcc_assert (!mh);
5734 : }
5735 : else
5736 : pre_shift = 0;
5737 :
5738 1219 : if (mh != 0)
5739 : {
5740 1219 : if (post_shift - 1 >= prec)
5741 : return NULL;
5742 :
5743 : /* t1 = oprnd0 h* ml;
5744 : t2 = oprnd0 - t1;
5745 : t3 = t2 >> 1;
5746 : t4 = t1 + t3;
5747 : q = t4 >> (post_shift - 1); */
5748 1219 : t1 = vect_recog_temp_ssa_var (itype, NULL);
5749 1219 : def_stmt = gimple_build_assign (t1, MULT_HIGHPART_EXPR, oprnd0,
5750 1219 : build_int_cst (itype, ml));
5751 1219 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5752 :
5753 1219 : t2 = vect_recog_temp_ssa_var (itype, NULL);
5754 1219 : def_stmt
5755 1219 : = gimple_build_assign (t2, MINUS_EXPR, oprnd0, t1);
5756 1219 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5757 :
5758 1219 : t3 = vect_recog_temp_ssa_var (itype, NULL);
5759 1219 : def_stmt
5760 1219 : = gimple_build_assign (t3, RSHIFT_EXPR, t2, integer_one_node);
5761 1219 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5762 :
5763 1219 : t4 = vect_recog_temp_ssa_var (itype, NULL);
5764 1219 : def_stmt
5765 1219 : = gimple_build_assign (t4, PLUS_EXPR, t1, t3);
5766 :
5767 1219 : if (post_shift != 1)
5768 : {
5769 1219 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5770 :
5771 1219 : q = vect_recog_temp_ssa_var (itype, NULL);
5772 1219 : pattern_stmt
5773 1219 : = gimple_build_assign (q, RSHIFT_EXPR, t4,
5774 1219 : build_int_cst (itype, post_shift - 1));
5775 : }
5776 : else
5777 : {
5778 : q = t4;
5779 : pattern_stmt = def_stmt;
5780 : }
5781 : }
5782 : else
5783 : {
5784 16259 : if (pre_shift >= prec || post_shift >= prec)
5785 : return NULL;
5786 :
5787 : /* t1 = oprnd0 >> pre_shift;
5788 : t2 = t1 h* ml;
5789 : q = t2 >> post_shift; */
5790 16259 : if (pre_shift)
5791 : {
5792 1551 : t1 = vect_recog_temp_ssa_var (itype, NULL);
5793 1551 : def_stmt
5794 1551 : = gimple_build_assign (t1, RSHIFT_EXPR, oprnd0,
5795 1551 : build_int_cst (integer_type_node, pre_shift));
5796 1551 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5797 : }
5798 : else
5799 : t1 = oprnd0;
5800 :
5801 16259 : t2 = vect_recog_temp_ssa_var (itype, NULL);
5802 16259 : def_stmt = gimple_build_assign (t2, MULT_HIGHPART_EXPR, t1,
5803 16259 : build_int_cst (itype, ml));
5804 :
5805 16259 : if (post_shift)
5806 : {
5807 16213 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5808 :
5809 16213 : q = vect_recog_temp_ssa_var (itype, NULL);
5810 16213 : def_stmt
5811 16213 : = gimple_build_assign (q, RSHIFT_EXPR, t2,
5812 16213 : build_int_cst (itype, post_shift));
5813 : }
5814 : else
5815 : q = t2;
5816 :
5817 : pattern_stmt = def_stmt;
5818 : }
5819 : }
5820 : else
5821 : {
5822 4892 : unsigned HOST_WIDE_INT ml;
5823 4892 : int post_shift;
5824 4892 : HOST_WIDE_INT d = TREE_INT_CST_LOW (oprnd1);
5825 4892 : unsigned HOST_WIDE_INT abs_d;
5826 4892 : bool add = false;
5827 4892 : tree t1, t2, t3, t4;
5828 :
5829 : /* Give up for -1. */
5830 4892 : if (d == -1)
5831 0 : return NULL;
5832 :
5833 : /* Since d might be INT_MIN, we have to cast to
5834 : unsigned HOST_WIDE_INT before negating to avoid
5835 : undefined signed overflow. */
5836 4892 : abs_d = (d >= 0
5837 4892 : ? (unsigned HOST_WIDE_INT) d
5838 : : - (unsigned HOST_WIDE_INT) d);
5839 :
5840 : /* n rem d = n rem -d */
5841 4892 : if (rhs_code == TRUNC_MOD_EXPR && d < 0)
5842 : {
5843 0 : d = abs_d;
5844 0 : oprnd1 = build_int_cst (itype, abs_d);
5845 : }
5846 4892 : if (HOST_BITS_PER_WIDE_INT >= prec
5847 4892 : && abs_d == HOST_WIDE_INT_1U << (prec - 1))
5848 : /* This case is not handled correctly below. */
5849 : return NULL;
5850 :
5851 4892 : choose_multiplier (abs_d, prec, prec - 1, &ml, &post_shift);
5852 4892 : if (ml >= HOST_WIDE_INT_1U << (prec - 1))
5853 : {
5854 1624 : add = true;
5855 1624 : ml |= HOST_WIDE_INT_M1U << (prec - 1);
5856 : }
5857 4892 : if (post_shift >= prec)
5858 : return NULL;
5859 :
5860 : /* t1 = oprnd0 h* ml; */
5861 4892 : t1 = vect_recog_temp_ssa_var (itype, NULL);
5862 4892 : def_stmt = gimple_build_assign (t1, MULT_HIGHPART_EXPR, oprnd0,
5863 4892 : build_int_cst (itype, ml));
5864 :
5865 4892 : if (add)
5866 : {
5867 : /* t2 = t1 + oprnd0; */
5868 1624 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5869 1624 : t2 = vect_recog_temp_ssa_var (itype, NULL);
5870 1624 : def_stmt = gimple_build_assign (t2, PLUS_EXPR, t1, oprnd0);
5871 : }
5872 : else
5873 : t2 = t1;
5874 :
5875 4892 : if (post_shift)
5876 : {
5877 : /* t3 = t2 >> post_shift; */
5878 4059 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5879 4059 : t3 = vect_recog_temp_ssa_var (itype, NULL);
5880 4059 : def_stmt = gimple_build_assign (t3, RSHIFT_EXPR, t2,
5881 4059 : build_int_cst (itype, post_shift));
5882 : }
5883 : else
5884 : t3 = t2;
5885 :
5886 4892 : int msb = 1;
5887 4892 : int_range_max r;
5888 9784 : get_range_query (cfun)->range_of_expr (r, oprnd0);
5889 4892 : if (!r.varying_p () && !r.undefined_p ())
5890 : {
5891 3040 : if (!wi::neg_p (r.lower_bound (), TYPE_SIGN (itype)))
5892 : msb = 0;
5893 736 : else if (wi::neg_p (r.upper_bound (), TYPE_SIGN (itype)))
5894 : msb = -1;
5895 : }
5896 :
5897 2304 : if (msb == 0 && d >= 0)
5898 : {
5899 : /* q = t3; */
5900 : q = t3;
5901 : pattern_stmt = def_stmt;
5902 : }
5903 : else
5904 : {
5905 : /* t4 = oprnd0 >> (prec - 1);
5906 : or if we know from VRP that oprnd0 >= 0
5907 : t4 = 0;
5908 : or if we know from VRP that oprnd0 < 0
5909 : t4 = -1; */
5910 2656 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5911 2656 : t4 = vect_recog_temp_ssa_var (itype, NULL);
5912 2656 : if (msb != 1)
5913 76 : def_stmt = gimple_build_assign (t4, INTEGER_CST,
5914 76 : build_int_cst (itype, msb));
5915 : else
5916 2580 : def_stmt = gimple_build_assign (t4, RSHIFT_EXPR, oprnd0,
5917 2580 : build_int_cst (itype, prec - 1));
5918 2656 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5919 :
5920 : /* q = t3 - t4; or q = t4 - t3; */
5921 2656 : q = vect_recog_temp_ssa_var (itype, NULL);
5922 5128 : pattern_stmt = gimple_build_assign (q, MINUS_EXPR, d < 0 ? t4 : t3,
5923 : d < 0 ? t3 : t4);
5924 : }
5925 4892 : }
5926 :
5927 22370 : if (rhs_code == TRUNC_MOD_EXPR || is_flclrd_moddiv_p)
5928 : {
5929 9980 : tree r, t1;
5930 :
5931 : /* We divided. Now finish by:
5932 : t1 = q * oprnd1;
5933 : r = oprnd0 - t1; */
5934 9980 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5935 :
5936 9980 : t1 = vect_recog_temp_ssa_var (itype, NULL);
5937 9980 : def_stmt = gimple_build_assign (t1, MULT_EXPR, q, oprnd1);
5938 9980 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5939 :
5940 9980 : r = vect_recog_temp_ssa_var (itype, NULL);
5941 9980 : pattern_stmt = gimple_build_assign (r, MINUS_EXPR, oprnd0, t1);
5942 :
5943 9980 : if (is_flclrd_moddiv_p)
5944 : {
5945 146 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5946 146 : pattern_stmt
5947 146 : = add_code_for_floorceilround_divmod (vectype, vinfo, stmt_vinfo,
5948 : rhs_code, q, r, oprnd0, oprnd1,
5949 : itype);
5950 146 : if (pattern_stmt == NULL)
5951 : return NULL;
5952 : }
5953 : }
5954 :
5955 : /* Pattern detected. */
5956 22370 : vect_pattern_detected ("vect_recog_divmod_pattern", last_stmt);
5957 :
5958 22370 : *type_out = vectype;
5959 22370 : return pattern_stmt;
5960 : }
5961 :
5962 : /* Detects pattern with a modulo operation (S1) where both arguments
5963 : are variables of integral type.
5964 : The statement is replaced by division, multiplication, and subtraction.
5965 : The last statement (S4) is returned.
5966 :
5967 : Example:
5968 : S1 c_t = a_t % b_t;
5969 :
5970 : is replaced by
5971 : S2 x_t = a_t / b_t;
5972 : S3 y_t = x_t * b_t;
5973 : S4 z_t = a_t - y_t; */
5974 :
5975 : static gimple *
5976 33083027 : vect_recog_mod_var_pattern (vec_info *vinfo,
5977 : stmt_vec_info stmt_vinfo, tree *type_out)
5978 : {
5979 33083027 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
5980 33083027 : tree oprnd0, oprnd1, vectype, itype;
5981 33083027 : gimple *pattern_stmt, *def_stmt;
5982 33083027 : enum tree_code rhs_code;
5983 :
5984 33083027 : if (!is_gimple_assign (last_stmt) || vect_is_reduction (stmt_vinfo))
5985 : return NULL;
5986 :
5987 23065137 : rhs_code = gimple_assign_rhs_code (last_stmt);
5988 23065137 : if (rhs_code != TRUNC_MOD_EXPR)
5989 : return NULL;
5990 :
5991 76804 : oprnd0 = gimple_assign_rhs1 (last_stmt);
5992 76804 : oprnd1 = gimple_assign_rhs2 (last_stmt);
5993 76804 : itype = TREE_TYPE (oprnd0);
5994 76804 : if (TREE_CODE (oprnd0) != SSA_NAME
5995 67899 : || TREE_CODE (oprnd1) != SSA_NAME
5996 54362 : || TREE_CODE (itype) != INTEGER_TYPE)
5997 : return NULL;
5998 :
5999 54190 : vectype = get_vectype_for_scalar_type (vinfo, itype);
6000 :
6001 54190 : if (!vectype
6002 45374 : || target_has_vecop_for_code (TRUNC_MOD_EXPR, vectype)
6003 45374 : || !target_has_vecop_for_code (TRUNC_DIV_EXPR, vectype)
6004 0 : || !target_has_vecop_for_code (MULT_EXPR, vectype)
6005 54190 : || !target_has_vecop_for_code (MINUS_EXPR, vectype))
6006 : return NULL;
6007 :
6008 0 : tree q, tmp, r;
6009 0 : q = vect_recog_temp_ssa_var (itype, NULL);
6010 0 : def_stmt = gimple_build_assign (q, TRUNC_DIV_EXPR, oprnd0, oprnd1);
6011 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vectype);
6012 :
6013 0 : tmp = vect_recog_temp_ssa_var (itype, NULL);
6014 0 : def_stmt = gimple_build_assign (tmp, MULT_EXPR, q, oprnd1);
6015 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vectype);
6016 :
6017 0 : r = vect_recog_temp_ssa_var (itype, NULL);
6018 0 : pattern_stmt = gimple_build_assign (r, MINUS_EXPR, oprnd0, tmp);
6019 :
6020 : /* Pattern detected. */
6021 0 : *type_out = vectype;
6022 0 : vect_pattern_detected ("vect_recog_mod_var_pattern", last_stmt);
6023 :
6024 0 : return pattern_stmt;
6025 : }
6026 :
6027 :
6028 : /* Return the proper type for converting bool VAR into
6029 : an integer value or NULL_TREE if no such type exists.
6030 : The type is chosen so that the converted value has the
6031 : same number of elements as VAR's vector type. */
6032 :
6033 : static tree
6034 4518138 : integer_type_for_mask (tree var, vec_info *vinfo, vect_def_type *dt = nullptr)
6035 : {
6036 4518138 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (var)))
6037 : return NULL_TREE;
6038 :
6039 2041070 : stmt_vec_info def_stmt_info = vinfo->lookup_def (var);
6040 2041070 : if (dt)
6041 : {
6042 383440 : if (!def_stmt_info)
6043 3487 : *dt = vect_external_def;
6044 : else
6045 379953 : *dt = STMT_VINFO_DEF_TYPE (def_stmt_info);
6046 : }
6047 383440 : if (!def_stmt_info
6048 1943450 : || STMT_VINFO_DEF_TYPE (def_stmt_info) == vect_external_def
6049 3601080 : || !vect_use_mask_type_p (def_stmt_info))
6050 : return NULL_TREE;
6051 :
6052 1204595 : return build_nonstandard_integer_type (def_stmt_info->mask_precision, 1);
6053 : }
6054 :
6055 : /* Function vect_recog_gcond_pattern
6056 :
6057 : Try to find pattern like following:
6058 :
6059 : if (a op b)
6060 :
6061 : where operator 'op' is not != and convert it to an adjusted boolean pattern
6062 :
6063 : mask = a op b
6064 : if (mask != 0)
6065 :
6066 : and set the mask type on MASK.
6067 :
6068 : Input:
6069 :
6070 : * STMT_VINFO: The stmt at the end from which the pattern
6071 : search begins, i.e. cast of a bool to
6072 : an integer type.
6073 :
6074 : Output:
6075 :
6076 : * TYPE_OUT: The type of the output of this pattern.
6077 :
6078 : * Return value: A new stmt that will be used to replace the pattern. */
6079 :
6080 : static gimple *
6081 33156906 : vect_recog_gcond_pattern (vec_info *vinfo,
6082 : stmt_vec_info stmt_vinfo, tree *type_out)
6083 : {
6084 : /* Currently we only support this for loop vectorization and when multiple
6085 : exits. */
6086 33156906 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6087 4685969 : if (!loop_vinfo || !LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
6088 : return NULL;
6089 :
6090 1747803 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
6091 1747803 : gcond* cond = NULL;
6092 1747803 : if (!(cond = dyn_cast <gcond *> (last_stmt)))
6093 : return NULL;
6094 :
6095 411921 : auto lhs = gimple_cond_lhs (cond);
6096 411921 : auto rhs = gimple_cond_rhs (cond);
6097 411921 : auto code = gimple_cond_code (cond);
6098 :
6099 411921 : tree scalar_type = TREE_TYPE (lhs);
6100 411921 : if (VECTOR_TYPE_P (scalar_type))
6101 : return NULL;
6102 :
6103 : /* If the input is a boolean then try to figure out the precision that the
6104 : vector type should use. We cannot use the scalar precision as this would
6105 : later mismatch. This is similar to what recog_bool does. */
6106 411921 : if (VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type))
6107 : {
6108 10433 : if (tree stype = integer_type_for_mask (lhs, vinfo))
6109 411921 : scalar_type = stype;
6110 : }
6111 :
6112 411921 : tree vectype = get_mask_type_for_scalar_type (vinfo, scalar_type);
6113 411921 : if (vectype == NULL_TREE)
6114 : return NULL;
6115 :
6116 390102 : tree new_lhs = vect_recog_temp_ssa_var (boolean_type_node, NULL);
6117 390102 : gimple *new_stmt = gimple_build_assign (new_lhs, code, lhs, rhs);
6118 390102 : append_pattern_def_seq (vinfo, stmt_vinfo, new_stmt, vectype, scalar_type);
6119 :
6120 390102 : gimple *pattern_stmt
6121 390102 : = gimple_build_cond (NE_EXPR, new_lhs,
6122 390102 : build_int_cst (TREE_TYPE (new_lhs), 0),
6123 : NULL_TREE, NULL_TREE);
6124 390102 : *type_out = vectype;
6125 390102 : vect_pattern_detected ("vect_recog_gcond_pattern", last_stmt);
6126 390102 : return pattern_stmt;
6127 : }
6128 :
6129 :
6130 : /* A helper for vect_recog_mask_conversion_pattern. Build
6131 : conversion of MASK to a type suitable for masking VECTYPE.
6132 : Built statement gets required vectype and is appended to
6133 : a pattern sequence of STMT_VINFO.
6134 : If VECTYPE is a mask type, SCALAR_TYPE_FOR_MASK is the scalar type
6135 : from which it was derived.
6136 :
6137 : Return converted mask. */
6138 :
6139 : static tree
6140 148232 : build_mask_conversion (vec_info *vinfo,
6141 : tree mask, tree vectype, stmt_vec_info stmt_vinfo,
6142 : tree scalar_type_for_mask = NULL_TREE)
6143 : {
6144 148232 : gimple *stmt;
6145 148232 : tree masktype, tmp;
6146 :
6147 239014 : gcc_assert (!scalar_type_for_mask == !VECTOR_BOOLEAN_TYPE_P (vectype));
6148 :
6149 148232 : masktype = truth_type_for (vectype);
6150 148232 : tmp = vect_recog_temp_ssa_var (TREE_TYPE (masktype), NULL);
6151 148232 : stmt = gimple_build_assign (tmp, CONVERT_EXPR, mask);
6152 239014 : append_pattern_def_seq (vinfo, stmt_vinfo,
6153 : stmt, masktype,
6154 : scalar_type_for_mask
6155 90782 : ? scalar_type_for_mask : TREE_TYPE (vectype));
6156 :
6157 148232 : return tmp;
6158 : }
6159 :
6160 :
6161 : /* Return MASK if MASK is suitable for masking an operation on vectors
6162 : of type VECTYPE, otherwise convert it into such a form and return
6163 : the result. Associate any conversion statements with STMT_INFO's
6164 : pattern. If VECTYPE is a mask type, SCALAR_TYPE_FOR_MASK is the scalar
6165 : type from which it was derived. */
6166 :
6167 : static tree
6168 81213 : vect_convert_mask_for_vectype (tree mask, tree vectype,
6169 : stmt_vec_info stmt_info, vec_info *vinfo,
6170 : tree scalar_type_for_mask = NULL_TREE)
6171 : {
6172 81213 : tree mask_type = integer_type_for_mask (mask, vinfo);
6173 81213 : if (mask_type)
6174 : {
6175 81213 : tree mask_vectype = get_mask_type_for_scalar_type (vinfo, mask_type);
6176 81213 : if (mask_vectype
6177 162426 : && maybe_ne (TYPE_VECTOR_SUBPARTS (vectype),
6178 99155 : TYPE_VECTOR_SUBPARTS (mask_vectype)))
6179 63271 : mask = build_mask_conversion (vinfo, mask, vectype, stmt_info,
6180 : scalar_type_for_mask);
6181 : }
6182 81213 : return mask;
6183 : }
6184 :
6185 :
6186 : /* Function vect_recog_bool_pattern
6187 :
6188 : Try to find pattern like following:
6189 :
6190 : bool a_b, b_b, c_b, d_b, e_b;
6191 : TYPE f_T;
6192 : loop:
6193 : S1 a_b = x1 CMP1 y1;
6194 : S2 b_b = x2 CMP2 y2;
6195 : S3 c_b = a_b & b_b;
6196 : S4 d_b = x3 CMP3 y3;
6197 : S5 e_b = c_b | d_b;
6198 : S6 f_T = (TYPE) e_b;
6199 :
6200 : where type 'TYPE' is an integral type. Or a similar pattern
6201 : ending in
6202 :
6203 : S6 f_Y = e_b ? r_Y : s_Y;
6204 :
6205 : as results from if-conversion of a complex condition.
6206 :
6207 : Input:
6208 :
6209 : * STMT_VINFO: The stmt at the end from which the pattern
6210 : search begins, i.e. cast of a bool to
6211 : an integer type.
6212 :
6213 : Output:
6214 :
6215 : * TYPE_OUT: The type of the output of this pattern.
6216 :
6217 : * Return value: A new stmt that will be used to replace the pattern.
6218 :
6219 : Assuming size of TYPE is the same as size of all comparisons
6220 : (otherwise some casts would be added where needed), the above
6221 : sequence we create related pattern stmts:
6222 : S1' a_T = x1 CMP1 y1 ? 1 : 0;
6223 : S3' c_T = x2 CMP2 y2 ? a_T : 0;
6224 : S4' d_T = x3 CMP3 y3 ? 1 : 0;
6225 : S5' e_T = c_T | d_T;
6226 : S6' f_T = e_T;
6227 :
6228 : Instead of the above S3' we could emit:
6229 : S2' b_T = x2 CMP2 y2 ? 1 : 0;
6230 : S3' c_T = a_T | b_T;
6231 : but the above is more efficient. */
6232 :
6233 : static gimple *
6234 33156906 : vect_recog_bool_pattern (vec_info *vinfo,
6235 : stmt_vec_info stmt_vinfo, tree *type_out)
6236 : {
6237 33156906 : gimple *last_stmt = stmt_vinfo->stmt;
6238 33156906 : enum tree_code rhs_code;
6239 33156906 : tree var, lhs, rhs, vectype;
6240 33156906 : gimple *pattern_stmt;
6241 :
6242 33156906 : if (!is_gimple_assign (last_stmt))
6243 : return NULL;
6244 :
6245 23639550 : var = gimple_assign_rhs1 (last_stmt);
6246 23639550 : lhs = gimple_assign_lhs (last_stmt);
6247 23639550 : rhs_code = gimple_assign_rhs_code (last_stmt);
6248 :
6249 23639550 : if (rhs_code == VIEW_CONVERT_EXPR)
6250 194754 : var = TREE_OPERAND (var, 0);
6251 :
6252 23639550 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (var)))
6253 : return NULL;
6254 :
6255 709491 : hash_set<gimple *> bool_stmts;
6256 :
6257 709491 : if (CONVERT_EXPR_CODE_P (rhs_code)
6258 : || rhs_code == VIEW_CONVERT_EXPR
6259 : || rhs_code == FLOAT_EXPR)
6260 : {
6261 170328 : if (! (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
6262 2087 : || SCALAR_FLOAT_TYPE_P (TREE_TYPE (lhs)))
6263 168775 : || VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)))
6264 : return NULL;
6265 78915 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6266 :
6267 78915 : tree type = integer_type_for_mask (var, vinfo);
6268 78915 : tree cst0, cst1, tmp;
6269 :
6270 78915 : if (!type)
6271 : return NULL;
6272 :
6273 : /* We may directly use cond with narrowed type to avoid multiple cond
6274 : exprs with following result packing and perform single cond with
6275 : packed mask instead. In case of widening we better make cond first
6276 : and then extract results. */
6277 38245 : if (TYPE_MODE (type) == TYPE_MODE (TREE_TYPE (lhs)))
6278 24949 : type = TREE_TYPE (lhs);
6279 :
6280 38245 : cst0 = build_int_cst (type, 0);
6281 38245 : cst1 = build_int_cst (type, 1);
6282 38245 : tmp = vect_recog_temp_ssa_var (type, NULL);
6283 38245 : pattern_stmt = gimple_build_assign (tmp, COND_EXPR, var, cst1, cst0);
6284 :
6285 38245 : if (!useless_type_conversion_p (type, TREE_TYPE (lhs)))
6286 : {
6287 13296 : tree new_vectype = get_vectype_for_scalar_type (vinfo, type);
6288 13296 : append_pattern_def_seq (vinfo, stmt_vinfo,
6289 : pattern_stmt, new_vectype);
6290 :
6291 13296 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6292 13296 : pattern_stmt
6293 26284 : = gimple_build_assign (lhs, (rhs_code == FLOAT_EXPR
6294 : ? FLOAT_EXPR : CONVERT_EXPR), tmp);
6295 : }
6296 :
6297 38245 : *type_out = vectype;
6298 38245 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6299 :
6300 38245 : return pattern_stmt;
6301 : }
6302 : else if (rhs_code == COND_EXPR
6303 155476 : && TREE_CODE (var) == SSA_NAME)
6304 : {
6305 155476 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6306 155476 : if (vectype == NULL_TREE)
6307 : return NULL;
6308 :
6309 : /* Build a scalar type for the boolean result that when
6310 : vectorized matches the vector type of the result in
6311 : size and number of elements. */
6312 140018 : unsigned prec
6313 140018 : = vector_element_size (tree_to_poly_uint64 (TYPE_SIZE (vectype)),
6314 : TYPE_VECTOR_SUBPARTS (vectype));
6315 :
6316 140018 : tree type
6317 280036 : = build_nonstandard_integer_type (prec,
6318 140018 : TYPE_UNSIGNED (TREE_TYPE (var)));
6319 140018 : if (get_vectype_for_scalar_type (vinfo, type) == NULL_TREE)
6320 : return NULL;
6321 :
6322 140018 : enum vect_def_type dt;
6323 140018 : if (integer_type_for_mask (var, vinfo))
6324 : return NULL;
6325 37539 : else if (TREE_CODE (TREE_TYPE (var)) == BOOLEAN_TYPE
6326 37539 : && vect_is_simple_use (var, vinfo, &dt)
6327 37539 : && (dt == vect_external_def
6328 37532 : || dt == vect_constant_def))
6329 : {
6330 : /* If the condition is already a boolean then manually convert it to a
6331 : mask of the given integer type but don't set a vectype. */
6332 1366 : tree lhs_ivar = vect_recog_temp_ssa_var (type, NULL);
6333 1366 : pattern_stmt = gimple_build_assign (lhs_ivar, COND_EXPR, var,
6334 : build_all_ones_cst (type),
6335 : build_zero_cst (type));
6336 1366 : append_inv_pattern_def_seq (vinfo, pattern_stmt);
6337 1366 : var = lhs_ivar;
6338 : }
6339 :
6340 37539 : tree lhs_var = vect_recog_temp_ssa_var (boolean_type_node, NULL);
6341 37539 : pattern_stmt = gimple_build_assign (lhs_var, NE_EXPR, var,
6342 37539 : build_zero_cst (TREE_TYPE (var)));
6343 :
6344 37539 : tree new_vectype = get_mask_type_for_scalar_type (vinfo, TREE_TYPE (var));
6345 37539 : if (!new_vectype)
6346 : return NULL;
6347 :
6348 37539 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, new_vectype,
6349 37539 : TREE_TYPE (var));
6350 :
6351 37539 : lhs_var = vect_convert_mask_for_vectype (lhs_var, vectype, stmt_vinfo,
6352 : vinfo);
6353 :
6354 37539 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6355 37539 : pattern_stmt
6356 37539 : = gimple_build_assign (lhs, COND_EXPR, lhs_var,
6357 : gimple_assign_rhs2 (last_stmt),
6358 : gimple_assign_rhs3 (last_stmt));
6359 37539 : *type_out = vectype;
6360 37539 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6361 :
6362 37539 : return pattern_stmt;
6363 : }
6364 468471 : else if (rhs_code == BIT_NOT_EXPR
6365 468471 : && !vect_use_mask_type_p (stmt_vinfo))
6366 : {
6367 : /* When we have a bool data inversion rewrite that to an XOR to
6368 : cope with the fact that we'll use a wider vector element type. */
6369 10318 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6370 10318 : pattern_stmt
6371 10318 : = gimple_build_assign (lhs, BIT_XOR_EXPR, var,
6372 10318 : build_all_ones_cst (TREE_TYPE (var)));
6373 10318 : *type_out = NULL_TREE;
6374 10318 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6375 :
6376 10318 : return pattern_stmt;
6377 : }
6378 458153 : else if ((rhs_code == BIT_XOR_EXPR
6379 : || rhs_code == BIT_AND_EXPR
6380 458153 : || rhs_code == BIT_IOR_EXPR)
6381 363137 : && TREE_CODE (var) == SSA_NAME)
6382 : {
6383 363137 : tree rhs2 = gimple_assign_rhs2 (last_stmt);
6384 363137 : if (TREE_CODE (rhs2) != SSA_NAME)
6385 : return NULL;
6386 363137 : tree lhs_type = integer_type_for_mask (lhs, vinfo);
6387 363137 : if (!lhs_type)
6388 : return NULL;
6389 191720 : vectype = get_mask_type_for_scalar_type (vinfo, lhs_type);
6390 191720 : if (!vectype)
6391 : return NULL;
6392 191720 : vect_def_type dt1, dt2;
6393 191720 : tree rhs1_type = integer_type_for_mask (var, vinfo, &dt1);
6394 191720 : tree rhs2_type = integer_type_for_mask (rhs2, vinfo, &dt2);
6395 191720 : if ((rhs1_type || dt1 == vect_external_def)
6396 175740 : && (rhs2_type || dt2 == vect_external_def))
6397 : return NULL;
6398 : /* When one input is a mask and the other is not create a pattern
6399 : stmt sequence that creates a mask for the non-mask input and
6400 : convert it to one suitable for the output mask used. */
6401 39201 : if (rhs1_type && !rhs2_type)
6402 : {
6403 23221 : tree rhs1_vectype = get_mask_type_for_scalar_type (vinfo, rhs1_type);
6404 23221 : if (!rhs1_vectype)
6405 : return NULL;
6406 23221 : tree rhs2_vectype = get_vectype_for_scalar_type (vinfo,
6407 23221 : TREE_TYPE (rhs2));
6408 23221 : if (!rhs2_vectype)
6409 : return NULL;
6410 23221 : tree new_vectype = truth_type_for (rhs2_vectype);
6411 23221 : tree tem = vect_recog_temp_ssa_var (TREE_TYPE (new_vectype), NULL);
6412 23221 : pattern_stmt = gimple_build_assign (tem, NE_EXPR, rhs2,
6413 : build_zero_cst
6414 23221 : (TREE_TYPE (rhs2)));
6415 23221 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
6416 23221 : new_vectype, TREE_TYPE (new_vectype));
6417 23221 : rhs2 = vect_convert_mask_for_vectype (tem, rhs1_vectype,
6418 : stmt_vinfo, vinfo, rhs1_type);
6419 : }
6420 15980 : else if (!rhs1_type && rhs2_type)
6421 : {
6422 15980 : tree rhs2_vectype = get_mask_type_for_scalar_type (vinfo, rhs2_type);
6423 15980 : if (!rhs2_vectype)
6424 : return NULL;
6425 15980 : tree rhs1_vectype = get_vectype_for_scalar_type (vinfo,
6426 15980 : TREE_TYPE (var));
6427 15980 : if (!rhs1_vectype)
6428 : return NULL;
6429 15980 : tree new_vectype = truth_type_for (rhs1_vectype);
6430 15980 : tree tem = vect_recog_temp_ssa_var (TREE_TYPE (new_vectype), NULL);
6431 15980 : pattern_stmt = gimple_build_assign (tem, NE_EXPR, var,
6432 : build_zero_cst
6433 15980 : (TREE_TYPE (var)));
6434 15980 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
6435 15980 : new_vectype, TREE_TYPE (new_vectype));
6436 15980 : var = vect_convert_mask_for_vectype (tem, rhs2_vectype,
6437 : stmt_vinfo, vinfo, rhs2_type);
6438 : }
6439 39201 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6440 39201 : pattern_stmt = gimple_build_assign (lhs, rhs_code, var, rhs2);
6441 39201 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6442 39201 : *type_out = vectype;
6443 39201 : return pattern_stmt;
6444 : }
6445 95016 : else if (rhs_code == SSA_NAME
6446 25973 : && STMT_VINFO_DATA_REF (stmt_vinfo))
6447 : {
6448 7888 : stmt_vec_info pattern_stmt_info;
6449 7888 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6450 7888 : if (!vectype || !VECTOR_MODE_P (TYPE_MODE (vectype)))
6451 : return NULL;
6452 :
6453 7888 : tree type = integer_type_for_mask (var, vinfo);
6454 7888 : if (!type)
6455 : return NULL;
6456 :
6457 4473 : var = vect_convert_mask_for_vectype (var, vectype, stmt_vinfo, vinfo);
6458 :
6459 4473 : tree cst0 = build_int_cst (TREE_TYPE (vectype), 0);
6460 4473 : tree cst1 = build_int_cst (TREE_TYPE (vectype), 1);
6461 4473 : rhs = vect_recog_temp_ssa_var (TREE_TYPE (vectype), NULL);
6462 4473 : pattern_stmt = gimple_build_assign (rhs, COND_EXPR, var, cst1, cst0);
6463 4473 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vectype);
6464 :
6465 4473 : lhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (vectype), lhs);
6466 4473 : pattern_stmt = gimple_build_assign (lhs, SSA_NAME, rhs);
6467 4473 : pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
6468 4473 : vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
6469 4473 : *type_out = vectype;
6470 4473 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6471 :
6472 4473 : return pattern_stmt;
6473 : }
6474 : else
6475 : return NULL;
6476 709491 : }
6477 :
6478 :
6479 : /* Function vect_recog_mask_conversion_pattern
6480 :
6481 : Try to find statements which require boolean type
6482 : conversion. Additional conversion statements are
6483 : added to handle such cases. For example:
6484 :
6485 : bool m_1, m_2, m_3;
6486 : int i_4, i_5;
6487 : double d_6, d_7;
6488 : char c_1, c_2, c_3;
6489 :
6490 : S1 m_1 = i_4 > i_5;
6491 : S2 m_2 = d_6 < d_7;
6492 : S3 m_3 = m_1 & m_2;
6493 : S4 c_1 = m_3 ? c_2 : c_3;
6494 :
6495 : Will be transformed into:
6496 :
6497 : S1 m_1 = i_4 > i_5;
6498 : S2 m_2 = d_6 < d_7;
6499 : S3'' m_2' = (_Bool[bitsize=32])m_2
6500 : S3' m_3' = m_1 & m_2';
6501 : S4'' m_3'' = (_Bool[bitsize=8])m_3'
6502 : S4' c_1' = m_3'' ? c_2 : c_3; */
6503 :
6504 : static gimple *
6505 33185044 : vect_recog_mask_conversion_pattern (vec_info *vinfo,
6506 : stmt_vec_info stmt_vinfo, tree *type_out)
6507 : {
6508 33185044 : gimple *last_stmt = stmt_vinfo->stmt;
6509 33185044 : enum tree_code rhs_code;
6510 33185044 : tree lhs = NULL_TREE, rhs1, rhs2, tmp, rhs1_type, rhs2_type;
6511 33185044 : tree vectype1, vectype2;
6512 33185044 : stmt_vec_info pattern_stmt_info;
6513 :
6514 : /* Check for MASK_LOAD and MASK_STORE as well as COND_OP calls requiring mask
6515 : conversion. */
6516 33185044 : if (is_gimple_call (last_stmt)
6517 33185044 : && gimple_call_internal_p (last_stmt))
6518 : {
6519 135902 : gcall *pattern_stmt;
6520 :
6521 135902 : internal_fn ifn = gimple_call_internal_fn (last_stmt);
6522 135902 : int mask_argno = internal_fn_mask_index (ifn);
6523 135902 : if (mask_argno < 0)
6524 : return NULL;
6525 :
6526 14814 : bool store_p = internal_store_fn_p (ifn);
6527 14814 : bool load_p = internal_store_fn_p (ifn);
6528 14814 : if (store_p)
6529 : {
6530 2513 : int rhs_index = internal_fn_stored_value_index (ifn);
6531 2513 : tree rhs = gimple_call_arg (last_stmt, rhs_index);
6532 2513 : vectype1 = get_vectype_for_scalar_type (vinfo, TREE_TYPE (rhs));
6533 : }
6534 : else
6535 : {
6536 12301 : lhs = gimple_call_lhs (last_stmt);
6537 12301 : if (!lhs)
6538 : return NULL;
6539 12301 : vectype1 = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6540 : }
6541 :
6542 14814 : if (!vectype1)
6543 : return NULL;
6544 :
6545 14494 : tree mask_arg = gimple_call_arg (last_stmt, mask_argno);
6546 14494 : tree mask_arg_type = integer_type_for_mask (mask_arg, vinfo);
6547 14494 : if (mask_arg_type)
6548 : {
6549 12862 : vectype2 = get_mask_type_for_scalar_type (vinfo, mask_arg_type);
6550 :
6551 12862 : if (!vectype2
6552 12862 : || known_eq (TYPE_VECTOR_SUBPARTS (vectype1),
6553 : TYPE_VECTOR_SUBPARTS (vectype2)))
6554 : return NULL;
6555 : }
6556 1632 : else if (store_p || load_p)
6557 : return NULL;
6558 :
6559 6298 : tmp = build_mask_conversion (vinfo, mask_arg, vectype1, stmt_vinfo);
6560 :
6561 6298 : auto_vec<tree, 8> args;
6562 6298 : unsigned int nargs = gimple_call_num_args (last_stmt);
6563 6298 : args.safe_grow (nargs, true);
6564 37788 : for (unsigned int i = 0; i < nargs; ++i)
6565 25192 : args[i] = ((int) i == mask_argno
6566 25192 : ? tmp
6567 18894 : : gimple_call_arg (last_stmt, i));
6568 6298 : pattern_stmt = gimple_build_call_internal_vec (ifn, args);
6569 :
6570 6298 : if (!store_p)
6571 : {
6572 6018 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6573 6018 : gimple_call_set_lhs (pattern_stmt, lhs);
6574 : }
6575 :
6576 6018 : if (load_p || store_p)
6577 280 : gimple_call_set_nothrow (pattern_stmt, true);
6578 :
6579 6298 : pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
6580 6298 : if (STMT_VINFO_DATA_REF (stmt_vinfo))
6581 1822 : vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
6582 :
6583 6298 : *type_out = vectype1;
6584 6298 : vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
6585 :
6586 6298 : return pattern_stmt;
6587 6298 : }
6588 :
6589 33049142 : if (!is_gimple_assign (last_stmt))
6590 : return NULL;
6591 :
6592 23667688 : gimple *pattern_stmt;
6593 23667688 : lhs = gimple_assign_lhs (last_stmt);
6594 23667688 : rhs1 = gimple_assign_rhs1 (last_stmt);
6595 23667688 : rhs_code = gimple_assign_rhs_code (last_stmt);
6596 :
6597 : /* Check for cond expression requiring mask conversion. */
6598 23667688 : if (rhs_code == COND_EXPR)
6599 : {
6600 135840 : vectype1 = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6601 :
6602 135840 : gcc_assert (! COMPARISON_CLASS_P (rhs1));
6603 135840 : if (TREE_CODE (rhs1) == SSA_NAME)
6604 : {
6605 135840 : rhs1_type = integer_type_for_mask (rhs1, vinfo);
6606 135840 : if (!rhs1_type)
6607 : return NULL;
6608 : }
6609 : else
6610 : return NULL;
6611 :
6612 122586 : vectype2 = get_mask_type_for_scalar_type (vinfo, rhs1_type);
6613 :
6614 122586 : if (!vectype1 || !vectype2)
6615 : return NULL;
6616 :
6617 : /* Continue if a conversion is needed. Also continue if we have
6618 : a comparison whose vector type would normally be different from
6619 : VECTYPE2 when considered in isolation. In that case we'll
6620 : replace the comparison with an SSA name (so that we can record
6621 : its vector type) and behave as though the comparison was an SSA
6622 : name from the outset. */
6623 120382 : if (known_eq (TYPE_VECTOR_SUBPARTS (vectype1),
6624 : TYPE_VECTOR_SUBPARTS (vectype2)))
6625 : return NULL;
6626 :
6627 45976 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1),
6628 91952 : TYPE_VECTOR_SUBPARTS (vectype2)))
6629 45976 : tmp = build_mask_conversion (vinfo, rhs1, vectype1, stmt_vinfo);
6630 : else
6631 : tmp = rhs1;
6632 :
6633 45976 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6634 45976 : pattern_stmt = gimple_build_assign (lhs, COND_EXPR, tmp,
6635 : gimple_assign_rhs2 (last_stmt),
6636 : gimple_assign_rhs3 (last_stmt));
6637 :
6638 45976 : *type_out = vectype1;
6639 45976 : vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
6640 :
6641 45976 : return pattern_stmt;
6642 : }
6643 :
6644 : /* Now check for binary boolean operations requiring conversion for
6645 : one of operands. */
6646 23531848 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)))
6647 : return NULL;
6648 :
6649 1838490 : if (rhs_code != BIT_IOR_EXPR
6650 : && rhs_code != BIT_XOR_EXPR
6651 1838490 : && rhs_code != BIT_AND_EXPR
6652 1514554 : && TREE_CODE_CLASS (rhs_code) != tcc_comparison)
6653 : return NULL;
6654 :
6655 1651380 : rhs2 = gimple_assign_rhs2 (last_stmt);
6656 :
6657 1651380 : rhs1_type = integer_type_for_mask (rhs1, vinfo);
6658 1651380 : rhs2_type = integer_type_for_mask (rhs2, vinfo);
6659 :
6660 1651380 : if (!rhs1_type || !rhs2_type
6661 1651380 : || TYPE_PRECISION (rhs1_type) == TYPE_PRECISION (rhs2_type))
6662 : return NULL;
6663 :
6664 32687 : if (TYPE_PRECISION (rhs1_type) < TYPE_PRECISION (rhs2_type))
6665 : {
6666 24640 : vectype1 = get_mask_type_for_scalar_type (vinfo, rhs1_type);
6667 24640 : if (!vectype1)
6668 : return NULL;
6669 24640 : rhs2 = build_mask_conversion (vinfo, rhs2, vectype1, stmt_vinfo,
6670 : rhs1_type);
6671 : }
6672 : else
6673 : {
6674 8047 : vectype1 = get_mask_type_for_scalar_type (vinfo, rhs2_type);
6675 8047 : if (!vectype1)
6676 : return NULL;
6677 8047 : rhs1 = build_mask_conversion (vinfo, rhs1, vectype1, stmt_vinfo,
6678 : rhs2_type);
6679 : }
6680 :
6681 32687 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6682 32687 : pattern_stmt = gimple_build_assign (lhs, rhs_code, rhs1, rhs2);
6683 :
6684 32687 : *type_out = vectype1;
6685 32687 : vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
6686 :
6687 32687 : return pattern_stmt;
6688 : }
6689 :
6690 : /* STMT_INFO is a load or store. If the load or store is conditional, return
6691 : the boolean condition under which it occurs, otherwise return null. */
6692 :
6693 : static tree
6694 95578 : vect_get_load_store_mask (stmt_vec_info stmt_info)
6695 : {
6696 95578 : if (gassign *def_assign = dyn_cast <gassign *> (stmt_info->stmt))
6697 : {
6698 94561 : gcc_assert (gimple_assign_single_p (def_assign));
6699 : return NULL_TREE;
6700 : }
6701 :
6702 1017 : if (gcall *def_call = dyn_cast <gcall *> (stmt_info->stmt))
6703 : {
6704 1017 : internal_fn ifn = gimple_call_internal_fn (def_call);
6705 1017 : int mask_index = internal_fn_mask_index (ifn);
6706 1017 : return gimple_call_arg (def_call, mask_index);
6707 : }
6708 :
6709 0 : gcc_unreachable ();
6710 : }
6711 :
6712 : /* Return the equivalent of:
6713 :
6714 : fold_convert (TYPE, VALUE)
6715 :
6716 : with the expectation that the operation will be vectorized.
6717 : If new statements are needed, add them as pattern statements
6718 : to STMT_INFO. */
6719 :
6720 : static tree
6721 0 : vect_add_conversion_to_pattern (vec_info *vinfo,
6722 : tree type, tree value, stmt_vec_info stmt_info)
6723 : {
6724 0 : if (useless_type_conversion_p (type, TREE_TYPE (value)))
6725 : return value;
6726 :
6727 0 : tree new_value = vect_recog_temp_ssa_var (type, NULL);
6728 0 : gassign *conversion = gimple_build_assign (new_value, CONVERT_EXPR, value);
6729 0 : append_pattern_def_seq (vinfo, stmt_info, conversion,
6730 : get_vectype_for_scalar_type (vinfo, type));
6731 0 : return new_value;
6732 : }
6733 :
6734 : /* Try to convert STMT_INFO into a call to a gather load or scatter store
6735 : internal function. Return the final statement on success and set
6736 : *TYPE_OUT to the vector type being loaded or stored.
6737 :
6738 : This function only handles gathers and scatters that were recognized
6739 : as such from the outset (indicated by STMT_VINFO_GATHER_SCATTER_P). */
6740 :
6741 : static gimple *
6742 33185044 : vect_recog_gather_scatter_pattern (vec_info *vinfo,
6743 : stmt_vec_info stmt_info, tree *type_out)
6744 : {
6745 : /* Currently we only support this for loop vectorization. */
6746 33185044 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6747 4699099 : if (!loop_vinfo)
6748 : return NULL;
6749 :
6750 : /* Make sure that we're looking at a gather load or scatter store. */
6751 4699099 : data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
6752 4699099 : if (!dr || !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
6753 : return NULL;
6754 :
6755 : /* Get the boolean that controls whether the load or store happens.
6756 : This is null if the operation is unconditional. */
6757 95578 : tree mask = vect_get_load_store_mask (stmt_info);
6758 :
6759 : /* DR analysis nailed down the vector type for the access. */
6760 95578 : tree gs_vectype = STMT_VINFO_VECTYPE (stmt_info);
6761 :
6762 : /* Make sure that the target supports an appropriate internal
6763 : function for the gather/scatter operation. */
6764 95578 : gather_scatter_info gs_info;
6765 95578 : if (!vect_check_gather_scatter (stmt_info, gs_vectype, loop_vinfo, &gs_info)
6766 95578 : || gs_info.ifn == IFN_LAST)
6767 : return NULL;
6768 :
6769 : /* Convert the mask to the right form. */
6770 0 : if (mask)
6771 0 : mask = vect_convert_mask_for_vectype (mask, gs_vectype, stmt_info,
6772 : loop_vinfo);
6773 0 : else if (gs_info.ifn == IFN_MASK_SCATTER_STORE
6774 0 : || gs_info.ifn == IFN_MASK_GATHER_LOAD
6775 0 : || gs_info.ifn == IFN_MASK_LEN_SCATTER_STORE
6776 0 : || gs_info.ifn == IFN_MASK_LEN_GATHER_LOAD)
6777 0 : mask = build_int_cst (TREE_TYPE (truth_type_for (gs_vectype)), -1);
6778 :
6779 : /* Get the invariant base and non-invariant offset, converting the
6780 : latter to the same width as the vector elements. */
6781 0 : tree base = gs_info.base;
6782 0 : tree offset_type = TREE_TYPE (gs_info.offset_vectype);
6783 0 : tree offset = vect_add_conversion_to_pattern (vinfo, offset_type,
6784 : gs_info.offset, stmt_info);
6785 :
6786 : /* Build the new pattern statement. */
6787 0 : tree scale = size_int (gs_info.scale);
6788 0 : gcall *pattern_stmt;
6789 :
6790 0 : if (DR_IS_READ (dr))
6791 : {
6792 0 : tree zero = build_zero_cst (gs_info.element_type);
6793 0 : if (mask != NULL)
6794 : {
6795 0 : int elsval = MASK_LOAD_ELSE_ZERO;
6796 :
6797 0 : tree vec_els
6798 0 : = vect_get_mask_load_else (elsval, TREE_TYPE (gs_vectype));
6799 0 : pattern_stmt = gimple_build_call_internal (gs_info.ifn, 7, base,
6800 : gs_info.alias_ptr,
6801 : offset, scale, zero, mask,
6802 : vec_els);
6803 : }
6804 : else
6805 0 : pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5, base,
6806 : gs_info.alias_ptr,
6807 : offset, scale, zero);
6808 0 : tree lhs = gimple_get_lhs (stmt_info->stmt);
6809 0 : tree load_lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6810 0 : gimple_call_set_lhs (pattern_stmt, load_lhs);
6811 : }
6812 : else
6813 : {
6814 0 : tree rhs = vect_get_store_rhs (stmt_info);
6815 0 : if (mask != NULL)
6816 0 : pattern_stmt = gimple_build_call_internal (gs_info.ifn, 6,
6817 : base, gs_info.alias_ptr,
6818 : offset, scale, rhs, mask);
6819 : else
6820 0 : pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5,
6821 : base, gs_info.alias_ptr,
6822 : offset, scale, rhs);
6823 : }
6824 0 : gimple_call_set_nothrow (pattern_stmt, true);
6825 :
6826 : /* Copy across relevant vectorization info and associate DR with the
6827 : new pattern statement instead of the original statement. */
6828 0 : stmt_vec_info pattern_stmt_info = loop_vinfo->add_stmt (pattern_stmt);
6829 0 : loop_vinfo->move_dr (pattern_stmt_info, stmt_info);
6830 :
6831 0 : *type_out = gs_vectype;
6832 0 : vect_pattern_detected ("gather/scatter pattern", stmt_info->stmt);
6833 :
6834 0 : return pattern_stmt;
6835 : }
6836 :
6837 : /* Helper method of vect_recog_cond_store_pattern, checks to see if COND_ARG
6838 : is points to a load statement that reads the same data as that of
6839 : STORE_VINFO. */
6840 :
6841 : static bool
6842 35512 : vect_cond_store_pattern_same_ref (vec_info *vinfo,
6843 : stmt_vec_info store_vinfo, tree cond_arg)
6844 : {
6845 35512 : stmt_vec_info load_stmt_vinfo = vinfo->lookup_def (cond_arg);
6846 35512 : if (!load_stmt_vinfo
6847 20313 : || !STMT_VINFO_DATA_REF (load_stmt_vinfo)
6848 11970 : || DR_IS_WRITE (STMT_VINFO_DATA_REF (load_stmt_vinfo))
6849 47482 : || !same_data_refs (STMT_VINFO_DATA_REF (store_vinfo),
6850 : STMT_VINFO_DATA_REF (load_stmt_vinfo)))
6851 26208 : return false;
6852 :
6853 : return true;
6854 : }
6855 :
6856 : /* Function vect_recog_cond_store_pattern
6857 :
6858 : Try to find the following pattern:
6859 :
6860 : x = *_3;
6861 : c = a CMP b;
6862 : y = c ? t_20 : x;
6863 : *_3 = y;
6864 :
6865 : where the store of _3 happens on a conditional select on a value loaded
6866 : from the same location. In such case we can elide the initial load if
6867 : MASK_STORE is supported and instead only conditionally write out the result.
6868 :
6869 : The pattern produces for the above:
6870 :
6871 : c = a CMP b;
6872 : .MASK_STORE (_3, c, t_20)
6873 :
6874 : Input:
6875 :
6876 : * STMT_VINFO: The stmt from which the pattern search begins. In the
6877 : example, when this function is called with _3 then the search begins.
6878 :
6879 : Output:
6880 :
6881 : * TYPE_OUT: The type of the output of this pattern.
6882 :
6883 : * Return value: A new stmt that will be used to replace the sequence. */
6884 :
6885 : static gimple *
6886 33185044 : vect_recog_cond_store_pattern (vec_info *vinfo,
6887 : stmt_vec_info stmt_vinfo, tree *type_out)
6888 : {
6889 33185044 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6890 4699099 : if (!loop_vinfo)
6891 : return NULL;
6892 :
6893 4699099 : gimple *store_stmt = STMT_VINFO_STMT (stmt_vinfo);
6894 :
6895 : /* Needs to be a gimple store where we have DR info for. */
6896 4699099 : if (!STMT_VINFO_DATA_REF (stmt_vinfo)
6897 1125081 : || DR_IS_READ (STMT_VINFO_DATA_REF (stmt_vinfo))
6898 5094693 : || !gimple_store_p (store_stmt))
6899 : return NULL;
6900 :
6901 393181 : tree st_rhs = gimple_assign_rhs1 (store_stmt);
6902 :
6903 393181 : if (TREE_CODE (st_rhs) != SSA_NAME)
6904 : return NULL;
6905 :
6906 306136 : auto cond_vinfo = vinfo->lookup_def (st_rhs);
6907 :
6908 : /* If the condition isn't part of the loop then bool recog wouldn't have seen
6909 : it and so this transformation may not be valid. */
6910 306136 : if (!cond_vinfo)
6911 : return NULL;
6912 :
6913 288049 : cond_vinfo = vect_stmt_to_vectorize (cond_vinfo);
6914 288049 : gassign *cond_stmt = dyn_cast<gassign *> (STMT_VINFO_STMT (cond_vinfo));
6915 355596 : if (!cond_stmt || gimple_assign_rhs_code (cond_stmt) != COND_EXPR)
6916 : return NULL;
6917 :
6918 : /* Check if the else value matches the original loaded one. */
6919 18797 : bool invert = false;
6920 18797 : tree cmp_ls = gimple_arg (cond_stmt, 0);
6921 18797 : if (TREE_CODE (cmp_ls) != SSA_NAME)
6922 : return NULL;
6923 :
6924 18797 : tree cond_arg1 = gimple_arg (cond_stmt, 1);
6925 18797 : tree cond_arg2 = gimple_arg (cond_stmt, 2);
6926 :
6927 18797 : if (!vect_cond_store_pattern_same_ref (vinfo, stmt_vinfo, cond_arg2)
6928 18797 : && !(invert = vect_cond_store_pattern_same_ref (vinfo, stmt_vinfo,
6929 : cond_arg1)))
6930 : return NULL;
6931 :
6932 9304 : vect_pattern_detected ("vect_recog_cond_store_pattern", store_stmt);
6933 :
6934 9304 : tree scalar_type = TREE_TYPE (st_rhs);
6935 9304 : if (VECTOR_TYPE_P (scalar_type))
6936 : return NULL;
6937 :
6938 9304 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
6939 9304 : if (vectype == NULL_TREE)
6940 : return NULL;
6941 :
6942 9304 : machine_mode mask_mode;
6943 9304 : machine_mode vecmode = TYPE_MODE (vectype);
6944 1864 : if (!VECTOR_MODE_P (vecmode)
6945 9304 : || targetm.vectorize.conditional_operation_is_expensive (IFN_MASK_STORE)
6946 9304 : || !targetm.vectorize.get_mask_mode (vecmode).exists (&mask_mode)
6947 9304 : || !can_vec_mask_load_store_p (vecmode, mask_mode, false))
6948 : return NULL;
6949 :
6950 0 : tree base = DR_REF (STMT_VINFO_DATA_REF (stmt_vinfo));
6951 0 : if (may_be_nonaddressable_p (base))
6952 : return NULL;
6953 :
6954 : /* We need to use the false parameter of the conditional select. */
6955 0 : tree cond_store_arg = invert ? cond_arg2 : cond_arg1;
6956 0 : tree cond_load_arg = invert ? cond_arg1 : cond_arg2;
6957 0 : gimple *load_stmt = SSA_NAME_DEF_STMT (cond_load_arg);
6958 :
6959 : /* This is a rough estimation to check that there aren't any aliasing stores
6960 : in between the load and store. It's a bit strict, but for now it's good
6961 : enough. */
6962 0 : if (gimple_vuse (load_stmt) != gimple_vuse (store_stmt))
6963 : return NULL;
6964 :
6965 : /* If we have to invert the condition, i.e. use the true argument rather than
6966 : the false argument, we have to negate the mask. */
6967 0 : if (invert)
6968 : {
6969 0 : tree var = vect_recog_temp_ssa_var (boolean_type_node, NULL);
6970 :
6971 : /* Invert the mask using ^ 1. */
6972 0 : tree itype = TREE_TYPE (cmp_ls);
6973 0 : gassign *conv = gimple_build_assign (var, BIT_XOR_EXPR, cmp_ls,
6974 : build_int_cst (itype, 1));
6975 :
6976 0 : tree mask_vec_type = get_mask_type_for_scalar_type (vinfo, itype);
6977 0 : append_pattern_def_seq (vinfo, stmt_vinfo, conv, mask_vec_type, itype);
6978 0 : cmp_ls= var;
6979 : }
6980 :
6981 0 : if (TREE_CODE (base) != MEM_REF)
6982 0 : base = build_fold_addr_expr (base);
6983 :
6984 0 : tree ptr = build_int_cst (reference_alias_ptr_type (base),
6985 0 : get_object_alignment (base));
6986 :
6987 : /* Convert the mask to the right form. */
6988 0 : tree mask = vect_convert_mask_for_vectype (cmp_ls, vectype, stmt_vinfo,
6989 : vinfo);
6990 :
6991 0 : gcall *call
6992 0 : = gimple_build_call_internal (IFN_MASK_STORE, 4, base, ptr, mask,
6993 : cond_store_arg);
6994 0 : gimple_set_location (call, gimple_location (store_stmt));
6995 :
6996 : /* Copy across relevant vectorization info and associate DR with the
6997 : new pattern statement instead of the original statement. */
6998 0 : stmt_vec_info pattern_stmt_info = loop_vinfo->add_stmt (call);
6999 0 : loop_vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
7000 :
7001 0 : *type_out = vectype;
7002 0 : return call;
7003 : }
7004 :
7005 : /* Return true if TYPE is a non-boolean integer type. These are the types
7006 : that we want to consider for narrowing. */
7007 :
7008 : static bool
7009 68517571 : vect_narrowable_type_p (tree type)
7010 : {
7011 68517571 : return INTEGRAL_TYPE_P (type) && !VECT_SCALAR_BOOLEAN_TYPE_P (type);
7012 : }
7013 :
7014 : /* Return true if the operation given by CODE can be truncated to N bits
7015 : when only N bits of the output are needed. This is only true if bit N+1
7016 : of the inputs has no effect on the low N bits of the result. */
7017 :
7018 : static bool
7019 17103790 : vect_truncatable_operation_p (tree_code code)
7020 : {
7021 17103790 : switch (code)
7022 : {
7023 : case NEGATE_EXPR:
7024 : case PLUS_EXPR:
7025 : case MINUS_EXPR:
7026 : case MULT_EXPR:
7027 : case BIT_NOT_EXPR:
7028 : case BIT_AND_EXPR:
7029 : case BIT_IOR_EXPR:
7030 : case BIT_XOR_EXPR:
7031 : case COND_EXPR:
7032 : return true;
7033 :
7034 6737256 : default:
7035 6737256 : return false;
7036 : }
7037 : }
7038 :
7039 : /* Record that STMT_INFO could be changed from operating on TYPE to
7040 : operating on a type with the precision and sign given by PRECISION
7041 : and SIGN respectively. PRECISION is an arbitrary bit precision;
7042 : it might not be a whole number of bytes. */
7043 :
7044 : static void
7045 2617084 : vect_set_operation_type (stmt_vec_info stmt_info, tree type,
7046 : unsigned int precision, signop sign)
7047 : {
7048 : /* Round the precision up to a whole number of bytes. */
7049 2617084 : precision = vect_element_precision (precision);
7050 2617084 : if (precision < TYPE_PRECISION (type)
7051 2617084 : && (!stmt_info->operation_precision
7052 41549 : || stmt_info->operation_precision > precision))
7053 : {
7054 1705145 : stmt_info->operation_precision = precision;
7055 1705145 : stmt_info->operation_sign = sign;
7056 : }
7057 2617084 : }
7058 :
7059 : /* Record that STMT_INFO only requires MIN_INPUT_PRECISION from its
7060 : non-boolean inputs, all of which have type TYPE. MIN_INPUT_PRECISION
7061 : is an arbitrary bit precision; it might not be a whole number of bytes. */
7062 :
7063 : static void
7064 12265555 : vect_set_min_input_precision (stmt_vec_info stmt_info, tree type,
7065 : unsigned int min_input_precision)
7066 : {
7067 : /* This operation in isolation only requires the inputs to have
7068 : MIN_INPUT_PRECISION of precision, However, that doesn't mean
7069 : that MIN_INPUT_PRECISION is a natural precision for the chain
7070 : as a whole. E.g. consider something like:
7071 :
7072 : unsigned short *x, *y;
7073 : *y = ((*x & 0xf0) >> 4) | (*y << 4);
7074 :
7075 : The right shift can be done on unsigned chars, and only requires the
7076 : result of "*x & 0xf0" to be done on unsigned chars. But taking that
7077 : approach would mean turning a natural chain of single-vector unsigned
7078 : short operations into one that truncates "*x" and then extends
7079 : "(*x & 0xf0) >> 4", with two vectors for each unsigned short
7080 : operation and one vector for each unsigned char operation.
7081 : This would be a significant pessimization.
7082 :
7083 : Instead only propagate the maximum of this precision and the precision
7084 : required by the users of the result. This means that we don't pessimize
7085 : the case above but continue to optimize things like:
7086 :
7087 : unsigned char *y;
7088 : unsigned short *x;
7089 : *y = ((*x & 0xf0) >> 4) | (*y << 4);
7090 :
7091 : Here we would truncate two vectors of *x to a single vector of
7092 : unsigned chars and use single-vector unsigned char operations for
7093 : everything else, rather than doing two unsigned short copies of
7094 : "(*x & 0xf0) >> 4" and then truncating the result. */
7095 12265555 : min_input_precision = MAX (min_input_precision,
7096 : stmt_info->min_output_precision);
7097 :
7098 12265555 : if (min_input_precision < TYPE_PRECISION (type)
7099 12265555 : && (!stmt_info->min_input_precision
7100 66220 : || stmt_info->min_input_precision > min_input_precision))
7101 640192 : stmt_info->min_input_precision = min_input_precision;
7102 12265555 : }
7103 :
7104 : /* Subroutine of vect_determine_min_output_precision. Return true if
7105 : we can calculate a reduced number of output bits for STMT_INFO,
7106 : whose result is LHS. */
7107 :
7108 : static bool
7109 15709240 : vect_determine_min_output_precision_1 (vec_info *vinfo,
7110 : stmt_vec_info stmt_info, tree lhs)
7111 : {
7112 : /* Take the maximum precision required by users of the result. */
7113 15709240 : unsigned int precision = 0;
7114 15709240 : imm_use_iterator iter;
7115 15709240 : use_operand_p use;
7116 16753994 : FOR_EACH_IMM_USE_FAST (use, iter, lhs)
7117 : {
7118 16448489 : gimple *use_stmt = USE_STMT (use);
7119 16448489 : if (is_gimple_debug (use_stmt))
7120 735054 : continue;
7121 15713435 : stmt_vec_info use_stmt_info = vinfo->lookup_stmt (use_stmt);
7122 15713435 : if (!use_stmt_info || !use_stmt_info->min_input_precision)
7123 : return false;
7124 : /* The input precision recorded for COND_EXPRs applies only to the
7125 : "then" and "else" values. */
7126 310344 : gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
7127 277561 : if (assign
7128 277561 : && gimple_assign_rhs_code (assign) == COND_EXPR
7129 644 : && use->use != gimple_assign_rhs2_ptr (assign)
7130 644 : && use->use != gimple_assign_rhs3_ptr (assign))
7131 : return false;
7132 1046478 : precision = MAX (precision, use_stmt_info->min_input_precision);
7133 15403735 : }
7134 :
7135 305505 : if (dump_enabled_p ())
7136 6098 : dump_printf_loc (MSG_NOTE, vect_location,
7137 : "only the low %d bits of %T are significant\n",
7138 : precision, lhs);
7139 305505 : stmt_info->min_output_precision = precision;
7140 305505 : return true;
7141 : }
7142 :
7143 : /* Calculate min_output_precision for STMT_INFO. */
7144 :
7145 : static void
7146 40232395 : vect_determine_min_output_precision (vec_info *vinfo, stmt_vec_info stmt_info)
7147 : {
7148 : /* We're only interested in statements with a narrowable result. */
7149 40232395 : tree lhs = gimple_get_lhs (stmt_info->stmt);
7150 40232395 : if (!lhs
7151 31888059 : || TREE_CODE (lhs) != SSA_NAME
7152 67194861 : || !vect_narrowable_type_p (TREE_TYPE (lhs)))
7153 : return;
7154 :
7155 15709240 : if (!vect_determine_min_output_precision_1 (vinfo, stmt_info, lhs))
7156 15403735 : stmt_info->min_output_precision = TYPE_PRECISION (TREE_TYPE (lhs));
7157 : }
7158 :
7159 : /* Use range information to decide whether STMT (described by STMT_INFO)
7160 : could be done in a narrower type. This is effectively a forward
7161 : propagation, since it uses context-independent information that applies
7162 : to all users of an SSA name. */
7163 :
7164 : static void
7165 23093102 : vect_determine_precisions_from_range (stmt_vec_info stmt_info, gassign *stmt)
7166 : {
7167 23093102 : tree lhs = gimple_assign_lhs (stmt);
7168 23093102 : if (!lhs || TREE_CODE (lhs) != SSA_NAME)
7169 20596814 : return;
7170 :
7171 18462003 : tree type = TREE_TYPE (lhs);
7172 18462003 : if (!vect_narrowable_type_p (type))
7173 : return;
7174 :
7175 : /* First see whether we have any useful range information for the result. */
7176 12170347 : unsigned int precision = TYPE_PRECISION (type);
7177 12170347 : signop sign = TYPE_SIGN (type);
7178 12170347 : wide_int min_value, max_value;
7179 12170347 : if (!vect_get_range_info (lhs, &min_value, &max_value))
7180 : return;
7181 :
7182 5856590 : tree_code code = gimple_assign_rhs_code (stmt);
7183 5856590 : unsigned int nops = gimple_num_ops (stmt);
7184 :
7185 5856590 : if (!vect_truncatable_operation_p (code))
7186 : {
7187 : /* Handle operations that can be computed in type T if all inputs
7188 : and outputs can be represented in type T. Also handle left and
7189 : right shifts, where (in addition) the maximum shift amount must
7190 : be less than the number of bits in T. */
7191 2173203 : bool is_shift;
7192 2173203 : switch (code)
7193 : {
7194 : case LSHIFT_EXPR:
7195 : case RSHIFT_EXPR:
7196 : is_shift = true;
7197 : break;
7198 :
7199 312682 : case ABS_EXPR:
7200 312682 : case MIN_EXPR:
7201 312682 : case MAX_EXPR:
7202 312682 : case TRUNC_DIV_EXPR:
7203 312682 : case CEIL_DIV_EXPR:
7204 312682 : case FLOOR_DIV_EXPR:
7205 312682 : case ROUND_DIV_EXPR:
7206 312682 : case EXACT_DIV_EXPR:
7207 : /* Modulus is excluded because it is typically calculated by doing
7208 : a division, for which minimum signed / -1 isn't representable in
7209 : the original signed type. We could take the division range into
7210 : account instead, if handling modulus ever becomes important. */
7211 312682 : is_shift = false;
7212 312682 : break;
7213 :
7214 : default:
7215 : return;
7216 : }
7217 1429445 : for (unsigned int i = 1; i < nops; ++i)
7218 : {
7219 1111510 : tree op = gimple_op (stmt, i);
7220 1111510 : wide_int op_min_value, op_max_value;
7221 1111510 : if (TREE_CODE (op) == INTEGER_CST)
7222 : {
7223 325195 : unsigned int op_precision = TYPE_PRECISION (TREE_TYPE (op));
7224 325195 : op_min_value = op_max_value = wi::to_wide (op, op_precision);
7225 : }
7226 786315 : else if (TREE_CODE (op) == SSA_NAME)
7227 : {
7228 786315 : if (!vect_get_range_info (op, &op_min_value, &op_max_value))
7229 : return;
7230 : }
7231 : else
7232 : return;
7233 :
7234 721206 : if (is_shift && i == 2)
7235 : {
7236 : /* There needs to be one more bit than the maximum shift amount.
7237 :
7238 : If the maximum shift amount is already 1 less than PRECISION
7239 : then we can't narrow the shift further. Dealing with that
7240 : case first ensures that we can safely use an unsigned range
7241 : below.
7242 :
7243 : op_min_value isn't relevant, since shifts by negative amounts
7244 : are UB. */
7245 208971 : if (wi::geu_p (op_max_value, precision - 1))
7246 : return;
7247 183569 : unsigned int min_bits = op_max_value.to_uhwi () + 1;
7248 :
7249 : /* As explained below, we can convert a signed shift into an
7250 : unsigned shift if the sign bit is always clear. At this
7251 : point we've already processed the ranges of the output and
7252 : the first input. */
7253 183569 : auto op_sign = sign;
7254 183569 : if (sign == SIGNED && !wi::neg_p (min_value))
7255 : op_sign = UNSIGNED;
7256 367138 : op_min_value = wide_int::from (wi::min_value (min_bits, op_sign),
7257 183569 : precision, op_sign);
7258 367138 : op_max_value = wide_int::from (wi::max_value (min_bits, op_sign),
7259 183569 : precision, op_sign);
7260 : }
7261 695804 : min_value = wi::min (min_value, op_min_value, sign);
7262 695804 : max_value = wi::max (max_value, op_max_value, sign);
7263 1111510 : }
7264 : }
7265 :
7266 : /* Try to switch signed types for unsigned types if we can.
7267 : This is better for two reasons. First, unsigned ops tend
7268 : to be cheaper than signed ops. Second, it means that we can
7269 : handle things like:
7270 :
7271 : signed char c;
7272 : int res = (int) c & 0xff00; // range [0x0000, 0xff00]
7273 :
7274 : as:
7275 :
7276 : signed char c;
7277 : unsigned short res_1 = (unsigned short) c & 0xff00;
7278 : int res = (int) res_1;
7279 :
7280 : where the intermediate result res_1 has unsigned rather than
7281 : signed type. */
7282 4001322 : if (sign == SIGNED && !wi::neg_p (min_value))
7283 : sign = UNSIGNED;
7284 :
7285 : /* See what precision is required for MIN_VALUE and MAX_VALUE. */
7286 4001322 : unsigned int precision1 = wi::min_precision (min_value, sign);
7287 4001322 : unsigned int precision2 = wi::min_precision (max_value, sign);
7288 4001322 : unsigned int value_precision = MAX (precision1, precision2);
7289 4001322 : if (value_precision >= precision)
7290 : return;
7291 :
7292 2496288 : if (dump_enabled_p ())
7293 118176 : dump_printf_loc (MSG_NOTE, vect_location, "can narrow to %s:%d"
7294 : " without loss of precision: %G",
7295 : sign == SIGNED ? "signed" : "unsigned",
7296 : value_precision, (gimple *) stmt);
7297 :
7298 2496288 : vect_set_operation_type (stmt_info, type, value_precision, sign);
7299 2496288 : vect_set_min_input_precision (stmt_info, type, value_precision);
7300 12170347 : }
7301 :
7302 : /* Use information about the users of STMT's result to decide whether
7303 : STMT (described by STMT_INFO) could be done in a narrower type.
7304 : This is effectively a backward propagation. */
7305 :
7306 : static void
7307 23093102 : vect_determine_precisions_from_users (stmt_vec_info stmt_info, gassign *stmt)
7308 : {
7309 23093102 : tree_code code = gimple_assign_rhs_code (stmt);
7310 23093102 : unsigned int opno = (code == COND_EXPR ? 2 : 1);
7311 23093102 : tree type = TREE_TYPE (gimple_op (stmt, opno));
7312 23093102 : if (!vect_narrowable_type_p (type))
7313 13323835 : return;
7314 :
7315 14368853 : unsigned int precision = TYPE_PRECISION (type);
7316 14368853 : unsigned int operation_precision, min_input_precision;
7317 14368853 : switch (code)
7318 : {
7319 2628122 : CASE_CONVERT:
7320 : /* Only the bits that contribute to the output matter. Don't change
7321 : the precision of the operation itself. */
7322 2628122 : operation_precision = precision;
7323 2628122 : min_input_precision = stmt_info->min_output_precision;
7324 2628122 : break;
7325 :
7326 493531 : case LSHIFT_EXPR:
7327 493531 : case RSHIFT_EXPR:
7328 493531 : {
7329 493531 : tree shift = gimple_assign_rhs2 (stmt);
7330 493531 : unsigned int min_const_shift, max_const_shift;
7331 493531 : wide_int min_shift, max_shift;
7332 493531 : if (TREE_CODE (shift) == SSA_NAME
7333 108655 : && vect_get_range_info (shift, &min_shift, &max_shift)
7334 83888 : && wi::ge_p (min_shift, 0, TYPE_SIGN (TREE_TYPE (shift)))
7335 574637 : && wi::lt_p (max_shift, TYPE_PRECISION (type),
7336 81106 : TYPE_SIGN (TREE_TYPE (shift))))
7337 : {
7338 73230 : min_const_shift = min_shift.to_uhwi ();
7339 73230 : max_const_shift = max_shift.to_uhwi ();
7340 : }
7341 420301 : else if (TREE_CODE (shift) == INTEGER_CST
7342 805177 : && wi::ltu_p (wi::to_widest (shift), precision))
7343 384768 : min_const_shift = max_const_shift = TREE_INT_CST_LOW (shift);
7344 : else
7345 35533 : return;
7346 457998 : if (code == LSHIFT_EXPR)
7347 : {
7348 : /* Avoid creating an undefined shift.
7349 :
7350 : ??? We could instead use min_output_precision as-is and
7351 : optimize out-of-range shifts to zero. However, only
7352 : degenerate testcases shift away all their useful input data,
7353 : and it isn't natural to drop input operations in the middle
7354 : of vectorization. This sort of thing should really be
7355 : handled before vectorization. */
7356 111196 : operation_precision = MAX (stmt_info->min_output_precision,
7357 : max_const_shift + 1);
7358 : /* We need CONST_SHIFT fewer bits of the input. */
7359 111196 : min_input_precision = (MAX (operation_precision, max_const_shift)
7360 : - min_const_shift);
7361 : }
7362 : else
7363 : {
7364 : /* We need CONST_SHIFT extra bits to do the operation. */
7365 346802 : operation_precision = (stmt_info->min_output_precision
7366 : + max_const_shift);
7367 346802 : min_input_precision = operation_precision;
7368 : }
7369 457998 : break;
7370 493531 : }
7371 :
7372 11247200 : default:
7373 11247200 : if (vect_truncatable_operation_p (code))
7374 : {
7375 : /* Input bit N has no effect on output bits N-1 and lower. */
7376 6683147 : operation_precision = stmt_info->min_output_precision;
7377 6683147 : min_input_precision = operation_precision;
7378 6683147 : break;
7379 : }
7380 : return;
7381 : }
7382 :
7383 9769267 : if (operation_precision < precision)
7384 : {
7385 120796 : if (dump_enabled_p ())
7386 2859 : dump_printf_loc (MSG_NOTE, vect_location, "can narrow to %s:%d"
7387 : " without affecting users: %G",
7388 2859 : TYPE_UNSIGNED (type) ? "unsigned" : "signed",
7389 : operation_precision, (gimple *) stmt);
7390 241592 : vect_set_operation_type (stmt_info, type, operation_precision,
7391 120796 : TYPE_SIGN (type));
7392 : }
7393 9769267 : vect_set_min_input_precision (stmt_info, type, min_input_precision);
7394 : }
7395 :
7396 : /* Return true if the statement described by STMT_INFO sets a boolean
7397 : SSA_NAME and if we know how to vectorize this kind of statement using
7398 : vector mask types. */
7399 :
7400 : static bool
7401 41341495 : possible_vector_mask_operation_p (stmt_vec_info stmt_info)
7402 : {
7403 41341495 : tree lhs = gimple_get_lhs (stmt_info->stmt);
7404 41341495 : tree_code code = ERROR_MARK;
7405 41341495 : gassign *assign = NULL;
7406 41341495 : gcond *cond = NULL;
7407 :
7408 41341495 : if ((assign = dyn_cast <gassign *> (stmt_info->stmt)))
7409 23885712 : code = gimple_assign_rhs_code (assign);
7410 17455783 : else if ((cond = dyn_cast <gcond *> (stmt_info->stmt)))
7411 : {
7412 5550885 : lhs = gimple_cond_lhs (cond);
7413 5550885 : code = gimple_cond_code (cond);
7414 : }
7415 :
7416 41341495 : if (!lhs
7417 38452582 : || TREE_CODE (lhs) != SSA_NAME
7418 74829323 : || !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)))
7419 : return false;
7420 :
7421 2255527 : if (code != ERROR_MARK)
7422 : {
7423 1993140 : switch (code)
7424 : {
7425 : CASE_CONVERT:
7426 : case SSA_NAME:
7427 : case BIT_NOT_EXPR:
7428 : case BIT_IOR_EXPR:
7429 : case BIT_XOR_EXPR:
7430 : case BIT_AND_EXPR:
7431 : return true;
7432 :
7433 1563254 : default:
7434 1563254 : return TREE_CODE_CLASS (code) == tcc_comparison;
7435 : }
7436 : }
7437 262387 : else if (is_a <gphi *> (stmt_info->stmt))
7438 147709 : return true;
7439 : return false;
7440 : }
7441 :
7442 : /* If STMT_INFO sets a boolean SSA_NAME, see whether we should use
7443 : a vector mask type instead of a normal vector type. Record the
7444 : result in STMT_INFO->mask_precision. Returns true when the
7445 : precision changed. */
7446 :
7447 : static bool
7448 41341495 : vect_determine_mask_precision (vec_info *vinfo, stmt_vec_info stmt_info)
7449 : {
7450 41341495 : if (!possible_vector_mask_operation_p (stmt_info))
7451 : return false;
7452 :
7453 : /* If at least one boolean input uses a vector mask type,
7454 : pick the mask type with the narrowest elements.
7455 :
7456 : ??? This is the traditional behavior. It should always produce
7457 : the smallest number of operations, but isn't necessarily the
7458 : optimal choice. For example, if we have:
7459 :
7460 : a = b & c
7461 :
7462 : where:
7463 :
7464 : - the user of a wants it to have a mask type for 16-bit elements (M16)
7465 : - b also uses M16
7466 : - c uses a mask type for 8-bit elements (M8)
7467 :
7468 : then picking M8 gives:
7469 :
7470 : - 1 M16->M8 pack for b
7471 : - 1 M8 AND for a
7472 : - 2 M8->M16 unpacks for the user of a
7473 :
7474 : whereas picking M16 would have given:
7475 :
7476 : - 2 M8->M16 unpacks for c
7477 : - 2 M16 ANDs for a
7478 :
7479 : The number of operations are equal, but M16 would have given
7480 : a shorter dependency chain and allowed more ILP. */
7481 2087742 : unsigned int precision = ~0U;
7482 2087742 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
7483 :
7484 : /* If the statement compares two values that shouldn't use vector masks,
7485 : try comparing the values as normal scalars instead. */
7486 2087742 : tree_code code = ERROR_MARK;
7487 2087742 : tree op0_type;
7488 2087742 : unsigned int nops = -1;
7489 2087742 : unsigned int ops_start = 0;
7490 :
7491 2087742 : if (gassign *assign = dyn_cast <gassign *> (stmt))
7492 : {
7493 1362752 : code = gimple_assign_rhs_code (assign);
7494 1362752 : op0_type = TREE_TYPE (gimple_assign_rhs1 (assign));
7495 1362752 : nops = gimple_num_ops (assign);
7496 1362752 : ops_start = 1;
7497 : }
7498 724990 : else if (gcond *cond = dyn_cast <gcond *> (stmt))
7499 : {
7500 577281 : code = gimple_cond_code (cond);
7501 577281 : op0_type = TREE_TYPE (gimple_cond_lhs (cond));
7502 577281 : nops = 2;
7503 577281 : ops_start = 0;
7504 : }
7505 :
7506 1940033 : if (code != ERROR_MARK)
7507 : {
7508 5774953 : for (unsigned int i = ops_start; i < nops; ++i)
7509 : {
7510 3834920 : tree rhs = gimple_op (stmt, i);
7511 3834920 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (rhs)))
7512 1875925 : continue;
7513 :
7514 1958995 : stmt_vec_info def_stmt_info = vinfo->lookup_def (rhs);
7515 1958995 : if (!def_stmt_info)
7516 : /* Don't let external or constant operands influence the choice.
7517 : We can convert them to whichever vector type we pick. */
7518 592218 : continue;
7519 :
7520 1366777 : if (def_stmt_info->mask_precision)
7521 : {
7522 1137929 : if (precision > def_stmt_info->mask_precision)
7523 3834920 : precision = def_stmt_info->mask_precision;
7524 : }
7525 : }
7526 :
7527 1940033 : if (precision == ~0U
7528 1563537 : && TREE_CODE_CLASS (code) == tcc_comparison)
7529 : {
7530 1354109 : scalar_mode mode;
7531 1354109 : tree vectype, mask_type;
7532 1354109 : if (is_a <scalar_mode> (TYPE_MODE (op0_type), &mode)
7533 : /* Do not allow this to set vinfo->vector_mode, this might
7534 : disrupt the result for the next iteration. */
7535 1354109 : && (vectype = get_related_vectype_for_scalar_type
7536 1625356 : (vinfo->vector_mode, op0_type))
7537 1183458 : && (mask_type = truth_type_for (vectype))
7538 1183458 : && expand_vec_cmp_expr_p (vectype, mask_type, code))
7539 1824422 : precision = GET_MODE_BITSIZE (mode);
7540 : }
7541 : }
7542 : else
7543 : {
7544 147709 : gphi *phi = as_a <gphi *> (stmt_info->stmt);
7545 759740 : for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
7546 : {
7547 464322 : tree rhs = gimple_phi_arg_def (phi, i);
7548 :
7549 464322 : stmt_vec_info def_stmt_info = vinfo->lookup_def (rhs);
7550 464322 : if (!def_stmt_info)
7551 : /* Don't let external or constant operands influence the choice.
7552 : We can convert them to whichever vector type we pick. */
7553 305846 : continue;
7554 :
7555 158476 : if (def_stmt_info->mask_precision)
7556 : {
7557 132718 : if (precision > def_stmt_info->mask_precision)
7558 464322 : precision = def_stmt_info->mask_precision;
7559 : }
7560 : }
7561 : }
7562 :
7563 2087742 : if (stmt_info->mask_precision != precision)
7564 : {
7565 1957358 : if (dump_enabled_p ())
7566 : {
7567 8262 : if (precision == ~0U)
7568 1825 : dump_printf_loc (MSG_NOTE, vect_location,
7569 : "using normal nonmask vectors for %G",
7570 : stmt_info->stmt);
7571 : else
7572 6437 : dump_printf_loc (MSG_NOTE, vect_location,
7573 : "using boolean precision %d for %G",
7574 : precision, stmt_info->stmt);
7575 : }
7576 :
7577 : /* ??? We'd like to assert stmt_info->mask_precision == 0
7578 : || stmt_info->mask_precision > precision, thus that we only
7579 : decrease mask precisions throughout iteration, but the
7580 : tcc_comparison handling above means for comparisons of bools
7581 : we start with 8 but might increase in case the bools get mask
7582 : precision on their own. */
7583 1957358 : stmt_info->mask_precision = precision;
7584 1957358 : return true;
7585 : }
7586 : return false;
7587 : }
7588 :
7589 : /* Handle vect_determine_precisions for STMT_INFO, given that we
7590 : have already done so for the users of its result. */
7591 :
7592 : void
7593 40232395 : vect_determine_stmt_precisions (vec_info *vinfo, stmt_vec_info stmt_info)
7594 : {
7595 40232395 : vect_determine_min_output_precision (vinfo, stmt_info);
7596 40232395 : if (gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt))
7597 : {
7598 23093102 : vect_determine_precisions_from_range (stmt_info, stmt);
7599 23093102 : vect_determine_precisions_from_users (stmt_info, stmt);
7600 : }
7601 40232395 : }
7602 :
7603 : /* Walk backwards through the vectorizable region to determine the
7604 : values of these fields:
7605 :
7606 : - min_output_precision
7607 : - min_input_precision
7608 : - operation_precision
7609 : - operation_sign. */
7610 :
7611 : void
7612 1142237 : vect_determine_precisions (vec_info *vinfo)
7613 : {
7614 1142237 : basic_block *bbs = vinfo->bbs;
7615 1142237 : unsigned int nbbs = vinfo->nbbs;
7616 :
7617 1163054 : DUMP_VECT_SCOPE ("vect_determine_precisions");
7618 :
7619 : /* For mask precisions we have to iterate since otherwise we do not
7620 : get reduction PHI precision correct. For now do this only for
7621 : loop vectorization. */
7622 1213984 : bool changed;
7623 1213984 : do
7624 : {
7625 1213984 : changed = false;
7626 13507012 : for (unsigned int i = 0; i < nbbs; i++)
7627 : {
7628 12293028 : basic_block bb = bbs[i];
7629 12293028 : for (auto gsi = gsi_start_phis (bb);
7630 19927835 : !gsi_end_p (gsi); gsi_next (&gsi))
7631 : {
7632 7634807 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi.phi ());
7633 7634807 : if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
7634 7443784 : changed |= vect_determine_mask_precision (vinfo, stmt_info);
7635 : }
7636 131625798 : for (auto gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
7637 : {
7638 107039742 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (gsi));
7639 107039742 : if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
7640 33897711 : changed |= vect_determine_mask_precision (vinfo, stmt_info);
7641 : }
7642 : }
7643 : }
7644 2356221 : while (changed && is_a <loop_vec_info> (vinfo));
7645 :
7646 13270236 : for (unsigned int i = 0; i < nbbs; i++)
7647 : {
7648 12127999 : basic_block bb = bbs[nbbs - i - 1];
7649 139107030 : for (auto gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
7650 : {
7651 105354943 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (gsi));
7652 105354943 : if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
7653 32996364 : vect_determine_stmt_precisions (vinfo, stmt_info);
7654 : }
7655 19555053 : for (auto gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
7656 : {
7657 7427054 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi.phi ());
7658 7427054 : if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
7659 7236031 : vect_determine_stmt_precisions (vinfo, stmt_info);
7660 : }
7661 : }
7662 1142237 : }
7663 :
7664 : typedef gimple *(*vect_recog_func_ptr) (vec_info *, stmt_vec_info, tree *);
7665 :
7666 : struct vect_recog_func
7667 : {
7668 : vect_recog_func_ptr fn;
7669 : const char *name;
7670 : };
7671 :
7672 : /* Note that ordering matters - the first pattern matching on a stmt is
7673 : taken which means usually the more complex one needs to precede the
7674 : less comples onex (widen_sum only after dot_prod or sad for example). */
7675 : static vect_recog_func vect_vect_recog_func_ptrs[] = {
7676 : { vect_recog_bitfield_ref_pattern, "bitfield_ref" },
7677 : { vect_recog_bit_insert_pattern, "bit_insert" },
7678 : { vect_recog_abd_pattern, "abd" },
7679 : { vect_recog_over_widening_pattern, "over_widening" },
7680 : /* Must come after over_widening, which narrows the shift as much as
7681 : possible beforehand. */
7682 : { vect_recog_average_pattern, "average" },
7683 : { vect_recog_cond_expr_convert_pattern, "cond_expr_convert" },
7684 : { vect_recog_mulhs_pattern, "mult_high" },
7685 : { vect_recog_cast_forwprop_pattern, "cast_forwprop" },
7686 : { vect_recog_widen_mult_pattern, "widen_mult" },
7687 : { vect_recog_dot_prod_pattern, "dot_prod" },
7688 : { vect_recog_sad_pattern, "sad" },
7689 : { vect_recog_widen_sum_pattern, "widen_sum" },
7690 : { vect_recog_pow_pattern, "pow" },
7691 : { vect_recog_popcount_clz_ctz_ffs_pattern, "popcount_clz_ctz_ffs" },
7692 : { vect_recog_ctz_ffs_pattern, "ctz_ffs" },
7693 : { vect_recog_widen_shift_pattern, "widen_shift" },
7694 : { vect_recog_rotate_pattern, "rotate" },
7695 : { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" },
7696 : { vect_recog_divmod_pattern, "divmod" },
7697 : { vect_recog_mod_var_pattern, "modvar" },
7698 : { vect_recog_mult_pattern, "mult" },
7699 : { vect_recog_sat_add_pattern, "sat_add" },
7700 : { vect_recog_sat_sub_pattern, "sat_sub" },
7701 : { vect_recog_sat_trunc_pattern, "sat_trunc" },
7702 : { vect_recog_gcond_pattern, "gcond" },
7703 : { vect_recog_bool_pattern, "bool" },
7704 : /* This must come before mask conversion, and includes the parts
7705 : of mask conversion that are needed for gather and scatter
7706 : internal functions. */
7707 : { vect_recog_gather_scatter_pattern, "gather_scatter" },
7708 : { vect_recog_cond_store_pattern, "cond_store" },
7709 : { vect_recog_mask_conversion_pattern, "mask_conversion" },
7710 : { vect_recog_widen_plus_pattern, "widen_plus" },
7711 : { vect_recog_widen_minus_pattern, "widen_minus" },
7712 : { vect_recog_widen_abd_pattern, "widen_abd" },
7713 : /* These must come after the double widening ones. */
7714 : };
7715 :
7716 : /* Mark statements that are involved in a pattern. */
7717 :
7718 : void
7719 1104520 : vect_mark_pattern_stmts (vec_info *vinfo,
7720 : stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
7721 : tree pattern_vectype)
7722 : {
7723 1104520 : stmt_vec_info orig_stmt_info_saved = orig_stmt_info;
7724 1104520 : gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
7725 :
7726 1104520 : gimple *orig_pattern_stmt = NULL;
7727 1104520 : if (is_pattern_stmt_p (orig_stmt_info))
7728 : {
7729 : /* We're replacing a statement in an existing pattern definition
7730 : sequence. */
7731 11894 : orig_pattern_stmt = orig_stmt_info->stmt;
7732 11894 : if (dump_enabled_p ())
7733 993 : dump_printf_loc (MSG_NOTE, vect_location,
7734 : "replacing earlier pattern %G", orig_pattern_stmt);
7735 :
7736 : /* To keep the book-keeping simple, just swap the lhs of the
7737 : old and new statements, so that the old one has a valid but
7738 : unused lhs. */
7739 11894 : tree old_lhs = gimple_get_lhs (orig_pattern_stmt);
7740 11894 : gimple_set_lhs (orig_pattern_stmt, gimple_get_lhs (pattern_stmt));
7741 11894 : gimple_set_lhs (pattern_stmt, old_lhs);
7742 :
7743 11894 : if (dump_enabled_p ())
7744 993 : dump_printf_loc (MSG_NOTE, vect_location, "with %G", pattern_stmt);
7745 :
7746 : /* Switch to the statement that ORIG replaces. */
7747 11894 : orig_stmt_info = STMT_VINFO_RELATED_STMT (orig_stmt_info);
7748 :
7749 : /* We shouldn't be replacing the main pattern statement. */
7750 11894 : gcc_assert (STMT_VINFO_RELATED_STMT (orig_stmt_info)->stmt
7751 : != orig_pattern_stmt);
7752 : }
7753 :
7754 1104520 : if (def_seq)
7755 2187740 : for (gimple_stmt_iterator si = gsi_start (def_seq);
7756 2187740 : !gsi_end_p (si); gsi_next (&si))
7757 : {
7758 1281352 : if (dump_enabled_p ())
7759 28510 : dump_printf_loc (MSG_NOTE, vect_location,
7760 : "extra pattern stmt: %G", gsi_stmt (si));
7761 1281352 : stmt_vec_info pattern_stmt_info
7762 1281352 : = vect_init_pattern_stmt (vinfo, gsi_stmt (si),
7763 : orig_stmt_info, pattern_vectype);
7764 : /* Stmts in the def sequence are not vectorizable cycle or
7765 : induction defs, instead they should all be vect_internal_def
7766 : feeding the main pattern stmt which retains this def type. */
7767 1281352 : STMT_VINFO_DEF_TYPE (pattern_stmt_info) = vect_internal_def;
7768 : }
7769 :
7770 1104520 : if (orig_pattern_stmt)
7771 : {
7772 11894 : vect_init_pattern_stmt (vinfo, pattern_stmt,
7773 : orig_stmt_info, pattern_vectype);
7774 :
7775 : /* Insert all the new pattern statements before the original one. */
7776 11894 : gimple_seq *orig_def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
7777 11894 : gimple_stmt_iterator gsi = gsi_for_stmt (orig_pattern_stmt,
7778 : orig_def_seq);
7779 11894 : gsi_insert_seq_before_without_update (&gsi, def_seq, GSI_SAME_STMT);
7780 11894 : gsi_insert_before_without_update (&gsi, pattern_stmt, GSI_SAME_STMT);
7781 :
7782 : /* Remove the pattern statement that this new pattern replaces. */
7783 11894 : gsi_remove (&gsi, false);
7784 : }
7785 : else
7786 1092626 : vect_set_pattern_stmt (vinfo,
7787 : pattern_stmt, orig_stmt_info, pattern_vectype);
7788 :
7789 : /* For any conditionals mark them as vect_condition_def. */
7790 1104520 : if (is_a <gcond *> (pattern_stmt))
7791 390721 : STMT_VINFO_DEF_TYPE (STMT_VINFO_RELATED_STMT (orig_stmt_info)) = vect_condition_def;
7792 :
7793 : /* Transfer reduction path info to the pattern. */
7794 1104520 : if (STMT_VINFO_REDUC_IDX (orig_stmt_info_saved) != -1)
7795 : {
7796 16317 : gimple_match_op op;
7797 16317 : if (!gimple_extract_op (orig_stmt_info_saved->stmt, &op))
7798 0 : gcc_unreachable ();
7799 16317 : tree lookfor = op.ops[STMT_VINFO_REDUC_IDX (orig_stmt_info)];
7800 : /* Search the pattern def sequence and the main pattern stmt. Note
7801 : we may have inserted all into a containing pattern def sequence
7802 : so the following is a bit awkward. */
7803 16317 : gimple_stmt_iterator si;
7804 16317 : gimple *s;
7805 16317 : if (def_seq)
7806 : {
7807 15110 : si = gsi_start (def_seq);
7808 15110 : s = gsi_stmt (si);
7809 15110 : gsi_next (&si);
7810 : }
7811 : else
7812 : {
7813 : si = gsi_none ();
7814 16317 : s = pattern_stmt;
7815 : }
7816 34432 : do
7817 : {
7818 34432 : bool found = false;
7819 34432 : if (gimple_extract_op (s, &op))
7820 : {
7821 84038 : for (unsigned i = 0; i < op.num_ops; ++i)
7822 65917 : if (op.ops[i] == lookfor)
7823 : {
7824 16311 : STMT_VINFO_REDUC_IDX (vinfo->lookup_stmt (s)) = i;
7825 16311 : lookfor = gimple_get_lhs (s);
7826 16311 : found = true;
7827 16311 : break;
7828 : }
7829 : /* Try harder to find a mid-entry into an earlier pattern
7830 : sequence. Likewise an entry to a stmt skipping a conversion
7831 : on an input. This means that the initial 'lookfor' was
7832 : bogus. */
7833 16311 : if (!found)
7834 : {
7835 39436 : for (unsigned i = 0; i < op.num_ops; ++i)
7836 21321 : if (TREE_CODE (op.ops[i]) == SSA_NAME)
7837 18121 : if (auto def = vinfo->lookup_def (op.ops[i]))
7838 17927 : if (vect_is_reduction (def)
7839 17927 : || (is_a <gphi *> (def->stmt)
7840 6 : && STMT_VINFO_REDUC_DEF (def) != NULL))
7841 : {
7842 6 : STMT_VINFO_REDUC_IDX (vinfo->lookup_stmt (s)) = i;
7843 6 : lookfor = gimple_get_lhs (s);
7844 6 : found = true;
7845 6 : break;
7846 : }
7847 : }
7848 : }
7849 34432 : if (s == pattern_stmt)
7850 : {
7851 16317 : if (!found && dump_enabled_p ())
7852 0 : dump_printf_loc (MSG_NOTE, vect_location,
7853 : "failed to update reduction index.\n");
7854 16317 : break;
7855 : }
7856 18115 : if (gsi_end_p (si))
7857 : s = pattern_stmt;
7858 : else
7859 : {
7860 3005 : s = gsi_stmt (si);
7861 3005 : if (s == pattern_stmt)
7862 : /* Found the end inside a bigger pattern def seq. */
7863 : si = gsi_none ();
7864 : else
7865 3005 : gsi_next (&si);
7866 : }
7867 : } while (1);
7868 : }
7869 1104520 : }
7870 :
7871 : /* Function vect_pattern_recog_1
7872 :
7873 : Input:
7874 : PATTERN_RECOG_FUNC: A pointer to a function that detects a certain
7875 : computation pattern.
7876 : STMT_INFO: A stmt from which the pattern search should start.
7877 :
7878 : If PATTERN_RECOG_FUNC successfully detected the pattern, it creates
7879 : a sequence of statements that has the same functionality and can be
7880 : used to replace STMT_INFO. It returns the last statement in the sequence
7881 : and adds any earlier statements to STMT_INFO's STMT_VINFO_PATTERN_DEF_SEQ.
7882 : PATTERN_RECOG_FUNC also sets *TYPE_OUT to the vector type of the final
7883 : statement, having first checked that the target supports the new operation
7884 : in that type.
7885 :
7886 : This function also does some bookkeeping, as explained in the documentation
7887 : for vect_recog_pattern. */
7888 :
7889 : static void
7890 1073279864 : vect_pattern_recog_1 (vec_info *vinfo,
7891 : const vect_recog_func &recog_func, stmt_vec_info stmt_info)
7892 : {
7893 1073279864 : gimple *pattern_stmt;
7894 1073279864 : tree pattern_vectype;
7895 :
7896 : /* If this statement has already been replaced with pattern statements,
7897 : leave the original statement alone, since the first match wins.
7898 : Instead try to match against the definition statements that feed
7899 : the main pattern statement. */
7900 1073279864 : if (STMT_VINFO_IN_PATTERN_P (stmt_info))
7901 : {
7902 14038424 : gimple_stmt_iterator gsi;
7903 14038424 : for (gsi = gsi_start (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
7904 31434640 : !gsi_end_p (gsi); gsi_next (&gsi))
7905 17396216 : vect_pattern_recog_1 (vinfo, recog_func,
7906 : vinfo->lookup_stmt (gsi_stmt (gsi)));
7907 : return;
7908 : }
7909 :
7910 1059241440 : gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
7911 1059241440 : pattern_stmt = recog_func.fn (vinfo, stmt_info, &pattern_vectype);
7912 1059241440 : if (!pattern_stmt)
7913 : {
7914 : /* Clear any half-formed pattern definition sequence. */
7915 1058136920 : STMT_VINFO_PATTERN_DEF_SEQ (stmt_info) = NULL;
7916 1058136920 : return;
7917 : }
7918 :
7919 : /* Found a vectorizable pattern. */
7920 1104520 : if (dump_enabled_p ())
7921 21151 : dump_printf_loc (MSG_NOTE, vect_location,
7922 : "%s pattern recognized: %G",
7923 21151 : recog_func.name, pattern_stmt);
7924 :
7925 : /* Mark the stmts that are involved in the pattern. */
7926 1104520 : vect_mark_pattern_stmts (vinfo, stmt_info, pattern_stmt, pattern_vectype);
7927 : }
7928 :
7929 :
7930 : /* Function vect_pattern_recog
7931 :
7932 : Input:
7933 : LOOP_VINFO - a struct_loop_info of a loop in which we want to look for
7934 : computation idioms.
7935 :
7936 : Output - for each computation idiom that is detected we create a new stmt
7937 : that provides the same functionality and that can be vectorized. We
7938 : also record some information in the struct_stmt_info of the relevant
7939 : stmts, as explained below:
7940 :
7941 : At the entry to this function we have the following stmts, with the
7942 : following initial value in the STMT_VINFO fields:
7943 :
7944 : stmt in_pattern_p related_stmt vec_stmt
7945 : S1: a_i = .... - - -
7946 : S2: a_2 = ..use(a_i).. - - -
7947 : S3: a_1 = ..use(a_2).. - - -
7948 : S4: a_0 = ..use(a_1).. - - -
7949 : S5: ... = ..use(a_0).. - - -
7950 :
7951 : Say the sequence {S1,S2,S3,S4} was detected as a pattern that can be
7952 : represented by a single stmt. We then:
7953 : - create a new stmt S6 equivalent to the pattern (the stmt is not
7954 : inserted into the code)
7955 : - fill in the STMT_VINFO fields as follows:
7956 :
7957 : in_pattern_p related_stmt vec_stmt
7958 : S1: a_i = .... - - -
7959 : S2: a_2 = ..use(a_i).. - - -
7960 : S3: a_1 = ..use(a_2).. - - -
7961 : S4: a_0 = ..use(a_1).. true S6 -
7962 : '---> S6: a_new = .... - S4 -
7963 : S5: ... = ..use(a_0).. - - -
7964 :
7965 : (the last stmt in the pattern (S4) and the new pattern stmt (S6) point
7966 : to each other through the RELATED_STMT field).
7967 :
7968 : S6 will be marked as relevant in vect_mark_stmts_to_be_vectorized instead
7969 : of S4 because it will replace all its uses. Stmts {S1,S2,S3} will
7970 : remain irrelevant unless used by stmts other than S4.
7971 :
7972 : If vectorization succeeds, vect_transform_stmt will skip over {S1,S2,S3}
7973 : (because they are marked as irrelevant). It will vectorize S6, and record
7974 : a pointer to the new vector stmt VS6 from S6 (as usual).
7975 : S4 will be skipped, and S5 will be vectorized as usual:
7976 :
7977 : in_pattern_p related_stmt vec_stmt
7978 : S1: a_i = .... - - -
7979 : S2: a_2 = ..use(a_i).. - - -
7980 : S3: a_1 = ..use(a_2).. - - -
7981 : > VS6: va_new = .... - - -
7982 : S4: a_0 = ..use(a_1).. true S6 VS6
7983 : '---> S6: a_new = .... - S4 VS6
7984 : > VS5: ... = ..vuse(va_new).. - - -
7985 : S5: ... = ..use(a_0).. - - -
7986 :
7987 : DCE could then get rid of {S1,S2,S3,S4,S5} (if their defs are not used
7988 : elsewhere), and we'll end up with:
7989 :
7990 : VS6: va_new = ....
7991 : VS5: ... = ..vuse(va_new)..
7992 :
7993 : In case of more than one pattern statements, e.g., widen-mult with
7994 : intermediate type:
7995 :
7996 : S1 a_t = ;
7997 : S2 a_T = (TYPE) a_t;
7998 : '--> S3: a_it = (interm_type) a_t;
7999 : S4 prod_T = a_T * CONST;
8000 : '--> S5: prod_T' = a_it w* CONST;
8001 :
8002 : there may be other users of a_T outside the pattern. In that case S2 will
8003 : be marked as relevant (as well as S3), and both S2 and S3 will be analyzed
8004 : and vectorized. The vector stmt VS2 will be recorded in S2, and VS3 will
8005 : be recorded in S3. */
8006 :
8007 : void
8008 1142237 : vect_pattern_recog (vec_info *vinfo)
8009 : {
8010 1142237 : basic_block *bbs = vinfo->bbs;
8011 1142237 : unsigned int nbbs = vinfo->nbbs;
8012 :
8013 1142237 : vect_determine_precisions (vinfo);
8014 :
8015 1142237 : DUMP_VECT_SCOPE ("vect_pattern_recog");
8016 :
8017 : /* Scan through the stmts in the region, applying the pattern recognition
8018 : functions starting at each stmt visited. */
8019 13270236 : for (unsigned i = 0; i < nbbs; i++)
8020 : {
8021 12127999 : basic_block bb = bbs[i];
8022 :
8023 129610941 : for (auto si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
8024 : {
8025 105354943 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (si));
8026 :
8027 105354943 : if (!stmt_info || !STMT_VINFO_VECTORIZABLE (stmt_info))
8028 72358579 : continue;
8029 :
8030 : /* Scan over all generic vect_recog_xxx_pattern functions. */
8031 1088880012 : for (const auto &func_ptr : vect_vect_recog_func_ptrs)
8032 1055883648 : vect_pattern_recog_1 (vinfo, func_ptr,
8033 : stmt_info);
8034 : }
8035 : }
8036 :
8037 : /* After this no more add_stmt calls are allowed. */
8038 1142237 : vinfo->stmt_vec_info_ro = true;
8039 1142237 : }
8040 :
8041 : /* Build a GIMPLE_ASSIGN or GIMPLE_CALL with the tree_code,
8042 : or internal_fn contained in ch, respectively. */
8043 : gimple *
8044 171687 : vect_gimple_build (tree lhs, code_helper ch, tree op0, tree op1)
8045 : {
8046 171687 : gcc_assert (op0 != NULL_TREE);
8047 171687 : if (ch.is_tree_code ())
8048 171687 : return gimple_build_assign (lhs, (tree_code) ch, op0, op1);
8049 :
8050 0 : gcc_assert (ch.is_internal_fn ());
8051 0 : gimple* stmt = gimple_build_call_internal (as_internal_fn ((combined_fn) ch),
8052 : op1 == NULL_TREE ? 1 : 2,
8053 : op0, op1);
8054 0 : gimple_call_set_lhs (stmt, lhs);
8055 0 : return stmt;
8056 : }
|