Line data Source code
1 : /* Analysis Utilities for Loop Vectorization.
2 : Copyright (C) 2006-2026 Free Software Foundation, Inc.
3 : Contributed by Dorit Nuzman <dorit@il.ibm.com>
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "gimple-iterator.h"
29 : #include "gimple-fold.h"
30 : #include "ssa.h"
31 : #include "expmed.h"
32 : #include "optabs-tree.h"
33 : #include "insn-config.h"
34 : #include "recog.h" /* FIXME: for insn_data */
35 : #include "fold-const.h"
36 : #include "stor-layout.h"
37 : #include "tree-eh.h"
38 : #include "gimplify.h"
39 : #include "gimple-iterator.h"
40 : #include "gimple-fold.h"
41 : #include "gimplify-me.h"
42 : #include "cfgloop.h"
43 : #include "tree-vectorizer.h"
44 : #include "dumpfile.h"
45 : #include "builtins.h"
46 : #include "internal-fn.h"
47 : #include "case-cfn-macros.h"
48 : #include "fold-const-call.h"
49 : #include "attribs.h"
50 : #include "cgraph.h"
51 : #include "omp-simd-clone.h"
52 : #include "predict.h"
53 : #include "tree-vector-builder.h"
54 : #include "tree-ssa-loop-ivopts.h"
55 : #include "vec-perm-indices.h"
56 : #include "gimple-range.h"
57 : #include "alias.h"
58 :
59 :
60 : /* TODO: Note the vectorizer still builds COND_EXPRs with GENERIC compares
61 : in the first operand. Disentangling this is future work, the
62 : IL is properly transferred to VEC_COND_EXPRs with separate compares. */
63 :
64 :
65 : /* Return true if we have a useful VR_RANGE range for VAR, storing it
66 : in *MIN_VALUE and *MAX_VALUE if so. Note the range in the dump files. */
67 :
68 : bool
69 12620512 : vect_get_range_info (tree var, wide_int *min_value, wide_int *max_value)
70 : {
71 12620512 : int_range_max vr;
72 12620512 : tree vr_min, vr_max;
73 25241024 : get_range_query (cfun)->range_of_expr (vr, var);
74 12620512 : if (vr.undefined_p ())
75 53 : vr.set_varying (TREE_TYPE (var));
76 12620512 : value_range_kind vr_type = get_legacy_range (vr, vr_min, vr_max);
77 12620512 : *min_value = wi::to_wide (vr_min);
78 12620512 : *max_value = wi::to_wide (vr_max);
79 12620512 : wide_int nonzero = get_nonzero_bits (var);
80 12620512 : signop sgn = TYPE_SIGN (TREE_TYPE (var));
81 12620512 : if (intersect_range_with_nonzero_bits (vr_type, min_value, max_value,
82 : nonzero, sgn) == VR_RANGE)
83 : {
84 6186070 : if (dump_enabled_p ())
85 : {
86 90648 : dump_generic_expr_loc (MSG_NOTE, vect_location, TDF_SLIM, var);
87 90648 : dump_printf (MSG_NOTE, " has range [");
88 90648 : dump_hex (MSG_NOTE, *min_value);
89 90648 : dump_printf (MSG_NOTE, ", ");
90 90648 : dump_hex (MSG_NOTE, *max_value);
91 90648 : dump_printf (MSG_NOTE, "]\n");
92 : }
93 : return true;
94 : }
95 : else
96 : {
97 6434442 : if (dump_enabled_p ())
98 : {
99 68738 : dump_generic_expr_loc (MSG_NOTE, vect_location, TDF_SLIM, var);
100 68738 : dump_printf (MSG_NOTE, " has no range info\n");
101 : }
102 : return false;
103 : }
104 12620512 : }
105 :
106 : /* Report that we've found an instance of pattern PATTERN in
107 : statement STMT. */
108 :
109 : static void
110 1328601 : vect_pattern_detected (const char *name, gimple *stmt)
111 : {
112 1328601 : if (dump_enabled_p ())
113 26326 : dump_printf_loc (MSG_NOTE, vect_location, "%s: detected: %G", name, stmt);
114 1328601 : }
115 :
116 : /* Associate pattern statement PATTERN_STMT with ORIG_STMT_INFO and
117 : return the pattern statement's stmt_vec_info. Set its vector type to
118 : VECTYPE if it doesn't have one already. */
119 :
120 : static stmt_vec_info
121 2515146 : vect_init_pattern_stmt (vec_info *vinfo, gimple *pattern_stmt,
122 : stmt_vec_info orig_stmt_info, tree vectype)
123 : {
124 2515146 : stmt_vec_info pattern_stmt_info = vinfo->lookup_stmt (pattern_stmt);
125 2515146 : if (pattern_stmt_info == NULL)
126 1330582 : pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
127 2515146 : gimple_set_bb (pattern_stmt, gimple_bb (orig_stmt_info->stmt));
128 :
129 2515146 : pattern_stmt_info->pattern_stmt_p = true;
130 2515146 : STMT_VINFO_RELATED_STMT (pattern_stmt_info) = orig_stmt_info;
131 2515146 : STMT_VINFO_DEF_TYPE (pattern_stmt_info)
132 2515146 : = STMT_VINFO_DEF_TYPE (orig_stmt_info);
133 2515146 : if (!STMT_VINFO_VECTYPE (pattern_stmt_info))
134 : {
135 2218483 : gcc_assert (!vectype
136 : || is_a <gcond *> (pattern_stmt)
137 : || (VECTOR_BOOLEAN_TYPE_P (vectype)
138 : == vect_use_mask_type_p (orig_stmt_info)));
139 1341116 : STMT_VINFO_VECTYPE (pattern_stmt_info) = vectype;
140 1341116 : pattern_stmt_info->mask_precision = orig_stmt_info->mask_precision;
141 1341116 : if (dump_enabled_p ())
142 : {
143 26464 : if (pattern_stmt_info->mask_precision == ~0U)
144 63 : dump_printf_loc (MSG_NOTE, vect_location,
145 : "using normal nonmask vectors for %G",
146 : pattern_stmt_info->stmt);
147 : else
148 26401 : dump_printf_loc (MSG_NOTE, vect_location,
149 : "using boolean precision %d for %G",
150 : pattern_stmt_info->mask_precision,
151 : pattern_stmt_info->stmt);
152 : }
153 : }
154 2515146 : return pattern_stmt_info;
155 : }
156 :
157 : /* Set the pattern statement of ORIG_STMT_INFO to PATTERN_STMT.
158 : Also set the vector type of PATTERN_STMT to VECTYPE, if it doesn't
159 : have one already. */
160 :
161 : static void
162 1076129 : vect_set_pattern_stmt (vec_info *vinfo, gimple *pattern_stmt,
163 : stmt_vec_info orig_stmt_info, tree vectype)
164 : {
165 1076129 : STMT_VINFO_IN_PATTERN_P (orig_stmt_info) = true;
166 1076129 : STMT_VINFO_RELATED_STMT (orig_stmt_info)
167 0 : = vect_init_pattern_stmt (vinfo, pattern_stmt, orig_stmt_info, vectype);
168 0 : }
169 :
170 : /* Add NEW_STMT to STMT_INFO's pattern definition statements. If VECTYPE
171 : is nonnull, record that NEW_STMT's vector type is VECTYPE, which might
172 : be different from the vector type of the final pattern statement.
173 : If VECTYPE is a mask type, SCALAR_TYPE_FOR_MASK is the scalar type
174 : from which it was derived. */
175 :
176 : static inline void
177 1400229 : append_pattern_def_seq (vec_info *vinfo,
178 : stmt_vec_info stmt_info, gimple *new_stmt,
179 : tree vectype = NULL_TREE,
180 : tree scalar_type_for_mask = NULL_TREE)
181 : {
182 2122421 : gcc_assert (!scalar_type_for_mask
183 : == (!vectype || !VECTOR_BOOLEAN_TYPE_P (vectype)));
184 1400229 : if (vectype)
185 : {
186 1174248 : stmt_vec_info new_stmt_info = vinfo->add_stmt (new_stmt);
187 1174248 : STMT_VINFO_VECTYPE (new_stmt_info) = vectype;
188 1174248 : if (scalar_type_for_mask)
189 : {
190 678037 : new_stmt_info->mask_precision
191 678037 : = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (scalar_type_for_mask));
192 678037 : if (dump_enabled_p ())
193 : {
194 6249 : if (new_stmt_info->mask_precision == ~0U)
195 : dump_printf_loc (MSG_NOTE, vect_location,
196 : "using normal nonmask vectors for %G",
197 : new_stmt_info->stmt);
198 : else
199 6249 : dump_printf_loc (MSG_NOTE, vect_location,
200 : "using boolean precision %d for %G",
201 : new_stmt_info->mask_precision,
202 : new_stmt_info->stmt);
203 : }
204 : }
205 : }
206 1400229 : gimple_seq_add_stmt_without_update (&STMT_VINFO_PATTERN_DEF_SEQ (stmt_info),
207 : new_stmt);
208 1400229 : }
209 :
210 :
211 : /* Add NEW_STMT to VINFO's invariant pattern definition statements. These
212 : statements are not vectorized but are materialized as scalar in the loop
213 : preheader. */
214 :
215 : static inline void
216 1383 : append_inv_pattern_def_seq (vec_info *vinfo, gimple *new_stmt)
217 : {
218 1383 : gimple_seq_add_stmt_without_update (&vinfo->inv_pattern_def_seq, new_stmt);
219 : }
220 :
221 : /* The caller wants to perform new operations on vect_external variable
222 : VAR, so that the result of the operations would also be vect_external.
223 : Return the edge on which the operations can be performed, if one exists.
224 : Return null if the operations should instead be treated as part of
225 : the pattern that needs them. */
226 :
227 : static edge
228 8611 : vect_get_external_def_edge (vec_info *vinfo, tree var)
229 : {
230 8611 : edge e = NULL;
231 8611 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
232 : {
233 769 : e = loop_preheader_edge (loop_vinfo->loop);
234 769 : if (!SSA_NAME_IS_DEFAULT_DEF (var))
235 : {
236 573 : basic_block bb = gimple_bb (SSA_NAME_DEF_STMT (var));
237 573 : if (bb == NULL
238 573 : || !dominated_by_p (CDI_DOMINATORS, e->dest, bb))
239 : e = NULL;
240 : }
241 : }
242 8611 : return e;
243 : }
244 :
245 : /* Return true if the target supports a vector version of CODE,
246 : where CODE is known to map to a direct optab with the given SUBTYPE.
247 : ITYPE specifies the type of (some of) the scalar inputs and OTYPE
248 : specifies the type of the scalar result.
249 :
250 : If CODE allows the inputs and outputs to have different type
251 : (such as for WIDEN_SUM_EXPR), it is the input mode rather
252 : than the output mode that determines the appropriate target pattern.
253 : Operand 0 of the target pattern then specifies the mode that the output
254 : must have.
255 :
256 : When returning true, set *VECOTYPE_OUT to the vector version of OTYPE.
257 : Also set *VECITYPE_OUT to the vector version of ITYPE if VECITYPE_OUT
258 : is nonnull. */
259 :
260 : static bool
261 865 : vect_supportable_direct_optab_p (vec_info *vinfo, tree otype, tree_code code,
262 : tree itype, tree *vecotype_out,
263 : tree *vecitype_out = NULL,
264 : enum optab_subtype subtype = optab_default)
265 : {
266 865 : tree vecitype = get_vectype_for_scalar_type (vinfo, itype);
267 865 : if (!vecitype)
268 : return false;
269 :
270 863 : tree vecotype = get_vectype_for_scalar_type (vinfo, otype);
271 863 : if (!vecotype)
272 : return false;
273 :
274 863 : optab optab = optab_for_tree_code (code, vecitype, subtype);
275 863 : if (!optab)
276 : return false;
277 :
278 863 : insn_code icode = optab_handler (optab, TYPE_MODE (vecitype));
279 863 : if (icode == CODE_FOR_nothing
280 863 : || insn_data[icode].operand[0].mode != TYPE_MODE (vecotype))
281 : return false;
282 :
283 469 : *vecotype_out = vecotype;
284 469 : if (vecitype_out)
285 460 : *vecitype_out = vecitype;
286 : return true;
287 : }
288 :
289 : /* Return true if the target supports a vector version of CODE,
290 : where CODE is known to map to a conversion optab with the given SUBTYPE.
291 : ITYPE specifies the type of (some of) the scalar inputs and OTYPE
292 : specifies the type of the scalar result.
293 :
294 : When returning true, set *VECOTYPE_OUT to the vector version of OTYPE.
295 : Also set *VECITYPE_OUT to the vector version of ITYPE if VECITYPE_OUT
296 : is nonnull. */
297 :
298 : static bool
299 5660 : vect_supportable_conv_optab_p (vec_info *vinfo, tree otype, tree_code code,
300 : tree itype, tree *vecotype_out,
301 : tree *vecitype_out = NULL,
302 : enum optab_subtype subtype = optab_default)
303 : {
304 5660 : tree vecitype = get_vectype_for_scalar_type (vinfo, itype);
305 5660 : tree vecotype = get_vectype_for_scalar_type (vinfo, otype);
306 5660 : if (!vecitype || !vecotype)
307 : return false;
308 :
309 5136 : if (!directly_supported_p (code, vecotype, vecitype, subtype))
310 : return false;
311 :
312 732 : *vecotype_out = vecotype;
313 732 : if (vecitype_out)
314 732 : *vecitype_out = vecitype;
315 : return true;
316 : }
317 :
318 : /* Round bit precision PRECISION up to a full element. */
319 :
320 : static unsigned int
321 3299929 : vect_element_precision (unsigned int precision)
322 : {
323 0 : precision = 1 << ceil_log2 (precision);
324 4958926 : return MAX (precision, BITS_PER_UNIT);
325 : }
326 :
327 : /* If OP is defined by a statement that's being considered for vectorization,
328 : return information about that statement, otherwise return NULL. */
329 :
330 : static stmt_vec_info
331 376081 : vect_get_internal_def (vec_info *vinfo, tree op)
332 : {
333 376081 : stmt_vec_info def_stmt_info = vinfo->lookup_def (op);
334 376081 : if (def_stmt_info
335 362318 : && STMT_VINFO_DEF_TYPE (def_stmt_info) == vect_internal_def)
336 344295 : return vect_stmt_to_vectorize (def_stmt_info);
337 : return NULL;
338 : }
339 :
340 : /* Holds information about an input operand after some sign changes
341 : and type promotions have been peeled away. */
342 : class vect_unpromoted_value {
343 : public:
344 : vect_unpromoted_value ();
345 :
346 : void set_op (tree, vect_def_type, stmt_vec_info = NULL);
347 :
348 : /* The value obtained after peeling away zero or more casts. */
349 : tree op;
350 :
351 : /* The type of OP. */
352 : tree type;
353 :
354 : /* The definition type of OP. */
355 : vect_def_type dt;
356 :
357 : /* If OP is the result of peeling at least one cast, and if the cast
358 : of OP itself is a vectorizable statement, CASTER identifies that
359 : statement, otherwise it is null. */
360 : stmt_vec_info caster;
361 : };
362 :
363 308955795 : inline vect_unpromoted_value::vect_unpromoted_value ()
364 308955795 : : op (NULL_TREE),
365 308955795 : type (NULL_TREE),
366 308955795 : dt (vect_uninitialized_def),
367 3304151 : caster (NULL)
368 : {
369 : }
370 :
371 : /* Set the operand to OP_IN, its definition type to DT_IN, and the
372 : statement that casts it to CASTER_IN. */
373 :
374 : inline void
375 11737986 : vect_unpromoted_value::set_op (tree op_in, vect_def_type dt_in,
376 : stmt_vec_info caster_in)
377 : {
378 11737986 : op = op_in;
379 11737986 : type = TREE_TYPE (op);
380 11737986 : dt = dt_in;
381 11737986 : caster = caster_in;
382 11737986 : }
383 :
384 : /* If OP is a vectorizable SSA name, strip a sequence of integer conversions
385 : to reach some vectorizable inner operand OP', continuing as long as it
386 : is possible to convert OP' back to OP using a possible sign change
387 : followed by a possible promotion P. Return this OP', or null if OP is
388 : not a vectorizable SSA name. If there is a promotion P, describe its
389 : input in UNPROM, otherwise describe OP' in UNPROM. If SINGLE_USE_P
390 : is nonnull, set *SINGLE_USE_P to false if any of the SSA names involved
391 : have more than one user.
392 :
393 : A successful return means that it is possible to go from OP' to OP
394 : via UNPROM. The cast from OP' to UNPROM is at most a sign change,
395 : whereas the cast from UNPROM to OP might be a promotion, a sign
396 : change, or a nop.
397 :
398 : E.g. say we have:
399 :
400 : signed short *ptr = ...;
401 : signed short C = *ptr;
402 : unsigned short B = (unsigned short) C; // sign change
403 : signed int A = (signed int) B; // unsigned promotion
404 : ...possible other uses of A...
405 : unsigned int OP = (unsigned int) A; // sign change
406 :
407 : In this case it's possible to go directly from C to OP using:
408 :
409 : OP = (unsigned int) (unsigned short) C;
410 : +------------+ +--------------+
411 : promotion sign change
412 :
413 : so OP' would be C. The input to the promotion is B, so UNPROM
414 : would describe B. */
415 :
416 : static tree
417 8645393 : vect_look_through_possible_promotion (vec_info *vinfo, tree op,
418 : vect_unpromoted_value *unprom,
419 : bool *single_use_p = NULL)
420 : {
421 8645393 : tree op_type = TREE_TYPE (op);
422 8645393 : if (!INTEGRAL_TYPE_P (op_type))
423 : return NULL_TREE;
424 :
425 8594141 : tree res = NULL_TREE;
426 8594141 : unsigned int orig_precision = TYPE_PRECISION (op_type);
427 8594141 : unsigned int min_precision = orig_precision;
428 8594141 : stmt_vec_info caster = NULL;
429 10290523 : while (TREE_CODE (op) == SSA_NAME && INTEGRAL_TYPE_P (op_type))
430 : {
431 : /* See whether OP is simple enough to vectorize. */
432 10078980 : stmt_vec_info def_stmt_info;
433 10078980 : gimple *def_stmt;
434 10078980 : vect_def_type dt;
435 10078980 : if (!vect_is_simple_use (op, vinfo, &dt, &def_stmt_info, &def_stmt))
436 : break;
437 :
438 : /* If OP is the input of a demotion, skip over it to see whether
439 : OP is itself the result of a promotion. If so, the combined
440 : effect of the promotion and the demotion might fit the required
441 : pattern, otherwise neither operation fits.
442 :
443 : This copes with cases such as the result of an arithmetic
444 : operation being truncated before being stored, and where that
445 : arithmetic operation has been recognized as an over-widened one. */
446 10062087 : if (TYPE_PRECISION (op_type) <= min_precision)
447 : {
448 : /* Use OP as the UNPROM described above if we haven't yet
449 : found a promotion, or if using the new input preserves the
450 : sign of the previous promotion. */
451 9930577 : if (!res
452 1454122 : || TYPE_PRECISION (unprom->type) == orig_precision
453 38910 : || TYPE_SIGN (unprom->type) == TYPE_SIGN (op_type)
454 9966159 : || (TYPE_UNSIGNED (op_type)
455 24951 : && TYPE_PRECISION (op_type) < TYPE_PRECISION (unprom->type)))
456 : {
457 9895425 : unprom->set_op (op, dt, caster);
458 9895425 : min_precision = TYPE_PRECISION (op_type);
459 : }
460 : /* Stop if we've already seen a promotion and if this
461 : conversion does more than change the sign. */
462 35152 : else if (TYPE_PRECISION (op_type)
463 35152 : != TYPE_PRECISION (unprom->type))
464 : break;
465 :
466 : /* The sequence now extends to OP. */
467 : res = op;
468 : }
469 :
470 : /* See whether OP is defined by a cast. Record it as CASTER if
471 : the cast is potentially vectorizable. */
472 10062057 : if (!def_stmt)
473 : break;
474 9848657 : caster = def_stmt_info;
475 :
476 : /* Ignore pattern statements, since we don't link uses for them. */
477 9848657 : if (caster
478 9848657 : && single_use_p
479 2004873 : && !STMT_VINFO_RELATED_STMT (caster)
480 11707003 : && !has_single_use (res))
481 1112556 : *single_use_p = false;
482 :
483 14602583 : gassign *assign = dyn_cast <gassign *> (def_stmt);
484 6219985 : if (!assign || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def_stmt)))
485 : break;
486 :
487 : /* Continue with the input to the cast. */
488 1696382 : op = gimple_assign_rhs1 (def_stmt);
489 1696382 : op_type = TREE_TYPE (op);
490 : }
491 : return res;
492 : }
493 :
494 : /* OP is an integer operand to an operation that returns TYPE, and we
495 : want to treat the operation as a widening one. So far we can treat
496 : it as widening from *COMMON_TYPE.
497 :
498 : Return true if OP is suitable for such a widening operation,
499 : either widening from *COMMON_TYPE or from some supertype of it.
500 : Update *COMMON_TYPE to the supertype in the latter case.
501 :
502 : SHIFT_P is true if OP is a shift amount. */
503 :
504 : static bool
505 328884 : vect_joust_widened_integer (tree type, bool shift_p, tree op,
506 : tree *common_type)
507 : {
508 : /* Calculate the minimum precision required by OP, without changing
509 : the sign of either operand. */
510 328884 : unsigned int precision;
511 328884 : if (shift_p)
512 : {
513 13324 : if (!wi::leu_p (wi::to_widest (op), TYPE_PRECISION (type) / 2))
514 : return false;
515 10724 : precision = TREE_INT_CST_LOW (op);
516 : }
517 : else
518 : {
519 315560 : precision = wi::min_precision (wi::to_widest (op),
520 315560 : TYPE_SIGN (*common_type));
521 315560 : if (precision * 2 > TYPE_PRECISION (type))
522 : return false;
523 : }
524 :
525 : /* If OP requires a wider type, switch to that type. The checks
526 : above ensure that this is still narrower than the result. */
527 311535 : precision = vect_element_precision (precision);
528 311535 : if (TYPE_PRECISION (*common_type) < precision)
529 6530 : *common_type = build_nonstandard_integer_type
530 6530 : (precision, TYPE_UNSIGNED (*common_type));
531 : return true;
532 : }
533 :
534 : /* Return true if the common supertype of NEW_TYPE and *COMMON_TYPE
535 : is narrower than type, storing the supertype in *COMMON_TYPE if so. */
536 :
537 : static bool
538 47543 : vect_joust_widened_type (tree type, tree new_type, tree *common_type)
539 : {
540 47543 : if (types_compatible_p (*common_type, new_type))
541 : return true;
542 :
543 : /* See if *COMMON_TYPE can hold all values of NEW_TYPE. */
544 3976 : if ((TYPE_PRECISION (new_type) < TYPE_PRECISION (*common_type))
545 3976 : && (TYPE_UNSIGNED (new_type) || !TYPE_UNSIGNED (*common_type)))
546 : return true;
547 :
548 : /* See if NEW_TYPE can hold all values of *COMMON_TYPE. */
549 3221 : if (TYPE_PRECISION (*common_type) < TYPE_PRECISION (new_type)
550 3221 : && (TYPE_UNSIGNED (*common_type) || !TYPE_UNSIGNED (new_type)))
551 : {
552 382 : *common_type = new_type;
553 382 : return true;
554 : }
555 :
556 : /* We have mismatched signs, with the signed type being
557 : no wider than the unsigned type. In this case we need
558 : a wider signed type. */
559 2839 : unsigned int precision = MAX (TYPE_PRECISION (*common_type),
560 : TYPE_PRECISION (new_type));
561 2839 : precision *= 2;
562 :
563 2839 : if (precision * 2 > TYPE_PRECISION (type))
564 : return false;
565 :
566 40 : *common_type = build_nonstandard_integer_type (precision, false);
567 40 : return true;
568 : }
569 :
570 : /* Check whether STMT_INFO can be viewed as a tree of integer operations
571 : in which each node either performs CODE or WIDENED_CODE, and where
572 : each leaf operand is narrower than the result of STMT_INFO. MAX_NOPS
573 : specifies the maximum number of leaf operands. SHIFT_P says whether
574 : CODE and WIDENED_CODE are some sort of shift.
575 :
576 : If STMT_INFO is such a tree, return the number of leaf operands
577 : and describe them in UNPROM[0] onwards. Also set *COMMON_TYPE
578 : to a type that (a) is narrower than the result of STMT_INFO and
579 : (b) can hold all leaf operand values.
580 :
581 : If SUBTYPE then allow that the signs of the operands
582 : may differ in signs but not in precision. SUBTYPE is updated to reflect
583 : this.
584 :
585 : Return 0 if STMT_INFO isn't such a tree, or if no such COMMON_TYPE
586 : exists. */
587 :
588 : static unsigned int
589 129911421 : vect_widened_op_tree (vec_info *vinfo, stmt_vec_info stmt_info, tree_code code,
590 : code_helper widened_code, bool shift_p,
591 : unsigned int max_nops,
592 : vect_unpromoted_value *unprom, tree *common_type,
593 : enum optab_subtype *subtype = NULL)
594 : {
595 : /* Check for an integer operation with the right code. */
596 129911421 : gimple* stmt = stmt_info->stmt;
597 129911421 : if (!(is_gimple_assign (stmt) || is_gimple_call (stmt)))
598 : return 0;
599 :
600 106753532 : code_helper rhs_code;
601 106753532 : if (is_gimple_assign (stmt))
602 91525016 : rhs_code = gimple_assign_rhs_code (stmt);
603 15228516 : else if (is_gimple_call (stmt))
604 15228516 : rhs_code = gimple_call_combined_fn (stmt);
605 : else
606 : return 0;
607 :
608 106753532 : if (rhs_code != code
609 106753532 : && rhs_code != widened_code)
610 : return 0;
611 :
612 6767747 : tree lhs = gimple_get_lhs (stmt);
613 6767747 : tree type = TREE_TYPE (lhs);
614 6767747 : if (!INTEGRAL_TYPE_P (type))
615 : return 0;
616 :
617 : /* Assume that both operands will be leaf operands. */
618 5882013 : max_nops -= 2;
619 :
620 : /* Check the operands. */
621 5882013 : unsigned int next_op = 0;
622 6695192 : for (unsigned int i = 0; i < 2; ++i)
623 : {
624 6338508 : vect_unpromoted_value *this_unprom = &unprom[next_op];
625 6338508 : unsigned int nops = 1;
626 6338508 : tree op = gimple_arg (stmt, i);
627 6338508 : if (i == 1 && TREE_CODE (op) == INTEGER_CST)
628 : {
629 : /* We already have a common type from earlier operands.
630 : Update it to account for OP. */
631 328884 : this_unprom->set_op (op, vect_constant_def);
632 328884 : if (!vect_joust_widened_integer (type, shift_p, op, common_type))
633 : return 0;
634 : }
635 : else
636 : {
637 : /* Only allow shifts by constants. */
638 6009624 : if (shift_p && i == 1)
639 : return 0;
640 :
641 6003536 : if (rhs_code != code)
642 : {
643 : /* If rhs_code is widened_code, don't look through further
644 : possible promotions, there is a promotion already embedded
645 : in the WIDEN_*_EXPR. */
646 1710 : if (TREE_CODE (op) != SSA_NAME
647 1710 : || !INTEGRAL_TYPE_P (TREE_TYPE (op)))
648 0 : return 0;
649 :
650 1710 : stmt_vec_info def_stmt_info;
651 1710 : gimple *def_stmt;
652 1710 : vect_def_type dt;
653 1710 : if (!vect_is_simple_use (op, vinfo, &dt, &def_stmt_info,
654 : &def_stmt))
655 : return 0;
656 1710 : this_unprom->set_op (op, dt, NULL);
657 : }
658 6001826 : else if (!vect_look_through_possible_promotion (vinfo, op,
659 : this_unprom))
660 : return 0;
661 :
662 5890184 : if (TYPE_PRECISION (this_unprom->type) == TYPE_PRECISION (type))
663 : {
664 : /* The operand isn't widened. If STMT_INFO has the code
665 : for an unwidened operation, recursively check whether
666 : this operand is a node of the tree. */
667 5382036 : if (rhs_code != code
668 5382036 : || max_nops == 0
669 5382561 : || this_unprom->dt != vect_internal_def)
670 : return 0;
671 :
672 : /* Give back the leaf slot allocated above now that we're
673 : not treating this as a leaf operand. */
674 525 : max_nops += 1;
675 :
676 : /* Recursively process the definition of the operand. */
677 525 : stmt_vec_info def_stmt_info
678 525 : = vect_get_internal_def (vinfo, this_unprom->op);
679 :
680 525 : nops = vect_widened_op_tree (vinfo, def_stmt_info, code,
681 : widened_code, shift_p, max_nops,
682 : this_unprom, common_type,
683 : subtype);
684 525 : if (nops == 0)
685 : return 0;
686 :
687 367 : max_nops -= nops;
688 : }
689 : else
690 : {
691 : /* Make sure that the operand is narrower than the result. */
692 508148 : if (TYPE_PRECISION (this_unprom->type) * 2
693 508148 : > TYPE_PRECISION (type))
694 : return 0;
695 :
696 : /* Update COMMON_TYPE for the new operand. */
697 503785 : if (i == 0)
698 456242 : *common_type = this_unprom->type;
699 47543 : else if (!vect_joust_widened_type (type, this_unprom->type,
700 : common_type))
701 : {
702 2799 : if (subtype)
703 : {
704 : /* See if we can sign extend the smaller type. */
705 291 : if (TYPE_PRECISION (this_unprom->type)
706 291 : > TYPE_PRECISION (*common_type))
707 27 : *common_type = this_unprom->type;
708 291 : *subtype = optab_vector_mixed_sign;
709 : }
710 : else
711 : return 0;
712 : }
713 : }
714 : }
715 813179 : next_op += nops;
716 : }
717 : return next_op;
718 : }
719 :
720 : /* Helper to return a new temporary for pattern of TYPE for STMT. If STMT
721 : is NULL, the caller must set SSA_NAME_DEF_STMT for the returned SSA var. */
722 :
723 : static tree
724 2133561 : vect_recog_temp_ssa_var (tree type, gimple *stmt = NULL)
725 : {
726 0 : return make_temp_ssa_name (type, stmt, "patt");
727 : }
728 :
729 : /* STMT2_INFO describes a type conversion that could be split into STMT1
730 : followed by a version of STMT2_INFO that takes NEW_RHS as its first
731 : input. Try to do this using pattern statements, returning true on
732 : success. */
733 :
734 : static bool
735 28465 : vect_split_statement (vec_info *vinfo, stmt_vec_info stmt2_info, tree new_rhs,
736 : gimple *stmt1, tree vectype)
737 : {
738 28465 : if (is_pattern_stmt_p (stmt2_info))
739 : {
740 : /* STMT2_INFO is part of a pattern. Get the statement to which
741 : the pattern is attached. */
742 454 : stmt_vec_info orig_stmt2_info = STMT_VINFO_RELATED_STMT (stmt2_info);
743 454 : vect_init_pattern_stmt (vinfo, stmt1, orig_stmt2_info, vectype);
744 :
745 454 : if (dump_enabled_p ())
746 19 : dump_printf_loc (MSG_NOTE, vect_location,
747 : "Splitting pattern statement: %G", stmt2_info->stmt);
748 :
749 : /* Since STMT2_INFO is a pattern statement, we can change it
750 : in-situ without worrying about changing the code for the
751 : containing block. */
752 454 : gimple_assign_set_rhs1 (stmt2_info->stmt, new_rhs);
753 :
754 454 : if (dump_enabled_p ())
755 : {
756 19 : dump_printf_loc (MSG_NOTE, vect_location, "into: %G", stmt1);
757 19 : dump_printf_loc (MSG_NOTE, vect_location, "and: %G",
758 : stmt2_info->stmt);
759 : }
760 :
761 454 : gimple_seq *def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt2_info);
762 454 : if (STMT_VINFO_RELATED_STMT (orig_stmt2_info) == stmt2_info)
763 : /* STMT2_INFO is the actual pattern statement. Add STMT1
764 : to the end of the definition sequence. */
765 451 : gimple_seq_add_stmt_without_update (def_seq, stmt1);
766 : else
767 : {
768 : /* STMT2_INFO belongs to the definition sequence. Insert STMT1
769 : before it. */
770 3 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt2_info->stmt, def_seq);
771 3 : gsi_insert_before_without_update (&gsi, stmt1, GSI_SAME_STMT);
772 : }
773 : return true;
774 : }
775 : else
776 : {
777 : /* STMT2_INFO doesn't yet have a pattern. Try to create a
778 : two-statement pattern now. */
779 28011 : gcc_assert (!STMT_VINFO_RELATED_STMT (stmt2_info));
780 28011 : tree lhs_type = TREE_TYPE (gimple_get_lhs (stmt2_info->stmt));
781 28011 : tree lhs_vectype = get_vectype_for_scalar_type (vinfo, lhs_type);
782 28011 : if (!lhs_vectype)
783 : return false;
784 :
785 28011 : if (dump_enabled_p ())
786 2049 : dump_printf_loc (MSG_NOTE, vect_location,
787 : "Splitting statement: %G", stmt2_info->stmt);
788 :
789 : /* Add STMT1 as a singleton pattern definition sequence. */
790 28011 : gimple_seq *def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (stmt2_info);
791 28011 : vect_init_pattern_stmt (vinfo, stmt1, stmt2_info, vectype);
792 28011 : gimple_seq_add_stmt_without_update (def_seq, stmt1);
793 :
794 : /* Build the second of the two pattern statements. */
795 28011 : tree new_lhs = vect_recog_temp_ssa_var (lhs_type, NULL);
796 28011 : gassign *new_stmt2 = gimple_build_assign (new_lhs, NOP_EXPR, new_rhs);
797 28011 : vect_set_pattern_stmt (vinfo, new_stmt2, stmt2_info, lhs_vectype);
798 :
799 28011 : if (dump_enabled_p ())
800 : {
801 2049 : dump_printf_loc (MSG_NOTE, vect_location,
802 : "into pattern statements: %G", stmt1);
803 2049 : dump_printf_loc (MSG_NOTE, vect_location, "and: %G",
804 : (gimple *) new_stmt2);
805 : }
806 :
807 : return true;
808 : }
809 : }
810 :
811 : /* Look for the following pattern
812 : X = x[i]
813 : Y = y[i]
814 : DIFF = X - Y
815 : DAD = ABS_EXPR<DIFF>
816 :
817 : ABS_STMT should point to a statement of code ABS_EXPR or ABSU_EXPR.
818 : HALF_TYPE and UNPROM will be set should the statement be found to
819 : be a widened operation.
820 : DIFF_STMT will be set to the MINUS_EXPR
821 : statement that precedes the ABS_STMT if it is a MINUS_EXPR..
822 : */
823 : static bool
824 22494113 : vect_recog_absolute_difference (vec_info *vinfo, gassign *abs_stmt,
825 : tree *half_type,
826 : vect_unpromoted_value unprom[2],
827 : gassign **diff_stmt)
828 : {
829 22494113 : if (!abs_stmt)
830 : return false;
831 :
832 : /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
833 : inside the loop (in case we are analyzing an outer-loop). */
834 22494113 : enum tree_code code = gimple_assign_rhs_code (abs_stmt);
835 22494113 : if (code != ABS_EXPR && code != ABSU_EXPR)
836 : return false;
837 :
838 34340 : tree abs_oprnd = gimple_assign_rhs1 (abs_stmt);
839 34340 : tree abs_type = TREE_TYPE (abs_oprnd);
840 34340 : if (!abs_oprnd)
841 : return false;
842 26930 : if (!ANY_INTEGRAL_TYPE_P (abs_type)
843 7664 : || TYPE_OVERFLOW_WRAPS (abs_type)
844 41850 : || TYPE_UNSIGNED (abs_type))
845 : return false;
846 :
847 : /* Peel off conversions from the ABS input. This can involve sign
848 : changes (e.g. from an unsigned subtraction to a signed ABS input)
849 : or signed promotion, but it can't include unsigned promotion.
850 : (Note that ABS of an unsigned promotion should have been folded
851 : away before now anyway.) */
852 7510 : vect_unpromoted_value unprom_diff;
853 7510 : abs_oprnd = vect_look_through_possible_promotion (vinfo, abs_oprnd,
854 : &unprom_diff);
855 7510 : if (!abs_oprnd)
856 : return false;
857 7256 : if (TYPE_PRECISION (unprom_diff.type) != TYPE_PRECISION (abs_type)
858 7256 : && TYPE_UNSIGNED (unprom_diff.type))
859 : return false;
860 :
861 : /* We then detect if the operand of abs_expr is defined by a minus_expr. */
862 7256 : stmt_vec_info diff_stmt_vinfo = vect_get_internal_def (vinfo, abs_oprnd);
863 7256 : if (!diff_stmt_vinfo)
864 : return false;
865 :
866 7048 : gassign *diff = dyn_cast <gassign *> (STMT_VINFO_STMT (diff_stmt_vinfo));
867 7048 : if (diff_stmt && diff
868 5806 : && gimple_assign_rhs_code (diff) == MINUS_EXPR
869 8901 : && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (abs_oprnd)))
870 251 : *diff_stmt = diff;
871 :
872 : /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
873 : inside the loop (in case we are analyzing an outer-loop). */
874 7048 : if (vect_widened_op_tree (vinfo, diff_stmt_vinfo,
875 7048 : MINUS_EXPR, IFN_VEC_WIDEN_MINUS,
876 : false, 2, unprom, half_type))
877 2408 : return true;
878 :
879 : return false;
880 : }
881 :
882 : /* Convert UNPROM to TYPE and return the result, adding new statements
883 : to STMT_INFO's pattern definition statements if no better way is
884 : available. VECTYPE is the vector form of TYPE.
885 :
886 : If SUBTYPE then convert the type based on the subtype. */
887 :
888 : static tree
889 495665 : vect_convert_input (vec_info *vinfo, stmt_vec_info stmt_info, tree type,
890 : vect_unpromoted_value *unprom, tree vectype,
891 : enum optab_subtype subtype = optab_default)
892 : {
893 : /* Update the type if the signs differ. */
894 495665 : if (subtype == optab_vector_mixed_sign)
895 : {
896 328 : gcc_assert (!TYPE_UNSIGNED (type));
897 328 : if (TYPE_UNSIGNED (TREE_TYPE (unprom->op)))
898 : {
899 164 : type = unsigned_type_for (type);
900 164 : vectype = unsigned_type_for (vectype);
901 : }
902 : }
903 :
904 : /* Check for a no-op conversion. */
905 495665 : if (types_compatible_p (type, TREE_TYPE (unprom->op)))
906 173071 : return unprom->op;
907 :
908 : /* Allow the caller to create constant vect_unpromoted_values. */
909 322594 : if (TREE_CODE (unprom->op) == INTEGER_CST)
910 200801 : return wide_int_to_tree (type, wi::to_widest (unprom->op));
911 :
912 121793 : tree input = unprom->op;
913 121793 : if (unprom->caster)
914 : {
915 63822 : tree lhs = gimple_get_lhs (unprom->caster->stmt);
916 63822 : tree lhs_type = TREE_TYPE (lhs);
917 :
918 : /* If the result of the existing cast is the right width, use it
919 : instead of the source of the cast. */
920 63822 : if (TYPE_PRECISION (lhs_type) == TYPE_PRECISION (type))
921 : input = lhs;
922 : /* If the precision we want is between the source and result
923 : precisions of the existing cast, try splitting the cast into
924 : two and tapping into a mid-way point. */
925 61591 : else if (TYPE_PRECISION (lhs_type) > TYPE_PRECISION (type)
926 61591 : && TYPE_PRECISION (type) > TYPE_PRECISION (unprom->type))
927 : {
928 : /* In order to preserve the semantics of the original cast,
929 : give the mid-way point the same signedness as the input value.
930 :
931 : It would be possible to use a signed type here instead if
932 : TYPE is signed and UNPROM->TYPE is unsigned, but that would
933 : make the sign of the midtype sensitive to the order in
934 : which we process the statements, since the signedness of
935 : TYPE is the signedness required by just one of possibly
936 : many users. Also, unsigned promotions are usually as cheap
937 : as or cheaper than signed ones, so it's better to keep an
938 : unsigned promotion. */
939 28465 : tree midtype = build_nonstandard_integer_type
940 28465 : (TYPE_PRECISION (type), TYPE_UNSIGNED (unprom->type));
941 28465 : tree vec_midtype = get_vectype_for_scalar_type (vinfo, midtype);
942 28465 : if (vec_midtype)
943 : {
944 28465 : input = vect_recog_temp_ssa_var (midtype, NULL);
945 28465 : gassign *new_stmt = gimple_build_assign (input, NOP_EXPR,
946 : unprom->op);
947 28465 : if (!vect_split_statement (vinfo, unprom->caster, input, new_stmt,
948 : vec_midtype))
949 0 : append_pattern_def_seq (vinfo, stmt_info,
950 : new_stmt, vec_midtype);
951 : }
952 : }
953 :
954 : /* See if we can reuse an existing result. */
955 63822 : if (types_compatible_p (type, TREE_TYPE (input)))
956 : return input;
957 : }
958 :
959 : /* We need a new conversion statement. */
960 101265 : tree new_op = vect_recog_temp_ssa_var (type, NULL);
961 101265 : gassign *new_stmt = gimple_build_assign (new_op, NOP_EXPR, input);
962 :
963 : /* If OP is an external value, see if we can insert the new statement
964 : on an incoming edge. */
965 101265 : if (input == unprom->op && unprom->dt == vect_external_def)
966 8596 : if (edge e = vect_get_external_def_edge (vinfo, input))
967 : {
968 754 : basic_block new_bb = gsi_insert_on_edge_immediate (e, new_stmt);
969 754 : gcc_assert (!new_bb);
970 : return new_op;
971 : }
972 :
973 : /* As a (common) last resort, add the statement to the pattern itself. */
974 100511 : append_pattern_def_seq (vinfo, stmt_info, new_stmt, vectype);
975 100511 : return new_op;
976 : }
977 :
978 : /* Invoke vect_convert_input for N elements of UNPROM and store the
979 : result in the corresponding elements of RESULT.
980 :
981 : If SUBTYPE then convert the type based on the subtype. */
982 :
983 : static void
984 251815 : vect_convert_inputs (vec_info *vinfo, stmt_vec_info stmt_info, unsigned int n,
985 : tree *result, tree type, vect_unpromoted_value *unprom,
986 : tree vectype, enum optab_subtype subtype = optab_default)
987 : {
988 747451 : for (unsigned int i = 0; i < n; ++i)
989 : {
990 : unsigned int j;
991 738978 : for (j = 0; j < i; ++j)
992 243821 : if (unprom[j].op == unprom[i].op)
993 : break;
994 :
995 495636 : if (j < i)
996 479 : result[i] = result[j];
997 : else
998 495157 : result[i] = vect_convert_input (vinfo, stmt_info,
999 495157 : type, &unprom[i], vectype, subtype);
1000 : }
1001 251815 : }
1002 :
1003 : /* The caller has created a (possibly empty) sequence of pattern definition
1004 : statements followed by a single statement PATTERN_STMT. Cast the result
1005 : of this final statement to TYPE. If a new statement is needed, add
1006 : PATTERN_STMT to the end of STMT_INFO's pattern definition statements
1007 : and return the new statement, otherwise return PATTERN_STMT as-is.
1008 : VECITYPE is the vector form of PATTERN_STMT's result type. */
1009 :
1010 : static gimple *
1011 278219 : vect_convert_output (vec_info *vinfo, stmt_vec_info stmt_info, tree type,
1012 : gimple *pattern_stmt, tree vecitype)
1013 : {
1014 278219 : tree lhs = gimple_get_lhs (pattern_stmt);
1015 278219 : if (!types_compatible_p (type, TREE_TYPE (lhs)))
1016 : {
1017 247638 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vecitype);
1018 247638 : tree cast_var = vect_recog_temp_ssa_var (type, NULL);
1019 247638 : pattern_stmt = gimple_build_assign (cast_var, NOP_EXPR, lhs);
1020 : }
1021 278219 : return pattern_stmt;
1022 : }
1023 :
1024 : /* Return true if STMT_VINFO describes a reduction for which reassociation
1025 : is allowed. If STMT_INFO is part of a group, assume that it's part of
1026 : a reduction chain and optimistically assume that all statements
1027 : except the last allow reassociation.
1028 : Also require it to have code CODE and to be a reduction
1029 : in the outermost loop. When returning true, store the operands in
1030 : *OP0_OUT and *OP1_OUT. */
1031 :
1032 : static bool
1033 96911619 : vect_reassociating_reduction_p (vec_info *vinfo,
1034 : stmt_vec_info stmt_info, tree_code code,
1035 : tree *op0_out, tree *op1_out)
1036 : {
1037 96911619 : loop_vec_info loop_info = dyn_cast <loop_vec_info> (vinfo);
1038 13735974 : if (!loop_info)
1039 : return false;
1040 :
1041 13735974 : gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
1042 14997741 : if (!assign || gimple_assign_rhs_code (assign) != code)
1043 : return false;
1044 :
1045 : /* We don't allow changing the order of the computation in the inner-loop
1046 : when doing outer-loop vectorization. */
1047 2678453 : class loop *loop = LOOP_VINFO_LOOP (loop_info);
1048 2678453 : if (loop && nested_in_vect_loop_p (loop, stmt_info))
1049 : return false;
1050 :
1051 2625227 : if (!vect_is_reduction (stmt_info))
1052 : return false;
1053 :
1054 172808 : if (needs_fold_left_reduction_p (TREE_TYPE (gimple_assign_lhs (assign)),
1055 172808 : code))
1056 : return false;
1057 :
1058 159887 : *op0_out = gimple_assign_rhs1 (assign);
1059 159887 : *op1_out = gimple_assign_rhs2 (assign);
1060 159887 : if (commutative_tree_code (code) && STMT_VINFO_REDUC_IDX (stmt_info) == 0)
1061 63894 : std::swap (*op0_out, *op1_out);
1062 : return true;
1063 : }
1064 :
1065 : /* Return true iff the target has a vector optab implementing the operation
1066 : CODE on type VECTYPE with SUBTYPE. */
1067 :
1068 : static bool
1069 878710 : target_has_vecop_for_code (tree_code code, tree vectype,
1070 : enum optab_subtype subtype = optab_vector)
1071 : {
1072 878710 : optab voptab = optab_for_tree_code (code, vectype, subtype);
1073 878710 : return voptab && can_implement_p (voptab, TYPE_MODE (vectype));
1074 : }
1075 :
1076 : /* match.pd function to match
1077 : (cond (cmp@3 a b) (convert@1 c) (convert@2 d))
1078 : with conditions:
1079 : 1) @1, @2, c, d, a, b are all integral type.
1080 : 2) There's single_use for both @1 and @2.
1081 : 3) a, c have same precision.
1082 : 4) c and @1 have different precision.
1083 : 5) c, d are the same type or they can differ in sign when convert is
1084 : truncation.
1085 :
1086 : record a and c and d and @3. */
1087 :
1088 : extern bool gimple_cond_expr_convert_p (tree, tree*, tree (*)(tree));
1089 :
1090 : /* Function vect_recog_cond_expr_convert
1091 :
1092 : Try to find the following pattern:
1093 :
1094 : TYPE_AB A,B;
1095 : TYPE_CD C,D;
1096 : TYPE_E E;
1097 : TYPE_E op_true = (TYPE_E) A;
1098 : TYPE_E op_false = (TYPE_E) B;
1099 :
1100 : E = C cmp D ? op_true : op_false;
1101 :
1102 : where
1103 : TYPE_PRECISION (TYPE_E) != TYPE_PRECISION (TYPE_CD);
1104 : TYPE_PRECISION (TYPE_AB) == TYPE_PRECISION (TYPE_CD);
1105 : single_use of op_true and op_false.
1106 : TYPE_AB could differ in sign when (TYPE_E) A is a truncation.
1107 :
1108 : Input:
1109 :
1110 : * STMT_VINFO: The stmt from which the pattern search begins.
1111 : here it starts with E = c cmp D ? op_true : op_false;
1112 :
1113 : Output:
1114 :
1115 : TYPE1 E' = C cmp D ? A : B;
1116 : TYPE3 E = (TYPE3) E';
1117 :
1118 : There may extra nop_convert for A or B to handle different signness.
1119 :
1120 : * TYPE_OUT: The vector type of the output of this pattern.
1121 :
1122 : * Return value: A new stmt that will be used to replace the sequence of
1123 : stmts that constitute the pattern. In this case it will be:
1124 : E = (TYPE3)E';
1125 : E' = C cmp D ? A : B; is recorded in pattern definition statements; */
1126 :
1127 : static gimple *
1128 32380528 : vect_recog_cond_expr_convert_pattern (vec_info *vinfo,
1129 : stmt_vec_info stmt_vinfo, tree *type_out)
1130 : {
1131 32380528 : gassign *last_stmt = dyn_cast <gassign *> (stmt_vinfo->stmt);
1132 22591240 : tree lhs, match[4], temp, type, new_lhs, op2, op1;
1133 22591240 : gimple *cond_stmt;
1134 22591240 : gimple *pattern_stmt;
1135 54971768 : enum tree_code code = NOP_EXPR;
1136 :
1137 22591240 : if (!last_stmt)
1138 : return NULL;
1139 :
1140 22591240 : lhs = gimple_assign_lhs (last_stmt);
1141 :
1142 : /* Find E = C cmp D ? (TYPE3) A ? (TYPE3) B;
1143 : TYPE_PRECISION (A) == TYPE_PRECISION (C). */
1144 22591240 : if (!gimple_cond_expr_convert_p (lhs, &match[0], NULL))
1145 : return NULL;
1146 :
1147 29 : if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (lhs)))
1148 20 : code = INTEGRAL_TYPE_P (TREE_TYPE (match[1])) ? FLOAT_EXPR : CONVERT_EXPR;
1149 9 : else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (match[1])))
1150 0 : code = FIX_TRUNC_EXPR;
1151 :
1152 29 : op1 = match[1];
1153 29 : op2 = match[2];
1154 29 : type = TREE_TYPE (op1);
1155 : /* When op1/op2 is REAL_CST, the conversion must be CONVERT_EXPR from
1156 : SCALAR_FLOAT_TYPE_P which is restricted in gimple_cond_expr_convert_p.
1157 : Otherwise, the conversion could be FLOAT_EXPR, FIX_TRUNC_EXPR
1158 : or CONVERT_EXPR. */
1159 29 : if (TREE_CODE (op1) == REAL_CST)
1160 : {
1161 20 : op1 = const_unop (CONVERT_EXPR, TREE_TYPE (op2), op1);
1162 20 : type = TREE_TYPE (op2);
1163 20 : if (op1 == NULL_TREE)
1164 : return NULL;
1165 : }
1166 9 : else if (TREE_CODE (op2) == REAL_CST)
1167 : {
1168 0 : op2 = const_unop (FLOAT_EXPR, TREE_TYPE (op1), op2);
1169 0 : if (op2 == NULL_TREE)
1170 : return NULL;
1171 : }
1172 9 : else if (code == NOP_EXPR)
1173 : {
1174 9 : if (TYPE_SIGN (type) != TYPE_SIGN (TREE_TYPE (match[2])))
1175 : {
1176 9 : op2 = vect_recog_temp_ssa_var (type, NULL);
1177 9 : gimple* nop_stmt = gimple_build_assign (op2, NOP_EXPR, match[2]);
1178 9 : append_pattern_def_seq (vinfo, stmt_vinfo, nop_stmt);
1179 : }
1180 : }
1181 :
1182 29 : vect_pattern_detected ("vect_recog_cond_expr_convert_pattern", last_stmt);
1183 :
1184 29 : temp = vect_recog_temp_ssa_var (type, NULL);
1185 29 : cond_stmt = gimple_build_assign (temp, build3 (COND_EXPR, type, match[3],
1186 : op1, op2));
1187 29 : append_pattern_def_seq (vinfo, stmt_vinfo, cond_stmt);
1188 29 : new_lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
1189 29 : pattern_stmt = gimple_build_assign (new_lhs, code, temp);
1190 29 : *type_out = NULL_TREE;
1191 :
1192 29 : if (dump_enabled_p ())
1193 20 : dump_printf_loc (MSG_NOTE, vect_location,
1194 : "created pattern stmt: %G", pattern_stmt);
1195 : return pattern_stmt;
1196 : }
1197 :
1198 : /* Function vect_recog_dot_prod_pattern
1199 :
1200 : Try to find the following pattern:
1201 :
1202 : type1a x_t
1203 : type1b y_t;
1204 : TYPE1 prod;
1205 : TYPE2 sum = init;
1206 : loop:
1207 : sum_0 = phi <init, sum_1>
1208 : S1 x_t = ...
1209 : S2 y_t = ...
1210 : S3 x_T = (TYPE1) x_t;
1211 : S4 y_T = (TYPE1) y_t;
1212 : S5 prod = x_T * y_T;
1213 : [S6 prod = (TYPE2) prod; #optional]
1214 : S7 sum_1 = prod + sum_0;
1215 :
1216 : where 'TYPE1' is exactly double the size of type 'type1a' and 'type1b',
1217 : the sign of 'TYPE1' must be one of 'type1a' or 'type1b' but the sign of
1218 : 'type1a' and 'type1b' can differ.
1219 :
1220 : Input:
1221 :
1222 : * STMT_VINFO: The stmt from which the pattern search begins. In the
1223 : example, when this function is called with S7, the pattern {S3,S4,S5,S6,S7}
1224 : will be detected.
1225 :
1226 : Output:
1227 :
1228 : * TYPE_OUT: The type of the output of this pattern.
1229 :
1230 : * Return value: A new stmt that will be used to replace the sequence of
1231 : stmts that constitute the pattern. In this case it will be:
1232 : WIDEN_DOT_PRODUCT <x_t, y_t, sum_0>
1233 :
1234 : Note: The dot-prod idiom is a widening reduction pattern that is
1235 : vectorized without preserving all the intermediate results. It
1236 : produces only N/2 (widened) results (by summing up pairs of
1237 : intermediate results) rather than all N results. Therefore, we
1238 : cannot allow this pattern when we want to get all the results and in
1239 : the correct order (as is the case when this computation is in an
1240 : inner-loop nested in an outer-loop that us being vectorized). */
1241 :
1242 : static gimple *
1243 32304511 : vect_recog_dot_prod_pattern (vec_info *vinfo,
1244 : stmt_vec_info stmt_vinfo, tree *type_out)
1245 : {
1246 32304511 : tree oprnd0, oprnd1;
1247 32304511 : gimple *last_stmt = stmt_vinfo->stmt;
1248 32304511 : tree type, half_type;
1249 32304511 : gimple *pattern_stmt;
1250 32304511 : tree var;
1251 :
1252 : /* Look for the following pattern
1253 : DX = (TYPE1) X;
1254 : DY = (TYPE1) Y;
1255 : DPROD = DX * DY;
1256 : DDPROD = (TYPE2) DPROD;
1257 : sum_1 = DDPROD + sum_0;
1258 : In which
1259 : - DX is double the size of X
1260 : - DY is double the size of Y
1261 : - DX, DY, DPROD all have the same type but the sign
1262 : between X, Y and DPROD can differ.
1263 : - sum is the same size of DPROD or bigger
1264 : - sum has been recognized as a reduction variable.
1265 :
1266 : This is equivalent to:
1267 : DPROD = X w* Y; #widen mult
1268 : sum_1 = DPROD w+ sum_0; #widen summation
1269 : or
1270 : DPROD = X w* Y; #widen mult
1271 : sum_1 = DPROD + sum_0; #summation
1272 : */
1273 :
1274 : /* Starting from LAST_STMT, follow the defs of its uses in search
1275 : of the above pattern. */
1276 :
1277 32304511 : if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, PLUS_EXPR,
1278 : &oprnd0, &oprnd1))
1279 : return NULL;
1280 :
1281 53937 : type = TREE_TYPE (gimple_get_lhs (last_stmt));
1282 :
1283 53937 : vect_unpromoted_value unprom_mult;
1284 53937 : oprnd0 = vect_look_through_possible_promotion (vinfo, oprnd0, &unprom_mult);
1285 :
1286 : /* So far so good. Since last_stmt was detected as a (summation) reduction,
1287 : we know that oprnd1 is the reduction variable (defined by a loop-header
1288 : phi), and oprnd0 is an ssa-name defined by a stmt in the loop body.
1289 : Left to check that oprnd0 is defined by a (widen_)mult_expr */
1290 53937 : if (!oprnd0)
1291 : return NULL;
1292 :
1293 36639 : stmt_vec_info mult_vinfo = vect_get_internal_def (vinfo, oprnd0);
1294 36639 : if (!mult_vinfo)
1295 : return NULL;
1296 :
1297 : /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
1298 : inside the loop (in case we are analyzing an outer-loop). */
1299 107364 : vect_unpromoted_value unprom0[2];
1300 35788 : enum optab_subtype subtype = optab_vector;
1301 35788 : if (!vect_widened_op_tree (vinfo, mult_vinfo, MULT_EXPR, WIDEN_MULT_EXPR,
1302 : false, 2, unprom0, &half_type, &subtype))
1303 : return NULL;
1304 :
1305 : /* If there are two widening operations, make sure they agree on the sign
1306 : of the extension. The result of an optab_vector_mixed_sign operation
1307 : is signed; otherwise, the result has the same sign as the operands. */
1308 1386 : if (TYPE_PRECISION (unprom_mult.type) != TYPE_PRECISION (type)
1309 2127 : && (subtype == optab_vector_mixed_sign
1310 741 : ? TYPE_UNSIGNED (unprom_mult.type)
1311 532 : : TYPE_SIGN (unprom_mult.type) != TYPE_SIGN (half_type)))
1312 : return NULL;
1313 :
1314 1305 : vect_pattern_detected ("vect_recog_dot_prod_pattern", last_stmt);
1315 :
1316 : /* If the inputs have mixed signs, canonicalize on using the signed
1317 : input type for analysis. This also helps when emulating mixed-sign
1318 : operations using signed operations. */
1319 1305 : if (subtype == optab_vector_mixed_sign)
1320 246 : half_type = signed_type_for (half_type);
1321 :
1322 1305 : tree half_vectype;
1323 1305 : if (!vect_supportable_conv_optab_p (vinfo, type, DOT_PROD_EXPR, half_type,
1324 : type_out, &half_vectype, subtype))
1325 : {
1326 : /* We can emulate a mixed-sign dot-product using a sequence of
1327 : signed dot-products; see vect_emulate_mixed_dot_prod for details. */
1328 588 : if (subtype != optab_vector_mixed_sign
1329 588 : || !vect_supportable_conv_optab_p (vinfo, signed_type_for (type),
1330 : DOT_PROD_EXPR, half_type,
1331 : type_out, &half_vectype,
1332 : optab_vector))
1333 : return NULL;
1334 :
1335 15 : *type_out = signed_or_unsigned_type_for (TYPE_UNSIGNED (type),
1336 : *type_out);
1337 : }
1338 :
1339 : /* Get the inputs in the appropriate types. */
1340 732 : tree mult_oprnd[2];
1341 732 : vect_convert_inputs (vinfo, stmt_vinfo, 2, mult_oprnd, half_type,
1342 : unprom0, half_vectype, subtype);
1343 :
1344 732 : var = vect_recog_temp_ssa_var (type, NULL);
1345 732 : pattern_stmt = gimple_build_assign (var, DOT_PROD_EXPR,
1346 : mult_oprnd[0], mult_oprnd[1], oprnd1);
1347 :
1348 732 : return pattern_stmt;
1349 : }
1350 :
1351 :
1352 : /* Function vect_recog_sad_pattern
1353 :
1354 : Try to find the following Sum of Absolute Difference (SAD) pattern:
1355 :
1356 : type x_t, y_t;
1357 : signed TYPE1 diff, abs_diff;
1358 : TYPE2 sum = init;
1359 : loop:
1360 : sum_0 = phi <init, sum_1>
1361 : S1 x_t = ...
1362 : S2 y_t = ...
1363 : S3 x_T = (TYPE1) x_t;
1364 : S4 y_T = (TYPE1) y_t;
1365 : S5 diff = x_T - y_T;
1366 : S6 abs_diff = ABS_EXPR <diff>;
1367 : [S7 abs_diff = (TYPE2) abs_diff; #optional]
1368 : S8 sum_1 = abs_diff + sum_0;
1369 :
1370 : where 'TYPE1' is at least double the size of type 'type', and 'TYPE2' is the
1371 : same size of 'TYPE1' or bigger. This is a special case of a reduction
1372 : computation.
1373 :
1374 : Input:
1375 :
1376 : * STMT_VINFO: The stmt from which the pattern search begins. In the
1377 : example, when this function is called with S8, the pattern
1378 : {S3,S4,S5,S6,S7,S8} will be detected.
1379 :
1380 : Output:
1381 :
1382 : * TYPE_OUT: The type of the output of this pattern.
1383 :
1384 : * Return value: A new stmt that will be used to replace the sequence of
1385 : stmts that constitute the pattern. In this case it will be:
1386 : SAD_EXPR <x_t, y_t, sum_0>
1387 : */
1388 :
1389 : static gimple *
1390 32303784 : vect_recog_sad_pattern (vec_info *vinfo,
1391 : stmt_vec_info stmt_vinfo, tree *type_out)
1392 : {
1393 32303784 : gimple *last_stmt = stmt_vinfo->stmt;
1394 32303784 : tree half_type;
1395 :
1396 : /* Look for the following pattern
1397 : DX = (TYPE1) X;
1398 : DY = (TYPE1) Y;
1399 : DDIFF = DX - DY;
1400 : DAD = ABS_EXPR <DDIFF>;
1401 : DDPROD = (TYPE2) DPROD;
1402 : sum_1 = DAD + sum_0;
1403 : In which
1404 : - DX is at least double the size of X
1405 : - DY is at least double the size of Y
1406 : - DX, DY, DDIFF, DAD all have the same type
1407 : - sum is the same size of DAD or bigger
1408 : - sum has been recognized as a reduction variable.
1409 :
1410 : This is equivalent to:
1411 : DDIFF = X w- Y; #widen sub
1412 : DAD = ABS_EXPR <DDIFF>;
1413 : sum_1 = DAD w+ sum_0; #widen summation
1414 : or
1415 : DDIFF = X w- Y; #widen sub
1416 : DAD = ABS_EXPR <DDIFF>;
1417 : sum_1 = DAD + sum_0; #summation
1418 : */
1419 :
1420 : /* Starting from LAST_STMT, follow the defs of its uses in search
1421 : of the above pattern. */
1422 :
1423 32303784 : tree plus_oprnd0, plus_oprnd1;
1424 32303784 : if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, PLUS_EXPR,
1425 : &plus_oprnd0, &plus_oprnd1))
1426 : return NULL;
1427 :
1428 53205 : tree sum_type = TREE_TYPE (gimple_get_lhs (last_stmt));
1429 :
1430 : /* Any non-truncating sequence of conversions is OK here, since
1431 : with a successful match, the result of the ABS(U) is known to fit
1432 : within the nonnegative range of the result type. (It cannot be the
1433 : negative of the minimum signed value due to the range of the widening
1434 : MINUS_EXPR.) */
1435 53205 : vect_unpromoted_value unprom_abs;
1436 53205 : plus_oprnd0 = vect_look_through_possible_promotion (vinfo, plus_oprnd0,
1437 : &unprom_abs);
1438 :
1439 : /* So far so good. Since last_stmt was detected as a (summation) reduction,
1440 : we know that plus_oprnd1 is the reduction variable (defined by a loop-header
1441 : phi), and plus_oprnd0 is an ssa-name defined by a stmt in the loop body.
1442 : Then check that plus_oprnd0 is defined by an abs_expr. */
1443 :
1444 53205 : if (!plus_oprnd0)
1445 : return NULL;
1446 :
1447 35907 : stmt_vec_info abs_stmt_vinfo = vect_get_internal_def (vinfo, plus_oprnd0);
1448 35907 : if (!abs_stmt_vinfo)
1449 : return NULL;
1450 :
1451 : /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
1452 : inside the loop (in case we are analyzing an outer-loop). */
1453 35056 : gassign *abs_stmt = dyn_cast <gassign *> (abs_stmt_vinfo->stmt);
1454 105168 : vect_unpromoted_value unprom[2];
1455 :
1456 35056 : if (!abs_stmt)
1457 : {
1458 367 : gcall *abd_stmt = dyn_cast <gcall *> (abs_stmt_vinfo->stmt);
1459 302 : if (!abd_stmt
1460 302 : || !gimple_call_internal_p (abd_stmt)
1461 0 : || gimple_call_num_args (abd_stmt) != 2)
1462 : return NULL;
1463 :
1464 0 : tree abd_oprnd0 = gimple_call_arg (abd_stmt, 0);
1465 0 : tree abd_oprnd1 = gimple_call_arg (abd_stmt, 1);
1466 :
1467 0 : if (gimple_call_internal_fn (abd_stmt) == IFN_ABD
1468 0 : || gimple_call_internal_fn (abd_stmt) == IFN_VEC_WIDEN_ABD)
1469 : {
1470 0 : unprom[0].op = abd_oprnd0;
1471 0 : unprom[0].type = TREE_TYPE (abd_oprnd0);
1472 0 : unprom[1].op = abd_oprnd1;
1473 0 : unprom[1].type = TREE_TYPE (abd_oprnd1);
1474 : }
1475 : else
1476 : return NULL;
1477 :
1478 0 : half_type = unprom[0].type;
1479 : }
1480 34689 : else if (!vect_recog_absolute_difference (vinfo, abs_stmt, &half_type,
1481 : unprom, NULL))
1482 : return NULL;
1483 :
1484 854 : vect_pattern_detected ("vect_recog_sad_pattern", last_stmt);
1485 :
1486 854 : tree half_vectype;
1487 854 : if (!vect_supportable_direct_optab_p (vinfo, sum_type, SAD_EXPR, half_type,
1488 : type_out, &half_vectype))
1489 : return NULL;
1490 :
1491 : /* Get the inputs to the SAD_EXPR in the appropriate types. */
1492 460 : tree sad_oprnd[2];
1493 460 : vect_convert_inputs (vinfo, stmt_vinfo, 2, sad_oprnd, half_type,
1494 : unprom, half_vectype);
1495 :
1496 460 : tree var = vect_recog_temp_ssa_var (sum_type, NULL);
1497 460 : gimple *pattern_stmt = gimple_build_assign (var, SAD_EXPR, sad_oprnd[0],
1498 : sad_oprnd[1], plus_oprnd1);
1499 :
1500 460 : return pattern_stmt;
1501 : }
1502 :
1503 : /* Function vect_recog_abd_pattern
1504 :
1505 : Try to find the following ABsolute Difference (ABD) or
1506 : widening ABD (WIDEN_ABD) pattern:
1507 :
1508 : TYPE1 x;
1509 : TYPE2 y;
1510 : TYPE3 x_cast = (TYPE3) x; // widening or no-op
1511 : TYPE3 y_cast = (TYPE3) y; // widening or no-op
1512 : TYPE3 diff = x_cast - y_cast;
1513 : TYPE4 diff_cast = (TYPE4) diff; // widening or no-op
1514 : TYPE5 abs = ABS(U)_EXPR <diff_cast>;
1515 :
1516 : WIDEN_ABD exists to optimize the case where TYPE4 is at least
1517 : twice as wide as TYPE3.
1518 :
1519 : Input:
1520 :
1521 : * STMT_VINFO: The stmt from which the pattern search begins
1522 :
1523 : Output:
1524 :
1525 : * TYPE_OUT: The type of the output of this pattern
1526 :
1527 : * Return value: A new stmt that will be used to replace the sequence of
1528 : stmts that constitute the pattern, principally:
1529 : out = IFN_ABD (x, y)
1530 : out = IFN_WIDEN_ABD (x, y)
1531 : */
1532 :
1533 : static gimple *
1534 32248506 : vect_recog_abd_pattern (vec_info *vinfo,
1535 : stmt_vec_info stmt_vinfo, tree *type_out)
1536 : {
1537 32248506 : gassign *last_stmt = dyn_cast <gassign *> (STMT_VINFO_STMT (stmt_vinfo));
1538 22459424 : if (!last_stmt)
1539 : return NULL;
1540 :
1541 22459424 : tree out_type = TREE_TYPE (gimple_assign_lhs (last_stmt));
1542 :
1543 67378272 : vect_unpromoted_value unprom[2];
1544 22459424 : gassign *diff_stmt = NULL;
1545 22459424 : tree abd_in_type;
1546 22459424 : if (!vect_recog_absolute_difference (vinfo, last_stmt, &abd_in_type,
1547 : unprom, &diff_stmt))
1548 : {
1549 : /* We cannot try further without having a non-widening MINUS. */
1550 22457870 : if (!diff_stmt)
1551 : return NULL;
1552 :
1553 251 : unprom[0].op = gimple_assign_rhs1 (diff_stmt);
1554 251 : unprom[1].op = gimple_assign_rhs2 (diff_stmt);
1555 251 : abd_in_type = signed_type_for (out_type);
1556 : }
1557 :
1558 1805 : tree abd_out_type = abd_in_type;
1559 :
1560 1805 : tree vectype_in = get_vectype_for_scalar_type (vinfo, abd_in_type);
1561 1805 : if (!vectype_in)
1562 : return NULL;
1563 :
1564 1787 : internal_fn ifn = IFN_ABD;
1565 1787 : tree vectype_out = vectype_in;
1566 :
1567 1787 : if (TYPE_PRECISION (out_type) >= TYPE_PRECISION (abd_in_type) * 2
1568 1787 : && stmt_vinfo->min_output_precision >= TYPE_PRECISION (abd_in_type) * 2)
1569 : {
1570 1445 : tree mid_type
1571 1445 : = build_nonstandard_integer_type (TYPE_PRECISION (abd_in_type) * 2,
1572 1445 : TYPE_UNSIGNED (abd_in_type));
1573 1445 : tree mid_vectype = get_vectype_for_scalar_type (vinfo, mid_type);
1574 :
1575 1445 : code_helper dummy_code;
1576 1445 : int dummy_int;
1577 1445 : auto_vec<tree> dummy_vec;
1578 1445 : if (mid_vectype
1579 1445 : && supportable_widening_operation (IFN_VEC_WIDEN_ABD,
1580 : mid_vectype, vectype_in, false,
1581 : &dummy_code, &dummy_code,
1582 : &dummy_int, &dummy_vec))
1583 : {
1584 0 : ifn = IFN_VEC_WIDEN_ABD;
1585 0 : abd_out_type = mid_type;
1586 0 : vectype_out = mid_vectype;
1587 : }
1588 1445 : }
1589 :
1590 1445 : if (ifn == IFN_ABD
1591 1787 : && !direct_internal_fn_supported_p (ifn, vectype_in,
1592 : OPTIMIZE_FOR_SPEED))
1593 : return NULL;
1594 :
1595 0 : vect_pattern_detected ("vect_recog_abd_pattern", last_stmt);
1596 :
1597 0 : tree abd_oprnds[2];
1598 0 : vect_convert_inputs (vinfo, stmt_vinfo, 2, abd_oprnds,
1599 : abd_in_type, unprom, vectype_in);
1600 :
1601 0 : *type_out = get_vectype_for_scalar_type (vinfo, out_type);
1602 :
1603 0 : tree abd_result = vect_recog_temp_ssa_var (abd_out_type, NULL);
1604 0 : gcall *abd_stmt = gimple_build_call_internal (ifn, 2,
1605 : abd_oprnds[0], abd_oprnds[1]);
1606 0 : gimple_call_set_lhs (abd_stmt, abd_result);
1607 0 : gimple_set_location (abd_stmt, gimple_location (last_stmt));
1608 :
1609 0 : gimple *stmt = abd_stmt;
1610 0 : if (TYPE_PRECISION (abd_in_type) == TYPE_PRECISION (abd_out_type)
1611 0 : && TYPE_PRECISION (abd_out_type) < TYPE_PRECISION (out_type)
1612 0 : && !TYPE_UNSIGNED (abd_out_type))
1613 : {
1614 0 : tree unsign = unsigned_type_for (abd_out_type);
1615 0 : stmt = vect_convert_output (vinfo, stmt_vinfo, unsign, stmt, vectype_out);
1616 0 : vectype_out = get_vectype_for_scalar_type (vinfo, unsign);
1617 : }
1618 :
1619 0 : return vect_convert_output (vinfo, stmt_vinfo, out_type, stmt, vectype_out);
1620 : }
1621 :
1622 : /* Recognize an operation that performs ORIG_CODE on widened inputs,
1623 : so that it can be treated as though it had the form:
1624 :
1625 : A_TYPE a;
1626 : B_TYPE b;
1627 : HALF_TYPE a_cast = (HALF_TYPE) a; // possible no-op
1628 : HALF_TYPE b_cast = (HALF_TYPE) b; // possible no-op
1629 : | RES_TYPE a_extend = (RES_TYPE) a_cast; // promotion from HALF_TYPE
1630 : | RES_TYPE b_extend = (RES_TYPE) b_cast; // promotion from HALF_TYPE
1631 : | RES_TYPE res = a_extend ORIG_CODE b_extend;
1632 :
1633 : Try to replace the pattern with:
1634 :
1635 : A_TYPE a;
1636 : B_TYPE b;
1637 : HALF_TYPE a_cast = (HALF_TYPE) a; // possible no-op
1638 : HALF_TYPE b_cast = (HALF_TYPE) b; // possible no-op
1639 : | EXT_TYPE ext = a_cast WIDE_CODE b_cast;
1640 : | RES_TYPE res = (EXT_TYPE) ext; // possible no-op
1641 :
1642 : where EXT_TYPE is wider than HALF_TYPE but has the same signedness.
1643 :
1644 : SHIFT_P is true if ORIG_CODE and WIDE_CODE are shifts. NAME is the
1645 : name of the pattern being matched, for dump purposes. */
1646 :
1647 : static gimple *
1648 129827369 : vect_recog_widen_op_pattern (vec_info *vinfo,
1649 : stmt_vec_info last_stmt_info, tree *type_out,
1650 : tree_code orig_code, code_helper wide_code,
1651 : bool shift_p, const char *name)
1652 : {
1653 129827369 : gimple *last_stmt = last_stmt_info->stmt;
1654 :
1655 389482107 : vect_unpromoted_value unprom[2];
1656 129827369 : tree half_type;
1657 129827369 : if (!vect_widened_op_tree (vinfo, last_stmt_info, orig_code, orig_code,
1658 : shift_p, 2, unprom, &half_type))
1659 :
1660 : return NULL;
1661 :
1662 : /* Pattern detected. */
1663 342725 : vect_pattern_detected (name, last_stmt);
1664 :
1665 342725 : tree type = TREE_TYPE (gimple_get_lhs (last_stmt));
1666 342725 : tree itype = type;
1667 342725 : if (TYPE_PRECISION (type) != TYPE_PRECISION (half_type) * 2
1668 342725 : || TYPE_UNSIGNED (type) != TYPE_UNSIGNED (half_type))
1669 234012 : itype = build_nonstandard_integer_type (TYPE_PRECISION (half_type) * 2,
1670 234012 : TYPE_UNSIGNED (half_type));
1671 :
1672 : /* Check target support */
1673 342725 : tree vectype = get_vectype_for_scalar_type (vinfo, half_type);
1674 342725 : tree vecitype = get_vectype_for_scalar_type (vinfo, itype);
1675 342725 : tree ctype = itype;
1676 342725 : tree vecctype = vecitype;
1677 342725 : if (orig_code == MINUS_EXPR
1678 6417 : && TYPE_UNSIGNED (itype)
1679 347231 : && TYPE_PRECISION (type) > TYPE_PRECISION (itype))
1680 : {
1681 : /* Subtraction is special, even if half_type is unsigned and no matter
1682 : whether type is signed or unsigned, if type is wider than itype,
1683 : we need to sign-extend from the widening operation result to the
1684 : result type.
1685 : Consider half_type unsigned char, operand 1 0xfe, operand 2 0xff,
1686 : itype unsigned short and type either int or unsigned int.
1687 : Widened (unsigned short) 0xfe - (unsigned short) 0xff is
1688 : (unsigned short) 0xffff, but for type int we want the result -1
1689 : and for type unsigned int 0xffffffff rather than 0xffff. */
1690 790 : ctype = build_nonstandard_integer_type (TYPE_PRECISION (itype), 0);
1691 790 : vecctype = get_vectype_for_scalar_type (vinfo, ctype);
1692 : }
1693 :
1694 342725 : code_helper dummy_code;
1695 342725 : int dummy_int;
1696 342725 : auto_vec<tree> dummy_vec;
1697 342725 : if (!vectype
1698 342725 : || !vecitype
1699 263551 : || !vecctype
1700 606276 : || !supportable_widening_operation (wide_code, vecitype, vectype, true,
1701 : &dummy_code, &dummy_code,
1702 : &dummy_int, &dummy_vec))
1703 : return NULL;
1704 :
1705 114699 : *type_out = get_vectype_for_scalar_type (vinfo, type);
1706 114699 : if (!*type_out)
1707 : return NULL;
1708 :
1709 114699 : tree oprnd[2];
1710 114699 : vect_convert_inputs (vinfo, last_stmt_info,
1711 : 2, oprnd, half_type, unprom, vectype);
1712 :
1713 114699 : tree var = vect_recog_temp_ssa_var (itype, NULL);
1714 114699 : gimple *pattern_stmt = vect_gimple_build (var, wide_code, oprnd[0], oprnd[1]);
1715 :
1716 114699 : if (vecctype != vecitype)
1717 0 : pattern_stmt = vect_convert_output (vinfo, last_stmt_info, ctype,
1718 : pattern_stmt, vecitype);
1719 :
1720 114699 : return vect_convert_output (vinfo, last_stmt_info,
1721 114699 : type, pattern_stmt, vecctype);
1722 342725 : }
1723 :
1724 : /* Try to detect multiplication on widened inputs, converting MULT_EXPR
1725 : to WIDEN_MULT_EXPR. See vect_recog_widen_op_pattern for details. */
1726 :
1727 : static gimple *
1728 32332351 : vect_recog_widen_mult_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
1729 : tree *type_out)
1730 : {
1731 32332351 : return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
1732 32332351 : MULT_EXPR, WIDEN_MULT_EXPR, false,
1733 32332351 : "vect_recog_widen_mult_pattern");
1734 : }
1735 :
1736 : /* Try to detect addition on widened inputs, converting PLUS_EXPR
1737 : to IFN_VEC_WIDEN_PLUS. See vect_recog_widen_op_pattern for details. */
1738 :
1739 : static gimple *
1740 32595576 : vect_recog_widen_plus_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
1741 : tree *type_out)
1742 : {
1743 32595576 : return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
1744 32595576 : PLUS_EXPR, IFN_VEC_WIDEN_PLUS,
1745 32595576 : false, "vect_recog_widen_plus_pattern");
1746 : }
1747 :
1748 : /* Try to detect subtraction on widened inputs, converting MINUS_EXPR
1749 : to IFN_VEC_WIDEN_MINUS. See vect_recog_widen_op_pattern for details. */
1750 : static gimple *
1751 32595576 : vect_recog_widen_minus_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
1752 : tree *type_out)
1753 : {
1754 32595576 : return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
1755 32595576 : MINUS_EXPR, IFN_VEC_WIDEN_MINUS,
1756 32595576 : false, "vect_recog_widen_minus_pattern");
1757 : }
1758 :
1759 : /* Try to detect abd on widened inputs, converting IFN_ABD
1760 : to IFN_VEC_WIDEN_ABD. */
1761 : static gimple *
1762 32595576 : vect_recog_widen_abd_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
1763 : tree *type_out)
1764 : {
1765 32595576 : gassign *last_stmt = dyn_cast <gassign *> (STMT_VINFO_STMT (stmt_vinfo));
1766 30708906 : if (!last_stmt || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (last_stmt)))
1767 : return NULL;
1768 :
1769 3236377 : tree last_rhs = gimple_assign_rhs1 (last_stmt);
1770 :
1771 3236377 : tree in_type = TREE_TYPE (last_rhs);
1772 3236377 : tree out_type = TREE_TYPE (gimple_assign_lhs (last_stmt));
1773 3236377 : if (!INTEGRAL_TYPE_P (in_type)
1774 2832116 : || !INTEGRAL_TYPE_P (out_type)
1775 2715299 : || TYPE_PRECISION (in_type) * 2 != TYPE_PRECISION (out_type)
1776 3900680 : || !TYPE_UNSIGNED (in_type))
1777 : return NULL;
1778 :
1779 239639 : vect_unpromoted_value unprom;
1780 239639 : tree op = vect_look_through_possible_promotion (vinfo, last_rhs, &unprom);
1781 239639 : if (!op || TYPE_PRECISION (TREE_TYPE (op)) != TYPE_PRECISION (in_type))
1782 : return NULL;
1783 :
1784 237093 : stmt_vec_info abd_pattern_vinfo = vect_get_internal_def (vinfo, op);
1785 237093 : if (!abd_pattern_vinfo)
1786 : return NULL;
1787 :
1788 211878 : gcall *abd_stmt = dyn_cast <gcall *> (STMT_VINFO_STMT (abd_pattern_vinfo));
1789 9184 : if (!abd_stmt
1790 9184 : || !gimple_call_internal_p (abd_stmt)
1791 314 : || gimple_call_internal_fn (abd_stmt) != IFN_ABD)
1792 : return NULL;
1793 :
1794 0 : tree vectype_in = get_vectype_for_scalar_type (vinfo, in_type);
1795 0 : tree vectype_out = get_vectype_for_scalar_type (vinfo, out_type);
1796 :
1797 0 : code_helper dummy_code;
1798 0 : int dummy_int;
1799 0 : auto_vec<tree> dummy_vec;
1800 0 : if (!supportable_widening_operation (IFN_VEC_WIDEN_ABD, vectype_out,
1801 : vectype_in, false,
1802 : &dummy_code, &dummy_code,
1803 : &dummy_int, &dummy_vec))
1804 : return NULL;
1805 :
1806 0 : vect_pattern_detected ("vect_recog_widen_abd_pattern", last_stmt);
1807 :
1808 0 : *type_out = vectype_out;
1809 :
1810 0 : tree abd_oprnd0 = gimple_call_arg (abd_stmt, 0);
1811 0 : tree abd_oprnd1 = gimple_call_arg (abd_stmt, 1);
1812 0 : tree widen_abd_result = vect_recog_temp_ssa_var (out_type, NULL);
1813 0 : gcall *widen_abd_stmt = gimple_build_call_internal (IFN_VEC_WIDEN_ABD, 2,
1814 : abd_oprnd0, abd_oprnd1);
1815 0 : gimple_call_set_lhs (widen_abd_stmt, widen_abd_result);
1816 0 : gimple_set_location (widen_abd_stmt, gimple_location (last_stmt));
1817 0 : return widen_abd_stmt;
1818 0 : }
1819 :
1820 : /* Function vect_recog_ctz_ffs_pattern
1821 :
1822 : Try to find the following pattern:
1823 :
1824 : TYPE1 A;
1825 : TYPE1 B;
1826 :
1827 : B = __builtin_ctz{,l,ll} (A);
1828 :
1829 : or
1830 :
1831 : B = __builtin_ffs{,l,ll} (A);
1832 :
1833 : Input:
1834 :
1835 : * STMT_VINFO: The stmt from which the pattern search begins.
1836 : here it starts with B = __builtin_* (A);
1837 :
1838 : Output:
1839 :
1840 : * TYPE_OUT: The vector type of the output of this pattern.
1841 :
1842 : * Return value: A new stmt that will be used to replace the sequence of
1843 : stmts that constitute the pattern, using clz or popcount builtins. */
1844 :
1845 : static gimple *
1846 32303673 : vect_recog_ctz_ffs_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
1847 : tree *type_out)
1848 : {
1849 32303673 : gimple *call_stmt = stmt_vinfo->stmt;
1850 32303673 : gimple *pattern_stmt;
1851 32303673 : tree rhs_oprnd, rhs_type, lhs_oprnd, lhs_type, vec_type, vec_rhs_type;
1852 32303673 : tree new_var;
1853 32303673 : internal_fn ifn = IFN_LAST, ifnnew = IFN_LAST;
1854 32303673 : bool defined_at_zero = true, defined_at_zero_new = false;
1855 32303673 : int val = 0, val_new = 0, val_cmp = 0;
1856 32303673 : int prec;
1857 32303673 : int sub = 0, add = 0;
1858 32303673 : location_t loc;
1859 :
1860 32303673 : if (!is_gimple_call (call_stmt))
1861 : return NULL;
1862 :
1863 3810263 : if (gimple_call_num_args (call_stmt) != 1
1864 3810263 : && gimple_call_num_args (call_stmt) != 2)
1865 : return NULL;
1866 :
1867 2171672 : rhs_oprnd = gimple_call_arg (call_stmt, 0);
1868 2171672 : rhs_type = TREE_TYPE (rhs_oprnd);
1869 2171672 : lhs_oprnd = gimple_call_lhs (call_stmt);
1870 2171672 : if (!lhs_oprnd)
1871 : return NULL;
1872 1043835 : lhs_type = TREE_TYPE (lhs_oprnd);
1873 1043835 : if (!INTEGRAL_TYPE_P (lhs_type)
1874 342219 : || !INTEGRAL_TYPE_P (rhs_type)
1875 47867 : || !type_has_mode_precision_p (rhs_type)
1876 1090115 : || TREE_CODE (rhs_oprnd) != SSA_NAME)
1877 : return NULL;
1878 :
1879 33112 : switch (gimple_call_combined_fn (call_stmt))
1880 : {
1881 1980 : CASE_CFN_CTZ:
1882 1980 : ifn = IFN_CTZ;
1883 1980 : if (!gimple_call_internal_p (call_stmt)
1884 1980 : || gimple_call_num_args (call_stmt) != 2)
1885 : defined_at_zero = false;
1886 : else
1887 121 : val = tree_to_shwi (gimple_call_arg (call_stmt, 1));
1888 : break;
1889 : CASE_CFN_FFS:
1890 : ifn = IFN_FFS;
1891 : break;
1892 : default:
1893 : return NULL;
1894 : }
1895 :
1896 2215 : prec = TYPE_PRECISION (rhs_type);
1897 2215 : loc = gimple_location (call_stmt);
1898 :
1899 2215 : vec_type = get_vectype_for_scalar_type (vinfo, lhs_type);
1900 2215 : if (!vec_type)
1901 : return NULL;
1902 :
1903 2209 : vec_rhs_type = get_vectype_for_scalar_type (vinfo, rhs_type);
1904 2209 : if (!vec_rhs_type)
1905 : return NULL;
1906 :
1907 : /* Do it only if the backend doesn't have ctz<vector_mode>2 or
1908 : ffs<vector_mode>2 pattern but does have clz<vector_mode>2 or
1909 : popcount<vector_mode>2. */
1910 1946 : if (!vec_type
1911 1946 : || direct_internal_fn_supported_p (ifn, vec_rhs_type,
1912 : OPTIMIZE_FOR_SPEED))
1913 : return NULL;
1914 :
1915 1946 : if (ifn == IFN_FFS
1916 1946 : && direct_internal_fn_supported_p (IFN_CTZ, vec_rhs_type,
1917 : OPTIMIZE_FOR_SPEED))
1918 : {
1919 0 : ifnnew = IFN_CTZ;
1920 0 : defined_at_zero_new
1921 0 : = CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (rhs_type),
1922 : val_new) == 2;
1923 : }
1924 1946 : else if (direct_internal_fn_supported_p (IFN_CLZ, vec_rhs_type,
1925 : OPTIMIZE_FOR_SPEED))
1926 : {
1927 166 : ifnnew = IFN_CLZ;
1928 166 : defined_at_zero_new
1929 166 : = CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (rhs_type),
1930 : val_new) == 2;
1931 : }
1932 166 : if ((ifnnew == IFN_LAST
1933 166 : || (defined_at_zero && !defined_at_zero_new))
1934 1780 : && direct_internal_fn_supported_p (IFN_POPCOUNT, vec_rhs_type,
1935 : OPTIMIZE_FOR_SPEED))
1936 : {
1937 : ifnnew = IFN_POPCOUNT;
1938 : defined_at_zero_new = true;
1939 : val_new = prec;
1940 : }
1941 1820 : if (ifnnew == IFN_LAST)
1942 : return NULL;
1943 :
1944 292 : vect_pattern_detected ("vec_recog_ctz_ffs_pattern", call_stmt);
1945 :
1946 292 : val_cmp = val_new;
1947 292 : if ((ifnnew == IFN_CLZ
1948 292 : && defined_at_zero
1949 106 : && defined_at_zero_new
1950 106 : && val == prec
1951 54 : && val_new == prec)
1952 238 : || (ifnnew == IFN_POPCOUNT && ifn == IFN_CTZ))
1953 : {
1954 137 : if (vect_is_reduction (stmt_vinfo))
1955 : return NULL;
1956 :
1957 : /* .CTZ (X) = PREC - .CLZ ((X - 1) & ~X)
1958 : .CTZ (X) = .POPCOUNT ((X - 1) & ~X). */
1959 137 : if (ifnnew == IFN_CLZ)
1960 54 : sub = prec;
1961 137 : val_cmp = prec;
1962 :
1963 137 : if (!TYPE_UNSIGNED (rhs_type))
1964 : {
1965 12 : rhs_type = unsigned_type_for (rhs_type);
1966 12 : vec_rhs_type = get_vectype_for_scalar_type (vinfo, rhs_type);
1967 12 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
1968 12 : pattern_stmt = gimple_build_assign (new_var, NOP_EXPR, rhs_oprnd);
1969 12 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
1970 : vec_rhs_type);
1971 12 : rhs_oprnd = new_var;
1972 : }
1973 :
1974 137 : tree m1 = vect_recog_temp_ssa_var (rhs_type, NULL);
1975 137 : pattern_stmt = gimple_build_assign (m1, PLUS_EXPR, rhs_oprnd,
1976 : build_int_cst (rhs_type, -1));
1977 137 : gimple_set_location (pattern_stmt, loc);
1978 137 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
1979 :
1980 137 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
1981 137 : pattern_stmt = gimple_build_assign (new_var, BIT_NOT_EXPR, rhs_oprnd);
1982 137 : gimple_set_location (pattern_stmt, loc);
1983 137 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
1984 137 : rhs_oprnd = new_var;
1985 :
1986 137 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
1987 137 : pattern_stmt = gimple_build_assign (new_var, BIT_AND_EXPR,
1988 : m1, rhs_oprnd);
1989 137 : gimple_set_location (pattern_stmt, loc);
1990 137 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
1991 137 : rhs_oprnd = new_var;
1992 137 : }
1993 155 : else if (ifnnew == IFN_CLZ)
1994 : {
1995 112 : if (vect_is_reduction (stmt_vinfo))
1996 : return NULL;
1997 :
1998 : /* .CTZ (X) = (PREC - 1) - .CLZ (X & -X)
1999 : .FFS (X) = PREC - .CLZ (X & -X). */
2000 106 : sub = prec - (ifn == IFN_CTZ);
2001 106 : val_cmp = sub - val_new;
2002 :
2003 106 : tree neg = vect_recog_temp_ssa_var (rhs_type, NULL);
2004 106 : pattern_stmt = gimple_build_assign (neg, NEGATE_EXPR, rhs_oprnd);
2005 106 : gimple_set_location (pattern_stmt, loc);
2006 106 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2007 :
2008 106 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
2009 106 : pattern_stmt = gimple_build_assign (new_var, BIT_AND_EXPR,
2010 : rhs_oprnd, neg);
2011 106 : gimple_set_location (pattern_stmt, loc);
2012 106 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2013 106 : rhs_oprnd = new_var;
2014 : }
2015 43 : else if (ifnnew == IFN_POPCOUNT)
2016 : {
2017 43 : if (vect_is_reduction (stmt_vinfo))
2018 : return NULL;
2019 :
2020 : /* .CTZ (X) = PREC - .POPCOUNT (X | -X)
2021 : .FFS (X) = (PREC + 1) - .POPCOUNT (X | -X). */
2022 43 : sub = prec + (ifn == IFN_FFS);
2023 43 : val_cmp = sub;
2024 :
2025 43 : tree neg = vect_recog_temp_ssa_var (rhs_type, NULL);
2026 43 : pattern_stmt = gimple_build_assign (neg, NEGATE_EXPR, rhs_oprnd);
2027 43 : gimple_set_location (pattern_stmt, loc);
2028 43 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2029 :
2030 43 : new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
2031 43 : pattern_stmt = gimple_build_assign (new_var, BIT_IOR_EXPR,
2032 : rhs_oprnd, neg);
2033 43 : gimple_set_location (pattern_stmt, loc);
2034 43 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
2035 43 : rhs_oprnd = new_var;
2036 : }
2037 0 : else if (ifnnew == IFN_CTZ)
2038 : {
2039 : /* .FFS (X) = .CTZ (X) + 1. */
2040 0 : add = 1;
2041 0 : val_cmp++;
2042 :
2043 0 : if (vect_is_reduction (stmt_vinfo)
2044 0 : && defined_at_zero
2045 0 : && (!defined_at_zero_new || val != val_cmp))
2046 : return NULL;
2047 : }
2048 :
2049 : /* Create B = .IFNNEW (A). */
2050 286 : new_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2051 286 : if ((ifnnew == IFN_CLZ || ifnnew == IFN_CTZ) && defined_at_zero_new)
2052 160 : pattern_stmt
2053 160 : = gimple_build_call_internal (ifnnew, 2, rhs_oprnd,
2054 : build_int_cst (integer_type_node,
2055 160 : val_new));
2056 : else
2057 126 : pattern_stmt = gimple_build_call_internal (ifnnew, 1, rhs_oprnd);
2058 286 : gimple_call_set_lhs (pattern_stmt, new_var);
2059 286 : gimple_set_location (pattern_stmt, loc);
2060 286 : *type_out = vec_type;
2061 :
2062 286 : if (sub)
2063 : {
2064 203 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2065 203 : tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2066 203 : pattern_stmt = gimple_build_assign (ret_var, MINUS_EXPR,
2067 203 : build_int_cst (lhs_type, sub),
2068 : new_var);
2069 203 : gimple_set_location (pattern_stmt, loc);
2070 203 : new_var = ret_var;
2071 : }
2072 83 : else if (add)
2073 : {
2074 0 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2075 0 : tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2076 0 : pattern_stmt = gimple_build_assign (ret_var, PLUS_EXPR, new_var,
2077 0 : build_int_cst (lhs_type, add));
2078 0 : gimple_set_location (pattern_stmt, loc);
2079 0 : new_var = ret_var;
2080 : }
2081 :
2082 286 : if (defined_at_zero
2083 210 : && (!defined_at_zero_new || val != val_cmp))
2084 : {
2085 43 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2086 43 : tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2087 43 : rhs_oprnd = gimple_call_arg (call_stmt, 0);
2088 43 : rhs_type = TREE_TYPE (rhs_oprnd);
2089 43 : tree cmp = vect_recog_temp_ssa_var (boolean_type_node, NULL);
2090 43 : pattern_stmt = gimple_build_assign (cmp, NE_EXPR, rhs_oprnd,
2091 : build_zero_cst (rhs_type));
2092 43 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
2093 : truth_type_for (vec_type), rhs_type);
2094 43 : pattern_stmt = gimple_build_assign (ret_var, COND_EXPR, cmp,
2095 : new_var,
2096 43 : build_int_cst (lhs_type, val));
2097 : }
2098 :
2099 286 : if (dump_enabled_p ())
2100 36 : dump_printf_loc (MSG_NOTE, vect_location,
2101 : "created pattern stmt: %G", pattern_stmt);
2102 :
2103 : return pattern_stmt;
2104 : }
2105 :
2106 : /* Function vect_recog_popcount_clz_ctz_ffs_pattern
2107 :
2108 : Try to find the following pattern:
2109 :
2110 : UTYPE1 A;
2111 : TYPE1 B;
2112 : UTYPE2 temp_in;
2113 : TYPE3 temp_out;
2114 : temp_in = (UTYPE2)A;
2115 :
2116 : temp_out = __builtin_popcount{,l,ll} (temp_in);
2117 : B = (TYPE1) temp_out;
2118 :
2119 : TYPE2 may or may not be equal to TYPE3.
2120 : i.e. TYPE2 is equal to TYPE3 for __builtin_popcount
2121 : i.e. TYPE2 is not equal to TYPE3 for __builtin_popcountll
2122 :
2123 : Input:
2124 :
2125 : * STMT_VINFO: The stmt from which the pattern search begins.
2126 : here it starts with B = (TYPE1) temp_out;
2127 :
2128 : Output:
2129 :
2130 : * TYPE_OUT: The vector type of the output of this pattern.
2131 :
2132 : * Return value: A new stmt that will be used to replace the sequence of
2133 : stmts that constitute the pattern. In this case it will be:
2134 : B = .POPCOUNT (A);
2135 :
2136 : Similarly for clz, ctz and ffs.
2137 : */
2138 :
2139 : static gimple *
2140 32303307 : vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
2141 : stmt_vec_info stmt_vinfo,
2142 : tree *type_out)
2143 : {
2144 32303307 : gassign *last_stmt = dyn_cast <gassign *> (stmt_vinfo->stmt);
2145 22513895 : gimple *call_stmt, *pattern_stmt;
2146 22513895 : tree rhs_oprnd, rhs_origin, lhs_oprnd, lhs_type, vec_type, new_var;
2147 54817202 : internal_fn ifn = IFN_LAST;
2148 54817202 : int addend = 0;
2149 :
2150 : /* Find B = (TYPE1) temp_out. */
2151 22513895 : if (!last_stmt)
2152 : return NULL;
2153 22513895 : tree_code code = gimple_assign_rhs_code (last_stmt);
2154 22513895 : if (!CONVERT_EXPR_CODE_P (code))
2155 : return NULL;
2156 :
2157 3080974 : lhs_oprnd = gimple_assign_lhs (last_stmt);
2158 3080974 : lhs_type = TREE_TYPE (lhs_oprnd);
2159 3080974 : if (!INTEGRAL_TYPE_P (lhs_type))
2160 : return NULL;
2161 :
2162 2802125 : rhs_oprnd = gimple_assign_rhs1 (last_stmt);
2163 2802125 : if (TREE_CODE (rhs_oprnd) != SSA_NAME
2164 2802125 : || !has_single_use (rhs_oprnd))
2165 : return NULL;
2166 1416883 : call_stmt = SSA_NAME_DEF_STMT (rhs_oprnd);
2167 :
2168 : /* Find temp_out = __builtin_popcount{,l,ll} (temp_in); */
2169 1416883 : if (!is_gimple_call (call_stmt))
2170 : return NULL;
2171 59731 : switch (gimple_call_combined_fn (call_stmt))
2172 : {
2173 : int val;
2174 : CASE_CFN_POPCOUNT:
2175 : ifn = IFN_POPCOUNT;
2176 : break;
2177 2376 : CASE_CFN_CLZ:
2178 2376 : ifn = IFN_CLZ;
2179 : /* Punt if call result is unsigned and defined value at zero
2180 : is negative, as the negative value doesn't extend correctly. */
2181 2376 : if (TYPE_UNSIGNED (TREE_TYPE (rhs_oprnd))
2182 0 : && gimple_call_internal_p (call_stmt)
2183 2376 : && CLZ_DEFINED_VALUE_AT_ZERO
2184 : (SCALAR_INT_TYPE_MODE (TREE_TYPE (rhs_oprnd)), val) == 2
2185 2376 : && val < 0)
2186 : return NULL;
2187 : break;
2188 744 : CASE_CFN_CTZ:
2189 744 : ifn = IFN_CTZ;
2190 : /* Punt if call result is unsigned and defined value at zero
2191 : is negative, as the negative value doesn't extend correctly. */
2192 744 : if (TYPE_UNSIGNED (TREE_TYPE (rhs_oprnd))
2193 0 : && gimple_call_internal_p (call_stmt)
2194 744 : && CTZ_DEFINED_VALUE_AT_ZERO
2195 : (SCALAR_INT_TYPE_MODE (TREE_TYPE (rhs_oprnd)), val) == 2
2196 744 : && val < 0)
2197 : return NULL;
2198 : break;
2199 57 : CASE_CFN_FFS:
2200 57 : ifn = IFN_FFS;
2201 57 : break;
2202 : default:
2203 : return NULL;
2204 : }
2205 :
2206 3530 : if (gimple_call_num_args (call_stmt) != 1
2207 3530 : && gimple_call_num_args (call_stmt) != 2)
2208 : return NULL;
2209 :
2210 3530 : rhs_oprnd = gimple_call_arg (call_stmt, 0);
2211 3530 : vect_unpromoted_value unprom_diff;
2212 3530 : rhs_origin
2213 3530 : = vect_look_through_possible_promotion (vinfo, rhs_oprnd, &unprom_diff);
2214 :
2215 3530 : if (!rhs_origin)
2216 : return NULL;
2217 :
2218 : /* Input and output of .POPCOUNT should be same-precision integer. */
2219 3520 : if (TYPE_PRECISION (unprom_diff.type) != TYPE_PRECISION (lhs_type))
2220 : return NULL;
2221 :
2222 : /* Also A should be unsigned or same precision as temp_in, otherwise
2223 : different builtins/internal functions have different behaviors. */
2224 1808 : if (TYPE_PRECISION (unprom_diff.type)
2225 1808 : != TYPE_PRECISION (TREE_TYPE (rhs_oprnd)))
2226 318 : switch (ifn)
2227 : {
2228 146 : case IFN_POPCOUNT:
2229 : /* For popcount require zero extension, which doesn't add any
2230 : further bits to the count. */
2231 146 : if (!TYPE_UNSIGNED (unprom_diff.type))
2232 : return NULL;
2233 : break;
2234 109 : case IFN_CLZ:
2235 : /* clzll (x) == clz (x) + 32 for unsigned x != 0, so ok
2236 : if it is undefined at zero or if it matches also for the
2237 : defined value there. */
2238 109 : if (!TYPE_UNSIGNED (unprom_diff.type))
2239 : return NULL;
2240 109 : if (!type_has_mode_precision_p (lhs_type)
2241 109 : || !type_has_mode_precision_p (TREE_TYPE (rhs_oprnd)))
2242 : return NULL;
2243 109 : addend = (TYPE_PRECISION (TREE_TYPE (rhs_oprnd))
2244 109 : - TYPE_PRECISION (lhs_type));
2245 109 : if (gimple_call_internal_p (call_stmt)
2246 109 : && gimple_call_num_args (call_stmt) == 2)
2247 : {
2248 0 : int val1, val2;
2249 0 : val1 = tree_to_shwi (gimple_call_arg (call_stmt, 1));
2250 0 : int d2
2251 0 : = CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (lhs_type),
2252 : val2);
2253 0 : if (d2 != 2 || val1 != val2 + addend)
2254 : return NULL;
2255 : }
2256 : break;
2257 43 : case IFN_CTZ:
2258 : /* ctzll (x) == ctz (x) for unsigned or signed x != 0, so ok
2259 : if it is undefined at zero or if it matches also for the
2260 : defined value there. */
2261 43 : if (gimple_call_internal_p (call_stmt)
2262 43 : && gimple_call_num_args (call_stmt) == 2)
2263 : {
2264 0 : int val1, val2;
2265 0 : val1 = tree_to_shwi (gimple_call_arg (call_stmt, 1));
2266 0 : int d2
2267 0 : = CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (lhs_type),
2268 : val2);
2269 0 : if (d2 != 2 || val1 != val2)
2270 : return NULL;
2271 : }
2272 : break;
2273 : case IFN_FFS:
2274 : /* ffsll (x) == ffs (x) for unsigned or signed x. */
2275 : break;
2276 : default:
2277 : gcc_unreachable ();
2278 : }
2279 :
2280 1808 : vec_type = get_vectype_for_scalar_type (vinfo, lhs_type);
2281 : /* Do it only if the backend has popcount<vector_mode>2 etc. pattern. */
2282 1808 : if (!vec_type)
2283 : return NULL;
2284 :
2285 1679 : bool supported
2286 1679 : = direct_internal_fn_supported_p (ifn, vec_type, OPTIMIZE_FOR_SPEED);
2287 : /* We cannot rely on the reduction check in vect_recog_ctz_ffs_pattern
2288 : because we do not set up the temporary pattern stmt finally for
2289 : the recursive analysis. */
2290 1679 : if (!supported && vect_is_reduction (stmt_vinfo))
2291 : return NULL;
2292 1663 : if (!supported)
2293 1486 : switch (ifn)
2294 : {
2295 : case IFN_POPCOUNT:
2296 : case IFN_CLZ:
2297 : return NULL;
2298 57 : case IFN_FFS:
2299 : /* vect_recog_ctz_ffs_pattern can implement ffs using ctz. */
2300 57 : if (direct_internal_fn_supported_p (IFN_CTZ, vec_type,
2301 : OPTIMIZE_FOR_SPEED))
2302 : break;
2303 : /* FALLTHRU */
2304 534 : case IFN_CTZ:
2305 : /* vect_recog_ctz_ffs_pattern can implement ffs or ctz using
2306 : clz or popcount. */
2307 534 : if (direct_internal_fn_supported_p (IFN_CLZ, vec_type,
2308 : OPTIMIZE_FOR_SPEED))
2309 : break;
2310 474 : if (direct_internal_fn_supported_p (IFN_POPCOUNT, vec_type,
2311 : OPTIMIZE_FOR_SPEED))
2312 : break;
2313 : return NULL;
2314 : default:
2315 : gcc_unreachable ();
2316 : }
2317 :
2318 318 : vect_pattern_detected ("vec_recog_popcount_clz_ctz_ffs_pattern",
2319 : call_stmt);
2320 :
2321 318 : tree orig_lhs_type = lhs_type;
2322 318 : gimple *cast_stmt = NULL;
2323 318 : if (!type_has_mode_precision_p (lhs_type))
2324 : {
2325 0 : lhs_type = TREE_TYPE (vec_type);
2326 0 : cast_stmt
2327 0 : = gimple_build_assign (vect_recog_temp_ssa_var (lhs_type, NULL),
2328 : NOP_EXPR, unprom_diff.op);
2329 0 : unprom_diff.op = gimple_assign_lhs (cast_stmt);
2330 : }
2331 :
2332 : /* Create B = .POPCOUNT (A). */
2333 318 : new_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2334 318 : tree arg2 = NULL_TREE;
2335 318 : int val;
2336 318 : if (ifn == IFN_CLZ
2337 368 : && CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (lhs_type),
2338 : val) == 2)
2339 48 : arg2 = build_int_cst (integer_type_node, val);
2340 270 : else if (ifn == IFN_CTZ
2341 363 : && CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (lhs_type),
2342 : val) == 2)
2343 93 : arg2 = build_int_cst (integer_type_node, val);
2344 318 : if (arg2)
2345 141 : pattern_stmt = gimple_build_call_internal (ifn, 2, unprom_diff.op, arg2);
2346 : else
2347 177 : pattern_stmt = gimple_build_call_internal (ifn, 1, unprom_diff.op);
2348 318 : gimple_call_set_lhs (pattern_stmt, new_var);
2349 318 : gimple_set_location (pattern_stmt, gimple_location (last_stmt));
2350 318 : *type_out = vec_type;
2351 :
2352 318 : if (dump_enabled_p ())
2353 24 : dump_printf_loc (MSG_NOTE, vect_location,
2354 : "created pattern stmt: %G", pattern_stmt);
2355 :
2356 318 : if (addend)
2357 : {
2358 12 : gcc_assert (supported);
2359 12 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2360 12 : tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
2361 12 : pattern_stmt = gimple_build_assign (ret_var, PLUS_EXPR, new_var,
2362 12 : build_int_cst (lhs_type, addend));
2363 : }
2364 306 : else if (!supported)
2365 : {
2366 141 : stmt_vec_info new_stmt_info = vinfo->add_stmt (pattern_stmt);
2367 141 : STMT_VINFO_VECTYPE (new_stmt_info) = vec_type;
2368 141 : pattern_stmt
2369 141 : = vect_recog_ctz_ffs_pattern (vinfo, new_stmt_info, type_out);
2370 141 : if (pattern_stmt == NULL)
2371 : return NULL;
2372 141 : if (cast_stmt)
2373 0 : append_pattern_def_seq (vinfo, stmt_vinfo, cast_stmt, vec_type);
2374 141 : if (gimple_seq seq = STMT_VINFO_PATTERN_DEF_SEQ (new_stmt_info))
2375 : {
2376 141 : gimple_seq *pseq = &STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo);
2377 141 : gimple_seq_add_seq_without_update (pseq, seq);
2378 : }
2379 : }
2380 165 : else if (cast_stmt)
2381 0 : append_pattern_def_seq (vinfo, stmt_vinfo, cast_stmt, vec_type);
2382 :
2383 153 : if (cast_stmt)
2384 : {
2385 0 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
2386 0 : tree ret_var = vect_recog_temp_ssa_var (orig_lhs_type, NULL);
2387 0 : pattern_stmt = gimple_build_assign (ret_var, NOP_EXPR,
2388 : gimple_get_lhs (pattern_stmt));
2389 : }
2390 : return pattern_stmt;
2391 : }
2392 :
2393 : /* Function vect_recog_pow_pattern
2394 :
2395 : Try to find the following pattern:
2396 :
2397 : x = POW (y, N);
2398 :
2399 : with POW being one of pow, powf, powi, powif and N being
2400 : either 2 or 0.5.
2401 :
2402 : Input:
2403 :
2404 : * STMT_VINFO: The stmt from which the pattern search begins.
2405 :
2406 : Output:
2407 :
2408 : * TYPE_OUT: The type of the output of this pattern.
2409 :
2410 : * Return value: A new stmt that will be used to replace the sequence of
2411 : stmts that constitute the pattern. In this case it will be:
2412 : x = x * x
2413 : or
2414 : x = sqrt (x)
2415 : */
2416 :
2417 : static gimple *
2418 32303324 : vect_recog_pow_pattern (vec_info *vinfo,
2419 : stmt_vec_info stmt_vinfo, tree *type_out)
2420 : {
2421 32303324 : gimple *last_stmt = stmt_vinfo->stmt;
2422 32303324 : tree base, exp;
2423 32303324 : gimple *stmt;
2424 32303324 : tree var;
2425 :
2426 32303324 : if (!is_gimple_call (last_stmt) || gimple_call_lhs (last_stmt) == NULL)
2427 : return NULL;
2428 :
2429 1533244 : switch (gimple_call_combined_fn (last_stmt))
2430 : {
2431 527 : CASE_CFN_POW:
2432 527 : CASE_CFN_POWI:
2433 527 : break;
2434 :
2435 : default:
2436 : return NULL;
2437 : }
2438 :
2439 527 : base = gimple_call_arg (last_stmt, 0);
2440 527 : exp = gimple_call_arg (last_stmt, 1);
2441 527 : if (TREE_CODE (exp) != REAL_CST
2442 472 : && TREE_CODE (exp) != INTEGER_CST)
2443 : {
2444 472 : if (flag_unsafe_math_optimizations
2445 37 : && TREE_CODE (base) == REAL_CST
2446 475 : && gimple_call_builtin_p (last_stmt, BUILT_IN_NORMAL))
2447 : {
2448 3 : combined_fn log_cfn;
2449 3 : built_in_function exp_bfn;
2450 3 : switch (DECL_FUNCTION_CODE (gimple_call_fndecl (last_stmt)))
2451 : {
2452 : case BUILT_IN_POW:
2453 : log_cfn = CFN_BUILT_IN_LOG;
2454 : exp_bfn = BUILT_IN_EXP;
2455 : break;
2456 0 : case BUILT_IN_POWF:
2457 0 : log_cfn = CFN_BUILT_IN_LOGF;
2458 0 : exp_bfn = BUILT_IN_EXPF;
2459 0 : break;
2460 0 : case BUILT_IN_POWL:
2461 0 : log_cfn = CFN_BUILT_IN_LOGL;
2462 0 : exp_bfn = BUILT_IN_EXPL;
2463 0 : break;
2464 : default:
2465 : return NULL;
2466 : }
2467 3 : tree logc = fold_const_call (log_cfn, TREE_TYPE (base), base);
2468 3 : tree exp_decl = builtin_decl_implicit (exp_bfn);
2469 : /* Optimize pow (C, x) as exp (log (C) * x). Normally match.pd
2470 : does that, but if C is a power of 2, we want to use
2471 : exp2 (log2 (C) * x) in the non-vectorized version, but for
2472 : vectorization we don't have vectorized exp2. */
2473 3 : if (logc
2474 3 : && TREE_CODE (logc) == REAL_CST
2475 3 : && exp_decl
2476 6 : && lookup_attribute ("omp declare simd",
2477 3 : DECL_ATTRIBUTES (exp_decl)))
2478 : {
2479 3 : cgraph_node *node = cgraph_node::get_create (exp_decl);
2480 3 : if (node->simd_clones == NULL)
2481 : {
2482 2 : if (targetm.simd_clone.compute_vecsize_and_simdlen == NULL
2483 2 : || node->definition)
2484 : return NULL;
2485 2 : expand_simd_clones (node);
2486 2 : if (node->simd_clones == NULL)
2487 : return NULL;
2488 : }
2489 3 : *type_out = get_vectype_for_scalar_type (vinfo, TREE_TYPE (base));
2490 3 : if (!*type_out)
2491 : return NULL;
2492 3 : tree def = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
2493 3 : gimple *g = gimple_build_assign (def, MULT_EXPR, exp, logc);
2494 3 : append_pattern_def_seq (vinfo, stmt_vinfo, g);
2495 3 : tree res = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
2496 3 : g = gimple_build_call (exp_decl, 1, def);
2497 3 : gimple_call_set_lhs (g, res);
2498 3 : return g;
2499 : }
2500 : }
2501 :
2502 : return NULL;
2503 : }
2504 :
2505 : /* We now have a pow or powi builtin function call with a constant
2506 : exponent. */
2507 :
2508 : /* Catch squaring. */
2509 55 : if ((tree_fits_shwi_p (exp)
2510 0 : && tree_to_shwi (exp) == 2)
2511 55 : || (TREE_CODE (exp) == REAL_CST
2512 55 : && real_equal (&TREE_REAL_CST (exp), &dconst2)))
2513 : {
2514 11 : if (!vect_supportable_direct_optab_p (vinfo, TREE_TYPE (base), MULT_EXPR,
2515 11 : TREE_TYPE (base), type_out))
2516 : return NULL;
2517 :
2518 9 : var = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
2519 9 : stmt = gimple_build_assign (var, MULT_EXPR, base, base);
2520 9 : return stmt;
2521 : }
2522 :
2523 : /* Catch square root. */
2524 44 : if (TREE_CODE (exp) == REAL_CST
2525 44 : && real_equal (&TREE_REAL_CST (exp), &dconsthalf))
2526 : {
2527 10 : *type_out = get_vectype_for_scalar_type (vinfo, TREE_TYPE (base));
2528 10 : if (*type_out
2529 10 : && direct_internal_fn_supported_p (IFN_SQRT, *type_out,
2530 : OPTIMIZE_FOR_SPEED))
2531 : {
2532 8 : gcall *stmt = gimple_build_call_internal (IFN_SQRT, 1, base);
2533 8 : var = vect_recog_temp_ssa_var (TREE_TYPE (base), stmt);
2534 8 : gimple_call_set_lhs (stmt, var);
2535 8 : gimple_call_set_nothrow (stmt, true);
2536 8 : return stmt;
2537 : }
2538 : }
2539 :
2540 : return NULL;
2541 : }
2542 :
2543 :
2544 : /* Function vect_recog_widen_sum_pattern
2545 :
2546 : Try to find the following pattern:
2547 :
2548 : type x_t;
2549 : TYPE x_T, sum = init;
2550 : loop:
2551 : sum_0 = phi <init, sum_1>
2552 : S1 x_t = *p;
2553 : S2 x_T = (TYPE) x_t;
2554 : S3 sum_1 = x_T + sum_0;
2555 :
2556 : where type 'TYPE' is at least double the size of type 'type', i.e - we're
2557 : summing elements of type 'type' into an accumulator of type 'TYPE'. This is
2558 : a special case of a reduction computation.
2559 :
2560 : Input:
2561 :
2562 : * STMT_VINFO: The stmt from which the pattern search begins. In the example,
2563 : when this function is called with S3, the pattern {S2,S3} will be detected.
2564 :
2565 : Output:
2566 :
2567 : * TYPE_OUT: The type of the output of this pattern.
2568 :
2569 : * Return value: A new stmt that will be used to replace the sequence of
2570 : stmts that constitute the pattern. In this case it will be:
2571 : WIDEN_SUM <x_t, sum_0>
2572 :
2573 : Note: The widening-sum idiom is a widening reduction pattern that is
2574 : vectorized without preserving all the intermediate results. It
2575 : produces only N/2 (widened) results (by summing up pairs of
2576 : intermediate results) rather than all N results. Therefore, we
2577 : cannot allow this pattern when we want to get all the results and in
2578 : the correct order (as is the case when this computation is in an
2579 : inner-loop nested in an outer-loop that us being vectorized). */
2580 :
2581 : static gimple *
2582 32303324 : vect_recog_widen_sum_pattern (vec_info *vinfo,
2583 : stmt_vec_info stmt_vinfo, tree *type_out)
2584 : {
2585 32303324 : gimple *last_stmt = stmt_vinfo->stmt;
2586 32303324 : tree oprnd0, oprnd1;
2587 32303324 : tree type;
2588 32303324 : gimple *pattern_stmt;
2589 32303324 : tree var;
2590 :
2591 : /* Look for the following pattern
2592 : DX = (TYPE) X;
2593 : sum_1 = DX + sum_0;
2594 : In which DX is at least double the size of X, and sum_1 has been
2595 : recognized as a reduction variable.
2596 : */
2597 :
2598 : /* Starting from LAST_STMT, follow the defs of its uses in search
2599 : of the above pattern. */
2600 :
2601 32303324 : if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, PLUS_EXPR,
2602 : &oprnd0, &oprnd1)
2603 52745 : || TREE_CODE (oprnd0) != SSA_NAME
2604 32355800 : || !vinfo->lookup_def (oprnd0))
2605 : return NULL;
2606 :
2607 52399 : type = TREE_TYPE (gimple_get_lhs (last_stmt));
2608 :
2609 : /* So far so good. Since last_stmt was detected as a (summation) reduction,
2610 : we know that oprnd1 is the reduction variable (defined by a loop-header
2611 : phi), and oprnd0 is an ssa-name defined by a stmt in the loop body.
2612 : Left to check that oprnd0 is defined by a cast from type 'type' to type
2613 : 'TYPE'. */
2614 :
2615 52399 : vect_unpromoted_value unprom0;
2616 52399 : if (!vect_look_through_possible_promotion (vinfo, oprnd0, &unprom0)
2617 52399 : || TYPE_PRECISION (unprom0.type) * 2 > TYPE_PRECISION (type))
2618 : return NULL;
2619 :
2620 2311 : vect_pattern_detected ("vect_recog_widen_sum_pattern", last_stmt);
2621 :
2622 : /* If the exact input type is not supported, widen it to an intermediate
2623 : type before adding it to the accumulator. Start with the narrowest
2624 : type in order to retain the largest vectorization factor. */
2625 2311 : tree input_type = unprom0.type;
2626 2311 : tree input_vectype = NULL_TREE;
2627 :
2628 4258 : while (!vect_supportable_conv_optab_p (vinfo, type, WIDEN_SUM_EXPR,
2629 : input_type, type_out, &input_vectype))
2630 : {
2631 4258 : unsigned int precision
2632 4258 : = vect_element_precision (TYPE_PRECISION (input_type) + 1);
2633 :
2634 4258 : if (precision > TYPE_PRECISION (type) / 2)
2635 : return NULL;
2636 :
2637 1947 : input_type = build_nonstandard_integer_type
2638 1947 : (precision, TYPE_UNSIGNED (unprom0.type));
2639 : }
2640 :
2641 0 : tree input = vect_convert_input (vinfo, stmt_vinfo, input_type, &unprom0,
2642 : input_vectype);
2643 :
2644 0 : var = vect_recog_temp_ssa_var (type, NULL);
2645 0 : pattern_stmt = gimple_build_assign (var, WIDEN_SUM_EXPR, input, oprnd1);
2646 :
2647 0 : return pattern_stmt;
2648 : }
2649 :
2650 : /* Function vect_recog_bitfield_ref_pattern
2651 :
2652 : Try to find the following pattern:
2653 :
2654 : bf_value = BIT_FIELD_REF (container, bitsize, bitpos);
2655 : result = (type_out) bf_value;
2656 :
2657 : or
2658 :
2659 : if (BIT_FIELD_REF (container, bitsize, bitpos) `cmp` <constant>)
2660 :
2661 : where type_out is a non-bitfield type, that is to say, it's precision matches
2662 : 2^(TYPE_SIZE(type_out) - (TYPE_UNSIGNED (type_out) ? 1 : 2)).
2663 :
2664 : Input:
2665 :
2666 : * STMT_VINFO: The stmt from which the pattern search begins.
2667 : here it starts with:
2668 : result = (type_out) bf_value;
2669 :
2670 : or
2671 :
2672 : if (BIT_FIELD_REF (container, bitsize, bitpos) `cmp` <constant>)
2673 :
2674 : Output:
2675 :
2676 : * TYPE_OUT: The vector type of the output of this pattern.
2677 :
2678 : * Return value: A new stmt that will be used to replace the sequence of
2679 : stmts that constitute the pattern. If the precision of type_out is bigger
2680 : than the precision type of _1 we perform the widening before the shifting,
2681 : since the new precision will be large enough to shift the value and moving
2682 : widening operations up the statement chain enables the generation of
2683 : widening loads. If we are widening and the operation after the pattern is
2684 : an addition then we mask first and shift later, to enable the generation of
2685 : shifting adds. In the case of narrowing we will always mask first, shift
2686 : last and then perform a narrowing operation. This will enable the
2687 : generation of narrowing shifts.
2688 :
2689 : Widening with mask first, shift later:
2690 : container = (type_out) container;
2691 : masked = container & (((1 << bitsize) - 1) << bitpos);
2692 : result = masked >> bitpos;
2693 :
2694 : Widening with shift first, mask last:
2695 : container = (type_out) container;
2696 : shifted = container >> bitpos;
2697 : result = shifted & ((1 << bitsize) - 1);
2698 :
2699 : Narrowing:
2700 : masked = container & (((1 << bitsize) - 1) << bitpos);
2701 : result = masked >> bitpos;
2702 : result = (type_out) result;
2703 :
2704 : If the bitfield is signed and it's wider than type_out, we need to
2705 : keep the result sign-extended:
2706 : container = (type) container;
2707 : masked = container << (prec - bitsize - bitpos);
2708 : result = (type_out) (masked >> (prec - bitsize));
2709 :
2710 : Here type is the signed variant of the wider of type_out and the type
2711 : of container.
2712 :
2713 : The shifting is always optional depending on whether bitpos != 0.
2714 :
2715 : When the original bitfield was inside a gcond then an new gcond is also
2716 : generated with the newly `result` as the operand to the comparison.
2717 :
2718 : */
2719 :
2720 : static gimple *
2721 32245000 : vect_recog_bitfield_ref_pattern (vec_info *vinfo, stmt_vec_info stmt_info,
2722 : tree *type_out)
2723 : {
2724 32245000 : gimple *bf_stmt = NULL;
2725 32245000 : tree lhs = NULL_TREE;
2726 32245000 : tree ret_type = NULL_TREE;
2727 32245000 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
2728 32245000 : if (gcond *cond_stmt = dyn_cast <gcond *> (stmt))
2729 : {
2730 5384665 : tree op = gimple_cond_lhs (cond_stmt);
2731 5384665 : if (TREE_CODE (op) != SSA_NAME)
2732 : return NULL;
2733 5384365 : bf_stmt = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (op));
2734 5384365 : if (TREE_CODE (gimple_cond_rhs (cond_stmt)) != INTEGER_CST)
2735 : return NULL;
2736 : }
2737 26860335 : else if (is_gimple_assign (stmt)
2738 22455299 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
2739 29860372 : && TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME)
2740 : {
2741 2958563 : gimple *second_stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmt));
2742 2958563 : bf_stmt = dyn_cast <gassign *> (second_stmt);
2743 2958563 : lhs = gimple_assign_lhs (stmt);
2744 2958563 : ret_type = TREE_TYPE (lhs);
2745 : }
2746 :
2747 6437924 : if (!bf_stmt
2748 6437924 : || gimple_assign_rhs_code (bf_stmt) != BIT_FIELD_REF)
2749 : return NULL;
2750 :
2751 16751 : tree bf_ref = gimple_assign_rhs1 (bf_stmt);
2752 16751 : tree container = TREE_OPERAND (bf_ref, 0);
2753 16751 : ret_type = ret_type ? ret_type : TREE_TYPE (container);
2754 :
2755 16751 : if (!bit_field_offset (bf_ref).is_constant ()
2756 16751 : || !bit_field_size (bf_ref).is_constant ()
2757 16751 : || !tree_fits_uhwi_p (TYPE_SIZE (TREE_TYPE (container))))
2758 : return NULL;
2759 :
2760 33124 : if (!INTEGRAL_TYPE_P (TREE_TYPE (bf_ref))
2761 16749 : || !INTEGRAL_TYPE_P (TREE_TYPE (container))
2762 18924 : || TYPE_MODE (TREE_TYPE (container)) == E_BLKmode)
2763 : return NULL;
2764 :
2765 2173 : gimple *use_stmt, *pattern_stmt;
2766 2173 : use_operand_p use_p;
2767 2173 : bool shift_first = true;
2768 2173 : tree container_type = TREE_TYPE (container);
2769 2173 : tree vectype = get_vectype_for_scalar_type (vinfo, container_type);
2770 :
2771 : /* Calculate shift_n before the adjustments for widening loads, otherwise
2772 : the container may change and we have to consider offset change for
2773 : widening loads on big endianness. The shift_n calculated here can be
2774 : independent of widening. */
2775 2173 : unsigned HOST_WIDE_INT shift_n = bit_field_offset (bf_ref).to_constant ();
2776 2173 : unsigned HOST_WIDE_INT mask_width = bit_field_size (bf_ref).to_constant ();
2777 2173 : unsigned HOST_WIDE_INT prec = tree_to_uhwi (TYPE_SIZE (container_type));
2778 2173 : if (BYTES_BIG_ENDIAN)
2779 : shift_n = prec - shift_n - mask_width;
2780 :
2781 2173 : bool ref_sext = (!TYPE_UNSIGNED (TREE_TYPE (bf_ref)) &&
2782 1400 : TYPE_PRECISION (ret_type) > mask_width);
2783 2173 : bool load_widen = (TYPE_PRECISION (TREE_TYPE (container)) <
2784 2173 : TYPE_PRECISION (ret_type));
2785 :
2786 : /* We move the conversion earlier if the loaded type is smaller than the
2787 : return type to enable the use of widening loads. And if we need a
2788 : sign extension, we need to convert the loaded value early to a signed
2789 : type as well. */
2790 2173 : if (ref_sext || load_widen)
2791 : {
2792 947 : tree type = load_widen ? ret_type : container_type;
2793 947 : if (ref_sext)
2794 908 : type = gimple_signed_type (type);
2795 947 : pattern_stmt = gimple_build_assign (vect_recog_temp_ssa_var (type),
2796 : NOP_EXPR, container);
2797 947 : container = gimple_get_lhs (pattern_stmt);
2798 947 : container_type = TREE_TYPE (container);
2799 947 : prec = tree_to_uhwi (TYPE_SIZE (container_type));
2800 947 : vectype = get_vectype_for_scalar_type (vinfo, container_type);
2801 947 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2802 : }
2803 1226 : else if (!useless_type_conversion_p (TREE_TYPE (container), ret_type))
2804 : /* If we are doing the conversion last then also delay the shift as we may
2805 : be able to combine the shift and conversion in certain cases. */
2806 : shift_first = false;
2807 :
2808 : /* If the only use of the result of this BIT_FIELD_REF + CONVERT is a
2809 : PLUS_EXPR then do the shift last as some targets can combine the shift and
2810 : add into a single instruction. */
2811 1416 : if (lhs && !is_pattern_stmt_p (stmt_info)
2812 3589 : && single_imm_use (lhs, &use_p, &use_stmt))
2813 : {
2814 1049 : if (gimple_code (use_stmt) == GIMPLE_ASSIGN
2815 1049 : && gimple_assign_rhs_code (use_stmt) == PLUS_EXPR)
2816 : shift_first = false;
2817 : }
2818 :
2819 : /* If we don't have to shift we only generate the mask, so just fix the
2820 : code-path to shift_first. */
2821 2173 : if (shift_n == 0)
2822 763 : shift_first = true;
2823 :
2824 2173 : tree result;
2825 2173 : if (shift_first && !ref_sext)
2826 : {
2827 513 : tree shifted = container;
2828 513 : if (shift_n)
2829 : {
2830 65 : pattern_stmt
2831 65 : = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
2832 : RSHIFT_EXPR, container,
2833 65 : build_int_cst (sizetype, shift_n));
2834 65 : shifted = gimple_assign_lhs (pattern_stmt);
2835 65 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2836 : }
2837 :
2838 513 : tree mask = wide_int_to_tree (container_type,
2839 513 : wi::mask (mask_width, false, prec));
2840 :
2841 513 : pattern_stmt
2842 513 : = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
2843 : BIT_AND_EXPR, shifted, mask);
2844 513 : result = gimple_assign_lhs (pattern_stmt);
2845 : }
2846 : else
2847 : {
2848 1660 : tree temp = vect_recog_temp_ssa_var (container_type);
2849 1660 : if (!ref_sext)
2850 : {
2851 752 : tree mask = wide_int_to_tree (container_type,
2852 752 : wi::shifted_mask (shift_n,
2853 : mask_width,
2854 : false, prec));
2855 752 : pattern_stmt = gimple_build_assign (temp, BIT_AND_EXPR,
2856 : container, mask);
2857 : }
2858 : else
2859 : {
2860 908 : HOST_WIDE_INT shl = prec - shift_n - mask_width;
2861 908 : shift_n += shl;
2862 908 : pattern_stmt = gimple_build_assign (temp, LSHIFT_EXPR,
2863 : container,
2864 : build_int_cst (sizetype,
2865 908 : shl));
2866 : }
2867 :
2868 1660 : tree masked = gimple_assign_lhs (pattern_stmt);
2869 1660 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2870 1660 : pattern_stmt
2871 1660 : = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
2872 : RSHIFT_EXPR, masked,
2873 1660 : build_int_cst (sizetype, shift_n));
2874 1660 : result = gimple_assign_lhs (pattern_stmt);
2875 : }
2876 :
2877 2173 : if (!useless_type_conversion_p (TREE_TYPE (result), ret_type))
2878 : {
2879 1444 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2880 1444 : pattern_stmt
2881 1444 : = gimple_build_assign (vect_recog_temp_ssa_var (ret_type),
2882 : NOP_EXPR, result);
2883 : }
2884 :
2885 2173 : if (!lhs)
2886 : {
2887 757 : if (!vectype)
2888 : return NULL;
2889 :
2890 619 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
2891 619 : vectype = truth_type_for (vectype);
2892 :
2893 : /* FIXME: This part extracts the boolean value out of the bitfield in the
2894 : same way as vect_recog_gcond_pattern does. However because
2895 : patterns cannot match the same root twice, when we handle and
2896 : lower the bitfield in the gcond, vect_recog_gcond_pattern can't
2897 : apply anymore. We should really fix it so that we don't need to
2898 : duplicate transformations like these. */
2899 619 : tree new_lhs = vect_recog_temp_ssa_var (boolean_type_node, NULL);
2900 619 : gcond *cond_stmt = dyn_cast <gcond *> (stmt_info->stmt);
2901 619 : tree cond_cst = gimple_cond_rhs (cond_stmt);
2902 619 : gimple *new_stmt
2903 619 : = gimple_build_assign (new_lhs, gimple_cond_code (cond_stmt),
2904 : gimple_get_lhs (pattern_stmt),
2905 : fold_convert (container_type, cond_cst));
2906 619 : append_pattern_def_seq (vinfo, stmt_info, new_stmt, vectype, container_type);
2907 619 : pattern_stmt
2908 619 : = gimple_build_cond (NE_EXPR, new_lhs,
2909 619 : build_zero_cst (TREE_TYPE (new_lhs)),
2910 : NULL_TREE, NULL_TREE);
2911 : }
2912 :
2913 2035 : *type_out = STMT_VINFO_VECTYPE (stmt_info);
2914 2035 : vect_pattern_detected ("bitfield_ref pattern", stmt_info->stmt);
2915 :
2916 2035 : return pattern_stmt;
2917 : }
2918 :
2919 : /* Function vect_recog_bit_insert_pattern
2920 :
2921 : Try to find the following pattern:
2922 :
2923 : written = BIT_INSERT_EXPR (container, value, bitpos);
2924 :
2925 : Input:
2926 :
2927 : * STMT_VINFO: The stmt we want to replace.
2928 :
2929 : Output:
2930 :
2931 : * TYPE_OUT: The vector type of the output of this pattern.
2932 :
2933 : * Return value: A new stmt that will be used to replace the sequence of
2934 : stmts that constitute the pattern. In this case it will be:
2935 : value = (container_type) value; // Make sure
2936 : shifted = value << bitpos; // Shift value into place
2937 : masked = shifted & (mask << bitpos); // Mask off the non-relevant bits in
2938 : // the 'to-write value'.
2939 : cleared = container & ~(mask << bitpos); // Clearing the bits we want to
2940 : // write to from the value we want
2941 : // to write to.
2942 : written = cleared | masked; // Write bits.
2943 :
2944 :
2945 : where mask = ((1 << TYPE_PRECISION (value)) - 1), a mask to keep the number of
2946 : bits corresponding to the real size of the bitfield value we are writing to.
2947 : The shifting is always optional depending on whether bitpos != 0.
2948 :
2949 : */
2950 :
2951 : static gimple *
2952 32248050 : vect_recog_bit_insert_pattern (vec_info *vinfo, stmt_vec_info stmt_info,
2953 : tree *type_out)
2954 : {
2955 32248050 : gassign *bf_stmt = dyn_cast <gassign *> (stmt_info->stmt);
2956 29980815 : if (!bf_stmt || gimple_assign_rhs_code (bf_stmt) != BIT_INSERT_EXPR)
2957 : return NULL;
2958 :
2959 680 : tree container = gimple_assign_rhs1 (bf_stmt);
2960 680 : tree value = gimple_assign_rhs2 (bf_stmt);
2961 680 : tree shift = gimple_assign_rhs3 (bf_stmt);
2962 :
2963 680 : tree bf_type = TREE_TYPE (value);
2964 680 : tree container_type = TREE_TYPE (container);
2965 :
2966 680 : if (!INTEGRAL_TYPE_P (container_type)
2967 680 : || !tree_fits_uhwi_p (TYPE_SIZE (container_type)))
2968 : return NULL;
2969 :
2970 508 : gimple *pattern_stmt;
2971 :
2972 508 : vect_unpromoted_value unprom;
2973 508 : unprom.set_op (value, vect_internal_def);
2974 508 : value = vect_convert_input (vinfo, stmt_info, container_type, &unprom,
2975 : get_vectype_for_scalar_type (vinfo,
2976 : container_type));
2977 :
2978 508 : unsigned HOST_WIDE_INT mask_width = TYPE_PRECISION (bf_type);
2979 508 : unsigned HOST_WIDE_INT prec = tree_to_uhwi (TYPE_SIZE (container_type));
2980 508 : unsigned HOST_WIDE_INT shift_n = tree_to_uhwi (shift);
2981 508 : if (BYTES_BIG_ENDIAN)
2982 : {
2983 : shift_n = prec - shift_n - mask_width;
2984 : shift = build_int_cst (TREE_TYPE (shift), shift_n);
2985 : }
2986 :
2987 508 : if (!useless_type_conversion_p (TREE_TYPE (value), container_type))
2988 : {
2989 0 : pattern_stmt =
2990 0 : gimple_build_assign (vect_recog_temp_ssa_var (container_type),
2991 : NOP_EXPR, value);
2992 0 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt);
2993 0 : value = gimple_get_lhs (pattern_stmt);
2994 : }
2995 :
2996 : /* Shift VALUE into place. */
2997 508 : tree shifted = value;
2998 508 : if (shift_n)
2999 : {
3000 249 : gimple_seq stmts = NULL;
3001 249 : shifted
3002 249 : = gimple_build (&stmts, LSHIFT_EXPR, container_type, value, shift);
3003 249 : if (!gimple_seq_empty_p (stmts))
3004 112 : append_pattern_def_seq (vinfo, stmt_info,
3005 : gimple_seq_first_stmt (stmts));
3006 : }
3007 :
3008 508 : tree mask_t
3009 508 : = wide_int_to_tree (container_type,
3010 508 : wi::shifted_mask (shift_n, mask_width, false, prec));
3011 :
3012 : /* Clear bits we don't want to write back from SHIFTED. */
3013 508 : gimple_seq stmts = NULL;
3014 508 : tree masked = gimple_build (&stmts, BIT_AND_EXPR, container_type, shifted,
3015 : mask_t);
3016 508 : if (!gimple_seq_empty_p (stmts))
3017 : {
3018 118 : pattern_stmt = gimple_seq_first_stmt (stmts);
3019 118 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt);
3020 : }
3021 :
3022 : /* Mask off the bits in the container that we are to write to. */
3023 508 : mask_t = wide_int_to_tree (container_type,
3024 508 : wi::shifted_mask (shift_n, mask_width, true, prec));
3025 508 : tree cleared = vect_recog_temp_ssa_var (container_type);
3026 508 : pattern_stmt = gimple_build_assign (cleared, BIT_AND_EXPR, container, mask_t);
3027 508 : append_pattern_def_seq (vinfo, stmt_info, pattern_stmt);
3028 :
3029 : /* Write MASKED into CLEARED. */
3030 508 : pattern_stmt
3031 508 : = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
3032 : BIT_IOR_EXPR, cleared, masked);
3033 :
3034 508 : *type_out = STMT_VINFO_VECTYPE (stmt_info);
3035 508 : vect_pattern_detected ("bit_insert pattern", stmt_info->stmt);
3036 :
3037 508 : return pattern_stmt;
3038 : }
3039 :
3040 :
3041 : /* Recognize cases in which an operation is performed in one type WTYPE
3042 : but could be done more efficiently in a narrower type NTYPE. For example,
3043 : if we have:
3044 :
3045 : ATYPE a; // narrower than NTYPE
3046 : BTYPE b; // narrower than NTYPE
3047 : WTYPE aw = (WTYPE) a;
3048 : WTYPE bw = (WTYPE) b;
3049 : WTYPE res = aw + bw; // only uses of aw and bw
3050 :
3051 : then it would be more efficient to do:
3052 :
3053 : NTYPE an = (NTYPE) a;
3054 : NTYPE bn = (NTYPE) b;
3055 : NTYPE resn = an + bn;
3056 : WTYPE res = (WTYPE) resn;
3057 :
3058 : Other situations include things like:
3059 :
3060 : ATYPE a; // NTYPE or narrower
3061 : WTYPE aw = (WTYPE) a;
3062 : WTYPE res = aw + b;
3063 :
3064 : when only "(NTYPE) res" is significant. In that case it's more efficient
3065 : to truncate "b" and do the operation on NTYPE instead:
3066 :
3067 : NTYPE an = (NTYPE) a;
3068 : NTYPE bn = (NTYPE) b; // truncation
3069 : NTYPE resn = an + bn;
3070 : WTYPE res = (WTYPE) resn;
3071 :
3072 : All users of "res" should then use "resn" instead, making the final
3073 : statement dead (not marked as relevant). The final statement is still
3074 : needed to maintain the type correctness of the IR.
3075 :
3076 : vect_determine_precisions has already determined the minimum
3077 : precision of the operation and the minimum precision required
3078 : by users of the result. */
3079 :
3080 : static gimple *
3081 32248506 : vect_recog_over_widening_pattern (vec_info *vinfo,
3082 : stmt_vec_info last_stmt_info, tree *type_out)
3083 : {
3084 32248506 : gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
3085 22459424 : if (!last_stmt)
3086 : return NULL;
3087 :
3088 : /* See whether we have found that this operation can be done on a
3089 : narrower type without changing its semantics. */
3090 22459424 : unsigned int new_precision = last_stmt_info->operation_precision;
3091 22459424 : if (!new_precision)
3092 : return NULL;
3093 :
3094 1669039 : tree lhs = gimple_assign_lhs (last_stmt);
3095 1669039 : tree type = TREE_TYPE (lhs);
3096 1669039 : tree_code code = gimple_assign_rhs_code (last_stmt);
3097 :
3098 : /* Punt for reductions where we don't handle the type conversions. */
3099 1669039 : if (vect_is_reduction (last_stmt_info))
3100 : return NULL;
3101 :
3102 : /* Keep the first operand of a COND_EXPR as-is: only the other two
3103 : operands are interesting. */
3104 1660709 : unsigned int first_op = (code == COND_EXPR ? 2 : 1);
3105 :
3106 : /* Check the operands. */
3107 1660709 : unsigned int nops = gimple_num_ops (last_stmt) - first_op;
3108 1660709 : auto_vec <vect_unpromoted_value, 3> unprom (nops);
3109 1660709 : unprom.quick_grow_cleared (nops);
3110 1660709 : unsigned int min_precision = 0;
3111 1660709 : bool single_use_p = false;
3112 6622510 : for (unsigned int i = 0; i < nops; ++i)
3113 : {
3114 3302804 : tree op = gimple_op (last_stmt, first_op + i);
3115 3302804 : if (TREE_CODE (op) == INTEGER_CST)
3116 1511459 : unprom[i].set_op (op, vect_constant_def);
3117 1791345 : else if (TREE_CODE (op) == SSA_NAME)
3118 : {
3119 1791345 : bool op_single_use_p = true;
3120 1791345 : if (!vect_look_through_possible_promotion (vinfo, op, &unprom[i],
3121 : &op_single_use_p))
3122 1712 : return NULL;
3123 : /* If:
3124 :
3125 : (1) N bits of the result are needed;
3126 : (2) all inputs are widened from M<N bits; and
3127 : (3) one operand OP is a single-use SSA name
3128 :
3129 : we can shift the M->N widening from OP to the output
3130 : without changing the number or type of extensions involved.
3131 : This then reduces the number of copies of STMT_INFO.
3132 :
3133 : If instead of (3) more than one operand is a single-use SSA name,
3134 : shifting the extension to the output is even more of a win.
3135 :
3136 : If instead:
3137 :
3138 : (1) N bits of the result are needed;
3139 : (2) one operand OP2 is widened from M2<N bits;
3140 : (3) another operand OP1 is widened from M1<M2 bits; and
3141 : (4) both OP1 and OP2 are single-use
3142 :
3143 : the choice is between:
3144 :
3145 : (a) truncating OP2 to M1, doing the operation on M1,
3146 : and then widening the result to N
3147 :
3148 : (b) widening OP1 to M2, doing the operation on M2, and then
3149 : widening the result to N
3150 :
3151 : Both shift the M2->N widening of the inputs to the output.
3152 : (a) additionally shifts the M1->M2 widening to the output;
3153 : it requires fewer copies of STMT_INFO but requires an extra
3154 : M2->M1 truncation.
3155 :
3156 : Which is better will depend on the complexity and cost of
3157 : STMT_INFO, which is hard to predict at this stage. However,
3158 : a clear tie-breaker in favor of (b) is the fact that the
3159 : truncation in (a) increases the length of the operation chain.
3160 :
3161 : If instead of (4) only one of OP1 or OP2 is single-use,
3162 : (b) is still a win over doing the operation in N bits:
3163 : it still shifts the M2->N widening on the single-use operand
3164 : to the output and reduces the number of STMT_INFO copies.
3165 :
3166 : If neither operand is single-use then operating on fewer than
3167 : N bits might lead to more extensions overall. Whether it does
3168 : or not depends on global information about the vectorization
3169 : region, and whether that's a good trade-off would again
3170 : depend on the complexity and cost of the statements involved,
3171 : as well as things like register pressure that are not normally
3172 : modelled at this stage. We therefore ignore these cases
3173 : and just optimize the clear single-use wins above.
3174 :
3175 : Thus we take the maximum precision of the unpromoted operands
3176 : and record whether any operand is single-use. */
3177 1789633 : if (unprom[i].dt == vect_internal_def)
3178 : {
3179 1029768 : min_precision = MAX (min_precision,
3180 : TYPE_PRECISION (unprom[i].type));
3181 1029768 : single_use_p |= op_single_use_p;
3182 : }
3183 : }
3184 : else
3185 : return NULL;
3186 : }
3187 :
3188 : /* Although the operation could be done in operation_precision, we have
3189 : to balance that against introducing extra truncations or extensions.
3190 : Calculate the minimum precision that can be handled efficiently.
3191 :
3192 : The loop above determined that the operation could be handled
3193 : efficiently in MIN_PRECISION if SINGLE_USE_P; this would shift an
3194 : extension from the inputs to the output without introducing more
3195 : instructions, and would reduce the number of instructions required
3196 : for STMT_INFO itself.
3197 :
3198 : vect_determine_precisions has also determined that the result only
3199 : needs min_output_precision bits. Truncating by a factor of N times
3200 : requires a tree of N - 1 instructions, so if TYPE is N times wider
3201 : than min_output_precision, doing the operation in TYPE and truncating
3202 : the result requires N + (N - 1) = 2N - 1 instructions per output vector.
3203 : In contrast:
3204 :
3205 : - truncating the input to a unary operation and doing the operation
3206 : in the new type requires at most N - 1 + 1 = N instructions per
3207 : output vector
3208 :
3209 : - doing the same for a binary operation requires at most
3210 : (N - 1) * 2 + 1 = 2N - 1 instructions per output vector
3211 :
3212 : Both unary and binary operations require fewer instructions than
3213 : this if the operands were extended from a suitable truncated form.
3214 : Thus there is usually nothing to lose by doing operations in
3215 : min_output_precision bits, but there can be something to gain. */
3216 1658997 : if (!single_use_p)
3217 1309752 : min_precision = last_stmt_info->min_output_precision;
3218 : else
3219 349245 : min_precision = MIN (min_precision, last_stmt_info->min_output_precision);
3220 :
3221 : /* Apply the minimum efficient precision we just calculated. */
3222 1658997 : if (new_precision < min_precision)
3223 : new_precision = min_precision;
3224 1658997 : new_precision = vect_element_precision (new_precision);
3225 1658997 : if (new_precision >= TYPE_PRECISION (type))
3226 : return NULL;
3227 :
3228 154270 : vect_pattern_detected ("vect_recog_over_widening_pattern", last_stmt);
3229 :
3230 154270 : *type_out = get_vectype_for_scalar_type (vinfo, type);
3231 154270 : if (!*type_out)
3232 : return NULL;
3233 :
3234 : /* We've found a viable pattern. Get the new type of the operation. */
3235 135989 : bool unsigned_p = (last_stmt_info->operation_sign == UNSIGNED);
3236 135989 : tree new_type = build_nonstandard_integer_type (new_precision, unsigned_p);
3237 :
3238 : /* If we're truncating an operation, we need to make sure that we
3239 : don't introduce new undefined overflow. The codes tested here are
3240 : a subset of those accepted by vect_truncatable_operation_p. */
3241 135989 : tree op_type = new_type;
3242 135989 : if (TYPE_OVERFLOW_UNDEFINED (new_type)
3243 176213 : && (code == PLUS_EXPR || code == MINUS_EXPR || code == MULT_EXPR))
3244 27603 : op_type = build_nonstandard_integer_type (new_precision, true);
3245 :
3246 135989 : tree new_vectype = get_vectype_for_scalar_type (vinfo, new_type);
3247 135989 : tree op_vectype = get_vectype_for_scalar_type (vinfo, op_type);
3248 135989 : if (!new_vectype || !op_vectype)
3249 : return NULL;
3250 :
3251 : /* Verify we can handle the new operation. For shifts and rotates
3252 : apply heuristic of whether we are likely facing vector-vector or
3253 : vector-scalar operation. Since we are eventually expecting that
3254 : a later pattern might eventually want to rewrite an unsupported
3255 : into a supported case error on that side in case the original
3256 : operation was not supported either or this is a binary operation
3257 : and the 2nd operand is constant. */
3258 135989 : if (code == RSHIFT_EXPR || code == LSHIFT_EXPR || code == RROTATE_EXPR)
3259 : {
3260 29124 : if (!target_has_vecop_for_code (code, op_vectype, optab_vector)
3261 28065 : && ((unprom[1].dt != vect_external_def
3262 27653 : && unprom[1].dt != vect_constant_def)
3263 19186 : || !target_has_vecop_for_code (code, op_vectype, optab_scalar))
3264 38026 : && !(!target_has_vecop_for_code (code, *type_out, optab_vector)
3265 7804 : && ((unprom[1].dt != vect_external_def
3266 7804 : || unprom[1].dt != vect_constant_def)
3267 : || !target_has_vecop_for_code (code, *type_out,
3268 : optab_scalar))))
3269 : return NULL;
3270 : }
3271 106865 : else if (!target_has_vecop_for_code (code, op_vectype, optab_vector)
3272 106865 : && (target_has_vecop_for_code (code, *type_out, optab_vector)
3273 29 : && !(nops == 2 && unprom[1].dt == vect_constant_def)))
3274 : return NULL;
3275 :
3276 134881 : if (dump_enabled_p ())
3277 4613 : dump_printf_loc (MSG_NOTE, vect_location, "demoting %T to %T\n",
3278 : type, new_type);
3279 :
3280 : /* Calculate the rhs operands for an operation on OP_TYPE. */
3281 134881 : tree ops[3] = {};
3282 135112 : for (unsigned int i = 1; i < first_op; ++i)
3283 231 : ops[i - 1] = gimple_op (last_stmt, i);
3284 134881 : vect_convert_inputs (vinfo, last_stmt_info, nops, &ops[first_op - 1],
3285 134881 : op_type, &unprom[0], op_vectype);
3286 :
3287 : /* Use the operation to produce a result of type OP_TYPE. */
3288 134881 : tree new_var = vect_recog_temp_ssa_var (op_type, NULL);
3289 134881 : gimple *pattern_stmt = gimple_build_assign (new_var, code,
3290 : ops[0], ops[1], ops[2]);
3291 134881 : gimple_set_location (pattern_stmt, gimple_location (last_stmt));
3292 :
3293 134881 : if (dump_enabled_p ())
3294 4613 : dump_printf_loc (MSG_NOTE, vect_location,
3295 : "created pattern stmt: %G", pattern_stmt);
3296 :
3297 : /* Convert back to the original signedness, if OP_TYPE is different
3298 : from NEW_TYPE. */
3299 134881 : if (op_type != new_type)
3300 27596 : pattern_stmt = vect_convert_output (vinfo, last_stmt_info, new_type,
3301 : pattern_stmt, op_vectype);
3302 :
3303 : /* Promote the result to the original type. */
3304 134881 : pattern_stmt = vect_convert_output (vinfo, last_stmt_info, type,
3305 : pattern_stmt, new_vectype);
3306 :
3307 134881 : return pattern_stmt;
3308 1660709 : }
3309 :
3310 : /* Recognize the following patterns:
3311 :
3312 : ATYPE a; // narrower than TYPE
3313 : BTYPE b; // narrower than TYPE
3314 :
3315 : 1) Multiply high with scaling
3316 : TYPE res = ((TYPE) a * (TYPE) b) >> c;
3317 : Here, c is bitsize (TYPE) / 2 - 1.
3318 :
3319 : 2) ... or also with rounding
3320 : TYPE res = (((TYPE) a * (TYPE) b) >> d + 1) >> 1;
3321 : Here, d is bitsize (TYPE) / 2 - 2.
3322 :
3323 : 3) Normal multiply high
3324 : TYPE res = ((TYPE) a * (TYPE) b) >> e;
3325 : Here, e is bitsize (TYPE) / 2.
3326 :
3327 : where only the bottom half of res is used. */
3328 :
3329 : static gimple *
3330 32380537 : vect_recog_mulhs_pattern (vec_info *vinfo,
3331 : stmt_vec_info last_stmt_info, tree *type_out)
3332 : {
3333 : /* Check for a right shift. */
3334 32380537 : gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
3335 22591249 : if (!last_stmt
3336 22591249 : || gimple_assign_rhs_code (last_stmt) != RSHIFT_EXPR)
3337 : return NULL;
3338 :
3339 : /* Check that the shift result is wider than the users of the
3340 : result need (i.e. that narrowing would be a natural choice). */
3341 353842 : tree lhs_type = TREE_TYPE (gimple_assign_lhs (last_stmt));
3342 353842 : unsigned int target_precision
3343 353842 : = vect_element_precision (last_stmt_info->min_output_precision);
3344 353842 : if (!INTEGRAL_TYPE_P (lhs_type)
3345 353842 : || target_precision >= TYPE_PRECISION (lhs_type))
3346 : return NULL;
3347 :
3348 : /* Look through any change in sign on the outer shift input. */
3349 61158 : vect_unpromoted_value unprom_rshift_input;
3350 61158 : tree rshift_input = vect_look_through_possible_promotion
3351 61158 : (vinfo, gimple_assign_rhs1 (last_stmt), &unprom_rshift_input);
3352 61158 : if (!rshift_input
3353 61158 : || TYPE_PRECISION (TREE_TYPE (rshift_input))
3354 60560 : != TYPE_PRECISION (lhs_type))
3355 : return NULL;
3356 :
3357 : /* Get the definition of the shift input. */
3358 56261 : stmt_vec_info rshift_input_stmt_info
3359 56261 : = vect_get_internal_def (vinfo, rshift_input);
3360 56261 : if (!rshift_input_stmt_info)
3361 : return NULL;
3362 51606 : gassign *rshift_input_stmt
3363 51606 : = dyn_cast <gassign *> (rshift_input_stmt_info->stmt);
3364 48290 : if (!rshift_input_stmt)
3365 : return NULL;
3366 :
3367 48290 : stmt_vec_info mulh_stmt_info;
3368 48290 : tree scale_term;
3369 48290 : bool rounding_p = false;
3370 :
3371 : /* Check for the presence of the rounding term. */
3372 55737 : if (gimple_assign_rhs_code (rshift_input_stmt) == PLUS_EXPR)
3373 : {
3374 : /* Check that the outer shift was by 1. */
3375 17860 : if (!integer_onep (gimple_assign_rhs2 (last_stmt)))
3376 8863 : return NULL;
3377 :
3378 : /* Check that the second operand of the PLUS_EXPR is 1. */
3379 436 : if (!integer_onep (gimple_assign_rhs2 (rshift_input_stmt)))
3380 : return NULL;
3381 :
3382 : /* Look through any change in sign on the addition input. */
3383 69 : vect_unpromoted_value unprom_plus_input;
3384 69 : tree plus_input = vect_look_through_possible_promotion
3385 69 : (vinfo, gimple_assign_rhs1 (rshift_input_stmt), &unprom_plus_input);
3386 69 : if (!plus_input
3387 69 : || TYPE_PRECISION (TREE_TYPE (plus_input))
3388 69 : != TYPE_PRECISION (TREE_TYPE (rshift_input)))
3389 : return NULL;
3390 :
3391 : /* Get the definition of the multiply-high-scale part. */
3392 69 : stmt_vec_info plus_input_stmt_info
3393 69 : = vect_get_internal_def (vinfo, plus_input);
3394 69 : if (!plus_input_stmt_info)
3395 : return NULL;
3396 69 : gassign *plus_input_stmt
3397 8932 : = dyn_cast <gassign *> (plus_input_stmt_info->stmt);
3398 69 : if (!plus_input_stmt
3399 69 : || gimple_assign_rhs_code (plus_input_stmt) != RSHIFT_EXPR)
3400 : return NULL;
3401 :
3402 : /* Look through any change in sign on the scaling input. */
3403 67 : vect_unpromoted_value unprom_scale_input;
3404 67 : tree scale_input = vect_look_through_possible_promotion
3405 67 : (vinfo, gimple_assign_rhs1 (plus_input_stmt), &unprom_scale_input);
3406 67 : if (!scale_input
3407 67 : || TYPE_PRECISION (TREE_TYPE (scale_input))
3408 67 : != TYPE_PRECISION (TREE_TYPE (plus_input)))
3409 : return NULL;
3410 :
3411 : /* Get the definition of the multiply-high part. */
3412 67 : mulh_stmt_info = vect_get_internal_def (vinfo, scale_input);
3413 67 : if (!mulh_stmt_info)
3414 : return NULL;
3415 :
3416 : /* Get the scaling term. */
3417 67 : scale_term = gimple_assign_rhs2 (plus_input_stmt);
3418 67 : rounding_p = true;
3419 : }
3420 : else
3421 : {
3422 39360 : mulh_stmt_info = rshift_input_stmt_info;
3423 39360 : scale_term = gimple_assign_rhs2 (last_stmt);
3424 : }
3425 :
3426 : /* Check that the scaling factor is constant. */
3427 39427 : if (TREE_CODE (scale_term) != INTEGER_CST)
3428 : return NULL;
3429 :
3430 : /* Check whether the scaling input term can be seen as two widened
3431 : inputs multiplied together. */
3432 115299 : vect_unpromoted_value unprom_mult[2];
3433 38433 : tree new_type;
3434 38433 : unsigned int nops
3435 38433 : = vect_widened_op_tree (vinfo, mulh_stmt_info, MULT_EXPR, WIDEN_MULT_EXPR,
3436 : false, 2, unprom_mult, &new_type);
3437 38433 : if (nops != 2)
3438 : return NULL;
3439 :
3440 : /* Adjust output precision. */
3441 8839 : if (TYPE_PRECISION (new_type) < target_precision)
3442 0 : new_type = build_nonstandard_integer_type
3443 0 : (target_precision, TYPE_UNSIGNED (new_type));
3444 :
3445 8839 : unsigned mult_precision = TYPE_PRECISION (new_type);
3446 8839 : internal_fn ifn;
3447 : /* Check that the scaling factor is expected. Instead of
3448 : target_precision, we should use the one that we actually
3449 : use for internal function. */
3450 8839 : if (rounding_p)
3451 : {
3452 : /* Check pattern 2). */
3453 134 : if (wi::to_widest (scale_term) + mult_precision + 2
3454 201 : != TYPE_PRECISION (lhs_type))
3455 : return NULL;
3456 :
3457 : ifn = IFN_MULHRS;
3458 : }
3459 : else
3460 : {
3461 : /* Check for pattern 1). */
3462 17544 : if (wi::to_widest (scale_term) + mult_precision + 1
3463 26316 : == TYPE_PRECISION (lhs_type))
3464 : ifn = IFN_MULHS;
3465 : /* Check for pattern 3). */
3466 8738 : else if (wi::to_widest (scale_term) + mult_precision
3467 17476 : == TYPE_PRECISION (lhs_type))
3468 : ifn = IFN_MULH;
3469 : else
3470 : return NULL;
3471 : }
3472 :
3473 8774 : vect_pattern_detected ("vect_recog_mulhs_pattern", last_stmt);
3474 :
3475 : /* Check for target support. */
3476 8774 : tree new_vectype = get_vectype_for_scalar_type (vinfo, new_type);
3477 8774 : if (!new_vectype
3478 15462 : || !direct_internal_fn_supported_p
3479 6688 : (ifn, new_vectype, OPTIMIZE_FOR_SPEED))
3480 : return NULL;
3481 :
3482 : /* The result is cast back to LHS_TYPE, a cast that the over-widening
3483 : machinery then removes. LHS_TYPE need not have a vector type, as for the
3484 : 128-bit product of a 64-bit high-part multiply, so leave *TYPE_OUT null
3485 : rather than giving up. */
3486 144 : *type_out = get_vectype_for_scalar_type (vinfo, lhs_type);
3487 :
3488 : /* Generate the IFN_MULHRS call. */
3489 144 : tree new_var = vect_recog_temp_ssa_var (new_type, NULL);
3490 144 : tree new_ops[2];
3491 144 : vect_convert_inputs (vinfo, last_stmt_info, 2, new_ops, new_type,
3492 : unprom_mult, new_vectype);
3493 144 : gcall *mulhrs_stmt
3494 144 : = gimple_build_call_internal (ifn, 2, new_ops[0], new_ops[1]);
3495 144 : gimple_call_set_lhs (mulhrs_stmt, new_var);
3496 144 : gimple_set_location (mulhrs_stmt, gimple_location (last_stmt));
3497 :
3498 144 : if (dump_enabled_p ())
3499 0 : dump_printf_loc (MSG_NOTE, vect_location,
3500 : "created pattern stmt: %G", (gimple *) mulhrs_stmt);
3501 :
3502 144 : return vect_convert_output (vinfo, last_stmt_info, lhs_type,
3503 144 : mulhrs_stmt, new_vectype);
3504 : }
3505 :
3506 : /* Recognize the patterns:
3507 :
3508 : ATYPE a; // narrower than TYPE
3509 : BTYPE b; // narrower than TYPE
3510 : (1) TYPE avg = ((TYPE) a + (TYPE) b) >> 1;
3511 : or (2) TYPE avg = ((TYPE) a + (TYPE) b + 1) >> 1;
3512 :
3513 : where only the bottom half of avg is used. Try to transform them into:
3514 :
3515 : (1) NTYPE avg' = .AVG_FLOOR ((NTYPE) a, (NTYPE) b);
3516 : or (2) NTYPE avg' = .AVG_CEIL ((NTYPE) a, (NTYPE) b);
3517 :
3518 : followed by:
3519 :
3520 : TYPE avg = (TYPE) avg';
3521 :
3522 : where NTYPE is no wider than half of TYPE. Since only the bottom half
3523 : of avg is used, all or part of the cast of avg' should become redundant.
3524 :
3525 : If there is no target support available, generate code to distribute rshift
3526 : over plus and add a carry. */
3527 :
3528 : static gimple *
3529 32376300 : vect_recog_average_pattern (vec_info *vinfo,
3530 : stmt_vec_info last_stmt_info, tree *type_out)
3531 : {
3532 : /* Check for a shift right by one bit. */
3533 32376300 : gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
3534 22587218 : if (!last_stmt
3535 22587218 : || gimple_assign_rhs_code (last_stmt) != RSHIFT_EXPR
3536 353355 : || !integer_onep (gimple_assign_rhs2 (last_stmt)))
3537 : return NULL;
3538 :
3539 : /* Check that the shift result is wider than the users of the
3540 : result need (i.e. that narrowing would be a natural choice). */
3541 57299 : tree lhs = gimple_assign_lhs (last_stmt);
3542 57299 : tree type = TREE_TYPE (lhs);
3543 57299 : unsigned int target_precision
3544 57299 : = vect_element_precision (last_stmt_info->min_output_precision);
3545 57299 : if (!INTEGRAL_TYPE_P (type) || target_precision >= TYPE_PRECISION (type))
3546 : return NULL;
3547 :
3548 : /* Look through any change in sign on the shift input. */
3549 2268 : tree rshift_rhs = gimple_assign_rhs1 (last_stmt);
3550 2268 : vect_unpromoted_value unprom_plus;
3551 2268 : rshift_rhs = vect_look_through_possible_promotion (vinfo, rshift_rhs,
3552 : &unprom_plus);
3553 2268 : if (!rshift_rhs
3554 2268 : || TYPE_PRECISION (TREE_TYPE (rshift_rhs)) != TYPE_PRECISION (type))
3555 : return NULL;
3556 :
3557 : /* Get the definition of the shift input. */
3558 2264 : stmt_vec_info plus_stmt_info = vect_get_internal_def (vinfo, rshift_rhs);
3559 2264 : if (!plus_stmt_info)
3560 : return NULL;
3561 :
3562 : /* Check whether the shift input can be seen as a tree of additions on
3563 : 2 or 3 widened inputs.
3564 :
3565 : Note that the pattern should be a win even if the result of one or
3566 : more additions is reused elsewhere: if the pattern matches, we'd be
3567 : replacing 2N RSHIFT_EXPRs and N VEC_PACK_*s with N IFN_AVG_*s. */
3568 2258 : internal_fn ifn = IFN_AVG_FLOOR;
3569 9032 : vect_unpromoted_value unprom[3];
3570 2258 : tree new_type;
3571 2258 : unsigned int nops = vect_widened_op_tree (vinfo, plus_stmt_info, PLUS_EXPR,
3572 2258 : IFN_VEC_WIDEN_PLUS, false, 3,
3573 : unprom, &new_type);
3574 2258 : if (nops == 0)
3575 : return NULL;
3576 959 : if (nops == 3)
3577 : {
3578 : /* Check that one operand is 1. */
3579 : unsigned int i;
3580 1155 : for (i = 0; i < 3; ++i)
3581 1101 : if (integer_onep (unprom[i].op))
3582 : break;
3583 367 : if (i == 3)
3584 : return NULL;
3585 : /* Throw away the 1 operand and keep the other two. */
3586 313 : if (i < 2)
3587 0 : unprom[i] = unprom[2];
3588 : ifn = IFN_AVG_CEIL;
3589 : }
3590 :
3591 905 : vect_pattern_detected ("vect_recog_average_pattern", last_stmt);
3592 :
3593 : /* We know that:
3594 :
3595 : (a) the operation can be viewed as:
3596 :
3597 : TYPE widened0 = (TYPE) UNPROM[0];
3598 : TYPE widened1 = (TYPE) UNPROM[1];
3599 : TYPE tmp1 = widened0 + widened1 {+ 1};
3600 : TYPE tmp2 = tmp1 >> 1; // LAST_STMT_INFO
3601 :
3602 : (b) the first two statements are equivalent to:
3603 :
3604 : TYPE widened0 = (TYPE) (NEW_TYPE) UNPROM[0];
3605 : TYPE widened1 = (TYPE) (NEW_TYPE) UNPROM[1];
3606 :
3607 : (c) vect_recog_over_widening_pattern has already tried to narrow TYPE
3608 : where sensible;
3609 :
3610 : (d) all the operations can be performed correctly at twice the width of
3611 : NEW_TYPE, due to the nature of the average operation; and
3612 :
3613 : (e) users of the result of the right shift need only TARGET_PRECISION
3614 : bits, where TARGET_PRECISION is no more than half of TYPE's
3615 : precision.
3616 :
3617 : Under these circumstances, the only situation in which NEW_TYPE
3618 : could be narrower than TARGET_PRECISION is if widened0, widened1
3619 : and an addition result are all used more than once. Thus we can
3620 : treat any widening of UNPROM[0] and UNPROM[1] to TARGET_PRECISION
3621 : as "free", whereas widening the result of the average instruction
3622 : from NEW_TYPE to TARGET_PRECISION would be a new operation. It's
3623 : therefore better not to go narrower than TARGET_PRECISION. */
3624 905 : if (TYPE_PRECISION (new_type) < target_precision)
3625 0 : new_type = build_nonstandard_integer_type (target_precision,
3626 0 : TYPE_UNSIGNED (new_type));
3627 :
3628 : /* Check for target support. */
3629 905 : tree new_vectype = get_vectype_for_scalar_type (vinfo, new_type);
3630 905 : if (!new_vectype)
3631 : return NULL;
3632 :
3633 905 : bool fallback_p = !direct_internal_fn_supported_p (ifn, new_vectype,
3634 905 : OPTIMIZE_FOR_SPEED);
3635 905 : if (fallback_p
3636 905 : && (!target_supports_op_p (new_vectype, RSHIFT_EXPR, optab_scalar)
3637 697 : || !target_supports_op_p (new_vectype, PLUS_EXPR, optab_default)
3638 697 : || !target_supports_op_p (new_vectype, BIT_AND_EXPR, optab_default)
3639 697 : || (ifn == IFN_AVG_CEIL
3640 105 : && !target_supports_op_p (new_vectype, BIT_IOR_EXPR,
3641 : optab_default))))
3642 : return NULL;
3643 :
3644 : /* The IR requires a valid vector type for the cast result, even though
3645 : it's likely to be discarded. */
3646 903 : *type_out = get_vectype_for_scalar_type (vinfo, type);
3647 903 : if (!*type_out)
3648 : return NULL;
3649 :
3650 899 : tree new_var = vect_recog_temp_ssa_var (new_type, NULL);
3651 899 : tree new_ops[2];
3652 899 : vect_convert_inputs (vinfo, last_stmt_info, 2, new_ops, new_type,
3653 : unprom, new_vectype);
3654 :
3655 899 : if (fallback_p)
3656 : {
3657 : /* As a fallback, generate code for following sequence:
3658 :
3659 : shifted_op0 = new_ops[0] >> 1;
3660 : shifted_op1 = new_ops[1] >> 1;
3661 : sum_of_shifted = shifted_op0 + shifted_op1;
3662 : unmasked_carry = new_ops[0] and/or new_ops[1];
3663 : carry = unmasked_carry & 1;
3664 : new_var = sum_of_shifted + carry;
3665 :
3666 : For signed types, arithmetic shifts round down and the carry is one
3667 : when both operands are odd (or either operand for IFN_AVG_CEIL). */
3668 :
3669 693 : tree one_cst = build_one_cst (new_type);
3670 693 : gassign *g;
3671 :
3672 693 : tree shifted_op0 = vect_recog_temp_ssa_var (new_type, NULL);
3673 693 : g = gimple_build_assign (shifted_op0, RSHIFT_EXPR, new_ops[0], one_cst);
3674 693 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3675 :
3676 693 : tree shifted_op1 = vect_recog_temp_ssa_var (new_type, NULL);
3677 693 : g = gimple_build_assign (shifted_op1, RSHIFT_EXPR, new_ops[1], one_cst);
3678 693 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3679 :
3680 693 : tree sum_of_shifted = vect_recog_temp_ssa_var (new_type, NULL);
3681 693 : g = gimple_build_assign (sum_of_shifted, PLUS_EXPR,
3682 : shifted_op0, shifted_op1);
3683 693 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3684 :
3685 693 : tree unmasked_carry = vect_recog_temp_ssa_var (new_type, NULL);
3686 693 : tree_code c = (ifn == IFN_AVG_CEIL) ? BIT_IOR_EXPR : BIT_AND_EXPR;
3687 693 : g = gimple_build_assign (unmasked_carry, c, new_ops[0], new_ops[1]);
3688 693 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3689 :
3690 693 : tree carry = vect_recog_temp_ssa_var (new_type, NULL);
3691 693 : g = gimple_build_assign (carry, BIT_AND_EXPR, unmasked_carry, one_cst);
3692 693 : append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
3693 :
3694 693 : g = gimple_build_assign (new_var, PLUS_EXPR, sum_of_shifted, carry);
3695 693 : return vect_convert_output (vinfo, last_stmt_info, type, g, new_vectype);
3696 : }
3697 :
3698 : /* Generate the IFN_AVG* call. */
3699 206 : gcall *average_stmt = gimple_build_call_internal (ifn, 2, new_ops[0],
3700 : new_ops[1]);
3701 206 : gimple_call_set_lhs (average_stmt, new_var);
3702 206 : gimple_set_location (average_stmt, gimple_location (last_stmt));
3703 :
3704 206 : if (dump_enabled_p ())
3705 79 : dump_printf_loc (MSG_NOTE, vect_location,
3706 : "created pattern stmt: %G", (gimple *) average_stmt);
3707 :
3708 206 : return vect_convert_output (vinfo, last_stmt_info,
3709 206 : type, average_stmt, new_vectype);
3710 : }
3711 :
3712 : /* Recognize cases in which the input to a cast is wider than its
3713 : output, and the input is fed by a widening operation. Fold this
3714 : by removing the unnecessary intermediate widening. E.g.:
3715 :
3716 : unsigned char a;
3717 : unsigned int b = (unsigned int) a;
3718 : unsigned short c = (unsigned short) b;
3719 :
3720 : -->
3721 :
3722 : unsigned short c = (unsigned short) a;
3723 :
3724 : Although this is rare in input IR, it is an expected side-effect
3725 : of the over-widening pattern above.
3726 :
3727 : This is beneficial also for integer-to-float conversions, if the
3728 : widened integer has more bits than the float, and if the unwidened
3729 : input doesn't. */
3730 :
3731 : static gimple *
3732 32380537 : vect_recog_cast_forwprop_pattern (vec_info *vinfo,
3733 : stmt_vec_info last_stmt_info, tree *type_out)
3734 : {
3735 : /* Check for a cast, including an integer-to-float conversion. */
3736 32380537 : gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
3737 22591105 : if (!last_stmt)
3738 : return NULL;
3739 22591105 : tree_code code = gimple_assign_rhs_code (last_stmt);
3740 22591105 : if (!CONVERT_EXPR_CODE_P (code) && code != FLOAT_EXPR)
3741 : return NULL;
3742 :
3743 : /* Make sure that the rhs is a scalar with a natural bitsize. */
3744 3255522 : tree lhs = gimple_assign_lhs (last_stmt);
3745 3255522 : if (!lhs)
3746 : return NULL;
3747 3255522 : tree lhs_type = TREE_TYPE (lhs);
3748 3255522 : scalar_mode lhs_mode;
3749 3234728 : if (VECT_SCALAR_BOOLEAN_TYPE_P (lhs_type)
3750 6488508 : || !is_a <scalar_mode> (TYPE_MODE (lhs_type), &lhs_mode))
3751 : return NULL;
3752 :
3753 : /* Check for a narrowing operation (from a vector point of view). */
3754 3229258 : tree rhs = gimple_assign_rhs1 (last_stmt);
3755 3229258 : tree rhs_type = TREE_TYPE (rhs);
3756 3229258 : if (!INTEGRAL_TYPE_P (rhs_type)
3757 2827208 : || VECT_SCALAR_BOOLEAN_TYPE_P (rhs_type)
3758 8734698 : || TYPE_PRECISION (rhs_type) <= GET_MODE_BITSIZE (lhs_mode))
3759 : return NULL;
3760 :
3761 : /* Try to find an unpromoted input. */
3762 378440 : vect_unpromoted_value unprom;
3763 378440 : if (!vect_look_through_possible_promotion (vinfo, rhs, &unprom)
3764 378440 : || TYPE_PRECISION (unprom.type) >= TYPE_PRECISION (rhs_type))
3765 : return NULL;
3766 :
3767 : /* If the bits above RHS_TYPE matter, make sure that they're the
3768 : same when extending from UNPROM as they are when extending from RHS. */
3769 48306 : if (!INTEGRAL_TYPE_P (lhs_type)
3770 48306 : && TYPE_SIGN (rhs_type) != TYPE_SIGN (unprom.type))
3771 : return NULL;
3772 :
3773 : /* We can get the same result by casting UNPROM directly, to avoid
3774 : the unnecessary widening and narrowing. */
3775 48186 : vect_pattern_detected ("vect_recog_cast_forwprop_pattern", last_stmt);
3776 :
3777 48186 : *type_out = get_vectype_for_scalar_type (vinfo, lhs_type);
3778 48186 : if (!*type_out)
3779 : return NULL;
3780 :
3781 48186 : tree new_var = vect_recog_temp_ssa_var (lhs_type, NULL);
3782 48186 : gimple *pattern_stmt = gimple_build_assign (new_var, code, unprom.op);
3783 48186 : gimple_set_location (pattern_stmt, gimple_location (last_stmt));
3784 :
3785 48186 : return pattern_stmt;
3786 : }
3787 :
3788 : /* Try to detect a shift left of a widened input, converting LSHIFT_EXPR
3789 : to WIDEN_LSHIFT_EXPR. See vect_recog_widen_op_pattern for details. */
3790 :
3791 : static gimple *
3792 32303866 : vect_recog_widen_shift_pattern (vec_info *vinfo,
3793 : stmt_vec_info last_stmt_info, tree *type_out)
3794 : {
3795 32303866 : return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
3796 32303866 : LSHIFT_EXPR, WIDEN_LSHIFT_EXPR, true,
3797 32303866 : "vect_recog_widen_shift_pattern");
3798 : }
3799 :
3800 : /* Detect a rotate pattern wouldn't be otherwise vectorized:
3801 :
3802 : type a_t, b_t, c_t;
3803 :
3804 : S0 a_t = b_t r<< c_t;
3805 :
3806 : Input/Output:
3807 :
3808 : * STMT_VINFO: The stmt from which the pattern search begins,
3809 : i.e. the shift/rotate stmt. The original stmt (S0) is replaced
3810 : with a sequence:
3811 :
3812 : S1 d_t = -c_t;
3813 : S2 e_t = d_t & (B - 1);
3814 : S3 f_t = b_t << c_t;
3815 : S4 g_t = b_t >> e_t;
3816 : S0 a_t = f_t | g_t;
3817 :
3818 : where B is element bitsize of type.
3819 :
3820 : Output:
3821 :
3822 : * TYPE_OUT: The type of the output of this pattern.
3823 :
3824 : * Return value: A new stmt that will be used to replace the rotate
3825 : S0 stmt. */
3826 :
3827 : static gimple *
3828 32303866 : vect_recog_rotate_pattern (vec_info *vinfo,
3829 : stmt_vec_info stmt_vinfo, tree *type_out)
3830 : {
3831 32303866 : gimple *last_stmt = stmt_vinfo->stmt;
3832 32303866 : tree oprnd0, oprnd1, lhs, var, var1, var2, vectype, type, stype, def, def2;
3833 32303866 : gimple *pattern_stmt, *def_stmt;
3834 32303866 : enum tree_code rhs_code;
3835 32303866 : enum vect_def_type dt;
3836 32303866 : optab optab1, optab2;
3837 32303866 : edge ext_def = NULL;
3838 32303866 : bool bswap16_p = false;
3839 :
3840 32303866 : if (is_gimple_assign (last_stmt))
3841 : {
3842 22514384 : rhs_code = gimple_assign_rhs_code (last_stmt);
3843 22514384 : switch (rhs_code)
3844 : {
3845 8519 : case LROTATE_EXPR:
3846 8519 : case RROTATE_EXPR:
3847 8519 : break;
3848 : default:
3849 : return NULL;
3850 : }
3851 :
3852 8519 : lhs = gimple_assign_lhs (last_stmt);
3853 8519 : oprnd0 = gimple_assign_rhs1 (last_stmt);
3854 8519 : type = TREE_TYPE (oprnd0);
3855 8519 : oprnd1 = gimple_assign_rhs2 (last_stmt);
3856 : }
3857 9789482 : else if (gimple_call_builtin_p (last_stmt, BUILT_IN_BSWAP16))
3858 : {
3859 : /* __builtin_bswap16 (x) is another form of x r>> 8.
3860 : The vectorizer has bswap support, but only if the argument isn't
3861 : promoted. */
3862 206 : lhs = gimple_call_lhs (last_stmt);
3863 206 : oprnd0 = gimple_call_arg (last_stmt, 0);
3864 206 : type = TREE_TYPE (oprnd0);
3865 206 : if (!lhs
3866 206 : || TYPE_PRECISION (TREE_TYPE (lhs)) != 16
3867 206 : || TYPE_PRECISION (type) <= 16
3868 0 : || TREE_CODE (oprnd0) != SSA_NAME
3869 206 : || BITS_PER_UNIT != 8)
3870 206 : return NULL;
3871 :
3872 0 : stmt_vec_info def_stmt_info;
3873 0 : if (!vect_is_simple_use (oprnd0, vinfo, &dt, &def_stmt_info, &def_stmt))
3874 : return NULL;
3875 :
3876 0 : if (dt != vect_internal_def)
3877 : return NULL;
3878 :
3879 0 : if (gimple_assign_cast_p (def_stmt))
3880 : {
3881 0 : def = gimple_assign_rhs1 (def_stmt);
3882 0 : if (INTEGRAL_TYPE_P (TREE_TYPE (def))
3883 0 : && TYPE_PRECISION (TREE_TYPE (def)) == 16)
3884 : oprnd0 = def;
3885 : }
3886 :
3887 0 : type = TREE_TYPE (lhs);
3888 0 : vectype = get_vectype_for_scalar_type (vinfo, type);
3889 0 : if (vectype == NULL_TREE)
3890 : return NULL;
3891 :
3892 0 : if (tree char_vectype = get_same_sized_vectype (char_type_node, vectype))
3893 : {
3894 : /* The encoding uses one stepped pattern for each byte in the
3895 : 16-bit word. */
3896 0 : vec_perm_builder elts (TYPE_VECTOR_SUBPARTS (char_vectype), 2, 3);
3897 0 : for (unsigned i = 0; i < 3; ++i)
3898 0 : for (unsigned j = 0; j < 2; ++j)
3899 0 : elts.quick_push ((i + 1) * 2 - j - 1);
3900 :
3901 0 : vec_perm_indices indices (elts, 1,
3902 0 : TYPE_VECTOR_SUBPARTS (char_vectype));
3903 0 : machine_mode vmode = TYPE_MODE (char_vectype);
3904 0 : if (can_vec_perm_const_p (vmode, vmode, indices))
3905 : {
3906 : /* vectorizable_bswap can handle the __builtin_bswap16 if we
3907 : undo the argument promotion. */
3908 0 : if (!useless_type_conversion_p (type, TREE_TYPE (oprnd0)))
3909 : {
3910 0 : def = vect_recog_temp_ssa_var (type, NULL);
3911 0 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd0);
3912 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
3913 0 : oprnd0 = def;
3914 : }
3915 :
3916 : /* Pattern detected. */
3917 0 : vect_pattern_detected ("vect_recog_rotate_pattern", last_stmt);
3918 :
3919 0 : *type_out = vectype;
3920 :
3921 : /* Pattern supported. Create a stmt to be used to replace the
3922 : pattern, with the unpromoted argument. */
3923 0 : var = vect_recog_temp_ssa_var (type, NULL);
3924 0 : pattern_stmt = gimple_build_call (gimple_call_fndecl (last_stmt),
3925 : 1, oprnd0);
3926 0 : gimple_call_set_lhs (pattern_stmt, var);
3927 0 : gimple_call_set_fntype (as_a <gcall *> (pattern_stmt),
3928 : gimple_call_fntype (last_stmt));
3929 0 : return pattern_stmt;
3930 : }
3931 0 : }
3932 :
3933 0 : oprnd1 = build_int_cst (integer_type_node, 8);
3934 0 : rhs_code = LROTATE_EXPR;
3935 0 : bswap16_p = true;
3936 : }
3937 : else
3938 : return NULL;
3939 :
3940 8519 : if (TREE_CODE (oprnd0) != SSA_NAME
3941 8399 : || !INTEGRAL_TYPE_P (type)
3942 16597 : || TYPE_PRECISION (TREE_TYPE (lhs)) != TYPE_PRECISION (type))
3943 : return NULL;
3944 :
3945 8078 : stmt_vec_info def_stmt_info;
3946 8078 : if (!vect_is_simple_use (oprnd1, vinfo, &dt, &def_stmt_info, &def_stmt))
3947 : return NULL;
3948 :
3949 8078 : if (dt != vect_internal_def
3950 7011 : && dt != vect_constant_def
3951 25 : && dt != vect_external_def)
3952 : return NULL;
3953 :
3954 8072 : vectype = get_vectype_for_scalar_type (vinfo, type);
3955 8072 : if (vectype == NULL_TREE)
3956 : return NULL;
3957 :
3958 : /* If vector/vector or vector/scalar rotate is supported by the target,
3959 : don't do anything here. */
3960 7777 : optab1 = optab_for_tree_code (rhs_code, vectype, optab_vector);
3961 7777 : if (optab1
3962 7777 : && can_implement_p (optab1, TYPE_MODE (vectype)))
3963 : {
3964 564 : use_rotate:
3965 564 : if (bswap16_p)
3966 : {
3967 0 : if (!useless_type_conversion_p (type, TREE_TYPE (oprnd0)))
3968 : {
3969 0 : def = vect_recog_temp_ssa_var (type, NULL);
3970 0 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd0);
3971 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
3972 0 : oprnd0 = def;
3973 : }
3974 :
3975 : /* Pattern detected. */
3976 0 : vect_pattern_detected ("vect_recog_rotate_pattern", last_stmt);
3977 :
3978 0 : *type_out = vectype;
3979 :
3980 : /* Pattern supported. Create a stmt to be used to replace the
3981 : pattern. */
3982 0 : var = vect_recog_temp_ssa_var (type, NULL);
3983 0 : pattern_stmt = gimple_build_assign (var, LROTATE_EXPR, oprnd0,
3984 : oprnd1);
3985 0 : return pattern_stmt;
3986 : }
3987 : return NULL;
3988 : }
3989 :
3990 7753 : if (is_a <bb_vec_info> (vinfo) || dt != vect_internal_def)
3991 : {
3992 7669 : optab2 = optab_for_tree_code (rhs_code, vectype, optab_scalar);
3993 7669 : if (optab2
3994 7669 : && can_implement_p (optab2, TYPE_MODE (vectype)))
3995 540 : goto use_rotate;
3996 : }
3997 :
3998 : /* We may not use a reduction operand twice. */
3999 7213 : if (vect_is_reduction (stmt_vinfo))
4000 : return NULL;
4001 :
4002 7192 : tree utype = unsigned_type_for (type);
4003 7192 : tree uvectype = get_vectype_for_scalar_type (vinfo, utype);
4004 7192 : if (!uvectype)
4005 : return NULL;
4006 :
4007 : /* If vector/vector or vector/scalar shifts aren't supported by the target,
4008 : don't do anything here either. */
4009 7192 : optab1 = optab_for_tree_code (LSHIFT_EXPR, uvectype, optab_vector);
4010 7192 : optab2 = optab_for_tree_code (RSHIFT_EXPR, uvectype, optab_vector);
4011 7192 : if (!optab1
4012 7192 : || !can_implement_p (optab1, TYPE_MODE (uvectype))
4013 746 : || !optab2
4014 7938 : || !can_implement_p (optab2, TYPE_MODE (uvectype)))
4015 : {
4016 6446 : if (! is_a <bb_vec_info> (vinfo) && dt == vect_internal_def)
4017 : return NULL;
4018 6383 : optab1 = optab_for_tree_code (LSHIFT_EXPR, uvectype, optab_scalar);
4019 6383 : optab2 = optab_for_tree_code (RSHIFT_EXPR, uvectype, optab_scalar);
4020 6383 : if (!optab1
4021 6383 : || !can_implement_p (optab1, TYPE_MODE (uvectype))
4022 4769 : || !optab2
4023 11152 : || !can_implement_p (optab2, TYPE_MODE (uvectype)))
4024 : return NULL;
4025 : }
4026 :
4027 5515 : *type_out = vectype;
4028 :
4029 5515 : if (!useless_type_conversion_p (utype, TREE_TYPE (oprnd0)))
4030 : {
4031 52 : def = vect_recog_temp_ssa_var (utype, NULL);
4032 52 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd0);
4033 52 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
4034 52 : oprnd0 = def;
4035 : }
4036 :
4037 5515 : if (dt == vect_external_def && TREE_CODE (oprnd1) == SSA_NAME)
4038 15 : ext_def = vect_get_external_def_edge (vinfo, oprnd1);
4039 :
4040 5515 : def = NULL_TREE;
4041 5515 : scalar_int_mode mode = SCALAR_INT_TYPE_MODE (utype);
4042 5515 : if (dt != vect_internal_def || TYPE_MODE (TREE_TYPE (oprnd1)) == mode)
4043 : def = oprnd1;
4044 124 : else if (def_stmt && gimple_assign_cast_p (def_stmt))
4045 : {
4046 96 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
4047 96 : if (TYPE_MODE (TREE_TYPE (rhs1)) == mode
4048 96 : && TYPE_PRECISION (TREE_TYPE (rhs1))
4049 0 : == TYPE_PRECISION (type))
4050 : def = rhs1;
4051 : }
4052 :
4053 5391 : if (def == NULL_TREE)
4054 : {
4055 124 : def = vect_recog_temp_ssa_var (utype, NULL);
4056 124 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd1);
4057 124 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
4058 : }
4059 5515 : stype = TREE_TYPE (def);
4060 :
4061 5515 : if (TREE_CODE (def) == INTEGER_CST)
4062 : {
4063 4937 : if (!tree_fits_uhwi_p (def)
4064 4937 : || tree_to_uhwi (def) >= GET_MODE_PRECISION (mode)
4065 9874 : || integer_zerop (def))
4066 : return NULL;
4067 4937 : def2 = build_int_cst (stype,
4068 4937 : GET_MODE_PRECISION (mode) - tree_to_uhwi (def));
4069 : }
4070 : else
4071 : {
4072 578 : tree vecstype = get_vectype_for_scalar_type (vinfo, stype);
4073 :
4074 578 : if (vecstype == NULL_TREE)
4075 : return NULL;
4076 578 : def2 = vect_recog_temp_ssa_var (stype, NULL);
4077 578 : def_stmt = gimple_build_assign (def2, NEGATE_EXPR, def);
4078 578 : if (ext_def)
4079 : {
4080 15 : basic_block new_bb
4081 15 : = gsi_insert_on_edge_immediate (ext_def, def_stmt);
4082 15 : gcc_assert (!new_bb);
4083 : }
4084 : else
4085 563 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecstype);
4086 :
4087 578 : def2 = vect_recog_temp_ssa_var (stype, NULL);
4088 578 : tree mask = build_int_cst (stype, GET_MODE_PRECISION (mode) - 1);
4089 578 : def_stmt = gimple_build_assign (def2, BIT_AND_EXPR,
4090 : gimple_assign_lhs (def_stmt), mask);
4091 578 : if (ext_def)
4092 : {
4093 15 : basic_block new_bb
4094 15 : = gsi_insert_on_edge_immediate (ext_def, def_stmt);
4095 15 : gcc_assert (!new_bb);
4096 : }
4097 : else
4098 563 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecstype);
4099 : }
4100 :
4101 5515 : var1 = vect_recog_temp_ssa_var (utype, NULL);
4102 10719 : def_stmt = gimple_build_assign (var1, rhs_code == LROTATE_EXPR
4103 : ? LSHIFT_EXPR : RSHIFT_EXPR,
4104 : oprnd0, def);
4105 5515 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
4106 :
4107 5515 : var2 = vect_recog_temp_ssa_var (utype, NULL);
4108 10719 : def_stmt = gimple_build_assign (var2, rhs_code == LROTATE_EXPR
4109 : ? RSHIFT_EXPR : LSHIFT_EXPR,
4110 : oprnd0, def2);
4111 5515 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
4112 :
4113 : /* Pattern detected. */
4114 5515 : vect_pattern_detected ("vect_recog_rotate_pattern", last_stmt);
4115 :
4116 : /* Pattern supported. Create a stmt to be used to replace the pattern. */
4117 5515 : var = vect_recog_temp_ssa_var (utype, NULL);
4118 5515 : pattern_stmt = gimple_build_assign (var, BIT_IOR_EXPR, var1, var2);
4119 :
4120 5515 : if (!useless_type_conversion_p (type, utype))
4121 : {
4122 52 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, uvectype);
4123 52 : tree result = vect_recog_temp_ssa_var (type, NULL);
4124 52 : pattern_stmt = gimple_build_assign (result, NOP_EXPR, var);
4125 : }
4126 : return pattern_stmt;
4127 : }
4128 :
4129 : /* Detect a vector by vector shift pattern that wouldn't be otherwise
4130 : vectorized:
4131 :
4132 : type a_t;
4133 : TYPE b_T, res_T;
4134 :
4135 : S1 a_t = ;
4136 : S2 b_T = ;
4137 : S3 res_T = b_T op a_t;
4138 :
4139 : where type 'TYPE' is a type with different size than 'type',
4140 : and op is <<, >> or rotate.
4141 :
4142 : Also detect cases:
4143 :
4144 : type a_t;
4145 : TYPE b_T, c_T, res_T;
4146 :
4147 : S0 c_T = ;
4148 : S1 a_t = (type) c_T;
4149 : S2 b_T = ;
4150 : S3 res_T = b_T op a_t;
4151 :
4152 : Input/Output:
4153 :
4154 : * STMT_VINFO: The stmt from which the pattern search begins,
4155 : i.e. the shift/rotate stmt. The original stmt (S3) is replaced
4156 : with a shift/rotate which has same type on both operands, in the
4157 : second case just b_T op c_T, in the first case with added cast
4158 : from a_t to c_T in STMT_VINFO_PATTERN_DEF_SEQ.
4159 :
4160 : Output:
4161 :
4162 : * TYPE_OUT: The type of the output of this pattern.
4163 :
4164 : * Return value: A new stmt that will be used to replace the shift/rotate
4165 : S3 stmt. */
4166 :
4167 : static gimple *
4168 32310735 : vect_recog_vector_vector_shift_pattern (vec_info *vinfo,
4169 : stmt_vec_info stmt_vinfo,
4170 : tree *type_out)
4171 : {
4172 32310735 : gimple *last_stmt = stmt_vinfo->stmt;
4173 32310735 : tree oprnd0, oprnd1, lhs, var;
4174 32310735 : gimple *pattern_stmt;
4175 32310735 : enum tree_code rhs_code;
4176 :
4177 32310735 : if (!is_gimple_assign (last_stmt))
4178 : return NULL;
4179 :
4180 22521253 : rhs_code = gimple_assign_rhs_code (last_stmt);
4181 22521253 : switch (rhs_code)
4182 : {
4183 503513 : case LSHIFT_EXPR:
4184 503513 : case RSHIFT_EXPR:
4185 503513 : case LROTATE_EXPR:
4186 503513 : case RROTATE_EXPR:
4187 503513 : break;
4188 : default:
4189 : return NULL;
4190 : }
4191 :
4192 503513 : lhs = gimple_assign_lhs (last_stmt);
4193 503513 : oprnd0 = gimple_assign_rhs1 (last_stmt);
4194 503513 : oprnd1 = gimple_assign_rhs2 (last_stmt);
4195 503513 : if (TREE_CODE (oprnd1) != SSA_NAME
4196 110693 : || TYPE_MODE (TREE_TYPE (oprnd0)) == TYPE_MODE (TREE_TYPE (oprnd1))
4197 49411 : || !INTEGRAL_TYPE_P (TREE_TYPE (oprnd0))
4198 48942 : || !type_has_mode_precision_p (TREE_TYPE (oprnd1))
4199 552455 : || TYPE_PRECISION (TREE_TYPE (lhs))
4200 48942 : != TYPE_PRECISION (TREE_TYPE (oprnd0)))
4201 : return NULL;
4202 :
4203 48942 : stmt_vec_info def_vinfo = vinfo->lookup_def (oprnd1);
4204 48942 : if (!def_vinfo || STMT_VINFO_DEF_TYPE (def_vinfo) == vect_external_def)
4205 : return NULL;
4206 :
4207 46098 : def_vinfo = vect_stmt_to_vectorize (def_vinfo);
4208 1128 : gcc_assert (def_vinfo);
4209 :
4210 46098 : *type_out = get_vectype_for_scalar_type (vinfo, TREE_TYPE (oprnd0));
4211 46098 : if (*type_out == NULL_TREE)
4212 : return NULL;
4213 :
4214 33324 : tree def = NULL_TREE;
4215 33324 : gassign *def_stmt = dyn_cast <gassign *> (def_vinfo->stmt);
4216 19505 : if (def_stmt && gimple_assign_cast_p (def_stmt))
4217 : {
4218 5490 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
4219 5490 : if (TYPE_MODE (TREE_TYPE (rhs1)) == TYPE_MODE (TREE_TYPE (oprnd0))
4220 5490 : && TYPE_PRECISION (TREE_TYPE (rhs1))
4221 1284 : == TYPE_PRECISION (TREE_TYPE (oprnd0)))
4222 : {
4223 1284 : if (TYPE_PRECISION (TREE_TYPE (oprnd1))
4224 1284 : >= TYPE_PRECISION (TREE_TYPE (rhs1)))
4225 : def = rhs1;
4226 : else
4227 : {
4228 1182 : tree mask
4229 1182 : = build_low_bits_mask (TREE_TYPE (rhs1),
4230 1182 : TYPE_PRECISION (TREE_TYPE (oprnd1)));
4231 1182 : def = vect_recog_temp_ssa_var (TREE_TYPE (rhs1), NULL);
4232 1182 : def_stmt = gimple_build_assign (def, BIT_AND_EXPR, rhs1, mask);
4233 1182 : tree vecstype = get_vectype_for_scalar_type (vinfo,
4234 1182 : TREE_TYPE (rhs1));
4235 1182 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecstype);
4236 : }
4237 : }
4238 : }
4239 :
4240 1284 : if (def == NULL_TREE)
4241 : {
4242 32040 : def = vect_recog_temp_ssa_var (TREE_TYPE (oprnd0), NULL);
4243 32040 : def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd1);
4244 32040 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
4245 : }
4246 :
4247 : /* Pattern detected. */
4248 33324 : vect_pattern_detected ("vect_recog_vector_vector_shift_pattern", last_stmt);
4249 :
4250 : /* Pattern supported. Create a stmt to be used to replace the pattern. */
4251 33324 : var = vect_recog_temp_ssa_var (TREE_TYPE (oprnd0), NULL);
4252 33324 : pattern_stmt = gimple_build_assign (var, rhs_code, oprnd0, def);
4253 :
4254 33324 : return pattern_stmt;
4255 : }
4256 :
4257 : /* Verify that the target has optabs of VECTYPE to perform all the steps
4258 : needed by the multiplication-by-immediate synthesis algorithm described by
4259 : ALG and VAR. If SYNTH_SHIFT_P is true ensure that vector addition is
4260 : present. Return true iff the target supports all the steps. */
4261 :
4262 : static bool
4263 306360 : target_supports_mult_synth_alg (struct algorithm *alg, mult_variant var,
4264 : tree vectype, bool synth_shift_p)
4265 : {
4266 306360 : if (alg->op[0] != alg_zero && alg->op[0] != alg_m)
4267 : return false;
4268 :
4269 306360 : bool supports_vminus = target_has_vecop_for_code (MINUS_EXPR, vectype);
4270 306360 : bool supports_vplus = target_has_vecop_for_code (PLUS_EXPR, vectype);
4271 :
4272 306360 : if (var == negate_variant
4273 306360 : && !target_has_vecop_for_code (NEGATE_EXPR, vectype))
4274 : return false;
4275 :
4276 : /* If we must synthesize shifts with additions make sure that vector
4277 : addition is available. */
4278 305759 : if ((var == add_variant || synth_shift_p) && !supports_vplus)
4279 : return false;
4280 :
4281 148113 : for (int i = 1; i < alg->ops; i++)
4282 : {
4283 111263 : switch (alg->op[i])
4284 : {
4285 : case alg_shift:
4286 : break;
4287 27312 : case alg_add_t_m2:
4288 27312 : case alg_add_t2_m:
4289 27312 : case alg_add_factor:
4290 27312 : if (!supports_vplus)
4291 : return false;
4292 : break;
4293 17434 : case alg_sub_t_m2:
4294 17434 : case alg_sub_t2_m:
4295 17434 : case alg_sub_factor:
4296 17434 : if (!supports_vminus)
4297 : return false;
4298 : break;
4299 : case alg_unknown:
4300 : case alg_m:
4301 : case alg_zero:
4302 : case alg_impossible:
4303 : return false;
4304 0 : default:
4305 0 : gcc_unreachable ();
4306 : }
4307 : }
4308 :
4309 : return true;
4310 : }
4311 :
4312 : /* Synthesize a left shift of OP by AMNT bits using a series of additions and
4313 : putting the final result in DEST. Append all statements but the last into
4314 : VINFO. Return the last statement. */
4315 :
4316 : static gimple *
4317 0 : synth_lshift_by_additions (vec_info *vinfo,
4318 : tree dest, tree op, HOST_WIDE_INT amnt,
4319 : stmt_vec_info stmt_info, tree vectype)
4320 : {
4321 0 : HOST_WIDE_INT i;
4322 0 : tree itype = TREE_TYPE (op);
4323 0 : tree prev_res = op;
4324 0 : gcc_assert (amnt >= 0);
4325 0 : for (i = 0; i < amnt; i++)
4326 : {
4327 0 : tree tmp_var = (i < amnt - 1) ? vect_recog_temp_ssa_var (itype, NULL)
4328 : : dest;
4329 0 : gimple *stmt
4330 0 : = gimple_build_assign (tmp_var, PLUS_EXPR, prev_res, prev_res);
4331 0 : prev_res = tmp_var;
4332 0 : if (i < amnt - 1)
4333 0 : append_pattern_def_seq (vinfo, stmt_info, stmt, vectype);
4334 : else
4335 0 : return stmt;
4336 : }
4337 0 : gcc_unreachable ();
4338 : return NULL;
4339 : }
4340 :
4341 : /* Helper for vect_synth_mult_by_constant. Apply a binary operation
4342 : CODE to operands OP1 and OP2, creating a new temporary SSA var in
4343 : the process if necessary. Append the resulting assignment statements
4344 : to the sequence in STMT_VINFO. Return the SSA variable that holds the
4345 : result of the binary operation. If SYNTH_SHIFT_P is true synthesize
4346 : left shifts using additions. */
4347 :
4348 : static tree
4349 44428 : apply_binop_and_append_stmt (vec_info *vinfo,
4350 : tree_code code, tree op1, tree op2,
4351 : stmt_vec_info stmt_vinfo, tree vectype,
4352 : bool synth_shift_p)
4353 : {
4354 44428 : if (integer_zerop (op2)
4355 44428 : && (code == LSHIFT_EXPR
4356 38608 : || code == PLUS_EXPR))
4357 : {
4358 38608 : gcc_assert (TREE_CODE (op1) == SSA_NAME);
4359 : return op1;
4360 : }
4361 :
4362 5820 : gimple *stmt;
4363 5820 : tree itype = TREE_TYPE (op1);
4364 5820 : tree tmp_var = vect_recog_temp_ssa_var (itype, NULL);
4365 :
4366 5820 : if (code == LSHIFT_EXPR
4367 5820 : && synth_shift_p)
4368 : {
4369 0 : stmt = synth_lshift_by_additions (vinfo, tmp_var, op1,
4370 0 : TREE_INT_CST_LOW (op2), stmt_vinfo,
4371 : vectype);
4372 0 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4373 0 : return tmp_var;
4374 : }
4375 :
4376 5820 : stmt = gimple_build_assign (tmp_var, code, op1, op2);
4377 5820 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4378 5820 : return tmp_var;
4379 : }
4380 :
4381 : /* Synthesize a multiplication of OP by an INTEGER_CST VAL using shifts
4382 : and simple arithmetic operations to be vectorized. Record the statements
4383 : produced in STMT_VINFO and return the last statement in the sequence or
4384 : NULL if it's not possible to synthesize such a multiplication.
4385 : This function mirrors the behavior of expand_mult_const in expmed.cc but
4386 : works on tree-ssa form. */
4387 :
4388 : static gimple *
4389 309719 : vect_synth_mult_by_constant (vec_info *vinfo, tree op, tree val,
4390 : stmt_vec_info stmt_vinfo)
4391 : {
4392 309719 : tree itype = TREE_TYPE (op);
4393 309719 : machine_mode mode = TYPE_MODE (itype);
4394 309719 : struct algorithm alg;
4395 309719 : mult_variant variant;
4396 309719 : if (!tree_fits_shwi_p (val))
4397 : return NULL;
4398 :
4399 : /* Multiplication synthesis by shifts, adds and subs can introduce
4400 : signed overflow where the original operation didn't. Perform the
4401 : operations on an unsigned type and cast back to avoid this.
4402 : In the future we may want to relax this for synthesis algorithms
4403 : that we can prove do not cause unexpected overflow. */
4404 306375 : bool cast_to_unsigned_p = !TYPE_OVERFLOW_WRAPS (itype);
4405 :
4406 59805 : tree multtype = cast_to_unsigned_p ? unsigned_type_for (itype) : itype;
4407 306375 : tree vectype = get_vectype_for_scalar_type (vinfo, multtype);
4408 306375 : if (!vectype)
4409 : return NULL;
4410 :
4411 : /* Targets that don't support vector shifts but support vector additions
4412 : can synthesize shifts that way. */
4413 306375 : bool synth_shift_p = !vect_supportable_shift (vinfo, LSHIFT_EXPR, multtype);
4414 :
4415 306375 : HOST_WIDE_INT hwval = tree_to_shwi (val);
4416 : /* Use MAX_COST here as we don't want to limit the sequence on rtx costs.
4417 : The vectorizer's benefit analysis will decide whether it's beneficial
4418 : to do this. */
4419 612554 : bool possible = choose_mult_variant (VECTOR_MODE_P (TYPE_MODE (vectype))
4420 306179 : ? TYPE_MODE (vectype) : mode,
4421 : hwval, &alg, &variant, MAX_COST);
4422 306375 : if (!possible)
4423 : return NULL;
4424 :
4425 306375 : if (vect_is_reduction (stmt_vinfo))
4426 : {
4427 26 : int op_uses = alg.op[0] != alg_zero;
4428 45 : for (int i = 1; i < alg.ops; i++)
4429 32 : switch (alg.op[i])
4430 : {
4431 4 : case alg_add_t_m2:
4432 4 : case alg_sub_t_m2:
4433 4 : if (synth_shift_p && alg.log[i])
4434 : return NULL;
4435 : else
4436 4 : op_uses++;
4437 4 : break;
4438 0 : case alg_add_t2_m:
4439 0 : case alg_sub_t2_m:
4440 0 : op_uses++;
4441 : /* Fallthru. */
4442 28 : case alg_shift:
4443 28 : if (synth_shift_p && alg.log[i])
4444 : return NULL;
4445 : break;
4446 : case alg_add_factor:
4447 : case alg_sub_factor:
4448 : return NULL;
4449 : default:
4450 : break;
4451 : }
4452 13 : if (variant == add_variant)
4453 0 : op_uses++;
4454 : /* When we'll synthesize more than a single use of the reduction
4455 : operand the reduction constraints are violated. Avoid this
4456 : situation. */
4457 13 : if (op_uses > 1)
4458 : return NULL;
4459 : }
4460 :
4461 306360 : if (!target_supports_mult_synth_alg (&alg, variant, vectype, synth_shift_p))
4462 : return NULL;
4463 :
4464 36850 : tree accumulator;
4465 :
4466 : /* Clear out the sequence of statements so we can populate it below. */
4467 36850 : gimple *stmt = NULL;
4468 :
4469 36850 : if (cast_to_unsigned_p)
4470 : {
4471 12436 : tree tmp_op = vect_recog_temp_ssa_var (multtype, NULL);
4472 12436 : stmt = gimple_build_assign (tmp_op, CONVERT_EXPR, op);
4473 12436 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4474 12436 : op = tmp_op;
4475 : }
4476 :
4477 36850 : if (alg.op[0] == alg_zero)
4478 205 : accumulator = build_int_cst (multtype, 0);
4479 : else
4480 : accumulator = op;
4481 :
4482 36850 : bool needs_fixup = (variant == negate_variant)
4483 36850 : || (variant == add_variant);
4484 :
4485 147510 : for (int i = 1; i < alg.ops; i++)
4486 : {
4487 110660 : tree shft_log = build_int_cst (multtype, alg.log[i]);
4488 110660 : tree accum_tmp = vect_recog_temp_ssa_var (multtype, NULL);
4489 110660 : tree tmp_var = NULL_TREE;
4490 :
4491 110660 : switch (alg.op[i])
4492 : {
4493 66232 : case alg_shift:
4494 66232 : if (synth_shift_p)
4495 0 : stmt
4496 0 : = synth_lshift_by_additions (vinfo, accum_tmp, accumulator,
4497 0 : alg.log[i], stmt_vinfo, vectype);
4498 : else
4499 66232 : stmt = gimple_build_assign (accum_tmp, LSHIFT_EXPR, accumulator,
4500 : shft_log);
4501 : break;
4502 22361 : case alg_add_t_m2:
4503 22361 : tmp_var
4504 22361 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, op, shft_log,
4505 : stmt_vinfo, vectype, synth_shift_p);
4506 22361 : stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, accumulator,
4507 : tmp_var);
4508 22361 : break;
4509 16450 : case alg_sub_t_m2:
4510 16450 : tmp_var = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, op,
4511 : shft_log, stmt_vinfo,
4512 : vectype, synth_shift_p);
4513 : /* In some algorithms the first step involves zeroing the
4514 : accumulator. If subtracting from such an accumulator
4515 : just emit the negation directly. */
4516 16450 : if (integer_zerop (accumulator))
4517 205 : stmt = gimple_build_assign (accum_tmp, NEGATE_EXPR, tmp_var);
4518 : else
4519 16245 : stmt = gimple_build_assign (accum_tmp, MINUS_EXPR, accumulator,
4520 : tmp_var);
4521 : break;
4522 0 : case alg_add_t2_m:
4523 0 : tmp_var
4524 0 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
4525 : shft_log, stmt_vinfo, vectype,
4526 : synth_shift_p);
4527 0 : stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, tmp_var, op);
4528 0 : break;
4529 0 : case alg_sub_t2_m:
4530 0 : tmp_var
4531 0 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
4532 : shft_log, stmt_vinfo, vectype,
4533 : synth_shift_p);
4534 0 : stmt = gimple_build_assign (accum_tmp, MINUS_EXPR, tmp_var, op);
4535 0 : break;
4536 4827 : case alg_add_factor:
4537 4827 : tmp_var
4538 4827 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
4539 : shft_log, stmt_vinfo, vectype,
4540 : synth_shift_p);
4541 4827 : stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, accumulator,
4542 : tmp_var);
4543 4827 : break;
4544 790 : case alg_sub_factor:
4545 790 : tmp_var
4546 790 : = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
4547 : shft_log, stmt_vinfo, vectype,
4548 : synth_shift_p);
4549 790 : stmt = gimple_build_assign (accum_tmp, MINUS_EXPR, tmp_var,
4550 : accumulator);
4551 790 : break;
4552 0 : default:
4553 0 : gcc_unreachable ();
4554 : }
4555 : /* We don't want to append the last stmt in the sequence to stmt_vinfo
4556 : but rather return it directly. */
4557 :
4558 110660 : if ((i < alg.ops - 1) || needs_fixup || cast_to_unsigned_p)
4559 86549 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4560 110660 : accumulator = accum_tmp;
4561 : }
4562 36850 : if (variant == negate_variant)
4563 : {
4564 443 : tree accum_tmp = vect_recog_temp_ssa_var (multtype, NULL);
4565 443 : stmt = gimple_build_assign (accum_tmp, NEGATE_EXPR, accumulator);
4566 443 : accumulator = accum_tmp;
4567 443 : if (cast_to_unsigned_p)
4568 150 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4569 : }
4570 36407 : else if (variant == add_variant)
4571 : {
4572 101 : tree accum_tmp = vect_recog_temp_ssa_var (multtype, NULL);
4573 101 : stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, accumulator, op);
4574 101 : accumulator = accum_tmp;
4575 101 : if (cast_to_unsigned_p)
4576 91 : append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
4577 : }
4578 : /* Move back to a signed if needed. */
4579 36547 : if (cast_to_unsigned_p)
4580 : {
4581 12436 : tree accum_tmp = vect_recog_temp_ssa_var (itype, NULL);
4582 12436 : stmt = gimple_build_assign (accum_tmp, CONVERT_EXPR, accumulator);
4583 : }
4584 :
4585 : return stmt;
4586 : }
4587 :
4588 : /* Detect multiplication by constant and convert it into a sequence of
4589 : shifts and additions, subtractions, negations. We reuse the
4590 : choose_mult_variant algorithms from expmed.cc
4591 :
4592 : Input/Output:
4593 :
4594 : STMT_VINFO: The stmt from which the pattern search begins,
4595 : i.e. the mult stmt.
4596 :
4597 : Output:
4598 :
4599 : * TYPE_OUT: The type of the output of this pattern.
4600 :
4601 : * Return value: A new stmt that will be used to replace
4602 : the multiplication. */
4603 :
4604 : static gimple *
4605 32494956 : vect_recog_mult_pattern (vec_info *vinfo,
4606 : stmt_vec_info stmt_vinfo, tree *type_out)
4607 : {
4608 32494956 : gimple *last_stmt = stmt_vinfo->stmt;
4609 32494956 : tree oprnd0, oprnd1, vectype, itype;
4610 32494956 : gimple *pattern_stmt;
4611 :
4612 32494956 : if (!is_gimple_assign (last_stmt))
4613 : return NULL;
4614 :
4615 22705474 : if (gimple_assign_rhs_code (last_stmt) != MULT_EXPR)
4616 : return NULL;
4617 :
4618 1541435 : oprnd0 = gimple_assign_rhs1 (last_stmt);
4619 1541435 : oprnd1 = gimple_assign_rhs2 (last_stmt);
4620 1541435 : itype = TREE_TYPE (oprnd0);
4621 :
4622 1541435 : if (TREE_CODE (oprnd0) != SSA_NAME
4623 1541368 : || TREE_CODE (oprnd1) != INTEGER_CST
4624 921809 : || !INTEGRAL_TYPE_P (itype)
4625 2463244 : || !type_has_mode_precision_p (itype))
4626 : return NULL;
4627 :
4628 921754 : vectype = get_vectype_for_scalar_type (vinfo, itype);
4629 921754 : if (vectype == NULL_TREE)
4630 : return NULL;
4631 :
4632 : /* If the target can handle vectorized multiplication natively,
4633 : don't attempt to optimize this. */
4634 750775 : optab mul_optab = optab_for_tree_code (MULT_EXPR, vectype, optab_default);
4635 750775 : if (mul_optab != unknown_optab
4636 750775 : && can_implement_p (mul_optab, TYPE_MODE (vectype)))
4637 : return NULL;
4638 :
4639 309719 : pattern_stmt = vect_synth_mult_by_constant (vinfo,
4640 : oprnd0, oprnd1, stmt_vinfo);
4641 309719 : if (!pattern_stmt)
4642 : return NULL;
4643 :
4644 : /* Pattern detected. */
4645 36850 : vect_pattern_detected ("vect_recog_mult_pattern", last_stmt);
4646 :
4647 36850 : *type_out = vectype;
4648 :
4649 36850 : return pattern_stmt;
4650 : }
4651 :
4652 : extern bool gimple_unsigned_integer_sat_add (tree, tree*, tree (*)(tree));
4653 : extern bool gimple_unsigned_integer_sat_sub (tree, tree*, tree (*)(tree));
4654 : extern bool gimple_unsigned_integer_sat_trunc (tree, tree*, tree (*)(tree));
4655 :
4656 : extern bool gimple_unsigned_integer_narrow_clip (tree, tree*, tree (*)(tree));
4657 :
4658 : extern bool gimple_signed_integer_sat_add (tree, tree*, tree (*)(tree));
4659 : extern bool gimple_signed_integer_sat_sub (tree, tree*, tree (*)(tree));
4660 : extern bool gimple_signed_integer_sat_trunc (tree, tree*, tree (*)(tree));
4661 :
4662 : static gimple *
4663 1177 : vect_recog_build_binary_gimple_stmt (vec_info *vinfo, stmt_vec_info stmt_info,
4664 : internal_fn fn, tree *type_out,
4665 : tree lhs, tree op_0, tree op_1)
4666 : {
4667 1177 : tree itype = TREE_TYPE (op_0);
4668 1177 : tree otype = TREE_TYPE (lhs);
4669 1177 : tree v_itype = get_vectype_for_scalar_type (vinfo, itype);
4670 1177 : tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
4671 :
4672 1177 : if (v_itype != NULL_TREE && v_otype != NULL_TREE
4673 1177 : && direct_internal_fn_supported_p (fn, v_itype, OPTIMIZE_FOR_BOTH))
4674 : {
4675 913 : gcall *call = gimple_build_call_internal (fn, 2, op_0, op_1);
4676 913 : tree in_ssa = vect_recog_temp_ssa_var (itype, NULL);
4677 :
4678 913 : gimple_call_set_lhs (call, in_ssa);
4679 913 : gimple_call_set_nothrow (call, /* nothrow_p */ false);
4680 913 : gimple_set_location (call, gimple_location (STMT_VINFO_STMT (stmt_info)));
4681 :
4682 913 : *type_out = v_otype;
4683 :
4684 913 : if (types_compatible_p (itype, otype))
4685 : return call;
4686 : else
4687 : {
4688 0 : append_pattern_def_seq (vinfo, stmt_info, call, v_itype);
4689 0 : tree out_ssa = vect_recog_temp_ssa_var (otype, NULL);
4690 :
4691 0 : return gimple_build_assign (out_ssa, NOP_EXPR, in_ssa);
4692 : }
4693 : }
4694 :
4695 : return NULL;
4696 : }
4697 :
4698 : /*
4699 : * Try to detect saturation add pattern (SAT_ADD), aka below gimple:
4700 : * _7 = _4 + _6;
4701 : * _8 = _4 > _7;
4702 : * _9 = (long unsigned int) _8;
4703 : * _10 = -_9;
4704 : * _12 = _7 | _10;
4705 : *
4706 : * And then simplified to
4707 : * _12 = .SAT_ADD (_4, _6);
4708 : */
4709 :
4710 : static gimple *
4711 32568312 : vect_recog_sat_add_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
4712 : tree *type_out)
4713 : {
4714 32568312 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
4715 :
4716 32568312 : if (!is_gimple_assign (last_stmt))
4717 : return NULL;
4718 :
4719 22778830 : tree ops[2];
4720 22778830 : tree lhs = gimple_assign_lhs (last_stmt);
4721 :
4722 22778830 : if (gimple_unsigned_integer_sat_add (lhs, ops, NULL)
4723 22778830 : || gimple_signed_integer_sat_add (lhs, ops, NULL))
4724 : {
4725 772 : if (TREE_CODE (ops[1]) == INTEGER_CST)
4726 13 : ops[1] = fold_convert (TREE_TYPE (ops[0]), ops[1]);
4727 :
4728 772 : gimple *stmt = vect_recog_build_binary_gimple_stmt (vinfo, stmt_vinfo,
4729 : IFN_SAT_ADD, type_out,
4730 : lhs, ops[0], ops[1]);
4731 772 : if (stmt)
4732 : {
4733 738 : vect_pattern_detected ("vect_recog_sat_add_pattern", last_stmt);
4734 738 : return stmt;
4735 : }
4736 : }
4737 :
4738 : return NULL;
4739 : }
4740 :
4741 : /*
4742 : * Try to transform the truncation for .SAT_SUB pattern, mostly occurs in
4743 : * the benchmark zip. Aka:
4744 : *
4745 : * unsigned int _1;
4746 : * unsigned int _2;
4747 : * unsigned short int _4;
4748 : * _9 = (unsigned short int).SAT_SUB (_1, _2);
4749 : *
4750 : * if _1 is known to be in the range of unsigned short int. For example
4751 : * there is a def _1 = (unsigned short int)_4. Then we can transform the
4752 : * truncation to:
4753 : *
4754 : * _3 = (unsigned short int) MIN (65535, _2); // aka _3 = .SAT_TRUNC (_2);
4755 : * _9 = .SAT_SUB (_4, _3);
4756 : *
4757 : * Then, we can better vectorized code and avoid the unnecessary narrowing
4758 : * stmt during vectorization with below stmt(s).
4759 : *
4760 : * _3 = .SAT_TRUNC(_2); // SI => HI
4761 : * _9 = .SAT_SUB (_4, _3);
4762 : */
4763 : static void
4764 405 : vect_recog_sat_sub_pattern_transform (vec_info *vinfo,
4765 : stmt_vec_info stmt_vinfo,
4766 : tree lhs, tree *ops)
4767 : {
4768 405 : tree otype = TREE_TYPE (lhs);
4769 405 : tree itype = TREE_TYPE (ops[0]);
4770 405 : unsigned itype_prec = TYPE_PRECISION (itype);
4771 405 : unsigned otype_prec = TYPE_PRECISION (otype);
4772 :
4773 405 : if (types_compatible_p (otype, itype) || otype_prec >= itype_prec)
4774 405 : return;
4775 :
4776 0 : tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
4777 0 : tree v_itype = get_vectype_for_scalar_type (vinfo, itype);
4778 0 : tree_pair v_pair = tree_pair (v_otype, v_itype);
4779 :
4780 0 : if (v_otype == NULL_TREE || v_itype == NULL_TREE
4781 0 : || !direct_internal_fn_supported_p (IFN_SAT_TRUNC, v_pair,
4782 : OPTIMIZE_FOR_BOTH))
4783 : return;
4784 :
4785 : /* 1. Find the _4 and update ops[0] as above example. */
4786 0 : vect_unpromoted_value unprom;
4787 0 : tree tmp = vect_look_through_possible_promotion (vinfo, ops[0], &unprom);
4788 :
4789 0 : if (tmp == NULL_TREE || TYPE_PRECISION (unprom.type) != otype_prec)
4790 : return;
4791 :
4792 0 : ops[0] = tmp;
4793 :
4794 : /* 2. Generate _3 = .SAT_TRUNC (_2) and update ops[1] as above example. */
4795 0 : tree trunc_lhs_ssa = vect_recog_temp_ssa_var (otype, NULL);
4796 0 : gcall *call = gimple_build_call_internal (IFN_SAT_TRUNC, 1, ops[1]);
4797 :
4798 0 : gimple_call_set_lhs (call, trunc_lhs_ssa);
4799 0 : gimple_call_set_nothrow (call, /* nothrow_p */ false);
4800 0 : append_pattern_def_seq (vinfo, stmt_vinfo, call, v_otype);
4801 :
4802 0 : ops[1] = trunc_lhs_ssa;
4803 : }
4804 :
4805 : /*
4806 : * Try to detect saturation sub pattern (SAT_ADD), aka below gimple:
4807 : * Unsigned:
4808 : * _7 = _1 >= _2;
4809 : * _8 = _1 - _2;
4810 : * _10 = (long unsigned int) _7;
4811 : * _9 = _8 * _10;
4812 : *
4813 : * And then simplified to
4814 : * _9 = .SAT_SUB (_1, _2);
4815 : *
4816 : * Signed:
4817 : * x.0_4 = (unsigned char) x_16;
4818 : * y.1_5 = (unsigned char) y_18;
4819 : * _6 = x.0_4 - y.1_5;
4820 : * minus_19 = (int8_t) _6;
4821 : * _7 = x_16 ^ y_18;
4822 : * _8 = x_16 ^ minus_19;
4823 : * _44 = _7 < 0;
4824 : * _23 = x_16 < 0;
4825 : * _24 = (signed char) _23;
4826 : * _58 = (unsigned char) _24;
4827 : * _59 = -_58;
4828 : * _25 = (signed char) _59;
4829 : * _26 = _25 ^ 127;
4830 : * _42 = _8 < 0;
4831 : * _41 = _42 & _44;
4832 : * iftmp.2_11 = _41 ? _26 : minus_19;
4833 : *
4834 : * And then simplified to
4835 : * iftmp.2_11 = .SAT_SUB (x_16, y_18);
4836 : */
4837 :
4838 : static gimple *
4839 32567574 : vect_recog_sat_sub_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
4840 : tree *type_out)
4841 : {
4842 32567574 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
4843 :
4844 32567574 : if (!is_gimple_assign (last_stmt))
4845 : return NULL;
4846 :
4847 22778092 : tree ops[2];
4848 22778092 : tree lhs = gimple_assign_lhs (last_stmt);
4849 :
4850 22778092 : if (gimple_unsigned_integer_sat_sub (lhs, ops, NULL)
4851 22778092 : || gimple_signed_integer_sat_sub (lhs, ops, NULL))
4852 : {
4853 405 : vect_recog_sat_sub_pattern_transform (vinfo, stmt_vinfo, lhs, ops);
4854 405 : gimple *stmt = vect_recog_build_binary_gimple_stmt (vinfo, stmt_vinfo,
4855 : IFN_SAT_SUB, type_out,
4856 : lhs, ops[0], ops[1]);
4857 405 : if (stmt)
4858 : {
4859 175 : vect_pattern_detected ("vect_recog_sat_sub_pattern", last_stmt);
4860 175 : return stmt;
4861 : }
4862 : }
4863 :
4864 : return NULL;
4865 : }
4866 :
4867 : /*
4868 : * Try to detect saturation truncation pattern (SAT_TRUNC), aka below gimple:
4869 : * overflow_5 = x_4(D) > 4294967295;
4870 : * _1 = (unsigned int) x_4(D);
4871 : * _2 = (unsigned int) overflow_5;
4872 : * _3 = -_2;
4873 : * _6 = _1 | _3;
4874 : *
4875 : * And then simplified to
4876 : * _6 = .SAT_TRUNC (x_4(D));
4877 : */
4878 :
4879 : static gimple *
4880 32567399 : vect_recog_sat_trunc_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
4881 : tree *type_out)
4882 : {
4883 32567399 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
4884 :
4885 32567399 : if (!is_gimple_assign (last_stmt))
4886 : return NULL;
4887 :
4888 22777917 : tree ops[1];
4889 22777917 : tree lhs = gimple_assign_lhs (last_stmt);
4890 22777917 : tree otype = TREE_TYPE (lhs);
4891 :
4892 22777917 : if ((gimple_unsigned_integer_narrow_clip (lhs, ops, NULL))
4893 22777917 : && type_has_mode_precision_p (otype))
4894 : {
4895 0 : tree itype = TREE_TYPE (ops[0]);
4896 0 : tree v_itype = get_vectype_for_scalar_type (vinfo, itype);
4897 0 : tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
4898 0 : internal_fn fn = IFN_SAT_TRUNC;
4899 :
4900 0 : if (v_itype != NULL_TREE && v_otype != NULL_TREE
4901 0 : && direct_internal_fn_supported_p (fn, tree_pair (v_otype, v_itype),
4902 : OPTIMIZE_FOR_BOTH))
4903 : {
4904 0 : tree temp = vect_recog_temp_ssa_var (itype, NULL);
4905 0 : gimple * max_stmt = gimple_build_assign (temp, build2 (MAX_EXPR, itype, build_zero_cst(itype), ops[0]));
4906 0 : append_pattern_def_seq (vinfo, stmt_vinfo, max_stmt, v_itype);
4907 :
4908 0 : gcall *call = gimple_build_call_internal (fn, 1, temp);
4909 0 : tree out_ssa = vect_recog_temp_ssa_var (otype, NULL);
4910 :
4911 0 : gimple_call_set_lhs (call, out_ssa);
4912 0 : gimple_call_set_nothrow (call, /* nothrow_p */ false);
4913 0 : gimple_set_location (call, gimple_location (last_stmt));
4914 :
4915 0 : *type_out = v_otype;
4916 :
4917 0 : return call;
4918 : }
4919 :
4920 : }
4921 :
4922 22777917 : if ((gimple_unsigned_integer_sat_trunc (lhs, ops, NULL)
4923 22777532 : || gimple_signed_integer_sat_trunc (lhs, ops, NULL))
4924 22777917 : && type_has_mode_precision_p (otype))
4925 : {
4926 373 : tree itype = TREE_TYPE (ops[0]);
4927 373 : tree v_itype = get_vectype_for_scalar_type (vinfo, itype);
4928 373 : tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
4929 373 : internal_fn fn = IFN_SAT_TRUNC;
4930 :
4931 367 : if (v_itype != NULL_TREE && v_otype != NULL_TREE
4932 740 : && direct_internal_fn_supported_p (fn, tree_pair (v_otype, v_itype),
4933 : OPTIMIZE_FOR_BOTH))
4934 : {
4935 0 : gcall *call = gimple_build_call_internal (fn, 1, ops[0]);
4936 0 : tree out_ssa = vect_recog_temp_ssa_var (otype, NULL);
4937 :
4938 0 : gimple_call_set_lhs (call, out_ssa);
4939 0 : gimple_call_set_nothrow (call, /* nothrow_p */ false);
4940 0 : gimple_set_location (call, gimple_location (last_stmt));
4941 :
4942 0 : *type_out = v_otype;
4943 :
4944 0 : return call;
4945 : }
4946 : }
4947 :
4948 : return NULL;
4949 : }
4950 :
4951 :
4952 : /* Function add_code_for_floorceilround_divmod
4953 : A helper function to add compensation code for implementing FLOOR_MOD_EXPR,
4954 : FLOOR_DIV_EXPR, CEIL_MOD_EXPR, CEIL_DIV_EXPR, ROUND_MOD_EXPR and
4955 : ROUND_DIV_EXPR
4956 : The quotient and remainder are needed for implemented these operators.
4957 : FLOOR cases
4958 : r = x %[fl] y; r = x/[fl] y;
4959 : is
4960 : r = x % y; if (r && (x ^ y) < 0) r += y;
4961 : r = x % y; d = x/y; if (r && (x ^ y) < 0) d--; Respectively
4962 : Produce following sequence
4963 : v0 = x^y
4964 : v1 = -r
4965 : v2 = r | -r
4966 : v3 = v0 & v2
4967 : v4 = v3 < 0
4968 : if (floor_mod)
4969 : v5 = v4 ? y : 0
4970 : v6 = r + v5
4971 : if (floor_div)
4972 : v5 = v4 ? 1 : 0
4973 : v6 = d - 1
4974 : Similar sequences of vector instructions are produces for following cases
4975 : CEIL cases
4976 : r = x %[cl] y; r = x/[cl] y;
4977 : is
4978 : r = x % y; if (r && (x ^ y) >= 0) r -= y;
4979 : r = x % y; if (r) r -= y; (unsigned)
4980 : r = x % y; d = x/y; if (r && (x ^ y) >= 0) d++;
4981 : r = x % y; d = x/y; if (r) d++; (unsigned)
4982 : ROUND cases
4983 : r = x %[rd] y; r = x/[rd] y;
4984 : is
4985 : r = x % y; if (r > ((y-1)/2)) if ((x ^ y) >= 0) r -= y; else r += y;
4986 : r = x % y; if (r > ((y-1)/2)) r -= y; (unsigned)
4987 : r = x % y; d = x/y; if (r > ((y-1)/2)) if ((x ^ y) >= 0) d++; else d--;
4988 : r = x % y; d = x/y; if (r > ((y-1)/2)) d++; (unsigned)
4989 : Inputs:
4990 : VECTYPE: Vector type of the operands
4991 : STMT_VINFO: Statement where pattern begins
4992 : RHS_CODE: Should either be FLOOR_MOD_EXPR or FLOOR_DIV_EXPR
4993 : Q: The quotient of division
4994 : R: Remainder of division
4995 : OPRDN0/OPRND1: Actual operands involved
4996 : ITYPE: tree type of oprnd0
4997 : Output:
4998 : NULL if vectorization not possible
4999 : Gimple statement based on rhs_code
5000 : */
5001 : static gimple *
5002 414 : add_code_for_floorceilround_divmod (tree vectype, vec_info *vinfo,
5003 : stmt_vec_info stmt_vinfo,
5004 : enum tree_code rhs_code, tree q, tree r,
5005 : tree oprnd0, tree oprnd1, tree itype)
5006 : {
5007 414 : gimple *def_stmt;
5008 414 : tree mask_vectype = truth_type_for (vectype);
5009 414 : if (!mask_vectype)
5010 : return NULL;
5011 414 : tree bool_cond;
5012 414 : bool unsigned_p = TYPE_UNSIGNED (itype);
5013 :
5014 414 : switch (rhs_code)
5015 : {
5016 378 : case FLOOR_MOD_EXPR:
5017 378 : case FLOOR_DIV_EXPR:
5018 378 : case CEIL_MOD_EXPR:
5019 378 : case CEIL_DIV_EXPR:
5020 378 : {
5021 378 : if (!target_has_vecop_for_code (NEGATE_EXPR, vectype)
5022 354 : || !target_has_vecop_for_code (BIT_XOR_EXPR, vectype)
5023 354 : || !target_has_vecop_for_code (BIT_IOR_EXPR, vectype)
5024 354 : || !target_has_vecop_for_code (PLUS_EXPR, vectype)
5025 354 : || !target_has_vecop_for_code (MINUS_EXPR, vectype)
5026 354 : || !expand_vec_cmp_expr_p (vectype, mask_vectype, LT_EXPR)
5027 602 : || !expand_vec_cond_expr_p (vectype, mask_vectype))
5028 : return NULL;
5029 224 : if (unsigned_p)
5030 : {
5031 18 : gcc_assert (rhs_code == CEIL_MOD_EXPR || rhs_code == CEIL_DIV_EXPR);
5032 :
5033 18 : if (!expand_vec_cmp_expr_p (vectype, mask_vectype, GT_EXPR))
5034 : return NULL;
5035 18 : bool is_mod = rhs_code == CEIL_MOD_EXPR;
5036 : // r > 0
5037 18 : bool_cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5038 18 : def_stmt = gimple_build_assign (bool_cond, GT_EXPR, r,
5039 : build_int_cst (itype, 0));
5040 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5041 : itype);
5042 :
5043 : // (r > 0) ? y : 0 (mod)
5044 : // (r > 0) ? 1 : 0 (ceil)
5045 18 : tree extr_cond = vect_recog_temp_ssa_var (itype, NULL);
5046 18 : def_stmt
5047 27 : = gimple_build_assign (extr_cond, COND_EXPR, bool_cond,
5048 9 : is_mod ? oprnd1 : build_int_cst (itype, 1),
5049 : build_int_cst (itype, 0));
5050 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5051 :
5052 : // r -= (r > 0) ? y : 0 (mod)
5053 : // d += (x^y < 0 && r) ? -1 : 0 (ceil)
5054 18 : tree result = vect_recog_temp_ssa_var (itype, NULL);
5055 27 : return gimple_build_assign (result, is_mod ? MINUS_EXPR : PLUS_EXPR,
5056 18 : is_mod ? r : q, extr_cond);
5057 : }
5058 : else
5059 : {
5060 206 : bool ceil_p
5061 206 : = (rhs_code == CEIL_MOD_EXPR || rhs_code == CEIL_DIV_EXPR);
5062 206 : if (ceil_p && !target_has_vecop_for_code (BIT_NOT_EXPR, vectype))
5063 : return NULL;
5064 : // x ^ y
5065 206 : tree xort = vect_recog_temp_ssa_var (itype, NULL);
5066 206 : def_stmt = gimple_build_assign (xort, BIT_XOR_EXPR, oprnd0, oprnd1);
5067 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5068 :
5069 206 : tree cond_reg = xort;
5070 : // ~(x ^ y) (ceil)
5071 206 : if (ceil_p)
5072 : {
5073 18 : cond_reg = vect_recog_temp_ssa_var (itype, NULL);
5074 18 : def_stmt = gimple_build_assign (cond_reg, BIT_NOT_EXPR, xort);
5075 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5076 : }
5077 :
5078 : // -r
5079 206 : tree negate_r = vect_recog_temp_ssa_var (itype, NULL);
5080 206 : def_stmt = gimple_build_assign (negate_r, NEGATE_EXPR, r);
5081 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5082 :
5083 : // r | -r , sign bit is set if r!=0
5084 206 : tree r_or_negr = vect_recog_temp_ssa_var (itype, NULL);
5085 206 : def_stmt
5086 206 : = gimple_build_assign (r_or_negr, BIT_IOR_EXPR, r, negate_r);
5087 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5088 :
5089 : // (x ^ y) & (r | -r)
5090 : // ~(x ^ y) & (r | -r) (ceil)
5091 206 : tree r_or_negr_and_xor = vect_recog_temp_ssa_var (itype, NULL);
5092 206 : def_stmt = gimple_build_assign (r_or_negr_and_xor, BIT_AND_EXPR,
5093 : r_or_negr, cond_reg);
5094 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5095 :
5096 : // (x ^ y) & (r | -r) < 0 which is equivalent to (x^y < 0 && r!=0)
5097 206 : bool_cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5098 206 : def_stmt
5099 206 : = gimple_build_assign (bool_cond, LT_EXPR, r_or_negr_and_xor,
5100 : build_int_cst (itype, 0));
5101 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5102 : itype);
5103 :
5104 : // (x^y < 0 && r) ? y : 0 (mod)
5105 : // (x^y < 0 && r) ? -1 : 0 (div)
5106 206 : bool is_mod
5107 206 : = (rhs_code == FLOOR_MOD_EXPR || rhs_code == CEIL_MOD_EXPR);
5108 206 : tree extr_cond = vect_recog_temp_ssa_var (itype, NULL);
5109 246 : def_stmt = gimple_build_assign (extr_cond, COND_EXPR, bool_cond,
5110 : is_mod ? oprnd1
5111 40 : : build_int_cst (itype, -1),
5112 : build_int_cst (itype, 0));
5113 206 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5114 :
5115 : // r += (x ^ y < 0 && r) ? y : 0 (floor mod)
5116 : // d += (x^y < 0 && r) ? -1 : 0 (floor div)
5117 : // r -= (x ^ y < 0 && r) ? y : 0 (ceil mod)
5118 : // d -= (x^y < 0 && r) ? -1 : 0 (ceil div)
5119 206 : tree result = vect_recog_temp_ssa_var (itype, NULL);
5120 412 : return gimple_build_assign (result,
5121 206 : (rhs_code == FLOOR_MOD_EXPR
5122 206 : || rhs_code == FLOOR_DIV_EXPR)
5123 : ? PLUS_EXPR
5124 : : MINUS_EXPR,
5125 206 : is_mod ? r : q, extr_cond);
5126 : }
5127 : }
5128 36 : case ROUND_MOD_EXPR:
5129 36 : case ROUND_DIV_EXPR:
5130 36 : {
5131 36 : if (!target_has_vecop_for_code (BIT_AND_EXPR, vectype)
5132 36 : || !target_has_vecop_for_code (PLUS_EXPR, vectype)
5133 36 : || !expand_vec_cmp_expr_p (vectype, mask_vectype, LT_EXPR)
5134 36 : || !expand_vec_cmp_expr_p (vectype, mask_vectype, GT_EXPR)
5135 72 : || !expand_vec_cond_expr_p (vectype, mask_vectype))
5136 : return NULL;
5137 :
5138 36 : bool is_mod = rhs_code == ROUND_MOD_EXPR;
5139 36 : HOST_WIDE_INT d = TREE_INT_CST_LOW (oprnd1);
5140 36 : unsigned HOST_WIDE_INT abs_d
5141 : = (d >= 0 ? (unsigned HOST_WIDE_INT) d : -(unsigned HOST_WIDE_INT) d);
5142 36 : unsigned HOST_WIDE_INT mid_d = (abs_d - 1) >> 1;
5143 36 : if (!unsigned_p)
5144 : {
5145 : // check availability of abs expression for vector
5146 18 : if (!target_has_vecop_for_code (ABS_EXPR, vectype))
5147 : return NULL;
5148 : // abs (r)
5149 18 : tree abs_r = vect_recog_temp_ssa_var (itype, NULL);
5150 18 : def_stmt = gimple_build_assign (abs_r, ABS_EXPR, r);
5151 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5152 :
5153 : // abs (r) > (abs (y-1) >> 1)
5154 18 : tree round_p = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5155 18 : def_stmt = gimple_build_assign (round_p, GT_EXPR, abs_r,
5156 18 : build_int_cst (itype, mid_d));
5157 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5158 : itype);
5159 :
5160 : // x ^ y
5161 18 : tree cond_reg = vect_recog_temp_ssa_var (itype, NULL);
5162 18 : def_stmt
5163 18 : = gimple_build_assign (cond_reg, BIT_XOR_EXPR, oprnd0, oprnd1);
5164 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5165 :
5166 : // x ^ y < 0
5167 18 : bool_cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5168 18 : def_stmt = gimple_build_assign (bool_cond, LT_EXPR, cond_reg,
5169 : build_int_cst (itype, 0));
5170 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5171 : itype);
5172 :
5173 : // x ^ y < 0 ? y : -y (mod)
5174 : // x ^ y < 0 ? -1 : 1 (div)
5175 18 : tree val1 = vect_recog_temp_ssa_var (itype, NULL);
5176 18 : def_stmt
5177 36 : = gimple_build_assign (val1, COND_EXPR, bool_cond,
5178 27 : build_int_cst (itype, is_mod ? d : -1),
5179 18 : build_int_cst (itype, is_mod ? -d : 1));
5180 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5181 18 : int precision = TYPE_PRECISION (itype);
5182 18 : wide_int wmask = wi::mask (precision, false, precision);
5183 :
5184 : // abs (r) > (abs (y-1) >> 1) ? 0xffffffff : 0
5185 18 : tree val2 = vect_recog_temp_ssa_var (itype, NULL);
5186 36 : def_stmt = gimple_build_assign (val2, COND_EXPR, round_p,
5187 18 : wide_int_to_tree (itype, wmask),
5188 : build_int_cst (itype, 0));
5189 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5190 :
5191 18 : tree fval = vect_recog_temp_ssa_var (itype, NULL);
5192 18 : def_stmt = gimple_build_assign (fval, BIT_AND_EXPR, val1, val2);
5193 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5194 :
5195 18 : tree result = vect_recog_temp_ssa_var (itype, NULL);
5196 27 : return gimple_build_assign (result, PLUS_EXPR, is_mod ? r : q,
5197 : fval);
5198 18 : }
5199 : else
5200 : {
5201 : // r > (y-1 >> 1)
5202 18 : tree round_p = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5203 18 : def_stmt = gimple_build_assign (round_p, GT_EXPR, r,
5204 18 : build_int_cst (itype, mid_d));
5205 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
5206 : itype);
5207 :
5208 : // (r > (y-1)>>1) ? -d : 1
5209 18 : tree val2 = vect_recog_temp_ssa_var (itype, NULL);
5210 18 : def_stmt
5211 36 : = gimple_build_assign (val2, COND_EXPR, round_p,
5212 18 : build_int_cst (itype, is_mod ? -d : 1),
5213 : build_int_cst (itype, 0));
5214 18 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5215 :
5216 18 : tree result = vect_recog_temp_ssa_var (itype, NULL);
5217 27 : return gimple_build_assign (result, PLUS_EXPR, is_mod ? r : q,
5218 18 : val2);
5219 : }
5220 : }
5221 : default:
5222 : return NULL;
5223 : }
5224 : }
5225 :
5226 : /* Detect a signed division by a constant that wouldn't be
5227 : otherwise vectorized:
5228 :
5229 : type a_t, b_t;
5230 :
5231 : S1 a_t = b_t / N;
5232 :
5233 : where type 'type' is an integral type and N is a constant.
5234 :
5235 : Similarly handle modulo by a constant:
5236 :
5237 : S4 a_t = b_t % N;
5238 :
5239 : Input/Output:
5240 :
5241 : * STMT_VINFO: The stmt from which the pattern search begins,
5242 : i.e. the division stmt. S1 is replaced by if N is a power
5243 : of two constant and type is signed:
5244 : S3 y_t = b_t < 0 ? N - 1 : 0;
5245 : S2 x_t = b_t + y_t;
5246 : S1' a_t = x_t >> log2 (N);
5247 :
5248 : S4 is replaced if N is a power of two constant and
5249 : type is signed by (where *_T temporaries have unsigned type):
5250 : S9 y_T = b_t < 0 ? -1U : 0U;
5251 : S8 z_T = y_T >> (sizeof (type_t) * CHAR_BIT - log2 (N));
5252 : S7 z_t = (type) z_T;
5253 : S6 w_t = b_t + z_t;
5254 : S5 x_t = w_t & (N - 1);
5255 : S4' a_t = x_t - z_t;
5256 :
5257 : Output:
5258 :
5259 : * TYPE_OUT: The type of the output of this pattern.
5260 :
5261 : * Return value: A new stmt that will be used to replace the division
5262 : S1 or modulo S4 stmt. */
5263 :
5264 : static gimple *
5265 32310633 : vect_recog_divmod_pattern (vec_info *vinfo,
5266 : stmt_vec_info stmt_vinfo, tree *type_out)
5267 : {
5268 32310633 : gimple *last_stmt = stmt_vinfo->stmt;
5269 32310633 : tree oprnd0, oprnd1, vectype, itype, cond;
5270 32310633 : gimple *pattern_stmt = NULL;
5271 32310633 : gimple *def_stmt = NULL;
5272 32310633 : enum tree_code rhs_code;
5273 32310633 : optab optab;
5274 32310633 : tree q, cst;
5275 32310633 : int prec;
5276 :
5277 32310633 : if (!is_gimple_assign (last_stmt)
5278 : /* The pattern will disrupt the reduction chain with multiple uses. */
5279 32310633 : || vect_is_reduction (stmt_vinfo))
5280 : return NULL;
5281 :
5282 22408001 : rhs_code = gimple_assign_rhs_code (last_stmt);
5283 22408001 : switch (rhs_code)
5284 : {
5285 288109 : case TRUNC_DIV_EXPR:
5286 288109 : case EXACT_DIV_EXPR:
5287 288109 : case TRUNC_MOD_EXPR:
5288 288109 : case FLOOR_MOD_EXPR:
5289 288109 : case FLOOR_DIV_EXPR:
5290 288109 : case CEIL_MOD_EXPR:
5291 288109 : case CEIL_DIV_EXPR:
5292 288109 : case ROUND_MOD_EXPR:
5293 288109 : case ROUND_DIV_EXPR:
5294 288109 : break;
5295 : default:
5296 : return NULL;
5297 : }
5298 :
5299 288109 : oprnd0 = gimple_assign_rhs1 (last_stmt);
5300 288109 : oprnd1 = gimple_assign_rhs2 (last_stmt);
5301 288109 : itype = TREE_TYPE (oprnd0);
5302 288109 : if (TREE_CODE (oprnd0) != SSA_NAME
5303 269484 : || TREE_CODE (oprnd1) != INTEGER_CST
5304 160157 : || TREE_CODE (itype) != INTEGER_TYPE
5305 448266 : || !type_has_mode_precision_p (itype))
5306 : return NULL;
5307 :
5308 160145 : scalar_int_mode itype_mode = SCALAR_INT_TYPE_MODE (itype);
5309 160145 : vectype = get_vectype_for_scalar_type (vinfo, itype);
5310 160145 : if (vectype == NULL_TREE)
5311 : return NULL;
5312 :
5313 129122 : if (optimize_bb_for_size_p (gimple_bb (last_stmt)))
5314 : {
5315 : /* If the target can handle vectorized division or modulo natively,
5316 : don't attempt to optimize this, since native division is likely
5317 : to give smaller code. */
5318 1981 : optab = optab_for_tree_code (rhs_code, vectype, optab_default);
5319 1981 : if (optab != unknown_optab
5320 1981 : && can_implement_p (optab, TYPE_MODE (vectype)))
5321 : return NULL;
5322 : }
5323 :
5324 129122 : prec = TYPE_PRECISION (itype);
5325 :
5326 258244 : bool is_flclrd_moddiv_p
5327 129122 : = rhs_code == FLOOR_MOD_EXPR || rhs_code == FLOOR_DIV_EXPR
5328 : || rhs_code == CEIL_MOD_EXPR || rhs_code == CEIL_DIV_EXPR
5329 128519 : || rhs_code == ROUND_MOD_EXPR || rhs_code == ROUND_DIV_EXPR;
5330 129122 : if (integer_pow2p (oprnd1))
5331 : {
5332 76575 : if ((TYPE_UNSIGNED (itype)
5333 36 : && (rhs_code == FLOOR_MOD_EXPR || rhs_code == FLOOR_DIV_EXPR))
5334 76611 : || tree_int_cst_sgn (oprnd1) != 1)
5335 : return NULL;
5336 :
5337 : /* Pattern detected. */
5338 76575 : vect_pattern_detected ("vect_recog_divmod_pattern", last_stmt);
5339 :
5340 76575 : *type_out = vectype;
5341 :
5342 : /* Check if the target supports this internal function. */
5343 76575 : internal_fn ifn = IFN_DIV_POW2;
5344 76575 : if (!TYPE_UNSIGNED (itype)
5345 76575 : && direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
5346 : {
5347 0 : tree shift = build_int_cst (itype, tree_log2 (oprnd1));
5348 :
5349 0 : tree var_div = vect_recog_temp_ssa_var (itype, NULL);
5350 0 : gimple *div_stmt = gimple_build_call_internal (ifn, 2, oprnd0, shift);
5351 0 : gimple_call_set_lhs (div_stmt, var_div);
5352 0 : if (rhs_code == TRUNC_MOD_EXPR || is_flclrd_moddiv_p)
5353 : {
5354 0 : append_pattern_def_seq (vinfo, stmt_vinfo, div_stmt);
5355 0 : tree t1 = vect_recog_temp_ssa_var (itype, NULL);
5356 0 : def_stmt
5357 0 : = gimple_build_assign (t1, LSHIFT_EXPR, var_div, shift);
5358 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5359 0 : tree r = vect_recog_temp_ssa_var (itype, NULL);
5360 0 : pattern_stmt = gimple_build_assign (r, MINUS_EXPR, oprnd0, t1);
5361 0 : if (is_flclrd_moddiv_p)
5362 : {
5363 0 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5364 0 : pattern_stmt
5365 0 : = add_code_for_floorceilround_divmod (vectype, vinfo,
5366 : stmt_vinfo, rhs_code,
5367 : var_div, r, oprnd0,
5368 : oprnd1, itype);
5369 0 : if (pattern_stmt == NULL)
5370 : return NULL;
5371 : }
5372 : }
5373 : else
5374 : pattern_stmt = div_stmt;
5375 0 : gimple_set_location (pattern_stmt, gimple_location (last_stmt));
5376 :
5377 0 : return pattern_stmt;
5378 : }
5379 :
5380 76575 : cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5381 76575 : def_stmt = gimple_build_assign (cond, LT_EXPR, oprnd0,
5382 : build_int_cst (itype, 0));
5383 76575 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt,
5384 : truth_type_for (vectype), itype);
5385 76575 : tree div_result = NULL_TREE;
5386 76575 : if (rhs_code == TRUNC_DIV_EXPR
5387 76575 : || rhs_code == EXACT_DIV_EXPR
5388 : || rhs_code == FLOOR_DIV_EXPR
5389 2702 : || rhs_code == CEIL_DIV_EXPR
5390 2540 : || rhs_code == ROUND_DIV_EXPR)
5391 : {
5392 74047 : tree var = vect_recog_temp_ssa_var (itype, NULL);
5393 74047 : tree shift;
5394 74047 : def_stmt
5395 74047 : = gimple_build_assign (var, COND_EXPR, cond,
5396 : fold_build2 (MINUS_EXPR, itype, oprnd1,
5397 : build_int_cst (itype, 1)),
5398 : build_int_cst (itype, 0));
5399 74047 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5400 74047 : var = vect_recog_temp_ssa_var (itype, NULL);
5401 74047 : def_stmt
5402 74047 : = gimple_build_assign (var, PLUS_EXPR, oprnd0,
5403 : gimple_assign_lhs (def_stmt));
5404 74047 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5405 :
5406 74047 : shift = build_int_cst (itype, tree_log2 (oprnd1));
5407 74047 : div_result = vect_recog_temp_ssa_var (itype, NULL);
5408 74047 : pattern_stmt
5409 74047 : = gimple_build_assign (div_result, RSHIFT_EXPR, var, shift);
5410 : }
5411 76575 : if (rhs_code == TRUNC_MOD_EXPR || is_flclrd_moddiv_p)
5412 : {
5413 2702 : if (rhs_code == FLOOR_DIV_EXPR
5414 : || rhs_code == CEIL_DIV_EXPR
5415 2702 : || rhs_code == ROUND_DIV_EXPR)
5416 174 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5417 :
5418 2702 : tree signmask;
5419 2702 : if (compare_tree_int (oprnd1, 2) == 0)
5420 : {
5421 1318 : signmask = vect_recog_temp_ssa_var (itype, NULL);
5422 1318 : def_stmt = gimple_build_assign (signmask, COND_EXPR, cond,
5423 : build_int_cst (itype, 1),
5424 : build_int_cst (itype, 0));
5425 1318 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5426 : }
5427 : else
5428 : {
5429 1384 : tree utype
5430 1384 : = build_nonstandard_integer_type (prec, 1);
5431 1384 : tree vecutype = get_vectype_for_scalar_type (vinfo, utype);
5432 1384 : tree shift
5433 1384 : = build_int_cst (utype, GET_MODE_BITSIZE (itype_mode)
5434 1384 : - tree_log2 (oprnd1));
5435 1384 : tree var = vect_recog_temp_ssa_var (utype, NULL);
5436 :
5437 1384 : def_stmt = gimple_build_assign (var, COND_EXPR, cond,
5438 : build_int_cst (utype, -1),
5439 : build_int_cst (utype, 0));
5440 1384 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecutype);
5441 1384 : var = vect_recog_temp_ssa_var (utype, NULL);
5442 1384 : def_stmt = gimple_build_assign (var, RSHIFT_EXPR,
5443 : gimple_assign_lhs (def_stmt),
5444 : shift);
5445 1384 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecutype);
5446 1384 : signmask = vect_recog_temp_ssa_var (itype, NULL);
5447 1384 : def_stmt
5448 1384 : = gimple_build_assign (signmask, NOP_EXPR, var);
5449 1384 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5450 : }
5451 2702 : def_stmt
5452 2702 : = gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL),
5453 : PLUS_EXPR, oprnd0, signmask);
5454 2702 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5455 2702 : def_stmt
5456 2702 : = gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL),
5457 : BIT_AND_EXPR, gimple_assign_lhs (def_stmt),
5458 : fold_build2 (MINUS_EXPR, itype, oprnd1,
5459 : build_int_cst (itype, 1)));
5460 2702 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5461 :
5462 2702 : tree r = vect_recog_temp_ssa_var (itype, NULL);
5463 2702 : pattern_stmt
5464 2702 : = gimple_build_assign (r, MINUS_EXPR, gimple_assign_lhs (def_stmt),
5465 : signmask);
5466 2702 : if (is_flclrd_moddiv_p)
5467 : {
5468 268 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5469 268 : pattern_stmt
5470 268 : = add_code_for_floorceilround_divmod (vectype, vinfo,
5471 : stmt_vinfo, rhs_code,
5472 : div_result, r, oprnd0,
5473 : oprnd1, itype);
5474 268 : if (pattern_stmt == NULL)
5475 : return NULL;
5476 : }
5477 : }
5478 :
5479 76421 : return pattern_stmt;
5480 : }
5481 :
5482 52547 : if ((cst = uniform_integer_cst_p (oprnd1))
5483 52547 : && TYPE_UNSIGNED (itype)
5484 : && rhs_code == TRUNC_DIV_EXPR
5485 29885 : && vectype
5486 70564 : && targetm.vectorize.preferred_div_as_shifts_over_mult (vectype))
5487 : {
5488 : /* We can use the relationship:
5489 :
5490 : x // N == ((x+N+2) // (N+1) + x) // (N+1) for 0 <= x < N(N+3)
5491 :
5492 : to optimize cases where N+1 is a power of 2, and where // (N+1)
5493 : is therefore a shift right. When operating in modes that are
5494 : multiples of a byte in size, there are two cases:
5495 :
5496 : (1) N(N+3) is not representable, in which case the question
5497 : becomes whether the replacement expression overflows.
5498 : It is enough to test that x+N+2 does not overflow,
5499 : i.e. that x < MAX-(N+1).
5500 :
5501 : (2) N(N+3) is representable, in which case it is the (only)
5502 : bound that we need to check.
5503 :
5504 : ??? For now we just handle the case where // (N+1) is a shift
5505 : right by half the precision, since some architectures can
5506 : optimize the associated addition and shift combinations
5507 : into single instructions. */
5508 :
5509 12203 : auto wcst = wi::to_wide (cst);
5510 12203 : int pow = wi::exact_log2 (wcst + 1);
5511 12203 : if (pow == prec / 2)
5512 : {
5513 : /* Check that no overflow will occur. If we don't have range
5514 : information we can't perform the optimization. */
5515 :
5516 472 : int_range_max r;
5517 944 : if (get_range_query (cfun)->range_of_expr (r, oprnd0, last_stmt)
5518 472 : && !r.undefined_p ())
5519 : {
5520 472 : wide_int max = r.upper_bound ();
5521 472 : wide_int one = wi::shwi (1, prec);
5522 472 : wide_int adder = wi::add (one, wi::lshift (one, pow));
5523 472 : wi::overflow_type ovf;
5524 472 : wi::add (max, adder, UNSIGNED, &ovf);
5525 472 : if (ovf == wi::OVF_NONE)
5526 : {
5527 305 : *type_out = vectype;
5528 305 : tree tadder = wide_int_to_tree (itype, adder);
5529 305 : tree rshift = wide_int_to_tree (itype, pow);
5530 :
5531 305 : tree new_lhs1 = vect_recog_temp_ssa_var (itype, NULL);
5532 305 : gassign *patt1
5533 305 : = gimple_build_assign (new_lhs1, PLUS_EXPR, oprnd0, tadder);
5534 305 : append_pattern_def_seq (vinfo, stmt_vinfo, patt1, vectype);
5535 :
5536 305 : tree new_lhs2 = vect_recog_temp_ssa_var (itype, NULL);
5537 305 : patt1 = gimple_build_assign (new_lhs2, RSHIFT_EXPR, new_lhs1,
5538 : rshift);
5539 305 : append_pattern_def_seq (vinfo, stmt_vinfo, patt1, vectype);
5540 :
5541 305 : tree new_lhs3 = vect_recog_temp_ssa_var (itype, NULL);
5542 305 : patt1 = gimple_build_assign (new_lhs3, PLUS_EXPR, new_lhs2,
5543 : oprnd0);
5544 305 : append_pattern_def_seq (vinfo, stmt_vinfo, patt1, vectype);
5545 :
5546 305 : tree new_lhs4 = vect_recog_temp_ssa_var (itype, NULL);
5547 305 : pattern_stmt = gimple_build_assign (new_lhs4, RSHIFT_EXPR,
5548 : new_lhs3, rshift);
5549 :
5550 305 : return pattern_stmt;
5551 : }
5552 472 : }
5553 472 : }
5554 : }
5555 :
5556 52242 : if (prec > HOST_BITS_PER_WIDE_INT
5557 52242 : || integer_zerop (oprnd1))
5558 : return NULL;
5559 :
5560 51951 : if (!can_mult_highpart_p (TYPE_MODE (vectype), TYPE_UNSIGNED (itype)))
5561 : return NULL;
5562 :
5563 13733 : if (TYPE_UNSIGNED (itype))
5564 : {
5565 8911 : unsigned HOST_WIDE_INT mh, ml;
5566 8911 : int pre_shift, post_shift;
5567 8911 : unsigned HOST_WIDE_INT d = (TREE_INT_CST_LOW (oprnd1)
5568 8911 : & GET_MODE_MASK (itype_mode));
5569 8911 : tree t1, t2, t3, t4;
5570 :
5571 8911 : if (d >= (HOST_WIDE_INT_1U << (prec - 1)))
5572 : /* FIXME: Can transform this into oprnd0 >= oprnd1 ? 1 : 0. */
5573 23 : return NULL;
5574 :
5575 : /* Find a suitable multiplier and right shift count instead of
5576 : directly dividing by D. */
5577 8888 : mh = choose_multiplier (d, prec, prec, &ml, &post_shift);
5578 :
5579 : /* If the suggested multiplier is more than PREC bits, we can do better
5580 : for even divisors, using an initial right shift. */
5581 8888 : if (mh != 0 && (d & 1) == 0)
5582 : {
5583 248 : pre_shift = ctz_or_zero (d);
5584 248 : mh = choose_multiplier (d >> pre_shift, prec, prec - pre_shift,
5585 : &ml, &post_shift);
5586 248 : gcc_assert (!mh);
5587 : }
5588 : else
5589 : pre_shift = 0;
5590 :
5591 651 : if (mh != 0)
5592 : {
5593 651 : if (post_shift - 1 >= prec)
5594 : return NULL;
5595 :
5596 : /* t1 = oprnd0 h* ml;
5597 : t2 = oprnd0 - t1;
5598 : t3 = t2 >> 1;
5599 : t4 = t1 + t3;
5600 : q = t4 >> (post_shift - 1); */
5601 651 : t1 = vect_recog_temp_ssa_var (itype, NULL);
5602 651 : def_stmt = gimple_build_assign (t1, MULT_HIGHPART_EXPR, oprnd0,
5603 651 : build_int_cst (itype, ml));
5604 651 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5605 :
5606 651 : t2 = vect_recog_temp_ssa_var (itype, NULL);
5607 651 : def_stmt
5608 651 : = gimple_build_assign (t2, MINUS_EXPR, oprnd0, t1);
5609 651 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5610 :
5611 651 : t3 = vect_recog_temp_ssa_var (itype, NULL);
5612 651 : def_stmt
5613 651 : = gimple_build_assign (t3, RSHIFT_EXPR, t2, integer_one_node);
5614 651 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5615 :
5616 651 : t4 = vect_recog_temp_ssa_var (itype, NULL);
5617 651 : def_stmt
5618 651 : = gimple_build_assign (t4, PLUS_EXPR, t1, t3);
5619 :
5620 651 : if (post_shift != 1)
5621 : {
5622 651 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5623 :
5624 651 : q = vect_recog_temp_ssa_var (itype, NULL);
5625 651 : pattern_stmt
5626 651 : = gimple_build_assign (q, RSHIFT_EXPR, t4,
5627 651 : build_int_cst (itype, post_shift - 1));
5628 : }
5629 : else
5630 : {
5631 : q = t4;
5632 : pattern_stmt = def_stmt;
5633 : }
5634 : }
5635 : else
5636 : {
5637 8237 : if (pre_shift >= prec || post_shift >= prec)
5638 : return NULL;
5639 :
5640 : /* t1 = oprnd0 >> pre_shift;
5641 : t2 = t1 h* ml;
5642 : q = t2 >> post_shift; */
5643 8237 : if (pre_shift)
5644 : {
5645 248 : t1 = vect_recog_temp_ssa_var (itype, NULL);
5646 248 : def_stmt
5647 248 : = gimple_build_assign (t1, RSHIFT_EXPR, oprnd0,
5648 248 : build_int_cst (integer_type_node, pre_shift));
5649 248 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5650 : }
5651 : else
5652 : t1 = oprnd0;
5653 :
5654 8237 : t2 = vect_recog_temp_ssa_var (itype, NULL);
5655 8237 : def_stmt = gimple_build_assign (t2, MULT_HIGHPART_EXPR, t1,
5656 8237 : build_int_cst (itype, ml));
5657 :
5658 8237 : if (post_shift)
5659 : {
5660 8227 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5661 :
5662 8227 : q = vect_recog_temp_ssa_var (itype, NULL);
5663 8227 : def_stmt
5664 8227 : = gimple_build_assign (q, RSHIFT_EXPR, t2,
5665 8227 : build_int_cst (itype, post_shift));
5666 : }
5667 : else
5668 : q = t2;
5669 :
5670 : pattern_stmt = def_stmt;
5671 : }
5672 : }
5673 : else
5674 : {
5675 4822 : unsigned HOST_WIDE_INT ml;
5676 4822 : int post_shift;
5677 4822 : HOST_WIDE_INT d = TREE_INT_CST_LOW (oprnd1);
5678 4822 : unsigned HOST_WIDE_INT abs_d;
5679 4822 : bool add = false;
5680 4822 : tree t1, t2, t3, t4;
5681 :
5682 : /* Give up for -1. */
5683 4822 : if (d == -1)
5684 0 : return NULL;
5685 :
5686 : /* Since d might be INT_MIN, we have to cast to
5687 : unsigned HOST_WIDE_INT before negating to avoid
5688 : undefined signed overflow. */
5689 4822 : abs_d = (d >= 0
5690 4822 : ? (unsigned HOST_WIDE_INT) d
5691 : : - (unsigned HOST_WIDE_INT) d);
5692 :
5693 : /* n rem d = n rem -d */
5694 4822 : if (rhs_code == TRUNC_MOD_EXPR && d < 0)
5695 : {
5696 0 : d = abs_d;
5697 0 : oprnd1 = build_int_cst (itype, abs_d);
5698 : }
5699 4822 : if (HOST_BITS_PER_WIDE_INT >= prec
5700 4822 : && abs_d == HOST_WIDE_INT_1U << (prec - 1))
5701 : /* This case is not handled correctly below. */
5702 : return NULL;
5703 :
5704 4822 : choose_multiplier (abs_d, prec, prec - 1, &ml, &post_shift);
5705 4822 : if (ml >= HOST_WIDE_INT_1U << (prec - 1))
5706 : {
5707 1619 : add = true;
5708 1619 : ml |= HOST_WIDE_INT_M1U << (prec - 1);
5709 : }
5710 4822 : if (post_shift >= prec)
5711 : return NULL;
5712 :
5713 : /* t1 = oprnd0 h* ml; */
5714 4822 : t1 = vect_recog_temp_ssa_var (itype, NULL);
5715 4822 : def_stmt = gimple_build_assign (t1, MULT_HIGHPART_EXPR, oprnd0,
5716 4822 : build_int_cst (itype, ml));
5717 :
5718 4822 : if (add)
5719 : {
5720 : /* t2 = t1 + oprnd0; */
5721 1619 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5722 1619 : t2 = vect_recog_temp_ssa_var (itype, NULL);
5723 1619 : def_stmt = gimple_build_assign (t2, PLUS_EXPR, t1, oprnd0);
5724 : }
5725 : else
5726 : t2 = t1;
5727 :
5728 4822 : if (post_shift)
5729 : {
5730 : /* t3 = t2 >> post_shift; */
5731 4009 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5732 4009 : t3 = vect_recog_temp_ssa_var (itype, NULL);
5733 4009 : def_stmt = gimple_build_assign (t3, RSHIFT_EXPR, t2,
5734 4009 : build_int_cst (itype, post_shift));
5735 : }
5736 : else
5737 : t3 = t2;
5738 :
5739 4822 : int msb = 1;
5740 4822 : int_range_max r;
5741 9644 : get_range_query (cfun)->range_of_expr (r, oprnd0);
5742 4822 : if (!r.varying_p () && !r.undefined_p ())
5743 : {
5744 2975 : if (!wi::neg_p (r.lower_bound (), TYPE_SIGN (itype)))
5745 : msb = 0;
5746 743 : else if (wi::neg_p (r.upper_bound (), TYPE_SIGN (itype)))
5747 : msb = -1;
5748 : }
5749 :
5750 2232 : if (msb == 0 && d >= 0)
5751 : {
5752 : /* q = t3; */
5753 : q = t3;
5754 : pattern_stmt = def_stmt;
5755 : }
5756 : else
5757 : {
5758 : /* t4 = oprnd0 >> (prec - 1);
5759 : or if we know from VRP that oprnd0 >= 0
5760 : t4 = 0;
5761 : or if we know from VRP that oprnd0 < 0
5762 : t4 = -1; */
5763 2650 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5764 2650 : t4 = vect_recog_temp_ssa_var (itype, NULL);
5765 2650 : if (msb != 1)
5766 68 : def_stmt = gimple_build_assign (t4, INTEGER_CST,
5767 68 : build_int_cst (itype, msb));
5768 : else
5769 2582 : def_stmt = gimple_build_assign (t4, RSHIFT_EXPR, oprnd0,
5770 2582 : build_int_cst (itype, prec - 1));
5771 2650 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5772 :
5773 : /* q = t3 - t4; or q = t4 - t3; */
5774 2650 : q = vect_recog_temp_ssa_var (itype, NULL);
5775 5116 : pattern_stmt = gimple_build_assign (q, MINUS_EXPR, d < 0 ? t4 : t3,
5776 : d < 0 ? t3 : t4);
5777 : }
5778 4822 : }
5779 :
5780 13710 : if (rhs_code == TRUNC_MOD_EXPR || is_flclrd_moddiv_p)
5781 : {
5782 6213 : tree r, t1;
5783 :
5784 : /* We divided. Now finish by:
5785 : t1 = q * oprnd1;
5786 : r = oprnd0 - t1; */
5787 6213 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5788 :
5789 6213 : t1 = vect_recog_temp_ssa_var (itype, NULL);
5790 6213 : def_stmt = gimple_build_assign (t1, MULT_EXPR, q, oprnd1);
5791 6213 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
5792 :
5793 6213 : r = vect_recog_temp_ssa_var (itype, NULL);
5794 6213 : pattern_stmt = gimple_build_assign (r, MINUS_EXPR, oprnd0, t1);
5795 :
5796 6213 : if (is_flclrd_moddiv_p)
5797 : {
5798 146 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
5799 146 : pattern_stmt
5800 146 : = add_code_for_floorceilround_divmod (vectype, vinfo, stmt_vinfo,
5801 : rhs_code, q, r, oprnd0, oprnd1,
5802 : itype);
5803 146 : if (pattern_stmt == NULL)
5804 : return NULL;
5805 : }
5806 : }
5807 :
5808 : /* Pattern detected. */
5809 13710 : vect_pattern_detected ("vect_recog_divmod_pattern", last_stmt);
5810 :
5811 13710 : *type_out = vectype;
5812 13710 : return pattern_stmt;
5813 : }
5814 :
5815 : /* Detects pattern with a modulo operation (S1) where both arguments
5816 : are variables of integral type.
5817 : The statement is replaced by division, multiplication, and subtraction.
5818 : The last statement (S4) is returned.
5819 :
5820 : Example:
5821 : S1 c_t = a_t % b_t;
5822 :
5823 : is replaced by
5824 : S2 x_t = a_t / b_t;
5825 : S3 y_t = x_t * b_t;
5826 : S4 z_t = a_t - y_t; */
5827 :
5828 : static gimple *
5829 32494956 : vect_recog_mod_var_pattern (vec_info *vinfo,
5830 : stmt_vec_info stmt_vinfo, tree *type_out)
5831 : {
5832 32494956 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
5833 32494956 : tree oprnd0, oprnd1, vectype, itype;
5834 32494956 : gimple *pattern_stmt, *def_stmt;
5835 32494956 : enum tree_code rhs_code;
5836 :
5837 32494956 : if (!is_gimple_assign (last_stmt) || vect_is_reduction (stmt_vinfo))
5838 : return NULL;
5839 :
5840 22592324 : rhs_code = gimple_assign_rhs_code (last_stmt);
5841 22592324 : if (rhs_code != TRUNC_MOD_EXPR)
5842 : return NULL;
5843 :
5844 78375 : oprnd0 = gimple_assign_rhs1 (last_stmt);
5845 78375 : oprnd1 = gimple_assign_rhs2 (last_stmt);
5846 78375 : itype = TREE_TYPE (oprnd0);
5847 78375 : if (TREE_CODE (oprnd0) != SSA_NAME
5848 69532 : || TREE_CODE (oprnd1) != SSA_NAME
5849 52905 : || TREE_CODE (itype) != INTEGER_TYPE)
5850 : return NULL;
5851 :
5852 52802 : vectype = get_vectype_for_scalar_type (vinfo, itype);
5853 :
5854 52802 : if (!vectype
5855 44227 : || target_has_vecop_for_code (TRUNC_MOD_EXPR, vectype)
5856 44227 : || !target_has_vecop_for_code (TRUNC_DIV_EXPR, vectype)
5857 0 : || !target_has_vecop_for_code (MULT_EXPR, vectype)
5858 52802 : || !target_has_vecop_for_code (MINUS_EXPR, vectype))
5859 : return NULL;
5860 :
5861 0 : tree q, tmp, r;
5862 0 : q = vect_recog_temp_ssa_var (itype, NULL);
5863 0 : def_stmt = gimple_build_assign (q, TRUNC_DIV_EXPR, oprnd0, oprnd1);
5864 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vectype);
5865 :
5866 0 : tmp = vect_recog_temp_ssa_var (itype, NULL);
5867 0 : def_stmt = gimple_build_assign (tmp, MULT_EXPR, q, oprnd1);
5868 0 : append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vectype);
5869 :
5870 0 : r = vect_recog_temp_ssa_var (itype, NULL);
5871 0 : pattern_stmt = gimple_build_assign (r, MINUS_EXPR, oprnd0, tmp);
5872 :
5873 : /* Pattern detected. */
5874 0 : *type_out = vectype;
5875 0 : vect_pattern_detected ("vect_recog_mod_var_pattern", last_stmt);
5876 :
5877 0 : return pattern_stmt;
5878 : }
5879 :
5880 :
5881 : /* Return the proper type for converting bool VAR into
5882 : an integer value or NULL_TREE if no such type exists.
5883 : The type is chosen so that the converted value has the
5884 : same number of elements as VAR's vector type. */
5885 :
5886 : static tree
5887 4793681 : integer_type_for_mask (tree var, vec_info *vinfo, vect_def_type *dt = nullptr)
5888 : {
5889 4793681 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (var)))
5890 : return NULL_TREE;
5891 :
5892 2206509 : stmt_vec_info def_stmt_info = vinfo->lookup_def (var);
5893 2206509 : if (dt)
5894 : {
5895 402628 : if (!def_stmt_info)
5896 3488 : *dt = vect_external_def;
5897 : else
5898 399140 : *dt = STMT_VINFO_DEF_TYPE (def_stmt_info);
5899 : }
5900 402628 : if (!def_stmt_info
5901 2110837 : || STMT_VINFO_DEF_TYPE (def_stmt_info) == vect_external_def
5902 3914718 : || !vect_use_mask_type_p (def_stmt_info))
5903 : return NULL_TREE;
5904 :
5905 1365812 : return build_nonstandard_integer_type (def_stmt_info->mask_precision, 1);
5906 : }
5907 :
5908 : /* Function vect_recog_gcond_pattern
5909 :
5910 : Try to find pattern like following:
5911 :
5912 : if (a op b)
5913 :
5914 : where operator 'op' is not != and convert it to an adjusted boolean pattern
5915 :
5916 : mask = a op b
5917 : if (mask != 0)
5918 :
5919 : and set the mask type on MASK.
5920 :
5921 : Input:
5922 :
5923 : * STMT_VINFO: The stmt at the end from which the pattern
5924 : search begins, i.e. cast of a bool to
5925 : an integer type.
5926 :
5927 : Output:
5928 :
5929 : * TYPE_OUT: The type of the output of this pattern.
5930 :
5931 : * Return value: A new stmt that will be used to replace the pattern. */
5932 :
5933 : static gimple *
5934 32567399 : vect_recog_gcond_pattern (vec_info *vinfo,
5935 : stmt_vec_info stmt_vinfo, tree *type_out)
5936 : {
5937 : /* Currently we only support this for loop vectorization and when multiple
5938 : exits. */
5939 32567399 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
5940 4599984 : if (!loop_vinfo || !LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
5941 : return NULL;
5942 :
5943 1708168 : gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
5944 1708168 : gcond* cond = NULL;
5945 1708168 : if (!(cond = dyn_cast <gcond *> (last_stmt)))
5946 : return NULL;
5947 :
5948 400261 : auto lhs = gimple_cond_lhs (cond);
5949 400261 : auto rhs = gimple_cond_rhs (cond);
5950 400261 : auto code = gimple_cond_code (cond);
5951 :
5952 400261 : tree scalar_type = TREE_TYPE (lhs);
5953 400261 : if (VECTOR_TYPE_P (scalar_type))
5954 : return NULL;
5955 :
5956 : /* If the input is a boolean then try to figure out the precision that the
5957 : vector type should use. We cannot use the scalar precision as this would
5958 : later mismatch. This is similar to what recog_bool does. */
5959 400261 : if (VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type))
5960 : {
5961 10420 : if (tree stype = integer_type_for_mask (lhs, vinfo))
5962 400261 : scalar_type = stype;
5963 : }
5964 :
5965 400261 : tree vectype = get_mask_type_for_scalar_type (vinfo, scalar_type);
5966 400261 : if (vectype == NULL_TREE)
5967 : return NULL;
5968 :
5969 380012 : tree new_lhs = vect_recog_temp_ssa_var (boolean_type_node, NULL);
5970 380012 : gimple *new_stmt = gimple_build_assign (new_lhs, code, lhs, rhs);
5971 380012 : append_pattern_def_seq (vinfo, stmt_vinfo, new_stmt, vectype, scalar_type);
5972 :
5973 380012 : gimple *pattern_stmt
5974 380012 : = gimple_build_cond (NE_EXPR, new_lhs,
5975 380012 : build_int_cst (TREE_TYPE (new_lhs), 0),
5976 : NULL_TREE, NULL_TREE);
5977 380012 : *type_out = vectype;
5978 380012 : vect_pattern_detected ("vect_recog_gcond_pattern", last_stmt);
5979 380012 : return pattern_stmt;
5980 : }
5981 :
5982 :
5983 : /* A helper for vect_recog_mask_conversion_pattern. Build
5984 : conversion of MASK to a type suitable for masking VECTYPE.
5985 : Built statement gets required vectype and is appended to
5986 : a pattern sequence of STMT_VINFO.
5987 : If VECTYPE is a mask type, SCALAR_TYPE_FOR_MASK is the scalar type
5988 : from which it was derived.
5989 :
5990 : Return converted mask. */
5991 :
5992 : static tree
5993 145693 : build_mask_conversion (vec_info *vinfo,
5994 : tree mask, tree vectype, stmt_vec_info stmt_vinfo,
5995 : tree scalar_type_for_mask = NULL_TREE)
5996 : {
5997 145693 : gimple *stmt;
5998 145693 : tree masktype, tmp;
5999 :
6000 235229 : gcc_assert (!scalar_type_for_mask == !VECTOR_BOOLEAN_TYPE_P (vectype));
6001 :
6002 145693 : masktype = truth_type_for (vectype);
6003 145693 : tmp = vect_recog_temp_ssa_var (TREE_TYPE (masktype), NULL);
6004 145693 : stmt = gimple_build_assign (tmp, CONVERT_EXPR, mask);
6005 235229 : append_pattern_def_seq (vinfo, stmt_vinfo,
6006 : stmt, masktype,
6007 : scalar_type_for_mask
6008 89536 : ? scalar_type_for_mask : TREE_TYPE (vectype));
6009 :
6010 145693 : return tmp;
6011 : }
6012 :
6013 :
6014 : /* Return MASK if MASK is suitable for masking an operation on vectors
6015 : of type VECTYPE, otherwise convert it into such a form and return
6016 : the result. Associate any conversion statements with STMT_INFO's
6017 : pattern. If VECTYPE is a mask type, SCALAR_TYPE_FOR_MASK is the scalar
6018 : type from which it was derived. */
6019 :
6020 : static tree
6021 79250 : vect_convert_mask_for_vectype (tree mask, tree vectype,
6022 : stmt_vec_info stmt_info, vec_info *vinfo,
6023 : tree scalar_type_for_mask = NULL_TREE)
6024 : {
6025 79250 : tree mask_type = integer_type_for_mask (mask, vinfo);
6026 79250 : if (mask_type)
6027 : {
6028 79250 : tree mask_vectype = get_mask_type_for_scalar_type (vinfo, mask_type);
6029 79250 : if (mask_vectype
6030 158500 : && maybe_ne (TYPE_VECTOR_SUBPARTS (vectype),
6031 96474 : TYPE_VECTOR_SUBPARTS (mask_vectype)))
6032 62026 : mask = build_mask_conversion (vinfo, mask, vectype, stmt_info,
6033 : scalar_type_for_mask);
6034 : }
6035 79250 : return mask;
6036 : }
6037 :
6038 :
6039 : /* Function vect_recog_bool_pattern
6040 :
6041 : Try to find pattern like following:
6042 :
6043 : bool a_b, b_b, c_b, d_b, e_b;
6044 : TYPE f_T;
6045 : loop:
6046 : S1 a_b = x1 CMP1 y1;
6047 : S2 b_b = x2 CMP2 y2;
6048 : S3 c_b = a_b & b_b;
6049 : S4 d_b = x3 CMP3 y3;
6050 : S5 e_b = c_b | d_b;
6051 : S6 f_T = (TYPE) e_b;
6052 :
6053 : where type 'TYPE' is an integral type. Or a similar pattern
6054 : ending in
6055 :
6056 : S6 f_Y = e_b ? r_Y : s_Y;
6057 :
6058 : as results from if-conversion of a complex condition.
6059 :
6060 : Input:
6061 :
6062 : * STMT_VINFO: The stmt at the end from which the pattern
6063 : search begins, i.e. cast of a bool to
6064 : an integer type.
6065 :
6066 : Output:
6067 :
6068 : * TYPE_OUT: The type of the output of this pattern.
6069 :
6070 : * Return value: A new stmt that will be used to replace the pattern.
6071 :
6072 : Assuming size of TYPE is the same as size of all comparisons
6073 : (otherwise some casts would be added where needed), the above
6074 : sequence we create related pattern stmts:
6075 : S1' a_T = x1 CMP1 y1 ? 1 : 0;
6076 : S3' c_T = x2 CMP2 y2 ? a_T : 0;
6077 : S4' d_T = x3 CMP3 y3 ? 1 : 0;
6078 : S5' e_T = c_T | d_T;
6079 : S6' f_T = e_T;
6080 :
6081 : Instead of the above S3' we could emit:
6082 : S2' b_T = x2 CMP2 y2 ? 1 : 0;
6083 : S3' c_T = a_T | b_T;
6084 : but the above is more efficient. */
6085 :
6086 : static gimple *
6087 32567399 : vect_recog_bool_pattern (vec_info *vinfo,
6088 : stmt_vec_info stmt_vinfo, tree *type_out)
6089 : {
6090 32567399 : gimple *last_stmt = stmt_vinfo->stmt;
6091 32567399 : enum tree_code rhs_code;
6092 32567399 : tree var, lhs, rhs, vectype;
6093 32567399 : gimple *pattern_stmt;
6094 :
6095 32567399 : if (!is_gimple_assign (last_stmt))
6096 : return NULL;
6097 :
6098 23157929 : var = gimple_assign_rhs1 (last_stmt);
6099 23157929 : lhs = gimple_assign_lhs (last_stmt);
6100 23157929 : rhs_code = gimple_assign_rhs_code (last_stmt);
6101 :
6102 23157929 : if (rhs_code == VIEW_CONVERT_EXPR)
6103 194692 : var = TREE_OPERAND (var, 0);
6104 :
6105 23157929 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (var)))
6106 : return NULL;
6107 :
6108 775161 : hash_set<gimple *> bool_stmts;
6109 :
6110 775161 : if (CONVERT_EXPR_CODE_P (rhs_code)
6111 : || rhs_code == VIEW_CONVERT_EXPR
6112 : || rhs_code == FLOAT_EXPR)
6113 : {
6114 165464 : if (! (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
6115 2411 : || SCALAR_FLOAT_TYPE_P (TREE_TYPE (lhs)))
6116 163587 : || VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)))
6117 : return NULL;
6118 76273 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6119 :
6120 76273 : tree type = integer_type_for_mask (var, vinfo);
6121 76273 : tree cst0, cst1, tmp;
6122 :
6123 76273 : if (!type)
6124 : return NULL;
6125 :
6126 : /* We may directly use cond with narrowed type to avoid multiple cond
6127 : exprs with following result packing and perform single cond with
6128 : packed mask instead. In case of widening we better make cond first
6129 : and then extract results. */
6130 37079 : if (TYPE_MODE (type) == TYPE_MODE (TREE_TYPE (lhs)))
6131 24157 : type = TREE_TYPE (lhs);
6132 :
6133 37079 : cst0 = build_int_cst (type, 0);
6134 37079 : cst1 = build_int_cst (type, 1);
6135 37079 : tmp = vect_recog_temp_ssa_var (type, NULL);
6136 37079 : pattern_stmt = gimple_build_assign (tmp, COND_EXPR, var, cst1, cst0);
6137 :
6138 37079 : if (!useless_type_conversion_p (type, TREE_TYPE (lhs)))
6139 : {
6140 12922 : tree new_vectype = get_vectype_for_scalar_type (vinfo, type);
6141 12922 : append_pattern_def_seq (vinfo, stmt_vinfo,
6142 : pattern_stmt, new_vectype);
6143 :
6144 12922 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6145 12922 : pattern_stmt
6146 25536 : = gimple_build_assign (lhs, (rhs_code == FLOAT_EXPR
6147 : ? FLOAT_EXPR : CONVERT_EXPR), tmp);
6148 : }
6149 :
6150 37079 : *type_out = vectype;
6151 37079 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6152 :
6153 37079 : return pattern_stmt;
6154 : }
6155 : else if (rhs_code == COND_EXPR
6156 211438 : && TREE_CODE (var) == SSA_NAME)
6157 : {
6158 211438 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6159 211438 : if (vectype == NULL_TREE)
6160 : return NULL;
6161 :
6162 : /* Build a scalar type for the boolean result that when
6163 : vectorized matches the vector type of the result in
6164 : size and number of elements. */
6165 196079 : unsigned prec
6166 196079 : = vector_element_size (tree_to_poly_uint64 (TYPE_SIZE (vectype)),
6167 : TYPE_VECTOR_SUBPARTS (vectype));
6168 :
6169 196079 : tree type
6170 392158 : = build_nonstandard_integer_type (prec,
6171 196079 : TYPE_UNSIGNED (TREE_TYPE (var)));
6172 196079 : if (get_vectype_for_scalar_type (vinfo, type) == NULL_TREE)
6173 : return NULL;
6174 :
6175 196079 : enum vect_def_type dt;
6176 196079 : if (integer_type_for_mask (var, vinfo))
6177 : return NULL;
6178 36746 : else if (TREE_CODE (TREE_TYPE (var)) == BOOLEAN_TYPE
6179 36746 : && vect_is_simple_use (var, vinfo, &dt)
6180 36746 : && (dt == vect_external_def
6181 36739 : || dt == vect_constant_def))
6182 : {
6183 : /* If the condition is already a boolean then manually convert it to a
6184 : mask of the given integer type but don't set a vectype. */
6185 1383 : tree lhs_ivar = vect_recog_temp_ssa_var (type, NULL);
6186 1383 : pattern_stmt = gimple_build_assign (lhs_ivar, COND_EXPR, var,
6187 : build_all_ones_cst (type),
6188 : build_zero_cst (type));
6189 1383 : append_inv_pattern_def_seq (vinfo, pattern_stmt);
6190 1383 : var = lhs_ivar;
6191 : }
6192 :
6193 36746 : tree lhs_var = vect_recog_temp_ssa_var (boolean_type_node, NULL);
6194 36746 : pattern_stmt = gimple_build_assign (lhs_var, NE_EXPR, var,
6195 36746 : build_zero_cst (TREE_TYPE (var)));
6196 :
6197 36746 : tree new_vectype = get_mask_type_for_scalar_type (vinfo, TREE_TYPE (var));
6198 36746 : if (!new_vectype)
6199 : return NULL;
6200 :
6201 36746 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, new_vectype,
6202 36746 : TREE_TYPE (var));
6203 :
6204 36746 : lhs_var = vect_convert_mask_for_vectype (lhs_var, vectype, stmt_vinfo,
6205 : vinfo);
6206 :
6207 36746 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6208 36746 : pattern_stmt
6209 36746 : = gimple_build_assign (lhs, COND_EXPR, lhs_var,
6210 : gimple_assign_rhs2 (last_stmt),
6211 : gimple_assign_rhs3 (last_stmt));
6212 36746 : *type_out = vectype;
6213 36746 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6214 :
6215 36746 : return pattern_stmt;
6216 : }
6217 480616 : else if (rhs_code == BIT_NOT_EXPR
6218 480616 : && !vect_use_mask_type_p (stmt_vinfo))
6219 : {
6220 : /* When we have a bool data inversion rewrite that to an XOR to
6221 : cope with the fact that we'll use a wider vector element type. */
6222 9874 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6223 9874 : pattern_stmt
6224 9874 : = gimple_build_assign (lhs, BIT_XOR_EXPR, var,
6225 9874 : build_all_ones_cst (TREE_TYPE (var)));
6226 9874 : *type_out = NULL_TREE;
6227 9874 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6228 :
6229 9874 : return pattern_stmt;
6230 : }
6231 470742 : else if ((rhs_code == BIT_XOR_EXPR
6232 : || rhs_code == BIT_AND_EXPR
6233 470742 : || rhs_code == BIT_IOR_EXPR)
6234 376631 : && TREE_CODE (var) == SSA_NAME)
6235 : {
6236 376631 : tree rhs2 = gimple_assign_rhs2 (last_stmt);
6237 376631 : if (TREE_CODE (rhs2) != SSA_NAME)
6238 : return NULL;
6239 376631 : tree lhs_type = integer_type_for_mask (lhs, vinfo);
6240 376631 : if (!lhs_type)
6241 : return NULL;
6242 201314 : vectype = get_mask_type_for_scalar_type (vinfo, lhs_type);
6243 201314 : if (!vectype)
6244 : return NULL;
6245 201314 : vect_def_type dt1, dt2;
6246 201314 : tree rhs1_type = integer_type_for_mask (var, vinfo, &dt1);
6247 201314 : tree rhs2_type = integer_type_for_mask (rhs2, vinfo, &dt2);
6248 201314 : if ((rhs1_type || dt1 == vect_external_def)
6249 185877 : && (rhs2_type || dt2 == vect_external_def))
6250 : return NULL;
6251 : /* When one input is a mask and the other is not create a pattern
6252 : stmt sequence that creates a mask for the non-mask input and
6253 : convert it to one suitable for the output mask used. */
6254 38071 : if (rhs1_type && !rhs2_type)
6255 : {
6256 22634 : tree rhs1_vectype = get_mask_type_for_scalar_type (vinfo, rhs1_type);
6257 22634 : if (!rhs1_vectype)
6258 : return NULL;
6259 22634 : tree rhs2_vectype = get_vectype_for_scalar_type (vinfo,
6260 22634 : TREE_TYPE (rhs2));
6261 22634 : if (!rhs2_vectype)
6262 : return NULL;
6263 22634 : tree new_vectype = truth_type_for (rhs2_vectype);
6264 22634 : tree tem = vect_recog_temp_ssa_var (TREE_TYPE (new_vectype), NULL);
6265 22634 : pattern_stmt = gimple_build_assign (tem, NE_EXPR, rhs2,
6266 : build_zero_cst
6267 22634 : (TREE_TYPE (rhs2)));
6268 22634 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
6269 22634 : new_vectype, TREE_TYPE (new_vectype));
6270 22634 : rhs2 = vect_convert_mask_for_vectype (tem, rhs1_vectype,
6271 : stmt_vinfo, vinfo, rhs1_type);
6272 : }
6273 15437 : else if (!rhs1_type && rhs2_type)
6274 : {
6275 15437 : tree rhs2_vectype = get_mask_type_for_scalar_type (vinfo, rhs2_type);
6276 15437 : if (!rhs2_vectype)
6277 : return NULL;
6278 15437 : tree rhs1_vectype = get_vectype_for_scalar_type (vinfo,
6279 15437 : TREE_TYPE (var));
6280 15437 : if (!rhs1_vectype)
6281 : return NULL;
6282 15437 : tree new_vectype = truth_type_for (rhs1_vectype);
6283 15437 : tree tem = vect_recog_temp_ssa_var (TREE_TYPE (new_vectype), NULL);
6284 15437 : pattern_stmt = gimple_build_assign (tem, NE_EXPR, var,
6285 : build_zero_cst
6286 15437 : (TREE_TYPE (var)));
6287 15437 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
6288 15437 : new_vectype, TREE_TYPE (new_vectype));
6289 15437 : var = vect_convert_mask_for_vectype (tem, rhs2_vectype,
6290 : stmt_vinfo, vinfo, rhs2_type);
6291 : }
6292 38071 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6293 38071 : pattern_stmt = gimple_build_assign (lhs, rhs_code, var, rhs2);
6294 38071 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6295 38071 : *type_out = vectype;
6296 38071 : return pattern_stmt;
6297 : }
6298 94111 : else if (rhs_code == SSA_NAME
6299 25705 : && STMT_VINFO_DATA_REF (stmt_vinfo))
6300 : {
6301 7807 : stmt_vec_info pattern_stmt_info;
6302 7807 : vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6303 7807 : if (!vectype || !VECTOR_MODE_P (TYPE_MODE (vectype)))
6304 : return NULL;
6305 :
6306 7807 : tree type = integer_type_for_mask (var, vinfo);
6307 7807 : if (!type)
6308 : return NULL;
6309 :
6310 4433 : var = vect_convert_mask_for_vectype (var, vectype, stmt_vinfo, vinfo);
6311 :
6312 4433 : tree cst0 = build_int_cst (TREE_TYPE (vectype), 0);
6313 4433 : tree cst1 = build_int_cst (TREE_TYPE (vectype), 1);
6314 4433 : rhs = vect_recog_temp_ssa_var (TREE_TYPE (vectype), NULL);
6315 4433 : pattern_stmt = gimple_build_assign (rhs, COND_EXPR, var, cst1, cst0);
6316 4433 : append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vectype);
6317 :
6318 4433 : lhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (vectype), lhs);
6319 4433 : pattern_stmt = gimple_build_assign (lhs, SSA_NAME, rhs);
6320 4433 : pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
6321 4433 : vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
6322 4433 : *type_out = vectype;
6323 4433 : vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
6324 :
6325 4433 : return pattern_stmt;
6326 : }
6327 : else
6328 : return NULL;
6329 775161 : }
6330 :
6331 :
6332 : /* Function vect_recog_mask_conversion_pattern
6333 :
6334 : Try to find statements which require boolean type
6335 : conversion. Additional conversion statements are
6336 : added to handle such cases. For example:
6337 :
6338 : bool m_1, m_2, m_3;
6339 : int i_4, i_5;
6340 : double d_6, d_7;
6341 : char c_1, c_2, c_3;
6342 :
6343 : S1 m_1 = i_4 > i_5;
6344 : S2 m_2 = d_6 < d_7;
6345 : S3 m_3 = m_1 & m_2;
6346 : S4 c_1 = m_3 ? c_2 : c_3;
6347 :
6348 : Will be transformed into:
6349 :
6350 : S1 m_1 = i_4 > i_5;
6351 : S2 m_2 = d_6 < d_7;
6352 : S3'' m_2' = (_Bool[bitsize=32])m_2
6353 : S3' m_3' = m_1 & m_2';
6354 : S4'' m_3'' = (_Bool[bitsize=8])m_3'
6355 : S4' c_1' = m_3'' ? c_2 : c_3; */
6356 :
6357 : static gimple *
6358 32595530 : vect_recog_mask_conversion_pattern (vec_info *vinfo,
6359 : stmt_vec_info stmt_vinfo, tree *type_out)
6360 : {
6361 32595530 : gimple *last_stmt = stmt_vinfo->stmt;
6362 32595530 : enum tree_code rhs_code;
6363 32595530 : tree lhs = NULL_TREE, rhs1, rhs2, tmp, rhs1_type, rhs2_type;
6364 32595530 : tree vectype1, vectype2;
6365 32595530 : stmt_vec_info pattern_stmt_info;
6366 :
6367 : /* Check for MASK_LOAD and MASK_STORE as well as COND_OP calls requiring mask
6368 : conversion. */
6369 32595530 : if (is_gimple_call (last_stmt)
6370 32595530 : && gimple_call_internal_p (last_stmt))
6371 : {
6372 127690 : gcall *pattern_stmt;
6373 :
6374 127690 : internal_fn ifn = gimple_call_internal_fn (last_stmt);
6375 127690 : int mask_argno = internal_fn_mask_index (ifn);
6376 127690 : if (mask_argno < 0)
6377 : return NULL;
6378 :
6379 14600 : bool store_p = internal_store_fn_p (ifn);
6380 14600 : bool load_p = internal_store_fn_p (ifn);
6381 14600 : if (store_p)
6382 : {
6383 2488 : int rhs_index = internal_fn_stored_value_index (ifn);
6384 2488 : tree rhs = gimple_call_arg (last_stmt, rhs_index);
6385 2488 : vectype1 = get_vectype_for_scalar_type (vinfo, TREE_TYPE (rhs));
6386 : }
6387 : else
6388 : {
6389 12112 : lhs = gimple_call_lhs (last_stmt);
6390 12112 : if (!lhs)
6391 : return NULL;
6392 12112 : vectype1 = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6393 : }
6394 :
6395 14600 : if (!vectype1)
6396 : return NULL;
6397 :
6398 14280 : tree mask_arg = gimple_call_arg (last_stmt, mask_argno);
6399 14280 : tree mask_arg_type = integer_type_for_mask (mask_arg, vinfo);
6400 14280 : if (mask_arg_type)
6401 : {
6402 12653 : vectype2 = get_mask_type_for_scalar_type (vinfo, mask_arg_type);
6403 :
6404 12653 : if (!vectype2
6405 12653 : || known_eq (TYPE_VECTOR_SUBPARTS (vectype1),
6406 : TYPE_VECTOR_SUBPARTS (vectype2)))
6407 : return NULL;
6408 : }
6409 1627 : else if (store_p || load_p)
6410 : return NULL;
6411 :
6412 6101 : tmp = build_mask_conversion (vinfo, mask_arg, vectype1, stmt_vinfo);
6413 :
6414 6101 : auto_vec<tree, 8> args;
6415 6101 : unsigned int nargs = gimple_call_num_args (last_stmt);
6416 6101 : args.safe_grow (nargs, true);
6417 36606 : for (unsigned int i = 0; i < nargs; ++i)
6418 24404 : args[i] = ((int) i == mask_argno
6419 24404 : ? tmp
6420 18303 : : gimple_call_arg (last_stmt, i));
6421 6101 : pattern_stmt = gimple_build_call_internal_vec (ifn, args);
6422 :
6423 6101 : if (!store_p)
6424 : {
6425 5829 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6426 5829 : gimple_call_set_lhs (pattern_stmt, lhs);
6427 : }
6428 :
6429 5829 : if (load_p || store_p)
6430 272 : gimple_call_set_nothrow (pattern_stmt, true);
6431 :
6432 6101 : pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
6433 6101 : if (STMT_VINFO_DATA_REF (stmt_vinfo))
6434 1804 : vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
6435 :
6436 6101 : *type_out = vectype1;
6437 6101 : vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
6438 :
6439 6101 : return pattern_stmt;
6440 6101 : }
6441 :
6442 32467840 : if (!is_gimple_assign (last_stmt))
6443 : return NULL;
6444 :
6445 23186060 : gimple *pattern_stmt;
6446 23186060 : lhs = gimple_assign_lhs (last_stmt);
6447 23186060 : rhs1 = gimple_assign_rhs1 (last_stmt);
6448 23186060 : rhs_code = gimple_assign_rhs_code (last_stmt);
6449 :
6450 : /* Check for cond expression requiring mask conversion. */
6451 23186060 : if (rhs_code == COND_EXPR)
6452 : {
6453 192183 : vectype1 = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
6454 :
6455 192183 : gcc_assert (! COMPARISON_CLASS_P (rhs1));
6456 192183 : if (TREE_CODE (rhs1) == SSA_NAME)
6457 : {
6458 192183 : rhs1_type = integer_type_for_mask (rhs1, vinfo);
6459 192183 : if (!rhs1_type)
6460 : return NULL;
6461 : }
6462 : else
6463 : return NULL;
6464 :
6465 178979 : vectype2 = get_mask_type_for_scalar_type (vinfo, rhs1_type);
6466 :
6467 178979 : if (!vectype1 || !vectype2)
6468 : return NULL;
6469 :
6470 : /* Continue if a conversion is needed. Also continue if we have
6471 : a comparison whose vector type would normally be different from
6472 : VECTYPE2 when considered in isolation. In that case we'll
6473 : replace the comparison with an SSA name (so that we can record
6474 : its vector type) and behave as though the comparison was an SSA
6475 : name from the outset. */
6476 176824 : if (known_eq (TYPE_VECTOR_SUBPARTS (vectype1),
6477 : TYPE_VECTOR_SUBPARTS (vectype2)))
6478 : return NULL;
6479 :
6480 45736 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1),
6481 91472 : TYPE_VECTOR_SUBPARTS (vectype2)))
6482 45736 : tmp = build_mask_conversion (vinfo, rhs1, vectype1, stmt_vinfo);
6483 : else
6484 : tmp = rhs1;
6485 :
6486 45736 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6487 45736 : pattern_stmt = gimple_build_assign (lhs, COND_EXPR, tmp,
6488 : gimple_assign_rhs2 (last_stmt),
6489 : gimple_assign_rhs3 (last_stmt));
6490 :
6491 45736 : *type_out = vectype1;
6492 45736 : vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
6493 :
6494 45736 : return pattern_stmt;
6495 : }
6496 :
6497 : /* Now check for binary boolean operations requiring conversion for
6498 : one of operands. */
6499 22993877 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)))
6500 : return NULL;
6501 :
6502 1902789 : if (rhs_code != BIT_IOR_EXPR
6503 : && rhs_code != BIT_XOR_EXPR
6504 1902789 : && rhs_code != BIT_AND_EXPR
6505 1564229 : && TREE_CODE_CLASS (rhs_code) != tcc_comparison)
6506 : return NULL;
6507 :
6508 1719065 : rhs2 = gimple_assign_rhs2 (last_stmt);
6509 :
6510 1719065 : rhs1_type = integer_type_for_mask (rhs1, vinfo);
6511 1719065 : rhs2_type = integer_type_for_mask (rhs2, vinfo);
6512 :
6513 1719065 : if (!rhs1_type || !rhs2_type
6514 1719065 : || TYPE_PRECISION (rhs1_type) == TYPE_PRECISION (rhs2_type))
6515 : return NULL;
6516 :
6517 31830 : if (TYPE_PRECISION (rhs1_type) < TYPE_PRECISION (rhs2_type))
6518 : {
6519 24499 : vectype1 = get_mask_type_for_scalar_type (vinfo, rhs1_type);
6520 24499 : if (!vectype1)
6521 : return NULL;
6522 24499 : rhs2 = build_mask_conversion (vinfo, rhs2, vectype1, stmt_vinfo,
6523 : rhs1_type);
6524 : }
6525 : else
6526 : {
6527 7331 : vectype1 = get_mask_type_for_scalar_type (vinfo, rhs2_type);
6528 7331 : if (!vectype1)
6529 : return NULL;
6530 7331 : rhs1 = build_mask_conversion (vinfo, rhs1, vectype1, stmt_vinfo,
6531 : rhs2_type);
6532 : }
6533 :
6534 31830 : lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6535 31830 : pattern_stmt = gimple_build_assign (lhs, rhs_code, rhs1, rhs2);
6536 :
6537 31830 : *type_out = vectype1;
6538 31830 : vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
6539 :
6540 31830 : return pattern_stmt;
6541 : }
6542 :
6543 : /* STMT_INFO is a load or store. If the load or store is conditional, return
6544 : the boolean condition under which it occurs, otherwise return null. */
6545 :
6546 : static tree
6547 104306 : vect_get_load_store_mask (stmt_vec_info stmt_info)
6548 : {
6549 104306 : if (gassign *def_assign = dyn_cast <gassign *> (stmt_info->stmt))
6550 : {
6551 103283 : gcc_assert (gimple_assign_single_p (def_assign));
6552 : return NULL_TREE;
6553 : }
6554 :
6555 1023 : if (gcall *def_call = dyn_cast <gcall *> (stmt_info->stmt))
6556 : {
6557 1023 : internal_fn ifn = gimple_call_internal_fn (def_call);
6558 1023 : int mask_index = internal_fn_mask_index (ifn);
6559 1023 : return gimple_call_arg (def_call, mask_index);
6560 : }
6561 :
6562 0 : gcc_unreachable ();
6563 : }
6564 :
6565 : /* Return the equivalent of:
6566 :
6567 : fold_convert (TYPE, VALUE)
6568 :
6569 : with the expectation that the operation will be vectorized.
6570 : If new statements are needed, add them as pattern statements
6571 : to STMT_INFO. */
6572 :
6573 : static tree
6574 0 : vect_add_conversion_to_pattern (vec_info *vinfo,
6575 : tree type, tree value, stmt_vec_info stmt_info)
6576 : {
6577 0 : if (useless_type_conversion_p (type, TREE_TYPE (value)))
6578 : return value;
6579 :
6580 0 : tree new_value = vect_recog_temp_ssa_var (type, NULL);
6581 0 : gassign *conversion = gimple_build_assign (new_value, CONVERT_EXPR, value);
6582 0 : append_pattern_def_seq (vinfo, stmt_info, conversion,
6583 : get_vectype_for_scalar_type (vinfo, type));
6584 0 : return new_value;
6585 : }
6586 :
6587 : /* Try to convert STMT_INFO into a call to a gather load or scatter store
6588 : internal function. Return the final statement on success and set
6589 : *TYPE_OUT to the vector type being loaded or stored.
6590 :
6591 : This function only handles gathers and scatters that were recognized
6592 : as such from the outset (indicated by STMT_VINFO_GATHER_SCATTER_P). */
6593 :
6594 : static gimple *
6595 32595530 : vect_recog_gather_scatter_pattern (vec_info *vinfo,
6596 : stmt_vec_info stmt_info, tree *type_out)
6597 : {
6598 : /* Currently we only support this for loop vectorization. */
6599 32595530 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6600 4612618 : if (!loop_vinfo)
6601 : return NULL;
6602 :
6603 : /* Make sure that we're looking at a gather load or scatter store. */
6604 4612618 : data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
6605 4612618 : if (!dr || !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
6606 : return NULL;
6607 :
6608 : /* Get the boolean that controls whether the load or store happens.
6609 : This is null if the operation is unconditional. */
6610 104306 : tree mask = vect_get_load_store_mask (stmt_info);
6611 :
6612 : /* DR analysis nailed down the vector type for the access. */
6613 104306 : tree gs_vectype = STMT_VINFO_VECTYPE (stmt_info);
6614 :
6615 : /* Make sure that the target supports an appropriate internal
6616 : function for the gather/scatter operation. */
6617 104306 : gather_scatter_info gs_info;
6618 104306 : if (!vect_check_gather_scatter (stmt_info, gs_vectype, loop_vinfo, &gs_info)
6619 104306 : || gs_info.ifn == IFN_LAST)
6620 : return NULL;
6621 :
6622 : /* Convert the mask to the right form. */
6623 0 : if (mask)
6624 0 : mask = vect_convert_mask_for_vectype (mask, gs_vectype, stmt_info,
6625 : loop_vinfo);
6626 0 : else if (gs_info.ifn == IFN_MASK_SCATTER_STORE
6627 0 : || gs_info.ifn == IFN_MASK_GATHER_LOAD
6628 0 : || gs_info.ifn == IFN_MASK_LEN_SCATTER_STORE
6629 0 : || gs_info.ifn == IFN_MASK_LEN_GATHER_LOAD)
6630 0 : mask = build_int_cst (TREE_TYPE (truth_type_for (gs_vectype)), -1);
6631 :
6632 : /* Get the invariant base and non-invariant offset, converting the
6633 : latter to the same width as the vector elements. */
6634 0 : tree base = gs_info.base;
6635 0 : tree offset_type = TREE_TYPE (gs_info.offset_vectype);
6636 0 : tree offset = vect_add_conversion_to_pattern (vinfo, offset_type,
6637 : gs_info.offset, stmt_info);
6638 :
6639 : /* Build the new pattern statement. */
6640 0 : tree scale = size_int (gs_info.scale);
6641 0 : gcall *pattern_stmt;
6642 :
6643 0 : if (DR_IS_READ (dr))
6644 : {
6645 0 : tree zero = build_zero_cst (gs_info.element_type);
6646 0 : if (mask != NULL)
6647 : {
6648 0 : int elsval = MASK_LOAD_ELSE_ZERO;
6649 :
6650 0 : tree vec_els
6651 0 : = vect_get_mask_load_else (elsval, TREE_TYPE (gs_vectype));
6652 0 : pattern_stmt = gimple_build_call_internal (gs_info.ifn, 7, base,
6653 : gs_info.alias_ptr,
6654 : offset, scale, zero, mask,
6655 : vec_els);
6656 : }
6657 : else
6658 0 : pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5, base,
6659 : gs_info.alias_ptr,
6660 : offset, scale, zero);
6661 0 : tree lhs = gimple_get_lhs (stmt_info->stmt);
6662 0 : tree load_lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
6663 0 : gimple_call_set_lhs (pattern_stmt, load_lhs);
6664 : }
6665 : else
6666 : {
6667 0 : tree rhs = vect_get_store_rhs (stmt_info);
6668 0 : if (mask != NULL)
6669 0 : pattern_stmt = gimple_build_call_internal (gs_info.ifn, 6,
6670 : base, gs_info.alias_ptr,
6671 : offset, scale, rhs, mask);
6672 : else
6673 0 : pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5,
6674 : base, gs_info.alias_ptr,
6675 : offset, scale, rhs);
6676 : }
6677 0 : gimple_call_set_nothrow (pattern_stmt, true);
6678 :
6679 : /* Copy across relevant vectorization info and associate DR with the
6680 : new pattern statement instead of the original statement. */
6681 0 : stmt_vec_info pattern_stmt_info = loop_vinfo->add_stmt (pattern_stmt);
6682 0 : loop_vinfo->move_dr (pattern_stmt_info, stmt_info);
6683 :
6684 0 : *type_out = gs_vectype;
6685 0 : vect_pattern_detected ("gather/scatter pattern", stmt_info->stmt);
6686 :
6687 0 : return pattern_stmt;
6688 : }
6689 :
6690 : /* Helper method of vect_recog_cond_store_pattern, checks to see if COND_ARG
6691 : is points to a load statement that reads the same data as that of
6692 : STORE_VINFO. */
6693 :
6694 : static bool
6695 35416 : vect_cond_store_pattern_same_ref (vec_info *vinfo,
6696 : stmt_vec_info store_vinfo, tree cond_arg)
6697 : {
6698 35416 : stmt_vec_info load_stmt_vinfo = vinfo->lookup_def (cond_arg);
6699 35416 : if (!load_stmt_vinfo
6700 20230 : || !STMT_VINFO_DATA_REF (load_stmt_vinfo)
6701 11986 : || DR_IS_WRITE (STMT_VINFO_DATA_REF (load_stmt_vinfo))
6702 47402 : || !same_data_refs (STMT_VINFO_DATA_REF (store_vinfo),
6703 : STMT_VINFO_DATA_REF (load_stmt_vinfo)))
6704 26096 : return false;
6705 :
6706 : return true;
6707 : }
6708 :
6709 : /* Function vect_recog_cond_store_pattern
6710 :
6711 : Try to find the following pattern:
6712 :
6713 : x = *_3;
6714 : c = a CMP b;
6715 : y = c ? t_20 : x;
6716 : *_3 = y;
6717 :
6718 : where the store of _3 happens on a conditional select on a value loaded
6719 : from the same location. In such case we can elide the initial load if
6720 : MASK_STORE is supported and instead only conditionally write out the result.
6721 :
6722 : The pattern produces for the above:
6723 :
6724 : c = a CMP b;
6725 : .MASK_STORE (_3, c, t_20)
6726 :
6727 : Input:
6728 :
6729 : * STMT_VINFO: The stmt from which the pattern search begins. In the
6730 : example, when this function is called with _3 then the search begins.
6731 :
6732 : Output:
6733 :
6734 : * TYPE_OUT: The type of the output of this pattern.
6735 :
6736 : * Return value: A new stmt that will be used to replace the sequence. */
6737 :
6738 : static gimple *
6739 32595530 : vect_recog_cond_store_pattern (vec_info *vinfo,
6740 : stmt_vec_info stmt_vinfo, tree *type_out)
6741 : {
6742 32595530 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6743 4612618 : if (!loop_vinfo)
6744 : return NULL;
6745 :
6746 4612618 : gimple *store_stmt = STMT_VINFO_STMT (stmt_vinfo);
6747 :
6748 : /* Needs to be a gimple store where we have DR info for. */
6749 4612618 : if (!STMT_VINFO_DATA_REF (stmt_vinfo)
6750 1105357 : || DR_IS_READ (STMT_VINFO_DATA_REF (stmt_vinfo))
6751 5002200 : || !gimple_store_p (store_stmt))
6752 : return NULL;
6753 :
6754 387194 : tree st_rhs = gimple_assign_rhs1 (store_stmt);
6755 :
6756 387194 : if (TREE_CODE (st_rhs) != SSA_NAME)
6757 : return NULL;
6758 :
6759 301043 : auto cond_vinfo = vinfo->lookup_def (st_rhs);
6760 :
6761 : /* If the condition isn't part of the loop then bool recog wouldn't have seen
6762 : it and so this transformation may not be valid. */
6763 301043 : if (!cond_vinfo)
6764 : return NULL;
6765 :
6766 283365 : cond_vinfo = vect_stmt_to_vectorize (cond_vinfo);
6767 283365 : gassign *cond_stmt = dyn_cast<gassign *> (STMT_VINFO_STMT (cond_vinfo));
6768 351791 : if (!cond_stmt || gimple_assign_rhs_code (cond_stmt) != COND_EXPR)
6769 : return NULL;
6770 :
6771 : /* Check if the else value matches the original loaded one. */
6772 18760 : bool invert = false;
6773 18760 : tree cmp_ls = gimple_arg (cond_stmt, 0);
6774 18760 : if (TREE_CODE (cmp_ls) != SSA_NAME)
6775 : return NULL;
6776 :
6777 18760 : tree cond_arg1 = gimple_arg (cond_stmt, 1);
6778 18760 : tree cond_arg2 = gimple_arg (cond_stmt, 2);
6779 :
6780 18760 : if (!vect_cond_store_pattern_same_ref (vinfo, stmt_vinfo, cond_arg2)
6781 18760 : && !(invert = vect_cond_store_pattern_same_ref (vinfo, stmt_vinfo,
6782 : cond_arg1)))
6783 : return NULL;
6784 :
6785 9320 : vect_pattern_detected ("vect_recog_cond_store_pattern", store_stmt);
6786 :
6787 9320 : tree scalar_type = TREE_TYPE (st_rhs);
6788 9320 : if (VECTOR_TYPE_P (scalar_type))
6789 : return NULL;
6790 :
6791 9320 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
6792 9320 : if (vectype == NULL_TREE)
6793 : return NULL;
6794 :
6795 9320 : machine_mode mask_mode;
6796 9320 : machine_mode vecmode = TYPE_MODE (vectype);
6797 1858 : if (!VECTOR_MODE_P (vecmode)
6798 9320 : || targetm.vectorize.conditional_operation_is_expensive (IFN_MASK_STORE)
6799 9320 : || !targetm.vectorize.get_mask_mode (vecmode).exists (&mask_mode)
6800 9320 : || !can_vec_mask_load_store_p (vecmode, mask_mode, false))
6801 : return NULL;
6802 :
6803 0 : tree base = DR_REF (STMT_VINFO_DATA_REF (stmt_vinfo));
6804 0 : if (may_be_nonaddressable_p (base))
6805 : return NULL;
6806 :
6807 : /* We need to use the false parameter of the conditional select. */
6808 0 : tree cond_store_arg = invert ? cond_arg2 : cond_arg1;
6809 0 : tree cond_load_arg = invert ? cond_arg1 : cond_arg2;
6810 0 : gimple *load_stmt = SSA_NAME_DEF_STMT (cond_load_arg);
6811 :
6812 : /* This is a rough estimation to check that there aren't any aliasing stores
6813 : in between the load and store. It's a bit strict, but for now it's good
6814 : enough. */
6815 0 : if (gimple_vuse (load_stmt) != gimple_vuse (store_stmt))
6816 : return NULL;
6817 :
6818 : /* If we have to invert the condition, i.e. use the true argument rather than
6819 : the false argument, we have to negate the mask. */
6820 0 : if (invert)
6821 : {
6822 0 : tree var = vect_recog_temp_ssa_var (boolean_type_node, NULL);
6823 :
6824 : /* Invert the mask using ^ 1. */
6825 0 : tree itype = TREE_TYPE (cmp_ls);
6826 0 : gassign *conv = gimple_build_assign (var, BIT_XOR_EXPR, cmp_ls,
6827 : build_int_cst (itype, 1));
6828 :
6829 0 : tree mask_vec_type = get_mask_type_for_scalar_type (vinfo, itype);
6830 0 : append_pattern_def_seq (vinfo, stmt_vinfo, conv, mask_vec_type, itype);
6831 0 : cmp_ls= var;
6832 : }
6833 :
6834 0 : if (TREE_CODE (base) != MEM_REF)
6835 0 : base = build_fold_addr_expr (base);
6836 :
6837 0 : tree ptr = build_int_cst (reference_alias_ptr_type (base),
6838 0 : get_object_alignment (base));
6839 :
6840 : /* Convert the mask to the right form. */
6841 0 : tree mask = vect_convert_mask_for_vectype (cmp_ls, vectype, stmt_vinfo,
6842 : vinfo);
6843 :
6844 0 : gcall *call
6845 0 : = gimple_build_call_internal (IFN_MASK_STORE, 4, base, ptr, mask,
6846 : cond_store_arg);
6847 0 : gimple_set_location (call, gimple_location (store_stmt));
6848 :
6849 : /* Copy across relevant vectorization info and associate DR with the
6850 : new pattern statement instead of the original statement. */
6851 0 : stmt_vec_info pattern_stmt_info = loop_vinfo->add_stmt (call);
6852 0 : loop_vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
6853 :
6854 0 : *type_out = vectype;
6855 0 : return call;
6856 : }
6857 :
6858 : /* Return true if TYPE is a non-boolean integer type. These are the types
6859 : that we want to consider for narrowing. */
6860 :
6861 : static bool
6862 66690349 : vect_narrowable_type_p (tree type)
6863 : {
6864 66690349 : return INTEGRAL_TYPE_P (type) && !VECT_SCALAR_BOOLEAN_TYPE_P (type);
6865 : }
6866 :
6867 : /* Return true if the operation given by CODE can be truncated to N bits
6868 : when only N bits of the output are needed. This is only true if bit N+1
6869 : of the inputs has no effect on the low N bits of the result. */
6870 :
6871 : static bool
6872 16545466 : vect_truncatable_operation_p (tree_code code)
6873 : {
6874 16545466 : switch (code)
6875 : {
6876 : case NEGATE_EXPR:
6877 : case PLUS_EXPR:
6878 : case MINUS_EXPR:
6879 : case MULT_EXPR:
6880 : case BIT_NOT_EXPR:
6881 : case BIT_AND_EXPR:
6882 : case BIT_IOR_EXPR:
6883 : case BIT_XOR_EXPR:
6884 : case COND_EXPR:
6885 : return true;
6886 :
6887 6487463 : default:
6888 6487463 : return false;
6889 : }
6890 : }
6891 :
6892 : /* Record that STMT_INFO could be changed from operating on TYPE to
6893 : operating on a type with the precision and sign given by PRECISION
6894 : and SIGN respectively. PRECISION is an arbitrary bit precision;
6895 : it might not be a whole number of bytes. */
6896 :
6897 : static void
6898 2572995 : vect_set_operation_type (stmt_vec_info stmt_info, tree type,
6899 : unsigned int precision, signop sign)
6900 : {
6901 : /* Round the precision up to a whole number of bytes. */
6902 2572995 : precision = vect_element_precision (precision);
6903 2572995 : if (precision < TYPE_PRECISION (type)
6904 2572995 : && (!stmt_info->operation_precision
6905 40590 : || stmt_info->operation_precision > precision))
6906 : {
6907 1676622 : stmt_info->operation_precision = precision;
6908 1676622 : stmt_info->operation_sign = sign;
6909 : }
6910 2572995 : }
6911 :
6912 : /* Record that STMT_INFO only requires MIN_INPUT_PRECISION from its
6913 : non-boolean inputs, all of which have type TYPE. MIN_INPUT_PRECISION
6914 : is an arbitrary bit precision; it might not be a whole number of bytes. */
6915 :
6916 : static void
6917 11864509 : vect_set_min_input_precision (stmt_vec_info stmt_info, tree type,
6918 : unsigned int min_input_precision)
6919 : {
6920 : /* This operation in isolation only requires the inputs to have
6921 : MIN_INPUT_PRECISION of precision, However, that doesn't mean
6922 : that MIN_INPUT_PRECISION is a natural precision for the chain
6923 : as a whole. E.g. consider something like:
6924 :
6925 : unsigned short *x, *y;
6926 : *y = ((*x & 0xf0) >> 4) | (*y << 4);
6927 :
6928 : The right shift can be done on unsigned chars, and only requires the
6929 : result of "*x & 0xf0" to be done on unsigned chars. But taking that
6930 : approach would mean turning a natural chain of single-vector unsigned
6931 : short operations into one that truncates "*x" and then extends
6932 : "(*x & 0xf0) >> 4", with two vectors for each unsigned short
6933 : operation and one vector for each unsigned char operation.
6934 : This would be a significant pessimization.
6935 :
6936 : Instead only propagate the maximum of this precision and the precision
6937 : required by the users of the result. This means that we don't pessimize
6938 : the case above but continue to optimize things like:
6939 :
6940 : unsigned char *y;
6941 : unsigned short *x;
6942 : *y = ((*x & 0xf0) >> 4) | (*y << 4);
6943 :
6944 : Here we would truncate two vectors of *x to a single vector of
6945 : unsigned chars and use single-vector unsigned char operations for
6946 : everything else, rather than doing two unsigned short copies of
6947 : "(*x & 0xf0) >> 4" and then truncating the result. */
6948 11864509 : min_input_precision = MAX (min_input_precision,
6949 : stmt_info->min_output_precision);
6950 :
6951 11864509 : if (min_input_precision < TYPE_PRECISION (type)
6952 11864509 : && (!stmt_info->min_input_precision
6953 64325 : || stmt_info->min_input_precision > min_input_precision))
6954 604266 : stmt_info->min_input_precision = min_input_precision;
6955 11864509 : }
6956 :
6957 : /* Subroutine of vect_determine_min_output_precision. Return true if
6958 : we can calculate a reduced number of output bits for STMT_INFO,
6959 : whose result is LHS. */
6960 :
6961 : static bool
6962 15217436 : vect_determine_min_output_precision_1 (vec_info *vinfo,
6963 : stmt_vec_info stmt_info, tree lhs)
6964 : {
6965 : /* Take the maximum precision required by users of the result. */
6966 15217436 : unsigned int precision = 0;
6967 15217436 : imm_use_iterator iter;
6968 15217436 : use_operand_p use;
6969 16256014 : FOR_EACH_IMM_USE_FAST (use, iter, lhs)
6970 : {
6971 15966069 : gimple *use_stmt = USE_STMT (use);
6972 15966069 : if (is_gimple_debug (use_stmt))
6973 745234 : continue;
6974 15220835 : stmt_vec_info use_stmt_info = vinfo->lookup_stmt (use_stmt);
6975 15220835 : if (!use_stmt_info || !use_stmt_info->min_input_precision)
6976 : return false;
6977 : /* The input precision recorded for COND_EXPRs applies only to the
6978 : "then" and "else" values. */
6979 293924 : gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
6980 263012 : if (assign
6981 263012 : && gimple_assign_rhs_code (assign) == COND_EXPR
6982 580 : && use->use != gimple_assign_rhs2_ptr (assign)
6983 580 : && use->use != gimple_assign_rhs3_ptr (assign))
6984 : return false;
6985 1040280 : precision = MAX (precision, use_stmt_info->min_input_precision);
6986 14927491 : }
6987 :
6988 289945 : if (dump_enabled_p ())
6989 5952 : dump_printf_loc (MSG_NOTE, vect_location,
6990 : "only the low %d bits of %T are significant\n",
6991 : precision, lhs);
6992 289945 : stmt_info->min_output_precision = precision;
6993 289945 : return true;
6994 : }
6995 :
6996 : /* Calculate min_output_precision for STMT_INFO. */
6997 :
6998 : static void
6999 39313161 : vect_determine_min_output_precision (vec_info *vinfo, stmt_vec_info stmt_info)
7000 : {
7001 : /* We're only interested in statements with a narrowable result. */
7002 39313161 : tree lhs = gimple_get_lhs (stmt_info->stmt);
7003 39313161 : if (!lhs
7004 31056354 : || TREE_CODE (lhs) != SSA_NAME
7005 65587450 : || !vect_narrowable_type_p (TREE_TYPE (lhs)))
7006 : return;
7007 :
7008 15217436 : if (!vect_determine_min_output_precision_1 (vinfo, stmt_info, lhs))
7009 14927491 : stmt_info->min_output_precision = TYPE_PRECISION (TREE_TYPE (lhs));
7010 : }
7011 :
7012 : /* Use range information to decide whether STMT (described by STMT_INFO)
7013 : could be done in a narrower type. This is effectively a forward
7014 : propagation, since it uses context-independent information that applies
7015 : to all users of an SSA name. */
7016 :
7017 : static void
7018 22455299 : vect_determine_precisions_from_range (stmt_vec_info stmt_info, gassign *stmt)
7019 : {
7020 22455299 : tree lhs = gimple_assign_lhs (stmt);
7021 22455299 : if (!lhs || TREE_CODE (lhs) != SSA_NAME)
7022 19998588 : return;
7023 :
7024 17960761 : tree type = TREE_TYPE (lhs);
7025 17960761 : if (!vect_narrowable_type_p (type))
7026 : return;
7027 :
7028 : /* First see whether we have any useful range information for the result. */
7029 11751576 : unsigned int precision = TYPE_PRECISION (type);
7030 11751576 : signop sign = TYPE_SIGN (type);
7031 11751576 : wide_int min_value, max_value;
7032 11751576 : if (!vect_get_range_info (lhs, &min_value, &max_value))
7033 : return;
7034 :
7035 5714646 : tree_code code = gimple_assign_rhs_code (stmt);
7036 5714646 : unsigned int nops = gimple_num_ops (stmt);
7037 :
7038 5714646 : if (!vect_truncatable_operation_p (code))
7039 : {
7040 : /* Handle operations that can be computed in type T if all inputs
7041 : and outputs can be represented in type T. Also handle left and
7042 : right shifts, where (in addition) the maximum shift amount must
7043 : be less than the number of bits in T. */
7044 2109520 : bool is_shift;
7045 2109520 : switch (code)
7046 : {
7047 : case LSHIFT_EXPR:
7048 : case RSHIFT_EXPR:
7049 : is_shift = true;
7050 : break;
7051 :
7052 308310 : case ABS_EXPR:
7053 308310 : case MIN_EXPR:
7054 308310 : case MAX_EXPR:
7055 308310 : case TRUNC_DIV_EXPR:
7056 308310 : case CEIL_DIV_EXPR:
7057 308310 : case FLOOR_DIV_EXPR:
7058 308310 : case ROUND_DIV_EXPR:
7059 308310 : case EXACT_DIV_EXPR:
7060 : /* Modulus is excluded because it is typically calculated by doing
7061 : a division, for which minimum signed / -1 isn't representable in
7062 : the original signed type. We could take the division range into
7063 : account instead, if handling modulus ever becomes important. */
7064 308310 : is_shift = false;
7065 308310 : break;
7066 :
7067 : default:
7068 : return;
7069 : }
7070 1394957 : for (unsigned int i = 1; i < nops; ++i)
7071 : {
7072 1081895 : tree op = gimple_op (stmt, i);
7073 1081895 : wide_int op_min_value, op_max_value;
7074 1081895 : if (TREE_CODE (op) == INTEGER_CST)
7075 : {
7076 320356 : unsigned int op_precision = TYPE_PRECISION (TREE_TYPE (op));
7077 320356 : op_min_value = op_max_value = wi::to_wide (op, op_precision);
7078 : }
7079 761539 : else if (TREE_CODE (op) == SSA_NAME)
7080 : {
7081 761539 : if (!vect_get_range_info (op, &op_min_value, &op_max_value))
7082 : return;
7083 : }
7084 : else
7085 : return;
7086 :
7087 708935 : if (is_shift && i == 2)
7088 : {
7089 : /* There needs to be one more bit than the maximum shift amount.
7090 :
7091 : If the maximum shift amount is already 1 less than PRECISION
7092 : then we can't narrow the shift further. Dealing with that
7093 : case first ensures that we can safely use an unsigned range
7094 : below.
7095 :
7096 : op_min_value isn't relevant, since shifts by negative amounts
7097 : are UB. */
7098 204672 : if (wi::geu_p (op_max_value, precision - 1))
7099 : return;
7100 179591 : unsigned int min_bits = op_max_value.to_uhwi () + 1;
7101 :
7102 : /* As explained below, we can convert a signed shift into an
7103 : unsigned shift if the sign bit is always clear. At this
7104 : point we've already processed the ranges of the output and
7105 : the first input. */
7106 179591 : auto op_sign = sign;
7107 179591 : if (sign == SIGNED && !wi::neg_p (min_value))
7108 : op_sign = UNSIGNED;
7109 359182 : op_min_value = wide_int::from (wi::min_value (min_bits, op_sign),
7110 179591 : precision, op_sign);
7111 359182 : op_max_value = wide_int::from (wi::max_value (min_bits, op_sign),
7112 179591 : precision, op_sign);
7113 : }
7114 683854 : min_value = wi::min (min_value, op_min_value, sign);
7115 683854 : max_value = wi::max (max_value, op_max_value, sign);
7116 1081895 : }
7117 : }
7118 :
7119 : /* Try to switch signed types for unsigned types if we can.
7120 : This is better for two reasons. First, unsigned ops tend
7121 : to be cheaper than signed ops. Second, it means that we can
7122 : handle things like:
7123 :
7124 : signed char c;
7125 : int res = (int) c & 0xff00; // range [0x0000, 0xff00]
7126 :
7127 : as:
7128 :
7129 : signed char c;
7130 : unsigned short res_1 = (unsigned short) c & 0xff00;
7131 : int res = (int) res_1;
7132 :
7133 : where the intermediate result res_1 has unsigned rather than
7134 : signed type. */
7135 3918188 : if (sign == SIGNED && !wi::neg_p (min_value))
7136 : sign = UNSIGNED;
7137 :
7138 : /* See what precision is required for MIN_VALUE and MAX_VALUE. */
7139 3918188 : unsigned int precision1 = wi::min_precision (min_value, sign);
7140 3918188 : unsigned int precision2 = wi::min_precision (max_value, sign);
7141 3918188 : unsigned int value_precision = MAX (precision1, precision2);
7142 3918188 : if (value_precision >= precision)
7143 : return;
7144 :
7145 2456711 : if (dump_enabled_p ())
7146 113309 : dump_printf_loc (MSG_NOTE, vect_location, "can narrow to %s:%d"
7147 : " without loss of precision: %G",
7148 : sign == SIGNED ? "signed" : "unsigned",
7149 : value_precision, (gimple *) stmt);
7150 :
7151 2456711 : vect_set_operation_type (stmt_info, type, value_precision, sign);
7152 2456711 : vect_set_min_input_precision (stmt_info, type, value_precision);
7153 11751576 : }
7154 :
7155 : /* Use information about the users of STMT's result to decide whether
7156 : STMT (described by STMT_INFO) could be done in a narrower type.
7157 : This is effectively a backward propagation. */
7158 :
7159 : static void
7160 22455299 : vect_determine_precisions_from_users (stmt_vec_info stmt_info, gassign *stmt)
7161 : {
7162 22455299 : tree_code code = gimple_assign_rhs_code (stmt);
7163 22455299 : unsigned int opno = (code == COND_EXPR ? 2 : 1);
7164 22455299 : tree type = TREE_TYPE (gimple_op (stmt, opno));
7165 22455299 : if (!vect_narrowable_type_p (type))
7166 13047501 : return;
7167 :
7168 13821188 : unsigned int precision = TYPE_PRECISION (type);
7169 13821188 : unsigned int operation_precision, min_input_precision;
7170 13821188 : switch (code)
7171 : {
7172 2516842 : CASE_CONVERT:
7173 : /* Only the bits that contribute to the output matter. Don't change
7174 : the precision of the operation itself. */
7175 2516842 : operation_precision = precision;
7176 2516842 : min_input_precision = stmt_info->min_output_precision;
7177 2516842 : break;
7178 :
7179 473526 : case LSHIFT_EXPR:
7180 473526 : case RSHIFT_EXPR:
7181 473526 : {
7182 473526 : tree shift = gimple_assign_rhs2 (stmt);
7183 473526 : unsigned int min_const_shift, max_const_shift;
7184 473526 : wide_int min_shift, max_shift;
7185 473526 : if (TREE_CODE (shift) == SSA_NAME
7186 107264 : && vect_get_range_info (shift, &min_shift, &max_shift)
7187 82816 : && wi::ge_p (min_shift, 0, TYPE_SIGN (TREE_TYPE (shift)))
7188 553583 : && wi::lt_p (max_shift, TYPE_PRECISION (type),
7189 80057 : TYPE_SIGN (TREE_TYPE (shift))))
7190 : {
7191 71925 : min_const_shift = min_shift.to_uhwi ();
7192 71925 : max_const_shift = max_shift.to_uhwi ();
7193 : }
7194 401601 : else if (TREE_CODE (shift) == INTEGER_CST
7195 767863 : && wi::ltu_p (wi::to_widest (shift), precision))
7196 366154 : min_const_shift = max_const_shift = TREE_INT_CST_LOW (shift);
7197 : else
7198 35447 : return;
7199 438079 : if (code == LSHIFT_EXPR)
7200 : {
7201 : /* Avoid creating an undefined shift.
7202 :
7203 : ??? We could instead use min_output_precision as-is and
7204 : optimize out-of-range shifts to zero. However, only
7205 : degenerate testcases shift away all their useful input data,
7206 : and it isn't natural to drop input operations in the middle
7207 : of vectorization. This sort of thing should really be
7208 : handled before vectorization. */
7209 109468 : operation_precision = MAX (stmt_info->min_output_precision,
7210 : max_const_shift + 1);
7211 : /* We need CONST_SHIFT fewer bits of the input. */
7212 109468 : min_input_precision = (MAX (operation_precision, max_const_shift)
7213 : - min_const_shift);
7214 : }
7215 : else
7216 : {
7217 : /* We need CONST_SHIFT extra bits to do the operation. */
7218 328611 : operation_precision = (stmt_info->min_output_precision
7219 : + max_const_shift);
7220 328611 : min_input_precision = operation_precision;
7221 : }
7222 438079 : break;
7223 473526 : }
7224 :
7225 10830820 : default:
7226 10830820 : if (vect_truncatable_operation_p (code))
7227 : {
7228 : /* Input bit N has no effect on output bits N-1 and lower. */
7229 6452877 : operation_precision = stmt_info->min_output_precision;
7230 6452877 : min_input_precision = operation_precision;
7231 6452877 : break;
7232 : }
7233 : return;
7234 : }
7235 :
7236 9407798 : if (operation_precision < precision)
7237 : {
7238 116284 : if (dump_enabled_p ())
7239 2820 : dump_printf_loc (MSG_NOTE, vect_location, "can narrow to %s:%d"
7240 : " without affecting users: %G",
7241 2820 : TYPE_UNSIGNED (type) ? "unsigned" : "signed",
7242 : operation_precision, (gimple *) stmt);
7243 232568 : vect_set_operation_type (stmt_info, type, operation_precision,
7244 116284 : TYPE_SIGN (type));
7245 : }
7246 9407798 : vect_set_min_input_precision (stmt_info, type, min_input_precision);
7247 : }
7248 :
7249 : /* Return true if the statement described by STMT_INFO sets a boolean
7250 : SSA_NAME and if we know how to vectorize this kind of statement using
7251 : vector mask types. */
7252 :
7253 : static bool
7254 40411044 : possible_vector_mask_operation_p (stmt_vec_info stmt_info)
7255 : {
7256 40411044 : tree lhs = gimple_get_lhs (stmt_info->stmt);
7257 40411044 : tree_code code = ERROR_MARK;
7258 40411044 : gassign *assign = NULL;
7259 40411044 : gcond *cond = NULL;
7260 :
7261 40411044 : if ((assign = dyn_cast <gassign *> (stmt_info->stmt)))
7262 23239674 : code = gimple_assign_rhs_code (assign);
7263 17171370 : else if ((cond = dyn_cast <gcond *> (stmt_info->stmt)))
7264 : {
7265 5474350 : lhs = gimple_cond_lhs (cond);
7266 5474350 : code = gimple_cond_code (cond);
7267 : }
7268 :
7269 40411044 : if (!lhs
7270 37533787 : || TREE_CODE (lhs) != SSA_NAME
7271 73124055 : || !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)))
7272 : return false;
7273 :
7274 2258974 : if (code != ERROR_MARK)
7275 : {
7276 2003953 : switch (code)
7277 : {
7278 : CASE_CONVERT:
7279 : case SSA_NAME:
7280 : case BIT_NOT_EXPR:
7281 : case BIT_IOR_EXPR:
7282 : case BIT_XOR_EXPR:
7283 : case BIT_AND_EXPR:
7284 : return true;
7285 :
7286 1561334 : default:
7287 1561334 : return TREE_CODE_CLASS (code) == tcc_comparison;
7288 : }
7289 : }
7290 255021 : else if (is_a <gphi *> (stmt_info->stmt))
7291 143897 : return true;
7292 : return false;
7293 : }
7294 :
7295 : /* If STMT_INFO sets a boolean SSA_NAME, see whether we should use
7296 : a vector mask type instead of a normal vector type. Record the
7297 : result in STMT_INFO->mask_precision. Returns true when the
7298 : precision changed. */
7299 :
7300 : static bool
7301 40411044 : vect_determine_mask_precision (vec_info *vinfo, stmt_vec_info stmt_info)
7302 : {
7303 40411044 : if (!possible_vector_mask_operation_p (stmt_info))
7304 : return false;
7305 :
7306 : /* If at least one boolean input uses a vector mask type,
7307 : pick the mask type with the narrowest elements.
7308 :
7309 : ??? This is the traditional behavior. It should always produce
7310 : the smallest number of operations, but isn't necessarily the
7311 : optimal choice. For example, if we have:
7312 :
7313 : a = b & c
7314 :
7315 : where:
7316 :
7317 : - the user of a wants it to have a mask type for 16-bit elements (M16)
7318 : - b also uses M16
7319 : - c uses a mask type for 8-bit elements (M8)
7320 :
7321 : then picking M8 gives:
7322 :
7323 : - 1 M16->M8 pack for b
7324 : - 1 M8 AND for a
7325 : - 2 M8->M16 unpacks for the user of a
7326 :
7327 : whereas picking M16 would have given:
7328 :
7329 : - 2 M8->M16 unpacks for c
7330 : - 2 M16 ANDs for a
7331 :
7332 : The number of operations are equal, but M16 would have given
7333 : a shorter dependency chain and allowed more ILP. */
7334 2096196 : unsigned int precision = ~0U;
7335 2096196 : gimple *stmt = STMT_VINFO_STMT (stmt_info);
7336 :
7337 : /* If the statement compares two values that shouldn't use vector masks,
7338 : try comparing the values as normal scalars instead. */
7339 2096196 : tree_code code = ERROR_MARK;
7340 2096196 : tree op0_type;
7341 2096196 : unsigned int nops = -1;
7342 2096196 : unsigned int ops_start = 0;
7343 :
7344 2096196 : if (gassign *assign = dyn_cast <gassign *> (stmt))
7345 : {
7346 1382586 : code = gimple_assign_rhs_code (assign);
7347 1382586 : op0_type = TREE_TYPE (gimple_assign_rhs1 (assign));
7348 1382586 : nops = gimple_num_ops (assign);
7349 1382586 : ops_start = 1;
7350 : }
7351 713610 : else if (gcond *cond = dyn_cast <gcond *> (stmt))
7352 : {
7353 569713 : code = gimple_cond_code (cond);
7354 569713 : op0_type = TREE_TYPE (gimple_cond_lhs (cond));
7355 569713 : nops = 2;
7356 569713 : ops_start = 0;
7357 : }
7358 :
7359 1952299 : if (code != ERROR_MARK)
7360 : {
7361 5812338 : for (unsigned int i = ops_start; i < nops; ++i)
7362 : {
7363 3860039 : tree rhs = gimple_op (stmt, i);
7364 3860039 : if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (rhs)))
7365 1890149 : continue;
7366 :
7367 1969890 : stmt_vec_info def_stmt_info = vinfo->lookup_def (rhs);
7368 1969890 : if (!def_stmt_info)
7369 : /* Don't let external or constant operands influence the choice.
7370 : We can convert them to whichever vector type we pick. */
7371 586350 : continue;
7372 :
7373 1383540 : if (def_stmt_info->mask_precision)
7374 : {
7375 1160511 : if (precision > def_stmt_info->mask_precision)
7376 3860039 : precision = def_stmt_info->mask_precision;
7377 : }
7378 : }
7379 :
7380 1952299 : if (precision == ~0U
7381 1567165 : && TREE_CODE_CLASS (code) == tcc_comparison)
7382 : {
7383 1354524 : scalar_mode mode;
7384 1354524 : tree vectype, mask_type;
7385 1354524 : if (is_a <scalar_mode> (TYPE_MODE (op0_type), &mode)
7386 : /* Do not allow this to set vinfo->vector_mode, this might
7387 : disrupt the result for the next iteration. */
7388 1354524 : && (vectype = get_related_vectype_for_scalar_type
7389 1627407 : (vinfo->vector_mode, op0_type))
7390 1186180 : && (mask_type = truth_type_for (vectype))
7391 1186180 : && expand_vec_cmp_expr_p (vectype, mask_type, code))
7392 1826594 : precision = GET_MODE_BITSIZE (mode);
7393 : }
7394 : }
7395 : else
7396 : {
7397 143897 : gphi *phi = as_a <gphi *> (stmt_info->stmt);
7398 740118 : for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
7399 : {
7400 452324 : tree rhs = gimple_phi_arg_def (phi, i);
7401 :
7402 452324 : stmt_vec_info def_stmt_info = vinfo->lookup_def (rhs);
7403 452324 : if (!def_stmt_info)
7404 : /* Don't let external or constant operands influence the choice.
7405 : We can convert them to whichever vector type we pick. */
7406 297363 : continue;
7407 :
7408 154961 : if (def_stmt_info->mask_precision)
7409 : {
7410 129863 : if (precision > def_stmt_info->mask_precision)
7411 452324 : precision = def_stmt_info->mask_precision;
7412 : }
7413 : }
7414 : }
7415 :
7416 2096196 : if (stmt_info->mask_precision != precision)
7417 : {
7418 1966690 : if (dump_enabled_p ())
7419 : {
7420 8021 : if (precision == ~0U)
7421 1764 : dump_printf_loc (MSG_NOTE, vect_location,
7422 : "using normal nonmask vectors for %G",
7423 : stmt_info->stmt);
7424 : else
7425 6257 : dump_printf_loc (MSG_NOTE, vect_location,
7426 : "using boolean precision %d for %G",
7427 : precision, stmt_info->stmt);
7428 : }
7429 :
7430 : /* ??? We'd like to assert stmt_info->mask_precision == 0
7431 : || stmt_info->mask_precision > precision, thus that we only
7432 : decrease mask precisions throughout iteration, but the
7433 : tcc_comparison handling above means for comparisons of bools
7434 : we start with 8 but might increase in case the bools get mask
7435 : precision on their own. */
7436 1966690 : stmt_info->mask_precision = precision;
7437 1966690 : return true;
7438 : }
7439 : return false;
7440 : }
7441 :
7442 : /* Handle vect_determine_precisions for STMT_INFO, given that we
7443 : have already done so for the users of its result. */
7444 :
7445 : void
7446 39313161 : vect_determine_stmt_precisions (vec_info *vinfo, stmt_vec_info stmt_info)
7447 : {
7448 39313161 : vect_determine_min_output_precision (vinfo, stmt_info);
7449 39313161 : if (gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt))
7450 : {
7451 22455299 : vect_determine_precisions_from_range (stmt_info, stmt);
7452 22455299 : vect_determine_precisions_from_users (stmt_info, stmt);
7453 : }
7454 39313161 : }
7455 :
7456 : /* Walk backwards through the vectorizable region to determine the
7457 : values of these fields:
7458 :
7459 : - min_output_precision
7460 : - min_input_precision
7461 : - operation_precision
7462 : - operation_sign. */
7463 :
7464 : void
7465 1114628 : vect_determine_precisions (vec_info *vinfo)
7466 : {
7467 1114628 : basic_block *bbs = vinfo->bbs;
7468 1114628 : unsigned int nbbs = vinfo->nbbs;
7469 :
7470 1134417 : DUMP_VECT_SCOPE ("vect_determine_precisions");
7471 :
7472 : /* For mask precisions we have to iterate since otherwise we do not
7473 : get reduction PHI precision correct. For now do this only for
7474 : loop vectorization. */
7475 1185700 : bool changed;
7476 1185700 : do
7477 : {
7478 1185700 : changed = false;
7479 13305782 : for (unsigned int i = 0; i < nbbs; i++)
7480 : {
7481 12120082 : basic_block bb = bbs[i];
7482 12120082 : for (auto gsi = gsi_start_phis (bb);
7483 19589864 : !gsi_end_p (gsi); gsi_next (&gsi))
7484 : {
7485 7469782 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi.phi ());
7486 7469782 : if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
7487 7273870 : changed |= vect_determine_mask_precision (vinfo, stmt_info);
7488 : }
7489 130814294 : for (auto gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
7490 : {
7491 106574130 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (gsi));
7492 106574130 : if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
7493 33137174 : changed |= vect_determine_mask_precision (vinfo, stmt_info);
7494 : }
7495 : }
7496 : }
7497 2300328 : while (changed && is_a <loop_vec_info> (vinfo));
7498 :
7499 13070997 : for (unsigned int i = 0; i < nbbs; i++)
7500 : {
7501 11956369 : basic_block bb = bbs[nbbs - i - 1];
7502 138204527 : for (auto gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
7503 : {
7504 104919890 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (gsi));
7505 104919890 : if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
7506 32245000 : vect_determine_stmt_precisions (vinfo, stmt_info);
7507 : }
7508 19220442 : for (auto gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
7509 : {
7510 7264073 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi.phi ());
7511 7264073 : if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
7512 7068161 : vect_determine_stmt_precisions (vinfo, stmt_info);
7513 : }
7514 : }
7515 1114628 : }
7516 :
7517 : typedef gimple *(*vect_recog_func_ptr) (vec_info *, stmt_vec_info, tree *);
7518 :
7519 : struct vect_recog_func
7520 : {
7521 : vect_recog_func_ptr fn;
7522 : const char *name;
7523 : };
7524 :
7525 : /* Note that ordering matters - the first pattern matching on a stmt is
7526 : taken which means usually the more complex one needs to precede the
7527 : less comples onex (widen_sum only after dot_prod or sad for example). */
7528 : static vect_recog_func vect_vect_recog_func_ptrs[] = {
7529 : { vect_recog_bitfield_ref_pattern, "bitfield_ref" },
7530 : { vect_recog_bit_insert_pattern, "bit_insert" },
7531 : { vect_recog_abd_pattern, "abd" },
7532 : { vect_recog_over_widening_pattern, "over_widening" },
7533 : /* Must come after over_widening, which narrows the shift as much as
7534 : possible beforehand. */
7535 : { vect_recog_average_pattern, "average" },
7536 : { vect_recog_cond_expr_convert_pattern, "cond_expr_convert" },
7537 : { vect_recog_mulhs_pattern, "mult_high" },
7538 : { vect_recog_cast_forwprop_pattern, "cast_forwprop" },
7539 : { vect_recog_widen_mult_pattern, "widen_mult" },
7540 : { vect_recog_dot_prod_pattern, "dot_prod" },
7541 : { vect_recog_sad_pattern, "sad" },
7542 : { vect_recog_widen_sum_pattern, "widen_sum" },
7543 : { vect_recog_pow_pattern, "pow" },
7544 : { vect_recog_popcount_clz_ctz_ffs_pattern, "popcount_clz_ctz_ffs" },
7545 : { vect_recog_ctz_ffs_pattern, "ctz_ffs" },
7546 : { vect_recog_widen_shift_pattern, "widen_shift" },
7547 : { vect_recog_rotate_pattern, "rotate" },
7548 : { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" },
7549 : { vect_recog_divmod_pattern, "divmod" },
7550 : { vect_recog_mod_var_pattern, "modvar" },
7551 : { vect_recog_mult_pattern, "mult" },
7552 : { vect_recog_sat_add_pattern, "sat_add" },
7553 : { vect_recog_sat_sub_pattern, "sat_sub" },
7554 : { vect_recog_sat_trunc_pattern, "sat_trunc" },
7555 : { vect_recog_gcond_pattern, "gcond" },
7556 : { vect_recog_bool_pattern, "bool" },
7557 : /* This must come before mask conversion, and includes the parts
7558 : of mask conversion that are needed for gather and scatter
7559 : internal functions. */
7560 : { vect_recog_gather_scatter_pattern, "gather_scatter" },
7561 : { vect_recog_cond_store_pattern, "cond_store" },
7562 : { vect_recog_mask_conversion_pattern, "mask_conversion" },
7563 : { vect_recog_widen_plus_pattern, "widen_plus" },
7564 : { vect_recog_widen_minus_pattern, "widen_minus" },
7565 : { vect_recog_widen_abd_pattern, "widen_abd" },
7566 : /* These must come after the double widening ones. */
7567 : };
7568 :
7569 : /* Mark statements that are involved in a pattern. */
7570 :
7571 : void
7572 1059976 : vect_mark_pattern_stmts (vec_info *vinfo,
7573 : stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
7574 : tree pattern_vectype)
7575 : {
7576 1059976 : stmt_vec_info orig_stmt_info_saved = orig_stmt_info;
7577 1059976 : gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
7578 :
7579 1059976 : gimple *orig_pattern_stmt = NULL;
7580 1059976 : if (is_pattern_stmt_p (orig_stmt_info))
7581 : {
7582 : /* We're replacing a statement in an existing pattern definition
7583 : sequence. */
7584 11858 : orig_pattern_stmt = orig_stmt_info->stmt;
7585 11858 : if (dump_enabled_p ())
7586 987 : dump_printf_loc (MSG_NOTE, vect_location,
7587 : "replacing earlier pattern %G", orig_pattern_stmt);
7588 :
7589 : /* To keep the book-keeping simple, just swap the lhs of the
7590 : old and new statements, so that the old one has a valid but
7591 : unused lhs. */
7592 11858 : tree old_lhs = gimple_get_lhs (orig_pattern_stmt);
7593 11858 : gimple_set_lhs (orig_pattern_stmt, gimple_get_lhs (pattern_stmt));
7594 11858 : gimple_set_lhs (pattern_stmt, old_lhs);
7595 :
7596 11858 : if (dump_enabled_p ())
7597 987 : dump_printf_loc (MSG_NOTE, vect_location, "with %G", pattern_stmt);
7598 :
7599 : /* Switch to the statement that ORIG replaces. */
7600 11858 : orig_stmt_info = STMT_VINFO_RELATED_STMT (orig_stmt_info);
7601 :
7602 : /* We shouldn't be replacing the main pattern statement. */
7603 11858 : gcc_assert (STMT_VINFO_RELATED_STMT (orig_stmt_info)->stmt
7604 : != orig_pattern_stmt);
7605 : }
7606 :
7607 1059976 : if (def_seq)
7608 2327181 : for (gimple_stmt_iterator si = gsi_start (def_seq);
7609 2327181 : !gsi_end_p (si); gsi_next (&si))
7610 : {
7611 1398694 : if (dump_enabled_p ())
7612 27153 : dump_printf_loc (MSG_NOTE, vect_location,
7613 : "extra pattern stmt: %G", gsi_stmt (si));
7614 1398694 : stmt_vec_info pattern_stmt_info
7615 1398694 : = vect_init_pattern_stmt (vinfo, gsi_stmt (si),
7616 : orig_stmt_info, pattern_vectype);
7617 : /* Stmts in the def sequence are not vectorizable cycle or
7618 : induction defs, instead they should all be vect_internal_def
7619 : feeding the main pattern stmt which retains this def type. */
7620 1398694 : STMT_VINFO_DEF_TYPE (pattern_stmt_info) = vect_internal_def;
7621 : }
7622 :
7623 1059976 : if (orig_pattern_stmt)
7624 : {
7625 11858 : vect_init_pattern_stmt (vinfo, pattern_stmt,
7626 : orig_stmt_info, pattern_vectype);
7627 :
7628 : /* Insert all the new pattern statements before the original one. */
7629 11858 : gimple_seq *orig_def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
7630 11858 : gimple_stmt_iterator gsi = gsi_for_stmt (orig_pattern_stmt,
7631 : orig_def_seq);
7632 11858 : gsi_insert_seq_before_without_update (&gsi, def_seq, GSI_SAME_STMT);
7633 11858 : gsi_insert_before_without_update (&gsi, pattern_stmt, GSI_SAME_STMT);
7634 :
7635 : /* Remove the pattern statement that this new pattern replaces. */
7636 11858 : gsi_remove (&gsi, false);
7637 : }
7638 : else
7639 1048118 : vect_set_pattern_stmt (vinfo,
7640 : pattern_stmt, orig_stmt_info, pattern_vectype);
7641 :
7642 : /* For any conditionals mark them as vect_condition_def. */
7643 1059976 : if (is_a <gcond *> (pattern_stmt))
7644 380631 : STMT_VINFO_DEF_TYPE (STMT_VINFO_RELATED_STMT (orig_stmt_info)) = vect_condition_def;
7645 :
7646 : /* Transfer reduction path info to the pattern. */
7647 1059976 : if (STMT_VINFO_REDUC_IDX (orig_stmt_info_saved) != -1)
7648 : {
7649 16130 : gimple_match_op op;
7650 16130 : if (!gimple_extract_op (orig_stmt_info_saved->stmt, &op))
7651 0 : gcc_unreachable ();
7652 16130 : tree lookfor = op.ops[STMT_VINFO_REDUC_IDX (orig_stmt_info)];
7653 : /* Search the pattern def sequence and the main pattern stmt. Note
7654 : we may have inserted all into a containing pattern def sequence
7655 : so the following is a bit awkward. */
7656 16130 : gimple_stmt_iterator si;
7657 16130 : gimple *s;
7658 16130 : if (def_seq)
7659 : {
7660 14923 : si = gsi_start (def_seq);
7661 14923 : s = gsi_stmt (si);
7662 14923 : gsi_next (&si);
7663 : }
7664 : else
7665 : {
7666 : si = gsi_none ();
7667 16130 : s = pattern_stmt;
7668 : }
7669 33869 : do
7670 : {
7671 33869 : bool found = false;
7672 33869 : if (gimple_extract_op (s, &op))
7673 : {
7674 82530 : for (unsigned i = 0; i < op.num_ops; ++i)
7675 64785 : if (op.ops[i] == lookfor)
7676 : {
7677 16124 : STMT_VINFO_REDUC_IDX (vinfo->lookup_stmt (s)) = i;
7678 16124 : lookfor = gimple_get_lhs (s);
7679 16124 : found = true;
7680 16124 : break;
7681 : }
7682 : /* Try harder to find a mid-entry into an earlier pattern
7683 : sequence. Likewise an entry to a stmt skipping a conversion
7684 : on an input. This means that the initial 'lookfor' was
7685 : bogus. */
7686 16124 : if (!found)
7687 : {
7688 38494 : for (unsigned i = 0; i < op.num_ops; ++i)
7689 20755 : if (TREE_CODE (op.ops[i]) == SSA_NAME)
7690 17745 : if (auto def = vinfo->lookup_def (op.ops[i]))
7691 17552 : if (vect_is_reduction (def)
7692 17552 : || (is_a <gphi *> (def->stmt)
7693 6 : && STMT_VINFO_REDUC_DEF (def) != NULL))
7694 : {
7695 6 : STMT_VINFO_REDUC_IDX (vinfo->lookup_stmt (s)) = i;
7696 6 : lookfor = gimple_get_lhs (s);
7697 6 : found = true;
7698 6 : break;
7699 : }
7700 : }
7701 : }
7702 33869 : if (s == pattern_stmt)
7703 : {
7704 16130 : if (!found && dump_enabled_p ())
7705 0 : dump_printf_loc (MSG_NOTE, vect_location,
7706 : "failed to update reduction index.\n");
7707 16130 : break;
7708 : }
7709 17739 : if (gsi_end_p (si))
7710 : s = pattern_stmt;
7711 : else
7712 : {
7713 2816 : s = gsi_stmt (si);
7714 2816 : if (s == pattern_stmt)
7715 : /* Found the end inside a bigger pattern def seq. */
7716 : si = gsi_none ();
7717 : else
7718 2816 : gsi_next (&si);
7719 : }
7720 : } while (1);
7721 : }
7722 1059976 : }
7723 :
7724 : /* Function vect_pattern_recog_1
7725 :
7726 : Input:
7727 : PATTERN_RECOG_FUNC: A pointer to a function that detects a certain
7728 : computation pattern.
7729 : STMT_INFO: A stmt from which the pattern search should start.
7730 :
7731 : If PATTERN_RECOG_FUNC successfully detected the pattern, it creates
7732 : a sequence of statements that has the same functionality and can be
7733 : used to replace STMT_INFO. It returns the last statement in the sequence
7734 : and adds any earlier statements to STMT_INFO's STMT_VINFO_PATTERN_DEF_SEQ.
7735 : PATTERN_RECOG_FUNC also sets *TYPE_OUT to the vector type of the final
7736 : statement, having first checked that the target supports the new operation
7737 : in that type.
7738 :
7739 : This function also does some bookkeeping, as explained in the documentation
7740 : for vect_recog_pattern. */
7741 :
7742 : static void
7743 1050632847 : vect_pattern_recog_1 (vec_info *vinfo,
7744 : const vect_recog_func &recog_func, stmt_vec_info stmt_info)
7745 : {
7746 1050632847 : gimple *pattern_stmt;
7747 1050632847 : tree pattern_vectype;
7748 :
7749 : /* If this statement has already been replaced with pattern statements,
7750 : leave the original statement alone, since the first match wins.
7751 : Instead try to match against the definition statements that feed
7752 : the main pattern statement. */
7753 1050632847 : if (STMT_VINFO_IN_PATTERN_P (stmt_info))
7754 : {
7755 13340337 : gimple_stmt_iterator gsi;
7756 13340337 : for (gsi = gsi_start (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
7757 32133184 : !gsi_end_p (gsi); gsi_next (&gsi))
7758 18792847 : vect_pattern_recog_1 (vinfo, recog_func,
7759 : vinfo->lookup_stmt (gsi_stmt (gsi)));
7760 : return;
7761 : }
7762 :
7763 1037292510 : gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
7764 1037292510 : pattern_stmt = recog_func.fn (vinfo, stmt_info, &pattern_vectype);
7765 1037292510 : if (!pattern_stmt)
7766 : {
7767 : /* Clear any half-formed pattern definition sequence. */
7768 1036232534 : STMT_VINFO_PATTERN_DEF_SEQ (stmt_info) = NULL;
7769 1036232534 : return;
7770 : }
7771 :
7772 : /* Found a vectorizable pattern. */
7773 1059976 : if (dump_enabled_p ())
7774 20057 : dump_printf_loc (MSG_NOTE, vect_location,
7775 : "%s pattern recognized: %G",
7776 20057 : recog_func.name, pattern_stmt);
7777 :
7778 : /* Mark the stmts that are involved in the pattern. */
7779 1059976 : vect_mark_pattern_stmts (vinfo, stmt_info, pattern_stmt, pattern_vectype);
7780 : }
7781 :
7782 :
7783 : /* Function vect_pattern_recog
7784 :
7785 : Input:
7786 : LOOP_VINFO - a struct_loop_info of a loop in which we want to look for
7787 : computation idioms.
7788 :
7789 : Output - for each computation idiom that is detected we create a new stmt
7790 : that provides the same functionality and that can be vectorized. We
7791 : also record some information in the struct_stmt_info of the relevant
7792 : stmts, as explained below:
7793 :
7794 : At the entry to this function we have the following stmts, with the
7795 : following initial value in the STMT_VINFO fields:
7796 :
7797 : stmt in_pattern_p related_stmt vec_stmt
7798 : S1: a_i = .... - - -
7799 : S2: a_2 = ..use(a_i).. - - -
7800 : S3: a_1 = ..use(a_2).. - - -
7801 : S4: a_0 = ..use(a_1).. - - -
7802 : S5: ... = ..use(a_0).. - - -
7803 :
7804 : Say the sequence {S1,S2,S3,S4} was detected as a pattern that can be
7805 : represented by a single stmt. We then:
7806 : - create a new stmt S6 equivalent to the pattern (the stmt is not
7807 : inserted into the code)
7808 : - fill in the STMT_VINFO fields as follows:
7809 :
7810 : in_pattern_p related_stmt vec_stmt
7811 : S1: a_i = .... - - -
7812 : S2: a_2 = ..use(a_i).. - - -
7813 : S3: a_1 = ..use(a_2).. - - -
7814 : S4: a_0 = ..use(a_1).. true S6 -
7815 : '---> S6: a_new = .... - S4 -
7816 : S5: ... = ..use(a_0).. - - -
7817 :
7818 : (the last stmt in the pattern (S4) and the new pattern stmt (S6) point
7819 : to each other through the RELATED_STMT field).
7820 :
7821 : S6 will be marked as relevant in vect_mark_stmts_to_be_vectorized instead
7822 : of S4 because it will replace all its uses. Stmts {S1,S2,S3} will
7823 : remain irrelevant unless used by stmts other than S4.
7824 :
7825 : If vectorization succeeds, vect_transform_stmt will skip over {S1,S2,S3}
7826 : (because they are marked as irrelevant). It will vectorize S6, and record
7827 : a pointer to the new vector stmt VS6 from S6 (as usual).
7828 : S4 will be skipped, and S5 will be vectorized as usual:
7829 :
7830 : in_pattern_p related_stmt vec_stmt
7831 : S1: a_i = .... - - -
7832 : S2: a_2 = ..use(a_i).. - - -
7833 : S3: a_1 = ..use(a_2).. - - -
7834 : > VS6: va_new = .... - - -
7835 : S4: a_0 = ..use(a_1).. true S6 VS6
7836 : '---> S6: a_new = .... - S4 VS6
7837 : > VS5: ... = ..vuse(va_new).. - - -
7838 : S5: ... = ..use(a_0).. - - -
7839 :
7840 : DCE could then get rid of {S1,S2,S3,S4,S5} (if their defs are not used
7841 : elsewhere), and we'll end up with:
7842 :
7843 : VS6: va_new = ....
7844 : VS5: ... = ..vuse(va_new)..
7845 :
7846 : In case of more than one pattern statements, e.g., widen-mult with
7847 : intermediate type:
7848 :
7849 : S1 a_t = ;
7850 : S2 a_T = (TYPE) a_t;
7851 : '--> S3: a_it = (interm_type) a_t;
7852 : S4 prod_T = a_T * CONST;
7853 : '--> S5: prod_T' = a_it w* CONST;
7854 :
7855 : there may be other users of a_T outside the pattern. In that case S2 will
7856 : be marked as relevant (as well as S3), and both S2 and S3 will be analyzed
7857 : and vectorized. The vector stmt VS2 will be recorded in S2, and VS3 will
7858 : be recorded in S3. */
7859 :
7860 : void
7861 1114628 : vect_pattern_recog (vec_info *vinfo)
7862 : {
7863 1114628 : basic_block *bbs = vinfo->bbs;
7864 1114628 : unsigned int nbbs = vinfo->nbbs;
7865 :
7866 1114628 : vect_determine_precisions (vinfo);
7867 :
7868 1114628 : DUMP_VECT_SCOPE ("vect_pattern_recog");
7869 :
7870 : /* Scan through the stmts in the region, applying the pattern recognition
7871 : functions starting at each stmt visited. */
7872 13070997 : for (unsigned i = 0; i < nbbs; i++)
7873 : {
7874 11956369 : basic_block bb = bbs[i];
7875 :
7876 128832628 : for (auto si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
7877 : {
7878 104919890 : stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (si));
7879 :
7880 104919890 : if (!stmt_info || !STMT_VINFO_VECTORIZABLE (stmt_info))
7881 72674890 : continue;
7882 :
7883 : /* Scan over all generic vect_recog_xxx_pattern functions. */
7884 1064085000 : for (const auto &func_ptr : vect_vect_recog_func_ptrs)
7885 1031840000 : vect_pattern_recog_1 (vinfo, func_ptr,
7886 : stmt_info);
7887 : }
7888 : }
7889 :
7890 : /* After this no more add_stmt calls are allowed. */
7891 1114628 : vinfo->stmt_vec_info_ro = true;
7892 1114628 : }
7893 :
7894 : /* Build a GIMPLE_ASSIGN or GIMPLE_CALL with the tree_code,
7895 : or internal_fn contained in ch, respectively. */
7896 : gimple *
7897 169084 : vect_gimple_build (tree lhs, code_helper ch, tree op0, tree op1)
7898 : {
7899 169084 : gcc_assert (op0 != NULL_TREE);
7900 169084 : if (ch.is_tree_code ())
7901 169084 : return gimple_build_assign (lhs, (tree_code) ch, op0, op1);
7902 :
7903 0 : gcc_assert (ch.is_internal_fn ());
7904 0 : gimple* stmt = gimple_build_call_internal (as_internal_fn ((combined_fn) ch),
7905 : op1 == NULL_TREE ? 1 : 2,
7906 : op0, op1);
7907 0 : gimple_call_set_lhs (stmt, lhs);
7908 0 : return stmt;
7909 : }
|
