LCOV - code coverage report
Current view: top level - gcc - tree-vect-patterns.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 89.1 % 3214 2863
Test Date: 2026-11-14 16:39:50 Functions: 97.7 % 87 85
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Analysis Utilities for Loop Vectorization.
       2              :    Copyright (C) 2006-2026 Free Software Foundation, Inc.
       3              :    Contributed by Dorit Nuzman <dorit@il.ibm.com>
       4              : 
       5              : This file is part of GCC.
       6              : 
       7              : GCC is free software; you can redistribute it and/or modify it under
       8              : the terms of the GNU General Public License as published by the Free
       9              : Software Foundation; either version 3, or (at your option) any later
      10              : version.
      11              : 
      12              : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      13              : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      14              : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      15              : for more details.
      16              : 
      17              : You should have received a copy of the GNU General Public License
      18              : along with GCC; see the file COPYING3.  If not see
      19              : <http://www.gnu.org/licenses/>.  */
      20              : 
      21              : #include "config.h"
      22              : #include "system.h"
      23              : #include "coretypes.h"
      24              : #include "backend.h"
      25              : #include "rtl.h"
      26              : #include "tree.h"
      27              : #include "gimple.h"
      28              : #include "gimple-iterator.h"
      29              : #include "gimple-fold.h"
      30              : #include "ssa.h"
      31              : #include "expmed.h"
      32              : #include "optabs-tree.h"
      33              : #include "insn-config.h"
      34              : #include "recog.h"            /* FIXME: for insn_data */
      35              : #include "fold-const.h"
      36              : #include "stor-layout.h"
      37              : #include "tree-eh.h"
      38              : #include "gimplify.h"
      39              : #include "gimplify-me.h"
      40              : #include "cfgloop.h"
      41              : #include "tree-vectorizer.h"
      42              : #include "dumpfile.h"
      43              : #include "builtins.h"
      44              : #include "internal-fn.h"
      45              : #include "case-cfn-macros.h"
      46              : #include "fold-const-call.h"
      47              : #include "attribs.h"
      48              : #include "cgraph.h"
      49              : #include "omp-simd-clone.h"
      50              : #include "predict.h"
      51              : #include "tree-vector-builder.h"
      52              : #include "tree-ssa-loop-ivopts.h"
      53              : #include "vec-perm-indices.h"
      54              : #include "gimple-range.h"
      55              : #include "alias.h"
      56              : 
      57              : 
      58              : /* TODO:  Note the vectorizer still builds COND_EXPRs with GENERIC compares
      59              :    in the first operand.  Disentangling this is future work, the
      60              :    IL is properly transferred to VEC_COND_EXPRs with separate compares.  */
      61              : 
      62              : 
      63              : /* Return true if we have a useful VR_RANGE range for VAR, storing it
      64              :    in *MIN_VALUE and *MAX_VALUE if so.  Note the range in the dump files.  */
      65              : 
      66              : bool
      67     13065450 : vect_get_range_info (tree var, wide_int *min_value, wide_int *max_value)
      68              : {
      69     13065450 :   int_range_max vr;
      70     13065450 :   tree vr_min, vr_max;
      71     26130900 :   get_range_query (cfun)->range_of_expr (vr, var);
      72     13065450 :   if (vr.undefined_p ())
      73           53 :     vr.set_varying (TREE_TYPE (var));
      74     13065450 :   value_range_kind vr_type = get_legacy_range (vr, vr_min, vr_max);
      75     13065450 :   *min_value = wi::to_wide (vr_min);
      76     13065450 :   *max_value = wi::to_wide (vr_max);
      77     13065450 :   wide_int nonzero = get_nonzero_bits (var);
      78     13065450 :   signop sgn = TYPE_SIGN (TREE_TYPE (var));
      79     13065450 :   if (intersect_range_with_nonzero_bits (vr_type, min_value, max_value,
      80              :                                          nonzero, sgn) == VR_RANGE)
      81              :     {
      82      6336518 :       if (dump_enabled_p ())
      83              :         {
      84        94805 :           dump_generic_expr_loc (MSG_NOTE, vect_location, TDF_SLIM, var);
      85        94805 :           dump_printf (MSG_NOTE, " has range [");
      86        94805 :           dump_hex (MSG_NOTE, *min_value);
      87        94805 :           dump_printf (MSG_NOTE, ", ");
      88        94805 :           dump_hex (MSG_NOTE, *max_value);
      89        94805 :           dump_printf (MSG_NOTE, "]\n");
      90              :         }
      91              :       return true;
      92              :     }
      93              :   else
      94              :     {
      95      6728932 :       if (dump_enabled_p ())
      96              :         {
      97        71326 :           dump_generic_expr_loc (MSG_NOTE, vect_location, TDF_SLIM, var);
      98        71326 :           dump_printf (MSG_NOTE, " has no range info\n");
      99              :         }
     100              :       return false;
     101              :     }
     102     13065450 : }
     103              : 
     104              : /* Report that we've found an instance of pattern PATTERN in
     105              :    statement STMT.  */
     106              : 
     107              : static void
     108      1398518 : vect_pattern_detected (const char *name, gimple *stmt)
     109              : {
     110      1398518 :   if (dump_enabled_p ())
     111        28002 :     dump_printf_loc (MSG_NOTE, vect_location, "%s: detected: %G", name, stmt);
     112      1398518 : }
     113              : 
     114              : /* Associate pattern statement PATTERN_STMT with ORIG_STMT_INFO and
     115              :    return the pattern statement's stmt_vec_info.  Set its vector type to
     116              :    VECTYPE if it doesn't have one already.  */
     117              : 
     118              : static stmt_vec_info
     119      2443389 : vect_init_pattern_stmt (vec_info *vinfo, gimple *pattern_stmt,
     120              :                         stmt_vec_info orig_stmt_info, tree vectype)
     121              : {
     122      2443389 :   stmt_vec_info pattern_stmt_info = vinfo->lookup_stmt (pattern_stmt);
     123      2443389 :   if (pattern_stmt_info == NULL)
     124      1280724 :     pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
     125      2443389 :   gimple_set_bb (pattern_stmt, gimple_bb (orig_stmt_info->stmt));
     126              : 
     127      2443389 :   pattern_stmt_info->pattern_stmt_p = true;
     128      2443389 :   STMT_VINFO_RELATED_STMT (pattern_stmt_info) = orig_stmt_info;
     129      2443389 :   STMT_VINFO_DEF_TYPE (pattern_stmt_info)
     130      2443389 :     = STMT_VINFO_DEF_TYPE (orig_stmt_info);
     131      2443389 :   if (!STMT_VINFO_VECTYPE (pattern_stmt_info))
     132              :     {
     133      2106658 :       gcc_assert (!vectype
     134              :                   || is_a <gcond *> (pattern_stmt)
     135              :                   || (VECTOR_BOOLEAN_TYPE_P (vectype)
     136              :                       == vect_use_mask_type_p (orig_stmt_info)));
     137      1291495 :       STMT_VINFO_VECTYPE (pattern_stmt_info) = vectype;
     138      1291495 :       pattern_stmt_info->mask_precision = orig_stmt_info->mask_precision;
     139      1291495 :       if (dump_enabled_p ())
     140              :         {
     141        27576 :           if (pattern_stmt_info->mask_precision == ~0U)
     142           63 :             dump_printf_loc (MSG_NOTE, vect_location,
     143              :                              "using normal nonmask vectors for %G",
     144              :                              pattern_stmt_info->stmt);
     145              :           else
     146        27513 :             dump_printf_loc (MSG_NOTE, vect_location,
     147              :                              "using boolean precision %d for %G",
     148              :                              pattern_stmt_info->mask_precision,
     149              :                              pattern_stmt_info->stmt);
     150              :         }
     151              :     }
     152      2443389 :   return pattern_stmt_info;
     153              : }
     154              : 
     155              : /* Set the pattern statement of ORIG_STMT_INFO to PATTERN_STMT.
     156              :    Also set the vector type of PATTERN_STMT to VECTYPE, if it doesn't
     157              :    have one already.  */
     158              : 
     159              : static void
     160      1121138 : vect_set_pattern_stmt (vec_info *vinfo, gimple *pattern_stmt,
     161              :                        stmt_vec_info orig_stmt_info, tree vectype)
     162              : {
     163      1121138 :   STMT_VINFO_IN_PATTERN_P (orig_stmt_info) = true;
     164      1121138 :   STMT_VINFO_RELATED_STMT (orig_stmt_info)
     165            0 :     = vect_init_pattern_stmt (vinfo, pattern_stmt, orig_stmt_info, vectype);
     166            0 : }
     167              : 
     168              : /* Add NEW_STMT to STMT_INFO's pattern definition statements.  If VECTYPE
     169              :    is nonnull, record that NEW_STMT's vector type is VECTYPE, which might
     170              :    be different from the vector type of the final pattern statement.
     171              :    If VECTYPE is a mask type, SCALAR_TYPE_FOR_MASK is the scalar type
     172              :    from which it was derived.  */
     173              : 
     174              : static inline void
     175      1282927 : append_pattern_def_seq (vec_info *vinfo,
     176              :                         stmt_vec_info stmt_info, gimple *new_stmt,
     177              :                         tree vectype = NULL_TREE,
     178              :                         tree scalar_type_for_mask = NULL_TREE)
     179              : {
     180      1930536 :   gcc_assert (!scalar_type_for_mask
     181              :               == (!vectype || !VECTOR_BOOLEAN_TYPE_P (vectype)));
     182      1282927 :   if (vectype)
     183              :     {
     184      1152124 :       stmt_vec_info new_stmt_info = vinfo->add_stmt (new_stmt);
     185      1152124 :       STMT_VINFO_VECTYPE (new_stmt_info) = vectype;
     186      1152124 :       if (scalar_type_for_mask)
     187              :         {
     188       635318 :           new_stmt_info->mask_precision
     189       635318 :             = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (scalar_type_for_mask));
     190       635318 :           if (dump_enabled_p ())
     191              :             {
     192         6889 :               if (new_stmt_info->mask_precision == ~0U)
     193              :                 dump_printf_loc (MSG_NOTE, vect_location,
     194              :                                  "using normal nonmask vectors for %G",
     195              :                                  new_stmt_info->stmt);
     196              :               else
     197         6889 :                 dump_printf_loc (MSG_NOTE, vect_location,
     198              :                                  "using boolean precision %d for %G",
     199              :                                  new_stmt_info->mask_precision,
     200              :                                  new_stmt_info->stmt);
     201              :             }
     202              :         }
     203              :     }
     204      1282927 :   gimple_seq_add_stmt_without_update (&STMT_VINFO_PATTERN_DEF_SEQ (stmt_info),
     205              :                                       new_stmt);
     206      1282927 : }
     207              : 
     208              : 
     209              : /* Add NEW_STMT to VINFO's invariant pattern definition statements.  These
     210              :    statements are not vectorized but are materialized as scalar in the loop
     211              :    preheader.  */
     212              : 
     213              : static inline void
     214         1366 : append_inv_pattern_def_seq (vec_info *vinfo, gimple *new_stmt)
     215              : {
     216         1366 :   gimple_seq_add_stmt_without_update (&vinfo->inv_pattern_def_seq, new_stmt);
     217              : }
     218              : 
     219              : /* The caller wants to perform new operations on vect_external variable
     220              :    VAR, so that the result of the operations would also be vect_external.
     221              :    Return the edge on which the operations can be performed, if one exists.
     222              :    Return null if the operations should instead be treated as part of
     223              :    the pattern that needs them.  */
     224              : 
     225              : static edge
     226         9713 : vect_get_external_def_edge (vec_info *vinfo, tree var)
     227              : {
     228         9713 :   edge e = NULL;
     229         9713 :   if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
     230              :     {
     231          977 :       e = loop_preheader_edge (loop_vinfo->loop);
     232          977 :       if (!SSA_NAME_IS_DEFAULT_DEF (var))
     233              :         {
     234          773 :           basic_block bb = gimple_bb (SSA_NAME_DEF_STMT (var));
     235          773 :           if (bb == NULL
     236          773 :               || !dominated_by_p (CDI_DOMINATORS, e->dest, bb))
     237              :             e = NULL;
     238              :         }
     239              :     }
     240         9713 :   return e;
     241              : }
     242              : 
     243              : /* Return true if the target supports a vector version of CODE,
     244              :    where CODE is known to map to a direct optab with the given SUBTYPE.
     245              :    ITYPE specifies the type of (some of) the scalar inputs and OTYPE
     246              :    specifies the type of the scalar result.
     247              : 
     248              :    If CODE allows the inputs and outputs to have different type
     249              :    (such as for WIDEN_SUM_EXPR), it is the input mode rather
     250              :    than the output mode that determines the appropriate target pattern.
     251              :    Operand 0 of the target pattern then specifies the mode that the output
     252              :    must have.
     253              : 
     254              :    When returning true, set *VECOTYPE_OUT to the vector version of OTYPE.
     255              :    Also set *VECITYPE_OUT to the vector version of ITYPE if VECITYPE_OUT
     256              :    is nonnull.  */
     257              : 
     258              : static bool
     259           84 : vect_supportable_direct_optab_p (vec_info *vinfo, tree otype, tree_code code,
     260              :                                  tree itype, tree *vecotype_out,
     261              :                                  tree *vecitype_out = NULL,
     262              :                                  enum optab_subtype subtype = optab_default)
     263              : {
     264           84 :   tree vecitype = get_vectype_for_scalar_type (vinfo, itype);
     265           84 :   if (!vecitype)
     266              :     return false;
     267              : 
     268           82 :   tree vecotype = get_vectype_for_scalar_type (vinfo, otype);
     269           82 :   if (!vecotype)
     270              :     return false;
     271              : 
     272           82 :   optab optab = optab_for_tree_code (code, vecitype, subtype);
     273           82 :   if (!optab)
     274              :     return false;
     275              : 
     276           82 :   insn_code icode = optab_handler (optab, TYPE_MODE (vecitype));
     277           82 :   if (icode == CODE_FOR_nothing
     278           82 :       || insn_data[icode].operand[0].mode != TYPE_MODE (vecotype))
     279              :     return false;
     280              : 
     281           31 :   *vecotype_out = vecotype;
     282           31 :   if (vecitype_out)
     283            0 :     *vecitype_out = vecitype;
     284              :   return true;
     285              : }
     286              : 
     287              : /* Return true if the target supports a vector version of CODE,
     288              :    where CODE is known to map to a conversion optab with the given SUBTYPE.
     289              :    ITYPE specifies the type of (some of) the scalar inputs and OTYPE
     290              :    specifies the type of the scalar result.
     291              : 
     292              :    When returning true, set *VECOTYPE_OUT to the vector version of OTYPE.
     293              :    Also set *VECITYPE_OUT to the vector version of ITYPE if VECITYPE_OUT
     294              :    is nonnull.  */
     295              : 
     296              : static bool
     297         6694 : vect_supportable_conv_optab_p (vec_info *vinfo, tree otype, tree_code code,
     298              :                                  tree itype, tree *vecotype_out,
     299              :                                  tree *vecitype_out = NULL,
     300              :                                  enum optab_subtype subtype = optab_default)
     301              : {
     302         6694 :   tree vecitype = get_vectype_for_scalar_type (vinfo, itype);
     303         6694 :   tree vecotype = get_vectype_for_scalar_type (vinfo, otype);
     304         6694 :   if (!vecitype || !vecotype)
     305              :     return false;
     306              : 
     307         6156 :   if (!directly_supported_p (code, vecotype, vecitype, subtype))
     308              :     return false;
     309              : 
     310         1192 :   *vecotype_out = vecotype;
     311         1192 :   if (vecitype_out)
     312         1192 :     *vecitype_out = vecitype;
     313              :   return true;
     314              : }
     315              : 
     316              : /* Round bit precision PRECISION up to a full element.  */
     317              : 
     318              : static unsigned int
     319      3669826 : vect_element_precision (unsigned int precision)
     320              : {
     321       277327 :   precision = 1 << ceil_log2 (precision);
     322      5357138 :   return MAX (precision, BITS_PER_UNIT);
     323              : }
     324              : 
     325              : /* If OP is defined by a statement that's being considered for vectorization,
     326              :    return information about that statement, otherwise return NULL.  */
     327              : 
     328              : static stmt_vec_info
     329       422597 : vect_get_internal_def (vec_info *vinfo, tree op)
     330              : {
     331       422597 :   stmt_vec_info def_stmt_info = vinfo->lookup_def (op);
     332       422597 :   if (def_stmt_info
     333       408619 :       && STMT_VINFO_DEF_TYPE (def_stmt_info) == vect_internal_def)
     334       390352 :     return vect_stmt_to_vectorize (def_stmt_info);
     335              :   return NULL;
     336              : }
     337              : 
     338              : /* Holds information about an input operand after some sign changes
     339              :    and type promotions have been peeled away.  */
     340              : class vect_unpromoted_value {
     341              : public:
     342              :   vect_unpromoted_value ();
     343              : 
     344              :   void set_op (tree, vect_def_type, stmt_vec_info = NULL);
     345              : 
     346              :   /* The value obtained after peeling away zero or more casts.  */
     347              :   tree op;
     348              : 
     349              :   /* The type of OP.  */
     350              :   tree type;
     351              : 
     352              :   /* The definition type of OP.  */
     353              :   vect_def_type dt;
     354              : 
     355              :   /* If OP is the result of peeling at least one cast, and if the cast
     356              :      of OP itself is a vectorizable statement, CASTER identifies that
     357              :      statement, otherwise it is null.  */
     358              :   stmt_vec_info caster;
     359              : };
     360              : 
     361    315722518 : inline vect_unpromoted_value::vect_unpromoted_value ()
     362    315722518 :   : op (NULL_TREE),
     363    315722518 :     type (NULL_TREE),
     364    315722518 :     dt (vect_uninitialized_def),
     365      3360877 :     caster (NULL)
     366              : {
     367              : }
     368              : 
     369              : /* Set the operand to OP_IN, its definition type to DT_IN, and the
     370              :    statement that casts it to CASTER_IN.  */
     371              : 
     372              : inline void
     373     12051193 : vect_unpromoted_value::set_op (tree op_in, vect_def_type dt_in,
     374              :                                stmt_vec_info caster_in)
     375              : {
     376     12051193 :   op = op_in;
     377     12051193 :   type = TREE_TYPE (op);
     378     12051193 :   dt = dt_in;
     379     12051193 :   caster = caster_in;
     380     12051193 : }
     381              : 
     382              : /* If OP is a vectorizable SSA name, strip a sequence of integer conversions
     383              :    to reach some vectorizable inner operand OP', continuing as long as it
     384              :    is possible to convert OP' back to OP using a possible sign change
     385              :    followed by a possible promotion P.  Return this OP', or null if OP is
     386              :    not a vectorizable SSA name.  If there is a promotion P, describe its
     387              :    input in UNPROM, otherwise describe OP' in UNPROM.  If SINGLE_USE_P
     388              :    is nonnull, set *SINGLE_USE_P to false if any of the SSA names involved
     389              :    have more than one user.
     390              : 
     391              :    A successful return means that it is possible to go from OP' to OP
     392              :    via UNPROM.  The cast from OP' to UNPROM is at most a sign change,
     393              :    whereas the cast from UNPROM to OP might be a promotion, a sign
     394              :    change, or a nop.
     395              : 
     396              :    E.g. say we have:
     397              : 
     398              :        signed short *ptr = ...;
     399              :        signed short C = *ptr;
     400              :        unsigned short B = (unsigned short) C;    // sign change
     401              :        signed int A = (signed int) B;            // unsigned promotion
     402              :        ...possible other uses of A...
     403              :        unsigned int OP = (unsigned int) A;       // sign change
     404              : 
     405              :    In this case it's possible to go directly from C to OP using:
     406              : 
     407              :        OP = (unsigned int) (unsigned short) C;
     408              :             +------------+ +--------------+
     409              :                promotion      sign change
     410              : 
     411              :    so OP' would be C.  The input to the promotion is B, so UNPROM
     412              :    would describe B.  */
     413              : 
     414              : static tree
     415      8833480 : vect_look_through_possible_promotion (vec_info *vinfo, tree op,
     416              :                                       vect_unpromoted_value *unprom,
     417              :                                       bool *single_use_p = NULL)
     418              : {
     419      8833480 :   tree op_type = TREE_TYPE (op);
     420      8833480 :   if (!INTEGRAL_TYPE_P (op_type))
     421              :     return NULL_TREE;
     422              : 
     423      8799131 :   tree res = NULL_TREE;
     424      8799131 :   unsigned int orig_precision = TYPE_PRECISION (op_type);
     425      8799131 :   unsigned int min_precision = orig_precision;
     426      8799131 :   stmt_vec_info caster = NULL;
     427     10572439 :   while (TREE_CODE (op) == SSA_NAME && INTEGRAL_TYPE_P (op_type))
     428              :     {
     429              :       /* See whether OP is simple enough to vectorize.  */
     430     10344116 :       stmt_vec_info def_stmt_info;
     431     10344116 :       gimple *def_stmt;
     432     10344116 :       vect_def_type dt;
     433     10344116 :       if (!vect_is_simple_use (op, vinfo, &dt, &def_stmt_info, &def_stmt))
     434              :         break;
     435              : 
     436              :       /* If OP is the input of a demotion, skip over it to see whether
     437              :          OP is itself the result of a promotion.  If so, the combined
     438              :          effect of the promotion and the demotion might fit the required
     439              :          pattern, otherwise neither operation fits.
     440              : 
     441              :          This copes with cases such as the result of an arithmetic
     442              :          operation being truncated before being stored, and where that
     443              :          arithmetic operation has been recognized as an over-widened one.  */
     444     10326659 :       if (TYPE_PRECISION (op_type) <= min_precision)
     445              :         {
     446              :           /* Use OP as the UNPROM described above if we haven't yet
     447              :              found a promotion, or if using the new input preserves the
     448              :              sign of the previous promotion.  */
     449     10191051 :           if (!res
     450      1512300 :               || TYPE_PRECISION (unprom->type) == orig_precision
     451        40519 :               || TYPE_SIGN (unprom->type) == TYPE_SIGN (op_type)
     452     10228118 :               || (TYPE_UNSIGNED (op_type)
     453        25925 :                   && TYPE_PRECISION (op_type) < TYPE_PRECISION (unprom->type)))
     454              :             {
     455     10154443 :               unprom->set_op (op, dt, caster);
     456     10154443 :               min_precision = TYPE_PRECISION (op_type);
     457              :             }
     458              :           /* Stop if we've already seen a promotion and if this
     459              :              conversion does more than change the sign.  */
     460        36608 :           else if (TYPE_PRECISION (op_type)
     461        36608 :                    != TYPE_PRECISION (unprom->type))
     462              :             break;
     463              : 
     464              :           /* The sequence now extends to OP.  */
     465              :           res = op;
     466              :         }
     467              : 
     468              :       /* See whether OP is defined by a cast.  Record it as CASTER if
     469              :          the cast is potentially vectorizable.  */
     470     10326629 :       if (!def_stmt)
     471              :         break;
     472     10104502 :       caster = def_stmt_info;
     473              : 
     474              :       /* Ignore pattern statements, since we don't link uses for them.  */
     475     10104502 :       if (caster
     476     10104502 :           && single_use_p
     477      2044253 :           && !STMT_VINFO_RELATED_STMT (caster)
     478     11992854 :           && !has_single_use (res))
     479      1129218 :         *single_use_p = false;
     480              : 
     481     14960653 :       gassign *assign = dyn_cast <gassign *> (def_stmt);
     482      6389845 :       if (!assign || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def_stmt)))
     483              :         break;
     484              : 
     485              :       /* Continue with the input to the cast.  */
     486      1773308 :       op = gimple_assign_rhs1 (def_stmt);
     487      1773308 :       op_type = TREE_TYPE (op);
     488              :     }
     489              :   return res;
     490              : }
     491              : 
     492              : /* OP is an integer operand to an operation that returns TYPE, and we
     493              :    want to treat the operation as a widening one.  So far we can treat
     494              :    it as widening from *COMMON_TYPE.
     495              : 
     496              :    Return true if OP is suitable for such a widening operation,
     497              :    either widening from *COMMON_TYPE or from some supertype of it.
     498              :    Update *COMMON_TYPE to the supertype in the latter case.
     499              : 
     500              :    SHIFT_P is true if OP is a shift amount.  */
     501              : 
     502              : static bool
     503       357414 : vect_joust_widened_integer (tree type, bool shift_p, tree op,
     504              :                             tree *common_type)
     505              : {
     506              :   /* Calculate the minimum precision required by OP, without changing
     507              :      the sign of either operand.  */
     508       357414 :   unsigned int precision;
     509       357414 :   if (shift_p)
     510              :     {
     511        13493 :       if (!wi::leu_p (wi::to_widest (op), TYPE_PRECISION (type) / 2))
     512              :         return false;
     513        10875 :       precision = TREE_INT_CST_LOW (op);
     514              :     }
     515              :   else
     516              :     {
     517       343921 :       precision = wi::min_precision (wi::to_widest (op),
     518       343921 :                                      TYPE_SIGN (*common_type));
     519       343921 :       if (precision * 2 > TYPE_PRECISION (type))
     520              :         return false;
     521              :     }
     522              : 
     523              :   /* If OP requires a wider type, switch to that type.  The checks
     524              :      above ensure that this is still narrower than the result.  */
     525       338183 :   precision = vect_element_precision (precision);
     526       338183 :   if (TYPE_PRECISION (*common_type) < precision)
     527         6829 :     *common_type = build_nonstandard_integer_type
     528         6829 :       (precision, TYPE_UNSIGNED (*common_type));
     529              :   return true;
     530              : }
     531              : 
     532              : /* Return true if the common supertype of NEW_TYPE and *COMMON_TYPE
     533              :    is narrower than type, storing the supertype in *COMMON_TYPE if so.  */
     534              : 
     535              : static bool
     536        51987 : vect_joust_widened_type (tree type, tree new_type, tree *common_type)
     537              : {
     538        51987 :   if (types_compatible_p (*common_type, new_type))
     539              :     return true;
     540              : 
     541              :   /* See if *COMMON_TYPE can hold all values of NEW_TYPE.  */
     542         4819 :   if ((TYPE_PRECISION (new_type) < TYPE_PRECISION (*common_type))
     543         4819 :       && (TYPE_UNSIGNED (new_type) || !TYPE_UNSIGNED (*common_type)))
     544              :     return true;
     545              : 
     546              :   /* See if NEW_TYPE can hold all values of *COMMON_TYPE.  */
     547         3858 :   if (TYPE_PRECISION (*common_type) < TYPE_PRECISION (new_type)
     548         3858 :       && (TYPE_UNSIGNED (*common_type) || !TYPE_UNSIGNED (new_type)))
     549              :     {
     550          932 :       *common_type = new_type;
     551          932 :       return true;
     552              :     }
     553              : 
     554              :   /* We have mismatched signs, with the signed type being
     555              :      no wider than the unsigned type.  In this case we need
     556              :      a wider signed type.  */
     557         2926 :   unsigned int precision = MAX (TYPE_PRECISION (*common_type),
     558              :                                 TYPE_PRECISION (new_type));
     559         2926 :   precision *= 2;
     560              : 
     561         2926 :   if (precision * 2 > TYPE_PRECISION (type))
     562              :     return false;
     563              : 
     564           40 :   *common_type = build_nonstandard_integer_type (precision, false);
     565           40 :   return true;
     566              : }
     567              : 
     568              : /* If the range of UNPROM->OP at STMT fits in a narrower element type than
     569              :    UNPROM->TYPE, change UNPROM->TYPE to that type and return true.  */
     570              : 
     571              : static bool
     572       278597 : vect_narrow_unprom_to_range (vect_unpromoted_value *unprom, gimple *stmt)
     573              : {
     574       278597 :   int_range_max r;
     575       557194 :   get_range_query (cfun)->range_of_expr (r, unprom->op, stmt);
     576       278597 :   if (r.undefined_p ())
     577              :     return false;
     578              : 
     579       277327 :   signop sgn = TYPE_SIGN (unprom->type);
     580       277327 :   unsigned int precision
     581       831981 :     = vect_element_precision (MAX (wi::min_precision (r.lower_bound (), sgn),
     582              :                                    wi::min_precision (r.upper_bound (), sgn)));
     583       277327 :   if (precision >= TYPE_PRECISION (unprom->type))
     584              :     return false;
     585              : 
     586        35814 :   unprom->type = build_nonstandard_integer_type (precision, sgn == UNSIGNED);
     587        35814 :   return true;
     588       278597 : }
     589              : 
     590              : /* Check whether STMT_INFO can be viewed as a tree of integer operations
     591              :    in which each node either performs CODE or WIDENED_CODE, and where
     592              :    each leaf operand is narrower than the result of STMT_INFO.  MAX_NOPS
     593              :    specifies the maximum number of leaf operands.  SHIFT_P says whether
     594              :    CODE and WIDENED_CODE are some sort of shift.
     595              : 
     596              :    If STMT_INFO is such a tree, return the number of leaf operands
     597              :    and describe them in UNPROM[0] onwards.  Also set *COMMON_TYPE
     598              :    to a type that (a) is narrower than the result of STMT_INFO and
     599              :    (b) can hold all leaf operand values.
     600              : 
     601              :    If SUBTYPE then allow that the signs of the operands
     602              :    may differ in signs but not in precision.  SUBTYPE is updated to reflect
     603              :    this.
     604              : 
     605              :    Return 0 if STMT_INFO isn't such a tree, or if no such COMMON_TYPE
     606              :    exists.  */
     607              : 
     608              : static unsigned int
     609    132595124 : vect_widened_op_tree (vec_info *vinfo, stmt_vec_info stmt_info, tree_code code,
     610              :                       code_helper widened_code, bool shift_p,
     611              :                       unsigned int max_nops,
     612              :                       vect_unpromoted_value *unprom, tree *common_type,
     613              :                       enum optab_subtype *subtype = NULL)
     614              : {
     615              :   /* Check for an integer operation with the right code.  */
     616    132595124 :   gimple* stmt = stmt_info->stmt;
     617    132595124 :   if (!(is_gimple_assign (stmt) || is_gimple_call (stmt)))
     618              :     return 0;
     619              : 
     620    109123395 :   code_helper rhs_code;
     621    109123395 :   if (is_gimple_assign (stmt))
     622     93757397 :     rhs_code = gimple_assign_rhs_code (stmt);
     623     15365998 :   else if (is_gimple_call (stmt))
     624     15365998 :     rhs_code = gimple_call_combined_fn (stmt);
     625              :   else
     626              :     return 0;
     627              : 
     628    109123395 :   if (rhs_code != code
     629    109123395 :       && rhs_code != widened_code)
     630              :     return 0;
     631              : 
     632      6895255 :   tree lhs = gimple_get_lhs (stmt);
     633      6895255 :   tree type = TREE_TYPE (lhs);
     634      6895255 :   if (!INTEGRAL_TYPE_P (type))
     635              :     return 0;
     636              : 
     637              :   /* Assume that both operands will be leaf operands.  */
     638      6003006 :   max_nops -= 2;
     639              : 
     640              :   /* Check the operands.  */
     641      6003006 :   unsigned int next_op = 0;
     642      6896820 :   for (unsigned int i = 0; i < 2; ++i)
     643              :     {
     644      6509131 :       vect_unpromoted_value *this_unprom = &unprom[next_op];
     645      6509131 :       unsigned int nops = 1;
     646      6509131 :       tree op = gimple_arg (stmt, i);
     647      6509131 :       if (i == 1 && TREE_CODE (op) == INTEGER_CST)
     648              :         {
     649              :           /* We already have a common type from earlier operands.
     650              :              Update it to account for OP.  */
     651       357414 :           this_unprom->set_op (op, vect_constant_def);
     652       357414 :           if (!vect_joust_widened_integer (type, shift_p, op, common_type))
     653              :             return 0;
     654              :         }
     655              :       else
     656              :         {
     657              :           /* Only allow shifts by constants.  */
     658      6151717 :           if (shift_p && i == 1)
     659              :             return 0;
     660              : 
     661      6145576 :           if (rhs_code != code)
     662              :             {
     663              :               /* If rhs_code is widened_code, don't look through further
     664              :                  possible promotions, there is a promotion already embedded
     665              :                  in the WIDEN_*_EXPR.  */
     666         1711 :               if (TREE_CODE (op) != SSA_NAME
     667         1711 :                   || !INTEGRAL_TYPE_P (TREE_TYPE (op)))
     668            0 :                 return 0;
     669              : 
     670         1711 :               stmt_vec_info def_stmt_info;
     671         1711 :               gimple *def_stmt;
     672         1711 :               vect_def_type dt;
     673         1711 :               if (!vect_is_simple_use (op, vinfo, &dt, &def_stmt_info,
     674              :                                        &def_stmt))
     675              :                 return 0;
     676         1711 :               this_unprom->set_op (op, dt, NULL);
     677              :             }
     678      6143865 :           else if (!vect_look_through_possible_promotion (vinfo, op,
     679              :                                                           this_unprom))
     680              :             return 0;
     681              : 
     682      6029741 :           if (TYPE_PRECISION (this_unprom->type) == TYPE_PRECISION (type)
     683      6308338 :               && !(this_unprom->dt == vect_external_def
     684       278597 :                    && vect_narrow_unprom_to_range (this_unprom, stmt)))
     685              :             {
     686              :               /* The operand isn't widened.  If STMT_INFO has the code
     687              :                  for an unwidened operation, recursively check whether
     688              :                  this operand is a node of the tree.  */
     689      5467519 :               if (rhs_code != code
     690      5467519 :                   || max_nops == 0
     691      5468070 :                   || this_unprom->dt != vect_internal_def)
     692              :                 return 0;
     693              : 
     694              :               /* Give back the leaf slot allocated above now that we're
     695              :                  not treating this as a leaf operand.  */
     696          551 :               max_nops += 1;
     697              : 
     698              :               /* Recursively process the definition of the operand.  */
     699          551 :               stmt_vec_info def_stmt_info
     700          551 :                 = vect_get_internal_def (vinfo, this_unprom->op);
     701              : 
     702          551 :               nops = vect_widened_op_tree (vinfo, def_stmt_info, code,
     703              :                                            widened_code, shift_p, max_nops,
     704              :                                            this_unprom, common_type,
     705              :                                            subtype);
     706          551 :               if (nops == 0)
     707              :                 return 0;
     708              : 
     709          367 :               max_nops -= nops;
     710              :             }
     711              :           else
     712              :             {
     713              :               /* Make sure that the operand is narrower than the result.  */
     714       562222 :               if (TYPE_PRECISION (this_unprom->type) * 2
     715       562222 :                   > TYPE_PRECISION (type))
     716              :                 return 0;
     717              : 
     718              :               /* Update COMMON_TYPE for the new operand.  */
     719       557859 :               if (i == 0)
     720       505872 :                 *common_type = this_unprom->type;
     721        51987 :               else if (!vect_joust_widened_type (type, this_unprom->type,
     722              :                                                  common_type))
     723              :                 {
     724         2886 :                   if (subtype)
     725              :                     {
     726              :                       /* See if we can sign extend the smaller type.  */
     727          291 :                       if (TYPE_PRECISION (this_unprom->type)
     728          291 :                           > TYPE_PRECISION (*common_type))
     729           27 :                         *common_type = this_unprom->type;
     730          291 :                       *subtype = optab_vector_mixed_sign;
     731              :                     }
     732              :                   else
     733              :                     return 0;
     734              :                 }
     735              :             }
     736              :         }
     737       893814 :       next_op += nops;
     738              :     }
     739              :   return next_op;
     740              : }
     741              : 
     742              : /* Helper to return a new temporary for pattern of TYPE for STMT.  If STMT
     743              :    is NULL, the caller must set SSA_NAME_DEF_STMT for the returned SSA var. */
     744              : 
     745              : static tree
     746      2051897 : vect_recog_temp_ssa_var (tree type, gimple *stmt = NULL)
     747              : {
     748            0 :   return make_temp_ssa_name (type, stmt, "patt");
     749              : }
     750              : 
     751              : /* STMT2_INFO describes a type conversion that could be split into STMT1
     752              :    followed by a version of STMT2_INFO that takes NEW_RHS as its first
     753              :    input.  Try to do this using pattern statements, returning true on
     754              :    success.  */
     755              : 
     756              : static bool
     757        29005 : vect_split_statement (vec_info *vinfo, stmt_vec_info stmt2_info, tree new_rhs,
     758              :                       gimple *stmt1, tree vectype)
     759              : {
     760        29005 :   if (is_pattern_stmt_p (stmt2_info))
     761              :     {
     762              :       /* STMT2_INFO is part of a pattern.  Get the statement to which
     763              :          the pattern is attached.  */
     764          493 :       stmt_vec_info orig_stmt2_info = STMT_VINFO_RELATED_STMT (stmt2_info);
     765          493 :       vect_init_pattern_stmt (vinfo, stmt1, orig_stmt2_info, vectype);
     766              : 
     767          493 :       if (dump_enabled_p ())
     768           19 :         dump_printf_loc (MSG_NOTE, vect_location,
     769              :                          "Splitting pattern statement: %G", stmt2_info->stmt);
     770              : 
     771              :       /* Since STMT2_INFO is a pattern statement, we can change it
     772              :          in-situ without worrying about changing the code for the
     773              :          containing block.  */
     774          493 :       gimple_assign_set_rhs1 (stmt2_info->stmt, new_rhs);
     775              : 
     776          493 :       if (dump_enabled_p ())
     777              :         {
     778           19 :           dump_printf_loc (MSG_NOTE, vect_location, "into: %G", stmt1);
     779           19 :           dump_printf_loc (MSG_NOTE, vect_location, "and: %G",
     780              :                            stmt2_info->stmt);
     781              :         }
     782              : 
     783          493 :       gimple_seq *def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt2_info);
     784          493 :       if (STMT_VINFO_RELATED_STMT (orig_stmt2_info) == stmt2_info)
     785              :         /* STMT2_INFO is the actual pattern statement.  Add STMT1
     786              :            to the end of the definition sequence.  */
     787          484 :         gimple_seq_add_stmt_without_update (def_seq, stmt1);
     788              :       else
     789              :         {
     790              :           /* STMT2_INFO belongs to the definition sequence.  Insert STMT1
     791              :              before it.  */
     792            9 :           gimple_stmt_iterator gsi = gsi_for_stmt (stmt2_info->stmt, def_seq);
     793            9 :           gsi_insert_before_without_update (&gsi, stmt1, GSI_SAME_STMT);
     794              :         }
     795              :       return true;
     796              :     }
     797              :   else
     798              :     {
     799              :       /* STMT2_INFO doesn't yet have a pattern.  Try to create a
     800              :          two-statement pattern now.  */
     801        28512 :       gcc_assert (!STMT_VINFO_RELATED_STMT (stmt2_info));
     802        28512 :       tree lhs_type = TREE_TYPE (gimple_get_lhs (stmt2_info->stmt));
     803        28512 :       tree lhs_vectype = get_vectype_for_scalar_type (vinfo, lhs_type);
     804        28512 :       if (!lhs_vectype)
     805              :         return false;
     806              : 
     807        28512 :       if (dump_enabled_p ())
     808         2059 :         dump_printf_loc (MSG_NOTE, vect_location,
     809              :                          "Splitting statement: %G", stmt2_info->stmt);
     810              : 
     811              :       /* Add STMT1 as a singleton pattern definition sequence.  */
     812        28512 :       gimple_seq *def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (stmt2_info);
     813        28512 :       vect_init_pattern_stmt (vinfo, stmt1, stmt2_info, vectype);
     814        28512 :       gimple_seq_add_stmt_without_update (def_seq, stmt1);
     815              : 
     816              :       /* Build the second of the two pattern statements.  */
     817        28512 :       tree new_lhs = vect_recog_temp_ssa_var (lhs_type, NULL);
     818        28512 :       gassign *new_stmt2 = gimple_build_assign (new_lhs, NOP_EXPR, new_rhs);
     819        28512 :       vect_set_pattern_stmt (vinfo, new_stmt2, stmt2_info, lhs_vectype);
     820              : 
     821        28512 :       if (dump_enabled_p ())
     822              :         {
     823         2059 :           dump_printf_loc (MSG_NOTE, vect_location,
     824              :                            "into pattern statements: %G", stmt1);
     825         2059 :           dump_printf_loc (MSG_NOTE, vect_location, "and: %G",
     826              :                            (gimple *) new_stmt2);
     827              :         }
     828              : 
     829              :       return true;
     830              :     }
     831              : }
     832              : 
     833              : /* Look for the following pattern
     834              :         X = x[i]
     835              :         Y = y[i]
     836              :         DIFF = X - Y
     837              :         DAD = ABS_EXPR<DIFF>
     838              : 
     839              :    ABS_STMT should point to a statement of code ABS_EXPR or ABSU_EXPR.
     840              :    HALF_TYPE and UNPROM will be set should the statement be found to
     841              :    be a widened operation.
     842              :    DIFF_STMT will be set to the MINUS_EXPR
     843              :    statement that precedes the ABS_STMT if it is a MINUS_EXPR..
     844              :  */
     845              : static bool
     846     23132316 : vect_recog_absolute_difference (vec_info *vinfo, gassign *abs_stmt,
     847              :                                 tree *half_type,
     848              :                                 vect_unpromoted_value unprom[2],
     849              :                                 gassign **diff_stmt)
     850              : {
     851     23132316 :   if (!abs_stmt)
     852              :     return false;
     853              : 
     854              :   /* FORNOW.  Can continue analyzing the def-use chain when this stmt in a phi
     855              :      inside the loop (in case we are analyzing an outer-loop).  */
     856     23132316 :   enum tree_code code = gimple_assign_rhs_code (abs_stmt);
     857     23132316 :   if (code != ABS_EXPR && code != ABSU_EXPR)
     858              :     return false;
     859              : 
     860        34421 :   tree abs_oprnd = gimple_assign_rhs1 (abs_stmt);
     861        34421 :   tree abs_type = TREE_TYPE (abs_oprnd);
     862        34421 :   if (!abs_oprnd)
     863              :     return false;
     864        27008 :   if (!ANY_INTEGRAL_TYPE_P (abs_type)
     865         7664 :       || TYPE_OVERFLOW_WRAPS (abs_type)
     866        41925 :       || TYPE_UNSIGNED (abs_type))
     867              :     return false;
     868              : 
     869              :   /* Peel off conversions from the ABS input.  This can involve sign
     870              :      changes (e.g. from an unsigned subtraction to a signed ABS input)
     871              :      or signed promotion, but it can't include unsigned promotion.
     872              :      (Note that ABS of an unsigned promotion should have been folded
     873              :      away before now anyway.)  */
     874         7504 :   vect_unpromoted_value unprom_diff;
     875         7504 :   abs_oprnd = vect_look_through_possible_promotion (vinfo, abs_oprnd,
     876              :                                                     &unprom_diff);
     877         7504 :   if (!abs_oprnd)
     878              :     return false;
     879         7253 :   if (TYPE_PRECISION (unprom_diff.type) != TYPE_PRECISION (abs_type)
     880         7253 :       && TYPE_UNSIGNED (unprom_diff.type))
     881              :     return false;
     882              : 
     883              :   /* We then detect if the operand of abs_expr is defined by a minus_expr.  */
     884         7253 :   stmt_vec_info diff_stmt_vinfo = vect_get_internal_def (vinfo, abs_oprnd);
     885         7253 :   if (!diff_stmt_vinfo)
     886              :     return false;
     887              : 
     888         7041 :   gassign *diff = dyn_cast <gassign *> (STMT_VINFO_STMT (diff_stmt_vinfo));
     889         7041 :   if (diff_stmt && diff
     890         5799 :       && gimple_assign_rhs_code (diff) == MINUS_EXPR
     891         8894 :       && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (abs_oprnd)))
     892          251 :     *diff_stmt = diff;
     893              : 
     894              :   /* FORNOW.  Can continue analyzing the def-use chain when this stmt in a phi
     895              :      inside the loop (in case we are analyzing an outer-loop).  */
     896         7041 :   if (vect_widened_op_tree (vinfo, diff_stmt_vinfo,
     897         7041 :                             MINUS_EXPR, IFN_VEC_WIDEN_MINUS,
     898              :                             false, 2, unprom, half_type))
     899         2408 :     return true;
     900              : 
     901              :   return false;
     902              : }
     903              : 
     904              : /* Convert UNPROM to TYPE and return the result, adding new statements
     905              :    to STMT_INFO's pattern definition statements if no better way is
     906              :    available.  VECTYPE is the vector form of TYPE.
     907              : 
     908              :    If SUBTYPE then convert the type based on the subtype.  */
     909              : 
     910              : static tree
     911       520266 : vect_convert_input (vec_info *vinfo, stmt_vec_info stmt_info, tree type,
     912              :                     vect_unpromoted_value *unprom, tree vectype,
     913              :                     enum optab_subtype subtype = optab_default)
     914              : {
     915              :   /* Update the type if the signs differ.  */
     916       520266 :   if (subtype == optab_vector_mixed_sign)
     917              :     {
     918          328 :       gcc_assert (!TYPE_UNSIGNED (type));
     919          328 :       if (TYPE_UNSIGNED (TREE_TYPE (unprom->op)))
     920              :         {
     921          164 :           type = unsigned_type_for (type);
     922          164 :           vectype = unsigned_type_for (vectype);
     923              :         }
     924              :     }
     925              : 
     926              :   /* Check for a no-op conversion.  */
     927       520266 :   if (types_compatible_p (type, TREE_TYPE (unprom->op)))
     928       183984 :     return unprom->op;
     929              : 
     930              :   /* Allow the caller to create constant vect_unpromoted_values.  */
     931       336282 :   if (TREE_CODE (unprom->op) == INTEGER_CST)
     932       208606 :     return wide_int_to_tree (type, wi::to_widest (unprom->op));
     933              : 
     934       127676 :   tree input = unprom->op;
     935       127676 :   if (unprom->caster)
     936              :     {
     937        67498 :       tree lhs = gimple_get_lhs (unprom->caster->stmt);
     938        67498 :       tree lhs_type = TREE_TYPE (lhs);
     939              : 
     940              :       /* If the result of the existing cast is the right width, use it
     941              :          instead of the source of the cast.  */
     942        67498 :       if (TYPE_PRECISION (lhs_type) == TYPE_PRECISION (type))
     943              :         input = lhs;
     944              :       /* If the precision we want is between the source and result
     945              :          precisions of the existing cast, try splitting the cast into
     946              :          two and tapping into a mid-way point.  */
     947        65213 :       else if (TYPE_PRECISION (lhs_type) > TYPE_PRECISION (type)
     948        65213 :                && TYPE_PRECISION (type) > TYPE_PRECISION (unprom->type))
     949              :         {
     950              :           /* In order to preserve the semantics of the original cast,
     951              :              give the mid-way point the same signedness as the input value.
     952              : 
     953              :              It would be possible to use a signed type here instead if
     954              :              TYPE is signed and UNPROM->TYPE is unsigned, but that would
     955              :              make the sign of the midtype sensitive to the order in
     956              :              which we process the statements, since the signedness of
     957              :              TYPE is the signedness required by just one of possibly
     958              :              many users.  Also, unsigned promotions are usually as cheap
     959              :              as or cheaper than signed ones, so it's better to keep an
     960              :              unsigned promotion.  */
     961        29005 :           tree midtype = build_nonstandard_integer_type
     962        29005 :             (TYPE_PRECISION (type), TYPE_UNSIGNED (unprom->type));
     963        29005 :           tree vec_midtype = get_vectype_for_scalar_type (vinfo, midtype);
     964        29005 :           if (vec_midtype)
     965              :             {
     966        29005 :               input = vect_recog_temp_ssa_var (midtype, NULL);
     967        29005 :               gassign *new_stmt = gimple_build_assign (input, NOP_EXPR,
     968              :                                                        unprom->op);
     969        29005 :               if (!vect_split_statement (vinfo, unprom->caster, input, new_stmt,
     970              :                                          vec_midtype))
     971            0 :                 append_pattern_def_seq (vinfo, stmt_info,
     972              :                                         new_stmt, vec_midtype);
     973              :             }
     974              :         }
     975              : 
     976              :       /* See if we can reuse an existing result.  */
     977        67498 :       if (types_compatible_p (type, TREE_TYPE (input)))
     978              :         return input;
     979              :     }
     980              : 
     981              :   /* We need a new conversion statement.  */
     982       106771 :   tree new_op = vect_recog_temp_ssa_var (type, NULL);
     983       106771 :   gassign *new_stmt = gimple_build_assign (new_op, NOP_EXPR, input);
     984              : 
     985              :   /* If OP is an external value, see if we can insert the new statement
     986              :      on an incoming edge.  */
     987       106771 :   if (input == unprom->op && unprom->dt == vect_external_def)
     988         9698 :     if (edge e = vect_get_external_def_edge (vinfo, input))
     989              :       {
     990          962 :         basic_block new_bb = gsi_insert_on_edge_immediate (e, new_stmt);
     991          962 :         gcc_assert (!new_bb);
     992              :         return new_op;
     993              :       }
     994              : 
     995              :   /* As a (common) last resort, add the statement to the pattern itself.  */
     996       105809 :   append_pattern_def_seq (vinfo, stmt_info, new_stmt, vectype);
     997       105809 :   return new_op;
     998              : }
     999              : 
    1000              : /* Invoke vect_convert_input for N elements of UNPROM and store the
    1001              :    result in the corresponding elements of RESULT.
    1002              : 
    1003              :    If SUBTYPE then convert the type based on the subtype.  */
    1004              : 
    1005              : static void
    1006       264230 : vect_convert_inputs (vec_info *vinfo, stmt_vec_info stmt_info, unsigned int n,
    1007              :                      tree *result, tree type, vect_unpromoted_value *unprom,
    1008              :                      tree vectype, enum optab_subtype subtype = optab_default)
    1009              : {
    1010       784577 :   for (unsigned int i = 0; i < n; ++i)
    1011              :     {
    1012              :       unsigned int j;
    1013       775875 :       for (j = 0; j < i; ++j)
    1014       256117 :         if (unprom[j].op == unprom[i].op)
    1015              :           break;
    1016              : 
    1017       520347 :       if (j < i)
    1018          589 :         result[i] = result[j];
    1019              :       else
    1020       519758 :         result[i] = vect_convert_input (vinfo, stmt_info,
    1021       519758 :                                         type, &unprom[i], vectype, subtype);
    1022              :     }
    1023       264230 : }
    1024              : 
    1025              : /* The caller has created a (possibly empty) sequence of pattern definition
    1026              :    statements followed by a single statement PATTERN_STMT.  Cast the result
    1027              :    of this final statement to TYPE.  If a new statement is needed, add
    1028              :    PATTERN_STMT to the end of STMT_INFO's pattern definition statements
    1029              :    and return the new statement, otherwise return PATTERN_STMT as-is.
    1030              :    VECITYPE is the vector form of PATTERN_STMT's result type.  */
    1031              : 
    1032              : static gimple *
    1033       291127 : vect_convert_output (vec_info *vinfo, stmt_vec_info stmt_info, tree type,
    1034              :                      gimple *pattern_stmt, tree vecitype)
    1035              : {
    1036       291127 :   tree lhs = gimple_get_lhs (pattern_stmt);
    1037       291127 :   if (!types_compatible_p (type, TREE_TYPE (lhs)))
    1038              :     {
    1039       259595 :       append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vecitype);
    1040       259595 :       tree cast_var = vect_recog_temp_ssa_var (type, NULL);
    1041       259595 :       pattern_stmt = gimple_build_assign (cast_var, NOP_EXPR, lhs);
    1042              :     }
    1043       291127 :   return pattern_stmt;
    1044              : }
    1045              : 
    1046              : /* Return true if STMT_VINFO describes a reduction for which reassociation
    1047              :    is allowed.  If STMT_INFO is part of a group, assume that it's part of
    1048              :    a reduction chain and optimistically assume that all statements
    1049              :    except the last allow reassociation.
    1050              :    Also require it to have code CODE and to be a reduction
    1051              :    in the outermost loop.  When returning true, store the operands in
    1052              :    *OP0_OUT and *OP1_OUT.  */
    1053              : 
    1054              : static bool
    1055     99163344 : vect_reassociating_reduction_p (vec_info *vinfo,
    1056              :                                 stmt_vec_info stmt_info, tree_code code,
    1057              :                                 tree *op0_out, tree *op1_out)
    1058              : {
    1059     99163344 :   loop_vec_info loop_info = dyn_cast <loop_vec_info> (vinfo);
    1060     13989699 :   if (!loop_info)
    1061              :     return false;
    1062              : 
    1063     13989699 :   gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
    1064     15255672 :   if (!assign || gimple_assign_rhs_code (assign) != code)
    1065              :     return false;
    1066              : 
    1067              :   /* We don't allow changing the order of the computation in the inner-loop
    1068              :      when doing outer-loop vectorization.  */
    1069      2718632 :   class loop *loop = LOOP_VINFO_LOOP (loop_info);
    1070      2718632 :   if (loop && nested_in_vect_loop_p (loop, stmt_info))
    1071              :     return false;
    1072              : 
    1073      2664608 :   if (!vect_is_reduction (stmt_info))
    1074              :     return false;
    1075              : 
    1076       174821 :   if (needs_fold_left_reduction_p (TREE_TYPE (gimple_assign_lhs (assign)),
    1077       174821 :                                    code))
    1078              :     return false;
    1079              : 
    1080       161234 :   *op0_out = gimple_assign_rhs1 (assign);
    1081       161234 :   *op1_out = gimple_assign_rhs2 (assign);
    1082       161234 :   if (commutative_tree_code (code) && STMT_VINFO_REDUC_IDX (stmt_info) == 0)
    1083        64149 :     std::swap (*op0_out, *op1_out);
    1084              :   return true;
    1085              : }
    1086              : 
    1087              : /* Return true iff the target has a vector optab implementing the operation
    1088              :    CODE on type VECTYPE with SUBTYPE.  */
    1089              : 
    1090              : static bool
    1091       908694 : target_has_vecop_for_code (tree_code code, tree vectype,
    1092              :                            enum optab_subtype subtype = optab_vector)
    1093              : {
    1094       908694 :   optab voptab = optab_for_tree_code (code, vectype, subtype);
    1095       908694 :   return voptab && can_implement_p (voptab, TYPE_MODE (vectype));
    1096              : }
    1097              : 
    1098              : /* match.pd function to match
    1099              :    (cond (cmp@3 a b) (convert@1 c) (convert@2 d))
    1100              :    with conditions:
    1101              :    1) @1, @2, c, d, a, b are all integral type.
    1102              :    2) There's single_use for both @1 and @2.
    1103              :    3) a, c have same precision.
    1104              :    4) c and @1 have different precision.
    1105              :    5) c, d are the same type or they can differ in sign when convert is
    1106              :    truncation.
    1107              : 
    1108              :    record a and c and d and @3.  */
    1109              : 
    1110              : extern bool gimple_cond_expr_convert_p (tree, tree*, tree (*)(tree));
    1111              : 
    1112              : /* Function vect_recog_cond_expr_convert
    1113              : 
    1114              :    Try to find the following pattern:
    1115              : 
    1116              :    TYPE_AB A,B;
    1117              :    TYPE_CD C,D;
    1118              :    TYPE_E E;
    1119              :    TYPE_E op_true = (TYPE_E) A;
    1120              :    TYPE_E op_false = (TYPE_E) B;
    1121              : 
    1122              :    E = C cmp D ? op_true : op_false;
    1123              : 
    1124              :    where
    1125              :    TYPE_PRECISION (TYPE_E) != TYPE_PRECISION (TYPE_CD);
    1126              :    TYPE_PRECISION (TYPE_AB) == TYPE_PRECISION (TYPE_CD);
    1127              :    single_use of op_true and op_false.
    1128              :    TYPE_AB could differ in sign when (TYPE_E) A is a truncation.
    1129              : 
    1130              :    Input:
    1131              : 
    1132              :    * STMT_VINFO: The stmt from which the pattern search begins.
    1133              :    here it starts with E = c cmp D ? op_true : op_false;
    1134              : 
    1135              :    Output:
    1136              : 
    1137              :    TYPE1 E' = C cmp D ? A : B;
    1138              :    TYPE3 E = (TYPE3) E';
    1139              : 
    1140              :    There may extra nop_convert for A or B to handle different signness.
    1141              : 
    1142              :    * TYPE_OUT: The vector type of the output of this pattern.
    1143              : 
    1144              :    * Return value: A new stmt that will be used to replace the sequence of
    1145              :    stmts that constitute the pattern. In this case it will be:
    1146              :    E = (TYPE3)E';
    1147              :    E' = C cmp D ? A : B; is recorded in pattern definition statements;  */
    1148              : 
    1149              : static gimple *
    1150     33136945 : vect_recog_cond_expr_convert_pattern (vec_info *vinfo,
    1151              :                                       stmt_vec_info stmt_vinfo, tree *type_out)
    1152              : {
    1153     33136945 :   gassign *last_stmt = dyn_cast <gassign *> (stmt_vinfo->stmt);
    1154     23234096 :   tree lhs, match[4], temp, type, new_lhs, op2, op1;
    1155     23234096 :   gimple *cond_stmt;
    1156     23234096 :   gimple *pattern_stmt;
    1157     56371041 :   enum tree_code code = NOP_EXPR;
    1158              : 
    1159     23234096 :   if (!last_stmt)
    1160              :     return NULL;
    1161              : 
    1162     23234096 :   lhs = gimple_assign_lhs (last_stmt);
    1163              : 
    1164              :   /* Find E = C cmp D ? (TYPE3) A ? (TYPE3) B;
    1165              :      TYPE_PRECISION (A) == TYPE_PRECISION (C).  */
    1166     23234096 :   if (!gimple_cond_expr_convert_p (lhs, &match[0], NULL))
    1167              :     return NULL;
    1168              : 
    1169           44 :   if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (lhs)))
    1170           20 :     code = INTEGRAL_TYPE_P (TREE_TYPE (match[1])) ? FLOAT_EXPR : CONVERT_EXPR;
    1171           24 :   else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (match[1])))
    1172            0 :     code = FIX_TRUNC_EXPR;
    1173              : 
    1174           44 :   op1 = match[1];
    1175           44 :   op2 = match[2];
    1176           44 :   type = TREE_TYPE (op1);
    1177              :   /* When op1/op2 is REAL_CST, the conversion must be CONVERT_EXPR from
    1178              :      SCALAR_FLOAT_TYPE_P which is restricted in gimple_cond_expr_convert_p.
    1179              :      Otherwise, the conversion could be FLOAT_EXPR, FIX_TRUNC_EXPR
    1180              :      or CONVERT_EXPR.  */
    1181           44 :   if (TREE_CODE (op1) == REAL_CST)
    1182              :     {
    1183           20 :       op1 = const_unop (CONVERT_EXPR, TREE_TYPE (op2), op1);
    1184           20 :       type = TREE_TYPE (op2);
    1185           20 :       if (op1 == NULL_TREE)
    1186              :         return NULL;
    1187              :     }
    1188           24 :   else if (TREE_CODE (op2) == REAL_CST)
    1189              :     {
    1190            0 :       op2 = const_unop (FLOAT_EXPR, TREE_TYPE (op1), op2);
    1191            0 :       if (op2 == NULL_TREE)
    1192              :         return NULL;
    1193              :     }
    1194           24 :   else if (code == NOP_EXPR)
    1195              :     {
    1196           24 :       if (TYPE_SIGN (type) != TYPE_SIGN (TREE_TYPE (match[2])))
    1197              :         {
    1198            9 :           op2 = vect_recog_temp_ssa_var (type, NULL);
    1199            9 :           gimple* nop_stmt = gimple_build_assign (op2, NOP_EXPR, match[2]);
    1200            9 :           append_pattern_def_seq (vinfo, stmt_vinfo, nop_stmt);
    1201              :         }
    1202              :     }
    1203              : 
    1204           44 :   vect_pattern_detected ("vect_recog_cond_expr_convert_pattern", last_stmt);
    1205              : 
    1206           44 :   temp = vect_recog_temp_ssa_var (type, NULL);
    1207           44 :   cond_stmt = gimple_build_assign (temp, build3 (COND_EXPR, type, match[3],
    1208              :                                                  op1, op2));
    1209           44 :   append_pattern_def_seq (vinfo, stmt_vinfo, cond_stmt);
    1210           44 :   new_lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
    1211           44 :   pattern_stmt = gimple_build_assign (new_lhs, code, temp);
    1212           44 :   *type_out = NULL_TREE;
    1213              : 
    1214           44 :   if (dump_enabled_p ())
    1215           24 :     dump_printf_loc (MSG_NOTE, vect_location,
    1216              :                      "created pattern stmt: %G", pattern_stmt);
    1217              :   return pattern_stmt;
    1218              : }
    1219              : 
    1220              : /* Function vect_recog_dot_prod_pattern
    1221              : 
    1222              :    Try to find the following pattern:
    1223              : 
    1224              :      type1a x_t
    1225              :      type1b y_t;
    1226              :      TYPE1 prod;
    1227              :      TYPE2 sum = init;
    1228              :    loop:
    1229              :      sum_0 = phi <init, sum_1>
    1230              :      S1  x_t = ...
    1231              :      S2  y_t = ...
    1232              :      S3  x_T = (TYPE1) x_t;
    1233              :      S4  y_T = (TYPE1) y_t;
    1234              :      S5  prod = x_T * y_T;
    1235              :      [S6  prod = (TYPE2) prod;  #optional]
    1236              :      S7  sum_1 = prod + sum_0;
    1237              : 
    1238              :    For integer reductions, 'TYPE1' is exactly double the size of 'type1a'
    1239              :    and 'type1b', and its sign must match one of them.  Floating-point
    1240              :    reductions require matching input types at most half the width of TYPE1,
    1241              :    with the inputs converted before multiplication, and TYPE2 the same as TYPE1.
    1242              : 
    1243              :    Input:
    1244              : 
    1245              :    * STMT_VINFO: The stmt from which the pattern search begins.  In the
    1246              :    example, when this function is called with S7, the pattern {S3,S4,S5,S6,S7}
    1247              :    will be detected.
    1248              : 
    1249              :    Output:
    1250              : 
    1251              :    * TYPE_OUT: The type of the output  of this pattern.
    1252              : 
    1253              :    * Return value: A new stmt that will be used to replace the sequence of
    1254              :    stmts that constitute the pattern. In this case it will be:
    1255              :         WIDEN_DOT_PRODUCT <x_t, y_t, sum_0>
    1256              : 
    1257              :    Note: The dot-prod idiom is a widening reduction pattern that is
    1258              :          vectorized without preserving all the intermediate results. It
    1259              :          produces only N/2 (widened) results (by summing up pairs of
    1260              :          intermediate results) rather than all N results.  Therefore, we
    1261              :          cannot allow this pattern when we want to get all the results and in
    1262              :          the correct order (as is the case when this computation is in an
    1263              :          inner-loop nested in an outer-loop that us being vectorized).  */
    1264              : 
    1265              : static gimple *
    1266     33055086 : vect_recog_dot_prod_pattern (vec_info *vinfo,
    1267              :                              stmt_vec_info stmt_vinfo, tree *type_out)
    1268              : {
    1269     33055086 :   tree oprnd0, oprnd1;
    1270     33055086 :   gimple *last_stmt = stmt_vinfo->stmt;
    1271     33055086 :   tree type, half_type;
    1272     33055086 :   gimple *pattern_stmt;
    1273     33055086 :   tree var;
    1274              : 
    1275              :   /* Look for the following pattern
    1276              :           DX = (TYPE1) X;
    1277              :           DY = (TYPE1) Y;
    1278              :           DPROD = DX * DY;
    1279              :           DDPROD = (TYPE2) DPROD;
    1280              :           sum_1 = DDPROD + sum_0;
    1281              :      In which
    1282              :      - DX and DY are at least twice as wide as X and Y (exactly twice for
    1283              :        integers).
    1284              :      - DX, DY, DPROD all have the same type but the sign
    1285              :        between X, Y and DPROD can differ.
    1286              :      - sum is the same size of DPROD or bigger
    1287              :      - sum has been recognized as a reduction variable.
    1288              : 
    1289              :      This is equivalent to:
    1290              :        DPROD = X w* Y;          #widen mult
    1291              :        sum_1 = DPROD w+ sum_0;  #widen summation
    1292              :      or
    1293              :        DPROD = X w* Y;          #widen mult
    1294              :        sum_1 = DPROD + sum_0;   #summation
    1295              :    */
    1296              : 
    1297              :   /* Starting from LAST_STMT, follow the defs of its uses in search
    1298              :      of the above pattern.  */
    1299              : 
    1300     33055086 :   if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, PLUS_EXPR,
    1301              :                                        &oprnd0, &oprnd1))
    1302              :     return NULL;
    1303              : 
    1304        54386 :   type = TREE_TYPE (gimple_get_lhs (last_stmt));
    1305              : 
    1306              :   /* So far so good.  Since LAST_STMT is a summation reduction, OPRND1 is
    1307              :      the reduction variable defined by a loop-header phi.  OPRND0 is the
    1308              :      value added to it, which must come from a multiplication.  For integer
    1309              :      reductions, first look through a possible promotion of OPRND0.  */
    1310        54386 :   bool float_p = SCALAR_FLOAT_TYPE_P (type);
    1311        54386 :   enum optab_subtype subtype = optab_vector;
    1312        54386 :   vect_unpromoted_value unprom_mult;
    1313        54386 :   if (!float_p)
    1314        37336 :     oprnd0 = vect_look_through_possible_promotion (vinfo, oprnd0, &unprom_mult);
    1315        54386 :   if (!oprnd0 || TREE_CODE (oprnd0) != SSA_NAME)
    1316              :     return NULL;
    1317              : 
    1318        54088 :   stmt_vec_info mult_vinfo = vect_get_internal_def (vinfo, oprnd0);
    1319        54088 :   if (!mult_vinfo)
    1320              :     return NULL;
    1321              : 
    1322              :   /* FORNOW.  We could continue following the def-use chain if this statement
    1323              :      is a phi inside the loop, as when analyzing an outer loop.  */
    1324       159678 :   vect_unpromoted_value unprom0[2];
    1325        53226 :   if (float_p)
    1326              :     {
    1327              :       /* Require multiplication of two directly promoted floating-point
    1328              :          inputs.  A product rounded before promotion is not equivalent.  */
    1329        17038 :       gassign *mult = dyn_cast <gassign *> (mult_vinfo->stmt);
    1330        17268 :       if (!mult || gimple_assign_rhs_code (mult) != MULT_EXPR)
    1331              :         return NULL;
    1332              : 
    1333        16772 :       for (unsigned int i = 0; i < 2; ++i)
    1334              :         {
    1335        16756 :           tree op = gimple_op (mult, i + 1);
    1336        16756 :           stmt_vec_info cast_info
    1337        16756 :             = (TREE_CODE (op) == SSA_NAME
    1338        16756 :                ? vect_get_internal_def (vinfo, op) : NULL);
    1339        16726 :           gassign *cast
    1340        16726 :             = cast_info ? dyn_cast <gassign *> (cast_info->stmt) : NULL;
    1341        33293 :           if (!cast || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (cast)))
    1342        16694 :             return NULL;
    1343              : 
    1344           62 :           op = gimple_assign_rhs1 (cast);
    1345           62 :           tree op_type = TREE_TYPE (op);
    1346           62 :           vect_def_type dt;
    1347           62 :           if (TREE_CODE (op) != SSA_NAME
    1348           62 :               || !SCALAR_FLOAT_TYPE_P (op_type)
    1349           62 :               || TYPE_PRECISION (op_type) * 2 > TYPE_PRECISION (type)
    1350          124 :               || !vect_is_simple_use (op, vinfo, &dt))
    1351              :             return NULL;
    1352           62 :           unprom0[i].set_op (op, dt, cast_info);
    1353              :         }
    1354           16 :       half_type = unprom0[0].type;
    1355           16 :       if (!types_compatible_p (half_type, unprom0[1].type))
    1356              :         return NULL;
    1357              :     }
    1358        36188 :   else if (!vect_widened_op_tree (vinfo, mult_vinfo, MULT_EXPR, WIDEN_MULT_EXPR,
    1359              :                                 false, 2, unprom0, &half_type, &subtype))
    1360              :     return NULL;
    1361              : 
    1362              :   /* If there are two integer widening operations, make sure they agree on
    1363              :      the sign of the extension.  The result of an optab_vector_mixed_sign
    1364              :      operation is signed; otherwise, it has the same sign as the operands.  */
    1365         1386 :   if (!float_p
    1366         1370 :       && TYPE_PRECISION (unprom_mult.type) != TYPE_PRECISION (type)
    1367         2127 :       && (subtype == optab_vector_mixed_sign
    1368          741 :           ? TYPE_UNSIGNED (unprom_mult.type)
    1369          532 :           : TYPE_SIGN (unprom_mult.type) != TYPE_SIGN (half_type)))
    1370              :     return NULL;
    1371              : 
    1372         1305 :   vect_pattern_detected ("vect_recog_dot_prod_pattern", last_stmt);
    1373              : 
    1374              :   /* If the inputs have mixed signs, canonicalize on using the signed
    1375              :      input type for analysis.  This also helps when emulating mixed-sign
    1376              :      operations using signed operations.  */
    1377         1305 :   if (subtype == optab_vector_mixed_sign)
    1378          246 :     half_type = signed_type_for (half_type);
    1379              : 
    1380         1305 :   tree half_vectype;
    1381         1305 :   if (!vect_supportable_conv_optab_p (vinfo, type, DOT_PROD_EXPR, half_type,
    1382              :                                         type_out, &half_vectype, subtype))
    1383              :     {
    1384              :       /* We can emulate a mixed-sign dot-product using a sequence of
    1385              :          signed dot-products; see vect_emulate_mixed_dot_prod for details.  */
    1386          588 :       if (subtype != optab_vector_mixed_sign
    1387          588 :           || !vect_supportable_conv_optab_p (vinfo, signed_type_for (type),
    1388              :                                                DOT_PROD_EXPR, half_type,
    1389              :                                                type_out, &half_vectype,
    1390              :                                                optab_vector))
    1391              :         return NULL;
    1392              : 
    1393           15 :       *type_out = signed_or_unsigned_type_for (TYPE_UNSIGNED (type),
    1394              :                                                *type_out);
    1395              :     }
    1396              : 
    1397              :   /* Get the inputs in the appropriate types.  */
    1398          732 :   tree mult_oprnd[2];
    1399          732 :   vect_convert_inputs (vinfo, stmt_vinfo, 2, mult_oprnd, half_type,
    1400              :                        unprom0, half_vectype, subtype);
    1401              : 
    1402          732 :   var = vect_recog_temp_ssa_var (type, NULL);
    1403          732 :   pattern_stmt = gimple_build_assign (var, DOT_PROD_EXPR,
    1404              :                                       mult_oprnd[0], mult_oprnd[1], oprnd1);
    1405              : 
    1406          732 :   return pattern_stmt;
    1407              : }
    1408              : 
    1409              : 
    1410              : /* Function vect_recog_sad_pattern
    1411              : 
    1412              :    Try to find the following Sum of Absolute Difference (SAD) pattern:
    1413              : 
    1414              :      type x_t, y_t;
    1415              :      signed TYPE1 diff, abs_diff;
    1416              :      TYPE2 sum = init;
    1417              :    loop:
    1418              :      sum_0 = phi <init, sum_1>
    1419              :      S1  x_t = ...
    1420              :      S2  y_t = ...
    1421              :      S3  x_T = (TYPE1) x_t;
    1422              :      S4  y_T = (TYPE1) y_t;
    1423              :      S5  diff = x_T - y_T;
    1424              :      S6  abs_diff = ABS_EXPR <diff>;
    1425              :      [S7  abs_diff = (TYPE2) abs_diff;  #optional]
    1426              :      S8  sum_1 = abs_diff + sum_0;
    1427              : 
    1428              :    where 'TYPE1' is at least double the size of type 'type', and 'TYPE2' is the
    1429              :    same size of 'TYPE1' or bigger. This is a special case of a reduction
    1430              :    computation.
    1431              : 
    1432              :    Input:
    1433              : 
    1434              :    * STMT_VINFO: The stmt from which the pattern search begins.  In the
    1435              :    example, when this function is called with S8, the pattern
    1436              :    {S3,S4,S5,S6,S7,S8} will be detected.
    1437              : 
    1438              :    Output:
    1439              : 
    1440              :    * TYPE_OUT: The type of the output of this pattern.
    1441              : 
    1442              :    * Return value: A new stmt that will be used to replace the sequence of
    1443              :    stmts that constitute the pattern. In this case it will be:
    1444              :         SAD_EXPR <x_t, y_t, sum_0>
    1445              :   */
    1446              : 
    1447              : static gimple *
    1448     33054359 : vect_recog_sad_pattern (vec_info *vinfo,
    1449              :                         stmt_vec_info stmt_vinfo, tree *type_out)
    1450              : {
    1451     33054359 :   gimple *last_stmt = stmt_vinfo->stmt;
    1452     33054359 :   tree half_type;
    1453              : 
    1454              :   /* Look for the following pattern
    1455              :           DX = (TYPE1) X;
    1456              :           DY = (TYPE1) Y;
    1457              :           DDIFF = DX - DY;
    1458              :           DAD = ABS_EXPR <DDIFF>;
    1459              :           DDPROD = (TYPE2) DPROD;
    1460              :           sum_1 = DAD + sum_0;
    1461              :      In which
    1462              :      - DX is at least double the size of X
    1463              :      - DY is at least double the size of Y
    1464              :      - DX, DY, DDIFF, DAD all have the same type
    1465              :      - sum is the same size of DAD or bigger
    1466              :      - sum has been recognized as a reduction variable.
    1467              : 
    1468              :      This is equivalent to:
    1469              :        DDIFF = X w- Y;          #widen sub
    1470              :        DAD = ABS_EXPR <DDIFF>;
    1471              :        sum_1 = DAD w+ sum_0;    #widen summation
    1472              :      or
    1473              :        DDIFF = X w- Y;          #widen sub
    1474              :        DAD = ABS_EXPR <DDIFF>;
    1475              :        sum_1 = DAD + sum_0;     #summation
    1476              :    */
    1477              : 
    1478              :   /* Starting from LAST_STMT, follow the defs of its uses in search
    1479              :      of the above pattern.  */
    1480              : 
    1481     33054359 :   tree plus_oprnd0, plus_oprnd1;
    1482     33054359 :   if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, PLUS_EXPR,
    1483              :                                        &plus_oprnd0, &plus_oprnd1))
    1484              :     return NULL;
    1485              : 
    1486        53654 :   tree sum_type = TREE_TYPE (gimple_get_lhs (last_stmt));
    1487              : 
    1488              :   /* Any non-truncating sequence of conversions is OK here, since
    1489              :      with a successful match, the result of the ABS(U) is known to fit
    1490              :      within the nonnegative range of the result type.  (It cannot be the
    1491              :      negative of the minimum signed value due to the range of the widening
    1492              :      MINUS_EXPR.)  */
    1493        53654 :   vect_unpromoted_value unprom_abs;
    1494        53654 :   plus_oprnd0 = vect_look_through_possible_promotion (vinfo, plus_oprnd0,
    1495              :                                                       &unprom_abs);
    1496              : 
    1497              :   /* So far so good.  Since last_stmt was detected as a (summation) reduction,
    1498              :      we know that plus_oprnd1 is the reduction variable (defined by a loop-header
    1499              :      phi), and plus_oprnd0 is an ssa-name defined by a stmt in the loop body.
    1500              :      Then check that plus_oprnd0 is defined by an abs_expr.  */
    1501              : 
    1502        53654 :   if (!plus_oprnd0)
    1503              :     return NULL;
    1504              : 
    1505        36306 :   stmt_vec_info abs_stmt_vinfo = vect_get_internal_def (vinfo, plus_oprnd0);
    1506        36306 :   if (!abs_stmt_vinfo)
    1507              :     return NULL;
    1508              : 
    1509              :   /* FORNOW.  Can continue analyzing the def-use chain when this stmt in a phi
    1510              :      inside the loop (in case we are analyzing an outer-loop).  */
    1511        35456 :   gassign *abs_stmt = dyn_cast <gassign *> (abs_stmt_vinfo->stmt);
    1512       106368 :   vect_unpromoted_value unprom[2];
    1513              : 
    1514        35456 :   if (!abs_stmt)
    1515              :     {
    1516          367 :       gcall *abd_stmt = dyn_cast <gcall *> (abs_stmt_vinfo->stmt);
    1517          302 :       if (!abd_stmt
    1518          302 :           || !gimple_call_internal_p (abd_stmt)
    1519            0 :           || gimple_call_num_args (abd_stmt) != 2)
    1520              :         return NULL;
    1521              : 
    1522            0 :       tree abd_oprnd0 = gimple_call_arg (abd_stmt, 0);
    1523            0 :       tree abd_oprnd1 = gimple_call_arg (abd_stmt, 1);
    1524              : 
    1525            0 :       if (gimple_call_internal_fn (abd_stmt) == IFN_ABD
    1526            0 :           || gimple_call_internal_fn (abd_stmt) == IFN_VEC_WIDEN_ABD)
    1527              :         {
    1528            0 :           unprom[0].op = abd_oprnd0;
    1529            0 :           unprom[0].type = TREE_TYPE (abd_oprnd0);
    1530            0 :           unprom[1].op = abd_oprnd1;
    1531            0 :           unprom[1].type = TREE_TYPE (abd_oprnd1);
    1532              :         }
    1533              :       else
    1534              :         return NULL;
    1535              : 
    1536            0 :       half_type = unprom[0].type;
    1537              :     }
    1538        35089 :   else if (!vect_recog_absolute_difference (vinfo, abs_stmt, &half_type,
    1539              :                                             unprom, NULL))
    1540              :     return NULL;
    1541              : 
    1542          854 :   vect_pattern_detected ("vect_recog_sad_pattern", last_stmt);
    1543              : 
    1544          854 :   tree half_vectype;
    1545          854 :   if (!vect_supportable_conv_optab_p (vinfo, sum_type, SAD_EXPR, half_type,
    1546              :                                       type_out, &half_vectype))
    1547              :     return NULL;
    1548              : 
    1549              :   /* Get the inputs to the SAD_EXPR in the appropriate types.  */
    1550          460 :   tree sad_oprnd[2];
    1551          460 :   vect_convert_inputs (vinfo, stmt_vinfo, 2, sad_oprnd, half_type,
    1552              :                        unprom, half_vectype);
    1553              : 
    1554          460 :   tree var = vect_recog_temp_ssa_var (sum_type, NULL);
    1555          460 :   gimple *pattern_stmt = gimple_build_assign (var, SAD_EXPR, sad_oprnd[0],
    1556              :                                               sad_oprnd[1], plus_oprnd1);
    1557              : 
    1558          460 :   return pattern_stmt;
    1559              : }
    1560              : 
    1561              : /* Function vect_recog_abd_pattern
    1562              : 
    1563              :    Try to find the following ABsolute Difference (ABD) or
    1564              :    widening ABD (WIDEN_ABD) pattern:
    1565              : 
    1566              :    TYPE1 x;
    1567              :    TYPE2 y;
    1568              :    TYPE3 x_cast = (TYPE3) x;              // widening or no-op
    1569              :    TYPE3 y_cast = (TYPE3) y;              // widening or no-op
    1570              :    TYPE3 diff = x_cast - y_cast;
    1571              :    TYPE4 diff_cast = (TYPE4) diff;        // widening or no-op
    1572              :    TYPE5 abs = ABS(U)_EXPR <diff_cast>;
    1573              : 
    1574              :    WIDEN_ABD exists to optimize the case where TYPE4 is at least
    1575              :    twice as wide as TYPE3.
    1576              : 
    1577              :    Input:
    1578              : 
    1579              :    * STMT_VINFO: The stmt from which the pattern search begins
    1580              : 
    1581              :    Output:
    1582              : 
    1583              :    * TYPE_OUT: The type of the output of this pattern
    1584              : 
    1585              :    * Return value: A new stmt that will be used to replace the sequence of
    1586              :      stmts that constitute the pattern, principally:
    1587              :         out = IFN_ABD (x, y)
    1588              :         out = IFN_WIDEN_ABD (x, y)
    1589              :  */
    1590              : 
    1591              : static gimple *
    1592     32999870 : vect_recog_abd_pattern (vec_info *vinfo,
    1593              :                         stmt_vec_info stmt_vinfo, tree *type_out)
    1594              : {
    1595     32999870 :   gassign *last_stmt = dyn_cast <gassign *> (STMT_VINFO_STMT (stmt_vinfo));
    1596     23097227 :   if (!last_stmt)
    1597              :     return NULL;
    1598              : 
    1599     23097227 :   tree out_type = TREE_TYPE (gimple_assign_lhs (last_stmt));
    1600              : 
    1601     69291681 :   vect_unpromoted_value unprom[2];
    1602     23097227 :   gassign *diff_stmt = NULL;
    1603     23097227 :   tree abd_in_type;
    1604     23097227 :   if (!vect_recog_absolute_difference (vinfo, last_stmt, &abd_in_type,
    1605              :                                        unprom, &diff_stmt))
    1606              :     {
    1607              :       /* We cannot try further without having a non-widening MINUS.  */
    1608     23095673 :       if (!diff_stmt)
    1609              :         return NULL;
    1610              : 
    1611          251 :       unprom[0].op = gimple_assign_rhs1 (diff_stmt);
    1612          251 :       unprom[1].op = gimple_assign_rhs2 (diff_stmt);
    1613          251 :       abd_in_type = signed_type_for (out_type);
    1614              :     }
    1615              : 
    1616         1805 :   tree abd_out_type = abd_in_type;
    1617              : 
    1618         1805 :   tree vectype_in = get_vectype_for_scalar_type (vinfo, abd_in_type);
    1619         1805 :   if (!vectype_in)
    1620              :     return NULL;
    1621              : 
    1622         1787 :   internal_fn ifn = IFN_ABD;
    1623         1787 :   tree vectype_out = vectype_in;
    1624              : 
    1625         1787 :   if (TYPE_PRECISION (out_type) >= TYPE_PRECISION (abd_in_type) * 2
    1626         1787 :       && stmt_vinfo->min_output_precision >= TYPE_PRECISION (abd_in_type) * 2)
    1627              :     {
    1628         1445 :       tree mid_type
    1629         1445 :         = build_nonstandard_integer_type (TYPE_PRECISION (abd_in_type) * 2,
    1630         1445 :                                           TYPE_UNSIGNED (abd_in_type));
    1631         1445 :       tree mid_vectype = get_vectype_for_scalar_type (vinfo, mid_type);
    1632              : 
    1633         1445 :       code_helper dummy_code;
    1634         1445 :       int dummy_int;
    1635         1445 :       auto_vec<tree> dummy_vec;
    1636         1445 :       if (mid_vectype
    1637         1445 :           && supportable_widening_operation (IFN_VEC_WIDEN_ABD,
    1638              :                                              mid_vectype, vectype_in, false,
    1639              :                                              &dummy_code, &dummy_code,
    1640              :                                              &dummy_int, &dummy_vec))
    1641              :         {
    1642            0 :           ifn = IFN_VEC_WIDEN_ABD;
    1643            0 :           abd_out_type = mid_type;
    1644            0 :           vectype_out = mid_vectype;
    1645              :         }
    1646         1445 :     }
    1647              : 
    1648         1445 :   if (ifn == IFN_ABD
    1649         1787 :       && !direct_internal_fn_supported_p (ifn, vectype_in,
    1650              :                                           OPTIMIZE_FOR_SPEED))
    1651              :     return NULL;
    1652              : 
    1653            0 :   vect_pattern_detected ("vect_recog_abd_pattern", last_stmt);
    1654              : 
    1655            0 :   tree abd_oprnds[2];
    1656            0 :   vect_convert_inputs (vinfo, stmt_vinfo, 2, abd_oprnds,
    1657              :                        abd_in_type, unprom, vectype_in);
    1658              : 
    1659            0 :   *type_out = get_vectype_for_scalar_type (vinfo, out_type);
    1660              : 
    1661            0 :   tree abd_result = vect_recog_temp_ssa_var (abd_out_type, NULL);
    1662            0 :   gcall *abd_stmt = gimple_build_call_internal (ifn, 2,
    1663              :                                                 abd_oprnds[0], abd_oprnds[1]);
    1664            0 :   gimple_call_set_lhs (abd_stmt, abd_result);
    1665            0 :   gimple_set_location (abd_stmt, gimple_location (last_stmt));
    1666              : 
    1667            0 :   gimple *stmt = abd_stmt;
    1668            0 :   if (TYPE_PRECISION (abd_in_type) == TYPE_PRECISION (abd_out_type)
    1669            0 :       && TYPE_PRECISION (abd_out_type) < TYPE_PRECISION (out_type)
    1670            0 :       && !TYPE_UNSIGNED (abd_out_type))
    1671              :     {
    1672            0 :       tree unsign = unsigned_type_for (abd_out_type);
    1673            0 :       stmt = vect_convert_output (vinfo, stmt_vinfo, unsign, stmt, vectype_out);
    1674            0 :       vectype_out = get_vectype_for_scalar_type (vinfo, unsign);
    1675              :     }
    1676              : 
    1677            0 :   return vect_convert_output (vinfo, stmt_vinfo, out_type, stmt, vectype_out);
    1678              : }
    1679              : 
    1680              : /* Recognize an operation that performs ORIG_CODE on widened inputs,
    1681              :    so that it can be treated as though it had the form:
    1682              : 
    1683              :       A_TYPE a;
    1684              :       B_TYPE b;
    1685              :       HALF_TYPE a_cast = (HALF_TYPE) a;  // possible no-op
    1686              :       HALF_TYPE b_cast = (HALF_TYPE) b;  // possible no-op
    1687              :     | RES_TYPE a_extend = (RES_TYPE) a_cast;  // promotion from HALF_TYPE
    1688              :     | RES_TYPE b_extend = (RES_TYPE) b_cast;  // promotion from HALF_TYPE
    1689              :     | RES_TYPE res = a_extend ORIG_CODE b_extend;
    1690              : 
    1691              :    Try to replace the pattern with:
    1692              : 
    1693              :       A_TYPE a;
    1694              :       B_TYPE b;
    1695              :       HALF_TYPE a_cast = (HALF_TYPE) a;  // possible no-op
    1696              :       HALF_TYPE b_cast = (HALF_TYPE) b;  // possible no-op
    1697              :     | EXT_TYPE ext = a_cast WIDE_CODE b_cast;
    1698              :     | RES_TYPE res = (EXT_TYPE) ext;  // possible no-op
    1699              : 
    1700              :    where EXT_TYPE is wider than HALF_TYPE but has the same signedness.
    1701              : 
    1702              :    SHIFT_P is true if ORIG_CODE and WIDE_CODE are shifts.  NAME is the
    1703              :    name of the pattern being matched, for dump purposes.  */
    1704              : 
    1705              : static gimple *
    1706    132507878 : vect_recog_widen_op_pattern (vec_info *vinfo,
    1707              :                              stmt_vec_info last_stmt_info, tree *type_out,
    1708              :                              tree_code orig_code, code_helper wide_code,
    1709              :                              bool shift_p, const char *name)
    1710              : {
    1711    132507878 :   gimple *last_stmt = last_stmt_info->stmt;
    1712              : 
    1713    397523634 :   vect_unpromoted_value unprom[2];
    1714    132507878 :   tree half_type;
    1715    132507878 :   if (!vect_widened_op_tree (vinfo, last_stmt_info, orig_code, orig_code,
    1716              :                              shift_p, 2, unprom, &half_type))
    1717              : 
    1718              :     return NULL;
    1719              : 
    1720              :   /* Pattern detected.  */
    1721       371775 :   vect_pattern_detected (name, last_stmt);
    1722              : 
    1723       371775 :   tree type = TREE_TYPE (gimple_get_lhs (last_stmt));
    1724       371775 :   tree itype = type;
    1725       371775 :   if (TYPE_PRECISION (type) != TYPE_PRECISION (half_type) * 2
    1726       371775 :       || TYPE_UNSIGNED (type) != TYPE_UNSIGNED (half_type))
    1727       257108 :     itype = build_nonstandard_integer_type (TYPE_PRECISION (half_type) * 2,
    1728       257108 :                                             TYPE_UNSIGNED (half_type));
    1729              : 
    1730              :   /* Check target support  */
    1731       371775 :   tree vectype = get_vectype_for_scalar_type (vinfo, half_type);
    1732       371775 :   tree vecitype = get_vectype_for_scalar_type (vinfo, itype);
    1733       371775 :   tree ctype = itype;
    1734       371775 :   tree vecctype = vecitype;
    1735       371775 :   if (orig_code == MINUS_EXPR
    1736         8951 :       && TYPE_UNSIGNED (itype)
    1737       378695 :       && TYPE_PRECISION (type) > TYPE_PRECISION (itype))
    1738              :     {
    1739              :       /* Subtraction is special, even if half_type is unsigned and no matter
    1740              :          whether type is signed or unsigned, if type is wider than itype,
    1741              :          we need to sign-extend from the widening operation result to the
    1742              :          result type.
    1743              :          Consider half_type unsigned char, operand 1 0xfe, operand 2 0xff,
    1744              :          itype unsigned short and type either int or unsigned int.
    1745              :          Widened (unsigned short) 0xfe - (unsigned short) 0xff is
    1746              :          (unsigned short) 0xffff, but for type int we want the result -1
    1747              :          and for type unsigned int 0xffffffff rather than 0xffff.  */
    1748         3066 :       ctype = build_nonstandard_integer_type (TYPE_PRECISION (itype), 0);
    1749         3066 :       vecctype = get_vectype_for_scalar_type (vinfo, ctype);
    1750              :     }
    1751              : 
    1752       371775 :   code_helper dummy_code;
    1753       371775 :   int dummy_int;
    1754       371775 :   auto_vec<tree> dummy_vec;
    1755       371775 :   if (!vectype
    1756       371775 :       || !vecitype
    1757       288808 :       || !vecctype
    1758       660583 :       || !supportable_widening_operation (wide_code, vecitype, vectype, true,
    1759              :                                           &dummy_code, &dummy_code,
    1760              :                                           &dummy_int, &dummy_vec))
    1761              :     return NULL;
    1762              : 
    1763       117484 :   *type_out = get_vectype_for_scalar_type (vinfo, type);
    1764       117484 :   if (!*type_out)
    1765              :     return NULL;
    1766              : 
    1767       117484 :   tree oprnd[2];
    1768       117484 :   vect_convert_inputs (vinfo, last_stmt_info,
    1769              :                        2, oprnd, half_type, unprom, vectype);
    1770              : 
    1771       117484 :   tree var = vect_recog_temp_ssa_var (itype, NULL);
    1772       117484 :   gimple *pattern_stmt = vect_gimple_build (var, wide_code, oprnd[0], oprnd[1]);
    1773              : 
    1774       117484 :   if (vecctype != vecitype)
    1775            0 :     pattern_stmt = vect_convert_output (vinfo, last_stmt_info, ctype,
    1776              :                                         pattern_stmt, vecitype);
    1777              : 
    1778       117484 :   return vect_convert_output (vinfo, last_stmt_info,
    1779       117484 :                               type, pattern_stmt, vecctype);
    1780       371775 : }
    1781              : 
    1782              : /* Try to detect multiplication on widened inputs, converting MULT_EXPR
    1783              :    to WIDEN_MULT_EXPR.  See vect_recog_widen_op_pattern for details.  */
    1784              : 
    1785              : static gimple *
    1786     33083170 : vect_recog_widen_mult_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
    1787              :                                tree *type_out)
    1788              : {
    1789     33083170 :   return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
    1790     33083170 :                                       MULT_EXPR, WIDEN_MULT_EXPR, false,
    1791     33083170 :                                       "vect_recog_widen_mult_pattern");
    1792              : }
    1793              : 
    1794              : /* Try to detect addition on widened inputs, converting PLUS_EXPR
    1795              :    to IFN_VEC_WIDEN_PLUS.  See vect_recog_widen_op_pattern for details.  */
    1796              : 
    1797              : static gimple *
    1798     33185090 : vect_recog_widen_plus_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
    1799              :                                tree *type_out)
    1800              : {
    1801     33185090 :   return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
    1802     33185090 :                                       PLUS_EXPR, IFN_VEC_WIDEN_PLUS,
    1803     33185090 :                                       false, "vect_recog_widen_plus_pattern");
    1804              : }
    1805              : 
    1806              : /* Try to detect subtraction on widened inputs, converting MINUS_EXPR
    1807              :    to IFN_VEC_WIDEN_MINUS.  See vect_recog_widen_op_pattern for details.  */
    1808              : static gimple *
    1809     33185090 : vect_recog_widen_minus_pattern (vec_info *vinfo, stmt_vec_info last_stmt_info,
    1810              :                                tree *type_out)
    1811              : {
    1812     33185090 :   return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
    1813     33185090 :                                       MINUS_EXPR, IFN_VEC_WIDEN_MINUS,
    1814     33185090 :                                       false, "vect_recog_widen_minus_pattern");
    1815              : }
    1816              : 
    1817              : /* Try to detect abd on widened inputs, converting IFN_ABD
    1818              :    to IFN_VEC_WIDEN_ABD.  */
    1819              : static gimple *
    1820     33185090 : vect_recog_widen_abd_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
    1821              :                               tree *type_out)
    1822              : {
    1823     33185090 :   gassign *last_stmt = dyn_cast <gassign *> (STMT_VINFO_STMT (stmt_vinfo));
    1824     31392613 :   if (!last_stmt || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (last_stmt)))
    1825              :     return NULL;
    1826              : 
    1827      3370826 :   tree last_rhs = gimple_assign_rhs1 (last_stmt);
    1828              : 
    1829      3370826 :   tree in_type = TREE_TYPE (last_rhs);
    1830      3370826 :   tree out_type = TREE_TYPE (gimple_assign_lhs (last_stmt));
    1831      3370826 :   if (!INTEGRAL_TYPE_P (in_type)
    1832      2947713 :       || !INTEGRAL_TYPE_P (out_type)
    1833      2819196 :       || TYPE_PRECISION (in_type) * 2 != TYPE_PRECISION (out_type)
    1834      4050393 :       || !TYPE_UNSIGNED (in_type))
    1835              :     return NULL;
    1836              : 
    1837       248498 :   vect_unpromoted_value unprom;
    1838       248498 :   tree op = vect_look_through_possible_promotion (vinfo, last_rhs, &unprom);
    1839       248498 :   if (!op || TYPE_PRECISION (TREE_TYPE (op)) != TYPE_PRECISION (in_type))
    1840              :     return NULL;
    1841              : 
    1842       245941 :   stmt_vec_info abd_pattern_vinfo = vect_get_internal_def (vinfo, op);
    1843       245941 :   if (!abd_pattern_vinfo)
    1844              :     return NULL;
    1845              : 
    1846       220365 :   gcall *abd_stmt = dyn_cast <gcall *> (STMT_VINFO_STMT (abd_pattern_vinfo));
    1847         9292 :   if (!abd_stmt
    1848         9292 :       || !gimple_call_internal_p (abd_stmt)
    1849          320 :       || gimple_call_internal_fn (abd_stmt) != IFN_ABD)
    1850              :     return NULL;
    1851              : 
    1852            0 :   tree vectype_in = get_vectype_for_scalar_type (vinfo, in_type);
    1853            0 :   tree vectype_out = get_vectype_for_scalar_type (vinfo, out_type);
    1854              : 
    1855            0 :   code_helper dummy_code;
    1856            0 :   int dummy_int;
    1857            0 :   auto_vec<tree> dummy_vec;
    1858            0 :   if (!supportable_widening_operation (IFN_VEC_WIDEN_ABD, vectype_out,
    1859              :                                        vectype_in, false,
    1860              :                                        &dummy_code, &dummy_code,
    1861              :                                        &dummy_int, &dummy_vec))
    1862              :     return NULL;
    1863              : 
    1864            0 :   vect_pattern_detected ("vect_recog_widen_abd_pattern", last_stmt);
    1865              : 
    1866            0 :   *type_out = vectype_out;
    1867              : 
    1868            0 :   tree abd_oprnd0 = gimple_call_arg (abd_stmt, 0);
    1869            0 :   tree abd_oprnd1 = gimple_call_arg (abd_stmt, 1);
    1870            0 :   tree widen_abd_result = vect_recog_temp_ssa_var (out_type, NULL);
    1871            0 :   gcall *widen_abd_stmt = gimple_build_call_internal (IFN_VEC_WIDEN_ABD, 2,
    1872              :                                                       abd_oprnd0, abd_oprnd1);
    1873            0 :   gimple_call_set_lhs (widen_abd_stmt, widen_abd_result);
    1874            0 :   gimple_set_location (widen_abd_stmt, gimple_location (last_stmt));
    1875            0 :   return widen_abd_stmt;
    1876            0 : }
    1877              : 
    1878              : /* Function vect_recog_ctz_ffs_pattern
    1879              : 
    1880              :    Try to find the following pattern:
    1881              : 
    1882              :    TYPE1 A;
    1883              :    TYPE1 B;
    1884              : 
    1885              :    B = __builtin_ctz{,l,ll} (A);
    1886              : 
    1887              :    or
    1888              : 
    1889              :    B = __builtin_ffs{,l,ll} (A);
    1890              : 
    1891              :    Input:
    1892              : 
    1893              :    * STMT_VINFO: The stmt from which the pattern search begins.
    1894              :    here it starts with B = __builtin_* (A);
    1895              : 
    1896              :    Output:
    1897              : 
    1898              :    * TYPE_OUT: The vector type of the output of this pattern.
    1899              : 
    1900              :    * Return value: A new stmt that will be used to replace the sequence of
    1901              :    stmts that constitute the pattern, using clz or popcount builtins.  */
    1902              : 
    1903              : static gimple *
    1904     33054288 : vect_recog_ctz_ffs_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
    1905              :                             tree *type_out)
    1906              : {
    1907     33054288 :   gimple *call_stmt = stmt_vinfo->stmt;
    1908     33054288 :   gimple *pattern_stmt;
    1909     33054288 :   tree rhs_oprnd, rhs_type, lhs_oprnd, lhs_type, vec_type, vec_rhs_type;
    1910     33054288 :   tree new_var;
    1911     33054288 :   internal_fn ifn = IFN_LAST, ifnnew = IFN_LAST;
    1912     33054288 :   bool defined_at_zero = true, defined_at_zero_new = false;
    1913     33054288 :   int val = 0, val_new = 0, val_cmp = 0;
    1914     33054288 :   int prec;
    1915     33054288 :   int sub = 0, add = 0;
    1916     33054288 :   location_t loc;
    1917              : 
    1918     33054288 :   if (!is_gimple_call (call_stmt))
    1919              :     return NULL;
    1920              : 
    1921      3844734 :   if (gimple_call_num_args (call_stmt) != 1
    1922      3844734 :       && gimple_call_num_args (call_stmt) != 2)
    1923              :     return NULL;
    1924              : 
    1925      2180799 :   rhs_oprnd = gimple_call_arg (call_stmt, 0);
    1926      2180799 :   rhs_type = TREE_TYPE (rhs_oprnd);
    1927      2180799 :   lhs_oprnd = gimple_call_lhs (call_stmt);
    1928      2180799 :   if (!lhs_oprnd)
    1929              :     return NULL;
    1930      1069052 :   lhs_type = TREE_TYPE (lhs_oprnd);
    1931      1069052 :   if (!INTEGRAL_TYPE_P (lhs_type)
    1932       357330 :       || !INTEGRAL_TYPE_P (rhs_type)
    1933        56128 :       || !type_has_mode_precision_p (rhs_type)
    1934      1122171 :       || TREE_CODE (rhs_oprnd) != SSA_NAME)
    1935              :     return NULL;
    1936              : 
    1937        39211 :   switch (gimple_call_combined_fn (call_stmt))
    1938              :     {
    1939         2075 :     CASE_CFN_CTZ:
    1940         2075 :       ifn = IFN_CTZ;
    1941         2075 :       if (!gimple_call_internal_p (call_stmt)
    1942         2075 :           || gimple_call_num_args (call_stmt) != 2)
    1943              :         defined_at_zero = false;
    1944              :       else
    1945          139 :         val = tree_to_shwi (gimple_call_arg (call_stmt, 1));
    1946              :       break;
    1947              :     CASE_CFN_FFS:
    1948              :       ifn = IFN_FFS;
    1949              :       break;
    1950              :     default:
    1951              :       return NULL;
    1952              :     }
    1953              : 
    1954         2310 :   prec = TYPE_PRECISION (rhs_type);
    1955         2310 :   loc = gimple_location (call_stmt);
    1956              : 
    1957         2310 :   vec_type = get_vectype_for_scalar_type (vinfo, lhs_type);
    1958         2310 :   if (!vec_type)
    1959              :     return NULL;
    1960              : 
    1961         2304 :   vec_rhs_type = get_vectype_for_scalar_type (vinfo, rhs_type);
    1962         2304 :   if (!vec_rhs_type)
    1963              :     return NULL;
    1964              : 
    1965              :   /* Do it only if the backend doesn't have ctz<vector_mode>2 or
    1966              :      ffs<vector_mode>2 pattern but does have clz<vector_mode>2 or
    1967              :      popcount<vector_mode>2.  */
    1968         2038 :   if (!vec_type
    1969         2038 :       || direct_internal_fn_supported_p (ifn, vec_rhs_type,
    1970              :                                          OPTIMIZE_FOR_SPEED))
    1971              :     return NULL;
    1972              : 
    1973         2038 :   if (ifn == IFN_FFS
    1974         2038 :       && direct_internal_fn_supported_p (IFN_CTZ, vec_rhs_type,
    1975              :                                          OPTIMIZE_FOR_SPEED))
    1976              :     {
    1977            0 :       ifnnew = IFN_CTZ;
    1978            0 :       defined_at_zero_new
    1979            0 :         = CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_rhs_type),
    1980              :                                      val_new) == 2;
    1981              :     }
    1982         2038 :   else if (direct_internal_fn_supported_p (IFN_CLZ, vec_rhs_type,
    1983              :                                            OPTIMIZE_FOR_SPEED))
    1984              :     {
    1985          166 :       ifnnew = IFN_CLZ;
    1986          166 :       defined_at_zero_new
    1987          166 :         = CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_rhs_type),
    1988              :                                      val_new) == 2;
    1989              :     }
    1990          166 :   if ((ifnnew == IFN_LAST
    1991          166 :        || (defined_at_zero && !defined_at_zero_new))
    1992         1872 :       && direct_internal_fn_supported_p (IFN_POPCOUNT, vec_rhs_type,
    1993              :                                          OPTIMIZE_FOR_SPEED))
    1994              :     {
    1995              :       ifnnew = IFN_POPCOUNT;
    1996              :       defined_at_zero_new = true;
    1997              :       val_new = prec;
    1998              :     }
    1999         1912 :   if (ifnnew == IFN_LAST)
    2000              :     return NULL;
    2001              : 
    2002              :   /* Only .FFS (X) = .CTZ (X) + 1 can be part of a reduction, and only
    2003              :      if the result for zero needs no select.  */
    2004          292 :   if (vect_is_reduction (stmt_vinfo)
    2005          292 :       && (ifnnew != IFN_CTZ
    2006            0 :           || (defined_at_zero
    2007            0 :               && (!defined_at_zero_new || val != val_new + 1))))
    2008              :     return NULL;
    2009              : 
    2010          286 :   vect_pattern_detected ("vec_recog_ctz_ffs_pattern", call_stmt);
    2011              : 
    2012              :   /* .CLZ, .CTZ and .POPCOUNT take an unsigned argument.  */
    2013          286 :   if (!TYPE_UNSIGNED (rhs_type))
    2014              :     {
    2015           51 :       rhs_type = unsigned_type_for (rhs_type);
    2016           51 :       vec_rhs_type = get_vectype_for_scalar_type (vinfo, rhs_type);
    2017           51 :       new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
    2018           51 :       pattern_stmt = gimple_build_assign (new_var, NOP_EXPR, rhs_oprnd);
    2019           51 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
    2020           51 :       rhs_oprnd = new_var;
    2021              :     }
    2022              : 
    2023          286 :   val_cmp = val_new;
    2024          286 :   if ((ifnnew == IFN_CLZ
    2025          286 :        && defined_at_zero
    2026          106 :        && defined_at_zero_new
    2027          106 :        && val == prec
    2028           54 :        && val_new == prec)
    2029          232 :       || (ifnnew == IFN_POPCOUNT && ifn == IFN_CTZ))
    2030              :     {
    2031              :       /* .CTZ (X) = PREC - .CLZ ((X - 1) & ~X)
    2032              :          .CTZ (X) = .POPCOUNT ((X - 1) & ~X).  */
    2033              :       if (ifnnew == IFN_CLZ)
    2034              :         sub = prec;
    2035          137 :       val_cmp = prec;
    2036              : 
    2037          137 :       tree m1 = vect_recog_temp_ssa_var (rhs_type, NULL);
    2038          137 :       pattern_stmt = gimple_build_assign (m1, PLUS_EXPR, rhs_oprnd,
    2039              :                                           build_int_cst (rhs_type, -1));
    2040          137 :       gimple_set_location (pattern_stmt, loc);
    2041          137 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
    2042              : 
    2043          137 :       new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
    2044          137 :       pattern_stmt = gimple_build_assign (new_var, BIT_NOT_EXPR, rhs_oprnd);
    2045          137 :       gimple_set_location (pattern_stmt, loc);
    2046          137 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
    2047          137 :       rhs_oprnd = new_var;
    2048              : 
    2049          137 :       new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
    2050          137 :       pattern_stmt = gimple_build_assign (new_var, BIT_AND_EXPR,
    2051              :                                           m1, rhs_oprnd);
    2052          137 :       gimple_set_location (pattern_stmt, loc);
    2053          137 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
    2054          137 :       rhs_oprnd = new_var;
    2055          137 :     }
    2056          149 :   else if (ifnnew == IFN_CLZ)
    2057              :     {
    2058              :       /* .CTZ (X) = (PREC - 1) - .CLZ (X & -X)
    2059              :          .FFS (X) = PREC - .CLZ (X & -X).  */
    2060          106 :       sub = prec - (ifn == IFN_CTZ);
    2061          106 :       val_cmp = sub - val_new;
    2062              : 
    2063          106 :       tree neg = vect_recog_temp_ssa_var (rhs_type, NULL);
    2064          106 :       pattern_stmt = gimple_build_assign (neg, NEGATE_EXPR, rhs_oprnd);
    2065          106 :       gimple_set_location (pattern_stmt, loc);
    2066          106 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
    2067              : 
    2068          106 :       new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
    2069          106 :       pattern_stmt = gimple_build_assign (new_var, BIT_AND_EXPR,
    2070              :                                           rhs_oprnd, neg);
    2071          106 :       gimple_set_location (pattern_stmt, loc);
    2072          106 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
    2073          106 :       rhs_oprnd = new_var;
    2074              :     }
    2075           43 :   else if (ifnnew == IFN_POPCOUNT)
    2076              :     {
    2077              :       /* .CTZ (X) = PREC - .POPCOUNT (X | -X)
    2078              :          .FFS (X) = (PREC + 1) - .POPCOUNT (X | -X).  */
    2079           43 :       sub = prec + (ifn == IFN_FFS);
    2080           43 :       val_cmp = sub;
    2081              : 
    2082           43 :       tree neg = vect_recog_temp_ssa_var (rhs_type, NULL);
    2083           43 :       pattern_stmt = gimple_build_assign (neg, NEGATE_EXPR, rhs_oprnd);
    2084           43 :       gimple_set_location (pattern_stmt, loc);
    2085           43 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
    2086              : 
    2087           43 :       new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
    2088           43 :       pattern_stmt = gimple_build_assign (new_var, BIT_IOR_EXPR,
    2089              :                                           rhs_oprnd, neg);
    2090           43 :       gimple_set_location (pattern_stmt, loc);
    2091           43 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
    2092           43 :       rhs_oprnd = new_var;
    2093              :     }
    2094            0 :   else if (ifnnew == IFN_CTZ)
    2095              :     {
    2096              :       /* .FFS (X) = .CTZ (X) + 1.  */
    2097            0 :       add = 1;
    2098            0 :       val_cmp++;
    2099              :     }
    2100              : 
    2101              :   /* Create B = .IFNNEW (A).  */
    2102          286 :   new_var = vect_recog_temp_ssa_var (lhs_type, NULL);
    2103          286 :   if ((ifnnew == IFN_CLZ || ifnnew == IFN_CTZ) && defined_at_zero_new)
    2104          160 :     pattern_stmt
    2105          160 :       = gimple_build_call_internal (ifnnew, 2, rhs_oprnd,
    2106              :                                     build_int_cst (integer_type_node,
    2107          160 :                                                    val_new));
    2108              :   else
    2109          126 :     pattern_stmt = gimple_build_call_internal (ifnnew, 1, rhs_oprnd);
    2110          286 :   gimple_call_set_lhs (pattern_stmt, new_var);
    2111          286 :   gimple_set_location (pattern_stmt, loc);
    2112          286 :   *type_out = vec_type;
    2113              : 
    2114          286 :   if (sub)
    2115              :     {
    2116          203 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
    2117          203 :       tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
    2118          203 :       pattern_stmt = gimple_build_assign (ret_var, MINUS_EXPR,
    2119          203 :                                           build_int_cst (lhs_type, sub),
    2120              :                                           new_var);
    2121          203 :       gimple_set_location (pattern_stmt, loc);
    2122          203 :       new_var = ret_var;
    2123              :     }
    2124           83 :   else if (add)
    2125              :     {
    2126            0 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
    2127            0 :       tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
    2128            0 :       pattern_stmt = gimple_build_assign (ret_var, PLUS_EXPR, new_var,
    2129            0 :                                           build_int_cst (lhs_type, add));
    2130            0 :       gimple_set_location (pattern_stmt, loc);
    2131            0 :       new_var = ret_var;
    2132              :     }
    2133              : 
    2134          286 :   if (defined_at_zero
    2135          210 :       && (!defined_at_zero_new || val != val_cmp))
    2136              :     {
    2137           43 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
    2138           43 :       tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
    2139           43 :       rhs_oprnd = gimple_call_arg (call_stmt, 0);
    2140           43 :       rhs_type = TREE_TYPE (rhs_oprnd);
    2141           43 :       tree cmp = vect_recog_temp_ssa_var (boolean_type_node, NULL);
    2142           43 :       pattern_stmt = gimple_build_assign (cmp, NE_EXPR, rhs_oprnd,
    2143              :                                           build_zero_cst (rhs_type));
    2144           43 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
    2145              :                               truth_type_for (vec_type), rhs_type);
    2146           43 :       pattern_stmt = gimple_build_assign (ret_var, COND_EXPR, cmp,
    2147              :                                           new_var,
    2148           43 :                                           build_int_cst (lhs_type, val));
    2149              :     }
    2150              : 
    2151          286 :   if (dump_enabled_p ())
    2152           36 :     dump_printf_loc (MSG_NOTE, vect_location,
    2153              :                      "created pattern stmt: %G", pattern_stmt);
    2154              : 
    2155              :   return pattern_stmt;
    2156              : }
    2157              : 
    2158              : /* Function vect_recog_popcount_clz_ctz_ffs_pattern
    2159              : 
    2160              :    Try to find the following pattern:
    2161              : 
    2162              :    UTYPE1 A;
    2163              :    TYPE1 B;
    2164              :    UTYPE2 temp_in;
    2165              :    TYPE3 temp_out;
    2166              :    temp_in = (UTYPE2)A;
    2167              : 
    2168              :    temp_out = __builtin_popcount{,l,ll} (temp_in);
    2169              :    B = (TYPE1) temp_out;
    2170              : 
    2171              :    TYPE2 may or may not be equal to TYPE3.
    2172              :    i.e. TYPE2 is equal to TYPE3 for __builtin_popcount
    2173              :    i.e. TYPE2 is not equal to TYPE3 for __builtin_popcountll
    2174              : 
    2175              :    Input:
    2176              : 
    2177              :    * STMT_VINFO: The stmt from which the pattern search begins.
    2178              :    here it starts with B = (TYPE1) temp_out;
    2179              : 
    2180              :    Output:
    2181              : 
    2182              :    * TYPE_OUT: The vector type of the output of this pattern.
    2183              : 
    2184              :    * Return value: A new stmt that will be used to replace the sequence of
    2185              :    stmts that constitute the pattern. In this case it will be:
    2186              :    B = .POPCOUNT (A);
    2187              : 
    2188              :    Similarly for clz, ctz and ffs.
    2189              : */
    2190              : 
    2191              : static gimple *
    2192     33053890 : vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
    2193              :                                          stmt_vec_info stmt_vinfo,
    2194              :                                          tree *type_out)
    2195              : {
    2196     33053890 :   gassign *last_stmt = dyn_cast <gassign *> (stmt_vinfo->stmt);
    2197     23146502 :   gimple *call_stmt, *pattern_stmt;
    2198     23146502 :   tree rhs_oprnd, rhs_origin, lhs_oprnd, lhs_type, vec_type, new_var;
    2199     56200392 :   internal_fn ifn = IFN_LAST;
    2200     56200392 :   int addend = 0;
    2201              : 
    2202              :   /* Find B = (TYPE1) temp_out. */
    2203     23146502 :   if (!last_stmt)
    2204              :     return NULL;
    2205     23146502 :   tree_code code = gimple_assign_rhs_code (last_stmt);
    2206     23146502 :   if (!CONVERT_EXPR_CODE_P (code))
    2207              :     return NULL;
    2208              : 
    2209      3213684 :   lhs_oprnd = gimple_assign_lhs (last_stmt);
    2210      3213684 :   lhs_type = TREE_TYPE (lhs_oprnd);
    2211      3213684 :   if (!INTEGRAL_TYPE_P (lhs_type))
    2212              :     return NULL;
    2213              : 
    2214      2922385 :   rhs_oprnd = gimple_assign_rhs1 (last_stmt);
    2215      2922385 :   if (TREE_CODE (rhs_oprnd) != SSA_NAME
    2216      2922385 :       || !has_single_use (rhs_oprnd))
    2217              :     return NULL;
    2218      1490258 :   call_stmt = SSA_NAME_DEF_STMT (rhs_oprnd);
    2219              : 
    2220              :   /* Find temp_out = __builtin_popcount{,l,ll} (temp_in);  */
    2221      1490258 :   if (!is_gimple_call (call_stmt))
    2222              :     return NULL;
    2223        60958 :   switch (gimple_call_combined_fn (call_stmt))
    2224              :     {
    2225              :       int val;
    2226              :     CASE_CFN_POPCOUNT:
    2227              :       ifn = IFN_POPCOUNT;
    2228              :       break;
    2229         2412 :     CASE_CFN_CLZ:
    2230         2412 :       ifn = IFN_CLZ;
    2231              :       /* Punt if call result is unsigned and defined value at zero
    2232              :          is negative, as the negative value doesn't extend correctly.  */
    2233         2412 :       if (TYPE_UNSIGNED (TREE_TYPE (rhs_oprnd))
    2234            0 :           && gimple_call_internal_p (call_stmt)
    2235         2412 :           && CLZ_DEFINED_VALUE_AT_ZERO
    2236              :                (SCALAR_INT_TYPE_MODE (TREE_TYPE (rhs_oprnd)), val) == 2
    2237         2412 :           && val < 0)
    2238              :         return NULL;
    2239              :       break;
    2240          765 :     CASE_CFN_CTZ:
    2241          765 :       ifn = IFN_CTZ;
    2242              :       /* Punt if call result is unsigned and defined value at zero
    2243              :          is negative, as the negative value doesn't extend correctly.  */
    2244          765 :       if (TYPE_UNSIGNED (TREE_TYPE (rhs_oprnd))
    2245            0 :           && gimple_call_internal_p (call_stmt)
    2246          765 :           && CTZ_DEFINED_VALUE_AT_ZERO
    2247              :                (SCALAR_INT_TYPE_MODE (TREE_TYPE (rhs_oprnd)), val) == 2
    2248          765 :           && val < 0)
    2249              :         return NULL;
    2250              :       break;
    2251           57 :     CASE_CFN_FFS:
    2252           57 :       ifn = IFN_FFS;
    2253           57 :       break;
    2254              :     default:
    2255              :       return NULL;
    2256              :     }
    2257              : 
    2258         3588 :   if (gimple_call_num_args (call_stmt) != 1
    2259         3588 :       && gimple_call_num_args (call_stmt) != 2)
    2260              :     return NULL;
    2261              : 
    2262         3588 :   rhs_oprnd = gimple_call_arg (call_stmt, 0);
    2263         3588 :   vect_unpromoted_value unprom_diff;
    2264         3588 :   rhs_origin
    2265         3588 :     = vect_look_through_possible_promotion (vinfo, rhs_oprnd, &unprom_diff);
    2266              : 
    2267         3588 :   if (!rhs_origin)
    2268              :     return NULL;
    2269              : 
    2270              :   /* Input and output of .POPCOUNT should be same-precision integer.  */
    2271         3578 :   if (TYPE_PRECISION (unprom_diff.type) != TYPE_PRECISION (lhs_type))
    2272              :     return NULL;
    2273              : 
    2274         1867 :   vec_type = get_vectype_for_scalar_type (vinfo, lhs_type);
    2275         1867 :   if (!vec_type || !VECTOR_MODE_P (TYPE_MODE (vec_type)))
    2276              :     return NULL;
    2277              : 
    2278              :   /* Also A should be unsigned or same precision as temp_in, otherwise
    2279              :      different builtins/internal functions have different behaviors.  */
    2280         1735 :   if (TYPE_PRECISION (unprom_diff.type)
    2281         1735 :       != TYPE_PRECISION (TREE_TYPE (rhs_oprnd)))
    2282          319 :     switch (ifn)
    2283              :       {
    2284          147 :       case IFN_POPCOUNT:
    2285              :         /* For popcount require zero extension, which doesn't add any
    2286              :            further bits to the count.  */
    2287          147 :         if (!TYPE_UNSIGNED (unprom_diff.type))
    2288              :           return NULL;
    2289              :         break;
    2290          109 :       case IFN_CLZ:
    2291              :         /* clzll (x) == clz (x) + 32 for unsigned x != 0.  */
    2292          109 :         if (!TYPE_UNSIGNED (unprom_diff.type))
    2293              :           return NULL;
    2294          109 :         if (!type_has_mode_precision_p (lhs_type)
    2295          109 :             || !type_has_mode_precision_p (TREE_TYPE (rhs_oprnd)))
    2296              :           return NULL;
    2297          109 :         addend = (TYPE_PRECISION (TREE_TYPE (rhs_oprnd))
    2298          109 :                   - TYPE_PRECISION (lhs_type));
    2299          109 :         break;
    2300              :       case IFN_CTZ:
    2301              :         /* ctzll (x) == ctz (x) for unsigned or signed x != 0.  */
    2302              :         break;
    2303              :       case IFN_FFS:
    2304              :         /* ffsll (x) == ffs (x) for unsigned or signed x.  */
    2305              :         break;
    2306              :       default:
    2307              :         gcc_unreachable ();
    2308              :       }
    2309              : 
    2310              :   /* For zero the pattern gives the value of the target for VEC_TYPE plus
    2311              :      ADDEND.  If the call has another value at zero, select it.  */
    2312         1735 :   tree zero_val = NULL_TREE;
    2313         1735 :   if ((ifn == IFN_CLZ || ifn == IFN_CTZ)
    2314         1383 :       && gimple_call_internal_p (call_stmt)
    2315         1887 :       && gimple_call_num_args (call_stmt) == 2)
    2316              :     {
    2317          138 :       int val;
    2318          138 :       int d = (ifn == IFN_CLZ
    2319          138 :                ? CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val)
    2320          174 :                : CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val));
    2321          138 :       if (d != 2
    2322          138 :           || tree_to_shwi (gimple_call_arg (call_stmt, 1)) != val + addend)
    2323           94 :         zero_val = gimple_call_arg (call_stmt, 1);
    2324              :     }
    2325              : 
    2326              :   /* Do it only if the backend has popcount<vector_mode>2 etc. pattern.  */
    2327         1735 :   bool supported
    2328         1735 :     = direct_internal_fn_supported_p (ifn, vec_type, OPTIMIZE_FOR_SPEED);
    2329              :   /* We cannot rely on the reduction check in vect_recog_ctz_ffs_pattern
    2330              :      because we do not set up the temporary pattern stmt finally for
    2331              :      the recursive analysis.  */
    2332         1735 :   if (!supported && vect_is_reduction (stmt_vinfo))
    2333              :     return NULL;
    2334         1719 :   if (!supported)
    2335         1542 :     switch (ifn)
    2336              :       {
    2337              :       case IFN_POPCOUNT:
    2338              :       case IFN_CLZ:
    2339              :         return NULL;
    2340           57 :       case IFN_FFS:
    2341              :         /* vect_recog_ctz_ffs_pattern can implement ffs using ctz.  */
    2342           57 :         if (direct_internal_fn_supported_p (IFN_CTZ, vec_type,
    2343              :                                             OPTIMIZE_FOR_SPEED))
    2344              :           break;
    2345              :         /* FALLTHRU */
    2346          552 :       case IFN_CTZ:
    2347              :         /* vect_recog_ctz_ffs_pattern can implement ffs or ctz using
    2348              :            clz or popcount.  */
    2349          552 :         if (direct_internal_fn_supported_p (IFN_CLZ, vec_type,
    2350              :                                             OPTIMIZE_FOR_SPEED))
    2351              :           break;
    2352          492 :         if (direct_internal_fn_supported_p (IFN_POPCOUNT, vec_type,
    2353              :                                             OPTIMIZE_FOR_SPEED))
    2354              :           break;
    2355              :         return NULL;
    2356              :       default:
    2357              :         gcc_unreachable ();
    2358              :       }
    2359              : 
    2360          318 :   vect_pattern_detected ("vec_recog_popcount_clz_ctz_ffs_pattern",
    2361              :                          call_stmt);
    2362              : 
    2363          318 :   tree orig_lhs_type = lhs_type;
    2364          318 :   if (!type_has_mode_precision_p (lhs_type))
    2365            0 :     lhs_type = TREE_TYPE (vec_type);
    2366          318 :   tree arg_type = (lhs_type != orig_lhs_type
    2367          318 :                    ? lhs_type : TREE_TYPE (unprom_diff.op));
    2368              :   /* .CLZ, .CTZ and .POPCOUNT take an unsigned argument.  */
    2369          318 :   if (ifn != IFN_FFS && !TYPE_UNSIGNED (arg_type))
    2370           40 :     arg_type = unsigned_type_for (arg_type);
    2371          318 :   gimple *cast_stmt = NULL;
    2372          318 :   tree arg_vectype = vec_type;
    2373          318 :   if (arg_type != TREE_TYPE (unprom_diff.op))
    2374              :     {
    2375           40 :       arg_vectype = get_vectype_for_scalar_type (vinfo, arg_type);
    2376           40 :       if (!arg_vectype)
    2377              :         return NULL;
    2378           40 :       cast_stmt
    2379           40 :         = gimple_build_assign (vect_recog_temp_ssa_var (arg_type, NULL),
    2380              :                                NOP_EXPR, unprom_diff.op);
    2381           40 :       unprom_diff.op = gimple_assign_lhs (cast_stmt);
    2382              :     }
    2383              : 
    2384              :   /* Create B = .POPCOUNT (A).  */
    2385          318 :   new_var = vect_recog_temp_ssa_var (lhs_type, NULL);
    2386          318 :   tree arg2 = NULL_TREE;
    2387          318 :   int val;
    2388          318 :   if (ifn == IFN_CLZ
    2389          368 :       && CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type),
    2390              :                                     val) == 2)
    2391           48 :     arg2 = build_int_cst (integer_type_node, val);
    2392          270 :   else if (ifn == IFN_CTZ
    2393          363 :            && CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type),
    2394              :                                          val) == 2)
    2395           93 :     arg2 = build_int_cst (integer_type_node, val);
    2396              :   /* vect_recog_ctz_ffs_pattern selects the value at zero itself.  */
    2397          318 :   if (zero_val && !supported)
    2398              :     {
    2399              :       arg2 = zero_val;
    2400              :       zero_val = NULL_TREE;
    2401              :     }
    2402          318 :   if (arg2)
    2403          141 :     pattern_stmt = gimple_build_call_internal (ifn, 2, unprom_diff.op, arg2);
    2404              :   else
    2405          177 :     pattern_stmt = gimple_build_call_internal (ifn, 1, unprom_diff.op);
    2406          318 :   gimple_call_set_lhs (pattern_stmt, new_var);
    2407          318 :   gimple_set_location (pattern_stmt, gimple_location (last_stmt));
    2408          318 :   *type_out = vec_type;
    2409              : 
    2410          318 :   if (dump_enabled_p ())
    2411           24 :     dump_printf_loc (MSG_NOTE, vect_location,
    2412              :                      "created pattern stmt: %G", pattern_stmt);
    2413              : 
    2414          318 :   if (cast_stmt && supported)
    2415           28 :     append_pattern_def_seq (vinfo, stmt_vinfo, cast_stmt, arg_vectype);
    2416              : 
    2417          318 :   if (addend)
    2418              :     {
    2419           12 :       gcc_assert (supported);
    2420           12 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
    2421           12 :       tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
    2422           12 :       pattern_stmt = gimple_build_assign (ret_var, PLUS_EXPR, new_var,
    2423           12 :                                           build_int_cst (lhs_type, addend));
    2424              :     }
    2425          306 :   else if (!supported)
    2426              :     {
    2427          141 :       stmt_vec_info new_stmt_info = vinfo->add_stmt (pattern_stmt);
    2428          141 :       STMT_VINFO_VECTYPE (new_stmt_info) = vec_type;
    2429          141 :       pattern_stmt
    2430          141 :         = vect_recog_ctz_ffs_pattern (vinfo, new_stmt_info, type_out);
    2431          141 :       if (pattern_stmt == NULL)
    2432              :         return NULL;
    2433          141 :       if (cast_stmt)
    2434           12 :         append_pattern_def_seq (vinfo, stmt_vinfo, cast_stmt, arg_vectype);
    2435          141 :       if (gimple_seq seq = STMT_VINFO_PATTERN_DEF_SEQ (new_stmt_info))
    2436              :         {
    2437          141 :           gimple_seq *pseq = &STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo);
    2438          141 :           gimple_seq_add_seq_without_update (pseq, seq);
    2439              :         }
    2440              :     }
    2441              : 
    2442          318 :   if (zero_val)
    2443              :     {
    2444            0 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
    2445            0 :       tree op = unprom_diff.op;
    2446            0 :       tree cmp = vect_recog_temp_ssa_var (boolean_type_node, NULL);
    2447            0 :       gimple *cmp_stmt = gimple_build_assign (cmp, NE_EXPR, op,
    2448            0 :                                               build_zero_cst (TREE_TYPE (op)));
    2449            0 :       append_pattern_def_seq (vinfo, stmt_vinfo, cmp_stmt,
    2450            0 :                               truth_type_for (vec_type), TREE_TYPE (op));
    2451            0 :       tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
    2452            0 :       pattern_stmt = gimple_build_assign (ret_var, COND_EXPR, cmp,
    2453              :                                   gimple_get_lhs (pattern_stmt),
    2454              :                                   fold_convert (lhs_type, zero_val));
    2455              :     }
    2456              : 
    2457          318 :   if (lhs_type != orig_lhs_type)
    2458              :     {
    2459            0 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
    2460            0 :       tree ret_var = vect_recog_temp_ssa_var (orig_lhs_type, NULL);
    2461            0 :       pattern_stmt = gimple_build_assign (ret_var, NOP_EXPR,
    2462              :                                           gimple_get_lhs (pattern_stmt));
    2463              :     }
    2464              :   return pattern_stmt;
    2465              : }
    2466              : 
    2467              : /* Function vect_recog_pow_pattern
    2468              : 
    2469              :    Try to find the following pattern:
    2470              : 
    2471              :      x = POW (y, N);
    2472              : 
    2473              :    with POW being one of pow, powf, powi, powif and N being
    2474              :    either 2 or 0.5.
    2475              : 
    2476              :    Input:
    2477              : 
    2478              :    * STMT_VINFO: The stmt from which the pattern search begins.
    2479              : 
    2480              :    Output:
    2481              : 
    2482              :    * TYPE_OUT: The type of the output of this pattern.
    2483              : 
    2484              :    * Return value: A new stmt that will be used to replace the sequence of
    2485              :    stmts that constitute the pattern. In this case it will be:
    2486              :         x = x * x
    2487              :    or
    2488              :         x = sqrt (x)
    2489              : */
    2490              : 
    2491              : static gimple *
    2492     33053899 : vect_recog_pow_pattern (vec_info *vinfo,
    2493              :                         stmt_vec_info stmt_vinfo, tree *type_out)
    2494              : {
    2495     33053899 :   gimple *last_stmt = stmt_vinfo->stmt;
    2496     33053899 :   tree base, exp;
    2497     33053899 :   gimple *stmt;
    2498     33053899 :   tree var;
    2499              : 
    2500     33053899 :   if (!is_gimple_call (last_stmt) || gimple_call_lhs (last_stmt) == NULL)
    2501              :     return NULL;
    2502              : 
    2503      1563683 :   switch (gimple_call_combined_fn (last_stmt))
    2504              :     {
    2505          550 :     CASE_CFN_POW:
    2506          550 :     CASE_CFN_POWI:
    2507          550 :       break;
    2508              : 
    2509              :     default:
    2510              :       return NULL;
    2511              :     }
    2512              : 
    2513          550 :   base = gimple_call_arg (last_stmt, 0);
    2514          550 :   exp = gimple_call_arg (last_stmt, 1);
    2515          550 :   if (TREE_CODE (exp) != REAL_CST
    2516          495 :       && TREE_CODE (exp) != INTEGER_CST)
    2517              :     {
    2518          495 :       if (flag_unsafe_math_optimizations
    2519           57 :           && TREE_CODE (base) == REAL_CST
    2520          498 :           && gimple_call_builtin_p (last_stmt, BUILT_IN_NORMAL))
    2521              :         {
    2522            3 :           combined_fn log_cfn;
    2523            3 :           built_in_function exp_bfn;
    2524            3 :           switch (DECL_FUNCTION_CODE (gimple_call_fndecl (last_stmt)))
    2525              :             {
    2526              :             case BUILT_IN_POW:
    2527              :               log_cfn = CFN_BUILT_IN_LOG;
    2528              :               exp_bfn = BUILT_IN_EXP;
    2529              :               break;
    2530            0 :             case BUILT_IN_POWF:
    2531            0 :               log_cfn = CFN_BUILT_IN_LOGF;
    2532            0 :               exp_bfn = BUILT_IN_EXPF;
    2533            0 :               break;
    2534            0 :             case BUILT_IN_POWL:
    2535            0 :               log_cfn = CFN_BUILT_IN_LOGL;
    2536            0 :               exp_bfn = BUILT_IN_EXPL;
    2537            0 :               break;
    2538              :             default:
    2539              :               return NULL;
    2540              :             }
    2541            3 :           tree logc = fold_const_call (log_cfn, TREE_TYPE (base), base);
    2542            3 :           tree exp_decl = builtin_decl_implicit (exp_bfn);
    2543              :           /* Optimize pow (C, x) as exp (log (C) * x).  Normally match.pd
    2544              :              does that, but if C is a power of 2, we want to use
    2545              :              exp2 (log2 (C) * x) in the non-vectorized version, but for
    2546              :              vectorization we don't have vectorized exp2.  */
    2547            3 :           if (logc
    2548            3 :               && TREE_CODE (logc) == REAL_CST
    2549            3 :               && exp_decl
    2550            6 :               && lookup_attribute ("omp declare simd",
    2551            3 :                                    DECL_ATTRIBUTES (exp_decl)))
    2552              :             {
    2553            3 :               cgraph_node *node = cgraph_node::get_create (exp_decl);
    2554            3 :               if (node->simd_clones == NULL)
    2555              :                 {
    2556            2 :                   if (targetm.simd_clone.compute_vecsize_and_simdlen == NULL
    2557            2 :                       || node->definition)
    2558              :                     return NULL;
    2559            2 :                   expand_simd_clones (node);
    2560            2 :                   if (node->simd_clones == NULL)
    2561              :                     return NULL;
    2562              :                 }
    2563            3 :               *type_out = get_vectype_for_scalar_type (vinfo, TREE_TYPE (base));
    2564            3 :               if (!*type_out)
    2565              :                 return NULL;
    2566            3 :               tree def = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
    2567            3 :               gimple *g = gimple_build_assign (def, MULT_EXPR, exp, logc);
    2568            3 :               append_pattern_def_seq (vinfo, stmt_vinfo, g);
    2569            3 :               tree res = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
    2570            3 :               g = gimple_build_call (exp_decl, 1, def);
    2571            3 :               gimple_call_set_lhs (g, res);
    2572            3 :               return g;
    2573              :             }
    2574              :         }
    2575              : 
    2576              :       return NULL;
    2577              :     }
    2578              : 
    2579              :   /* We now have a pow or powi builtin function call with a constant
    2580              :      exponent.  */
    2581              : 
    2582              :   /* Catch squaring.  */
    2583           55 :   if ((tree_fits_shwi_p (exp)
    2584            0 :        && tree_to_shwi (exp) == 2)
    2585           55 :       || (TREE_CODE (exp) == REAL_CST
    2586           55 :           && real_equal (&TREE_REAL_CST (exp), &dconst2)))
    2587              :     {
    2588           11 :       if (!vect_supportable_direct_optab_p (vinfo, TREE_TYPE (base), MULT_EXPR,
    2589           11 :                                             TREE_TYPE (base), type_out))
    2590              :         return NULL;
    2591              : 
    2592            9 :       var = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
    2593            9 :       stmt = gimple_build_assign (var, MULT_EXPR, base, base);
    2594            9 :       return stmt;
    2595              :     }
    2596              : 
    2597              :   /* Catch square root.  Transforming pow(x,0.5) to sqrt(x) is unsafe
    2598              :      for signed zeroes and negative infinity.
    2599              :      pow(-0,0.5) = +0, while sqrt(-0) = -0.
    2600              :      pow(-Inf,0.5) = +Inf, while sqrt(-Inf) is NaN.  */
    2601           44 :   if (TREE_CODE (exp) == REAL_CST
    2602           44 :       && real_equal (&TREE_REAL_CST (exp), &dconsthalf)
    2603           10 :       && !tree_expr_maybe_real_minus_zero_p (base)
    2604           44 :       && (!tree_expr_maybe_infinite_p (base)
    2605            0 :           || tree_expr_nonnegative_p (base)))
    2606              :     {
    2607            0 :       *type_out = get_vectype_for_scalar_type (vinfo, TREE_TYPE (base));
    2608            0 :       if (*type_out
    2609            0 :           && direct_internal_fn_supported_p (IFN_SQRT, *type_out,
    2610              :                                              OPTIMIZE_FOR_SPEED))
    2611              :         {
    2612            0 :           gcall *stmt = gimple_build_call_internal (IFN_SQRT, 1, base);
    2613            0 :           var = vect_recog_temp_ssa_var (TREE_TYPE (base), stmt);
    2614            0 :           gimple_call_set_lhs (stmt, var);
    2615            0 :           gimple_call_set_nothrow (stmt, true);
    2616            0 :           return stmt;
    2617              :         }
    2618              :     }
    2619              : 
    2620              :   return NULL;
    2621              : }
    2622              : 
    2623              : 
    2624              : /* Function vect_recog_widen_sum_pattern
    2625              : 
    2626              :    Try to find the following pattern:
    2627              : 
    2628              :      type x_t;
    2629              :      TYPE x_T, sum = init;
    2630              :    loop:
    2631              :      sum_0 = phi <init, sum_1>
    2632              :      S1  x_t = *p;
    2633              :      S2  x_T = (TYPE) x_t;
    2634              :      S3  sum_1 = x_T + sum_0;
    2635              : 
    2636              :    where type 'TYPE' is at least double the size of type 'type', i.e - we're
    2637              :    summing elements of type 'type' into an accumulator of type 'TYPE'. This is
    2638              :    a special case of a reduction computation.
    2639              : 
    2640              :    Input:
    2641              : 
    2642              :    * STMT_VINFO: The stmt from which the pattern search begins. In the example,
    2643              :    when this function is called with S3, the pattern {S2,S3} will be detected.
    2644              : 
    2645              :    Output:
    2646              : 
    2647              :    * TYPE_OUT: The type of the output of this pattern.
    2648              : 
    2649              :    * Return value: A new stmt that will be used to replace the sequence of
    2650              :    stmts that constitute the pattern. In this case it will be:
    2651              :         WIDEN_SUM <x_t, sum_0>
    2652              : 
    2653              :    Note: The widening-sum idiom is a widening reduction pattern that is
    2654              :          vectorized without preserving all the intermediate results. It
    2655              :          produces only N/2 (widened) results (by summing up pairs of
    2656              :          intermediate results) rather than all N results.  Therefore, we
    2657              :          cannot allow this pattern when we want to get all the results and in
    2658              :          the correct order (as is the case when this computation is in an
    2659              :          inner-loop nested in an outer-loop that us being vectorized).  */
    2660              : 
    2661              : static gimple *
    2662     33053899 : vect_recog_widen_sum_pattern (vec_info *vinfo,
    2663              :                               stmt_vec_info stmt_vinfo, tree *type_out)
    2664              : {
    2665     33053899 :   gimple *last_stmt = stmt_vinfo->stmt;
    2666     33053899 :   tree oprnd0, oprnd1;
    2667     33053899 :   tree type;
    2668     33053899 :   gimple *pattern_stmt;
    2669     33053899 :   tree var;
    2670              : 
    2671              :   /* Look for the following pattern
    2672              :           DX = (TYPE) X;
    2673              :           sum_1 = DX + sum_0;
    2674              :      In which DX is at least double the size of X, and sum_1 has been
    2675              :      recognized as a reduction variable.
    2676              :    */
    2677              : 
    2678              :   /* Starting from LAST_STMT, follow the defs of its uses in search
    2679              :      of the above pattern.  */
    2680              : 
    2681     33053899 :   if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, PLUS_EXPR,
    2682              :                                        &oprnd0, &oprnd1)
    2683        53194 :       || TREE_CODE (oprnd0) != SSA_NAME
    2684     33106822 :       || !vinfo->lookup_def (oprnd0))
    2685              :     return NULL;
    2686              : 
    2687        52846 :   type = TREE_TYPE (gimple_get_lhs (last_stmt));
    2688              : 
    2689              :   /* So far so good.  Since last_stmt was detected as a (summation) reduction,
    2690              :      we know that oprnd1 is the reduction variable (defined by a loop-header
    2691              :      phi), and oprnd0 is an ssa-name defined by a stmt in the loop body.
    2692              :      Left to check that oprnd0 is defined by a cast from type 'type' to type
    2693              :      'TYPE'.  */
    2694              : 
    2695        52846 :   vect_unpromoted_value unprom0;
    2696        52846 :   if (!vect_look_through_possible_promotion (vinfo, oprnd0, &unprom0)
    2697        52846 :       || TYPE_PRECISION (unprom0.type) * 2 > TYPE_PRECISION (type))
    2698              :     return NULL;
    2699              : 
    2700         2372 :   vect_pattern_detected ("vect_recog_widen_sum_pattern", last_stmt);
    2701              : 
    2702              :   /* If the exact input type is not supported, widen it to an intermediate
    2703              :      type before adding it to the accumulator.  Start with the narrowest
    2704              :      type in order to retain the largest vectorization factor.  */
    2705         2372 :   tree input_type = unprom0.type;
    2706         2372 :   tree input_vectype = NULL_TREE;
    2707              : 
    2708         4438 :   while (!vect_supportable_conv_optab_p (vinfo, type, WIDEN_SUM_EXPR,
    2709              :                                          input_type, type_out, &input_vectype))
    2710              :     {
    2711         4438 :       unsigned int precision
    2712         4438 :         = vect_element_precision (TYPE_PRECISION (input_type) + 1);
    2713              : 
    2714         4438 :       if (precision > TYPE_PRECISION (type) / 2)
    2715              :         return NULL;
    2716              : 
    2717         2066 :       input_type = build_nonstandard_integer_type
    2718         2066 :         (precision, TYPE_UNSIGNED (unprom0.type));
    2719              :     }
    2720              : 
    2721            0 :   tree input = vect_convert_input (vinfo, stmt_vinfo, input_type, &unprom0,
    2722              :                                    input_vectype);
    2723              : 
    2724            0 :   var = vect_recog_temp_ssa_var (type, NULL);
    2725            0 :   pattern_stmt = gimple_build_assign (var, WIDEN_SUM_EXPR, input, oprnd1);
    2726              : 
    2727            0 :   return pattern_stmt;
    2728              : }
    2729              : 
    2730              : /* Function vect_recog_bitfield_ref_pattern
    2731              : 
    2732              :    Try to find the following pattern:
    2733              : 
    2734              :    bf_value = BIT_FIELD_REF (container, bitsize, bitpos);
    2735              :    result = (type_out) bf_value;
    2736              : 
    2737              :    or
    2738              : 
    2739              :    if (BIT_FIELD_REF (container, bitsize, bitpos) `cmp` <constant>)
    2740              : 
    2741              :    where type_out is a non-bitfield type, that is to say, it's precision matches
    2742              :    2^(TYPE_SIZE(type_out) - (TYPE_UNSIGNED (type_out) ? 1 : 2)).
    2743              : 
    2744              :    Input:
    2745              : 
    2746              :    * STMT_VINFO: The stmt from which the pattern search begins.
    2747              :    here it starts with:
    2748              :    result = (type_out) bf_value;
    2749              : 
    2750              :    or
    2751              : 
    2752              :    if (BIT_FIELD_REF (container, bitsize, bitpos) `cmp` <constant>)
    2753              : 
    2754              :    Output:
    2755              : 
    2756              :    * TYPE_OUT: The vector type of the output of this pattern.
    2757              : 
    2758              :    * Return value: A new stmt that will be used to replace the sequence of
    2759              :    stmts that constitute the pattern. If the precision of type_out is bigger
    2760              :    than the precision type of _1 we perform the widening before the shifting,
    2761              :    since the new precision will be large enough to shift the value and moving
    2762              :    widening operations up the statement chain enables the generation of
    2763              :    widening loads.  If we are widening and the operation after the pattern is
    2764              :    an addition then we mask first and shift later, to enable the generation of
    2765              :    shifting adds.  In the case of narrowing we will always mask first, shift
    2766              :    last and then perform a narrowing operation.  This will enable the
    2767              :    generation of narrowing shifts.
    2768              : 
    2769              :    Widening with mask first, shift later:
    2770              :    container = (type_out) container;
    2771              :    masked = container & (((1 << bitsize) - 1) << bitpos);
    2772              :    result = masked >> bitpos;
    2773              : 
    2774              :    Widening with shift first, mask last:
    2775              :    container = (type_out) container;
    2776              :    shifted = container >> bitpos;
    2777              :    result = shifted & ((1 << bitsize) - 1);
    2778              : 
    2779              :    Narrowing:
    2780              :    masked = container & (((1 << bitsize) - 1) << bitpos);
    2781              :    result = masked >> bitpos;
    2782              :    result = (type_out) result;
    2783              : 
    2784              :    If the bitfield is signed and it's wider than type_out, we need to
    2785              :    keep the result sign-extended:
    2786              :    container = (type) container;
    2787              :    masked = container << (prec - bitsize - bitpos);
    2788              :    result = (type_out) (masked >> (prec - bitsize));
    2789              : 
    2790              :    Here type is the signed variant of the wider of type_out and the type
    2791              :    of container.
    2792              : 
    2793              :    The shifting is always optional depending on whether bitpos != 0.
    2794              : 
    2795              :    When the original bitfield was inside a gcond then an new gcond is also
    2796              :    generated with the newly `result` as the operand to the comparison.
    2797              : 
    2798              : */
    2799              : 
    2800              : static gimple *
    2801     32996364 : vect_recog_bitfield_ref_pattern (vec_info *vinfo, stmt_vec_info stmt_info,
    2802              :                                  tree *type_out)
    2803              : {
    2804     32996364 :   gimple *bf_stmt = NULL;
    2805     32996364 :   tree lhs = NULL_TREE;
    2806     32996364 :   tree ret_type = NULL_TREE;
    2807     32996364 :   gimple *stmt = STMT_VINFO_STMT (stmt_info);
    2808     32996364 :   if (gcond *cond_stmt = dyn_cast <gcond *> (stmt))
    2809              :     {
    2810      5460563 :       tree op = gimple_cond_lhs (cond_stmt);
    2811      5460563 :       if (TREE_CODE (op) != SSA_NAME)
    2812              :         return NULL;
    2813      5460254 :       bf_stmt = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (op));
    2814      5460254 :       if (TREE_CODE (gimple_cond_rhs (cond_stmt)) != INTEGER_CST)
    2815              :         return NULL;
    2816              :     }
    2817     27535801 :   else if (is_gimple_assign (stmt)
    2818     23093102 :            && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
    2819     30668353 :            && TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME)
    2820              :     {
    2821      3089984 :       gimple *second_stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmt));
    2822      3089984 :       bf_stmt = dyn_cast <gassign *> (second_stmt);
    2823      3089984 :       lhs = gimple_assign_lhs (stmt);
    2824      3089984 :       ret_type = TREE_TYPE (lhs);
    2825              :     }
    2826              : 
    2827      6620822 :   if (!bf_stmt
    2828      6620822 :       || gimple_assign_rhs_code (bf_stmt) != BIT_FIELD_REF)
    2829              :     return NULL;
    2830              : 
    2831        17399 :   tree bf_ref = gimple_assign_rhs1 (bf_stmt);
    2832        17399 :   tree container = TREE_OPERAND (bf_ref, 0);
    2833        17399 :   ret_type = ret_type ? ret_type : TREE_TYPE (container);
    2834              : 
    2835        17399 :   if (!bit_field_offset (bf_ref).is_constant ()
    2836        17399 :       || !bit_field_size (bf_ref).is_constant ()
    2837        17399 :       || !tree_fits_uhwi_p (TYPE_SIZE (TREE_TYPE (container))))
    2838              :     return NULL;
    2839              : 
    2840        34420 :   if (!INTEGRAL_TYPE_P (TREE_TYPE (bf_ref))
    2841        17397 :       || !INTEGRAL_TYPE_P (TREE_TYPE (container))
    2842        19572 :       || TYPE_MODE (TREE_TYPE (container)) == E_BLKmode)
    2843              :     return NULL;
    2844              : 
    2845         2173 :   gimple *use_stmt, *pattern_stmt;
    2846         2173 :   use_operand_p use_p;
    2847         2173 :   bool shift_first = true;
    2848         2173 :   tree container_type = TREE_TYPE (container);
    2849         2173 :   tree vectype = get_vectype_for_scalar_type (vinfo, container_type);
    2850              : 
    2851              :   /* Calculate shift_n before the adjustments for widening loads, otherwise
    2852              :      the container may change and we have to consider offset change for
    2853              :      widening loads on big endianness.  The shift_n calculated here can be
    2854              :      independent of widening.  */
    2855         2173 :   unsigned HOST_WIDE_INT shift_n = bit_field_offset (bf_ref).to_constant ();
    2856         2173 :   unsigned HOST_WIDE_INT mask_width = bit_field_size (bf_ref).to_constant ();
    2857         2173 :   unsigned HOST_WIDE_INT prec = tree_to_uhwi (TYPE_SIZE (container_type));
    2858         2173 :   if (BYTES_BIG_ENDIAN)
    2859              :     shift_n = prec - shift_n - mask_width;
    2860              : 
    2861         2173 :   bool ref_sext = (!TYPE_UNSIGNED (TREE_TYPE (bf_ref)) &&
    2862         1400 :                    TYPE_PRECISION (ret_type) > mask_width);
    2863         2173 :   bool load_widen = (TYPE_PRECISION (TREE_TYPE (container)) <
    2864         2173 :                      TYPE_PRECISION (ret_type));
    2865              : 
    2866              :   /* We move the conversion earlier if the loaded type is smaller than the
    2867              :      return type to enable the use of widening loads.  And if we need a
    2868              :      sign extension, we need to convert the loaded value early to a signed
    2869              :      type as well.  */
    2870         2173 :   if (ref_sext || load_widen)
    2871              :     {
    2872          947 :       tree type = load_widen ? ret_type : container_type;
    2873          947 :       if (ref_sext)
    2874          908 :         type = gimple_signed_type (type);
    2875          947 :       pattern_stmt = gimple_build_assign (vect_recog_temp_ssa_var (type),
    2876              :                                           NOP_EXPR, container);
    2877          947 :       container = gimple_get_lhs (pattern_stmt);
    2878          947 :       container_type = TREE_TYPE (container);
    2879          947 :       prec = tree_to_uhwi (TYPE_SIZE (container_type));
    2880          947 :       vectype = get_vectype_for_scalar_type (vinfo, container_type);
    2881          947 :       append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
    2882              :     }
    2883         1226 :   else if (!useless_type_conversion_p (TREE_TYPE (container), ret_type))
    2884              :     /* If we are doing the conversion last then also delay the shift as we may
    2885              :        be able to combine the shift and conversion in certain cases.  */
    2886              :     shift_first = false;
    2887              : 
    2888              :   /* If the only use of the result of this BIT_FIELD_REF + CONVERT is a
    2889              :      PLUS_EXPR then do the shift last as some targets can combine the shift and
    2890              :      add into a single instruction.  */
    2891         1416 :   if (lhs && !is_pattern_stmt_p (stmt_info)
    2892         3589 :       && single_imm_use (lhs, &use_p, &use_stmt))
    2893              :     {
    2894         1049 :       if (gimple_code (use_stmt) == GIMPLE_ASSIGN
    2895         1049 :           && gimple_assign_rhs_code (use_stmt) == PLUS_EXPR)
    2896              :         shift_first = false;
    2897              :     }
    2898              : 
    2899              :   /* If we don't have to shift we only generate the mask, so just fix the
    2900              :      code-path to shift_first.  */
    2901         2173 :   if (shift_n == 0)
    2902          763 :     shift_first = true;
    2903              : 
    2904         2173 :   tree result;
    2905         2173 :   if (shift_first && !ref_sext)
    2906              :     {
    2907          513 :       tree shifted = container;
    2908          513 :       if (shift_n)
    2909              :         {
    2910           65 :           pattern_stmt
    2911           65 :             = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
    2912              :                                    RSHIFT_EXPR, container,
    2913           65 :                                    build_int_cst (sizetype, shift_n));
    2914           65 :           shifted = gimple_assign_lhs (pattern_stmt);
    2915           65 :           append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
    2916              :         }
    2917              : 
    2918          513 :       tree mask = wide_int_to_tree (container_type,
    2919          513 :                                     wi::mask (mask_width, false, prec));
    2920              : 
    2921          513 :       pattern_stmt
    2922          513 :         = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
    2923              :                                BIT_AND_EXPR, shifted, mask);
    2924          513 :       result = gimple_assign_lhs (pattern_stmt);
    2925              :     }
    2926              :   else
    2927              :     {
    2928         1660 :       tree temp = vect_recog_temp_ssa_var (container_type);
    2929         1660 :       if (!ref_sext)
    2930              :         {
    2931          752 :           tree mask = wide_int_to_tree (container_type,
    2932          752 :                                         wi::shifted_mask (shift_n,
    2933              :                                                           mask_width,
    2934              :                                                           false, prec));
    2935          752 :           pattern_stmt = gimple_build_assign (temp, BIT_AND_EXPR,
    2936              :                                               container, mask);
    2937              :         }
    2938              :       else
    2939              :         {
    2940          908 :           HOST_WIDE_INT shl = prec - shift_n - mask_width;
    2941          908 :           shift_n += shl;
    2942          908 :           pattern_stmt = gimple_build_assign (temp, LSHIFT_EXPR,
    2943              :                                               container,
    2944              :                                               build_int_cst (sizetype,
    2945          908 :                                                              shl));
    2946              :         }
    2947              : 
    2948         1660 :       tree masked = gimple_assign_lhs (pattern_stmt);
    2949         1660 :       append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
    2950         1660 :       pattern_stmt
    2951         1660 :         = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
    2952              :                                RSHIFT_EXPR, masked,
    2953         1660 :                                build_int_cst (sizetype, shift_n));
    2954         1660 :       result = gimple_assign_lhs (pattern_stmt);
    2955              :     }
    2956              : 
    2957         2173 :   if (!useless_type_conversion_p (TREE_TYPE (result), ret_type))
    2958              :     {
    2959         1444 :       append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
    2960         1444 :       pattern_stmt
    2961         1444 :         = gimple_build_assign (vect_recog_temp_ssa_var (ret_type),
    2962              :                                NOP_EXPR, result);
    2963              :     }
    2964              : 
    2965         2173 :   if (!lhs)
    2966              :     {
    2967          757 :       if (!vectype)
    2968              :         return NULL;
    2969              : 
    2970          619 :       append_pattern_def_seq (vinfo, stmt_info, pattern_stmt, vectype);
    2971          619 :       vectype = truth_type_for (vectype);
    2972              : 
    2973              :       /* FIXME: This part extracts the boolean value out of the bitfield in the
    2974              :                 same way as vect_recog_gcond_pattern does.  However because
    2975              :                 patterns cannot match the same root twice,  when we handle and
    2976              :                 lower the bitfield in the gcond, vect_recog_gcond_pattern can't
    2977              :                 apply anymore.  We should really fix it so that we don't need to
    2978              :                 duplicate transformations like these.  */
    2979          619 :       tree new_lhs = vect_recog_temp_ssa_var (boolean_type_node, NULL);
    2980          619 :       gcond *cond_stmt = dyn_cast <gcond *> (stmt_info->stmt);
    2981          619 :       tree cond_cst = gimple_cond_rhs (cond_stmt);
    2982          619 :       gimple *new_stmt
    2983          619 :         = gimple_build_assign (new_lhs, gimple_cond_code (cond_stmt),
    2984              :                                gimple_get_lhs (pattern_stmt),
    2985              :                                fold_convert (container_type, cond_cst));
    2986          619 :       append_pattern_def_seq (vinfo, stmt_info, new_stmt, vectype, container_type);
    2987          619 :       pattern_stmt
    2988          619 :         = gimple_build_cond (NE_EXPR, new_lhs,
    2989          619 :                              build_zero_cst (TREE_TYPE (new_lhs)),
    2990              :                              NULL_TREE, NULL_TREE);
    2991              :     }
    2992              : 
    2993         2035 :   *type_out = STMT_VINFO_VECTYPE (stmt_info);
    2994         2035 :   vect_pattern_detected ("bitfield_ref pattern", stmt_info->stmt);
    2995              : 
    2996         2035 :   return pattern_stmt;
    2997              : }
    2998              : 
    2999              : /* Function vect_recog_bit_insert_pattern
    3000              : 
    3001              :    Try to find the following pattern:
    3002              : 
    3003              :    written = BIT_INSERT_EXPR (container, value, bitpos);
    3004              : 
    3005              :    Input:
    3006              : 
    3007              :    * STMT_VINFO: The stmt we want to replace.
    3008              : 
    3009              :    Output:
    3010              : 
    3011              :    * TYPE_OUT: The vector type of the output of this pattern.
    3012              : 
    3013              :    * Return value: A new stmt that will be used to replace the sequence of
    3014              :    stmts that constitute the pattern. In this case it will be:
    3015              :    value = (container_type) value;          // Make sure
    3016              :    shifted = value << bitpos;                 // Shift value into place
    3017              :    masked = shifted & (mask << bitpos);           // Mask off the non-relevant bits in
    3018              :                                             // the 'to-write value'.
    3019              :    cleared = container & ~(mask << bitpos); // Clearing the bits we want to
    3020              :                                             // write to from the value we want
    3021              :                                             // to write to.
    3022              :    written = cleared | masked;              // Write bits.
    3023              : 
    3024              : 
    3025              :    where mask = ((1 << TYPE_PRECISION (value)) - 1), a mask to keep the number of
    3026              :    bits corresponding to the real size of the bitfield value we are writing to.
    3027              :    The shifting is always optional depending on whether bitpos != 0.
    3028              : 
    3029              : */
    3030              : 
    3031              : static gimple *
    3032     32999414 : vect_recog_bit_insert_pattern (vec_info *vinfo, stmt_vec_info stmt_info,
    3033              :                                tree *type_out)
    3034              : {
    3035     32999414 :   gassign *bf_stmt = dyn_cast <gassign *> (stmt_info->stmt);
    3036     30820529 :   if (!bf_stmt || gimple_assign_rhs_code (bf_stmt) != BIT_INSERT_EXPR)
    3037              :     return NULL;
    3038              : 
    3039          690 :   tree container = gimple_assign_rhs1 (bf_stmt);
    3040          690 :   tree value = gimple_assign_rhs2 (bf_stmt);
    3041          690 :   tree shift = gimple_assign_rhs3 (bf_stmt);
    3042              : 
    3043          690 :   tree bf_type = TREE_TYPE (value);
    3044          690 :   tree container_type = TREE_TYPE (container);
    3045              : 
    3046          690 :   if (!INTEGRAL_TYPE_P (container_type)
    3047          690 :       || !tree_fits_uhwi_p (TYPE_SIZE (container_type)))
    3048              :     return NULL;
    3049              : 
    3050          508 :   gimple *pattern_stmt;
    3051              : 
    3052          508 :   vect_unpromoted_value unprom;
    3053          508 :   unprom.set_op (value, vect_internal_def);
    3054          508 :   value = vect_convert_input (vinfo, stmt_info, container_type, &unprom,
    3055              :                               get_vectype_for_scalar_type (vinfo,
    3056              :                                                            container_type));
    3057              : 
    3058          508 :   unsigned HOST_WIDE_INT mask_width = TYPE_PRECISION (bf_type);
    3059          508 :   unsigned HOST_WIDE_INT prec = tree_to_uhwi (TYPE_SIZE (container_type));
    3060          508 :   unsigned HOST_WIDE_INT shift_n = tree_to_uhwi (shift);
    3061          508 :   if (BYTES_BIG_ENDIAN)
    3062              :     {
    3063              :       shift_n = prec - shift_n - mask_width;
    3064              :       shift = build_int_cst (TREE_TYPE (shift), shift_n);
    3065              :     }
    3066              : 
    3067          508 :   if (!useless_type_conversion_p (TREE_TYPE (value), container_type))
    3068              :     {
    3069            0 :       pattern_stmt =
    3070            0 :         gimple_build_assign (vect_recog_temp_ssa_var (container_type),
    3071              :                              NOP_EXPR, value);
    3072            0 :       append_pattern_def_seq (vinfo, stmt_info, pattern_stmt);
    3073            0 :       value = gimple_get_lhs (pattern_stmt);
    3074              :     }
    3075              : 
    3076              :   /* Shift VALUE into place.  */
    3077          508 :   tree shifted = value;
    3078          508 :   if (shift_n)
    3079              :     {
    3080          249 :       gimple_seq stmts = NULL;
    3081          249 :       shifted
    3082          249 :         = gimple_build (&stmts, LSHIFT_EXPR, container_type, value, shift);
    3083          249 :       if (!gimple_seq_empty_p (stmts))
    3084          112 :         append_pattern_def_seq (vinfo, stmt_info,
    3085              :                                 gimple_seq_first_stmt (stmts));
    3086              :     }
    3087              : 
    3088          508 :   tree mask_t
    3089          508 :     = wide_int_to_tree (container_type,
    3090          508 :                         wi::shifted_mask (shift_n, mask_width, false, prec));
    3091              : 
    3092              :   /* Clear bits we don't want to write back from SHIFTED.  */
    3093          508 :   gimple_seq stmts = NULL;
    3094          508 :   tree masked = gimple_build (&stmts, BIT_AND_EXPR, container_type, shifted,
    3095              :                               mask_t);
    3096          508 :   if (!gimple_seq_empty_p (stmts))
    3097              :     {
    3098          118 :       pattern_stmt = gimple_seq_first_stmt (stmts);
    3099          118 :       append_pattern_def_seq (vinfo, stmt_info, pattern_stmt);
    3100              :     }
    3101              : 
    3102              :   /* Mask off the bits in the container that we are to write to.  */
    3103          508 :   mask_t = wide_int_to_tree (container_type,
    3104          508 :                              wi::shifted_mask (shift_n, mask_width, true, prec));
    3105          508 :   tree cleared = vect_recog_temp_ssa_var (container_type);
    3106          508 :   pattern_stmt = gimple_build_assign (cleared, BIT_AND_EXPR, container, mask_t);
    3107          508 :   append_pattern_def_seq (vinfo, stmt_info, pattern_stmt);
    3108              : 
    3109              :   /* Write MASKED into CLEARED.  */
    3110          508 :   pattern_stmt
    3111          508 :     = gimple_build_assign (vect_recog_temp_ssa_var (container_type),
    3112              :                            BIT_IOR_EXPR, cleared, masked);
    3113              : 
    3114          508 :   *type_out = STMT_VINFO_VECTYPE (stmt_info);
    3115          508 :   vect_pattern_detected ("bit_insert pattern", stmt_info->stmt);
    3116              : 
    3117          508 :   return pattern_stmt;
    3118              : }
    3119              : 
    3120              : 
    3121              : /* Recognize cases in which an operation is performed in one type WTYPE
    3122              :    but could be done more efficiently in a narrower type NTYPE.  For example,
    3123              :    if we have:
    3124              : 
    3125              :      ATYPE a;  // narrower than NTYPE
    3126              :      BTYPE b;  // narrower than NTYPE
    3127              :      WTYPE aw = (WTYPE) a;
    3128              :      WTYPE bw = (WTYPE) b;
    3129              :      WTYPE res = aw + bw;  // only uses of aw and bw
    3130              : 
    3131              :    then it would be more efficient to do:
    3132              : 
    3133              :      NTYPE an = (NTYPE) a;
    3134              :      NTYPE bn = (NTYPE) b;
    3135              :      NTYPE resn = an + bn;
    3136              :      WTYPE res = (WTYPE) resn;
    3137              : 
    3138              :    Other situations include things like:
    3139              : 
    3140              :      ATYPE a;  // NTYPE or narrower
    3141              :      WTYPE aw = (WTYPE) a;
    3142              :      WTYPE res = aw + b;
    3143              : 
    3144              :    when only "(NTYPE) res" is significant.  In that case it's more efficient
    3145              :    to truncate "b" and do the operation on NTYPE instead:
    3146              : 
    3147              :      NTYPE an = (NTYPE) a;
    3148              :      NTYPE bn = (NTYPE) b;  // truncation
    3149              :      NTYPE resn = an + bn;
    3150              :      WTYPE res = (WTYPE) resn;
    3151              : 
    3152              :    All users of "res" should then use "resn" instead, making the final
    3153              :    statement dead (not marked as relevant).  The final statement is still
    3154              :    needed to maintain the type correctness of the IR.
    3155              : 
    3156              :    vect_determine_precisions has already determined the minimum
    3157              :    precision of the operation and the minimum precision required
    3158              :    by users of the result.  */
    3159              : 
    3160              : static gimple *
    3161     32999870 : vect_recog_over_widening_pattern (vec_info *vinfo,
    3162              :                                   stmt_vec_info last_stmt_info, tree *type_out)
    3163              : {
    3164     32999870 :   gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
    3165     23097227 :   if (!last_stmt)
    3166              :     return NULL;
    3167              : 
    3168              :   /* See whether we have found that this operation can be done on a
    3169              :      narrower type without changing its semantics.  */
    3170     23097227 :   unsigned int new_precision = last_stmt_info->operation_precision;
    3171     23097227 :   if (!new_precision)
    3172              :     return NULL;
    3173              : 
    3174      1697544 :   tree lhs = gimple_assign_lhs (last_stmt);
    3175      1697544 :   tree type = TREE_TYPE (lhs);
    3176      1697544 :   tree_code code = gimple_assign_rhs_code (last_stmt);
    3177              : 
    3178              :   /* Punt for reductions where we don't handle the type conversions.  */
    3179      1697544 :   if (vect_is_reduction (last_stmt_info))
    3180              :     return NULL;
    3181              : 
    3182              :   /* Keep the first operand of a COND_EXPR as-is: only the other two
    3183              :      operands are interesting.  */
    3184      1689197 :   unsigned int first_op = (code == COND_EXPR ? 2 : 1);
    3185              : 
    3186              :   /* Check the operands.  */
    3187      1689197 :   unsigned int nops = gimple_num_ops (last_stmt) - first_op;
    3188      1689197 :   auto_vec <vect_unpromoted_value, 3> unprom (nops);
    3189      1689197 :   unprom.quick_grow_cleared (nops);
    3190      1689197 :   unsigned int min_precision = 0;
    3191      1689197 :   bool single_use_p = false;
    3192      6735864 :   for (unsigned int i = 0; i < nops; ++i)
    3193              :     {
    3194      3359355 :       tree op = gimple_op (last_stmt, first_op + i);
    3195      3359355 :       if (TREE_CODE (op) == INTEGER_CST)
    3196      1537055 :         unprom[i].set_op (op, vect_constant_def);
    3197      1822300 :       else if (TREE_CODE (op) == SSA_NAME)
    3198              :         {
    3199      1822300 :           bool op_single_use_p = true;
    3200      1822300 :           if (!vect_look_through_possible_promotion (vinfo, op, &unprom[i],
    3201              :                                                      &op_single_use_p))
    3202         1885 :             return NULL;
    3203              :           /* If:
    3204              : 
    3205              :              (1) N bits of the result are needed;
    3206              :              (2) all inputs are widened from M<N bits; and
    3207              :              (3) one operand OP is a single-use SSA name
    3208              : 
    3209              :              we can shift the M->N widening from OP to the output
    3210              :              without changing the number or type of extensions involved.
    3211              :              This then reduces the number of copies of STMT_INFO.
    3212              : 
    3213              :              If instead of (3) more than one operand is a single-use SSA name,
    3214              :              shifting the extension to the output is even more of a win.
    3215              : 
    3216              :              If instead:
    3217              : 
    3218              :              (1) N bits of the result are needed;
    3219              :              (2) one operand OP2 is widened from M2<N bits;
    3220              :              (3) another operand OP1 is widened from M1<M2 bits; and
    3221              :              (4) both OP1 and OP2 are single-use
    3222              : 
    3223              :              the choice is between:
    3224              : 
    3225              :              (a) truncating OP2 to M1, doing the operation on M1,
    3226              :                  and then widening the result to N
    3227              : 
    3228              :              (b) widening OP1 to M2, doing the operation on M2, and then
    3229              :                  widening the result to N
    3230              : 
    3231              :              Both shift the M2->N widening of the inputs to the output.
    3232              :              (a) additionally shifts the M1->M2 widening to the output;
    3233              :              it requires fewer copies of STMT_INFO but requires an extra
    3234              :              M2->M1 truncation.
    3235              : 
    3236              :              Which is better will depend on the complexity and cost of
    3237              :              STMT_INFO, which is hard to predict at this stage.  However,
    3238              :              a clear tie-breaker in favor of (b) is the fact that the
    3239              :              truncation in (a) increases the length of the operation chain.
    3240              : 
    3241              :              If instead of (4) only one of OP1 or OP2 is single-use,
    3242              :              (b) is still a win over doing the operation in N bits:
    3243              :              it still shifts the M2->N widening on the single-use operand
    3244              :              to the output and reduces the number of STMT_INFO copies.
    3245              : 
    3246              :              If neither operand is single-use then operating on fewer than
    3247              :              N bits might lead to more extensions overall.  Whether it does
    3248              :              or not depends on global information about the vectorization
    3249              :              region, and whether that's a good trade-off would again
    3250              :              depend on the complexity and cost of the statements involved,
    3251              :              as well as things like register pressure that are not normally
    3252              :              modelled at this stage.  We therefore ignore these cases
    3253              :              and just optimize the clear single-use wins above.
    3254              : 
    3255              :              Thus we take the maximum precision of the unpromoted operands
    3256              :              and record whether any operand is single-use.  */
    3257      1820415 :           if (unprom[i].dt == vect_internal_def)
    3258              :             {
    3259      1048758 :               min_precision = MAX (min_precision,
    3260              :                                    TYPE_PRECISION (unprom[i].type));
    3261      1048758 :               single_use_p |= op_single_use_p;
    3262              :             }
    3263              :         }
    3264              :       else
    3265              :         return NULL;
    3266              :     }
    3267              : 
    3268              :   /* Although the operation could be done in operation_precision, we have
    3269              :      to balance that against introducing extra truncations or extensions.
    3270              :      Calculate the minimum precision that can be handled efficiently.
    3271              : 
    3272              :      The loop above determined that the operation could be handled
    3273              :      efficiently in MIN_PRECISION if SINGLE_USE_P; this would shift an
    3274              :      extension from the inputs to the output without introducing more
    3275              :      instructions, and would reduce the number of instructions required
    3276              :      for STMT_INFO itself.
    3277              : 
    3278              :      vect_determine_precisions has also determined that the result only
    3279              :      needs min_output_precision bits.  Truncating by a factor of N times
    3280              :      requires a tree of N - 1 instructions, so if TYPE is N times wider
    3281              :      than min_output_precision, doing the operation in TYPE and truncating
    3282              :      the result requires N + (N - 1) = 2N - 1 instructions per output vector.
    3283              :      In contrast:
    3284              : 
    3285              :      - truncating the input to a unary operation and doing the operation
    3286              :        in the new type requires at most N - 1 + 1 = N instructions per
    3287              :        output vector
    3288              : 
    3289              :      - doing the same for a binary operation requires at most
    3290              :        (N - 1) * 2 + 1 = 2N - 1 instructions per output vector
    3291              : 
    3292              :      Both unary and binary operations require fewer instructions than
    3293              :      this if the operands were extended from a suitable truncated form.
    3294              :      Thus there is usually nothing to lose by doing operations in
    3295              :      min_output_precision bits, but there can be something to gain.  */
    3296      1687312 :   if (!single_use_p)
    3297      1328141 :     min_precision = last_stmt_info->min_output_precision;
    3298              :   else
    3299       359171 :     min_precision = MIN (min_precision, last_stmt_info->min_output_precision);
    3300              : 
    3301              :   /* Apply the minimum efficient precision we just calculated.  */
    3302      1687312 :   if (new_precision < min_precision)
    3303              :     new_precision = min_precision;
    3304      1687312 :   new_precision = vect_element_precision (new_precision);
    3305      1687312 :   if (new_precision >= TYPE_PRECISION (type))
    3306              :     return NULL;
    3307              : 
    3308       161026 :   vect_pattern_detected ("vect_recog_over_widening_pattern", last_stmt);
    3309              : 
    3310       161026 :   *type_out = get_vectype_for_scalar_type (vinfo, type);
    3311       161026 :   if (!*type_out)
    3312              :     return NULL;
    3313              : 
    3314              :   /* We've found a viable pattern.  Get the new type of the operation.  */
    3315       141212 :   bool unsigned_p = (last_stmt_info->operation_sign == UNSIGNED);
    3316       141212 :   tree new_type = build_nonstandard_integer_type (new_precision, unsigned_p);
    3317              : 
    3318              :   /* If we're truncating an operation, we need to make sure that we
    3319              :      don't introduce new undefined overflow.  The codes tested here are
    3320              :      a subset of those accepted by vect_truncatable_operation_p.  */
    3321       141212 :   tree op_type = new_type;
    3322       141212 :   if (TYPE_OVERFLOW_UNDEFINED (new_type)
    3323       182273 :       && (code == PLUS_EXPR || code == MINUS_EXPR || code == MULT_EXPR))
    3324        28096 :     op_type = build_nonstandard_integer_type (new_precision, true);
    3325              : 
    3326       141212 :   tree new_vectype = get_vectype_for_scalar_type (vinfo, new_type);
    3327       141212 :   tree op_vectype = get_vectype_for_scalar_type (vinfo, op_type);
    3328       141212 :   if (!new_vectype || !op_vectype)
    3329              :     return NULL;
    3330              : 
    3331              :   /* Verify we can handle the new operation.  For shifts and rotates
    3332              :      apply heuristic of whether we are likely facing vector-vector or
    3333              :      vector-scalar operation.  Since we are eventually expecting that
    3334              :      a later pattern might eventually want to rewrite an unsupported
    3335              :      into a supported case error on that side in case the original
    3336              :      operation was not supported either or this is a binary operation
    3337              :      and the 2nd operand is constant.  */
    3338       141212 :   if (code == RSHIFT_EXPR || code == LSHIFT_EXPR || code == RROTATE_EXPR)
    3339              :     {
    3340        29693 :       if (!target_has_vecop_for_code (code, op_vectype, optab_vector)
    3341        28616 :           && ((unprom[1].dt != vect_external_def
    3342        28198 :                && unprom[1].dt != vect_constant_def)
    3343        19630 :               || !target_has_vecop_for_code (code, op_vectype, optab_scalar))
    3344        38702 :           && !(!target_has_vecop_for_code (code, *type_out, optab_vector)
    3345         7911 :                && ((unprom[1].dt != vect_external_def
    3346         7911 :                     || unprom[1].dt != vect_constant_def)
    3347              :                    || !target_has_vecop_for_code (code, *type_out,
    3348              :                                                   optab_scalar))))
    3349              :         return NULL;
    3350              :     }
    3351       111519 :   else if (!target_has_vecop_for_code (code, op_vectype, optab_vector)
    3352       111519 :            && (target_has_vecop_for_code (code, *type_out, optab_vector)
    3353           29 :                && !(nops == 2 && unprom[1].dt == vect_constant_def)))
    3354              :     return NULL;
    3355              : 
    3356       140104 :   if (dump_enabled_p ())
    3357         4691 :     dump_printf_loc (MSG_NOTE, vect_location, "demoting %T to %T\n",
    3358              :                      type, new_type);
    3359              : 
    3360              :   /* Calculate the rhs operands for an operation on OP_TYPE.  */
    3361       140104 :   tree ops[3] = {};
    3362       140377 :   for (unsigned int i = 1; i < first_op; ++i)
    3363          273 :     ops[i - 1] = gimple_op (last_stmt, i);
    3364       140104 :   vect_convert_inputs (vinfo, last_stmt_info, nops, &ops[first_op - 1],
    3365       140104 :                        op_type, &unprom[0], op_vectype);
    3366              : 
    3367              :   /* Use the operation to produce a result of type OP_TYPE.  */
    3368       140104 :   tree new_var = vect_recog_temp_ssa_var (op_type, NULL);
    3369       140104 :   gimple *pattern_stmt = gimple_build_assign (new_var, code,
    3370              :                                               ops[0], ops[1], ops[2]);
    3371       140104 :   gimple_set_location (pattern_stmt, gimple_location (last_stmt));
    3372              : 
    3373       140104 :   if (dump_enabled_p ())
    3374         4691 :     dump_printf_loc (MSG_NOTE, vect_location,
    3375              :                      "created pattern stmt: %G", pattern_stmt);
    3376              : 
    3377              :   /* Convert back to the original signedness, if OP_TYPE is different
    3378              :      from NEW_TYPE.  */
    3379       140104 :   if (op_type != new_type)
    3380        28089 :     pattern_stmt = vect_convert_output (vinfo, last_stmt_info, new_type,
    3381              :                                         pattern_stmt, op_vectype);
    3382              : 
    3383              :   /* Promote the result to the original type.  */
    3384       140104 :   pattern_stmt = vect_convert_output (vinfo, last_stmt_info, type,
    3385              :                                       pattern_stmt, new_vectype);
    3386              : 
    3387       140104 :   return pattern_stmt;
    3388      1689197 : }
    3389              : 
    3390              : /* Recognize the following patterns:
    3391              : 
    3392              :      ATYPE a;  // narrower than TYPE
    3393              :      BTYPE b;  // narrower than TYPE
    3394              : 
    3395              :    1) Multiply high with scaling
    3396              :      TYPE res = ((TYPE) a * (TYPE) b) >> c;
    3397              :      Here, c is bitsize (TYPE) / 2 - 1.
    3398              : 
    3399              :    2) ... or also with rounding
    3400              :      TYPE res = (((TYPE) a * (TYPE) b) >> d + 1) >> 1;
    3401              :      Here, d is bitsize (TYPE) / 2 - 2.
    3402              : 
    3403              :    3) Normal multiply high
    3404              :      TYPE res = ((TYPE) a * (TYPE) b) >> e;
    3405              :      Here, e is bitsize (TYPE) / 2.
    3406              : 
    3407              :    where only the bottom half of res is used.  */
    3408              : 
    3409              : static gimple *
    3410     33136954 : vect_recog_mulhs_pattern (vec_info *vinfo,
    3411              :                           stmt_vec_info last_stmt_info, tree *type_out)
    3412              : {
    3413              :   /* Check for a right shift.  */
    3414     33136954 :   gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
    3415     23234105 :   if (!last_stmt
    3416     23234105 :       || gimple_assign_rhs_code (last_stmt) != RSHIFT_EXPR)
    3417              :     return NULL;
    3418              : 
    3419              :   /* Check that the shift result is wider than the users of the
    3420              :      result need (i.e. that narrowing would be a natural choice).  */
    3421       373839 :   tree lhs_type = TREE_TYPE (gimple_assign_lhs (last_stmt));
    3422       373839 :   unsigned int target_precision
    3423       373839 :     = vect_element_precision (last_stmt_info->min_output_precision);
    3424       373839 :   if (!INTEGRAL_TYPE_P (lhs_type)
    3425       373839 :       || target_precision >= TYPE_PRECISION (lhs_type))
    3426              :     return NULL;
    3427              : 
    3428              :   /* Look through any change in sign on the outer shift input.  */
    3429        64238 :   vect_unpromoted_value unprom_rshift_input;
    3430        64238 :   tree rshift_input = vect_look_through_possible_promotion
    3431        64238 :     (vinfo, gimple_assign_rhs1 (last_stmt), &unprom_rshift_input);
    3432        64238 :   if (!rshift_input
    3433        64238 :       || TYPE_PRECISION (TREE_TYPE (rshift_input))
    3434        63612 :            != TYPE_PRECISION (lhs_type))
    3435              :     return NULL;
    3436              : 
    3437              :   /* Get the definition of the shift input.  */
    3438        59302 :   stmt_vec_info rshift_input_stmt_info
    3439        59302 :     = vect_get_internal_def (vinfo, rshift_input);
    3440        59302 :   if (!rshift_input_stmt_info)
    3441              :     return NULL;
    3442        54570 :   gassign *rshift_input_stmt
    3443        54570 :     = dyn_cast <gassign *> (rshift_input_stmt_info->stmt);
    3444        51128 :   if (!rshift_input_stmt)
    3445              :     return NULL;
    3446              : 
    3447        51128 :   stmt_vec_info mulh_stmt_info;
    3448        51128 :   tree scale_term;
    3449        51128 :   bool rounding_p = false;
    3450              : 
    3451              :   /* Check for the presence of the rounding term.  */
    3452        59008 :   if (gimple_assign_rhs_code (rshift_input_stmt) == PLUS_EXPR)
    3453              :     {
    3454              :       /* Check that the outer shift was by 1.  */
    3455        17990 :       if (!integer_onep (gimple_assign_rhs2 (last_stmt)))
    3456         8928 :         return NULL;
    3457              : 
    3458              :       /* Check that the second operand of the PLUS_EXPR is 1.  */
    3459          514 :       if (!integer_onep (gimple_assign_rhs2 (rshift_input_stmt)))
    3460              :         return NULL;
    3461              : 
    3462              :       /* Look through any change in sign on the addition input.  */
    3463          108 :       vect_unpromoted_value unprom_plus_input;
    3464          108 :       tree plus_input = vect_look_through_possible_promotion
    3465          108 :         (vinfo, gimple_assign_rhs1 (rshift_input_stmt), &unprom_plus_input);
    3466          108 :       if (!plus_input
    3467          108 :            || TYPE_PRECISION (TREE_TYPE (plus_input))
    3468          108 :                 != TYPE_PRECISION (TREE_TYPE (rshift_input)))
    3469              :         return NULL;
    3470              : 
    3471              :       /* Get the definition of the multiply-high-scale part.  */
    3472           95 :       stmt_vec_info plus_input_stmt_info
    3473           95 :         = vect_get_internal_def (vinfo, plus_input);
    3474           95 :       if (!plus_input_stmt_info)
    3475              :         return NULL;
    3476           95 :       gassign *plus_input_stmt
    3477         9023 :         = dyn_cast <gassign *> (plus_input_stmt_info->stmt);
    3478           95 :       if (!plus_input_stmt
    3479           95 :           || gimple_assign_rhs_code (plus_input_stmt) != RSHIFT_EXPR)
    3480              :         return NULL;
    3481              : 
    3482              :       /* Look through any change in sign on the scaling input.  */
    3483           67 :       vect_unpromoted_value unprom_scale_input;
    3484           67 :       tree scale_input = vect_look_through_possible_promotion
    3485           67 :         (vinfo, gimple_assign_rhs1 (plus_input_stmt), &unprom_scale_input);
    3486           67 :       if (!scale_input
    3487           67 :           || TYPE_PRECISION (TREE_TYPE (scale_input))
    3488           67 :                != TYPE_PRECISION (TREE_TYPE (plus_input)))
    3489              :         return NULL;
    3490              : 
    3491              :       /* Get the definition of the multiply-high part.  */
    3492           67 :       mulh_stmt_info = vect_get_internal_def (vinfo, scale_input);
    3493           67 :       if (!mulh_stmt_info)
    3494              :         return NULL;
    3495              : 
    3496              :       /* Get the scaling term.  */
    3497           67 :       scale_term = gimple_assign_rhs2 (plus_input_stmt);
    3498           67 :       rounding_p = true;
    3499              :     }
    3500              :   else
    3501              :     {
    3502        42133 :       mulh_stmt_info = rshift_input_stmt_info;
    3503        42133 :       scale_term = gimple_assign_rhs2 (last_stmt);
    3504              :     }
    3505              : 
    3506              :   /* Check that the scaling factor is constant.  */
    3507        42200 :   if (TREE_CODE (scale_term) != INTEGER_CST)
    3508              :     return NULL;
    3509              : 
    3510              :   /* Check whether the scaling input term can be seen as two widened
    3511              :      inputs multiplied together.  */
    3512       123612 :   vect_unpromoted_value unprom_mult[2];
    3513        41204 :   tree new_type;
    3514        41204 :   unsigned int nops
    3515        41204 :     = vect_widened_op_tree (vinfo, mulh_stmt_info, MULT_EXPR, WIDEN_MULT_EXPR,
    3516              :                             false, 2, unprom_mult, &new_type);
    3517        41204 :   if (nops != 2)
    3518              :     return NULL;
    3519              : 
    3520              :   /* Adjust output precision.  */
    3521        10797 :   if (TYPE_PRECISION (new_type) < target_precision)
    3522            0 :     new_type = build_nonstandard_integer_type
    3523            0 :       (target_precision, TYPE_UNSIGNED (new_type));
    3524              : 
    3525        10797 :   unsigned mult_precision = TYPE_PRECISION (new_type);
    3526        10797 :   internal_fn ifn;
    3527              :   /* Check that the scaling factor is expected.  Instead of
    3528              :      target_precision, we should use the one that we actually
    3529              :      use for internal function.  */
    3530        10797 :   if (rounding_p)
    3531              :     {
    3532              :       /* Check pattern 2).  */
    3533          134 :       if (wi::to_widest (scale_term) + mult_precision + 2
    3534          201 :           != TYPE_PRECISION (lhs_type))
    3535              :         return NULL;
    3536              : 
    3537              :       ifn = IFN_MULHRS;
    3538              :     }
    3539              :   else
    3540              :     {
    3541              :       /* Check for pattern 1).  */
    3542        21460 :       if (wi::to_widest (scale_term) + mult_precision + 1
    3543        32190 :           == TYPE_PRECISION (lhs_type))
    3544              :         ifn = IFN_MULHS;
    3545              :       /* Check for pattern 3).  */
    3546        10696 :       else if (wi::to_widest (scale_term) + mult_precision
    3547        21392 :                == TYPE_PRECISION (lhs_type))
    3548              :         ifn = IFN_MULH;
    3549              :       else
    3550              :         return NULL;
    3551              :     }
    3552              : 
    3553        10708 :   vect_pattern_detected ("vect_recog_mulhs_pattern", last_stmt);
    3554              : 
    3555              :   /* Check for target support.  */
    3556        10708 :   tree new_vectype = get_vectype_for_scalar_type (vinfo, new_type);
    3557        10708 :   if (!new_vectype
    3558        18070 :       || !direct_internal_fn_supported_p
    3559         7362 :             (ifn, new_vectype, OPTIMIZE_FOR_SPEED))
    3560              :     return NULL;
    3561              : 
    3562              :   /* The result is cast back to LHS_TYPE, a cast that the over-widening
    3563              :      machinery then removes.  LHS_TYPE need not have a vector type, as for the
    3564              :      128-bit product of a 64-bit high-part multiply, so leave *TYPE_OUT null
    3565              :      rather than giving up.  */
    3566         4551 :   *type_out = get_vectype_for_scalar_type (vinfo, lhs_type);
    3567              : 
    3568              :   /* Generate the IFN_MULHRS call.  */
    3569         4551 :   tree new_var = vect_recog_temp_ssa_var (new_type, NULL);
    3570         4551 :   tree new_ops[2];
    3571         4551 :   vect_convert_inputs (vinfo, last_stmt_info, 2, new_ops, new_type,
    3572              :                        unprom_mult, new_vectype);
    3573         4551 :   gcall *mulhrs_stmt
    3574         4551 :     = gimple_build_call_internal (ifn, 2, new_ops[0], new_ops[1]);
    3575         4551 :   gimple_call_set_lhs (mulhrs_stmt, new_var);
    3576         4551 :   gimple_set_location (mulhrs_stmt, gimple_location (last_stmt));
    3577              : 
    3578         4551 :   if (dump_enabled_p ())
    3579            3 :     dump_printf_loc (MSG_NOTE, vect_location,
    3580              :                      "created pattern stmt: %G", (gimple *) mulhrs_stmt);
    3581              : 
    3582         4551 :   return vect_convert_output (vinfo, last_stmt_info, lhs_type,
    3583         4551 :                               mulhrs_stmt, new_vectype);
    3584              : }
    3585              : 
    3586              : /* Recognize the patterns:
    3587              : 
    3588              :             ATYPE a;  // narrower than TYPE
    3589              :             BTYPE b;  // narrower than TYPE
    3590              :         (1) TYPE avg = ((TYPE) a + (TYPE) b) >> 1;
    3591              :      or (2) TYPE avg = ((TYPE) a + (TYPE) b + 1) >> 1;
    3592              : 
    3593              :    where only the bottom half of avg is used.  Try to transform them into:
    3594              : 
    3595              :         (1) NTYPE avg' = .AVG_FLOOR ((NTYPE) a, (NTYPE) b);
    3596              :      or (2) NTYPE avg' = .AVG_CEIL ((NTYPE) a, (NTYPE) b);
    3597              : 
    3598              :   followed by:
    3599              : 
    3600              :             TYPE avg = (TYPE) avg';
    3601              : 
    3602              :   where NTYPE is no wider than half of TYPE.  Since only the bottom half
    3603              :   of avg is used, all or part of the cast of avg' should become redundant.
    3604              : 
    3605              :   If there is no target support available, generate code to distribute rshift
    3606              :   over plus and add a carry.  */
    3607              : 
    3608              : static gimple *
    3609     33132717 : vect_recog_average_pattern (vec_info *vinfo,
    3610              :                             stmt_vec_info last_stmt_info, tree *type_out)
    3611              : {
    3612              :   /* Check for a shift right by one bit.  */
    3613     33132717 :   gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
    3614     23230074 :   if (!last_stmt
    3615     23230074 :       || gimple_assign_rhs_code (last_stmt) != RSHIFT_EXPR
    3616       373352 :       || !integer_onep (gimple_assign_rhs2 (last_stmt)))
    3617              :     return NULL;
    3618              : 
    3619              :   /* Check that the shift result is wider than the users of the
    3620              :      result need (i.e. that narrowing would be a natural choice).  */
    3621        58955 :   tree lhs = gimple_assign_lhs (last_stmt);
    3622        58955 :   tree type = TREE_TYPE (lhs);
    3623        58955 :   unsigned int target_precision
    3624        58955 :     = vect_element_precision (last_stmt_info->min_output_precision);
    3625        58955 :   if (!INTEGRAL_TYPE_P (type) || target_precision >= TYPE_PRECISION (type))
    3626              :     return NULL;
    3627              : 
    3628              :   /* Look through any change in sign on the shift input.  */
    3629         2274 :   tree rshift_rhs = gimple_assign_rhs1 (last_stmt);
    3630         2274 :   vect_unpromoted_value unprom_plus;
    3631         2274 :   rshift_rhs = vect_look_through_possible_promotion (vinfo, rshift_rhs,
    3632              :                                                      &unprom_plus);
    3633         2274 :   if (!rshift_rhs
    3634         2274 :       || TYPE_PRECISION (TREE_TYPE (rshift_rhs)) != TYPE_PRECISION (type))
    3635              :     return NULL;
    3636              : 
    3637              :   /* Get the definition of the shift input.  */
    3638         2268 :   stmt_vec_info plus_stmt_info = vect_get_internal_def (vinfo, rshift_rhs);
    3639         2268 :   if (!plus_stmt_info)
    3640              :     return NULL;
    3641              : 
    3642              :   /* Check whether the shift input can be seen as a tree of additions on
    3643              :      2 or 3 widened inputs.
    3644              : 
    3645              :      Note that the pattern should be a win even if the result of one or
    3646              :      more additions is reused elsewhere: if the pattern matches, we'd be
    3647              :      replacing 2N RSHIFT_EXPRs and N VEC_PACK_*s with N IFN_AVG_*s.  */
    3648         2262 :   internal_fn ifn = IFN_AVG_FLOOR;
    3649         9048 :   vect_unpromoted_value unprom[3];
    3650         2262 :   tree new_type;
    3651         2262 :   unsigned int nops = vect_widened_op_tree (vinfo, plus_stmt_info, PLUS_EXPR,
    3652         2262 :                                             IFN_VEC_WIDEN_PLUS, false, 3,
    3653              :                                             unprom, &new_type);
    3654         2262 :   if (nops == 0)
    3655              :     return NULL;
    3656          972 :   if (nops == 3)
    3657              :     {
    3658              :       /* Check that one operand is 1.  */
    3659              :       unsigned int i;
    3660         1155 :       for (i = 0; i < 3; ++i)
    3661         1101 :         if (integer_onep (unprom[i].op))
    3662              :           break;
    3663          367 :       if (i == 3)
    3664              :         return NULL;
    3665              :       /* Throw away the 1 operand and keep the other two.  */
    3666          313 :       if (i < 2)
    3667            0 :         unprom[i] = unprom[2];
    3668              :       ifn = IFN_AVG_CEIL;
    3669              :     }
    3670              : 
    3671          918 :   vect_pattern_detected ("vect_recog_average_pattern", last_stmt);
    3672              : 
    3673              :   /* We know that:
    3674              : 
    3675              :      (a) the operation can be viewed as:
    3676              : 
    3677              :            TYPE widened0 = (TYPE) UNPROM[0];
    3678              :            TYPE widened1 = (TYPE) UNPROM[1];
    3679              :            TYPE tmp1 = widened0 + widened1 {+ 1};
    3680              :            TYPE tmp2 = tmp1 >> 1;   // LAST_STMT_INFO
    3681              : 
    3682              :      (b) the first two statements are equivalent to:
    3683              : 
    3684              :            TYPE widened0 = (TYPE) (NEW_TYPE) UNPROM[0];
    3685              :            TYPE widened1 = (TYPE) (NEW_TYPE) UNPROM[1];
    3686              : 
    3687              :      (c) vect_recog_over_widening_pattern has already tried to narrow TYPE
    3688              :          where sensible;
    3689              : 
    3690              :      (d) all the operations can be performed correctly at twice the width of
    3691              :          NEW_TYPE, due to the nature of the average operation; and
    3692              : 
    3693              :      (e) users of the result of the right shift need only TARGET_PRECISION
    3694              :          bits, where TARGET_PRECISION is no more than half of TYPE's
    3695              :          precision.
    3696              : 
    3697              :      Under these circumstances, the only situation in which NEW_TYPE
    3698              :      could be narrower than TARGET_PRECISION is if widened0, widened1
    3699              :      and an addition result are all used more than once.  Thus we can
    3700              :      treat any widening of UNPROM[0] and UNPROM[1] to TARGET_PRECISION
    3701              :      as "free", whereas widening the result of the average instruction
    3702              :      from NEW_TYPE to TARGET_PRECISION would be a new operation.  It's
    3703              :      therefore better not to go narrower than TARGET_PRECISION.  */
    3704          918 :   if (TYPE_PRECISION (new_type) < target_precision)
    3705            0 :     new_type = build_nonstandard_integer_type (target_precision,
    3706            0 :                                                TYPE_UNSIGNED (new_type));
    3707              : 
    3708              :   /* Check for target support.  */
    3709          918 :   tree new_vectype = get_vectype_for_scalar_type (vinfo, new_type);
    3710          918 :   if (!new_vectype)
    3711              :     return NULL;
    3712              : 
    3713          918 :   bool fallback_p = !direct_internal_fn_supported_p (ifn, new_vectype,
    3714          918 :                                                       OPTIMIZE_FOR_SPEED);
    3715          918 :   if (fallback_p
    3716          918 :       && (!target_supports_op_p (new_vectype, RSHIFT_EXPR, optab_scalar)
    3717          697 :           || !target_supports_op_p (new_vectype, PLUS_EXPR, optab_default)
    3718          697 :           || !target_supports_op_p (new_vectype, BIT_AND_EXPR, optab_default)
    3719          697 :           || (ifn == IFN_AVG_CEIL
    3720          105 :               && !target_supports_op_p (new_vectype, BIT_IOR_EXPR,
    3721              :                                         optab_default))))
    3722              :     return NULL;
    3723              : 
    3724              :   /* The IR requires a valid vector type for the cast result, even though
    3725              :      it's likely to be discarded.  */
    3726          903 :   *type_out = get_vectype_for_scalar_type (vinfo, type);
    3727          903 :   if (!*type_out)
    3728              :     return NULL;
    3729              : 
    3730          899 :   tree new_var = vect_recog_temp_ssa_var (new_type, NULL);
    3731          899 :   tree new_ops[2];
    3732          899 :   vect_convert_inputs (vinfo, last_stmt_info, 2, new_ops, new_type,
    3733              :                        unprom, new_vectype);
    3734              : 
    3735          899 :   if (fallback_p)
    3736              :     {
    3737              :       /* As a fallback, generate code for following sequence:
    3738              : 
    3739              :          shifted_op0 = new_ops[0] >> 1;
    3740              :          shifted_op1 = new_ops[1] >> 1;
    3741              :          sum_of_shifted = shifted_op0 + shifted_op1;
    3742              :          unmasked_carry = new_ops[0] and/or new_ops[1];
    3743              :          carry = unmasked_carry & 1;
    3744              :          new_var = sum_of_shifted + carry;
    3745              : 
    3746              :          For signed types, arithmetic shifts round down and the carry is one
    3747              :          when both operands are odd (or either operand for IFN_AVG_CEIL).  */
    3748              : 
    3749          693 :       tree one_cst = build_one_cst (new_type);
    3750          693 :       gassign *g;
    3751              : 
    3752          693 :       tree shifted_op0 = vect_recog_temp_ssa_var (new_type, NULL);
    3753          693 :       g = gimple_build_assign (shifted_op0, RSHIFT_EXPR, new_ops[0], one_cst);
    3754          693 :       append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
    3755              : 
    3756          693 :       tree shifted_op1 = vect_recog_temp_ssa_var (new_type, NULL);
    3757          693 :       g = gimple_build_assign (shifted_op1, RSHIFT_EXPR, new_ops[1], one_cst);
    3758          693 :       append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
    3759              : 
    3760          693 :       tree sum_of_shifted = vect_recog_temp_ssa_var (new_type, NULL);
    3761          693 :       g = gimple_build_assign (sum_of_shifted, PLUS_EXPR,
    3762              :                                shifted_op0, shifted_op1);
    3763          693 :       append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
    3764              : 
    3765          693 :       tree unmasked_carry = vect_recog_temp_ssa_var (new_type, NULL);
    3766          693 :       tree_code c = (ifn == IFN_AVG_CEIL) ? BIT_IOR_EXPR : BIT_AND_EXPR;
    3767          693 :       g = gimple_build_assign (unmasked_carry, c, new_ops[0], new_ops[1]);
    3768          693 :       append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
    3769              : 
    3770          693 :       tree carry = vect_recog_temp_ssa_var (new_type, NULL);
    3771          693 :       g = gimple_build_assign (carry, BIT_AND_EXPR, unmasked_carry, one_cst);
    3772          693 :       append_pattern_def_seq (vinfo, last_stmt_info, g, new_vectype);
    3773              : 
    3774          693 :       g = gimple_build_assign (new_var, PLUS_EXPR, sum_of_shifted, carry);
    3775          693 :       return vect_convert_output (vinfo, last_stmt_info, type, g, new_vectype);
    3776              :     }
    3777              : 
    3778              :   /* Generate the IFN_AVG* call.  */
    3779          206 :   gcall *average_stmt = gimple_build_call_internal (ifn, 2, new_ops[0],
    3780              :                                                     new_ops[1]);
    3781          206 :   gimple_call_set_lhs (average_stmt, new_var);
    3782          206 :   gimple_set_location (average_stmt, gimple_location (last_stmt));
    3783              : 
    3784          206 :   if (dump_enabled_p ())
    3785           79 :     dump_printf_loc (MSG_NOTE, vect_location,
    3786              :                      "created pattern stmt: %G", (gimple *) average_stmt);
    3787              : 
    3788          206 :   return vect_convert_output (vinfo, last_stmt_info,
    3789          206 :                               type, average_stmt, new_vectype);
    3790              : }
    3791              : 
    3792              : /* Recognize cases in which the input to a cast is wider than its
    3793              :    output, and the input is fed by a widening operation.  Fold this
    3794              :    by removing the unnecessary intermediate widening.  E.g.:
    3795              : 
    3796              :      unsigned char a;
    3797              :      unsigned int b = (unsigned int) a;
    3798              :      unsigned short c = (unsigned short) b;
    3799              : 
    3800              :    -->
    3801              : 
    3802              :      unsigned short c = (unsigned short) a;
    3803              : 
    3804              :    Although this is rare in input IR, it is an expected side-effect
    3805              :    of the over-widening pattern above.
    3806              : 
    3807              :    This is beneficial also for integer-to-float conversions, if the
    3808              :    widened integer has more bits than the float, and if the unwidened
    3809              :    input doesn't.  */
    3810              : 
    3811              : static gimple *
    3812     33136954 : vect_recog_cast_forwprop_pattern (vec_info *vinfo,
    3813              :                                   stmt_vec_info last_stmt_info, tree *type_out)
    3814              : {
    3815              :   /* Check for a cast, including an integer-to-float conversion.  */
    3816     33136954 :   gassign *last_stmt = dyn_cast <gassign *> (last_stmt_info->stmt);
    3817     23229554 :   if (!last_stmt)
    3818              :     return NULL;
    3819     23229554 :   tree_code code = gimple_assign_rhs_code (last_stmt);
    3820     23229554 :   if (!CONVERT_EXPR_CODE_P (code) && code != FLOAT_EXPR)
    3821              :     return NULL;
    3822              : 
    3823              :   /* Make sure that the rhs is a scalar with a natural bitsize.  */
    3824      3394286 :   tree lhs = gimple_assign_lhs (last_stmt);
    3825      3394286 :   if (!lhs)
    3826              :     return NULL;
    3827      3394286 :   tree lhs_type = TREE_TYPE (lhs);
    3828      3394286 :   scalar_mode lhs_mode;
    3829      3373277 :   if (VECT_SCALAR_BOOLEAN_TYPE_P (lhs_type)
    3830      6765773 :       || !is_a <scalar_mode> (TYPE_MODE (lhs_type), &lhs_mode))
    3831              :     return NULL;
    3832              : 
    3833              :   /* Check for a narrowing operation (from a vector point of view).  */
    3834      3367685 :   tree rhs = gimple_assign_rhs1 (last_stmt);
    3835      3367685 :   tree rhs_type = TREE_TYPE (rhs);
    3836      3367685 :   if (!INTEGRAL_TYPE_P (rhs_type)
    3837      2946785 :       || VECT_SCALAR_BOOLEAN_TYPE_P (rhs_type)
    3838      9107751 :       || TYPE_PRECISION (rhs_type) <= GET_MODE_BITSIZE (lhs_mode))
    3839              :     return NULL;
    3840              : 
    3841              :   /* Try to find an unpromoted input.  */
    3842       397129 :   vect_unpromoted_value unprom;
    3843       397129 :   if (!vect_look_through_possible_promotion (vinfo, rhs, &unprom)
    3844       397129 :       || TYPE_PRECISION (unprom.type) >= TYPE_PRECISION (rhs_type))
    3845              :     return NULL;
    3846              : 
    3847              :   /* If the bits above RHS_TYPE matter, make sure that they're the
    3848              :      same when extending from UNPROM as they are when extending from RHS.  */
    3849        53904 :   if (!INTEGRAL_TYPE_P (lhs_type)
    3850        53904 :       && TYPE_SIGN (rhs_type) != TYPE_SIGN (unprom.type))
    3851              :     return NULL;
    3852              : 
    3853              :   /* We can get the same result by casting UNPROM directly, to avoid
    3854              :      the unnecessary widening and narrowing.  */
    3855        53784 :   vect_pattern_detected ("vect_recog_cast_forwprop_pattern", last_stmt);
    3856              : 
    3857        53784 :   *type_out = get_vectype_for_scalar_type (vinfo, lhs_type);
    3858        53784 :   if (!*type_out)
    3859              :     return NULL;
    3860              : 
    3861        53784 :   tree new_var = vect_recog_temp_ssa_var (lhs_type, NULL);
    3862        53784 :   gimple *pattern_stmt = gimple_build_assign (new_var, code, unprom.op);
    3863        53784 :   gimple_set_location (pattern_stmt, gimple_location (last_stmt));
    3864              : 
    3865        53784 :   return pattern_stmt;
    3866              : }
    3867              : 
    3868              : /* Try to detect a shift left of a widened input, converting LSHIFT_EXPR
    3869              :    to WIDEN_LSHIFT_EXPR.  See vect_recog_widen_op_pattern for details.  */
    3870              : 
    3871              : static gimple *
    3872     33054528 : vect_recog_widen_shift_pattern (vec_info *vinfo,
    3873              :                                 stmt_vec_info last_stmt_info, tree *type_out)
    3874              : {
    3875     33054528 :   return vect_recog_widen_op_pattern (vinfo, last_stmt_info, type_out,
    3876     33054528 :                                       LSHIFT_EXPR, WIDEN_LSHIFT_EXPR, true,
    3877     33054528 :                                       "vect_recog_widen_shift_pattern");
    3878              : }
    3879              : 
    3880              : /* Detect a rotate pattern wouldn't be otherwise vectorized:
    3881              : 
    3882              :    type a_t, b_t, c_t;
    3883              : 
    3884              :    S0 a_t = b_t r<< c_t;
    3885              : 
    3886              :   Input/Output:
    3887              : 
    3888              :   * STMT_VINFO: The stmt from which the pattern search begins,
    3889              :     i.e. the shift/rotate stmt.  The original stmt (S0) is replaced
    3890              :     with a sequence:
    3891              : 
    3892              :    S1 d_t = -c_t;
    3893              :    S2 e_t = d_t & (B - 1);
    3894              :    S3 f_t = b_t << c_t;
    3895              :    S4 g_t = b_t >> e_t;
    3896              :    S0 a_t = f_t | g_t;
    3897              : 
    3898              :     where B is element bitsize of type.
    3899              : 
    3900              :   Output:
    3901              : 
    3902              :   * TYPE_OUT: The type of the output of this pattern.
    3903              : 
    3904              :   * Return value: A new stmt that will be used to replace the rotate
    3905              :     S0 stmt.  */
    3906              : 
    3907              : static gimple *
    3908     33054528 : vect_recog_rotate_pattern (vec_info *vinfo,
    3909              :                            stmt_vec_info stmt_vinfo, tree *type_out)
    3910              : {
    3911     33054528 :   gimple *last_stmt = stmt_vinfo->stmt;
    3912     33054528 :   tree oprnd0, oprnd1, lhs, var, var1, var2, vectype, type, stype, def, def2;
    3913     33054528 :   gimple *pattern_stmt, *def_stmt;
    3914     33054528 :   enum tree_code rhs_code;
    3915     33054528 :   enum vect_def_type dt;
    3916     33054528 :   optab optab1, optab2;
    3917     33054528 :   edge ext_def = NULL;
    3918     33054528 :   bool bswap16_p = false;
    3919              : 
    3920     33054528 :   if (is_gimple_assign (last_stmt))
    3921              :     {
    3922     23147070 :       rhs_code = gimple_assign_rhs_code (last_stmt);
    3923     23147070 :       switch (rhs_code)
    3924              :         {
    3925         8586 :         case LROTATE_EXPR:
    3926         8586 :         case RROTATE_EXPR:
    3927         8586 :           break;
    3928              :         default:
    3929              :           return NULL;
    3930              :         }
    3931              : 
    3932         8586 :       lhs = gimple_assign_lhs (last_stmt);
    3933         8586 :       oprnd0 = gimple_assign_rhs1 (last_stmt);
    3934         8586 :       type = TREE_TYPE (oprnd0);
    3935         8586 :       oprnd1 = gimple_assign_rhs2 (last_stmt);
    3936              :     }
    3937      9907458 :   else if (gimple_call_builtin_p (last_stmt, BUILT_IN_BSWAP16))
    3938              :     {
    3939              :       /* __builtin_bswap16 (x) is another form of x r>> 8.
    3940              :          The vectorizer has bswap support, but only if the argument isn't
    3941              :          promoted.  */
    3942          206 :       lhs = gimple_call_lhs (last_stmt);
    3943          206 :       oprnd0 = gimple_call_arg (last_stmt, 0);
    3944          206 :       type = TREE_TYPE (oprnd0);
    3945          206 :       if (!lhs
    3946          206 :           || TYPE_PRECISION (TREE_TYPE (lhs)) != 16
    3947          206 :           || TYPE_PRECISION (type) <= 16
    3948            0 :           || TREE_CODE (oprnd0) != SSA_NAME
    3949          206 :           || BITS_PER_UNIT != 8)
    3950          206 :         return NULL;
    3951              : 
    3952            0 :       stmt_vec_info def_stmt_info;
    3953            0 :       if (!vect_is_simple_use (oprnd0, vinfo, &dt, &def_stmt_info, &def_stmt))
    3954              :         return NULL;
    3955              : 
    3956            0 :       if (dt != vect_internal_def)
    3957              :         return NULL;
    3958              : 
    3959            0 :       if (gimple_assign_cast_p (def_stmt))
    3960              :         {
    3961            0 :           def = gimple_assign_rhs1 (def_stmt);
    3962            0 :           if (INTEGRAL_TYPE_P (TREE_TYPE (def))
    3963            0 :               && TYPE_PRECISION (TREE_TYPE (def)) == 16)
    3964              :             oprnd0 = def;
    3965              :         }
    3966              : 
    3967            0 :       type = TREE_TYPE (lhs);
    3968            0 :       vectype = get_vectype_for_scalar_type (vinfo, type);
    3969            0 :       if (vectype == NULL_TREE)
    3970              :         return NULL;
    3971              : 
    3972            0 :       if (tree char_vectype = get_same_sized_vectype (char_type_node, vectype))
    3973              :         {
    3974              :           /* The encoding uses one stepped pattern for each byte in the
    3975              :              16-bit word.  */
    3976            0 :           vec_perm_builder elts (TYPE_VECTOR_SUBPARTS (char_vectype), 2, 3);
    3977            0 :           for (unsigned i = 0; i < 3; ++i)
    3978            0 :             for (unsigned j = 0; j < 2; ++j)
    3979            0 :               elts.quick_push ((i + 1) * 2 - j - 1);
    3980              : 
    3981            0 :           vec_perm_indices indices (elts, 1,
    3982            0 :                                     TYPE_VECTOR_SUBPARTS (char_vectype));
    3983            0 :           machine_mode vmode = TYPE_MODE (char_vectype);
    3984            0 :           if (can_vec_perm_const_p (vmode, vmode, indices))
    3985              :             {
    3986              :               /* vectorizable_bswap can handle the __builtin_bswap16 if we
    3987              :                  undo the argument promotion.  */
    3988            0 :               if (!useless_type_conversion_p (type, TREE_TYPE (oprnd0)))
    3989              :                 {
    3990            0 :                   def = vect_recog_temp_ssa_var (type, NULL);
    3991            0 :                   def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd0);
    3992            0 :                   append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    3993            0 :                   oprnd0 = def;
    3994              :                 }
    3995              : 
    3996              :               /* Pattern detected.  */
    3997            0 :               vect_pattern_detected ("vect_recog_rotate_pattern", last_stmt);
    3998              : 
    3999            0 :               *type_out = vectype;
    4000              : 
    4001              :               /* Pattern supported.  Create a stmt to be used to replace the
    4002              :                  pattern, with the unpromoted argument.  */
    4003            0 :               var = vect_recog_temp_ssa_var (type, NULL);
    4004            0 :               pattern_stmt = gimple_build_call (gimple_call_fndecl (last_stmt),
    4005              :                                                 1, oprnd0);
    4006            0 :               gimple_call_set_lhs (pattern_stmt, var);
    4007            0 :               gimple_call_set_fntype (as_a <gcall *> (pattern_stmt),
    4008              :                                       gimple_call_fntype (last_stmt));
    4009            0 :               return pattern_stmt;
    4010              :             }
    4011            0 :         }
    4012              : 
    4013            0 :       oprnd1 = build_int_cst (integer_type_node, 8);
    4014            0 :       rhs_code = LROTATE_EXPR;
    4015            0 :       bswap16_p = true;
    4016              :     }
    4017              :   else
    4018              :     return NULL;
    4019              : 
    4020         8586 :   if (TREE_CODE (oprnd0) != SSA_NAME
    4021         8466 :       || !INTEGRAL_TYPE_P (type)
    4022        16731 :       || TYPE_PRECISION (TREE_TYPE (lhs)) != TYPE_PRECISION (type))
    4023              :     return NULL;
    4024              : 
    4025         8145 :   stmt_vec_info def_stmt_info;
    4026         8145 :   if (!vect_is_simple_use (oprnd1, vinfo, &dt, &def_stmt_info, &def_stmt))
    4027              :     return NULL;
    4028              : 
    4029         8145 :   if (dt != vect_internal_def
    4030         7078 :       && dt != vect_constant_def
    4031           25 :       && dt != vect_external_def)
    4032              :     return NULL;
    4033              : 
    4034         8139 :   vectype = get_vectype_for_scalar_type (vinfo, type);
    4035         8139 :   if (vectype == NULL_TREE)
    4036              :     return NULL;
    4037              : 
    4038              :   /* If vector/vector or vector/scalar rotate is supported by the target,
    4039              :      don't do anything here.  */
    4040         7772 :   optab1 = optab_for_tree_code (rhs_code, vectype, optab_vector);
    4041         7772 :   if (optab1
    4042         7772 :       && can_implement_p (optab1, TYPE_MODE (vectype)))
    4043              :     {
    4044          564 :      use_rotate:
    4045          564 :       if (bswap16_p)
    4046              :         {
    4047            0 :           if (!useless_type_conversion_p (type, TREE_TYPE (oprnd0)))
    4048              :             {
    4049            0 :               def = vect_recog_temp_ssa_var (type, NULL);
    4050            0 :               def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd0);
    4051            0 :               append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    4052            0 :               oprnd0 = def;
    4053              :             }
    4054              : 
    4055              :           /* Pattern detected.  */
    4056            0 :           vect_pattern_detected ("vect_recog_rotate_pattern", last_stmt);
    4057              : 
    4058            0 :           *type_out = vectype;
    4059              : 
    4060              :           /* Pattern supported.  Create a stmt to be used to replace the
    4061              :              pattern.  */
    4062            0 :           var = vect_recog_temp_ssa_var (type, NULL);
    4063            0 :           pattern_stmt = gimple_build_assign (var, LROTATE_EXPR, oprnd0,
    4064              :                                               oprnd1);
    4065            0 :           return pattern_stmt;
    4066              :         }
    4067              :       return NULL;
    4068              :     }
    4069              : 
    4070         7748 :   if (is_a <bb_vec_info> (vinfo) || dt != vect_internal_def)
    4071              :     {
    4072         7664 :       optab2 = optab_for_tree_code (rhs_code, vectype, optab_scalar);
    4073         7664 :       if (optab2
    4074         7664 :           && can_implement_p (optab2, TYPE_MODE (vectype)))
    4075          540 :         goto use_rotate;
    4076              :     }
    4077              : 
    4078              :   /* We may not use a reduction operand twice.  */
    4079         7208 :   if (vect_is_reduction (stmt_vinfo))
    4080              :     return NULL;
    4081              : 
    4082         7187 :   tree utype = unsigned_type_for (type);
    4083         7187 :   tree uvectype = get_vectype_for_scalar_type (vinfo, utype);
    4084         7187 :   if (!uvectype)
    4085              :     return NULL;
    4086              : 
    4087              :   /* If vector/vector or vector/scalar shifts aren't supported by the target,
    4088              :      don't do anything here either.  */
    4089         7187 :   optab1 = optab_for_tree_code (LSHIFT_EXPR, uvectype, optab_vector);
    4090         7187 :   optab2 = optab_for_tree_code (RSHIFT_EXPR, uvectype, optab_vector);
    4091         7187 :   if (!optab1
    4092         7187 :       || !can_implement_p (optab1, TYPE_MODE (uvectype))
    4093          746 :       || !optab2
    4094         7933 :       || !can_implement_p (optab2, TYPE_MODE (uvectype)))
    4095              :     {
    4096         6441 :       if (! is_a <bb_vec_info> (vinfo) && dt == vect_internal_def)
    4097              :         return NULL;
    4098         6378 :       optab1 = optab_for_tree_code (LSHIFT_EXPR, uvectype, optab_scalar);
    4099         6378 :       optab2 = optab_for_tree_code (RSHIFT_EXPR, uvectype, optab_scalar);
    4100         6378 :       if (!optab1
    4101         6378 :           || !can_implement_p (optab1, TYPE_MODE (uvectype))
    4102         4728 :           || !optab2
    4103        11106 :           || !can_implement_p (optab2, TYPE_MODE (uvectype)))
    4104              :         return NULL;
    4105              :     }
    4106              : 
    4107         5474 :   *type_out = vectype;
    4108              : 
    4109         5474 :   if (!useless_type_conversion_p (utype, TREE_TYPE (oprnd0)))
    4110              :     {
    4111           58 :       def = vect_recog_temp_ssa_var (utype, NULL);
    4112           58 :       def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd0);
    4113           58 :       append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
    4114           58 :       oprnd0 = def;
    4115              :     }
    4116              : 
    4117         5474 :   if (dt == vect_external_def && TREE_CODE (oprnd1) == SSA_NAME)
    4118           15 :     ext_def = vect_get_external_def_edge (vinfo, oprnd1);
    4119              : 
    4120         5474 :   def = NULL_TREE;
    4121         5474 :   scalar_int_mode mode = SCALAR_INT_TYPE_MODE (utype);
    4122         5474 :   if (dt != vect_internal_def || TYPE_MODE (TREE_TYPE (oprnd1)) == mode)
    4123              :     def = oprnd1;
    4124          124 :   else if (def_stmt && gimple_assign_cast_p (def_stmt))
    4125              :     {
    4126           96 :       tree rhs1 = gimple_assign_rhs1 (def_stmt);
    4127           96 :       if (TYPE_MODE (TREE_TYPE (rhs1)) == mode
    4128           96 :           && TYPE_PRECISION (TREE_TYPE (rhs1))
    4129            0 :              == TYPE_PRECISION (type))
    4130              :         def = rhs1;
    4131              :     }
    4132              : 
    4133         5350 :   if (def == NULL_TREE)
    4134              :     {
    4135          124 :       def = vect_recog_temp_ssa_var (utype, NULL);
    4136          124 :       def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd1);
    4137          124 :       append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
    4138              :     }
    4139         5474 :   stype = TREE_TYPE (def);
    4140              : 
    4141         5474 :   if (TREE_CODE (def) == INTEGER_CST)
    4142              :     {
    4143         4896 :       if (!tree_fits_uhwi_p (def)
    4144         4896 :           || tree_to_uhwi (def) >= GET_MODE_PRECISION (mode)
    4145         9792 :           || integer_zerop (def))
    4146              :         return NULL;
    4147         4896 :       def2 = build_int_cst (stype,
    4148         4896 :                             GET_MODE_PRECISION (mode) - tree_to_uhwi (def));
    4149              :     }
    4150              :   else
    4151              :     {
    4152          578 :       tree vecstype = get_vectype_for_scalar_type (vinfo, stype);
    4153              : 
    4154          578 :       if (vecstype == NULL_TREE)
    4155              :         return NULL;
    4156          578 :       def2 = vect_recog_temp_ssa_var (stype, NULL);
    4157          578 :       def_stmt = gimple_build_assign (def2, NEGATE_EXPR, def);
    4158          578 :       if (ext_def)
    4159              :         {
    4160           15 :           basic_block new_bb
    4161           15 :             = gsi_insert_on_edge_immediate (ext_def, def_stmt);
    4162           15 :           gcc_assert (!new_bb);
    4163              :         }
    4164              :       else
    4165          563 :         append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecstype);
    4166              : 
    4167          578 :       def2 = vect_recog_temp_ssa_var (stype, NULL);
    4168          578 :       tree mask = build_int_cst (stype, GET_MODE_PRECISION (mode) - 1);
    4169          578 :       def_stmt = gimple_build_assign (def2, BIT_AND_EXPR,
    4170              :                                       gimple_assign_lhs (def_stmt), mask);
    4171          578 :       if (ext_def)
    4172              :         {
    4173           15 :           basic_block new_bb
    4174           15 :             = gsi_insert_on_edge_immediate (ext_def, def_stmt);
    4175           15 :           gcc_assert (!new_bb);
    4176              :         }
    4177              :       else
    4178          563 :         append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecstype);
    4179              :     }
    4180              : 
    4181         5474 :   var1 = vect_recog_temp_ssa_var (utype, NULL);
    4182        10637 :   def_stmt = gimple_build_assign (var1, rhs_code == LROTATE_EXPR
    4183              :                                         ? LSHIFT_EXPR : RSHIFT_EXPR,
    4184              :                                   oprnd0, def);
    4185         5474 :   append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
    4186              : 
    4187         5474 :   var2 = vect_recog_temp_ssa_var (utype, NULL);
    4188        10637 :   def_stmt = gimple_build_assign (var2, rhs_code == LROTATE_EXPR
    4189              :                                         ? RSHIFT_EXPR : LSHIFT_EXPR,
    4190              :                                   oprnd0, def2);
    4191         5474 :   append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, uvectype);
    4192              : 
    4193              :   /* Pattern detected.  */
    4194         5474 :   vect_pattern_detected ("vect_recog_rotate_pattern", last_stmt);
    4195              : 
    4196              :   /* Pattern supported.  Create a stmt to be used to replace the pattern.  */
    4197         5474 :   var = vect_recog_temp_ssa_var (utype, NULL);
    4198         5474 :   pattern_stmt = gimple_build_assign (var, BIT_IOR_EXPR, var1, var2);
    4199              : 
    4200         5474 :   if (!useless_type_conversion_p (type, utype))
    4201              :     {
    4202           58 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, uvectype);
    4203           58 :       tree result = vect_recog_temp_ssa_var (type, NULL);
    4204           58 :       pattern_stmt = gimple_build_assign (result, NOP_EXPR, var);
    4205              :     }
    4206              :   return pattern_stmt;
    4207              : }
    4208              : 
    4209              : /* Detect a vector by vector shift pattern that wouldn't be otherwise
    4210              :    vectorized:
    4211              : 
    4212              :    type a_t;
    4213              :    TYPE b_T, res_T;
    4214              : 
    4215              :    S1 a_t = ;
    4216              :    S2 b_T = ;
    4217              :    S3 res_T = b_T op a_t;
    4218              : 
    4219              :   where type 'TYPE' is a type with different size than 'type',
    4220              :   and op is <<, >> or rotate.
    4221              : 
    4222              :   Also detect cases:
    4223              : 
    4224              :    type a_t;
    4225              :    TYPE b_T, c_T, res_T;
    4226              : 
    4227              :    S0 c_T = ;
    4228              :    S1 a_t = (type) c_T;
    4229              :    S2 b_T = ;
    4230              :    S3 res_T = b_T op a_t;
    4231              : 
    4232              :   Input/Output:
    4233              : 
    4234              :   * STMT_VINFO: The stmt from which the pattern search begins,
    4235              :     i.e. the shift/rotate stmt.  The original stmt (S3) is replaced
    4236              :     with a shift/rotate which has same type on both operands, in the
    4237              :     second case just b_T op c_T, in the first case with added cast
    4238              :     from a_t to c_T in STMT_VINFO_PATTERN_DEF_SEQ.
    4239              : 
    4240              :   Output:
    4241              : 
    4242              :   * TYPE_OUT: The type of the output of this pattern.
    4243              : 
    4244              :   * Return value: A new stmt that will be used to replace the shift/rotate
    4245              :     S3 stmt.  */
    4246              : 
    4247              : static gimple *
    4248     33061368 : vect_recog_vector_vector_shift_pattern (vec_info *vinfo,
    4249              :                                         stmt_vec_info stmt_vinfo,
    4250              :                                         tree *type_out)
    4251              : {
    4252     33061368 :   gimple *last_stmt = stmt_vinfo->stmt;
    4253     33061368 :   tree oprnd0, oprnd1, lhs, var;
    4254     33061368 :   gimple *pattern_stmt;
    4255     33061368 :   enum tree_code rhs_code;
    4256              : 
    4257     33061368 :   if (!is_gimple_assign (last_stmt))
    4258              :     return NULL;
    4259              : 
    4260     23153910 :   rhs_code = gimple_assign_rhs_code (last_stmt);
    4261     23153910 :   switch (rhs_code)
    4262              :     {
    4263       521116 :     case LSHIFT_EXPR:
    4264       521116 :     case RSHIFT_EXPR:
    4265       521116 :     case LROTATE_EXPR:
    4266       521116 :     case RROTATE_EXPR:
    4267       521116 :       break;
    4268              :     default:
    4269              :       return NULL;
    4270              :     }
    4271              : 
    4272       521116 :   lhs = gimple_assign_lhs (last_stmt);
    4273       521116 :   oprnd0 = gimple_assign_rhs1 (last_stmt);
    4274       521116 :   oprnd1 = gimple_assign_rhs2 (last_stmt);
    4275       521116 :   if (TREE_CODE (oprnd1) != SSA_NAME
    4276       112640 :       || TYPE_MODE (TREE_TYPE (oprnd0)) == TYPE_MODE (TREE_TYPE (oprnd1))
    4277        49988 :       || !INTEGRAL_TYPE_P (TREE_TYPE (oprnd0))
    4278        49367 :       || !type_has_mode_precision_p (TREE_TYPE (oprnd1))
    4279       570483 :       || TYPE_PRECISION (TREE_TYPE (lhs))
    4280        49367 :          != TYPE_PRECISION (TREE_TYPE (oprnd0)))
    4281              :     return NULL;
    4282              : 
    4283        49367 :   stmt_vec_info def_vinfo = vinfo->lookup_def (oprnd1);
    4284        49367 :   if (!def_vinfo || STMT_VINFO_DEF_TYPE (def_vinfo) == vect_external_def)
    4285              :     return NULL;
    4286              : 
    4287        46523 :   def_vinfo = vect_stmt_to_vectorize (def_vinfo);
    4288         1134 :   gcc_assert (def_vinfo);
    4289              : 
    4290        46523 :   *type_out = get_vectype_for_scalar_type (vinfo, TREE_TYPE (oprnd0));
    4291        46523 :   if (*type_out == NULL_TREE)
    4292              :     return NULL;
    4293              : 
    4294        33556 :   tree def = NULL_TREE;
    4295        33556 :   gassign *def_stmt = dyn_cast <gassign *> (def_vinfo->stmt);
    4296        19643 :   if (def_stmt && gimple_assign_cast_p (def_stmt))
    4297              :     {
    4298         5515 :       tree rhs1 = gimple_assign_rhs1 (def_stmt);
    4299         5515 :       if (TYPE_MODE (TREE_TYPE (rhs1)) == TYPE_MODE (TREE_TYPE (oprnd0))
    4300         5515 :           && TYPE_PRECISION (TREE_TYPE (rhs1))
    4301         1296 :              == TYPE_PRECISION (TREE_TYPE (oprnd0)))
    4302              :         {
    4303         1296 :           if (TYPE_PRECISION (TREE_TYPE (oprnd1))
    4304         1296 :               >= TYPE_PRECISION (TREE_TYPE (rhs1)))
    4305              :             def = rhs1;
    4306              :           else
    4307              :             {
    4308         1194 :               tree mask
    4309         1194 :                 = build_low_bits_mask (TREE_TYPE (rhs1),
    4310         1194 :                                        TYPE_PRECISION (TREE_TYPE (oprnd1)));
    4311         1194 :               def = vect_recog_temp_ssa_var (TREE_TYPE (rhs1), NULL);
    4312         1194 :               def_stmt = gimple_build_assign (def, BIT_AND_EXPR, rhs1, mask);
    4313         1194 :               tree vecstype = get_vectype_for_scalar_type (vinfo,
    4314         1194 :                                                            TREE_TYPE (rhs1));
    4315         1194 :               append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecstype);
    4316              :             }
    4317              :         }
    4318              :     }
    4319              : 
    4320         1296 :   if (def == NULL_TREE)
    4321              :     {
    4322        32260 :       def = vect_recog_temp_ssa_var (TREE_TYPE (oprnd0), NULL);
    4323        32260 :       def_stmt = gimple_build_assign (def, NOP_EXPR, oprnd1);
    4324        32260 :       append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    4325              :     }
    4326              : 
    4327              :   /* Pattern detected.  */
    4328        33556 :   vect_pattern_detected ("vect_recog_vector_vector_shift_pattern", last_stmt);
    4329              : 
    4330              :   /* Pattern supported.  Create a stmt to be used to replace the pattern.  */
    4331        33556 :   var = vect_recog_temp_ssa_var (TREE_TYPE (oprnd0), NULL);
    4332        33556 :   pattern_stmt = gimple_build_assign (var, rhs_code, oprnd0, def);
    4333              : 
    4334        33556 :   return pattern_stmt;
    4335              : }
    4336              : 
    4337              : /* Verify that the target has optabs of VECTYPE to perform all the steps
    4338              :    needed by the multiplication-by-immediate synthesis algorithm described by
    4339              :    ALG and VAR.  If SYNTH_SHIFT_P is true ensure that vector addition is
    4340              :    present.  Return true iff the target supports all the steps.  */
    4341              : 
    4342              : static bool
    4343       317185 : target_supports_mult_synth_alg (struct algorithm *alg, mult_variant var,
    4344              :                                  tree vectype, bool synth_shift_p)
    4345              : {
    4346       317185 :   if (alg->op[0] != alg_zero && alg->op[0] != alg_m)
    4347              :     return false;
    4348              : 
    4349       317185 :   bool supports_vminus = target_has_vecop_for_code (MINUS_EXPR, vectype);
    4350       317185 :   bool supports_vplus = target_has_vecop_for_code (PLUS_EXPR, vectype);
    4351              : 
    4352       317185 :   if (var == negate_variant
    4353       317185 :       && !target_has_vecop_for_code (NEGATE_EXPR, vectype))
    4354              :     return false;
    4355              : 
    4356              :   /* If we must synthesize shifts with additions make sure that vector
    4357              :      addition is available.  */
    4358       316510 :   if ((var == add_variant || synth_shift_p) && !supports_vplus)
    4359              :     return false;
    4360              : 
    4361       153497 :   for (int i = 1; i < alg->ops; i++)
    4362              :     {
    4363       115125 :       switch (alg->op[i])
    4364              :         {
    4365              :         case alg_shift:
    4366              :           break;
    4367        28352 :         case alg_add_t_m2:
    4368        28352 :         case alg_add_t2_m:
    4369        28352 :         case alg_add_factor:
    4370        28352 :           if (!supports_vplus)
    4371              :             return false;
    4372              :           break;
    4373        18071 :         case alg_sub_t_m2:
    4374        18071 :         case alg_sub_t2_m:
    4375        18071 :         case alg_sub_factor:
    4376        18071 :           if (!supports_vminus)
    4377              :             return false;
    4378              :           break;
    4379              :         case alg_unknown:
    4380              :         case alg_m:
    4381              :         case alg_zero:
    4382              :         case alg_impossible:
    4383              :           return false;
    4384            0 :         default:
    4385            0 :           gcc_unreachable ();
    4386              :         }
    4387              :     }
    4388              : 
    4389              :   return true;
    4390              : }
    4391              : 
    4392              : /* Synthesize a left shift of OP by AMNT bits using a series of additions and
    4393              :    putting the final result in DEST.  Append all statements but the last into
    4394              :    VINFO.  Return the last statement.  */
    4395              : 
    4396              : static gimple *
    4397            0 : synth_lshift_by_additions (vec_info *vinfo,
    4398              :                            tree dest, tree op, HOST_WIDE_INT amnt,
    4399              :                            stmt_vec_info stmt_info, tree vectype)
    4400              : {
    4401            0 :   HOST_WIDE_INT i;
    4402            0 :   tree itype = TREE_TYPE (op);
    4403            0 :   tree prev_res = op;
    4404            0 :   gcc_assert (amnt >= 0);
    4405            0 :   for (i = 0; i < amnt; i++)
    4406              :     {
    4407            0 :       tree tmp_var = (i < amnt - 1) ? vect_recog_temp_ssa_var (itype, NULL)
    4408              :                       : dest;
    4409            0 :       gimple *stmt
    4410            0 :         = gimple_build_assign (tmp_var, PLUS_EXPR, prev_res, prev_res);
    4411            0 :       prev_res = tmp_var;
    4412            0 :       if (i < amnt - 1)
    4413            0 :         append_pattern_def_seq (vinfo, stmt_info, stmt, vectype);
    4414              :       else
    4415            0 :         return stmt;
    4416              :     }
    4417            0 :   gcc_unreachable ();
    4418              :   return NULL;
    4419              : }
    4420              : 
    4421              : /* Helper for vect_synth_mult_by_constant.  Apply a binary operation
    4422              :    CODE to operands OP1 and OP2, creating a new temporary SSA var in
    4423              :    the process if necessary.  Append the resulting assignment statements
    4424              :    to the sequence in STMT_VINFO.  Return the SSA variable that holds the
    4425              :    result of the binary operation.  If SYNTH_SHIFT_P is true synthesize
    4426              :    left shifts using additions.  */
    4427              : 
    4428              : static tree
    4429        46105 : apply_binop_and_append_stmt (vec_info *vinfo,
    4430              :                              tree_code code, tree op1, tree op2,
    4431              :                              stmt_vec_info stmt_vinfo, tree vectype,
    4432              :                              bool synth_shift_p)
    4433              : {
    4434        46105 :   if (integer_zerop (op2)
    4435        46105 :       && (code == LSHIFT_EXPR
    4436        40146 :           || code == PLUS_EXPR))
    4437              :     {
    4438        40146 :       gcc_assert (TREE_CODE (op1) == SSA_NAME);
    4439              :       return op1;
    4440              :     }
    4441              : 
    4442         5959 :   gimple *stmt;
    4443         5959 :   tree itype = TREE_TYPE (op1);
    4444         5959 :   tree tmp_var = vect_recog_temp_ssa_var (itype, NULL);
    4445              : 
    4446         5959 :   if (code == LSHIFT_EXPR
    4447         5959 :       && synth_shift_p)
    4448              :     {
    4449            0 :       stmt = synth_lshift_by_additions (vinfo, tmp_var, op1,
    4450            0 :                                         TREE_INT_CST_LOW (op2), stmt_vinfo,
    4451              :                                         vectype);
    4452            0 :       append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
    4453            0 :       return tmp_var;
    4454              :     }
    4455              : 
    4456         5959 :   stmt = gimple_build_assign (tmp_var, code, op1, op2);
    4457         5959 :   append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
    4458         5959 :   return tmp_var;
    4459              : }
    4460              : 
    4461              : /* Synthesize a multiplication of OP by an INTEGER_CST VAL using shifts
    4462              :    and simple arithmetic operations to be vectorized.  Record the statements
    4463              :    produced in STMT_VINFO and return the last statement in the sequence or
    4464              :    NULL if it's not possible to synthesize such a multiplication.
    4465              :    This function mirrors the behavior of expand_mult_const in expmed.cc but
    4466              :    works on tree-ssa form.  */
    4467              : 
    4468              : static gimple *
    4469       320734 : vect_synth_mult_by_constant (vec_info *vinfo, tree op, tree val,
    4470              :                              stmt_vec_info stmt_vinfo)
    4471              : {
    4472       320734 :   tree itype = TREE_TYPE (op);
    4473       320734 :   machine_mode mode = TYPE_MODE (itype);
    4474       320734 :   struct algorithm alg;
    4475       320734 :   mult_variant variant;
    4476       320734 :   if (!tree_fits_shwi_p (val))
    4477              :     return NULL;
    4478              : 
    4479              :   /* Multiplication synthesis by shifts, adds and subs can introduce
    4480              :      signed overflow where the original operation didn't.  Perform the
    4481              :      operations on an unsigned type and cast back to avoid this.
    4482              :      In the future we may want to relax this for synthesis algorithms
    4483              :      that we can prove do not cause unexpected overflow.  */
    4484       317200 :   bool cast_to_unsigned_p = !TYPE_OVERFLOW_WRAPS (itype);
    4485              : 
    4486        61142 :   tree multtype = cast_to_unsigned_p ? unsigned_type_for (itype) : itype;
    4487       317200 :   tree vectype = get_vectype_for_scalar_type (vinfo, multtype);
    4488       317200 :   if (!vectype)
    4489              :     return NULL;
    4490              : 
    4491              :   /* Targets that don't support vector shifts but support vector additions
    4492              :      can synthesize shifts that way.  */
    4493       317200 :   bool synth_shift_p = !vect_supportable_shift (vinfo, LSHIFT_EXPR, multtype);
    4494              : 
    4495       317200 :   HOST_WIDE_INT hwval = tree_to_shwi (val);
    4496              :   /* Use MAX_COST here as we don't want to limit the sequence on rtx costs.
    4497              :      The vectorizer's benefit analysis will decide whether it's beneficial
    4498              :      to do this.  */
    4499       634204 :   bool possible = choose_mult_variant (VECTOR_MODE_P (TYPE_MODE (vectype))
    4500       317004 :                                        ? TYPE_MODE (vectype) : mode,
    4501              :                                        hwval, &alg, &variant, MAX_COST);
    4502       317200 :   if (!possible)
    4503              :     return NULL;
    4504              : 
    4505       317200 :   if (vect_is_reduction (stmt_vinfo))
    4506              :     {
    4507           26 :       int op_uses = alg.op[0] != alg_zero;
    4508           45 :       for (int i = 1; i < alg.ops; i++)
    4509           32 :         switch (alg.op[i])
    4510              :           {
    4511            4 :           case alg_add_t_m2:
    4512            4 :           case alg_sub_t_m2:
    4513            4 :             if (synth_shift_p && alg.log[i])
    4514              :               return NULL;
    4515              :             else
    4516            4 :               op_uses++;
    4517            4 :             break;
    4518            0 :           case alg_add_t2_m:
    4519            0 :           case alg_sub_t2_m:
    4520            0 :             op_uses++;
    4521              :             /* Fallthru.  */
    4522           28 :           case alg_shift:
    4523           28 :             if (synth_shift_p && alg.log[i])
    4524              :               return NULL;
    4525              :             break;
    4526              :           case alg_add_factor:
    4527              :           case alg_sub_factor:
    4528              :             return NULL;
    4529              :           default:
    4530              :             break;
    4531              :           }
    4532           13 :       if (variant == add_variant)
    4533            0 :         op_uses++;
    4534              :       /* When we'll synthesize more than a single use of the reduction
    4535              :          operand the reduction constraints are violated.  Avoid this
    4536              :          situation.  */
    4537           13 :       if (op_uses > 1)
    4538              :         return NULL;
    4539              :     }
    4540              : 
    4541       317185 :   if (!target_supports_mult_synth_alg (&alg, variant, vectype, synth_shift_p))
    4542              :     return NULL;
    4543              : 
    4544        38372 :   tree accumulator;
    4545              : 
    4546              :   /* Clear out the sequence of statements so we can populate it below.  */
    4547        38372 :   gimple *stmt = NULL;
    4548              : 
    4549        38372 :   if (cast_to_unsigned_p)
    4550              :     {
    4551        12676 :       tree tmp_op = vect_recog_temp_ssa_var (multtype, NULL);
    4552        12676 :       stmt = gimple_build_assign (tmp_op, CONVERT_EXPR, op);
    4553        12676 :       append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
    4554        12676 :       op = tmp_op;
    4555              :     }
    4556              : 
    4557        38372 :   if (alg.op[0] == alg_zero)
    4558          198 :     accumulator = build_int_cst (multtype, 0);
    4559              :   else
    4560              :     accumulator = op;
    4561              : 
    4562        38372 :   bool needs_fixup = (variant == negate_variant)
    4563        38372 :                       || (variant == add_variant);
    4564              : 
    4565       152894 :   for (int i = 1; i < alg.ops; i++)
    4566              :     {
    4567       114522 :       tree shft_log = build_int_cst (multtype, alg.log[i]);
    4568       114522 :       tree accum_tmp = vect_recog_temp_ssa_var (multtype, NULL);
    4569       114522 :       tree tmp_var = NULL_TREE;
    4570              : 
    4571       114522 :       switch (alg.op[i])
    4572              :         {
    4573        68417 :         case alg_shift:
    4574        68417 :           if (synth_shift_p)
    4575            0 :             stmt
    4576            0 :               = synth_lshift_by_additions (vinfo, accum_tmp, accumulator,
    4577            0 :                                            alg.log[i], stmt_vinfo, vectype);
    4578              :           else
    4579        68417 :             stmt = gimple_build_assign (accum_tmp, LSHIFT_EXPR, accumulator,
    4580              :                                          shft_log);
    4581              :           break;
    4582        23358 :         case alg_add_t_m2:
    4583        23358 :           tmp_var
    4584        23358 :             = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, op, shft_log,
    4585              :                                            stmt_vinfo, vectype, synth_shift_p);
    4586        23358 :           stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, accumulator,
    4587              :                                        tmp_var);
    4588        23358 :           break;
    4589        16991 :         case alg_sub_t_m2:
    4590        16991 :           tmp_var = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, op,
    4591              :                                                  shft_log, stmt_vinfo,
    4592              :                                                  vectype, synth_shift_p);
    4593              :           /* In some algorithms the first step involves zeroing the
    4594              :              accumulator.  If subtracting from such an accumulator
    4595              :              just emit the negation directly.  */
    4596        16991 :           if (integer_zerop (accumulator))
    4597          198 :             stmt = gimple_build_assign (accum_tmp, NEGATE_EXPR, tmp_var);
    4598              :           else
    4599        16793 :             stmt = gimple_build_assign (accum_tmp, MINUS_EXPR, accumulator,
    4600              :                                         tmp_var);
    4601              :           break;
    4602            0 :         case alg_add_t2_m:
    4603            0 :           tmp_var
    4604            0 :             = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
    4605              :                                            shft_log, stmt_vinfo, vectype,
    4606              :                                            synth_shift_p);
    4607            0 :           stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, tmp_var, op);
    4608            0 :           break;
    4609            0 :         case alg_sub_t2_m:
    4610            0 :           tmp_var
    4611            0 :             = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
    4612              :                                            shft_log, stmt_vinfo, vectype,
    4613              :                                            synth_shift_p);
    4614            0 :           stmt = gimple_build_assign (accum_tmp, MINUS_EXPR, tmp_var, op);
    4615            0 :           break;
    4616         4870 :         case alg_add_factor:
    4617         4870 :           tmp_var
    4618         4870 :             = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
    4619              :                                            shft_log, stmt_vinfo, vectype,
    4620              :                                            synth_shift_p);
    4621         4870 :           stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, accumulator,
    4622              :                                        tmp_var);
    4623         4870 :           break;
    4624          886 :         case alg_sub_factor:
    4625          886 :           tmp_var
    4626          886 :             = apply_binop_and_append_stmt (vinfo, LSHIFT_EXPR, accumulator,
    4627              :                                            shft_log, stmt_vinfo, vectype,
    4628              :                                            synth_shift_p);
    4629          886 :           stmt = gimple_build_assign (accum_tmp, MINUS_EXPR, tmp_var,
    4630              :                                       accumulator);
    4631          886 :           break;
    4632            0 :         default:
    4633            0 :           gcc_unreachable ();
    4634              :         }
    4635              :       /* We don't want to append the last stmt in the sequence to stmt_vinfo
    4636              :          but rather return it directly.  */
    4637              : 
    4638       114522 :       if ((i < alg.ops - 1) || needs_fixup || cast_to_unsigned_p)
    4639        89156 :         append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
    4640       114522 :       accumulator = accum_tmp;
    4641              :     }
    4642        38372 :   if (variant == negate_variant)
    4643              :     {
    4644          493 :       tree accum_tmp = vect_recog_temp_ssa_var (multtype, NULL);
    4645          493 :       stmt = gimple_build_assign (accum_tmp, NEGATE_EXPR, accumulator);
    4646          493 :       accumulator = accum_tmp;
    4647          493 :       if (cast_to_unsigned_p)
    4648          169 :         append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
    4649              :     }
    4650        37879 :   else if (variant == add_variant)
    4651              :     {
    4652           97 :       tree accum_tmp = vect_recog_temp_ssa_var (multtype, NULL);
    4653           97 :       stmt = gimple_build_assign (accum_tmp, PLUS_EXPR, accumulator, op);
    4654           97 :       accumulator = accum_tmp;
    4655           97 :       if (cast_to_unsigned_p)
    4656           91 :         append_pattern_def_seq (vinfo, stmt_vinfo, stmt, vectype);
    4657              :     }
    4658              :   /* Move back to a signed if needed.  */
    4659        38042 :   if (cast_to_unsigned_p)
    4660              :     {
    4661        12676 :       tree accum_tmp = vect_recog_temp_ssa_var (itype, NULL);
    4662        12676 :       stmt = gimple_build_assign (accum_tmp, CONVERT_EXPR, accumulator);
    4663              :     }
    4664              : 
    4665              :   return stmt;
    4666              : }
    4667              : 
    4668              : /* Detect multiplication by constant and convert it into a sequence of
    4669              :    shifts and additions, subtractions, negations.  We reuse the
    4670              :    choose_mult_variant algorithms from expmed.cc
    4671              : 
    4672              :    Input/Output:
    4673              : 
    4674              :    STMT_VINFO: The stmt from which the pattern search begins,
    4675              :    i.e. the mult stmt.
    4676              : 
    4677              :  Output:
    4678              : 
    4679              :   * TYPE_OUT: The type of the output of this pattern.
    4680              : 
    4681              :   * Return value: A new stmt that will be used to replace
    4682              :     the multiplication.  */
    4683              : 
    4684              : static gimple *
    4685     33083027 : vect_recog_mult_pattern (vec_info *vinfo,
    4686              :                          stmt_vec_info stmt_vinfo, tree *type_out)
    4687              : {
    4688     33083027 :   gimple *last_stmt = stmt_vinfo->stmt;
    4689     33083027 :   tree oprnd0, oprnd1, vectype, itype;
    4690     33083027 :   gimple *pattern_stmt;
    4691              : 
    4692     33083027 :   if (!is_gimple_assign (last_stmt))
    4693              :     return NULL;
    4694              : 
    4695     23175569 :   if (gimple_assign_rhs_code (last_stmt) != MULT_EXPR)
    4696              :     return NULL;
    4697              : 
    4698      1583282 :   oprnd0 = gimple_assign_rhs1 (last_stmt);
    4699      1583282 :   oprnd1 = gimple_assign_rhs2 (last_stmt);
    4700      1583282 :   itype = TREE_TYPE (oprnd0);
    4701              : 
    4702      1583282 :   if (TREE_CODE (oprnd0) != SSA_NAME
    4703      1583215 :       || TREE_CODE (oprnd1) != INTEGER_CST
    4704       954416 :       || !INTEGRAL_TYPE_P (itype)
    4705      2537698 :       || !type_has_mode_precision_p (itype))
    4706              :     return NULL;
    4707              : 
    4708       954361 :   vectype = get_vectype_for_scalar_type (vinfo, itype);
    4709       954361 :   if (vectype == NULL_TREE)
    4710              :     return NULL;
    4711              : 
    4712              :   /* If the target can handle vectorized multiplication natively,
    4713              :      don't attempt to optimize this.  */
    4714       776453 :   optab mul_optab = optab_for_tree_code (MULT_EXPR, vectype, optab_default);
    4715       776453 :   if (mul_optab != unknown_optab
    4716       776453 :       && can_implement_p (mul_optab, TYPE_MODE (vectype)))
    4717              :     return NULL;
    4718              : 
    4719       320734 :   pattern_stmt = vect_synth_mult_by_constant (vinfo,
    4720              :                                               oprnd0, oprnd1, stmt_vinfo);
    4721       320734 :   if (!pattern_stmt)
    4722              :     return NULL;
    4723              : 
    4724              :   /* Pattern detected.  */
    4725        38372 :   vect_pattern_detected ("vect_recog_mult_pattern", last_stmt);
    4726              : 
    4727        38372 :   *type_out = vectype;
    4728              : 
    4729        38372 :   return pattern_stmt;
    4730              : }
    4731              : 
    4732              : extern bool gimple_unsigned_integer_sat_add (tree, tree*, tree (*)(tree));
    4733              : extern bool gimple_unsigned_integer_sat_sub (tree, tree*, tree (*)(tree));
    4734              : extern bool gimple_unsigned_integer_sat_trunc (tree, tree*, tree (*)(tree));
    4735              : 
    4736              : extern bool gimple_unsigned_integer_narrow_clip (tree, tree*, tree (*)(tree));
    4737              : 
    4738              : extern bool gimple_signed_integer_sat_add (tree, tree*, tree (*)(tree));
    4739              : extern bool gimple_signed_integer_sat_sub (tree, tree*, tree (*)(tree));
    4740              : extern bool gimple_signed_integer_sat_trunc (tree, tree*, tree (*)(tree));
    4741              : 
    4742              : static gimple *
    4743         1269 : vect_recog_build_binary_gimple_stmt (vec_info *vinfo, stmt_vec_info stmt_info,
    4744              :                                      internal_fn fn, tree *type_out,
    4745              :                                      tree lhs, tree op_0, tree op_1)
    4746              : {
    4747         1269 :   tree itype = TREE_TYPE (op_0);
    4748         1269 :   tree otype = TREE_TYPE (lhs);
    4749         1269 :   tree v_itype = get_vectype_for_scalar_type (vinfo, itype);
    4750         1269 :   tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
    4751              : 
    4752         1269 :   if (v_itype != NULL_TREE && v_otype != NULL_TREE
    4753         1269 :     && direct_internal_fn_supported_p (fn, v_itype, OPTIMIZE_FOR_BOTH))
    4754              :     {
    4755         1011 :       gcall *call = gimple_build_call_internal (fn, 2, op_0, op_1);
    4756         1011 :       tree in_ssa = vect_recog_temp_ssa_var (itype, NULL);
    4757              : 
    4758         1011 :       gimple_call_set_lhs (call, in_ssa);
    4759         1011 :       gimple_call_set_nothrow (call, /* nothrow_p */ false);
    4760         1011 :       gimple_set_location (call, gimple_location (STMT_VINFO_STMT (stmt_info)));
    4761              : 
    4762         1011 :       *type_out = v_otype;
    4763              : 
    4764         1011 :       if (types_compatible_p (itype, otype))
    4765              :         return call;
    4766              :       else
    4767              :         {
    4768            0 :           append_pattern_def_seq (vinfo, stmt_info, call, v_itype);
    4769            0 :           tree out_ssa = vect_recog_temp_ssa_var (otype, NULL);
    4770              : 
    4771            0 :           return gimple_build_assign (out_ssa, NOP_EXPR, in_ssa);
    4772              :         }
    4773              :     }
    4774              : 
    4775              :   return NULL;
    4776              : }
    4777              : 
    4778              : /*
    4779              :  * Try to detect saturation add pattern (SAT_ADD), aka below gimple:
    4780              :  *   _7 = _4 + _6;
    4781              :  *   _8 = _4 > _7;
    4782              :  *   _9 = (long unsigned int) _8;
    4783              :  *   _10 = -_9;
    4784              :  *   _12 = _7 | _10;
    4785              :  *
    4786              :  * And then simplified to
    4787              :  *   _12 = .SAT_ADD (_4, _6);
    4788              :  */
    4789              : 
    4790              : static gimple *
    4791     33157917 : vect_recog_sat_add_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
    4792              :                             tree *type_out)
    4793              : {
    4794     33157917 :   gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
    4795              : 
    4796     33157917 :   if (!is_gimple_assign (last_stmt))
    4797              :     return NULL;
    4798              : 
    4799     23250459 :   tree ops[2];
    4800     23250459 :   tree lhs = gimple_assign_lhs (last_stmt);
    4801              : 
    4802     23250459 :   if (gimple_unsigned_integer_sat_add (lhs, ops, NULL)
    4803     23250459 :       || gimple_signed_integer_sat_add (lhs, ops, NULL))
    4804              :     {
    4805              :       /* .SAT_ADD saturates in the type of its first operand, which must be
    4806              :          the type of the result.  */
    4807          831 :       gcc_checking_assert (types_compatible_p (TREE_TYPE (lhs),
    4808              :                                                TREE_TYPE (ops[0])));
    4809          831 :       if (TREE_CODE (ops[1]) == INTEGER_CST)
    4810           13 :         ops[1] = fold_convert (TREE_TYPE (ops[0]), ops[1]);
    4811              : 
    4812          831 :       gimple *stmt = vect_recog_build_binary_gimple_stmt (vinfo, stmt_vinfo,
    4813              :                                                           IFN_SAT_ADD, type_out,
    4814              :                                                           lhs, ops[0], ops[1]);
    4815          831 :       if (stmt)
    4816              :         {
    4817          796 :           vect_pattern_detected ("vect_recog_sat_add_pattern", last_stmt);
    4818          796 :           return stmt;
    4819              :         }
    4820              :     }
    4821              : 
    4822              :   return NULL;
    4823              : }
    4824              : 
    4825              : /*
    4826              :  * Try to transform the truncation for .SAT_SUB pattern,  mostly occurs in
    4827              :  * the benchmark zip.  Aka:
    4828              :  *
    4829              :  *   unsigned int _1;
    4830              :  *   unsigned int _2;
    4831              :  *   unsigned short int _4;
    4832              :  *   _9 = (unsigned short int).SAT_SUB (_1, _2);
    4833              :  *
    4834              :  *   if _1 is known to be in the range of unsigned short int.  For example
    4835              :  *   there is a def _1 = (unsigned short int)_4.  Then we can transform the
    4836              :  *   truncation to:
    4837              :  *
    4838              :  *   _3 = (unsigned short int) MIN (65535, _2); // aka _3 = .SAT_TRUNC (_2);
    4839              :  *   _9 = .SAT_SUB (_4, _3);
    4840              :  *
    4841              :  *   Then,  we can better vectorized code and avoid the unnecessary narrowing
    4842              :  *   stmt during vectorization with below stmt(s).
    4843              :  *
    4844              :  *   _3 = .SAT_TRUNC(_2); // SI => HI
    4845              :  *   _9 = .SAT_SUB (_4, _3);
    4846              :  */
    4847              : static void
    4848          438 : vect_recog_sat_sub_pattern_transform (vec_info *vinfo,
    4849              :                                       stmt_vec_info stmt_vinfo,
    4850              :                                       tree lhs, tree *ops)
    4851              : {
    4852          438 :   tree otype = TREE_TYPE (lhs);
    4853          438 :   tree itype = TREE_TYPE (ops[0]);
    4854          438 :   unsigned itype_prec = TYPE_PRECISION (itype);
    4855          438 :   unsigned otype_prec = TYPE_PRECISION (otype);
    4856              : 
    4857          438 :   if (types_compatible_p (otype, itype) || otype_prec >= itype_prec)
    4858          438 :     return;
    4859              : 
    4860            0 :   tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
    4861            0 :   tree v_itype = get_vectype_for_scalar_type (vinfo, itype);
    4862            0 :   tree_pair v_pair = tree_pair (v_otype, v_itype);
    4863              : 
    4864            0 :   if (v_otype == NULL_TREE || v_itype == NULL_TREE
    4865            0 :     || !direct_internal_fn_supported_p (IFN_SAT_TRUNC, v_pair,
    4866              :                                         OPTIMIZE_FOR_BOTH))
    4867              :     return;
    4868              : 
    4869              :   /* 1. Find the _4 and update ops[0] as above example.  */
    4870            0 :   vect_unpromoted_value unprom;
    4871            0 :   tree tmp = vect_look_through_possible_promotion (vinfo, ops[0], &unprom);
    4872              : 
    4873            0 :   if (tmp == NULL_TREE || TYPE_PRECISION (unprom.type) != otype_prec)
    4874              :     return;
    4875              : 
    4876            0 :   ops[0] = tmp;
    4877              : 
    4878              :   /* 2. Generate _3 = .SAT_TRUNC (_2) and update ops[1] as above example.  */
    4879            0 :   tree trunc_lhs_ssa = vect_recog_temp_ssa_var (otype, NULL);
    4880            0 :   gcall *call = gimple_build_call_internal (IFN_SAT_TRUNC, 1, ops[1]);
    4881              : 
    4882            0 :   gimple_call_set_lhs (call, trunc_lhs_ssa);
    4883            0 :   gimple_call_set_nothrow (call, /* nothrow_p */ false);
    4884            0 :   append_pattern_def_seq (vinfo, stmt_vinfo, call, v_otype);
    4885              : 
    4886            0 :   ops[1] = trunc_lhs_ssa;
    4887              : }
    4888              : 
    4889              : /*
    4890              :  * Try to detect saturation sub pattern (SAT_ADD), aka below gimple:
    4891              :  * Unsigned:
    4892              :  *   _7 = _1 >= _2;
    4893              :  *   _8 = _1 - _2;
    4894              :  *   _10 = (long unsigned int) _7;
    4895              :  *   _9 = _8 * _10;
    4896              :  *
    4897              :  * And then simplified to
    4898              :  *   _9 = .SAT_SUB (_1, _2);
    4899              :  *
    4900              :  * Signed:
    4901              :  *   x.0_4 = (unsigned char) x_16;
    4902              :  *   y.1_5 = (unsigned char) y_18;
    4903              :  *   _6 = x.0_4 - y.1_5;
    4904              :  *   minus_19 = (int8_t) _6;
    4905              :  *   _7 = x_16 ^ y_18;
    4906              :  *   _8 = x_16 ^ minus_19;
    4907              :  *   _44 = _7 < 0;
    4908              :  *   _23 = x_16 < 0;
    4909              :  *   _24 = (signed char) _23;
    4910              :  *   _58 = (unsigned char) _24;
    4911              :  *   _59 = -_58;
    4912              :  *   _25 = (signed char) _59;
    4913              :  *   _26 = _25 ^ 127;
    4914              :  *   _42 = _8 < 0;
    4915              :  *   _41 = _42 & _44;
    4916              :  *   iftmp.2_11 = _41 ? _26 : minus_19;
    4917              :  *
    4918              :  * And then simplified to
    4919              :  *   iftmp.2_11 = .SAT_SUB (x_16, y_18);
    4920              :  */
    4921              : 
    4922              : static gimple *
    4923     33157121 : vect_recog_sat_sub_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
    4924              :                             tree *type_out)
    4925              : {
    4926     33157121 :   gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
    4927              : 
    4928     33157121 :   if (!is_gimple_assign (last_stmt))
    4929              :     return NULL;
    4930              : 
    4931     23249663 :   tree ops[2];
    4932     23249663 :   tree lhs = gimple_assign_lhs (last_stmt);
    4933              : 
    4934     23249663 :   if (gimple_unsigned_integer_sat_sub (lhs, ops, NULL)
    4935     23249663 :       || gimple_signed_integer_sat_sub (lhs, ops, NULL))
    4936              :     {
    4937          438 :       vect_recog_sat_sub_pattern_transform (vinfo, stmt_vinfo, lhs, ops);
    4938          438 :       gimple *stmt = vect_recog_build_binary_gimple_stmt (vinfo, stmt_vinfo,
    4939              :                                                           IFN_SAT_SUB, type_out,
    4940              :                                                           lhs, ops[0], ops[1]);
    4941          438 :       if (stmt)
    4942              :         {
    4943          215 :           vect_pattern_detected ("vect_recog_sat_sub_pattern", last_stmt);
    4944          215 :           return stmt;
    4945              :         }
    4946              :     }
    4947              : 
    4948              :   return NULL;
    4949              : }
    4950              : 
    4951              : /* Return the vector type for ITYPE if the target supports the MAX_EXPR
    4952              :    and SAT_TRUNC sequence.  */
    4953              : 
    4954              : static tree
    4955           73 : vect_sat_trunc_clip_input_vectype (vec_info *vinfo, tree v_otype,
    4956              :                                    tree itype)
    4957              : {
    4958           73 :   tree v_itype;
    4959           73 :   if (!vect_supportable_direct_optab_p
    4960           73 :         (vinfo, itype, MAX_EXPR, itype, &v_itype)
    4961           95 :       || !direct_internal_fn_supported_p
    4962           95 :            (IFN_SAT_TRUNC, tree_pair (v_otype, v_itype),
    4963              :             OPTIMIZE_FOR_BOTH))
    4964              :     return NULL_TREE;
    4965              : 
    4966            0 :   return v_itype;
    4967              : }
    4968              : 
    4969              : /*
    4970              :  * Try to detect saturation truncation pattern (SAT_TRUNC), aka below gimple:
    4971              :  *   overflow_5 = x_4(D) > 4294967295;
    4972              :  *   _1 = (unsigned int) x_4(D);
    4973              :  *   _2 = (unsigned int) overflow_5;
    4974              :  *   _3 = -_2;
    4975              :  *   _6 = _1 | _3;
    4976              :  *
    4977              :  * And then simplified to
    4978              :  *   _6 = .SAT_TRUNC (x_4(D));
    4979              :  */
    4980              : 
    4981              : static gimple *
    4982     33156906 : vect_recog_sat_trunc_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
    4983              :                               tree *type_out)
    4984              : {
    4985     33156906 :   gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
    4986              : 
    4987     33156906 :   if (!is_gimple_assign (last_stmt))
    4988              :     return NULL;
    4989              : 
    4990     23249448 :   tree ops[1];
    4991     23249448 :   tree lhs = gimple_assign_lhs (last_stmt);
    4992     23249448 :   tree otype = TREE_TYPE (lhs);
    4993              : 
    4994     23249448 :   if ((gimple_unsigned_integer_narrow_clip (lhs, ops, NULL))
    4995     23249448 :        && type_has_mode_precision_p (otype))
    4996              :     {
    4997           73 :       tree input = ops[0];
    4998           73 :       tree itype = TREE_TYPE (input);
    4999           73 :       tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
    5000           73 :       internal_fn fn = IFN_SAT_TRUNC;
    5001           73 :       tree v_itype = NULL_TREE;
    5002              : 
    5003           73 :       if (v_otype != NULL_TREE)
    5004              :         {
    5005              :           /* Avoid widening an input only to narrow it again.  A signed
    5006              :              promotion does not affect MAX (X, 0), so use the unpromoted
    5007              :              value when the target can narrow it directly to the output
    5008              :              type.  */
    5009           73 :           vect_unpromoted_value unprom;
    5010           73 :           bool single_use_p = true;
    5011           73 :           if (vect_look_through_possible_promotion (vinfo, input, &unprom,
    5012              :                                                       &single_use_p)
    5013           73 :               && single_use_p
    5014           34 :               && INTEGRAL_TYPE_P (unprom.type)
    5015           34 :               && TYPE_SIGN (unprom.type) == TYPE_SIGN (itype)
    5016           34 :               && type_has_mode_precision_p (unprom.type)
    5017           34 :               && TYPE_PRECISION (otype) < TYPE_PRECISION (unprom.type)
    5018          107 :               && TYPE_PRECISION (unprom.type) < TYPE_PRECISION (itype))
    5019              :             {
    5020            0 :               tree v_unprom_type
    5021            0 :                 = vect_sat_trunc_clip_input_vectype (vinfo, v_otype,
    5022              :                                                        unprom.type);
    5023            0 :               if (v_unprom_type != NULL_TREE)
    5024              :                 {
    5025            0 :                   input = unprom.op;
    5026            0 :                   itype = unprom.type;
    5027            0 :                   v_itype = v_unprom_type;
    5028              :                 }
    5029              :             }
    5030              : 
    5031            0 :           if (v_itype == NULL_TREE)
    5032           73 :             v_itype
    5033           73 :               = vect_sat_trunc_clip_input_vectype (vinfo, v_otype, itype);
    5034              : 
    5035           73 :           if (v_itype != NULL_TREE)
    5036              :             {
    5037            0 :               tree temp = vect_recog_temp_ssa_var (itype, NULL);
    5038            0 :               gimple *max_stmt
    5039            0 :                 = gimple_build_assign (temp, MAX_EXPR,
    5040              :                                        build_zero_cst (itype), input);
    5041            0 :               append_pattern_def_seq (vinfo, stmt_vinfo, max_stmt, v_itype);
    5042              : 
    5043            0 :               gcall *call = gimple_build_call_internal (fn, 1, temp);
    5044            0 :               tree out_ssa = vect_recog_temp_ssa_var (otype, NULL);
    5045              : 
    5046            0 :               gimple_call_set_lhs (call, out_ssa);
    5047            0 :               gimple_call_set_nothrow (call, false);
    5048            0 :               gimple_set_location (call, gimple_location (last_stmt));
    5049              : 
    5050            0 :               *type_out = v_otype;
    5051              : 
    5052            0 :               return call;
    5053              :             }
    5054              :         }
    5055              :     }
    5056              : 
    5057     23249448 :   if ((gimple_unsigned_integer_sat_trunc (lhs, ops, NULL)
    5058     23249063 :        || gimple_signed_integer_sat_trunc (lhs, ops, NULL))
    5059     23249620 :       && type_has_mode_precision_p (otype))
    5060              :     {
    5061          545 :       tree itype = TREE_TYPE (ops[0]);
    5062          545 :       tree v_itype = get_vectype_for_scalar_type (vinfo, itype);
    5063          545 :       tree v_otype = get_vectype_for_scalar_type (vinfo, otype);
    5064          545 :       internal_fn fn = IFN_SAT_TRUNC;
    5065              : 
    5066          537 :       if (v_itype != NULL_TREE && v_otype != NULL_TREE
    5067         1082 :         && direct_internal_fn_supported_p (fn, tree_pair (v_otype, v_itype),
    5068              :                                            OPTIMIZE_FOR_BOTH))
    5069              :         {
    5070            0 :           gcall *call = gimple_build_call_internal (fn, 1, ops[0]);
    5071            0 :           tree out_ssa = vect_recog_temp_ssa_var (otype, NULL);
    5072              : 
    5073            0 :           gimple_call_set_lhs (call, out_ssa);
    5074            0 :           gimple_call_set_nothrow (call, /* nothrow_p */ false);
    5075            0 :           gimple_set_location (call, gimple_location (last_stmt));
    5076              : 
    5077            0 :           *type_out = v_otype;
    5078              : 
    5079            0 :           return call;
    5080              :         }
    5081              :     }
    5082              : 
    5083              :   return NULL;
    5084              : }
    5085              : 
    5086              : 
    5087              : /* Function add_code_for_floorceilround_divmod
    5088              :    A helper function to add compensation code for implementing FLOOR_MOD_EXPR,
    5089              :    FLOOR_DIV_EXPR, CEIL_MOD_EXPR, CEIL_DIV_EXPR, ROUND_MOD_EXPR and
    5090              :    ROUND_DIV_EXPR
    5091              :    The quotient and remainder are needed for implemented these operators.
    5092              :    FLOOR cases
    5093              :    r = x %[fl] y; r = x/[fl] y;
    5094              :    is
    5095              :    r = x % y; if (r && (x ^ y) < 0) r += y;
    5096              :    r = x % y; d = x/y; if (r && (x ^ y) < 0) d--; Respectively
    5097              :    Produce following sequence
    5098              :    v0 = x^y
    5099              :    v1 = -r
    5100              :    v2 = r | -r
    5101              :    v3 = v0 & v2
    5102              :    v4 = v3 < 0
    5103              :    if (floor_mod)
    5104              :      v5 = v4 ? y : 0
    5105              :      v6 = r + v5
    5106              :    if (floor_div)
    5107              :      v5 = v4 ? 1 : 0
    5108              :      v6 = d - 1
    5109              :    Similar sequences of vector instructions are produces for following cases
    5110              :    CEIL cases
    5111              :    r = x %[cl] y; r = x/[cl] y;
    5112              :    is
    5113              :    r = x % y; if (r && (x ^ y) >= 0) r -= y;
    5114              :    r = x % y; if (r) r -= y; (unsigned)
    5115              :    r = x % y; d = x/y; if (r && (x ^ y) >= 0) d++;
    5116              :    r = x % y; d = x/y; if (r) d++; (unsigned)
    5117              :    ROUND cases
    5118              :    r = x %[rd] y; r = x/[rd] y;
    5119              :    is
    5120              :    r = x % y; if (r > ((y-1)/2)) if ((x ^ y) >= 0) r -= y; else r += y;
    5121              :    r = x % y; if (r > ((y-1)/2)) r -= y; (unsigned)
    5122              :    r = x % y; d = x/y; if (r > ((y-1)/2)) if ((x ^ y) >= 0) d++; else d--;
    5123              :    r = x % y; d = x/y; if (r > ((y-1)/2)) d++; (unsigned)
    5124              :    Inputs:
    5125              :      VECTYPE: Vector type of the operands
    5126              :      STMT_VINFO: Statement where pattern begins
    5127              :      RHS_CODE: Should either be FLOOR_MOD_EXPR or FLOOR_DIV_EXPR
    5128              :      Q: The quotient of division
    5129              :      R: Remainder of division
    5130              :      OPRDN0/OPRND1: Actual operands involved
    5131              :      ITYPE: tree type of oprnd0
    5132              :    Output:
    5133              :      NULL if vectorization not possible
    5134              :      Gimple statement based on rhs_code
    5135              : */
    5136              : static gimple *
    5137          418 : add_code_for_floorceilround_divmod (tree vectype, vec_info *vinfo,
    5138              :                                     stmt_vec_info stmt_vinfo,
    5139              :                                     enum tree_code rhs_code, tree q, tree r,
    5140              :                                     tree oprnd0, tree oprnd1, tree itype)
    5141              : {
    5142          418 :   gimple *def_stmt;
    5143          418 :   tree mask_vectype = truth_type_for (vectype);
    5144          418 :   if (!mask_vectype)
    5145              :     return NULL;
    5146          418 :   tree bool_cond;
    5147          418 :   bool unsigned_p = TYPE_UNSIGNED (itype);
    5148              : 
    5149          418 :   switch (rhs_code)
    5150              :     {
    5151          382 :     case FLOOR_MOD_EXPR:
    5152          382 :     case FLOOR_DIV_EXPR:
    5153          382 :     case CEIL_MOD_EXPR:
    5154          382 :     case CEIL_DIV_EXPR:
    5155          382 :       {
    5156          382 :         if (!target_has_vecop_for_code (NEGATE_EXPR, vectype)
    5157          358 :             || !target_has_vecop_for_code (BIT_XOR_EXPR, vectype)
    5158          358 :             || !target_has_vecop_for_code (BIT_IOR_EXPR, vectype)
    5159          358 :             || !target_has_vecop_for_code (PLUS_EXPR, vectype)
    5160          358 :             || !target_has_vecop_for_code (MINUS_EXPR, vectype)
    5161          358 :             || !expand_vec_cmp_expr_p (vectype, mask_vectype, LT_EXPR)
    5162          606 :             || !expand_vec_cond_expr_p (vectype, mask_vectype))
    5163              :           return NULL;
    5164          224 :         if (unsigned_p)
    5165              :           {
    5166           18 :             gcc_assert (rhs_code == CEIL_MOD_EXPR || rhs_code == CEIL_DIV_EXPR);
    5167              : 
    5168           18 :             if (!expand_vec_cmp_expr_p (vectype, mask_vectype, GT_EXPR))
    5169              :               return NULL;
    5170           18 :             bool is_mod = rhs_code == CEIL_MOD_EXPR;
    5171              :             // r > 0
    5172           18 :             bool_cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
    5173           18 :             def_stmt = gimple_build_assign (bool_cond, GT_EXPR, r,
    5174              :                                             build_int_cst (itype, 0));
    5175           18 :             append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
    5176              :                                     itype);
    5177              : 
    5178              :             // (r > 0) ? y : 0 (mod)
    5179              :             // (r > 0) ? 1 : 0 (ceil)
    5180           18 :             tree extr_cond = vect_recog_temp_ssa_var (itype, NULL);
    5181           18 :             def_stmt
    5182           27 :               = gimple_build_assign (extr_cond, COND_EXPR, bool_cond,
    5183            9 :                                      is_mod ? oprnd1 : build_int_cst (itype, 1),
    5184              :                                      build_int_cst (itype, 0));
    5185           18 :             append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5186              : 
    5187              :             // r -= (r > 0) ? y : 0 (mod)
    5188              :             // d += (x^y < 0 && r) ? -1 : 0 (ceil)
    5189           18 :             tree result = vect_recog_temp_ssa_var (itype, NULL);
    5190           27 :             return gimple_build_assign (result, is_mod ? MINUS_EXPR : PLUS_EXPR,
    5191           18 :                                         is_mod ? r : q, extr_cond);
    5192              :           }
    5193              :         else
    5194              :           {
    5195          206 :             bool ceil_p
    5196          206 :               = (rhs_code == CEIL_MOD_EXPR || rhs_code == CEIL_DIV_EXPR);
    5197          206 :             if (ceil_p && !target_has_vecop_for_code (BIT_NOT_EXPR, vectype))
    5198              :               return NULL;
    5199              :             // x ^ y
    5200          206 :             tree xort = vect_recog_temp_ssa_var (itype, NULL);
    5201          206 :             def_stmt = gimple_build_assign (xort, BIT_XOR_EXPR, oprnd0, oprnd1);
    5202          206 :             append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5203              : 
    5204          206 :             tree cond_reg = xort;
    5205              :             // ~(x ^ y) (ceil)
    5206          206 :             if (ceil_p)
    5207              :               {
    5208           18 :                 cond_reg = vect_recog_temp_ssa_var (itype, NULL);
    5209           18 :                 def_stmt = gimple_build_assign (cond_reg, BIT_NOT_EXPR, xort);
    5210           18 :                 append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5211              :               }
    5212              : 
    5213              :             // -r
    5214          206 :             tree negate_r = vect_recog_temp_ssa_var (itype, NULL);
    5215          206 :             def_stmt = gimple_build_assign (negate_r, NEGATE_EXPR, r);
    5216          206 :             append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5217              : 
    5218              :             // r | -r , sign bit is set if r!=0
    5219          206 :             tree r_or_negr = vect_recog_temp_ssa_var (itype, NULL);
    5220          206 :             def_stmt
    5221          206 :               = gimple_build_assign (r_or_negr, BIT_IOR_EXPR, r, negate_r);
    5222          206 :             append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5223              : 
    5224              :             // (x ^ y) & (r | -r)
    5225              :             // ~(x ^ y) & (r | -r) (ceil)
    5226          206 :             tree r_or_negr_and_xor = vect_recog_temp_ssa_var (itype, NULL);
    5227          206 :             def_stmt = gimple_build_assign (r_or_negr_and_xor, BIT_AND_EXPR,
    5228              :                                             r_or_negr, cond_reg);
    5229          206 :             append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5230              : 
    5231              :             // (x ^ y) & (r | -r) < 0 which is equivalent to (x^y < 0 && r!=0)
    5232          206 :             bool_cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
    5233          206 :             def_stmt
    5234          206 :               = gimple_build_assign (bool_cond, LT_EXPR, r_or_negr_and_xor,
    5235              :                                      build_int_cst (itype, 0));
    5236          206 :             append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
    5237              :                                     itype);
    5238              : 
    5239              :             // (x^y < 0 && r) ? y : 0 (mod)
    5240              :             // (x^y < 0 && r) ? -1 : 0 (div)
    5241          206 :             bool is_mod
    5242          206 :               = (rhs_code == FLOOR_MOD_EXPR || rhs_code == CEIL_MOD_EXPR);
    5243          206 :             tree extr_cond = vect_recog_temp_ssa_var (itype, NULL);
    5244          246 :             def_stmt = gimple_build_assign (extr_cond, COND_EXPR, bool_cond,
    5245              :                                             is_mod ? oprnd1
    5246           40 :                                                    : build_int_cst (itype, -1),
    5247              :                                             build_int_cst (itype, 0));
    5248          206 :             append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5249              : 
    5250              :             // r += (x ^ y < 0 && r) ? y : 0 (floor mod)
    5251              :             // d += (x^y < 0 && r) ? -1 : 0 (floor div)
    5252              :             // r -= (x ^ y < 0 && r) ? y : 0 (ceil mod)
    5253              :             // d -= (x^y < 0 && r) ? -1 : 0 (ceil div)
    5254          206 :             tree result = vect_recog_temp_ssa_var (itype, NULL);
    5255          412 :             return gimple_build_assign (result,
    5256          206 :                                         (rhs_code == FLOOR_MOD_EXPR
    5257          206 :                                          || rhs_code == FLOOR_DIV_EXPR)
    5258              :                                           ? PLUS_EXPR
    5259              :                                           : MINUS_EXPR,
    5260          206 :                                         is_mod ? r : q, extr_cond);
    5261              :           }
    5262              :       }
    5263           36 :     case ROUND_MOD_EXPR:
    5264           36 :     case ROUND_DIV_EXPR:
    5265           36 :       {
    5266           36 :         if (!target_has_vecop_for_code (BIT_AND_EXPR, vectype)
    5267           36 :             || !target_has_vecop_for_code (PLUS_EXPR, vectype)
    5268           36 :             || !expand_vec_cmp_expr_p (vectype, mask_vectype, LT_EXPR)
    5269           36 :             || !expand_vec_cmp_expr_p (vectype, mask_vectype, GT_EXPR)
    5270           72 :             || !expand_vec_cond_expr_p (vectype, mask_vectype))
    5271              :           return NULL;
    5272              : 
    5273           36 :         bool is_mod = rhs_code == ROUND_MOD_EXPR;
    5274           36 :         HOST_WIDE_INT d = TREE_INT_CST_LOW (oprnd1);
    5275           36 :         unsigned HOST_WIDE_INT abs_d
    5276              :           = (d >= 0 ? (unsigned HOST_WIDE_INT) d : -(unsigned HOST_WIDE_INT) d);
    5277           36 :         unsigned HOST_WIDE_INT mid_d = (abs_d - 1) >> 1;
    5278           36 :         if (!unsigned_p)
    5279              :           {
    5280              :             // check availability of abs expression for vector
    5281           18 :             if (!target_has_vecop_for_code (ABS_EXPR, vectype))
    5282              :               return NULL;
    5283              :             // abs (r)
    5284           18 :             tree abs_r = vect_recog_temp_ssa_var (itype, NULL);
    5285           18 :             def_stmt = gimple_build_assign (abs_r, ABS_EXPR, r);
    5286           18 :             append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5287              : 
    5288              :             // abs (r) > (abs (y-1) >> 1)
    5289           18 :             tree round_p = vect_recog_temp_ssa_var (boolean_type_node, NULL);
    5290           18 :             def_stmt = gimple_build_assign (round_p, GT_EXPR, abs_r,
    5291           18 :                                             build_int_cst (itype, mid_d));
    5292           18 :             append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
    5293              :                                     itype);
    5294              : 
    5295              :             // x ^ y
    5296           18 :             tree cond_reg = vect_recog_temp_ssa_var (itype, NULL);
    5297           18 :             def_stmt
    5298           18 :               = gimple_build_assign (cond_reg, BIT_XOR_EXPR, oprnd0, oprnd1);
    5299           18 :             append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5300              : 
    5301              :             // x ^ y < 0
    5302           18 :             bool_cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
    5303           18 :             def_stmt = gimple_build_assign (bool_cond, LT_EXPR, cond_reg,
    5304              :                                             build_int_cst (itype, 0));
    5305           18 :             append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
    5306              :                                     itype);
    5307              : 
    5308              :             // x ^ y < 0 ? y : -y (mod)
    5309              :             // x ^ y < 0 ? -1 : 1 (div)
    5310           18 :             tree val1 = vect_recog_temp_ssa_var (itype, NULL);
    5311           18 :             def_stmt
    5312           36 :               = gimple_build_assign (val1, COND_EXPR, bool_cond,
    5313           27 :                                      build_int_cst (itype, is_mod ? d : -1),
    5314           18 :                                      build_int_cst (itype, is_mod ? -d : 1));
    5315           18 :             append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5316           18 :             int precision = TYPE_PRECISION (itype);
    5317           18 :             wide_int wmask = wi::mask (precision, false, precision);
    5318              : 
    5319              :             // abs (r) > (abs (y-1) >> 1) ? 0xffffffff : 0
    5320           18 :             tree val2 = vect_recog_temp_ssa_var (itype, NULL);
    5321           36 :             def_stmt = gimple_build_assign (val2, COND_EXPR, round_p,
    5322           18 :                                             wide_int_to_tree (itype, wmask),
    5323              :                                             build_int_cst (itype, 0));
    5324           18 :             append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5325              : 
    5326           18 :             tree fval = vect_recog_temp_ssa_var (itype, NULL);
    5327           18 :             def_stmt = gimple_build_assign (fval, BIT_AND_EXPR, val1, val2);
    5328           18 :             append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5329              : 
    5330           18 :             tree result = vect_recog_temp_ssa_var (itype, NULL);
    5331           27 :             return gimple_build_assign (result, PLUS_EXPR, is_mod ? r : q,
    5332              :                                         fval);
    5333           18 :           }
    5334              :         else
    5335              :           {
    5336              :             // r > (y-1 >> 1)
    5337           18 :             tree round_p = vect_recog_temp_ssa_var (boolean_type_node, NULL);
    5338           18 :             def_stmt = gimple_build_assign (round_p, GT_EXPR, r,
    5339           18 :                                             build_int_cst (itype, mid_d));
    5340           18 :             append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, mask_vectype,
    5341              :                                     itype);
    5342              : 
    5343              :             // (r > (y-1)>>1) ? -d : 1
    5344           18 :             tree val2 = vect_recog_temp_ssa_var (itype, NULL);
    5345           18 :             def_stmt
    5346           36 :               = gimple_build_assign (val2, COND_EXPR, round_p,
    5347           18 :                                      build_int_cst (itype, is_mod ? -d : 1),
    5348              :                                      build_int_cst (itype, 0));
    5349           18 :             append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5350              : 
    5351           18 :             tree result = vect_recog_temp_ssa_var (itype, NULL);
    5352           27 :             return gimple_build_assign (result, PLUS_EXPR, is_mod ? r : q,
    5353           18 :                                         val2);
    5354              :           }
    5355              :       }
    5356              :     default:
    5357              :       return NULL;
    5358              :     }
    5359              : }
    5360              : 
    5361              : /* Detect a signed division by a constant that wouldn't be
    5362              :    otherwise vectorized:
    5363              : 
    5364              :    type a_t, b_t;
    5365              : 
    5366              :    S1 a_t = b_t / N;
    5367              : 
    5368              :   where type 'type' is an integral type and N is a constant.
    5369              : 
    5370              :   Similarly handle modulo by a constant:
    5371              : 
    5372              :    S4 a_t = b_t % N;
    5373              : 
    5374              :   Input/Output:
    5375              : 
    5376              :   * STMT_VINFO: The stmt from which the pattern search begins,
    5377              :     i.e. the division stmt.  S1 is replaced by if N is a power
    5378              :     of two constant and type is signed:
    5379              :   S3  y_t = b_t < 0 ? N - 1 : 0;
    5380              :   S2  x_t = b_t + y_t;
    5381              :   S1' a_t = x_t >> log2 (N);
    5382              : 
    5383              :     S4 is replaced if N is a power of two constant and
    5384              :     type is signed by (where *_T temporaries have unsigned type):
    5385              :   S9  y_T = b_t < 0 ? -1U : 0U;
    5386              :   S8  z_T = y_T >> (sizeof (type_t) * CHAR_BIT - log2 (N));
    5387              :   S7  z_t = (type) z_T;
    5388              :   S6  w_t = b_t + z_t;
    5389              :   S5  x_t = w_t & (N - 1);
    5390              :   S4' a_t = x_t - z_t;
    5391              : 
    5392              :   Output:
    5393              : 
    5394              :   * TYPE_OUT: The type of the output of this pattern.
    5395              : 
    5396              :   * Return value: A new stmt that will be used to replace the division
    5397              :     S1 or modulo S4 stmt.  */
    5398              : 
    5399              : static gimple *
    5400     33061266 : vect_recog_divmod_pattern (vec_info *vinfo,
    5401              :                            stmt_vec_info stmt_vinfo, tree *type_out)
    5402              : {
    5403     33061266 :   gimple *last_stmt = stmt_vinfo->stmt;
    5404     33061266 :   tree oprnd0, oprnd1, vectype, itype, cond;
    5405     33061266 :   gimple *pattern_stmt = NULL;
    5406     33061266 :   gimple *def_stmt = NULL;
    5407     33061266 :   enum tree_code rhs_code;
    5408     33061266 :   optab optab;
    5409     33061266 :   tree q, cst;
    5410     33061266 :   int prec;
    5411              : 
    5412     33061266 :   if (!is_gimple_assign (last_stmt)
    5413              :       /* The pattern will disrupt the reduction chain with multiple uses.  */
    5414     33061266 :       || vect_is_reduction (stmt_vinfo))
    5415              :     return NULL;
    5416              : 
    5417     23043376 :   rhs_code = gimple_assign_rhs_code (last_stmt);
    5418     23043376 :   switch (rhs_code)
    5419              :     {
    5420       294974 :     case TRUNC_DIV_EXPR:
    5421       294974 :     case EXACT_DIV_EXPR:
    5422       294974 :     case TRUNC_MOD_EXPR:
    5423       294974 :     case FLOOR_MOD_EXPR:
    5424       294974 :     case FLOOR_DIV_EXPR:
    5425       294974 :     case CEIL_MOD_EXPR:
    5426       294974 :     case CEIL_DIV_EXPR:
    5427       294974 :     case ROUND_MOD_EXPR:
    5428       294974 :     case ROUND_DIV_EXPR:
    5429       294974 :       break;
    5430              :     default:
    5431              :       return NULL;
    5432              :     }
    5433              : 
    5434       294974 :   oprnd0 = gimple_assign_rhs1 (last_stmt);
    5435       294974 :   oprnd1 = gimple_assign_rhs2 (last_stmt);
    5436       294974 :   itype = TREE_TYPE (oprnd0);
    5437       294974 :   if (TREE_CODE (oprnd0) != SSA_NAME
    5438       276241 :       || TREE_CODE (oprnd1) != INTEGER_CST
    5439       162859 :       || TREE_CODE (itype) != INTEGER_TYPE
    5440       457833 :       || !type_has_mode_precision_p (itype))
    5441              :     return NULL;
    5442              : 
    5443       162845 :   scalar_int_mode itype_mode = SCALAR_INT_TYPE_MODE (itype);
    5444       162845 :   vectype = get_vectype_for_scalar_type (vinfo, itype);
    5445       162845 :   if (vectype == NULL_TREE)
    5446              :     return NULL;
    5447              : 
    5448       131175 :   if (optimize_bb_for_size_p (gimple_bb (last_stmt)))
    5449              :     {
    5450              :       /* If the target can handle vectorized division or modulo natively,
    5451              :          don't attempt to optimize this, since native division is likely
    5452              :          to give smaller code.  */
    5453         2264 :       optab = optab_for_tree_code (rhs_code, vectype, optab_default);
    5454         2264 :       if (optab != unknown_optab
    5455         2264 :           && can_implement_p (optab, TYPE_MODE (vectype)))
    5456              :         return NULL;
    5457              :     }
    5458              : 
    5459       131175 :   prec = TYPE_PRECISION (itype);
    5460              : 
    5461       262350 :   bool is_flclrd_moddiv_p
    5462       131175 :     = rhs_code == FLOOR_MOD_EXPR || rhs_code == FLOOR_DIV_EXPR
    5463              :     || rhs_code == CEIL_MOD_EXPR || rhs_code == CEIL_DIV_EXPR
    5464       130559 :     || rhs_code == ROUND_MOD_EXPR || rhs_code == ROUND_DIV_EXPR;
    5465       131175 :   if (integer_pow2p (oprnd1))
    5466              :     {
    5467        77659 :       if ((TYPE_UNSIGNED (itype)
    5468           36 :            && (rhs_code == FLOOR_MOD_EXPR || rhs_code == FLOOR_DIV_EXPR))
    5469        77695 :           || tree_int_cst_sgn (oprnd1) != 1)
    5470              :         return NULL;
    5471              : 
    5472              :       /* Pattern detected.  */
    5473        77659 :       vect_pattern_detected ("vect_recog_divmod_pattern", last_stmt);
    5474              : 
    5475        77659 :       *type_out = vectype;
    5476              : 
    5477              :       /* OPRND1 is a positive power of two here and the dividend of an
    5478              :          exact division is a multiple of it, so the right shift is exact
    5479              :          also for negative dividends and needs no rounding fixup.  */
    5480        77659 :       if (rhs_code == EXACT_DIV_EXPR)
    5481              :         {
    5482        58355 :           tree shift = build_int_cst (itype, tree_log2 (oprnd1));
    5483        58355 :           pattern_stmt
    5484        58355 :             = gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL),
    5485              :                                    RSHIFT_EXPR, oprnd0, shift);
    5486        58355 :           gimple_set_location (pattern_stmt, gimple_location (last_stmt));
    5487        58355 :           return pattern_stmt;
    5488              :         }
    5489              : 
    5490              :       /* Check if the target supports this internal function.  */
    5491        19304 :       internal_fn ifn = IFN_DIV_POW2;
    5492        19304 :       if (!TYPE_UNSIGNED (itype)
    5493        19304 :           && direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
    5494              :         {
    5495            0 :           tree shift = build_int_cst (itype, tree_log2 (oprnd1));
    5496              : 
    5497            0 :           tree var_div = vect_recog_temp_ssa_var (itype, NULL);
    5498            0 :           gimple *div_stmt = gimple_build_call_internal (ifn, 2, oprnd0, shift);
    5499            0 :           gimple_call_set_lhs (div_stmt, var_div);
    5500            0 :           if (rhs_code == TRUNC_MOD_EXPR || is_flclrd_moddiv_p)
    5501              :             {
    5502            0 :               append_pattern_def_seq (vinfo, stmt_vinfo, div_stmt);
    5503            0 :               tree t1 = vect_recog_temp_ssa_var (itype, NULL);
    5504            0 :               def_stmt
    5505            0 :                 = gimple_build_assign (t1, LSHIFT_EXPR, var_div, shift);
    5506            0 :               append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5507            0 :               tree r = vect_recog_temp_ssa_var (itype, NULL);
    5508            0 :               pattern_stmt = gimple_build_assign (r, MINUS_EXPR, oprnd0, t1);
    5509            0 :               if (is_flclrd_moddiv_p)
    5510              :                 {
    5511            0 :                   append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
    5512            0 :                   pattern_stmt
    5513            0 :                     = add_code_for_floorceilround_divmod (vectype, vinfo,
    5514              :                                                           stmt_vinfo, rhs_code,
    5515              :                                                           var_div, r, oprnd0,
    5516              :                                                           oprnd1, itype);
    5517            0 :                   if (pattern_stmt == NULL)
    5518              :                     return NULL;
    5519              :                 }
    5520              :             }
    5521              :           else
    5522              :             pattern_stmt = div_stmt;
    5523            0 :           gimple_set_location (pattern_stmt, gimple_location (last_stmt));
    5524              : 
    5525            0 :           return pattern_stmt;
    5526              :         }
    5527              : 
    5528        19304 :       cond = vect_recog_temp_ssa_var (boolean_type_node, NULL);
    5529        19304 :       def_stmt = gimple_build_assign (cond, LT_EXPR, oprnd0,
    5530              :                                       build_int_cst (itype, 0));
    5531        19304 :       append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt,
    5532              :                               truth_type_for (vectype), itype);
    5533        19304 :       tree div_result = NULL_TREE;
    5534        19304 :       if (rhs_code == TRUNC_DIV_EXPR
    5535        19304 :           || rhs_code == FLOOR_DIV_EXPR
    5536         2518 :           || rhs_code == CEIL_DIV_EXPR
    5537         2518 :           || rhs_code == ROUND_DIV_EXPR)
    5538              :         {
    5539        16829 :           tree var = vect_recog_temp_ssa_var (itype, NULL);
    5540        16829 :           tree shift;
    5541        16829 :           def_stmt
    5542        16829 :             = gimple_build_assign (var, COND_EXPR, cond,
    5543              :                                    fold_build2 (MINUS_EXPR, itype, oprnd1,
    5544              :                                                 build_int_cst (itype, 1)),
    5545              :                                    build_int_cst (itype, 0));
    5546        16829 :           append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5547        16829 :           var = vect_recog_temp_ssa_var (itype, NULL);
    5548        16829 :           def_stmt
    5549        16829 :             = gimple_build_assign (var, PLUS_EXPR, oprnd0,
    5550              :                                    gimple_assign_lhs (def_stmt));
    5551        16829 :           append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5552              : 
    5553        16829 :           shift = build_int_cst (itype, tree_log2 (oprnd1));
    5554        16829 :           div_result = vect_recog_temp_ssa_var (itype, NULL);
    5555        16829 :           pattern_stmt
    5556        16829 :             = gimple_build_assign (div_result, RSHIFT_EXPR, var, shift);
    5557              :         }
    5558        19304 :       if (rhs_code == TRUNC_MOD_EXPR || is_flclrd_moddiv_p)
    5559              :         {
    5560         2653 :           if (rhs_code == FLOOR_DIV_EXPR
    5561              :               || rhs_code == CEIL_DIV_EXPR
    5562         2653 :               || rhs_code == ROUND_DIV_EXPR)
    5563          178 :             append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
    5564              : 
    5565         2653 :           tree signmask;
    5566         2653 :           if (compare_tree_int (oprnd1, 2) == 0)
    5567              :             {
    5568         1343 :               signmask = vect_recog_temp_ssa_var (itype, NULL);
    5569         1343 :               def_stmt = gimple_build_assign (signmask, COND_EXPR, cond,
    5570              :                                               build_int_cst (itype, 1),
    5571              :                                               build_int_cst (itype, 0));
    5572         1343 :               append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5573              :             }
    5574              :           else
    5575              :             {
    5576         1310 :               tree utype
    5577         1310 :                 = build_nonstandard_integer_type (prec, 1);
    5578         1310 :               tree vecutype = get_vectype_for_scalar_type (vinfo, utype);
    5579         1310 :               tree shift
    5580         1310 :                 = build_int_cst (utype, GET_MODE_BITSIZE (itype_mode)
    5581         1310 :                                         - tree_log2 (oprnd1));
    5582         1310 :               tree var = vect_recog_temp_ssa_var (utype, NULL);
    5583              : 
    5584         1310 :               def_stmt = gimple_build_assign (var, COND_EXPR, cond,
    5585              :                                               build_int_cst (utype, -1),
    5586              :                                               build_int_cst (utype, 0));
    5587         1310 :               append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecutype);
    5588         1310 :               var = vect_recog_temp_ssa_var (utype, NULL);
    5589         1310 :               def_stmt = gimple_build_assign (var, RSHIFT_EXPR,
    5590              :                                               gimple_assign_lhs (def_stmt),
    5591              :                                               shift);
    5592         1310 :               append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vecutype);
    5593         1310 :               signmask = vect_recog_temp_ssa_var (itype, NULL);
    5594         1310 :               def_stmt
    5595         1310 :                 = gimple_build_assign (signmask, NOP_EXPR, var);
    5596         1310 :               append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5597              :             }
    5598         2653 :           def_stmt
    5599         2653 :             = gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL),
    5600              :                                    PLUS_EXPR, oprnd0, signmask);
    5601         2653 :           append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5602         2653 :           def_stmt
    5603         2653 :             = gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL),
    5604              :                                    BIT_AND_EXPR, gimple_assign_lhs (def_stmt),
    5605              :                                    fold_build2 (MINUS_EXPR, itype, oprnd1,
    5606              :                                                 build_int_cst (itype, 1)));
    5607         2653 :           append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5608              : 
    5609         2653 :           tree r = vect_recog_temp_ssa_var (itype, NULL);
    5610         2653 :           pattern_stmt
    5611         2653 :             = gimple_build_assign (r, MINUS_EXPR, gimple_assign_lhs (def_stmt),
    5612              :                                    signmask);
    5613         2653 :           if (is_flclrd_moddiv_p)
    5614              :             {
    5615          272 :               append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
    5616          272 :               pattern_stmt
    5617          272 :                 = add_code_for_floorceilround_divmod (vectype, vinfo,
    5618              :                                                       stmt_vinfo, rhs_code,
    5619              :                                                       div_result, r, oprnd0,
    5620              :                                                       oprnd1, itype);
    5621          272 :               if (pattern_stmt == NULL)
    5622              :                 return NULL;
    5623              :             }
    5624              :         }
    5625              : 
    5626        19146 :       return pattern_stmt;
    5627              :     }
    5628              : 
    5629        53516 :   if ((cst = uniform_integer_cst_p (oprnd1))
    5630        53516 :       && TYPE_UNSIGNED (itype)
    5631              :       && rhs_code == TRUNC_DIV_EXPR
    5632        30758 :       && vectype
    5633        71786 :       && targetm.vectorize.preferred_div_as_shifts_over_mult (vectype))
    5634              :     {
    5635              :       /* We can use the relationship:
    5636              : 
    5637              :            x // N == ((x+N+2) // (N+1) + x) // (N+1)  for 0 <= x < N(N+3)
    5638              : 
    5639              :          to optimize cases where N+1 is a power of 2, and where // (N+1)
    5640              :          is therefore a shift right.  When operating in modes that are
    5641              :          multiples of a byte in size, there are two cases:
    5642              : 
    5643              :          (1) N(N+3) is not representable, in which case the question
    5644              :              becomes whether the replacement expression overflows.
    5645              :              It is enough to test that x+N+2 does not overflow,
    5646              :              i.e. that x < MAX-(N+1).
    5647              : 
    5648              :          (2) N(N+3) is representable, in which case it is the (only)
    5649              :              bound that we need to check.
    5650              : 
    5651              :          ??? For now we just handle the case where // (N+1) is a shift
    5652              :          right by half the precision, since some architectures can
    5653              :          optimize the associated addition and shift combinations
    5654              :          into single instructions.  */
    5655              : 
    5656         7553 :       auto wcst = wi::to_wide (cst);
    5657         7553 :       int pow = wi::exact_log2 (wcst + 1);
    5658         7553 :       if (pow == prec / 2)
    5659              :         {
    5660              :           /* Check that no overflow will occur.  If we don't have range
    5661              :              information we can't perform the optimization.  */
    5662              : 
    5663          503 :           int_range_max r;
    5664         1006 :           if (get_range_query (cfun)->range_of_expr (r, oprnd0, last_stmt)
    5665          503 :               && !r.undefined_p ())
    5666              :             {
    5667          503 :               wide_int max = r.upper_bound ();
    5668          503 :               wide_int one = wi::shwi (1, prec);
    5669          503 :               wide_int adder = wi::add (one, wi::lshift (one, pow));
    5670          503 :               wi::overflow_type ovf;
    5671          503 :               wi::add (max, adder, UNSIGNED, &ovf);
    5672          503 :               if (ovf == wi::OVF_NONE)
    5673              :                 {
    5674          321 :                   *type_out = vectype;
    5675          321 :                   tree tadder = wide_int_to_tree (itype, adder);
    5676          321 :                   tree rshift = wide_int_to_tree (itype, pow);
    5677              : 
    5678          321 :                   tree new_lhs1 = vect_recog_temp_ssa_var (itype, NULL);
    5679          321 :                   gassign *patt1
    5680          321 :                     = gimple_build_assign (new_lhs1, PLUS_EXPR, oprnd0, tadder);
    5681          321 :                   append_pattern_def_seq (vinfo, stmt_vinfo, patt1, vectype);
    5682              : 
    5683          321 :                   tree new_lhs2 = vect_recog_temp_ssa_var (itype, NULL);
    5684          321 :                   patt1 = gimple_build_assign (new_lhs2, RSHIFT_EXPR, new_lhs1,
    5685              :                                                rshift);
    5686          321 :                   append_pattern_def_seq (vinfo, stmt_vinfo, patt1, vectype);
    5687              : 
    5688          321 :                   tree new_lhs3 = vect_recog_temp_ssa_var (itype, NULL);
    5689          321 :                   patt1 = gimple_build_assign (new_lhs3, PLUS_EXPR, new_lhs2,
    5690              :                                                oprnd0);
    5691          321 :                   append_pattern_def_seq (vinfo, stmt_vinfo, patt1, vectype);
    5692              : 
    5693          321 :                   tree new_lhs4 = vect_recog_temp_ssa_var (itype, NULL);
    5694          321 :                   pattern_stmt = gimple_build_assign (new_lhs4, RSHIFT_EXPR,
    5695              :                                                       new_lhs3, rshift);
    5696              : 
    5697          321 :                   return pattern_stmt;
    5698              :                 }
    5699          503 :             }
    5700          503 :         }
    5701              :     }
    5702              : 
    5703        53195 :   if (prec > HOST_BITS_PER_WIDE_INT
    5704        53195 :       || integer_zerop (oprnd1))
    5705              :     return NULL;
    5706              : 
    5707        52898 :   if (!can_mult_highpart_p (TYPE_MODE (vectype), TYPE_UNSIGNED (itype)))
    5708              :     return NULL;
    5709              : 
    5710        22415 :   if (TYPE_UNSIGNED (itype))
    5711              :     {
    5712        17523 :       unsigned HOST_WIDE_INT mh, ml;
    5713        17523 :       int pre_shift, post_shift;
    5714        17523 :       unsigned HOST_WIDE_INT d = (TREE_INT_CST_LOW (oprnd1)
    5715        17523 :                                   & GET_MODE_MASK (itype_mode));
    5716        17523 :       tree t1, t2, t3, t4;
    5717              : 
    5718        17523 :       if (d >= (HOST_WIDE_INT_1U << (prec - 1)))
    5719              :         /* FIXME: Can transform this into oprnd0 >= oprnd1 ? 1 : 0.  */
    5720           45 :         return NULL;
    5721              : 
    5722              :       /* Find a suitable multiplier and right shift count instead of
    5723              :          directly dividing by D.  */
    5724        17478 :       mh = choose_multiplier (d, prec, prec, &ml, &post_shift);
    5725              : 
    5726              :       /* If the suggested multiplier is more than PREC bits, we can do better
    5727              :          for even divisors, using an initial right shift.  */
    5728        17478 :       if (mh != 0 && (d & 1) == 0)
    5729              :         {
    5730         1551 :           pre_shift = ctz_or_zero (d);
    5731         1551 :           mh = choose_multiplier (d >> pre_shift, prec, prec - pre_shift,
    5732              :                                   &ml, &post_shift);
    5733         1551 :           gcc_assert (!mh);
    5734              :         }
    5735              :       else
    5736              :         pre_shift = 0;
    5737              : 
    5738         1219 :       if (mh != 0)
    5739              :         {
    5740         1219 :           if (post_shift - 1 >= prec)
    5741              :             return NULL;
    5742              : 
    5743              :           /* t1 = oprnd0 h* ml;
    5744              :              t2 = oprnd0 - t1;
    5745              :              t3 = t2 >> 1;
    5746              :              t4 = t1 + t3;
    5747              :              q = t4 >> (post_shift - 1);  */
    5748         1219 :           t1 = vect_recog_temp_ssa_var (itype, NULL);
    5749         1219 :           def_stmt = gimple_build_assign (t1, MULT_HIGHPART_EXPR, oprnd0,
    5750         1219 :                                           build_int_cst (itype, ml));
    5751         1219 :           append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5752              : 
    5753         1219 :           t2 = vect_recog_temp_ssa_var (itype, NULL);
    5754         1219 :           def_stmt
    5755         1219 :             = gimple_build_assign (t2, MINUS_EXPR, oprnd0, t1);
    5756         1219 :           append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5757              : 
    5758         1219 :           t3 = vect_recog_temp_ssa_var (itype, NULL);
    5759         1219 :           def_stmt
    5760         1219 :             = gimple_build_assign (t3, RSHIFT_EXPR, t2, integer_one_node);
    5761         1219 :           append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5762              : 
    5763         1219 :           t4 = vect_recog_temp_ssa_var (itype, NULL);
    5764         1219 :           def_stmt
    5765         1219 :             = gimple_build_assign (t4, PLUS_EXPR, t1, t3);
    5766              : 
    5767         1219 :           if (post_shift != 1)
    5768              :             {
    5769         1219 :               append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5770              : 
    5771         1219 :               q = vect_recog_temp_ssa_var (itype, NULL);
    5772         1219 :               pattern_stmt
    5773         1219 :                 = gimple_build_assign (q, RSHIFT_EXPR, t4,
    5774         1219 :                                        build_int_cst (itype, post_shift - 1));
    5775              :             }
    5776              :           else
    5777              :             {
    5778              :               q = t4;
    5779              :               pattern_stmt = def_stmt;
    5780              :             }
    5781              :         }
    5782              :       else
    5783              :         {
    5784        16259 :           if (pre_shift >= prec || post_shift >= prec)
    5785              :             return NULL;
    5786              : 
    5787              :           /* t1 = oprnd0 >> pre_shift;
    5788              :              t2 = t1 h* ml;
    5789              :              q = t2 >> post_shift;  */
    5790        16259 :           if (pre_shift)
    5791              :             {
    5792         1551 :               t1 = vect_recog_temp_ssa_var (itype, NULL);
    5793         1551 :               def_stmt
    5794         1551 :                 = gimple_build_assign (t1, RSHIFT_EXPR, oprnd0,
    5795         1551 :                                        build_int_cst (integer_type_node, pre_shift));
    5796         1551 :               append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5797              :             }
    5798              :           else
    5799              :             t1 = oprnd0;
    5800              : 
    5801        16259 :           t2 = vect_recog_temp_ssa_var (itype, NULL);
    5802        16259 :           def_stmt = gimple_build_assign (t2, MULT_HIGHPART_EXPR, t1,
    5803        16259 :                                           build_int_cst (itype, ml));
    5804              : 
    5805        16259 :           if (post_shift)
    5806              :             {
    5807        16213 :               append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5808              : 
    5809        16213 :               q = vect_recog_temp_ssa_var (itype, NULL);
    5810        16213 :               def_stmt
    5811        16213 :                 = gimple_build_assign (q, RSHIFT_EXPR, t2,
    5812        16213 :                                        build_int_cst (itype, post_shift));
    5813              :             }
    5814              :           else
    5815              :             q = t2;
    5816              : 
    5817              :           pattern_stmt = def_stmt;
    5818              :         }
    5819              :     }
    5820              :   else
    5821              :     {
    5822         4892 :       unsigned HOST_WIDE_INT ml;
    5823         4892 :       int post_shift;
    5824         4892 :       HOST_WIDE_INT d = TREE_INT_CST_LOW (oprnd1);
    5825         4892 :       unsigned HOST_WIDE_INT abs_d;
    5826         4892 :       bool add = false;
    5827         4892 :       tree t1, t2, t3, t4;
    5828              : 
    5829              :       /* Give up for -1.  */
    5830         4892 :       if (d == -1)
    5831            0 :         return NULL;
    5832              : 
    5833              :       /* Since d might be INT_MIN, we have to cast to
    5834              :          unsigned HOST_WIDE_INT before negating to avoid
    5835              :          undefined signed overflow.  */
    5836         4892 :       abs_d = (d >= 0
    5837         4892 :                ? (unsigned HOST_WIDE_INT) d
    5838              :                : - (unsigned HOST_WIDE_INT) d);
    5839              : 
    5840              :       /* n rem d = n rem -d */
    5841         4892 :       if (rhs_code == TRUNC_MOD_EXPR && d < 0)
    5842              :         {
    5843            0 :           d = abs_d;
    5844            0 :           oprnd1 = build_int_cst (itype, abs_d);
    5845              :         }
    5846         4892 :       if (HOST_BITS_PER_WIDE_INT >= prec
    5847         4892 :           && abs_d == HOST_WIDE_INT_1U << (prec - 1))
    5848              :         /* This case is not handled correctly below.  */
    5849              :         return NULL;
    5850              : 
    5851         4892 :       choose_multiplier (abs_d, prec, prec - 1, &ml, &post_shift);
    5852         4892 :       if (ml >= HOST_WIDE_INT_1U << (prec - 1))
    5853              :         {
    5854         1624 :           add = true;
    5855         1624 :           ml |= HOST_WIDE_INT_M1U << (prec - 1);
    5856              :         }
    5857         4892 :       if (post_shift >= prec)
    5858              :         return NULL;
    5859              : 
    5860              :       /* t1 = oprnd0 h* ml;  */
    5861         4892 :       t1 = vect_recog_temp_ssa_var (itype, NULL);
    5862         4892 :       def_stmt = gimple_build_assign (t1, MULT_HIGHPART_EXPR, oprnd0,
    5863         4892 :                                       build_int_cst (itype, ml));
    5864              : 
    5865         4892 :       if (add)
    5866              :         {
    5867              :           /* t2 = t1 + oprnd0;  */
    5868         1624 :           append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5869         1624 :           t2 = vect_recog_temp_ssa_var (itype, NULL);
    5870         1624 :           def_stmt = gimple_build_assign (t2, PLUS_EXPR, t1, oprnd0);
    5871              :         }
    5872              :       else
    5873              :         t2 = t1;
    5874              : 
    5875         4892 :       if (post_shift)
    5876              :         {
    5877              :           /* t3 = t2 >> post_shift;  */
    5878         4059 :           append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5879         4059 :           t3 = vect_recog_temp_ssa_var (itype, NULL);
    5880         4059 :           def_stmt = gimple_build_assign (t3, RSHIFT_EXPR, t2,
    5881         4059 :                                           build_int_cst (itype, post_shift));
    5882              :         }
    5883              :       else
    5884              :         t3 = t2;
    5885              : 
    5886         4892 :       int msb = 1;
    5887         4892 :       int_range_max r;
    5888         9784 :       get_range_query (cfun)->range_of_expr (r, oprnd0);
    5889         4892 :       if (!r.varying_p () && !r.undefined_p ())
    5890              :         {
    5891         3040 :           if (!wi::neg_p (r.lower_bound (), TYPE_SIGN (itype)))
    5892              :             msb = 0;
    5893          736 :           else if (wi::neg_p (r.upper_bound (), TYPE_SIGN (itype)))
    5894              :             msb = -1;
    5895              :         }
    5896              : 
    5897         2304 :       if (msb == 0 && d >= 0)
    5898              :         {
    5899              :           /* q = t3;  */
    5900              :           q = t3;
    5901              :           pattern_stmt = def_stmt;
    5902              :         }
    5903              :       else
    5904              :         {
    5905              :           /* t4 = oprnd0 >> (prec - 1);
    5906              :              or if we know from VRP that oprnd0 >= 0
    5907              :              t4 = 0;
    5908              :              or if we know from VRP that oprnd0 < 0
    5909              :              t4 = -1;  */
    5910         2656 :           append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5911         2656 :           t4 = vect_recog_temp_ssa_var (itype, NULL);
    5912         2656 :           if (msb != 1)
    5913           76 :             def_stmt = gimple_build_assign (t4, INTEGER_CST,
    5914           76 :                                             build_int_cst (itype, msb));
    5915              :           else
    5916         2580 :             def_stmt = gimple_build_assign (t4, RSHIFT_EXPR, oprnd0,
    5917         2580 :                                             build_int_cst (itype, prec - 1));
    5918         2656 :           append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5919              : 
    5920              :           /* q = t3 - t4;  or q = t4 - t3;  */
    5921         2656 :           q = vect_recog_temp_ssa_var (itype, NULL);
    5922         5128 :           pattern_stmt = gimple_build_assign (q, MINUS_EXPR, d < 0 ? t4 : t3,
    5923              :                                               d < 0 ? t3 : t4);
    5924              :         }
    5925         4892 :     }
    5926              : 
    5927        22370 :   if (rhs_code == TRUNC_MOD_EXPR || is_flclrd_moddiv_p)
    5928              :     {
    5929         9980 :       tree r, t1;
    5930              : 
    5931              :       /* We divided.  Now finish by:
    5932              :          t1 = q * oprnd1;
    5933              :          r = oprnd0 - t1;  */
    5934         9980 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
    5935              : 
    5936         9980 :       t1 = vect_recog_temp_ssa_var (itype, NULL);
    5937         9980 :       def_stmt = gimple_build_assign (t1, MULT_EXPR, q, oprnd1);
    5938         9980 :       append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt);
    5939              : 
    5940         9980 :       r = vect_recog_temp_ssa_var (itype, NULL);
    5941         9980 :       pattern_stmt = gimple_build_assign (r, MINUS_EXPR, oprnd0, t1);
    5942              : 
    5943         9980 :       if (is_flclrd_moddiv_p)
    5944              :         {
    5945          146 :         append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt);
    5946          146 :         pattern_stmt
    5947          146 :           = add_code_for_floorceilround_divmod (vectype, vinfo, stmt_vinfo,
    5948              :                                                 rhs_code, q, r, oprnd0, oprnd1,
    5949              :                                                 itype);
    5950          146 :         if (pattern_stmt == NULL)
    5951              :           return NULL;
    5952              :         }
    5953              :     }
    5954              : 
    5955              :   /* Pattern detected.  */
    5956        22370 :   vect_pattern_detected ("vect_recog_divmod_pattern", last_stmt);
    5957              : 
    5958        22370 :   *type_out = vectype;
    5959        22370 :   return pattern_stmt;
    5960              : }
    5961              : 
    5962              : /* Detects pattern with a modulo operation (S1) where both arguments
    5963              :    are variables of integral type.
    5964              :    The statement is replaced by division, multiplication, and subtraction.
    5965              :    The last statement (S4) is returned.
    5966              : 
    5967              :    Example:
    5968              :    S1 c_t = a_t % b_t;
    5969              : 
    5970              :    is replaced by
    5971              :    S2 x_t = a_t / b_t;
    5972              :    S3 y_t = x_t * b_t;
    5973              :    S4 z_t = a_t - y_t;  */
    5974              : 
    5975              : static gimple *
    5976     33083027 : vect_recog_mod_var_pattern (vec_info *vinfo,
    5977              :                             stmt_vec_info stmt_vinfo, tree *type_out)
    5978              : {
    5979     33083027 :   gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
    5980     33083027 :   tree oprnd0, oprnd1, vectype, itype;
    5981     33083027 :   gimple *pattern_stmt, *def_stmt;
    5982     33083027 :   enum tree_code rhs_code;
    5983              : 
    5984     33083027 :   if (!is_gimple_assign (last_stmt) || vect_is_reduction (stmt_vinfo))
    5985              :     return NULL;
    5986              : 
    5987     23065137 :   rhs_code = gimple_assign_rhs_code (last_stmt);
    5988     23065137 :   if (rhs_code != TRUNC_MOD_EXPR)
    5989              :     return NULL;
    5990              : 
    5991        76804 :   oprnd0 = gimple_assign_rhs1 (last_stmt);
    5992        76804 :   oprnd1 = gimple_assign_rhs2 (last_stmt);
    5993        76804 :   itype = TREE_TYPE (oprnd0);
    5994        76804 :   if (TREE_CODE (oprnd0) != SSA_NAME
    5995        67899 :       || TREE_CODE (oprnd1) != SSA_NAME
    5996        54362 :       || TREE_CODE (itype) != INTEGER_TYPE)
    5997              :     return NULL;
    5998              : 
    5999        54190 :   vectype = get_vectype_for_scalar_type (vinfo, itype);
    6000              : 
    6001        54190 :   if (!vectype
    6002        45374 :       || target_has_vecop_for_code (TRUNC_MOD_EXPR, vectype)
    6003        45374 :       || !target_has_vecop_for_code (TRUNC_DIV_EXPR, vectype)
    6004            0 :       || !target_has_vecop_for_code (MULT_EXPR, vectype)
    6005        54190 :       || !target_has_vecop_for_code (MINUS_EXPR, vectype))
    6006              :     return NULL;
    6007              : 
    6008            0 :   tree q, tmp, r;
    6009            0 :   q = vect_recog_temp_ssa_var (itype, NULL);
    6010            0 :   def_stmt = gimple_build_assign (q, TRUNC_DIV_EXPR, oprnd0, oprnd1);
    6011            0 :   append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vectype);
    6012              : 
    6013            0 :   tmp = vect_recog_temp_ssa_var (itype, NULL);
    6014            0 :   def_stmt = gimple_build_assign (tmp, MULT_EXPR, q, oprnd1);
    6015            0 :   append_pattern_def_seq (vinfo, stmt_vinfo, def_stmt, vectype);
    6016              : 
    6017            0 :   r = vect_recog_temp_ssa_var (itype, NULL);
    6018            0 :   pattern_stmt = gimple_build_assign (r, MINUS_EXPR, oprnd0, tmp);
    6019              : 
    6020              :   /* Pattern detected.  */
    6021            0 :   *type_out = vectype;
    6022            0 :   vect_pattern_detected ("vect_recog_mod_var_pattern", last_stmt);
    6023              : 
    6024            0 :   return pattern_stmt;
    6025              : }
    6026              : 
    6027              : 
    6028              : /* Return the proper type for converting bool VAR into
    6029              :    an integer value or NULL_TREE if no such type exists.
    6030              :    The type is chosen so that the converted value has the
    6031              :    same number of elements as VAR's vector type.  */
    6032              : 
    6033              : static tree
    6034      4518138 : integer_type_for_mask (tree var, vec_info *vinfo, vect_def_type *dt = nullptr)
    6035              : {
    6036      4518138 :   if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (var)))
    6037              :     return NULL_TREE;
    6038              : 
    6039      2041070 :   stmt_vec_info def_stmt_info = vinfo->lookup_def (var);
    6040      2041070 :   if (dt)
    6041              :     {
    6042       383440 :       if (!def_stmt_info)
    6043         3487 :         *dt = vect_external_def;
    6044              :       else
    6045       379953 :         *dt = STMT_VINFO_DEF_TYPE (def_stmt_info);
    6046              :     }
    6047       383440 :   if (!def_stmt_info
    6048      1943450 :       || STMT_VINFO_DEF_TYPE (def_stmt_info) == vect_external_def
    6049      3601080 :       || !vect_use_mask_type_p (def_stmt_info))
    6050              :     return NULL_TREE;
    6051              : 
    6052      1204595 :   return build_nonstandard_integer_type (def_stmt_info->mask_precision, 1);
    6053              : }
    6054              : 
    6055              : /* Function vect_recog_gcond_pattern
    6056              : 
    6057              :    Try to find pattern like following:
    6058              : 
    6059              :      if (a op b)
    6060              : 
    6061              :    where operator 'op' is not != and convert it to an adjusted boolean pattern
    6062              : 
    6063              :      mask = a op b
    6064              :      if (mask != 0)
    6065              : 
    6066              :    and set the mask type on MASK.
    6067              : 
    6068              :    Input:
    6069              : 
    6070              :    * STMT_VINFO: The stmt at the end from which the pattern
    6071              :                  search begins, i.e. cast of a bool to
    6072              :                  an integer type.
    6073              : 
    6074              :    Output:
    6075              : 
    6076              :    * TYPE_OUT: The type of the output of this pattern.
    6077              : 
    6078              :    * Return value: A new stmt that will be used to replace the pattern.  */
    6079              : 
    6080              : static gimple *
    6081     33156906 : vect_recog_gcond_pattern (vec_info *vinfo,
    6082              :                          stmt_vec_info stmt_vinfo, tree *type_out)
    6083              : {
    6084              :   /* Currently we only support this for loop vectorization and when multiple
    6085              :      exits.  */
    6086     33156906 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    6087      4685969 :   if (!loop_vinfo || !LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
    6088              :     return NULL;
    6089              : 
    6090      1747803 :   gimple *last_stmt = STMT_VINFO_STMT (stmt_vinfo);
    6091      1747803 :   gcond* cond = NULL;
    6092      1747803 :   if (!(cond = dyn_cast <gcond *> (last_stmt)))
    6093              :     return NULL;
    6094              : 
    6095       411921 :   auto lhs = gimple_cond_lhs (cond);
    6096       411921 :   auto rhs = gimple_cond_rhs (cond);
    6097       411921 :   auto code = gimple_cond_code (cond);
    6098              : 
    6099       411921 :   tree scalar_type = TREE_TYPE (lhs);
    6100       411921 :   if (VECTOR_TYPE_P (scalar_type))
    6101              :     return NULL;
    6102              : 
    6103              :   /* If the input is a boolean then try to figure out the precision that the
    6104              :      vector type should use.  We cannot use the scalar precision as this would
    6105              :      later mismatch.  This is similar to what recog_bool does.  */
    6106       411921 :   if (VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type))
    6107              :     {
    6108        10433 :       if (tree stype = integer_type_for_mask (lhs, vinfo))
    6109       411921 :         scalar_type = stype;
    6110              :     }
    6111              : 
    6112       411921 :   tree vectype = get_mask_type_for_scalar_type (vinfo, scalar_type);
    6113       411921 :   if (vectype == NULL_TREE)
    6114              :     return NULL;
    6115              : 
    6116       390102 :   tree new_lhs = vect_recog_temp_ssa_var (boolean_type_node, NULL);
    6117       390102 :   gimple *new_stmt = gimple_build_assign (new_lhs, code, lhs, rhs);
    6118       390102 :   append_pattern_def_seq (vinfo, stmt_vinfo, new_stmt, vectype, scalar_type);
    6119              : 
    6120       390102 :   gimple *pattern_stmt
    6121       390102 :     = gimple_build_cond (NE_EXPR, new_lhs,
    6122       390102 :                          build_int_cst (TREE_TYPE (new_lhs), 0),
    6123              :                          NULL_TREE, NULL_TREE);
    6124       390102 :   *type_out = vectype;
    6125       390102 :   vect_pattern_detected ("vect_recog_gcond_pattern", last_stmt);
    6126       390102 :   return pattern_stmt;
    6127              : }
    6128              : 
    6129              : 
    6130              : /* A helper for vect_recog_mask_conversion_pattern.  Build
    6131              :    conversion of MASK to a type suitable for masking VECTYPE.
    6132              :    Built statement gets required vectype and is appended to
    6133              :    a pattern sequence of STMT_VINFO.
    6134              :    If VECTYPE is a mask type, SCALAR_TYPE_FOR_MASK is the scalar type
    6135              :    from which it was derived.
    6136              : 
    6137              :    Return converted mask.  */
    6138              : 
    6139              : static tree
    6140       148232 : build_mask_conversion (vec_info *vinfo,
    6141              :                        tree mask, tree vectype, stmt_vec_info stmt_vinfo,
    6142              :                        tree scalar_type_for_mask = NULL_TREE)
    6143              : {
    6144       148232 :   gimple *stmt;
    6145       148232 :   tree masktype, tmp;
    6146              : 
    6147       239014 :   gcc_assert (!scalar_type_for_mask == !VECTOR_BOOLEAN_TYPE_P (vectype));
    6148              : 
    6149       148232 :   masktype = truth_type_for (vectype);
    6150       148232 :   tmp = vect_recog_temp_ssa_var (TREE_TYPE (masktype), NULL);
    6151       148232 :   stmt = gimple_build_assign (tmp, CONVERT_EXPR, mask);
    6152       239014 :   append_pattern_def_seq (vinfo, stmt_vinfo,
    6153              :                           stmt, masktype,
    6154              :                           scalar_type_for_mask
    6155        90782 :                           ? scalar_type_for_mask : TREE_TYPE (vectype));
    6156              : 
    6157       148232 :   return tmp;
    6158              : }
    6159              : 
    6160              : 
    6161              : /* Return MASK if MASK is suitable for masking an operation on vectors
    6162              :    of type VECTYPE, otherwise convert it into such a form and return
    6163              :    the result.  Associate any conversion statements with STMT_INFO's
    6164              :    pattern.  If VECTYPE is a mask type, SCALAR_TYPE_FOR_MASK is the scalar
    6165              :    type from which it was derived.  */
    6166              : 
    6167              : static tree
    6168        81213 : vect_convert_mask_for_vectype (tree mask, tree vectype,
    6169              :                                stmt_vec_info stmt_info, vec_info *vinfo,
    6170              :                                tree scalar_type_for_mask = NULL_TREE)
    6171              : {
    6172        81213 :   tree mask_type = integer_type_for_mask (mask, vinfo);
    6173        81213 :   if (mask_type)
    6174              :     {
    6175        81213 :       tree mask_vectype = get_mask_type_for_scalar_type (vinfo, mask_type);
    6176        81213 :       if (mask_vectype
    6177       162426 :           && maybe_ne (TYPE_VECTOR_SUBPARTS (vectype),
    6178        99155 :                        TYPE_VECTOR_SUBPARTS (mask_vectype)))
    6179        63271 :         mask = build_mask_conversion (vinfo, mask, vectype, stmt_info,
    6180              :                                       scalar_type_for_mask);
    6181              :     }
    6182        81213 :   return mask;
    6183              : }
    6184              : 
    6185              : 
    6186              : /* Function vect_recog_bool_pattern
    6187              : 
    6188              :    Try to find pattern like following:
    6189              : 
    6190              :      bool a_b, b_b, c_b, d_b, e_b;
    6191              :      TYPE f_T;
    6192              :    loop:
    6193              :      S1  a_b = x1 CMP1 y1;
    6194              :      S2  b_b = x2 CMP2 y2;
    6195              :      S3  c_b = a_b & b_b;
    6196              :      S4  d_b = x3 CMP3 y3;
    6197              :      S5  e_b = c_b | d_b;
    6198              :      S6  f_T = (TYPE) e_b;
    6199              : 
    6200              :    where type 'TYPE' is an integral type.  Or a similar pattern
    6201              :    ending in
    6202              : 
    6203              :      S6  f_Y = e_b ? r_Y : s_Y;
    6204              : 
    6205              :    as results from if-conversion of a complex condition.
    6206              : 
    6207              :    Input:
    6208              : 
    6209              :    * STMT_VINFO: The stmt at the end from which the pattern
    6210              :                  search begins, i.e. cast of a bool to
    6211              :                  an integer type.
    6212              : 
    6213              :    Output:
    6214              : 
    6215              :    * TYPE_OUT: The type of the output of this pattern.
    6216              : 
    6217              :    * Return value: A new stmt that will be used to replace the pattern.
    6218              : 
    6219              :         Assuming size of TYPE is the same as size of all comparisons
    6220              :         (otherwise some casts would be added where needed), the above
    6221              :         sequence we create related pattern stmts:
    6222              :         S1'  a_T = x1 CMP1 y1 ? 1 : 0;
    6223              :         S3'  c_T = x2 CMP2 y2 ? a_T : 0;
    6224              :         S4'  d_T = x3 CMP3 y3 ? 1 : 0;
    6225              :         S5'  e_T = c_T | d_T;
    6226              :         S6'  f_T = e_T;
    6227              : 
    6228              :         Instead of the above S3' we could emit:
    6229              :         S2'  b_T = x2 CMP2 y2 ? 1 : 0;
    6230              :         S3'  c_T = a_T | b_T;
    6231              :         but the above is more efficient.  */
    6232              : 
    6233              : static gimple *
    6234     33156906 : vect_recog_bool_pattern (vec_info *vinfo,
    6235              :                          stmt_vec_info stmt_vinfo, tree *type_out)
    6236              : {
    6237     33156906 :   gimple *last_stmt = stmt_vinfo->stmt;
    6238     33156906 :   enum tree_code rhs_code;
    6239     33156906 :   tree var, lhs, rhs, vectype;
    6240     33156906 :   gimple *pattern_stmt;
    6241              : 
    6242     33156906 :   if (!is_gimple_assign (last_stmt))
    6243              :     return NULL;
    6244              : 
    6245     23639550 :   var = gimple_assign_rhs1 (last_stmt);
    6246     23639550 :   lhs = gimple_assign_lhs (last_stmt);
    6247     23639550 :   rhs_code = gimple_assign_rhs_code (last_stmt);
    6248              : 
    6249     23639550 :   if (rhs_code == VIEW_CONVERT_EXPR)
    6250       194754 :     var = TREE_OPERAND (var, 0);
    6251              : 
    6252     23639550 :   if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (var)))
    6253              :     return NULL;
    6254              : 
    6255       709491 :   hash_set<gimple *> bool_stmts;
    6256              : 
    6257       709491 :   if (CONVERT_EXPR_CODE_P (rhs_code)
    6258              :       || rhs_code == VIEW_CONVERT_EXPR
    6259              :       || rhs_code == FLOAT_EXPR)
    6260              :     {
    6261       170328 :       if (! (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
    6262         2087 :              || SCALAR_FLOAT_TYPE_P (TREE_TYPE (lhs)))
    6263       168775 :           || VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)))
    6264              :         return NULL;
    6265        78915 :       vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
    6266              : 
    6267        78915 :       tree type = integer_type_for_mask (var, vinfo);
    6268        78915 :       tree cst0, cst1, tmp;
    6269              : 
    6270        78915 :       if (!type)
    6271              :         return NULL;
    6272              : 
    6273              :       /* We may directly use cond with narrowed type to avoid multiple cond
    6274              :          exprs with following result packing and perform single cond with
    6275              :          packed mask instead.  In case of widening we better make cond first
    6276              :          and then extract results.  */
    6277        38245 :       if (TYPE_MODE (type) == TYPE_MODE (TREE_TYPE (lhs)))
    6278        24949 :         type = TREE_TYPE (lhs);
    6279              : 
    6280        38245 :       cst0 = build_int_cst (type, 0);
    6281        38245 :       cst1 = build_int_cst (type, 1);
    6282        38245 :       tmp = vect_recog_temp_ssa_var (type, NULL);
    6283        38245 :       pattern_stmt = gimple_build_assign (tmp, COND_EXPR, var, cst1, cst0);
    6284              : 
    6285        38245 :       if (!useless_type_conversion_p (type, TREE_TYPE (lhs)))
    6286              :         {
    6287        13296 :           tree new_vectype = get_vectype_for_scalar_type (vinfo, type);
    6288        13296 :           append_pattern_def_seq (vinfo, stmt_vinfo,
    6289              :                                   pattern_stmt, new_vectype);
    6290              : 
    6291        13296 :           lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
    6292        13296 :           pattern_stmt
    6293        26284 :             = gimple_build_assign (lhs, (rhs_code == FLOAT_EXPR
    6294              :                                          ? FLOAT_EXPR : CONVERT_EXPR), tmp);
    6295              :         }
    6296              : 
    6297        38245 :       *type_out = vectype;
    6298        38245 :       vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
    6299              : 
    6300        38245 :       return pattern_stmt;
    6301              :     }
    6302              :   else if (rhs_code == COND_EXPR
    6303       155476 :            && TREE_CODE (var) == SSA_NAME)
    6304              :     {
    6305       155476 :       vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
    6306       155476 :       if (vectype == NULL_TREE)
    6307              :         return NULL;
    6308              : 
    6309              :       /* Build a scalar type for the boolean result that when
    6310              :          vectorized matches the vector type of the result in
    6311              :          size and number of elements.  */
    6312       140018 :       unsigned prec
    6313       140018 :         = vector_element_size (tree_to_poly_uint64 (TYPE_SIZE (vectype)),
    6314              :                                TYPE_VECTOR_SUBPARTS (vectype));
    6315              : 
    6316       140018 :       tree type
    6317       280036 :         = build_nonstandard_integer_type (prec,
    6318       140018 :                                           TYPE_UNSIGNED (TREE_TYPE (var)));
    6319       140018 :       if (get_vectype_for_scalar_type (vinfo, type) == NULL_TREE)
    6320              :         return NULL;
    6321              : 
    6322       140018 :       enum vect_def_type dt;
    6323       140018 :       if (integer_type_for_mask (var, vinfo))
    6324              :         return NULL;
    6325        37539 :       else if (TREE_CODE (TREE_TYPE (var)) == BOOLEAN_TYPE
    6326        37539 :                && vect_is_simple_use (var, vinfo, &dt)
    6327        37539 :                && (dt == vect_external_def
    6328        37532 :                    || dt == vect_constant_def))
    6329              :         {
    6330              :           /* If the condition is already a boolean then manually convert it to a
    6331              :              mask of the given integer type but don't set a vectype.  */
    6332         1366 :           tree lhs_ivar = vect_recog_temp_ssa_var (type, NULL);
    6333         1366 :           pattern_stmt = gimple_build_assign (lhs_ivar, COND_EXPR, var,
    6334              :                                               build_all_ones_cst (type),
    6335              :                                               build_zero_cst (type));
    6336         1366 :           append_inv_pattern_def_seq (vinfo, pattern_stmt);
    6337         1366 :           var = lhs_ivar;
    6338              :         }
    6339              : 
    6340        37539 :       tree lhs_var = vect_recog_temp_ssa_var (boolean_type_node, NULL);
    6341        37539 :       pattern_stmt = gimple_build_assign (lhs_var, NE_EXPR, var,
    6342        37539 :                                           build_zero_cst (TREE_TYPE (var)));
    6343              : 
    6344        37539 :       tree new_vectype = get_mask_type_for_scalar_type (vinfo, TREE_TYPE (var));
    6345        37539 :       if (!new_vectype)
    6346              :         return NULL;
    6347              : 
    6348        37539 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, new_vectype,
    6349        37539 :                               TREE_TYPE (var));
    6350              : 
    6351        37539 :       lhs_var = vect_convert_mask_for_vectype (lhs_var, vectype, stmt_vinfo,
    6352              :                                                vinfo);
    6353              : 
    6354        37539 :       lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
    6355        37539 :       pattern_stmt
    6356        37539 :         = gimple_build_assign (lhs, COND_EXPR, lhs_var,
    6357              :                                gimple_assign_rhs2 (last_stmt),
    6358              :                                gimple_assign_rhs3 (last_stmt));
    6359        37539 :       *type_out = vectype;
    6360        37539 :       vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
    6361              : 
    6362        37539 :       return pattern_stmt;
    6363              :     }
    6364       468471 :   else if (rhs_code == BIT_NOT_EXPR
    6365       468471 :            && !vect_use_mask_type_p (stmt_vinfo))
    6366              :     {
    6367              :       /* When we have a bool data inversion rewrite that to an XOR to
    6368              :          cope with the fact that we'll use a wider vector element type.  */
    6369        10318 :       lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
    6370        10318 :       pattern_stmt
    6371        10318 :         = gimple_build_assign (lhs, BIT_XOR_EXPR, var,
    6372        10318 :                                build_all_ones_cst (TREE_TYPE (var)));
    6373        10318 :       *type_out = NULL_TREE;
    6374        10318 :       vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
    6375              : 
    6376        10318 :       return pattern_stmt;
    6377              :     }
    6378       458153 :   else if ((rhs_code == BIT_XOR_EXPR
    6379              :             || rhs_code == BIT_AND_EXPR
    6380       458153 :             || rhs_code == BIT_IOR_EXPR)
    6381       363137 :            && TREE_CODE (var) == SSA_NAME)
    6382              :     {
    6383       363137 :       tree rhs2 = gimple_assign_rhs2 (last_stmt);
    6384       363137 :       if (TREE_CODE (rhs2) != SSA_NAME)
    6385              :         return NULL;
    6386       363137 :       tree lhs_type = integer_type_for_mask (lhs, vinfo);
    6387       363137 :       if (!lhs_type)
    6388              :         return NULL;
    6389       191720 :       vectype = get_mask_type_for_scalar_type (vinfo, lhs_type);
    6390       191720 :       if (!vectype)
    6391              :         return NULL;
    6392       191720 :       vect_def_type dt1, dt2;
    6393       191720 :       tree rhs1_type = integer_type_for_mask (var, vinfo, &dt1);
    6394       191720 :       tree rhs2_type = integer_type_for_mask (rhs2, vinfo, &dt2);
    6395       191720 :       if ((rhs1_type || dt1 == vect_external_def)
    6396       175740 :           && (rhs2_type || dt2 == vect_external_def))
    6397              :         return NULL;
    6398              :       /* When one input is a mask and the other is not create a pattern
    6399              :          stmt sequence that creates a mask for the non-mask input and
    6400              :          convert it to one suitable for the output mask used.  */
    6401        39201 :       if (rhs1_type && !rhs2_type)
    6402              :         {
    6403        23221 :           tree rhs1_vectype = get_mask_type_for_scalar_type (vinfo, rhs1_type);
    6404        23221 :           if (!rhs1_vectype)
    6405              :             return NULL;
    6406        23221 :           tree rhs2_vectype = get_vectype_for_scalar_type (vinfo,
    6407        23221 :                                                            TREE_TYPE (rhs2));
    6408        23221 :           if (!rhs2_vectype)
    6409              :             return NULL;
    6410        23221 :           tree new_vectype = truth_type_for (rhs2_vectype);
    6411        23221 :           tree tem = vect_recog_temp_ssa_var (TREE_TYPE (new_vectype), NULL);
    6412        23221 :           pattern_stmt = gimple_build_assign (tem, NE_EXPR, rhs2,
    6413              :                                               build_zero_cst
    6414        23221 :                                                 (TREE_TYPE (rhs2)));
    6415        23221 :           append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
    6416        23221 :                                   new_vectype, TREE_TYPE (new_vectype));
    6417        23221 :           rhs2 = vect_convert_mask_for_vectype (tem, rhs1_vectype,
    6418              :                                                 stmt_vinfo, vinfo, rhs1_type);
    6419              :         }
    6420        15980 :       else if (!rhs1_type && rhs2_type)
    6421              :         {
    6422        15980 :           tree rhs2_vectype = get_mask_type_for_scalar_type (vinfo, rhs2_type);
    6423        15980 :           if (!rhs2_vectype)
    6424              :             return NULL;
    6425        15980 :           tree rhs1_vectype = get_vectype_for_scalar_type (vinfo,
    6426        15980 :                                                            TREE_TYPE (var));
    6427        15980 :           if (!rhs1_vectype)
    6428              :             return NULL;
    6429        15980 :           tree new_vectype = truth_type_for (rhs1_vectype);
    6430        15980 :           tree tem = vect_recog_temp_ssa_var (TREE_TYPE (new_vectype), NULL);
    6431        15980 :           pattern_stmt = gimple_build_assign (tem, NE_EXPR, var,
    6432              :                                               build_zero_cst
    6433        15980 :                                                 (TREE_TYPE (var)));
    6434        15980 :           append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
    6435        15980 :                                   new_vectype, TREE_TYPE (new_vectype));
    6436        15980 :           var = vect_convert_mask_for_vectype (tem, rhs2_vectype,
    6437              :                                                stmt_vinfo, vinfo, rhs2_type);
    6438              :         }
    6439        39201 :       lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
    6440        39201 :       pattern_stmt = gimple_build_assign (lhs, rhs_code, var, rhs2);
    6441        39201 :       vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
    6442        39201 :       *type_out = vectype;
    6443        39201 :       return pattern_stmt;
    6444              :     }
    6445        95016 :   else if (rhs_code == SSA_NAME
    6446        25973 :            && STMT_VINFO_DATA_REF (stmt_vinfo))
    6447              :     {
    6448         7888 :       stmt_vec_info pattern_stmt_info;
    6449         7888 :       vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
    6450         7888 :       if (!vectype || !VECTOR_MODE_P (TYPE_MODE (vectype)))
    6451              :         return NULL;
    6452              : 
    6453         7888 :       tree type = integer_type_for_mask (var, vinfo);
    6454         7888 :       if (!type)
    6455              :         return NULL;
    6456              : 
    6457         4473 :       var = vect_convert_mask_for_vectype (var, vectype, stmt_vinfo, vinfo);
    6458              : 
    6459         4473 :       tree cst0 = build_int_cst (TREE_TYPE (vectype), 0);
    6460         4473 :       tree cst1 = build_int_cst (TREE_TYPE (vectype), 1);
    6461         4473 :       rhs = vect_recog_temp_ssa_var (TREE_TYPE (vectype), NULL);
    6462         4473 :       pattern_stmt = gimple_build_assign (rhs, COND_EXPR, var, cst1, cst0);
    6463         4473 :       append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vectype);
    6464              : 
    6465         4473 :       lhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (vectype), lhs);
    6466         4473 :       pattern_stmt = gimple_build_assign (lhs, SSA_NAME, rhs);
    6467         4473 :       pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
    6468         4473 :       vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
    6469         4473 :       *type_out = vectype;
    6470         4473 :       vect_pattern_detected ("vect_recog_bool_pattern", last_stmt);
    6471              : 
    6472         4473 :       return pattern_stmt;
    6473              :     }
    6474              :   else
    6475              :     return NULL;
    6476       709491 : }
    6477              : 
    6478              : 
    6479              : /* Function vect_recog_mask_conversion_pattern
    6480              : 
    6481              :    Try to find statements which require boolean type
    6482              :    conversion.  Additional conversion statements are
    6483              :    added to handle such cases.  For example:
    6484              : 
    6485              :    bool m_1, m_2, m_3;
    6486              :    int i_4, i_5;
    6487              :    double d_6, d_7;
    6488              :    char c_1, c_2, c_3;
    6489              : 
    6490              :    S1   m_1 = i_4 > i_5;
    6491              :    S2   m_2 = d_6 < d_7;
    6492              :    S3   m_3 = m_1 & m_2;
    6493              :    S4   c_1 = m_3 ? c_2 : c_3;
    6494              : 
    6495              :    Will be transformed into:
    6496              : 
    6497              :    S1   m_1 = i_4 > i_5;
    6498              :    S2   m_2 = d_6 < d_7;
    6499              :    S3'' m_2' = (_Bool[bitsize=32])m_2
    6500              :    S3'  m_3' = m_1 & m_2';
    6501              :    S4'' m_3'' = (_Bool[bitsize=8])m_3'
    6502              :    S4'  c_1' = m_3'' ? c_2 : c_3;  */
    6503              : 
    6504              : static gimple *
    6505     33185044 : vect_recog_mask_conversion_pattern (vec_info *vinfo,
    6506              :                                     stmt_vec_info stmt_vinfo, tree *type_out)
    6507              : {
    6508     33185044 :   gimple *last_stmt = stmt_vinfo->stmt;
    6509     33185044 :   enum tree_code rhs_code;
    6510     33185044 :   tree lhs = NULL_TREE, rhs1, rhs2, tmp, rhs1_type, rhs2_type;
    6511     33185044 :   tree vectype1, vectype2;
    6512     33185044 :   stmt_vec_info pattern_stmt_info;
    6513              : 
    6514              :   /* Check for MASK_LOAD and MASK_STORE as well as COND_OP calls requiring mask
    6515              :      conversion.  */
    6516     33185044 :   if (is_gimple_call (last_stmt)
    6517     33185044 :       && gimple_call_internal_p (last_stmt))
    6518              :     {
    6519       135902 :       gcall *pattern_stmt;
    6520              : 
    6521       135902 :       internal_fn ifn = gimple_call_internal_fn (last_stmt);
    6522       135902 :       int mask_argno = internal_fn_mask_index (ifn);
    6523       135902 :       if (mask_argno < 0)
    6524              :         return NULL;
    6525              : 
    6526        14814 :       bool store_p = internal_store_fn_p (ifn);
    6527        14814 :       bool load_p = internal_store_fn_p (ifn);
    6528        14814 :       if (store_p)
    6529              :         {
    6530         2513 :           int rhs_index = internal_fn_stored_value_index (ifn);
    6531         2513 :           tree rhs = gimple_call_arg (last_stmt, rhs_index);
    6532         2513 :           vectype1 = get_vectype_for_scalar_type (vinfo, TREE_TYPE (rhs));
    6533              :         }
    6534              :       else
    6535              :         {
    6536        12301 :           lhs = gimple_call_lhs (last_stmt);
    6537        12301 :           if (!lhs)
    6538              :             return NULL;
    6539        12301 :           vectype1 = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
    6540              :         }
    6541              : 
    6542        14814 :       if (!vectype1)
    6543              :         return NULL;
    6544              : 
    6545        14494 :       tree mask_arg = gimple_call_arg (last_stmt, mask_argno);
    6546        14494 :       tree mask_arg_type = integer_type_for_mask (mask_arg, vinfo);
    6547        14494 :       if (mask_arg_type)
    6548              :         {
    6549        12862 :           vectype2 = get_mask_type_for_scalar_type (vinfo, mask_arg_type);
    6550              : 
    6551        12862 :           if (!vectype2
    6552        12862 :               || known_eq (TYPE_VECTOR_SUBPARTS (vectype1),
    6553              :                            TYPE_VECTOR_SUBPARTS (vectype2)))
    6554              :             return NULL;
    6555              :         }
    6556         1632 :       else if (store_p || load_p)
    6557              :         return NULL;
    6558              : 
    6559         6298 :       tmp = build_mask_conversion (vinfo, mask_arg, vectype1, stmt_vinfo);
    6560              : 
    6561         6298 :       auto_vec<tree, 8> args;
    6562         6298 :       unsigned int nargs = gimple_call_num_args (last_stmt);
    6563         6298 :       args.safe_grow (nargs, true);
    6564        37788 :       for (unsigned int i = 0; i < nargs; ++i)
    6565        25192 :         args[i] = ((int) i == mask_argno
    6566        25192 :                    ? tmp
    6567        18894 :                    : gimple_call_arg (last_stmt, i));
    6568         6298 :       pattern_stmt = gimple_build_call_internal_vec (ifn, args);
    6569              : 
    6570         6298 :       if (!store_p)
    6571              :         {
    6572         6018 :           lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
    6573         6018 :           gimple_call_set_lhs (pattern_stmt, lhs);
    6574              :         }
    6575              : 
    6576         6018 :       if (load_p || store_p)
    6577          280 :         gimple_call_set_nothrow (pattern_stmt, true);
    6578              : 
    6579         6298 :       pattern_stmt_info = vinfo->add_stmt (pattern_stmt);
    6580         6298 :       if (STMT_VINFO_DATA_REF (stmt_vinfo))
    6581         1822 :         vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
    6582              : 
    6583         6298 :       *type_out = vectype1;
    6584         6298 :       vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
    6585              : 
    6586         6298 :       return pattern_stmt;
    6587         6298 :     }
    6588              : 
    6589     33049142 :   if (!is_gimple_assign (last_stmt))
    6590              :     return NULL;
    6591              : 
    6592     23667688 :   gimple *pattern_stmt;
    6593     23667688 :   lhs = gimple_assign_lhs (last_stmt);
    6594     23667688 :   rhs1 = gimple_assign_rhs1 (last_stmt);
    6595     23667688 :   rhs_code = gimple_assign_rhs_code (last_stmt);
    6596              : 
    6597              :   /* Check for cond expression requiring mask conversion.  */
    6598     23667688 :   if (rhs_code == COND_EXPR)
    6599              :     {
    6600       135840 :       vectype1 = get_vectype_for_scalar_type (vinfo, TREE_TYPE (lhs));
    6601              : 
    6602       135840 :       gcc_assert (! COMPARISON_CLASS_P (rhs1));
    6603       135840 :       if (TREE_CODE (rhs1) == SSA_NAME)
    6604              :         {
    6605       135840 :           rhs1_type = integer_type_for_mask (rhs1, vinfo);
    6606       135840 :           if (!rhs1_type)
    6607              :             return NULL;
    6608              :         }
    6609              :       else
    6610              :         return NULL;
    6611              : 
    6612       122586 :       vectype2 = get_mask_type_for_scalar_type (vinfo, rhs1_type);
    6613              : 
    6614       122586 :       if (!vectype1 || !vectype2)
    6615              :         return NULL;
    6616              : 
    6617              :       /* Continue if a conversion is needed.  Also continue if we have
    6618              :          a comparison whose vector type would normally be different from
    6619              :          VECTYPE2 when considered in isolation.  In that case we'll
    6620              :          replace the comparison with an SSA name (so that we can record
    6621              :          its vector type) and behave as though the comparison was an SSA
    6622              :          name from the outset.  */
    6623       120382 :       if (known_eq (TYPE_VECTOR_SUBPARTS (vectype1),
    6624              :                     TYPE_VECTOR_SUBPARTS (vectype2)))
    6625              :         return NULL;
    6626              : 
    6627        45976 :       if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1),
    6628        91952 :                     TYPE_VECTOR_SUBPARTS (vectype2)))
    6629        45976 :         tmp = build_mask_conversion (vinfo, rhs1, vectype1, stmt_vinfo);
    6630              :       else
    6631              :         tmp = rhs1;
    6632              : 
    6633        45976 :       lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
    6634        45976 :       pattern_stmt = gimple_build_assign (lhs, COND_EXPR, tmp,
    6635              :                                           gimple_assign_rhs2 (last_stmt),
    6636              :                                           gimple_assign_rhs3 (last_stmt));
    6637              : 
    6638        45976 :       *type_out = vectype1;
    6639        45976 :       vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
    6640              : 
    6641        45976 :       return pattern_stmt;
    6642              :     }
    6643              : 
    6644              :   /* Now check for binary boolean operations requiring conversion for
    6645              :      one of operands.  */
    6646     23531848 :   if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)))
    6647              :     return NULL;
    6648              : 
    6649      1838490 :   if (rhs_code != BIT_IOR_EXPR
    6650              :       && rhs_code != BIT_XOR_EXPR
    6651      1838490 :       && rhs_code != BIT_AND_EXPR
    6652      1514554 :       && TREE_CODE_CLASS (rhs_code) != tcc_comparison)
    6653              :     return NULL;
    6654              : 
    6655      1651380 :   rhs2 = gimple_assign_rhs2 (last_stmt);
    6656              : 
    6657      1651380 :   rhs1_type = integer_type_for_mask (rhs1, vinfo);
    6658      1651380 :   rhs2_type = integer_type_for_mask (rhs2, vinfo);
    6659              : 
    6660      1651380 :   if (!rhs1_type || !rhs2_type
    6661      1651380 :       || TYPE_PRECISION (rhs1_type) == TYPE_PRECISION (rhs2_type))
    6662              :     return NULL;
    6663              : 
    6664        32687 :   if (TYPE_PRECISION (rhs1_type) < TYPE_PRECISION (rhs2_type))
    6665              :     {
    6666        24640 :       vectype1 = get_mask_type_for_scalar_type (vinfo, rhs1_type);
    6667        24640 :       if (!vectype1)
    6668              :         return NULL;
    6669        24640 :       rhs2 = build_mask_conversion (vinfo, rhs2, vectype1, stmt_vinfo,
    6670              :                                     rhs1_type);
    6671              :     }
    6672              :   else
    6673              :     {
    6674         8047 :       vectype1 = get_mask_type_for_scalar_type (vinfo, rhs2_type);
    6675         8047 :       if (!vectype1)
    6676              :         return NULL;
    6677         8047 :       rhs1 = build_mask_conversion (vinfo, rhs1, vectype1, stmt_vinfo,
    6678              :                                     rhs2_type);
    6679              :     }
    6680              : 
    6681        32687 :   lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
    6682        32687 :   pattern_stmt = gimple_build_assign (lhs, rhs_code, rhs1, rhs2);
    6683              : 
    6684        32687 :   *type_out = vectype1;
    6685        32687 :   vect_pattern_detected ("vect_recog_mask_conversion_pattern", last_stmt);
    6686              : 
    6687        32687 :   return pattern_stmt;
    6688              : }
    6689              : 
    6690              : /* STMT_INFO is a load or store.  If the load or store is conditional, return
    6691              :    the boolean condition under which it occurs, otherwise return null.  */
    6692              : 
    6693              : static tree
    6694        95578 : vect_get_load_store_mask (stmt_vec_info stmt_info)
    6695              : {
    6696        95578 :   if (gassign *def_assign = dyn_cast <gassign *> (stmt_info->stmt))
    6697              :     {
    6698        94561 :       gcc_assert (gimple_assign_single_p (def_assign));
    6699              :       return NULL_TREE;
    6700              :     }
    6701              : 
    6702         1017 :   if (gcall *def_call = dyn_cast <gcall *> (stmt_info->stmt))
    6703              :     {
    6704         1017 :       internal_fn ifn = gimple_call_internal_fn (def_call);
    6705         1017 :       int mask_index = internal_fn_mask_index (ifn);
    6706         1017 :       return gimple_call_arg (def_call, mask_index);
    6707              :     }
    6708              : 
    6709            0 :   gcc_unreachable ();
    6710              : }
    6711              : 
    6712              : /* Return the equivalent of:
    6713              : 
    6714              :      fold_convert (TYPE, VALUE)
    6715              : 
    6716              :    with the expectation that the operation will be vectorized.
    6717              :    If new statements are needed, add them as pattern statements
    6718              :    to STMT_INFO.  */
    6719              : 
    6720              : static tree
    6721            0 : vect_add_conversion_to_pattern (vec_info *vinfo,
    6722              :                                 tree type, tree value, stmt_vec_info stmt_info)
    6723              : {
    6724            0 :   if (useless_type_conversion_p (type, TREE_TYPE (value)))
    6725              :     return value;
    6726              : 
    6727            0 :   tree new_value = vect_recog_temp_ssa_var (type, NULL);
    6728            0 :   gassign *conversion = gimple_build_assign (new_value, CONVERT_EXPR, value);
    6729            0 :   append_pattern_def_seq (vinfo, stmt_info, conversion,
    6730              :                           get_vectype_for_scalar_type (vinfo, type));
    6731            0 :   return new_value;
    6732              : }
    6733              : 
    6734              : /* Try to convert STMT_INFO into a call to a gather load or scatter store
    6735              :    internal function.  Return the final statement on success and set
    6736              :    *TYPE_OUT to the vector type being loaded or stored.
    6737              : 
    6738              :    This function only handles gathers and scatters that were recognized
    6739              :    as such from the outset (indicated by STMT_VINFO_GATHER_SCATTER_P).  */
    6740              : 
    6741              : static gimple *
    6742     33185044 : vect_recog_gather_scatter_pattern (vec_info *vinfo,
    6743              :                                    stmt_vec_info stmt_info, tree *type_out)
    6744              : {
    6745              :   /* Currently we only support this for loop vectorization.  */
    6746     33185044 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    6747      4699099 :   if (!loop_vinfo)
    6748              :     return NULL;
    6749              : 
    6750              :   /* Make sure that we're looking at a gather load or scatter store.  */
    6751      4699099 :   data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
    6752      4699099 :   if (!dr || !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
    6753              :     return NULL;
    6754              : 
    6755              :   /* Get the boolean that controls whether the load or store happens.
    6756              :      This is null if the operation is unconditional.  */
    6757        95578 :   tree mask = vect_get_load_store_mask (stmt_info);
    6758              : 
    6759              :   /* DR analysis nailed down the vector type for the access.  */
    6760        95578 :   tree gs_vectype = STMT_VINFO_VECTYPE (stmt_info);
    6761              : 
    6762              :   /* Make sure that the target supports an appropriate internal
    6763              :      function for the gather/scatter operation.  */
    6764        95578 :   gather_scatter_info gs_info;
    6765        95578 :   if (!vect_check_gather_scatter (stmt_info, gs_vectype, loop_vinfo, &gs_info)
    6766        95578 :       || gs_info.ifn == IFN_LAST)
    6767              :     return NULL;
    6768              : 
    6769              :   /* Convert the mask to the right form.  */
    6770            0 :   if (mask)
    6771            0 :     mask = vect_convert_mask_for_vectype (mask, gs_vectype, stmt_info,
    6772              :                                           loop_vinfo);
    6773            0 :   else if (gs_info.ifn == IFN_MASK_SCATTER_STORE
    6774            0 :            || gs_info.ifn == IFN_MASK_GATHER_LOAD
    6775            0 :            || gs_info.ifn == IFN_MASK_LEN_SCATTER_STORE
    6776            0 :            || gs_info.ifn == IFN_MASK_LEN_GATHER_LOAD)
    6777            0 :     mask = build_int_cst (TREE_TYPE (truth_type_for (gs_vectype)), -1);
    6778              : 
    6779              :   /* Get the invariant base and non-invariant offset, converting the
    6780              :      latter to the same width as the vector elements.  */
    6781            0 :   tree base = gs_info.base;
    6782            0 :   tree offset_type = TREE_TYPE (gs_info.offset_vectype);
    6783            0 :   tree offset = vect_add_conversion_to_pattern (vinfo, offset_type,
    6784              :                                                 gs_info.offset, stmt_info);
    6785              : 
    6786              :   /* Build the new pattern statement.  */
    6787            0 :   tree scale = size_int (gs_info.scale);
    6788            0 :   gcall *pattern_stmt;
    6789              : 
    6790            0 :   if (DR_IS_READ (dr))
    6791              :     {
    6792            0 :       tree zero = build_zero_cst (gs_info.element_type);
    6793            0 :       if (mask != NULL)
    6794              :         {
    6795            0 :           int elsval = MASK_LOAD_ELSE_ZERO;
    6796              : 
    6797            0 :           tree vec_els
    6798            0 :             = vect_get_mask_load_else (elsval, TREE_TYPE (gs_vectype));
    6799            0 :           pattern_stmt = gimple_build_call_internal (gs_info.ifn, 7, base,
    6800              :                                                      gs_info.alias_ptr,
    6801              :                                                      offset, scale, zero, mask,
    6802              :                                                      vec_els);
    6803              :         }
    6804              :       else
    6805            0 :         pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5, base,
    6806              :                                                    gs_info.alias_ptr,
    6807              :                                                    offset, scale, zero);
    6808            0 :       tree lhs = gimple_get_lhs (stmt_info->stmt);
    6809            0 :       tree load_lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
    6810            0 :       gimple_call_set_lhs (pattern_stmt, load_lhs);
    6811              :     }
    6812              :   else
    6813              :     {
    6814            0 :       tree rhs = vect_get_store_rhs (stmt_info);
    6815            0 :       if (mask != NULL)
    6816            0 :         pattern_stmt = gimple_build_call_internal (gs_info.ifn, 6,
    6817              :                                                    base, gs_info.alias_ptr,
    6818              :                                                    offset, scale, rhs, mask);
    6819              :       else
    6820            0 :         pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5,
    6821              :                                                    base, gs_info.alias_ptr,
    6822              :                                                    offset, scale, rhs);
    6823              :     }
    6824            0 :   gimple_call_set_nothrow (pattern_stmt, true);
    6825              : 
    6826              :   /* Copy across relevant vectorization info and associate DR with the
    6827              :      new pattern statement instead of the original statement.  */
    6828            0 :   stmt_vec_info pattern_stmt_info = loop_vinfo->add_stmt (pattern_stmt);
    6829            0 :   loop_vinfo->move_dr (pattern_stmt_info, stmt_info);
    6830              : 
    6831            0 :   *type_out = gs_vectype;
    6832            0 :   vect_pattern_detected ("gather/scatter pattern", stmt_info->stmt);
    6833              : 
    6834            0 :   return pattern_stmt;
    6835              : }
    6836              : 
    6837              : /* Helper method of vect_recog_cond_store_pattern,  checks to see if COND_ARG
    6838              :    is points to a load statement that reads the same data as that of
    6839              :    STORE_VINFO.  */
    6840              : 
    6841              : static bool
    6842        35512 : vect_cond_store_pattern_same_ref (vec_info *vinfo,
    6843              :                                   stmt_vec_info store_vinfo, tree cond_arg)
    6844              : {
    6845        35512 :   stmt_vec_info load_stmt_vinfo = vinfo->lookup_def (cond_arg);
    6846        35512 :   if (!load_stmt_vinfo
    6847        20313 :       || !STMT_VINFO_DATA_REF (load_stmt_vinfo)
    6848        11970 :       || DR_IS_WRITE (STMT_VINFO_DATA_REF (load_stmt_vinfo))
    6849        47482 :       || !same_data_refs (STMT_VINFO_DATA_REF (store_vinfo),
    6850              :                           STMT_VINFO_DATA_REF (load_stmt_vinfo)))
    6851        26208 :     return false;
    6852              : 
    6853              :   return true;
    6854              : }
    6855              : 
    6856              : /* Function vect_recog_cond_store_pattern
    6857              : 
    6858              :    Try to find the following pattern:
    6859              : 
    6860              :    x = *_3;
    6861              :    c = a CMP b;
    6862              :    y = c ? t_20 : x;
    6863              :    *_3 = y;
    6864              : 
    6865              :    where the store of _3 happens on a conditional select on a value loaded
    6866              :    from the same location.  In such case we can elide the initial load if
    6867              :    MASK_STORE is supported and instead only conditionally write out the result.
    6868              : 
    6869              :    The pattern produces for the above:
    6870              : 
    6871              :    c = a CMP b;
    6872              :    .MASK_STORE (_3, c, t_20)
    6873              : 
    6874              :    Input:
    6875              : 
    6876              :    * STMT_VINFO: The stmt from which the pattern search begins.  In the
    6877              :    example, when this function is called with _3 then the search begins.
    6878              : 
    6879              :    Output:
    6880              : 
    6881              :    * TYPE_OUT: The type of the output  of this pattern.
    6882              : 
    6883              :    * Return value: A new stmt that will be used to replace the sequence.  */
    6884              : 
    6885              : static gimple *
    6886     33185044 : vect_recog_cond_store_pattern (vec_info *vinfo,
    6887              :                                stmt_vec_info stmt_vinfo, tree *type_out)
    6888              : {
    6889     33185044 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    6890      4699099 :   if (!loop_vinfo)
    6891              :     return NULL;
    6892              : 
    6893      4699099 :   gimple *store_stmt = STMT_VINFO_STMT (stmt_vinfo);
    6894              : 
    6895              :   /* Needs to be a gimple store where we have DR info for.  */
    6896      4699099 :   if (!STMT_VINFO_DATA_REF (stmt_vinfo)
    6897      1125081 :       || DR_IS_READ (STMT_VINFO_DATA_REF (stmt_vinfo))
    6898      5094693 :       || !gimple_store_p (store_stmt))
    6899              :     return NULL;
    6900              : 
    6901       393181 :   tree st_rhs = gimple_assign_rhs1 (store_stmt);
    6902              : 
    6903       393181 :   if (TREE_CODE (st_rhs) != SSA_NAME)
    6904              :     return NULL;
    6905              : 
    6906       306136 :   auto cond_vinfo = vinfo->lookup_def (st_rhs);
    6907              : 
    6908              :   /* If the condition isn't part of the loop then bool recog wouldn't have seen
    6909              :      it and so this transformation may not be valid.  */
    6910       306136 :   if (!cond_vinfo)
    6911              :     return NULL;
    6912              : 
    6913       288049 :   cond_vinfo = vect_stmt_to_vectorize (cond_vinfo);
    6914       288049 :   gassign *cond_stmt = dyn_cast<gassign *> (STMT_VINFO_STMT (cond_vinfo));
    6915       355596 :   if (!cond_stmt || gimple_assign_rhs_code (cond_stmt) != COND_EXPR)
    6916              :     return NULL;
    6917              : 
    6918              :   /* Check if the else value matches the original loaded one.  */
    6919        18797 :   bool invert = false;
    6920        18797 :   tree cmp_ls = gimple_arg (cond_stmt, 0);
    6921        18797 :   if (TREE_CODE (cmp_ls) != SSA_NAME)
    6922              :     return NULL;
    6923              : 
    6924        18797 :   tree cond_arg1 = gimple_arg (cond_stmt, 1);
    6925        18797 :   tree cond_arg2 = gimple_arg (cond_stmt, 2);
    6926              : 
    6927        18797 :   if (!vect_cond_store_pattern_same_ref (vinfo, stmt_vinfo, cond_arg2)
    6928        18797 :       && !(invert = vect_cond_store_pattern_same_ref (vinfo, stmt_vinfo,
    6929              :                                                       cond_arg1)))
    6930              :     return NULL;
    6931              : 
    6932         9304 :   vect_pattern_detected ("vect_recog_cond_store_pattern", store_stmt);
    6933              : 
    6934         9304 :   tree scalar_type = TREE_TYPE (st_rhs);
    6935         9304 :   if (VECTOR_TYPE_P (scalar_type))
    6936              :     return NULL;
    6937              : 
    6938         9304 :   tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
    6939         9304 :   if (vectype == NULL_TREE)
    6940              :     return NULL;
    6941              : 
    6942         9304 :   machine_mode mask_mode;
    6943         9304 :   machine_mode vecmode = TYPE_MODE (vectype);
    6944         1864 :   if (!VECTOR_MODE_P (vecmode)
    6945         9304 :       || targetm.vectorize.conditional_operation_is_expensive (IFN_MASK_STORE)
    6946         9304 :       || !targetm.vectorize.get_mask_mode (vecmode).exists (&mask_mode)
    6947         9304 :       || !can_vec_mask_load_store_p (vecmode, mask_mode, false))
    6948              :     return NULL;
    6949              : 
    6950            0 :   tree base = DR_REF (STMT_VINFO_DATA_REF (stmt_vinfo));
    6951            0 :   if (may_be_nonaddressable_p (base))
    6952              :     return NULL;
    6953              : 
    6954              :   /* We need to use the false parameter of the conditional select.  */
    6955            0 :   tree cond_store_arg = invert ? cond_arg2 : cond_arg1;
    6956            0 :   tree cond_load_arg = invert ? cond_arg1 : cond_arg2;
    6957            0 :   gimple *load_stmt = SSA_NAME_DEF_STMT (cond_load_arg);
    6958              : 
    6959              :   /* This is a rough estimation to check that there aren't any aliasing stores
    6960              :      in between the load and store.  It's a bit strict, but for now it's good
    6961              :      enough.  */
    6962            0 :   if (gimple_vuse (load_stmt) != gimple_vuse (store_stmt))
    6963              :     return NULL;
    6964              : 
    6965              :   /* If we have to invert the condition, i.e. use the true argument rather than
    6966              :      the false argument, we have to negate the mask.  */
    6967            0 :   if (invert)
    6968              :     {
    6969            0 :       tree var = vect_recog_temp_ssa_var (boolean_type_node, NULL);
    6970              : 
    6971              :       /* Invert the mask using ^ 1.  */
    6972            0 :       tree itype = TREE_TYPE (cmp_ls);
    6973            0 :       gassign *conv = gimple_build_assign (var, BIT_XOR_EXPR, cmp_ls,
    6974              :                                            build_int_cst (itype, 1));
    6975              : 
    6976            0 :       tree mask_vec_type = get_mask_type_for_scalar_type (vinfo, itype);
    6977            0 :       append_pattern_def_seq (vinfo, stmt_vinfo, conv, mask_vec_type, itype);
    6978            0 :       cmp_ls= var;
    6979              :     }
    6980              : 
    6981            0 :   if (TREE_CODE (base) != MEM_REF)
    6982            0 :    base = build_fold_addr_expr (base);
    6983              : 
    6984            0 :   tree ptr = build_int_cst (reference_alias_ptr_type (base),
    6985            0 :                             get_object_alignment (base));
    6986              : 
    6987              :   /* Convert the mask to the right form.  */
    6988            0 :   tree mask = vect_convert_mask_for_vectype (cmp_ls, vectype, stmt_vinfo,
    6989              :                                              vinfo);
    6990              : 
    6991            0 :   gcall *call
    6992            0 :     = gimple_build_call_internal (IFN_MASK_STORE, 4, base, ptr, mask,
    6993              :                                   cond_store_arg);
    6994            0 :   gimple_set_location (call, gimple_location (store_stmt));
    6995              : 
    6996              :   /* Copy across relevant vectorization info and associate DR with the
    6997              :      new pattern statement instead of the original statement.  */
    6998            0 :   stmt_vec_info pattern_stmt_info = loop_vinfo->add_stmt (call);
    6999            0 :   loop_vinfo->move_dr (pattern_stmt_info, stmt_vinfo);
    7000              : 
    7001            0 :   *type_out = vectype;
    7002            0 :   return call;
    7003              : }
    7004              : 
    7005              : /* Return true if TYPE is a non-boolean integer type.  These are the types
    7006              :    that we want to consider for narrowing.  */
    7007              : 
    7008              : static bool
    7009     68517571 : vect_narrowable_type_p (tree type)
    7010              : {
    7011     68517571 :   return INTEGRAL_TYPE_P (type) && !VECT_SCALAR_BOOLEAN_TYPE_P (type);
    7012              : }
    7013              : 
    7014              : /* Return true if the operation given by CODE can be truncated to N bits
    7015              :    when only N bits of the output are needed.  This is only true if bit N+1
    7016              :    of the inputs has no effect on the low N bits of the result.  */
    7017              : 
    7018              : static bool
    7019     17103790 : vect_truncatable_operation_p (tree_code code)
    7020              : {
    7021     17103790 :   switch (code)
    7022              :     {
    7023              :     case NEGATE_EXPR:
    7024              :     case PLUS_EXPR:
    7025              :     case MINUS_EXPR:
    7026              :     case MULT_EXPR:
    7027              :     case BIT_NOT_EXPR:
    7028              :     case BIT_AND_EXPR:
    7029              :     case BIT_IOR_EXPR:
    7030              :     case BIT_XOR_EXPR:
    7031              :     case COND_EXPR:
    7032              :       return true;
    7033              : 
    7034      6737256 :     default:
    7035      6737256 :       return false;
    7036              :     }
    7037              : }
    7038              : 
    7039              : /* Record that STMT_INFO could be changed from operating on TYPE to
    7040              :    operating on a type with the precision and sign given by PRECISION
    7041              :    and SIGN respectively.  PRECISION is an arbitrary bit precision;
    7042              :    it might not be a whole number of bytes.  */
    7043              : 
    7044              : static void
    7045      2617084 : vect_set_operation_type (stmt_vec_info stmt_info, tree type,
    7046              :                          unsigned int precision, signop sign)
    7047              : {
    7048              :   /* Round the precision up to a whole number of bytes.  */
    7049      2617084 :   precision = vect_element_precision (precision);
    7050      2617084 :   if (precision < TYPE_PRECISION (type)
    7051      2617084 :       && (!stmt_info->operation_precision
    7052        41549 :           || stmt_info->operation_precision > precision))
    7053              :     {
    7054      1705145 :       stmt_info->operation_precision = precision;
    7055      1705145 :       stmt_info->operation_sign = sign;
    7056              :     }
    7057      2617084 : }
    7058              : 
    7059              : /* Record that STMT_INFO only requires MIN_INPUT_PRECISION from its
    7060              :    non-boolean inputs, all of which have type TYPE.  MIN_INPUT_PRECISION
    7061              :    is an arbitrary bit precision; it might not be a whole number of bytes.  */
    7062              : 
    7063              : static void
    7064     12265555 : vect_set_min_input_precision (stmt_vec_info stmt_info, tree type,
    7065              :                               unsigned int min_input_precision)
    7066              : {
    7067              :   /* This operation in isolation only requires the inputs to have
    7068              :      MIN_INPUT_PRECISION of precision,  However, that doesn't mean
    7069              :      that MIN_INPUT_PRECISION is a natural precision for the chain
    7070              :      as a whole.  E.g. consider something like:
    7071              : 
    7072              :          unsigned short *x, *y;
    7073              :          *y = ((*x & 0xf0) >> 4) | (*y << 4);
    7074              : 
    7075              :      The right shift can be done on unsigned chars, and only requires the
    7076              :      result of "*x & 0xf0" to be done on unsigned chars.  But taking that
    7077              :      approach would mean turning a natural chain of single-vector unsigned
    7078              :      short operations into one that truncates "*x" and then extends
    7079              :      "(*x & 0xf0) >> 4", with two vectors for each unsigned short
    7080              :      operation and one vector for each unsigned char operation.
    7081              :      This would be a significant pessimization.
    7082              : 
    7083              :      Instead only propagate the maximum of this precision and the precision
    7084              :      required by the users of the result.  This means that we don't pessimize
    7085              :      the case above but continue to optimize things like:
    7086              : 
    7087              :          unsigned char *y;
    7088              :          unsigned short *x;
    7089              :          *y = ((*x & 0xf0) >> 4) | (*y << 4);
    7090              : 
    7091              :      Here we would truncate two vectors of *x to a single vector of
    7092              :      unsigned chars and use single-vector unsigned char operations for
    7093              :      everything else, rather than doing two unsigned short copies of
    7094              :      "(*x & 0xf0) >> 4" and then truncating the result.  */
    7095     12265555 :   min_input_precision = MAX (min_input_precision,
    7096              :                              stmt_info->min_output_precision);
    7097              : 
    7098     12265555 :   if (min_input_precision < TYPE_PRECISION (type)
    7099     12265555 :       && (!stmt_info->min_input_precision
    7100        66220 :           || stmt_info->min_input_precision > min_input_precision))
    7101       640192 :     stmt_info->min_input_precision = min_input_precision;
    7102     12265555 : }
    7103              : 
    7104              : /* Subroutine of vect_determine_min_output_precision.  Return true if
    7105              :    we can calculate a reduced number of output bits for STMT_INFO,
    7106              :    whose result is LHS.  */
    7107              : 
    7108              : static bool
    7109     15709240 : vect_determine_min_output_precision_1 (vec_info *vinfo,
    7110              :                                        stmt_vec_info stmt_info, tree lhs)
    7111              : {
    7112              :   /* Take the maximum precision required by users of the result.  */
    7113     15709240 :   unsigned int precision = 0;
    7114     15709240 :   imm_use_iterator iter;
    7115     15709240 :   use_operand_p use;
    7116     16753994 :   FOR_EACH_IMM_USE_FAST (use, iter, lhs)
    7117              :     {
    7118     16448489 :       gimple *use_stmt = USE_STMT (use);
    7119     16448489 :       if (is_gimple_debug (use_stmt))
    7120       735054 :         continue;
    7121     15713435 :       stmt_vec_info use_stmt_info = vinfo->lookup_stmt (use_stmt);
    7122     15713435 :       if (!use_stmt_info || !use_stmt_info->min_input_precision)
    7123              :         return false;
    7124              :       /* The input precision recorded for COND_EXPRs applies only to the
    7125              :          "then" and "else" values.  */
    7126       310344 :       gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
    7127       277561 :       if (assign
    7128       277561 :           && gimple_assign_rhs_code (assign) == COND_EXPR
    7129          644 :           && use->use != gimple_assign_rhs2_ptr (assign)
    7130          644 :           && use->use != gimple_assign_rhs3_ptr (assign))
    7131              :         return false;
    7132      1046478 :       precision = MAX (precision, use_stmt_info->min_input_precision);
    7133     15403735 :     }
    7134              : 
    7135       305505 :   if (dump_enabled_p ())
    7136         6098 :     dump_printf_loc (MSG_NOTE, vect_location,
    7137              :                      "only the low %d bits of %T are significant\n",
    7138              :                      precision, lhs);
    7139       305505 :   stmt_info->min_output_precision = precision;
    7140       305505 :   return true;
    7141              : }
    7142              : 
    7143              : /* Calculate min_output_precision for STMT_INFO.  */
    7144              : 
    7145              : static void
    7146     40232395 : vect_determine_min_output_precision (vec_info *vinfo, stmt_vec_info stmt_info)
    7147              : {
    7148              :   /* We're only interested in statements with a narrowable result.  */
    7149     40232395 :   tree lhs = gimple_get_lhs (stmt_info->stmt);
    7150     40232395 :   if (!lhs
    7151     31888059 :       || TREE_CODE (lhs) != SSA_NAME
    7152     67194861 :       || !vect_narrowable_type_p (TREE_TYPE (lhs)))
    7153              :     return;
    7154              : 
    7155     15709240 :   if (!vect_determine_min_output_precision_1 (vinfo, stmt_info, lhs))
    7156     15403735 :     stmt_info->min_output_precision = TYPE_PRECISION (TREE_TYPE (lhs));
    7157              : }
    7158              : 
    7159              : /* Use range information to decide whether STMT (described by STMT_INFO)
    7160              :    could be done in a narrower type.  This is effectively a forward
    7161              :    propagation, since it uses context-independent information that applies
    7162              :    to all users of an SSA name.  */
    7163              : 
    7164              : static void
    7165     23093102 : vect_determine_precisions_from_range (stmt_vec_info stmt_info, gassign *stmt)
    7166              : {
    7167     23093102 :   tree lhs = gimple_assign_lhs (stmt);
    7168     23093102 :   if (!lhs || TREE_CODE (lhs) != SSA_NAME)
    7169     20596814 :     return;
    7170              : 
    7171     18462003 :   tree type = TREE_TYPE (lhs);
    7172     18462003 :   if (!vect_narrowable_type_p (type))
    7173              :     return;
    7174              : 
    7175              :   /* First see whether we have any useful range information for the result.  */
    7176     12170347 :   unsigned int precision = TYPE_PRECISION (type);
    7177     12170347 :   signop sign = TYPE_SIGN (type);
    7178     12170347 :   wide_int min_value, max_value;
    7179     12170347 :   if (!vect_get_range_info (lhs, &min_value, &max_value))
    7180              :     return;
    7181              : 
    7182      5856590 :   tree_code code = gimple_assign_rhs_code (stmt);
    7183      5856590 :   unsigned int nops = gimple_num_ops (stmt);
    7184              : 
    7185      5856590 :   if (!vect_truncatable_operation_p (code))
    7186              :     {
    7187              :       /* Handle operations that can be computed in type T if all inputs
    7188              :          and outputs can be represented in type T.  Also handle left and
    7189              :          right shifts, where (in addition) the maximum shift amount must
    7190              :          be less than the number of bits in T.  */
    7191      2173203 :       bool is_shift;
    7192      2173203 :       switch (code)
    7193              :         {
    7194              :         case LSHIFT_EXPR:
    7195              :         case RSHIFT_EXPR:
    7196              :           is_shift = true;
    7197              :           break;
    7198              : 
    7199       312682 :         case ABS_EXPR:
    7200       312682 :         case MIN_EXPR:
    7201       312682 :         case MAX_EXPR:
    7202       312682 :         case TRUNC_DIV_EXPR:
    7203       312682 :         case CEIL_DIV_EXPR:
    7204       312682 :         case FLOOR_DIV_EXPR:
    7205       312682 :         case ROUND_DIV_EXPR:
    7206       312682 :         case EXACT_DIV_EXPR:
    7207              :           /* Modulus is excluded because it is typically calculated by doing
    7208              :              a division, for which minimum signed / -1 isn't representable in
    7209              :              the original signed type.  We could take the division range into
    7210              :              account instead, if handling modulus ever becomes important.  */
    7211       312682 :           is_shift = false;
    7212       312682 :           break;
    7213              : 
    7214              :         default:
    7215              :           return;
    7216              :         }
    7217      1429445 :       for (unsigned int i = 1; i < nops; ++i)
    7218              :         {
    7219      1111510 :           tree op = gimple_op (stmt, i);
    7220      1111510 :           wide_int op_min_value, op_max_value;
    7221      1111510 :           if (TREE_CODE (op) == INTEGER_CST)
    7222              :             {
    7223       325195 :               unsigned int op_precision = TYPE_PRECISION (TREE_TYPE (op));
    7224       325195 :               op_min_value = op_max_value = wi::to_wide (op, op_precision);
    7225              :             }
    7226       786315 :           else if (TREE_CODE (op) == SSA_NAME)
    7227              :             {
    7228       786315 :               if (!vect_get_range_info (op, &op_min_value, &op_max_value))
    7229              :                 return;
    7230              :             }
    7231              :           else
    7232              :             return;
    7233              : 
    7234       721206 :           if (is_shift && i == 2)
    7235              :             {
    7236              :               /* There needs to be one more bit than the maximum shift amount.
    7237              : 
    7238              :                  If the maximum shift amount is already 1 less than PRECISION
    7239              :                  then we can't narrow the shift further.  Dealing with that
    7240              :                  case first ensures that we can safely use an unsigned range
    7241              :                  below.
    7242              : 
    7243              :                  op_min_value isn't relevant, since shifts by negative amounts
    7244              :                  are UB.  */
    7245       208971 :               if (wi::geu_p (op_max_value, precision - 1))
    7246              :                 return;
    7247       183569 :               unsigned int min_bits = op_max_value.to_uhwi () + 1;
    7248              : 
    7249              :               /* As explained below, we can convert a signed shift into an
    7250              :                  unsigned shift if the sign bit is always clear.  At this
    7251              :                  point we've already processed the ranges of the output and
    7252              :                  the first input.  */
    7253       183569 :               auto op_sign = sign;
    7254       183569 :               if (sign == SIGNED && !wi::neg_p (min_value))
    7255              :                 op_sign = UNSIGNED;
    7256       367138 :               op_min_value = wide_int::from (wi::min_value (min_bits, op_sign),
    7257       183569 :                                              precision, op_sign);
    7258       367138 :               op_max_value = wide_int::from (wi::max_value (min_bits, op_sign),
    7259       183569 :                                              precision, op_sign);
    7260              :             }
    7261       695804 :           min_value = wi::min (min_value, op_min_value, sign);
    7262       695804 :           max_value = wi::max (max_value, op_max_value, sign);
    7263      1111510 :         }
    7264              :     }
    7265              : 
    7266              :   /* Try to switch signed types for unsigned types if we can.
    7267              :      This is better for two reasons.  First, unsigned ops tend
    7268              :      to be cheaper than signed ops.  Second, it means that we can
    7269              :      handle things like:
    7270              : 
    7271              :         signed char c;
    7272              :         int res = (int) c & 0xff00; // range [0x0000, 0xff00]
    7273              : 
    7274              :      as:
    7275              : 
    7276              :         signed char c;
    7277              :         unsigned short res_1 = (unsigned short) c & 0xff00;
    7278              :         int res = (int) res_1;
    7279              : 
    7280              :      where the intermediate result res_1 has unsigned rather than
    7281              :      signed type.  */
    7282      4001322 :   if (sign == SIGNED && !wi::neg_p (min_value))
    7283              :     sign = UNSIGNED;
    7284              : 
    7285              :   /* See what precision is required for MIN_VALUE and MAX_VALUE.  */
    7286      4001322 :   unsigned int precision1 = wi::min_precision (min_value, sign);
    7287      4001322 :   unsigned int precision2 = wi::min_precision (max_value, sign);
    7288      4001322 :   unsigned int value_precision = MAX (precision1, precision2);
    7289      4001322 :   if (value_precision >= precision)
    7290              :     return;
    7291              : 
    7292      2496288 :   if (dump_enabled_p ())
    7293       118176 :     dump_printf_loc (MSG_NOTE, vect_location, "can narrow to %s:%d"
    7294              :                      " without loss of precision: %G",
    7295              :                      sign == SIGNED ? "signed" : "unsigned",
    7296              :                      value_precision, (gimple *) stmt);
    7297              : 
    7298      2496288 :   vect_set_operation_type (stmt_info, type, value_precision, sign);
    7299      2496288 :   vect_set_min_input_precision (stmt_info, type, value_precision);
    7300     12170347 : }
    7301              : 
    7302              : /* Use information about the users of STMT's result to decide whether
    7303              :    STMT (described by STMT_INFO) could be done in a narrower type.
    7304              :    This is effectively a backward propagation.  */
    7305              : 
    7306              : static void
    7307     23093102 : vect_determine_precisions_from_users (stmt_vec_info stmt_info, gassign *stmt)
    7308              : {
    7309     23093102 :   tree_code code = gimple_assign_rhs_code (stmt);
    7310     23093102 :   unsigned int opno = (code == COND_EXPR ? 2 : 1);
    7311     23093102 :   tree type = TREE_TYPE (gimple_op (stmt, opno));
    7312     23093102 :   if (!vect_narrowable_type_p (type))
    7313     13323835 :     return;
    7314              : 
    7315     14368853 :   unsigned int precision = TYPE_PRECISION (type);
    7316     14368853 :   unsigned int operation_precision, min_input_precision;
    7317     14368853 :   switch (code)
    7318              :     {
    7319      2628122 :     CASE_CONVERT:
    7320              :       /* Only the bits that contribute to the output matter.  Don't change
    7321              :          the precision of the operation itself.  */
    7322      2628122 :       operation_precision = precision;
    7323      2628122 :       min_input_precision = stmt_info->min_output_precision;
    7324      2628122 :       break;
    7325              : 
    7326       493531 :     case LSHIFT_EXPR:
    7327       493531 :     case RSHIFT_EXPR:
    7328       493531 :       {
    7329       493531 :         tree shift = gimple_assign_rhs2 (stmt);
    7330       493531 :         unsigned int min_const_shift, max_const_shift;
    7331       493531 :         wide_int min_shift, max_shift;
    7332       493531 :         if (TREE_CODE (shift) == SSA_NAME
    7333       108655 :             && vect_get_range_info (shift, &min_shift, &max_shift)
    7334        83888 :             && wi::ge_p (min_shift, 0, TYPE_SIGN (TREE_TYPE (shift)))
    7335       574637 :             && wi::lt_p (max_shift, TYPE_PRECISION (type),
    7336        81106 :                          TYPE_SIGN (TREE_TYPE (shift))))
    7337              :           {
    7338        73230 :             min_const_shift = min_shift.to_uhwi ();
    7339        73230 :             max_const_shift = max_shift.to_uhwi ();
    7340              :           }
    7341       420301 :         else if (TREE_CODE (shift) == INTEGER_CST
    7342       805177 :                  && wi::ltu_p (wi::to_widest (shift), precision))
    7343       384768 :           min_const_shift = max_const_shift = TREE_INT_CST_LOW (shift);
    7344              :         else
    7345        35533 :           return;
    7346       457998 :         if (code == LSHIFT_EXPR)
    7347              :           {
    7348              :             /* Avoid creating an undefined shift.
    7349              : 
    7350              :                ??? We could instead use min_output_precision as-is and
    7351              :                optimize out-of-range shifts to zero.  However, only
    7352              :                degenerate testcases shift away all their useful input data,
    7353              :                and it isn't natural to drop input operations in the middle
    7354              :                of vectorization.  This sort of thing should really be
    7355              :                handled before vectorization.  */
    7356       111196 :             operation_precision = MAX (stmt_info->min_output_precision,
    7357              :                                        max_const_shift + 1);
    7358              :             /* We need CONST_SHIFT fewer bits of the input.  */
    7359       111196 :             min_input_precision = (MAX (operation_precision, max_const_shift)
    7360              :                                    - min_const_shift);
    7361              :           }
    7362              :         else
    7363              :           {
    7364              :             /* We need CONST_SHIFT extra bits to do the operation.  */
    7365       346802 :             operation_precision = (stmt_info->min_output_precision
    7366              :                                    + max_const_shift);
    7367       346802 :             min_input_precision = operation_precision;
    7368              :           }
    7369       457998 :         break;
    7370       493531 :       }
    7371              : 
    7372     11247200 :     default:
    7373     11247200 :       if (vect_truncatable_operation_p (code))
    7374              :         {
    7375              :           /* Input bit N has no effect on output bits N-1 and lower.  */
    7376      6683147 :           operation_precision = stmt_info->min_output_precision;
    7377      6683147 :           min_input_precision = operation_precision;
    7378      6683147 :           break;
    7379              :         }
    7380              :       return;
    7381              :     }
    7382              : 
    7383      9769267 :   if (operation_precision < precision)
    7384              :     {
    7385       120796 :       if (dump_enabled_p ())
    7386         2859 :         dump_printf_loc (MSG_NOTE, vect_location, "can narrow to %s:%d"
    7387              :                          " without affecting users: %G",
    7388         2859 :                          TYPE_UNSIGNED (type) ? "unsigned" : "signed",
    7389              :                          operation_precision, (gimple *) stmt);
    7390       241592 :       vect_set_operation_type (stmt_info, type, operation_precision,
    7391       120796 :                                TYPE_SIGN (type));
    7392              :     }
    7393      9769267 :   vect_set_min_input_precision (stmt_info, type, min_input_precision);
    7394              : }
    7395              : 
    7396              : /* Return true if the statement described by STMT_INFO sets a boolean
    7397              :    SSA_NAME and if we know how to vectorize this kind of statement using
    7398              :    vector mask types.  */
    7399              : 
    7400              : static bool
    7401     41341495 : possible_vector_mask_operation_p (stmt_vec_info stmt_info)
    7402              : {
    7403     41341495 :   tree lhs = gimple_get_lhs (stmt_info->stmt);
    7404     41341495 :   tree_code code = ERROR_MARK;
    7405     41341495 :   gassign *assign = NULL;
    7406     41341495 :   gcond *cond = NULL;
    7407              : 
    7408     41341495 :   if ((assign = dyn_cast <gassign *> (stmt_info->stmt)))
    7409     23885712 :     code = gimple_assign_rhs_code (assign);
    7410     17455783 :   else if ((cond = dyn_cast <gcond *> (stmt_info->stmt)))
    7411              :     {
    7412      5550885 :       lhs = gimple_cond_lhs (cond);
    7413      5550885 :       code = gimple_cond_code (cond);
    7414              :     }
    7415              : 
    7416     41341495 :   if (!lhs
    7417     38452582 :       || TREE_CODE (lhs) != SSA_NAME
    7418     74829323 :       || !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)))
    7419              :     return false;
    7420              : 
    7421      2255527 :   if (code != ERROR_MARK)
    7422              :     {
    7423      1993140 :       switch (code)
    7424              :         {
    7425              :         CASE_CONVERT:
    7426              :         case SSA_NAME:
    7427              :         case BIT_NOT_EXPR:
    7428              :         case BIT_IOR_EXPR:
    7429              :         case BIT_XOR_EXPR:
    7430              :         case BIT_AND_EXPR:
    7431              :           return true;
    7432              : 
    7433      1563254 :         default:
    7434      1563254 :           return TREE_CODE_CLASS (code) == tcc_comparison;
    7435              :         }
    7436              :     }
    7437       262387 :   else if (is_a <gphi *> (stmt_info->stmt))
    7438       147709 :     return true;
    7439              :   return false;
    7440              : }
    7441              : 
    7442              : /* If STMT_INFO sets a boolean SSA_NAME, see whether we should use
    7443              :    a vector mask type instead of a normal vector type.  Record the
    7444              :    result in STMT_INFO->mask_precision.  Returns true when the
    7445              :    precision changed.  */
    7446              : 
    7447              : static bool
    7448     41341495 : vect_determine_mask_precision (vec_info *vinfo, stmt_vec_info stmt_info)
    7449              : {
    7450     41341495 :   if (!possible_vector_mask_operation_p (stmt_info))
    7451              :     return false;
    7452              : 
    7453              :   /* If at least one boolean input uses a vector mask type,
    7454              :      pick the mask type with the narrowest elements.
    7455              : 
    7456              :      ??? This is the traditional behavior.  It should always produce
    7457              :      the smallest number of operations, but isn't necessarily the
    7458              :      optimal choice.  For example, if we have:
    7459              : 
    7460              :        a = b & c
    7461              : 
    7462              :      where:
    7463              : 
    7464              :        - the user of a wants it to have a mask type for 16-bit elements (M16)
    7465              :        - b also uses M16
    7466              :        - c uses a mask type for 8-bit elements (M8)
    7467              : 
    7468              :      then picking M8 gives:
    7469              : 
    7470              :        - 1 M16->M8 pack for b
    7471              :        - 1 M8 AND for a
    7472              :        - 2 M8->M16 unpacks for the user of a
    7473              : 
    7474              :      whereas picking M16 would have given:
    7475              : 
    7476              :        - 2 M8->M16 unpacks for c
    7477              :        - 2 M16 ANDs for a
    7478              : 
    7479              :      The number of operations are equal, but M16 would have given
    7480              :      a shorter dependency chain and allowed more ILP.  */
    7481      2087742 :   unsigned int precision = ~0U;
    7482      2087742 :   gimple *stmt = STMT_VINFO_STMT (stmt_info);
    7483              : 
    7484              :   /* If the statement compares two values that shouldn't use vector masks,
    7485              :      try comparing the values as normal scalars instead.  */
    7486      2087742 :   tree_code code = ERROR_MARK;
    7487      2087742 :   tree op0_type;
    7488      2087742 :   unsigned int nops = -1;
    7489      2087742 :   unsigned int ops_start = 0;
    7490              : 
    7491      2087742 :   if (gassign *assign = dyn_cast <gassign *> (stmt))
    7492              :     {
    7493      1362752 :       code = gimple_assign_rhs_code (assign);
    7494      1362752 :       op0_type = TREE_TYPE (gimple_assign_rhs1 (assign));
    7495      1362752 :       nops = gimple_num_ops (assign);
    7496      1362752 :       ops_start = 1;
    7497              :     }
    7498       724990 :   else if (gcond *cond = dyn_cast <gcond *> (stmt))
    7499              :     {
    7500       577281 :       code = gimple_cond_code (cond);
    7501       577281 :       op0_type = TREE_TYPE (gimple_cond_lhs (cond));
    7502       577281 :       nops = 2;
    7503       577281 :       ops_start = 0;
    7504              :     }
    7505              : 
    7506      1940033 :   if (code != ERROR_MARK)
    7507              :     {
    7508      5774953 :       for (unsigned int i = ops_start; i < nops; ++i)
    7509              :         {
    7510      3834920 :           tree rhs = gimple_op (stmt, i);
    7511      3834920 :           if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (rhs)))
    7512      1875925 :             continue;
    7513              : 
    7514      1958995 :           stmt_vec_info def_stmt_info = vinfo->lookup_def (rhs);
    7515      1958995 :           if (!def_stmt_info)
    7516              :             /* Don't let external or constant operands influence the choice.
    7517              :                We can convert them to whichever vector type we pick.  */
    7518       592218 :             continue;
    7519              : 
    7520      1366777 :           if (def_stmt_info->mask_precision)
    7521              :             {
    7522      1137929 :               if (precision > def_stmt_info->mask_precision)
    7523      3834920 :                 precision = def_stmt_info->mask_precision;
    7524              :             }
    7525              :         }
    7526              : 
    7527      1940033 :       if (precision == ~0U
    7528      1563537 :           && TREE_CODE_CLASS (code) == tcc_comparison)
    7529              :         {
    7530      1354109 :           scalar_mode mode;
    7531      1354109 :           tree vectype, mask_type;
    7532      1354109 :           if (is_a <scalar_mode> (TYPE_MODE (op0_type), &mode)
    7533              :               /* Do not allow this to set vinfo->vector_mode, this might
    7534              :                  disrupt the result for the next iteration.  */
    7535      1354109 :               && (vectype = get_related_vectype_for_scalar_type
    7536      1625356 :                                                 (vinfo->vector_mode, op0_type))
    7537      1183458 :               && (mask_type = truth_type_for (vectype))
    7538      1183458 :               && expand_vec_cmp_expr_p (vectype, mask_type, code))
    7539      1824422 :             precision = GET_MODE_BITSIZE (mode);
    7540              :         }
    7541              :     }
    7542              :   else
    7543              :     {
    7544       147709 :       gphi *phi = as_a <gphi *> (stmt_info->stmt);
    7545       759740 :       for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
    7546              :         {
    7547       464322 :           tree rhs = gimple_phi_arg_def (phi, i);
    7548              : 
    7549       464322 :           stmt_vec_info def_stmt_info = vinfo->lookup_def (rhs);
    7550       464322 :           if (!def_stmt_info)
    7551              :             /* Don't let external or constant operands influence the choice.
    7552              :                We can convert them to whichever vector type we pick.  */
    7553       305846 :             continue;
    7554              : 
    7555       158476 :           if (def_stmt_info->mask_precision)
    7556              :             {
    7557       132718 :               if (precision > def_stmt_info->mask_precision)
    7558       464322 :                 precision = def_stmt_info->mask_precision;
    7559              :             }
    7560              :         }
    7561              :     }
    7562              : 
    7563      2087742 :   if (stmt_info->mask_precision != precision)
    7564              :     {
    7565      1957358 :       if (dump_enabled_p ())
    7566              :         {
    7567         8262 :           if (precision == ~0U)
    7568         1825 :             dump_printf_loc (MSG_NOTE, vect_location,
    7569              :                              "using normal nonmask vectors for %G",
    7570              :                              stmt_info->stmt);
    7571              :           else
    7572         6437 :             dump_printf_loc (MSG_NOTE, vect_location,
    7573              :                              "using boolean precision %d for %G",
    7574              :                              precision, stmt_info->stmt);
    7575              :         }
    7576              : 
    7577              :       /* ???  We'd like to assert stmt_info->mask_precision == 0
    7578              :          || stmt_info->mask_precision > precision, thus that we only
    7579              :          decrease mask precisions throughout iteration, but the
    7580              :          tcc_comparison handling above means for comparisons of bools
    7581              :          we start with 8 but might increase in case the bools get mask
    7582              :          precision on their own.  */
    7583      1957358 :       stmt_info->mask_precision = precision;
    7584      1957358 :       return true;
    7585              :     }
    7586              :   return false;
    7587              : }
    7588              : 
    7589              : /* Handle vect_determine_precisions for STMT_INFO, given that we
    7590              :    have already done so for the users of its result.  */
    7591              : 
    7592              : void
    7593     40232395 : vect_determine_stmt_precisions (vec_info *vinfo, stmt_vec_info stmt_info)
    7594              : {
    7595     40232395 :   vect_determine_min_output_precision (vinfo, stmt_info);
    7596     40232395 :   if (gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt))
    7597              :     {
    7598     23093102 :       vect_determine_precisions_from_range (stmt_info, stmt);
    7599     23093102 :       vect_determine_precisions_from_users (stmt_info, stmt);
    7600              :     }
    7601     40232395 : }
    7602              : 
    7603              : /* Walk backwards through the vectorizable region to determine the
    7604              :    values of these fields:
    7605              : 
    7606              :    - min_output_precision
    7607              :    - min_input_precision
    7608              :    - operation_precision
    7609              :    - operation_sign.  */
    7610              : 
    7611              : void
    7612      1142237 : vect_determine_precisions (vec_info *vinfo)
    7613              : {
    7614      1142237 :   basic_block *bbs = vinfo->bbs;
    7615      1142237 :   unsigned int nbbs = vinfo->nbbs;
    7616              : 
    7617      1163054 :   DUMP_VECT_SCOPE ("vect_determine_precisions");
    7618              : 
    7619              :   /* For mask precisions we have to iterate since otherwise we do not
    7620              :      get reduction PHI precision correct.  For now do this only for
    7621              :      loop vectorization.  */
    7622      1213984 :   bool changed;
    7623      1213984 :   do
    7624              :     {
    7625      1213984 :       changed = false;
    7626     13507012 :       for (unsigned int i = 0; i < nbbs; i++)
    7627              :         {
    7628     12293028 :           basic_block bb = bbs[i];
    7629     12293028 :           for (auto gsi = gsi_start_phis (bb);
    7630     19927835 :                !gsi_end_p (gsi); gsi_next (&gsi))
    7631              :             {
    7632      7634807 :               stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi.phi ());
    7633      7634807 :               if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
    7634      7443784 :                 changed |= vect_determine_mask_precision (vinfo, stmt_info);
    7635              :             }
    7636    131625798 :           for (auto gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
    7637              :             {
    7638    107039742 :               stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (gsi));
    7639    107039742 :               if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
    7640     33897711 :                 changed |= vect_determine_mask_precision (vinfo, stmt_info);
    7641              :             }
    7642              :         }
    7643              :     }
    7644      2356221 :   while (changed && is_a <loop_vec_info> (vinfo));
    7645              : 
    7646     13270236 :   for (unsigned int i = 0; i < nbbs; i++)
    7647              :     {
    7648     12127999 :       basic_block bb = bbs[nbbs - i - 1];
    7649    139107030 :       for (auto gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
    7650              :         {
    7651    105354943 :           stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (gsi));
    7652    105354943 :           if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
    7653     32996364 :             vect_determine_stmt_precisions (vinfo, stmt_info);
    7654              :         }
    7655     19555053 :       for (auto gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
    7656              :         {
    7657      7427054 :           stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi.phi ());
    7658      7427054 :           if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
    7659      7236031 :             vect_determine_stmt_precisions (vinfo, stmt_info);
    7660              :         }
    7661              :     }
    7662      1142237 : }
    7663              : 
    7664              : typedef gimple *(*vect_recog_func_ptr) (vec_info *, stmt_vec_info, tree *);
    7665              : 
    7666              : struct vect_recog_func
    7667              : {
    7668              :   vect_recog_func_ptr fn;
    7669              :   const char *name;
    7670              : };
    7671              : 
    7672              : /* Note that ordering matters - the first pattern matching on a stmt is
    7673              :    taken which means usually the more complex one needs to precede the
    7674              :    less comples onex (widen_sum only after dot_prod or sad for example).  */
    7675              : static vect_recog_func vect_vect_recog_func_ptrs[] = {
    7676              :   { vect_recog_bitfield_ref_pattern, "bitfield_ref" },
    7677              :   { vect_recog_bit_insert_pattern, "bit_insert" },
    7678              :   { vect_recog_abd_pattern, "abd" },
    7679              :   { vect_recog_over_widening_pattern, "over_widening" },
    7680              :   /* Must come after over_widening, which narrows the shift as much as
    7681              :      possible beforehand.  */
    7682              :   { vect_recog_average_pattern, "average" },
    7683              :   { vect_recog_cond_expr_convert_pattern, "cond_expr_convert" },
    7684              :   { vect_recog_mulhs_pattern, "mult_high" },
    7685              :   { vect_recog_cast_forwprop_pattern, "cast_forwprop" },
    7686              :   { vect_recog_widen_mult_pattern, "widen_mult" },
    7687              :   { vect_recog_dot_prod_pattern, "dot_prod" },
    7688              :   { vect_recog_sad_pattern, "sad" },
    7689              :   { vect_recog_widen_sum_pattern, "widen_sum" },
    7690              :   { vect_recog_pow_pattern, "pow" },
    7691              :   { vect_recog_popcount_clz_ctz_ffs_pattern, "popcount_clz_ctz_ffs" },
    7692              :   { vect_recog_ctz_ffs_pattern, "ctz_ffs" },
    7693              :   { vect_recog_widen_shift_pattern, "widen_shift" },
    7694              :   { vect_recog_rotate_pattern, "rotate" },
    7695              :   { vect_recog_vector_vector_shift_pattern, "vector_vector_shift" },
    7696              :   { vect_recog_divmod_pattern, "divmod" },
    7697              :   { vect_recog_mod_var_pattern, "modvar" },
    7698              :   { vect_recog_mult_pattern, "mult" },
    7699              :   { vect_recog_sat_add_pattern, "sat_add" },
    7700              :   { vect_recog_sat_sub_pattern, "sat_sub" },
    7701              :   { vect_recog_sat_trunc_pattern, "sat_trunc" },
    7702              :   { vect_recog_gcond_pattern, "gcond" },
    7703              :   { vect_recog_bool_pattern, "bool" },
    7704              :   /* This must come before mask conversion, and includes the parts
    7705              :      of mask conversion that are needed for gather and scatter
    7706              :      internal functions.  */
    7707              :   { vect_recog_gather_scatter_pattern, "gather_scatter" },
    7708              :   { vect_recog_cond_store_pattern, "cond_store" },
    7709              :   { vect_recog_mask_conversion_pattern, "mask_conversion" },
    7710              :   { vect_recog_widen_plus_pattern, "widen_plus" },
    7711              :   { vect_recog_widen_minus_pattern, "widen_minus" },
    7712              :   { vect_recog_widen_abd_pattern, "widen_abd" },
    7713              :   /* These must come after the double widening ones.  */
    7714              : };
    7715              : 
    7716              : /* Mark statements that are involved in a pattern.  */
    7717              : 
    7718              : void
    7719      1104520 : vect_mark_pattern_stmts (vec_info *vinfo,
    7720              :                          stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
    7721              :                          tree pattern_vectype)
    7722              : {
    7723      1104520 :   stmt_vec_info orig_stmt_info_saved = orig_stmt_info;
    7724      1104520 :   gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
    7725              : 
    7726      1104520 :   gimple *orig_pattern_stmt = NULL;
    7727      1104520 :   if (is_pattern_stmt_p (orig_stmt_info))
    7728              :     {
    7729              :       /* We're replacing a statement in an existing pattern definition
    7730              :          sequence.  */
    7731        11894 :       orig_pattern_stmt = orig_stmt_info->stmt;
    7732        11894 :       if (dump_enabled_p ())
    7733          993 :         dump_printf_loc (MSG_NOTE, vect_location,
    7734              :                          "replacing earlier pattern %G", orig_pattern_stmt);
    7735              : 
    7736              :       /* To keep the book-keeping simple, just swap the lhs of the
    7737              :          old and new statements, so that the old one has a valid but
    7738              :          unused lhs.  */
    7739        11894 :       tree old_lhs = gimple_get_lhs (orig_pattern_stmt);
    7740        11894 :       gimple_set_lhs (orig_pattern_stmt, gimple_get_lhs (pattern_stmt));
    7741        11894 :       gimple_set_lhs (pattern_stmt, old_lhs);
    7742              : 
    7743        11894 :       if (dump_enabled_p ())
    7744          993 :         dump_printf_loc (MSG_NOTE, vect_location, "with %G", pattern_stmt);
    7745              : 
    7746              :       /* Switch to the statement that ORIG replaces.  */
    7747        11894 :       orig_stmt_info = STMT_VINFO_RELATED_STMT (orig_stmt_info);
    7748              : 
    7749              :       /* We shouldn't be replacing the main pattern statement.  */
    7750        11894 :       gcc_assert (STMT_VINFO_RELATED_STMT (orig_stmt_info)->stmt
    7751              :                   != orig_pattern_stmt);
    7752              :     }
    7753              : 
    7754      1104520 :   if (def_seq)
    7755      2187740 :     for (gimple_stmt_iterator si = gsi_start (def_seq);
    7756      2187740 :          !gsi_end_p (si); gsi_next (&si))
    7757              :       {
    7758      1281352 :         if (dump_enabled_p ())
    7759        28510 :           dump_printf_loc (MSG_NOTE, vect_location,
    7760              :                            "extra pattern stmt: %G", gsi_stmt (si));
    7761      1281352 :         stmt_vec_info pattern_stmt_info
    7762      1281352 :           = vect_init_pattern_stmt (vinfo, gsi_stmt (si),
    7763              :                                     orig_stmt_info, pattern_vectype);
    7764              :         /* Stmts in the def sequence are not vectorizable cycle or
    7765              :            induction defs, instead they should all be vect_internal_def
    7766              :            feeding the main pattern stmt which retains this def type.  */
    7767      1281352 :         STMT_VINFO_DEF_TYPE (pattern_stmt_info) = vect_internal_def;
    7768              :       }
    7769              : 
    7770      1104520 :   if (orig_pattern_stmt)
    7771              :     {
    7772        11894 :       vect_init_pattern_stmt (vinfo, pattern_stmt,
    7773              :                               orig_stmt_info, pattern_vectype);
    7774              : 
    7775              :       /* Insert all the new pattern statements before the original one.  */
    7776        11894 :       gimple_seq *orig_def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
    7777        11894 :       gimple_stmt_iterator gsi = gsi_for_stmt (orig_pattern_stmt,
    7778              :                                                orig_def_seq);
    7779        11894 :       gsi_insert_seq_before_without_update (&gsi, def_seq, GSI_SAME_STMT);
    7780        11894 :       gsi_insert_before_without_update (&gsi, pattern_stmt, GSI_SAME_STMT);
    7781              : 
    7782              :       /* Remove the pattern statement that this new pattern replaces.  */
    7783        11894 :       gsi_remove (&gsi, false);
    7784              :     }
    7785              :   else
    7786      1092626 :     vect_set_pattern_stmt (vinfo,
    7787              :                            pattern_stmt, orig_stmt_info, pattern_vectype);
    7788              : 
    7789              :   /* For any conditionals mark them as vect_condition_def.  */
    7790      1104520 :   if (is_a <gcond *> (pattern_stmt))
    7791       390721 :     STMT_VINFO_DEF_TYPE (STMT_VINFO_RELATED_STMT (orig_stmt_info)) = vect_condition_def;
    7792              : 
    7793              :   /* Transfer reduction path info to the pattern.  */
    7794      1104520 :   if (STMT_VINFO_REDUC_IDX (orig_stmt_info_saved) != -1)
    7795              :     {
    7796        16317 :       gimple_match_op op;
    7797        16317 :       if (!gimple_extract_op (orig_stmt_info_saved->stmt, &op))
    7798            0 :         gcc_unreachable ();
    7799        16317 :       tree lookfor = op.ops[STMT_VINFO_REDUC_IDX (orig_stmt_info)];
    7800              :       /* Search the pattern def sequence and the main pattern stmt.  Note
    7801              :          we may have inserted all into a containing pattern def sequence
    7802              :          so the following is a bit awkward.  */
    7803        16317 :       gimple_stmt_iterator si;
    7804        16317 :       gimple *s;
    7805        16317 :       if (def_seq)
    7806              :         {
    7807        15110 :           si = gsi_start (def_seq);
    7808        15110 :           s = gsi_stmt (si);
    7809        15110 :           gsi_next (&si);
    7810              :         }
    7811              :       else
    7812              :         {
    7813              :           si = gsi_none ();
    7814        16317 :           s = pattern_stmt;
    7815              :         }
    7816        34432 :       do
    7817              :         {
    7818        34432 :           bool found = false;
    7819        34432 :           if (gimple_extract_op (s, &op))
    7820              :             {
    7821        84038 :               for (unsigned i = 0; i < op.num_ops; ++i)
    7822        65917 :                 if (op.ops[i] == lookfor)
    7823              :                   {
    7824        16311 :                     STMT_VINFO_REDUC_IDX (vinfo->lookup_stmt (s)) = i;
    7825        16311 :                     lookfor = gimple_get_lhs (s);
    7826        16311 :                     found = true;
    7827        16311 :                     break;
    7828              :                   }
    7829              :               /* Try harder to find a mid-entry into an earlier pattern
    7830              :                  sequence.  Likewise an entry to a stmt skipping a conversion
    7831              :                  on an input.  This means that the initial 'lookfor' was
    7832              :                  bogus.  */
    7833        16311 :               if (!found)
    7834              :                 {
    7835        39436 :                   for (unsigned i = 0; i < op.num_ops; ++i)
    7836        21321 :                     if (TREE_CODE (op.ops[i]) == SSA_NAME)
    7837        18121 :                       if (auto def = vinfo->lookup_def (op.ops[i]))
    7838        17927 :                         if (vect_is_reduction (def)
    7839        17927 :                             || (is_a <gphi *> (def->stmt)
    7840            6 :                                 && STMT_VINFO_REDUC_DEF (def) != NULL))
    7841              :                           {
    7842            6 :                             STMT_VINFO_REDUC_IDX (vinfo->lookup_stmt (s)) = i;
    7843            6 :                             lookfor = gimple_get_lhs (s);
    7844            6 :                             found = true;
    7845            6 :                             break;
    7846              :                           }
    7847              :                 }
    7848              :             }
    7849        34432 :           if (s == pattern_stmt)
    7850              :             {
    7851        16317 :               if (!found && dump_enabled_p ())
    7852            0 :                 dump_printf_loc (MSG_NOTE, vect_location,
    7853              :                                  "failed to update reduction index.\n");
    7854        16317 :               break;
    7855              :             }
    7856        18115 :           if (gsi_end_p (si))
    7857              :             s = pattern_stmt;
    7858              :           else
    7859              :             {
    7860         3005 :               s = gsi_stmt (si);
    7861         3005 :               if (s == pattern_stmt)
    7862              :                 /* Found the end inside a bigger pattern def seq.  */
    7863              :                 si = gsi_none ();
    7864              :               else
    7865         3005 :                 gsi_next (&si);
    7866              :             }
    7867              :         } while (1);
    7868              :     }
    7869      1104520 : }
    7870              : 
    7871              : /* Function vect_pattern_recog_1
    7872              : 
    7873              :    Input:
    7874              :    PATTERN_RECOG_FUNC: A pointer to a function that detects a certain
    7875              :         computation pattern.
    7876              :    STMT_INFO: A stmt from which the pattern search should start.
    7877              : 
    7878              :    If PATTERN_RECOG_FUNC successfully detected the pattern, it creates
    7879              :    a sequence of statements that has the same functionality and can be
    7880              :    used to replace STMT_INFO.  It returns the last statement in the sequence
    7881              :    and adds any earlier statements to STMT_INFO's STMT_VINFO_PATTERN_DEF_SEQ.
    7882              :    PATTERN_RECOG_FUNC also sets *TYPE_OUT to the vector type of the final
    7883              :    statement, having first checked that the target supports the new operation
    7884              :    in that type.
    7885              : 
    7886              :    This function also does some bookkeeping, as explained in the documentation
    7887              :    for vect_recog_pattern.  */
    7888              : 
    7889              : static void
    7890   1073279864 : vect_pattern_recog_1 (vec_info *vinfo,
    7891              :                       const vect_recog_func &recog_func, stmt_vec_info stmt_info)
    7892              : {
    7893   1073279864 :   gimple *pattern_stmt;
    7894   1073279864 :   tree pattern_vectype;
    7895              : 
    7896              :   /* If this statement has already been replaced with pattern statements,
    7897              :      leave the original statement alone, since the first match wins.
    7898              :      Instead try to match against the definition statements that feed
    7899              :      the main pattern statement.  */
    7900   1073279864 :   if (STMT_VINFO_IN_PATTERN_P (stmt_info))
    7901              :     {
    7902     14038424 :       gimple_stmt_iterator gsi;
    7903     14038424 :       for (gsi = gsi_start (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
    7904     31434640 :            !gsi_end_p (gsi); gsi_next (&gsi))
    7905     17396216 :         vect_pattern_recog_1 (vinfo, recog_func,
    7906              :                               vinfo->lookup_stmt (gsi_stmt (gsi)));
    7907              :       return;
    7908              :     }
    7909              : 
    7910   1059241440 :   gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_info));
    7911   1059241440 :   pattern_stmt = recog_func.fn (vinfo, stmt_info, &pattern_vectype);
    7912   1059241440 :   if (!pattern_stmt)
    7913              :     {
    7914              :       /* Clear any half-formed pattern definition sequence.  */
    7915   1058136920 :       STMT_VINFO_PATTERN_DEF_SEQ (stmt_info) = NULL;
    7916   1058136920 :       return;
    7917              :     }
    7918              : 
    7919              :   /* Found a vectorizable pattern.  */
    7920      1104520 :   if (dump_enabled_p ())
    7921        21151 :     dump_printf_loc (MSG_NOTE, vect_location,
    7922              :                      "%s pattern recognized: %G",
    7923        21151 :                      recog_func.name, pattern_stmt);
    7924              : 
    7925              :   /* Mark the stmts that are involved in the pattern. */
    7926      1104520 :   vect_mark_pattern_stmts (vinfo, stmt_info, pattern_stmt, pattern_vectype);
    7927              : }
    7928              : 
    7929              : 
    7930              : /* Function vect_pattern_recog
    7931              : 
    7932              :    Input:
    7933              :    LOOP_VINFO - a struct_loop_info of a loop in which we want to look for
    7934              :         computation idioms.
    7935              : 
    7936              :    Output - for each computation idiom that is detected we create a new stmt
    7937              :         that provides the same functionality and that can be vectorized.  We
    7938              :         also record some information in the struct_stmt_info of the relevant
    7939              :         stmts, as explained below:
    7940              : 
    7941              :    At the entry to this function we have the following stmts, with the
    7942              :    following initial value in the STMT_VINFO fields:
    7943              : 
    7944              :          stmt                     in_pattern_p  related_stmt    vec_stmt
    7945              :          S1: a_i = ....                 -       -               -
    7946              :          S2: a_2 = ..use(a_i)..         -       -               -
    7947              :          S3: a_1 = ..use(a_2)..         -       -               -
    7948              :          S4: a_0 = ..use(a_1)..         -       -               -
    7949              :          S5: ... = ..use(a_0)..         -       -               -
    7950              : 
    7951              :    Say the sequence {S1,S2,S3,S4} was detected as a pattern that can be
    7952              :    represented by a single stmt.  We then:
    7953              :    - create a new stmt S6 equivalent to the pattern (the stmt is not
    7954              :      inserted into the code)
    7955              :    - fill in the STMT_VINFO fields as follows:
    7956              : 
    7957              :                                   in_pattern_p  related_stmt    vec_stmt
    7958              :          S1: a_i = ....                 -       -               -
    7959              :          S2: a_2 = ..use(a_i)..         -       -               -
    7960              :          S3: a_1 = ..use(a_2)..         -       -               -
    7961              :          S4: a_0 = ..use(a_1)..         true    S6              -
    7962              :           '---> S6: a_new = ....        -       S4              -
    7963              :          S5: ... = ..use(a_0)..         -       -               -
    7964              : 
    7965              :    (the last stmt in the pattern (S4) and the new pattern stmt (S6) point
    7966              :    to each other through the RELATED_STMT field).
    7967              : 
    7968              :    S6 will be marked as relevant in vect_mark_stmts_to_be_vectorized instead
    7969              :    of S4 because it will replace all its uses.  Stmts {S1,S2,S3} will
    7970              :    remain irrelevant unless used by stmts other than S4.
    7971              : 
    7972              :    If vectorization succeeds, vect_transform_stmt will skip over {S1,S2,S3}
    7973              :    (because they are marked as irrelevant).  It will vectorize S6, and record
    7974              :    a pointer to the new vector stmt VS6 from S6 (as usual).
    7975              :    S4 will be skipped, and S5 will be vectorized as usual:
    7976              : 
    7977              :                                   in_pattern_p  related_stmt    vec_stmt
    7978              :          S1: a_i = ....                 -       -               -
    7979              :          S2: a_2 = ..use(a_i)..         -       -               -
    7980              :          S3: a_1 = ..use(a_2)..         -       -               -
    7981              :        > VS6: va_new = ....             -       -               -
    7982              :          S4: a_0 = ..use(a_1)..         true    S6              VS6
    7983              :           '---> S6: a_new = ....        -       S4              VS6
    7984              :        > VS5: ... = ..vuse(va_new)..    -       -               -
    7985              :          S5: ... = ..use(a_0)..         -       -               -
    7986              : 
    7987              :    DCE could then get rid of {S1,S2,S3,S4,S5} (if their defs are not used
    7988              :    elsewhere), and we'll end up with:
    7989              : 
    7990              :         VS6: va_new = ....
    7991              :         VS5: ... = ..vuse(va_new)..
    7992              : 
    7993              :    In case of more than one pattern statements, e.g., widen-mult with
    7994              :    intermediate type:
    7995              : 
    7996              :      S1  a_t = ;
    7997              :      S2  a_T = (TYPE) a_t;
    7998              :            '--> S3: a_it = (interm_type) a_t;
    7999              :      S4  prod_T = a_T * CONST;
    8000              :            '--> S5: prod_T' = a_it w* CONST;
    8001              : 
    8002              :    there may be other users of a_T outside the pattern.  In that case S2 will
    8003              :    be marked as relevant (as well as S3), and both S2 and S3 will be analyzed
    8004              :    and vectorized.  The vector stmt VS2 will be recorded in S2, and VS3 will
    8005              :    be recorded in S3.  */
    8006              : 
    8007              : void
    8008      1142237 : vect_pattern_recog (vec_info *vinfo)
    8009              : {
    8010      1142237 :   basic_block *bbs = vinfo->bbs;
    8011      1142237 :   unsigned int nbbs = vinfo->nbbs;
    8012              : 
    8013      1142237 :   vect_determine_precisions (vinfo);
    8014              : 
    8015      1142237 :   DUMP_VECT_SCOPE ("vect_pattern_recog");
    8016              : 
    8017              :   /* Scan through the stmts in the region, applying the pattern recognition
    8018              :      functions starting at each stmt visited.  */
    8019     13270236 :   for (unsigned i = 0; i < nbbs; i++)
    8020              :     {
    8021     12127999 :       basic_block bb = bbs[i];
    8022              : 
    8023    129610941 :       for (auto si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
    8024              :         {
    8025    105354943 :           stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (si));
    8026              : 
    8027    105354943 :           if (!stmt_info || !STMT_VINFO_VECTORIZABLE (stmt_info))
    8028     72358579 :             continue;
    8029              : 
    8030              :           /* Scan over all generic vect_recog_xxx_pattern functions.  */
    8031   1088880012 :           for (const auto &func_ptr : vect_vect_recog_func_ptrs)
    8032   1055883648 :             vect_pattern_recog_1 (vinfo, func_ptr,
    8033              :                                   stmt_info);
    8034              :         }
    8035              :     }
    8036              : 
    8037              :   /* After this no more add_stmt calls are allowed.  */
    8038      1142237 :   vinfo->stmt_vec_info_ro = true;
    8039      1142237 : }
    8040              : 
    8041              : /* Build a GIMPLE_ASSIGN or GIMPLE_CALL with the tree_code,
    8042              :    or internal_fn contained in ch, respectively.  */
    8043              : gimple *
    8044       171687 : vect_gimple_build (tree lhs, code_helper ch, tree op0, tree op1)
    8045              : {
    8046       171687 :   gcc_assert (op0 != NULL_TREE);
    8047       171687 :   if (ch.is_tree_code ())
    8048       171687 :     return gimple_build_assign (lhs, (tree_code) ch, op0, op1);
    8049              : 
    8050            0 :   gcc_assert (ch.is_internal_fn ());
    8051            0 :   gimple* stmt = gimple_build_call_internal (as_internal_fn ((combined_fn) ch),
    8052              :                                              op1 == NULL_TREE ? 1 : 2,
    8053              :                                              op0, op1);
    8054            0 :   gimple_call_set_lhs (stmt, lhs);
    8055            0 :   return stmt;
    8056              : }
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.